git clone https://www.github.com/valeoai/VideoActionModel
cd VideoActionModel
# Installs all runtime dependencies, except for PyTorch:
pip install -e .
# Installs all runtime dependencies, including PyTorch 2.4.0 (recommended version):
# pip install -e ".[torch]"
# Installs all runtime and development dependencies:
# pip install -e ".[dev]"

VideoActionModel
|––inference    => NeuroNCAP evaluation
     \--scripts => Scripts to run the NeuroNCAP evaluation
|––mup_shapes   => mup parameterization
|––notebooks
     |--qualitative*.ipynb => Qualitative examples for VaViM an VaVAM
     \--scaling_laws.ipynb => Scaling laws of VaViM
|––scripts  => Utilities: fusing deepspeed checkpoints, evaluations...
\––vam      => Main source code
     |--action_expert     => Action expert
     |--datalib           => Data pre-processing and loading
     |--evaluation        => Evaluation utils
     \--video_pretraining => GPT-style model

python jeanzay_slurm_job_submit.py \
-n VaViM_768_pretraining_OpenDV \
--gpus_per_node 4 --nodes 16 \
-wt 20:00:00 \
-p 'experiment=video_pretraining_GPT2_llamagen_ds16_16384_opendv data.batch_size=6 paths.output_dir=XXXXX model.network.embedding_dim=768 callbacks=callbacks_opendv_training model.optimizer_conf.weight_decay=1e-07 model.network.init_std=0.0289 model.optimizer_conf.lr=0.0041 data.num_workers=6 ++trainer.max_epochs=1 ++trainer.val_check_interval=0.25'

python scripts/regain_index_from_train.py \
--ckpt /path/to/ckpt.pt \
--outdir $ycy_ALL_CCFRWORK \
--name checkpoint_pretraining

python jeanzay_slurm_job_submit.py \
-n VaViM_768_finetuning_data_mix \
--gpus_per_node 4 --nodes 16 \
-wt 02:30:00 \
-p 'experiment=finetune_mix_complet data.batch_size=6 paths.output_dir=XXXXX  model.network.embedding_dim=768 model.optimizer_conf.weight_decay=1e-07 model.optimizer_conf.lr=0.0041 data.num_workers=6 ckpt_path="XXXXX"'

python jeanzay_slurm_job_submit.py \
-n VaVAM_768_action_learning_nuPlan_nuScenes \
--gpus_per_node 4 \
--nodes 6 \
-wt 10:00:00 \
-p 'experiment=action_learning data.batch_size=16 model.vam_conf.gpt_config.embedding_dim=768 model.vam_conf.action_config.embedding_dim=192 model.vam_conf.action_config.init_std=0.0086 model.optimizer_conf.lr=0.0194 model.optimizer_conf.weight_decay=1e-07 paths.output_dir=XXXXX ++trainer.max_epochs=1 data.num_workers=6 model.vam_conf.gpt_checkpoint_path="XXXXX" trainer.strategy=ddp +model.grad_logging=100'

name=XXX \  # Name of the experiment
++trainer.devices=XXX \  # Number of GPUs per node
++trainer.num_nodes=XXX \  # Number of nodes

import torch

from vam.datalib import OpenDVTokensDataset, torch_image_to_plot
from vam.utils import expand_path, plot_multiple_images
from vam.video_pretraining import load_pretrained_gpt

vm_checkpoint_path = "/path/to/trained/vavim"
detokenizer_path = "/path/to/trained/detokenizer"
opendv_data_root_dir = "/path/to/preprocessed/opendv"

# Load the pretrained model and the tokenizer decoder.
gpt = load_pretrained_gpt(expand_path(vm_checkpoint_path))
image_detokenizer = torch.jit.load(expand_path(detokenizer_path)).to("cuda")

# Load the dataset.
dts = OpenDVTokensDataset(
    data_root_dir=expand_path(opendv_data_root_dir),
    video_list=["5pAf38x5z9Q"],  # This is one of the validation video from OpenDV
    sequence_length=8,
    subsampling_factor=5,
)

# Upper bound quality with ground truth tokens.
visual_tokens = dts[100]["visual_tokens"].to("cuda", non_blocking=True)
gt_images = image_detokenizer(dts[100]["visual_tokens"].to("cuda", non_blocking=True))
gt_images = torch_image_to_plot(gt_images)
_ = plot_multiple_images(gt_images, 2, 4)

# Generate 4 frames in the future from the first 6 frames.
# Note: we can use bloat16 on A100 or H100 GPUs.
with torch.amp.autocast("cuda", dtype=torch.bfloat16):
    generated_frames = gpt.forward_inference(
        number_of_future_frames=4,
        burnin_visual_tokens=visual_tokens.unsqueeze(0)[:, :6],
    )

pred_images = image_detokenizer(generated_frames.squeeze(0))
pred_images = torch_image_to_plot(pred_images)
_ = plot_multiple_images(pred_images, 1, 4)

import pickle

import torch
from einops import rearrange, repeat

from vam.action_expert import load_inference_VAM
from vam.datalib import EgoTrajectoryDataset
from vam.evaluation import min_ade
from vam.utils import expand_path

vam_checkpoint_path = "/path/to/trained/vavam"
nuscenes_pickle_data_path = "/path/to/nuscenes/pickle"
nuscenes_tokens_rootdir = "/path/to/nuscenes/tokens_dir"

# Load the pretrained model.
vam = load_inference_VAM(expand_path(vam_checkpoint_path), "cuda")

with open(expand_path(nuscenes_pickle_data_path), "rb") as f:
    nuscenes_pickle_data = pickle.load(f)

dataset = EgoTrajectoryDataset(
    pickle_data=nuscenes_pickle_data,
    tokens_rootdir=expand_path(nuscenes_tokens_rootdir),
)

num_sampling = 5
sample = dataset[0]
visual_tokens = sample["visual_tokens"].to("cuda", non_blocking=True)
visual_tokens = repeat(visual_tokens, "t h w -> b t h w", b=num_sampling)
commands = sample["high_level_command"].to("cuda", non_blocking=True)[-1:]
commands = repeat(commands, "t -> b t", b=num_sampling)

with torch.amp.autocast("cuda", dtype=torch.bfloat16):
    trajectory = vam(visual_tokens, commands, torch.bfloat16)

ground_truth = sample["positions"].to("cuda", non_blocking=True)[-1:]

ground_truth = sample["positions"].to("cuda", non_blocking=True)[-1:]
trajectory = rearrange(trajectory, "s 1 t a -> 1 s t a")
loss = min_ade(trajectory, ground_truth)
print(loss)

import torch
from einops import rearrange
from torch import Tensor

from vam.evaluation.datasets import CityscapesDataset
from vam.evaluation.hbird import hbird_evaluation
from vam.utils import expand_path
from vam.video_pretraining import load_pretrained_gpt

vm_checkpoint_path = "/path/to/trained/vavim"
tokenizer_path = "/path/to/trained/tokenizer"
cityscapes_root = "/path/to/cityscapes/root"

# Load the pretrained model and the tokenizer.
gpt = load_pretrained_gpt(expand_path(vm_checkpoint_path))
image_tokenizer = torch.jit.load(expand_path(tokenizer_path)).to("cuda")
model_info = {
    "patch_size": 16,
    "d_model": gpt.embedding_dim,
}


def forward_fn(x: Tensor, inference: bool) -> Tensor:
    x = image_tokenizer(x)
    x = rearrange(x, "b h w -> b 1 h w")
    x = gpt.get_intermediate_layers(x, 12)
    x = rearrange(x[:, -1], "b h w d -> b (h w) d")
    return x


train_dts = CityscapesDataset(root=expand_path(cityscapes_root), split="train")
val_dts = CityscapesDataset(root=expand_path(cityscapes_root), split="val")

logs, _ = hbird_evaluation(
    ftr_extr_fn=forward_fn,
    model_info=model_info,
    train_dataset=train_dts,
    val_dataset=val_dts,
    batch_size=16,
    batch_size_eval=16,
    augmentation_epoch=1,
    device="cuda",
    dtype="bf16",
    return_labels=False,
    num_neighbour=30,
    nn_params=None,
    memory_size="x10",  # you can set this to reduce memory size
    f_mem_p=None,
    l_mem_p=None,
)

print(logs["mIoU"], logs["IoU"])

python scripts/depth_anything_a_dataset.py --dataset_name cityscapes
python scripts/depth_anything_a_dataset.py --dataset_name cityscapes --compute_only_issues

@article{vavam2025,
  title={VaViM and VaVAM: Autonomous Driving through Video Generative Modeling},
  author={Bartoccioni, Florent and Ramzi, Elias and Besnier, Victor and Venkataramanan, Shashanka and Vu, Tuan-Hung and Xu, Yihong and Chambon, Loick and Gidaris, Spyros and Odabas, Serkan and Hurych, David and Marlet, Renaud and Boulch, Alexandre and Chen, Mickael and Zablocki, Eloi and Bursuc, Andrei and Valle, Eduardo and Cord, Matthieu},
  journal={arXiv preprint arXiv:2502.15672},
  year={2025}
}

@software{vavam2025_github,
    title = {VaViM and VaVAM: Autonomous Driving through Video Generative Modeling},
    author={Bartoccioni, Florent and Ramzi, Elias and Besnier, Victor and Venkataramanan, Shashanka and Vu, Tuan-Hung and Xu, Yihong and Chambon, Loick and Gidaris, Spyros and Odabas, Serkan and Hurych, David and Marlet, Renaud and Boulch, Alexandre and Chen, Mickael and Zablocki, Eloi and Bursuc, Andrei and Valle, Eduardo and Cord, Matthieu},
    license = {MIT},
    url = {https://github.com/valeoai/VideoActionModel}
}