diff --git a/mmf/configs/zoo/models.yaml b/mmf/configs/zoo/models.yaml index f6fad4643..0ef82971a 100644 --- a/mmf/configs/zoo/models.yaml +++ b/mmf/configs/zoo/models.yaml @@ -564,3 +564,19 @@ villa: - url: mmf://models/uniter/villa.pretrained.tar.gz file_name: villa.pretrained.tar.gz hashcode: 7a8f31421ef644fddc99bd142a0090660573dd526a779d025253c3fd996754fc + +vilt: + defaults: ${vilt.vqa} + vqa: + # Model from project : projects/vilt, Val: 69.55 + version: 1.0_2021_011_29 + resources: + - url: mmf://models/vilt/vilt.finetuned.vqa2.tar.gz + file_name: vilt.finetuned.vqa2.tar.gz + hashcode: c1a96abfaffa140c0970592691c2cb828b3bb16be7a6c610f7d02c9c05951c1f + pretrained: + version: 1.0_2021_011_29 + resources: + - url: mmf://models/vilt/vilt.pretrained.tar.gz + file_name: vilt.pretrained.tar.gz + hashcode: fb4478665ba6709bd454cec40b66dcb89570b998853745cc400bfb6f99aaeb69 diff --git a/website/docs/projects/vilt.md b/website/docs/projects/vilt.md index a6381ab9f..4a16923fa 100644 --- a/website/docs/projects/vilt.md +++ b/website/docs/projects/vilt.md @@ -24,9 +24,24 @@ Follow installation instructions in the [documentation](https://mmf.readthedocs. ## Training -To train ViLT model on the VQA2.0 dataset, run the following command +To train ViLT model from scratch on the VQA2.0 dataset, run the following command ``` mmf_run config=projects/vilt/configs/vqa2/defaults.yaml run_type=train_val dataset=vqa2 model=vilt ``` To finetune using different pretrained starting weights, change the `pretrained_model_name` under image_encoder in the config yaml to reference a huggingface model. + +To finetrain a pretrained ViLT model on the VQA2.0 dataset, +``` +mmf_run config=projects/vilt/configs/vqa2/defaults.yaml run_type=train_val dataset=vqa2 model=vilt checkpoint.resume_zoo=vilt.pretrained +``` + +To test a ViLT model already finetuned on the VQA2.0 dataset, +``` +mmf_run config=projects/vilt/configs/vqa2/defaults.yaml run_type=val dataset=vqa2 model=vilt checkpoint.resume_zoo=vilt.vqa +``` + +To pretrain a ViLT model from scratch on the COCO dataset, +``` +mmf_run config=projects/vilt/configs/masked_coco/pretrain.yaml run_type=train_val dataset=masked_coco model=vilt +```