Image-QA benchmarks (4/5)

pyproject.toml

@@ -49,6 +49,15 @@ crawl4ai = [
 ]
 hf = [
     "transformers>=5.4.0",
+    # Required by the released Molmo2 remote code (image_processing_molmo2.py imports both at
+    # module top); not pulled transitively, so declare them here for the HF multimodal path.
+    "einops",
+    "torchvision ; platform_system != 'Darwin'",  # GPU-coupled; skip on macOS like the vllm extra
+    # Image-pointing benchmarks (pixmo_points_eval, sa_co_gold_subset): pycocotools decodes the
+    # COCO-RLE instance masks and scipy provides the point↔mask bipartite matching. Both are
+    # imported lazily by the pointing scorer, so non-pointing HF runs don't need them at import.
+    "pycocotools",
+    "scipy~=1.17.1",
 ]
 olmo_core = [
     "ai2-olmo-core[torchao,transformers]==2.4.0",
@@ -191,6 +200,16 @@ select = [
 "src/olmo_eval/evals/tasks/squad.py" = ["E501"]
 # Static fewshot data with long string literals
 "src/olmo_eval/evals/tasks/constants/*" = ["E501"]
+# Verbatim prompt templates vendored from the mm_olmo reference implementation
+"src/olmo_eval/evals/vision/scoring/prompt_templates.py" = ["E501"]
+"src/olmo_eval/evals/vision/scoring/vqa_normalization.py" = ["E501"]
+"src/olmo_eval/evals/vision/scoring/multiple_choice.py" = ["E501"]
+"src/olmo_eval/evals/vision/scoring/math_vista_offline.py" = ["E501"]
+"src/olmo_eval/evals/vision/scoring/mmmu_pro.py" = ["E501"]
+# CharXiv grading prompts are vendored byte-for-byte from the official repo; their trailing
+# whitespace is part of the official prompt bytes and must not be stripped.
+"src/olmo_eval/evals/vision/scoring/charxiv.py" = ["E501", "W291", "W293"]
+"tests/evals/vision/test_image_qa_scorers.py" = ["E501"]
 [tool.ruff.format]
 docstring-code-format = true
 

src/olmo_eval/cli/beaker/job_assembler.py

@@ -130,7 +130,7 @@ def _normalize_olmo_core_package(package: str) -> str:
 
 def normalize_provider_package_for_kind(provider_kind: str | None, package: str) -> str:
     """Normalize a provider package override for the effective provider kind."""
-    if provider_kind == "olmo_core":
+    if provider_kind in {"olmo_core", "olmo_core_vlm"}:
         return _normalize_olmo_core_package(package)
     return package
 
@@ -476,6 +476,7 @@ def assemble(self, exp: ExperimentPlan) -> BeakerJobConfig:
         provider_extra_override = (
             {
                 "olmo_core": "olmo_core",
+                "olmo_core_vlm": "olmo_core",
                 "vllm": "vllm",
                 "vllm_server": "vllm",
             }.get(provider_kind)

src/olmo_eval/cli/utils.py

@@ -107,6 +107,7 @@ def reconstruct_ordered_args(args: list[str]) -> list[FlaggedArg]:
         "metrics",
         "batching",
         "scorer_startup_timeout",
+        "max_hard_failure_rate",
     }
 )
 
@@ -126,6 +127,8 @@ def reconstruct_ordered_args(args: list[str]) -> list[FlaggedArg]:
         "sampling_params",
         "dependencies",
         "sandbox_allocation_weight",
+        "prompt_templates",
+        "system_prompt_style",
         "priority",  # Special: extracted for job priority, not a real TaskConfig field
     }
 )

src/olmo_eval/common/constants/infrastructure.py

@@ -193,6 +193,7 @@ def cluster_has_weka(cluster: str) -> bool:
     "vllm_server": "vllm",
     "hf": "hf",
     "olmo_core": "olmo_core",
+    "olmo_core_vlm": "olmo_core",
     "litellm": "litellm",
     "mock": None,
 }

src/olmo_eval/common/constants/models.py

@@ -136,4 +136,30 @@ def get_model_presets() -> dict[str, ProviderConfig]:
             api_base="https://ai2-model-hub.allen.ai",
             required_secrets=("LITELLM_PROXY_API_KEY",),
         ),
+        # Multimodal (image-text-to-text) models. Run the released HF Molmo2
+        # checkpoint directly via AutoProcessor + AutoModelForImageTextToText;
+        # tasks attach the image to LMRequest.images. Requires the `hf` extra.
+        # fp32 weights + bf16 autocast matches mm_olmo's official amp_bf16 eval
+        # numerics (the released config declares dtype=float32, float32_attention).
+        "molmo2-4b": ProviderConfig(
+            kind=ProviderKind.HF,
+            model="allenai/Molmo2-4B",
+            trust_remote_code=True,
+            dtype="float32",
+            kwargs={"multimodal": True, "max_crops": 24, "autocast_dtype": "bfloat16"},
+        ),
+        "molmo2-8b": ProviderConfig(
+            kind=ProviderKind.HF,
+            model="allenai/Molmo2-8B",
+            trust_remote_code=True,
+            dtype="float32",
+            kwargs={"multimodal": True, "max_crops": 24, "autocast_dtype": "bfloat16"},
+        ),
+        "molmo2-o-7b": ProviderConfig(
+            kind=ProviderKind.HF,
+            model="allenai/Molmo2-O-7B",
+            trust_remote_code=True,
+            dtype="float32",
+            kwargs={"multimodal": True, "max_crops": 24, "autocast_dtype": "bfloat16"},
+        ),
     }

src/olmo_eval/common/types/base.py

@@ -231,6 +231,7 @@ class LMRequest:
     tools: tuple[ToolSchema, ...] | None = None
     system_prompt: str | None = None
     max_length: int | None = None
+    images: tuple[Any, ...] | None = None
 
 
 @hide_unset()

src/olmo_eval/common/types/literals.py

@@ -11,11 +11,14 @@ class ProviderKind(StrEnum):
     VLLM_SERVER = "vllm_server"
     HF = "hf"
     OLMO_CORE = "olmo_core"
+    OLMO_CORE_VLM = "olmo_core_vlm"
     MOCK = "mock"
     LITELLM = "litellm"
 
 
-ProviderLiteral = Literal["vllm", "vllm_server", "hf", "olmo_core", "mock", "litellm"]
+ProviderLiteral = Literal[
+    "vllm", "vllm_server", "hf", "olmo_core", "olmo_core_vlm", "mock", "litellm"
+]
 DtypeLiteral = Literal["auto", "float16", "bfloat16", "float32"]
 PriorityLiteral = Literal["low", "normal", "high", "urgent"]
 LoadFormatLiteral = Literal[

src/olmo_eval/evals/__init__.py

@@ -4,3 +4,4 @@
 from . import external as _external  # noqa: F401
 from . import suites as _suites  # noqa: F401
 from . import tasks as _tasks  # noqa: F401
+from . import vision as _vision  # noqa: F401

src/olmo_eval/evals/suites/molmo2.py

@@ -0,0 +1,72 @@
+"""Molmo2 multimodal benchmark suites.
+
+Grows with the vision families as they land: captioning now, pointing/counting,
+image QA, and multi-image next.
+"""
+
+from olmo_eval.evals.suites.registry import AggregationStrategy, make_suite
+
+# The image-QA benchmarks Molmo2 reports. ``mmmu_pro`` is a single task whose primary metric is the
+# MMMU-Pro Overall = (standard-10 + vision)/2, so it contributes one 0-1 entry like the others.
+MOLMO2_IMAGE_QA_TASKS = (
+    "chart_qa",
+    "vqa2",
+    "doc_qa",
+    "info_qa",
+    "text_vqa",
+    "real_world_qa",
+    "mmmu",
+    "mmmu_pro",
+    "math_vista",
+    "countbench_qa",
+    "pixmo_count",
+    "ai2d",
+    "charxiv_descriptive",
+    "charxiv_reasoning",
+)
+
+make_suite(
+    "molmo2_imageqa",
+    MOLMO2_IMAGE_QA_TASKS,
+    aggregation=AggregationStrategy.AVERAGE,
+    description="Molmo2's image-QA benchmarks (primary metrics are all 0-1).",
+)
+
+make_suite(
+    "molmo2_imageqa_caption",
+    (*MOLMO2_IMAGE_QA_TASKS, "dense_caption"),
+    # dense_caption's primary metric is 0-100, so no cross-task average is computed.
+    aggregation=AggregationStrategy.DISPLAY_ONLY,
+    description="Molmo2's image-QA benchmarks plus PixMo-Cap dense caption (GPT judge).",
+)
+
+# Image-pointing benchmarks — point-in-mask precision/recall/f1, not VQA-style answers.
+MOLMO2_POINTING_TASKS = (
+    "pixmo_points_eval",
+    "sa_co_gold_subset",
+)
+
+make_suite(
+    "molmo2_pointing",
+    MOLMO2_POINTING_TASKS,
+    aggregation=AggregationStrategy.AVERAGE,
+    description="Molmo2's image-pointing benchmarks (primary metric is f1, 0-1).",
+)
+
+# The mm_olmo `_mp` variants: same scoring, but the prompt is built from a bare label by
+# the model's own formatter, so it follows the checkpoint (see `vision.scoring.prompts`).
+MOLMO2_POINTING_MP_TASKS = (
+    "pixmo_points_eval_mp",
+    "sa_co_gold_subset_mp",
+    "sa_co_gold_point_4k_mp",
+)
+
+# `sa_co_gold_point_mp` (the unsampled 166,766-example gold set) is registered but kept out
+# of the suite: it is ~6x the 4k variant and measures the same thing.
+
+make_suite(
+    "molmo2_pointing_mp",
+    MOLMO2_POINTING_MP_TASKS,
+    aggregation=AggregationStrategy.AVERAGE,
+    description="Molmo2's image-pointing benchmarks with mm_olmo's model-prompt (_mp) inputs.",
+)

src/olmo_eval/evals/tasks/common/base.py

@@ -172,6 +172,12 @@ class TaskConfig:
     #: beaker launcher mounts each as the user-scoped secret ``{user}_{NAME}``.
     required_secrets: tuple[str, ...] = ()
 
+    #: How a task that builds its prompt from a bare label should render it. The
+    #: vision tasks read these; they follow the checkpoint, since instruction-tuned
+    #: and pretrain models were trained on different prompt forms.
+    prompt_templates: str | None = None
+    system_prompt_style: str | None = None
+
     def __post_init__(self) -> None:
         """Validate scheduler-only sandbox allocation hints."""
         if isinstance(self.output_score_aggregation, str):
@@ -283,6 +289,8 @@ def serialize_primary_metric(pm: Any) -> Any:
             "max_length": self.max_length,
             "answer_extractor": getattr(self.answer_extractor, "__name__", None),
             "dependencies": self.dependencies,
+            "prompt_templates": self.prompt_templates,
+            "system_prompt_style": self.system_prompt_style,
         }
 
     def get_primary_metric(self) -> Metric | None:

src/olmo_eval/evals/vision/__init__.py

@@ -0,0 +1,9 @@
+"""Vision (multimodal) evaluation: tasks, scoring, benchmarks, and data access.
+
+Importing this package registers every vision benchmark, mirroring how
+``olmo_eval.evals`` imports ``tasks``.
+"""
+
+from olmo_eval.evals.vision import benchmarks as _benchmarks  # noqa: F401
+
+__all__: list[str] = []

src/olmo_eval/evals/vision/benchmarks/__init__.py

@@ -0,0 +1,25 @@
+"""The vision benchmarks; importing this package registers them.
+
+One module per benchmark. Imports are explicit (unlike ``evals/tasks``\'
+pkgutil scan) so a missing module is an ImportError here rather than a silently
+absent task.
+"""
+
+from olmo_eval.evals.vision.benchmarks import ai2d as _ai2d  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import chart_qa as _chart_qa  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import charxiv as _charxiv  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import countbench_qa as _countbench_qa  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import dense_caption as _dense_caption  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import doc_qa as _doc_qa  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import info_qa as _info_qa  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import math_vista as _math_vista  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import mmmu as _mmmu  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import mmmu_pro as _mmmu_pro  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import pixmo_count as _pixmo_count  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import pixmo_points_eval as _pixmo_points_eval  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import real_world_qa as _real_world_qa  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import sa_co_gold as _sa_co_gold  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import text_vqa as _text_vqa  # noqa: F401
+from olmo_eval.evals.vision.benchmarks import vqa2 as _vqa2  # noqa: F401
+
+__all__: list[str] = []

src/olmo_eval/evals/vision/benchmarks/ai2d.py

@@ -0,0 +1,86 @@
+"""AI2D (validation by default; ``ai2d:test`` for the official test split).
+
+Mirrors mm_olmo's ``AI2DConfig(boxes="both")`` (task name
+``ai2_diagram_v2_mix_transparent``): loads the prepared arrow dataset at
+``torch_datasets/academic_datasets/ai2d``, where every abc-label question
+appears twice — once with opaque answer boxes drawn on the diagram and once
+with transparent ones (``has_transparent_box``).
+
+Formatting follows ``AI2DConfig.format_example``: when a question's answer
+options are (almost all) the on-diagram letters themselves, options are
+listed without ``A./B.`` prefixes and the model must answer with the option
+text (``ai2_diagram_no_letter``); otherwise standard lettered options are
+used.  Multiple-choice style — no style tag.
+
+Reference (Molmo2-4B ck2000, val): mc_ai2d_opaque=0.8537,
+mc_ai2d_transparent=0.9481.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from olmo_eval.common.types import Instance, SamplingParams, Split
+from olmo_eval.evals.tasks.common import register, register_variant
+from olmo_eval.evals.vision.data.images import lazy_hf_image
+from olmo_eval.evals.vision.data.paths import torch_datasets_dir
+from olmo_eval.evals.vision.scoring.prompt_templates import format_mc_question
+from olmo_eval.evals.vision.scoring.vqa import Ai2dScorer
+from olmo_eval.evals.vision.tasks.single_image import Ai2dMetric, ImageQATask
+
+_SCORER = Ai2dScorer()
+_OPAQUE = Ai2dMetric(name="mc_ai2d_opaque", scorer=_SCORER, transparent=False)
+_TRANSPARENT = Ai2dMetric(name="mc_ai2d_transparent", scorer=_SCORER, transparent=True)
+
+
+@register("ai2d")
+class Ai2dTask(ImageQATask):
+    sampling_params = SamplingParams(temperature=0.0, max_tokens=32)
+    metrics = (_OPAQUE, _TRANSPARENT)
+    primary_metric = _OPAQUE
+    split = Split.VALIDATION
+
+    def _build_instances(self) -> Iterator[Instance]:
+        import datasets
+
+        ds = datasets.load_from_disk(str(torch_datasets_dir() / "academic_datasets" / "ai2d"))
+        ds = ds[self.config.split.value]
+        ds_nodecode = ds.cast_column("image", datasets.Image(decode=False))
+
+        for idx in range(len(ds_nodecode)):
+            ex = ds_nodecode[idx]
+            options = ex["answer_texts"]
+            answer_idx = ex["correct_answer"]
+            if ex["abc_label"] and sum(ex["option_is_abc"]) >= (len(options) - 1):
+                # ai2_diagram_no_letter: unlabelled options, abc options uppercased
+                unlabelled = [
+                    opt.upper() if abc else opt
+                    for opt, abc in zip(options, ex["option_is_abc"], strict=True)
+                ]
+                question, option_names = format_mc_question(
+                    ex["question"], unlabelled, labelled=False
+                )
+                gold = unlabelled[answer_idx]
+            else:
+                question, option_names = format_mc_question(ex["question"], options)
+                gold = option_names[answer_idx]
+            yield Instance(
+                question=question,
+                gold_answer=gold,
+                metadata={
+                    "example_id": ex["question_id"],
+                    "image_id": ex["image_id"],
+                    "abc_label": ex["abc_label"],
+                    "has_transparent_box": ex["has_transparent_box"],
+                    "answer_idx": answer_idx,
+                    "option_names": option_names,
+                    "options": options,
+                    "image": lazy_hf_image(ds_nodecode, idx, "image"),
+                },
+            )
+
+
+register_variant("ai2d", "test", split=Split.TEST)
+# `:transparent` makes mc_ai2d_transparent the primary metric (shown in the summary table);
+# both metrics are still computed. Stack with `:test`, e.g. `ai2d:test:transparent`.
+register_variant("ai2d", "transparent", primary_metric=_TRANSPARENT)