diff --git a/src/olmo_eval/common/scorers/ifeval.py b/src/olmo_eval/common/scorers/ifeval.py index 16dad3bdb..ff708dcf6 100644 --- a/src/olmo_eval/common/scorers/ifeval.py +++ b/src/olmo_eval/common/scorers/ifeval.py @@ -1,7 +1,7 @@ """Scorer for IFBench / IFEval instruction-following evaluation. Uses the ``ifbench`` package registry, which covers the original IFEval -(DEFAULT) verifiers, the OOD verifiers used by ``allenai/IFBench_test2``, +(DEFAULT) verifiers, the OOD verifiers used by ``allenai/IFBench_test``, and the verifiers used by the multi-turn ``VGraf/ifeval_mt`` slices. The scorer evaluates a response against per-instance instructions (looked up in ``instance.metadata["instruction_id_list"]`` / ``"kwargs"``) and writes diff --git a/src/olmo_eval/evals/tasks/ifeval_ood.py b/src/olmo_eval/evals/tasks/ifeval_ood.py index d3c65991c..11b583aa0 100644 --- a/src/olmo_eval/evals/tasks/ifeval_ood.py +++ b/src/olmo_eval/evals/tasks/ifeval_ood.py @@ -1,6 +1,6 @@ """IFEval OOD: out-of-distribution instruction-following slice of IFBench. -Dataset: ``allenai/IFBench_test2`` (300 prompts), each carrying a list of +Dataset: ``allenai/IFBench_test`` (300 prompts), each carrying a list of instruction IDs and per-instruction kwargs. Verifiers come from the vendored registry in :mod:`olmo_eval.common.scorers.ifeval_deps`. @@ -35,7 +35,7 @@ @register("ifeval_ood") class IFEvalOOD(Task): - data_source = DataSource(path="allenai/IFBench_test2", split="train") + data_source = DataSource(path="allenai/IFBench_test", split="train") split = Split.TRAIN metrics = ( IFEvalPromptStrictAccuracy(),