refactor: make language_aggregation_mode a non-optional parameter in evaluate()

Address PR #34 review feedback from @Mattdl. The aggregation mode now flows
consistently through the entire evaluation and aggregation pipeline
instead of being an optional parameter.

Author: federetyk

src/workrb/metrics/reporting.py

@@ -7,6 +7,7 @@
 from typing import Literal
 
 from workrb.results import BenchmarkResults
+from workrb.types import LanguageAggregationMode
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +22,7 @@ def format_results(
     show_error: bool = True,
     error_type: Literal["ci_margin", "stderr", "std"] = "ci_margin",
     show_only_key_metrics: bool = True,
+    language_aggregation_mode: LanguageAggregationMode | None = None,
 ) -> str:
     """
     Display benchmark results using BenchmarkResults aggregation methods.
@@ -36,11 +38,19 @@ def format_results(
         show_error: Whether to show error bars
         error_type: Type of error to show - "ci_margin", "stderr", or "std"
         show_only_key_metrics: If True, only show key metrics defined in task groups
+        language_aggregation_mode: How to determine the grouping language for
+            aggregation. When ``None``, reads the mode stored in
+            ``results.metadata.language_aggregation_mode``.
 
     Returns
     -------
         String containing formatted results
     """
+    if language_aggregation_mode is None:
+        language_aggregation_mode = LanguageAggregationMode(
+            results.metadata.language_aggregation_mode
+        )
+
     # Get aggregations - always include mean and error_type
     aggregations = ("mean", error_type) if show_error else ("mean",)
 
@@ -50,30 +60,67 @@ def format_results(
         for metrics in results.key_metrics_by_task_group.values():
             key_metrics.update(metrics)
 
+    # Compute all aggregation levels at once
+    all_results = results._get_summary_metrics(
+        aggregations=aggregations,
+        language_aggregation_mode=language_aggregation_mode,
+    )
+
+    # Partition results by tag name prefix for selective display
+    results_by_level: dict[str, dict] = {
+        "mean_per_task": {},
+        "mean_per_task_group": {},
+        "mean_per_language": {},
+        "mean_benchmark": {},
+    }
+    for tag, value in all_results.items():
+        if tag.name in results_by_level:
+            results_by_level[tag.name][tag] = value
+
     # Display each requested aggregation level
     metric_strs = []
     if display_per_task:
-        agg_results = results._aggregate_per_task(aggregations=aggregations)
         metric_strs.append(
-            _display_aggregation(agg_results, key_metrics, value_format, show_error, error_type)
+            _display_aggregation(
+                results_by_level["mean_per_task"],
+                key_metrics,
+                value_format,
+                show_error,
+                error_type,
+            )
         )
 
     if display_per_task_group:
-        agg_results = results._aggregate_per_task_group(aggregations=aggregations)
         metric_strs.append(
-            _display_aggregation(agg_results, key_metrics, value_format, show_error, error_type)
+            _display_aggregation(
+                results_by_level["mean_per_task_group"],
+                key_metrics,
+                value_format,
+                show_error,
+                error_type,
+            )
         )
 
     if display_per_language:
-        agg_results = results._aggregate_per_language(aggregations=aggregations)
         metric_strs.append(
-            _display_aggregation(agg_results, key_metrics, value_format, show_error, error_type)
+            _display_aggregation(
+                results_by_level["mean_per_language"],
+                key_metrics,
+                value_format,
+                show_error,
+                error_type,
+            )
         )
 
     if display_overall:
-        agg_results = results._aggregate_benchmark(aggregations=aggregations)
         metric_strs.append(
-            _display_aggregation(agg_results, key_metrics, value_format, show_error, error_type)
+            _display_aggregation(
+                results_by_level["mean_benchmark"],
+                key_metrics,
+                value_format,
+                show_error,
+                error_type,
+            )
         )
 
     return "\n".join(metric_strs)

src/workrb/results.py

@@ -60,6 +60,7 @@ class BenchmarkMetadata(BaseModel):
     num_tasks: int = Field(ge=1)
     languages: list[str]
     resumed_from_checkpoint: bool = False
+    language_aggregation_mode: str = LanguageAggregationMode.MONOLINGUAL_ONLY.value
 
 
 class ResultTagString(BaseModel):
@@ -99,10 +100,11 @@ class BenchmarkResults(BaseModel):
 
     def __str__(self) -> str:
         """String representation of the benchmark results."""
+        mode = LanguageAggregationMode(self.metadata.language_aggregation_mode)
         lines = [
             "BenchmarkResults",
             "=" * 80,
-            pprint.pformat(self.get_summary_metrics()),
+            pprint.pformat(self.get_summary_metrics(language_aggregation_mode=mode)),
         ]
         return "\n".join(lines)
 
@@ -126,43 +128,98 @@ def get_summary_metrics(
             How to determine the grouping language for per-language aggregation.
             Defaults to ``MONOLINGUAL_ONLY``.
         """
-        mean_per_task = self._aggregate_per_task(
+        combined = self._get_summary_metrics(
+            aggregations=aggregations,
+            language_aggregation_mode=language_aggregation_mode,
+        )
+        return {str(k): v for k, v in combined.items()}
+
+    def _get_summary_metrics(
+        self,
+        aggregations: tuple = ("mean", "ci_margin"),
+        language_aggregation_mode: LanguageAggregationMode = LanguageAggregationMode.MONOLINGUAL_ONLY,
+    ) -> dict[ResultTagString, float]:
+        """Compute all aggregation levels and return combined results.
+
+        Returns a single dict with ``ResultTagString`` keys covering:
+        ``mean_per_task``, ``mean_per_task_group``, ``mean_per_task_type``,
+        ``mean_per_language``, and ``mean_benchmark``.
+
+        Parameters
+        ----------
+        aggregations : tuple
+            Statistics to compute (e.g. ``"mean"``, ``"ci_margin"``).
+        language_aggregation_mode : LanguageAggregationMode
+            How to determine the grouping language for aggregation.
+        """
+        mean_per_task = self._aggregate_datasetids_per_task(
+            language_aggregation_mode=language_aggregation_mode,
             aggregations=aggregations,
         )
         mean_per_task_group = self._aggregate_per_task_group(
-            aggregations=aggregations, task_results=mean_per_task
+            language_aggregation_mode=language_aggregation_mode,
+            aggregations=aggregations,
+            task_results=mean_per_task,
         )
         mean_per_task_type = self._aggregate_per_task_type(
-            aggregations=aggregations, task_group_results=mean_per_task_group
+            language_aggregation_mode=language_aggregation_mode,
+            aggregations=aggregations,
+            task_group_results=mean_per_task_group,
         )
         mean_benchmark = self._aggregate_benchmark(
-            aggregations=aggregations, task_type_results=mean_per_task_type
+            language_aggregation_mode=language_aggregation_mode,
+            aggregations=aggregations,
+            task_type_results=mean_per_task_type,
         )
         mean_per_language = self._aggregate_per_language(
             aggregations=aggregations,
             aggregation_mode=language_aggregation_mode,
         )
 
-        combined = {
+        return {
             **mean_per_language,
             **mean_per_task,
             **mean_per_task_group,
             **mean_per_task_type,
             **mean_benchmark,
         }
-        return {str(k): v for k, v in combined.items()}
 
-    def _aggregate_per_task(
+    def _aggregate_datasetids_per_task(
         self,
+        language_aggregation_mode: LanguageAggregationMode,
         tag_name: str = "mean_per_task",
         aggregations: tuple = ("mean", "stderr", "ci_margin"),
     ) -> dict[ResultTagString, float]:
-        """Aggregate results per task, by aggregating over languages within tasks."""
-        # Collect metric values per task
+        """Aggregate dataset results per task, filtering by language aggregation mode.
+
+        For each task, only datasets compatible with the given
+        ``language_aggregation_mode`` are included in the per-task average.
+        Incompatible datasets are skipped with a warning, using the same
+        ``_get_language_grouping_key`` logic as ``_aggregate_per_language``.
+
+        This is the root aggregation level: per-task results feed into
+        per-task-group, per-task-type, and benchmark-level aggregations,
+        so filtering here ensures consistency across the entire chain.
+        """
         raw_results = defaultdict(list)
         for task_name, task_result in self.task_results.items():
-            for lang_metrics_result in task_result.datasetid_results.values():
-                for metric_name, metric_value in lang_metrics_result.metrics_dict.items():
+            for dataset_id, metrics_result in task_result.datasetid_results.items():
+                language_key = self._get_language_grouping_key(
+                    metrics_result, language_aggregation_mode
+                )
+                if language_key is None:
+                    logger.warning(
+                        "Skipping dataset '%s' of task '%s' in per-task aggregation: "
+                        "incompatible with mode '%s' "
+                        "(input_languages=%s, output_languages=%s).",
+                        dataset_id,
+                        task_name,
+                        language_aggregation_mode.value,
+                        metrics_result.input_languages,
+                        metrics_result.output_languages,
+                    )
+                    continue
+                for metric_name, metric_value in metrics_result.metrics_dict.items():
                     raw_results[(task_name, metric_name)].append(metric_value)
 
         # Compute stats
@@ -179,6 +236,7 @@ def _aggregate_per_task(
 
     def _aggregate_per_task_group(
         self,
+        language_aggregation_mode: LanguageAggregationMode,
         tag_name: str = "mean_per_task_group",
         aggregations: tuple = ("mean", "stderr", "ci_margin"),
         task_results: dict[ResultTagString, float] | None = None,
@@ -187,7 +245,9 @@ def _aggregate_per_task_group(
 
         First aggregates over languages within tasks, then over tasks within task groups.
         """
-        task_results = task_results or self._aggregate_per_task(aggregations=("mean",))
+        task_results = task_results or self._aggregate_datasetids_per_task(
+            language_aggregation_mode=language_aggregation_mode, aggregations=("mean",)
+        )
 
         task_group_list_results = defaultdict(list)
         for task_result_tag, value in task_results.items():
@@ -221,6 +281,7 @@ def _aggregate_per_task_group(
 
     def _aggregate_per_task_type(
         self,
+        language_aggregation_mode: LanguageAggregationMode,
         tag_name: str = "mean_per_task_type",
         aggregations: tuple = ("mean", "stderr", "ci_margin"),
         task_group_results: dict[ResultTagString, float] | None = None,
@@ -231,7 +292,7 @@ def _aggregate_per_task_type(
         then over task groups within task types.
         """
         task_group_results = task_group_results or self._aggregate_per_task_group(
-            aggregations=("mean",)
+            language_aggregation_mode=language_aggregation_mode, aggregations=("mean",)
         )
 
         # Mapping from task group name to task type name
@@ -275,6 +336,7 @@ def _aggregate_per_task_type(
 
     def _aggregate_benchmark(
         self,
+        language_aggregation_mode: LanguageAggregationMode,
         tag_name: str = "mean_benchmark",
         aggregations: tuple = ("mean", "stderr", "ci_margin"),
         task_type_results: dict[ResultTagString, float] | None = None,
@@ -288,7 +350,7 @@ def _aggregate_benchmark(
         4. Aggregates over task types for final benchmark scores
         """
         task_type_results = task_type_results or self._aggregate_per_task_type(
-            aggregations=("mean",)
+            language_aggregation_mode=language_aggregation_mode, aggregations=("mean",)
         )
 
         metric_list_results = defaultdict(list)