Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion nemo/collections/common/metrics/classification_accuracy.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,10 @@ def compute_topk_accuracy(correct_counts_k, total_counts_k):


class ExactStringPerCategoryMatchMetric(Metric):
def __init__(self, categories=[], dist_sync_on_step=False, *args, **kwargs):
def __init__(self, categories=None, dist_sync_on_step=False, *args, **kwargs):
super().__init__(dist_sync_on_step=dist_sync_on_step)
if categories is None:
categories = []
self.categories = set(categories)

self.add_state("correct", default=torch.tensor(0), dist_reduce_fx="sum")
Expand Down
25 changes: 24 additions & 1 deletion tests/collections/common/test_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@
import pytest
import torch

from nemo.collections.common.metrics.classification_accuracy import TopKClassificationAccuracy
from nemo.collections.common.metrics.classification_accuracy import (
ExactStringPerCategoryMatchMetric,
TopKClassificationAccuracy,
)
from nemo.collections.common.metrics.punct_er import (
DatasetPunctuationErrorRate,
OccurancePunctuationErrorRate,
Expand Down Expand Up @@ -86,6 +89,26 @@ def test_top_1_accuracy_distributed(self):

assert abs(acc_top1 - 0.5) < 1e-3 # 3/6

@pytest.mark.unit
def test_exact_string_per_category_match_metric_default_categories_is_none(self):
"""Default categories should be None to avoid shared mutable state."""
import inspect

sig = inspect.signature(ExactStringPerCategoryMatchMetric.__init__)
assert sig.parameters['categories'].default is None

@pytest.mark.unit
def test_exact_string_per_category_match_metric_tracks_categories(self):
"""Metric should track per-category correct/total counts."""
metric = ExactStringPerCategoryMatchMetric(categories=['a', 'b'])
metric.update(pred='x', target='x', category='a')
metric.update(pred='x', target='y', category='b')

assert metric.a_total == 1
assert metric.a_correct == 1
assert metric.b_total == 1
assert metric.b_correct == 0

@pytest.mark.unit
def test_top_1_accuracy_distributed_uneven_batch(self):
# Simulate test on 2 process DDP execution
Expand Down
Loading