Enable callbacks for loading netcdf with multiple variables (#6754)

* enable callbacks for loading netcdf with multiple variables

* ruff format

* update docstring

* add integration test

* address review comments

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* address review comment

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: stephenworsley

docs/src/whatsnew/latest.rst

@@ -91,6 +91,11 @@ This document explains the changes made to Iris for this release
    is hoped that a future ``libnetcdf`` release will recover the original
    performance. See `netcdf-c#3183`_ for more details. (:pull:`6747`)
 
+#. `@stephenworsley`_ made NetCDF loading more efficient by filtering variables
+   before they become instantiated as cubes in the case where multiple name
+   constraints are given. This was previously only implemented where one such
+   constraint was given. (:issue:`6228`, :pull:`6754`)
+
 
 🔥 Deprecations
 ===============

lib/iris/fileformats/netcdf/loader.py

@@ -601,24 +601,26 @@ def _translate_constraints_to_var_callback(constraints):
 
     Notes
     -----
-    For now, ONLY handles a single NameConstraint with no 'STASH' component.
+    For now, ONLY handles NameConstraints with no 'STASH' component.
 
     """
     import iris._constraints
 
     constraints = iris._constraints.list_of_constraints(constraints)
-    result = None
-    if len(constraints) == 1:
-        (constraint,) = constraints
-        if (
-            isinstance(constraint, iris._constraints.NameConstraint)
-            and constraint.STASH == "none"
-        ):
-            # As long as it doesn't use a STASH match, then we can treat it as
-            # a testing against name properties of cf_var.
-            # That's just like testing against name properties of a cube, except that they may not all exist.
-            def inner(cf_datavar):
-                match = True
+    if len(constraints) == 0 or not all(
+        isinstance(constraint, iris._constraints.NameConstraint)
+        and constraint.STASH == "none"
+        for constraint in constraints
+    ):
+        # We can define a var-filtering function to speedup the load, *ONLY* when we
+        #  have some constraints, and all are simple NameConstraints with no STASH.
+        result = None
+    else:
+
+        def inner(cf_datavar):
+            match_any_constraint = False
+            for constraint in constraints:
+                match_this_constraint = True
                 for name in constraint._names:
                     expected = getattr(constraint, name)
                     if name != "STASH" and expected != "none":
@@ -629,11 +631,14 @@ def inner(cf_datavar):
                             continue
                         actual = getattr(cf_datavar, attr_name, "")
                         if actual != expected:
-                            match = False
+                            match_this_constraint = False
                             break
-                return match
+                if match_this_constraint:
+                    match_any_constraint = True
+                    break
+            return match_any_constraint
 
-            result = inner
+        result = inner
     return result
 
 

lib/iris/tests/integration/netcdf/test_general.py

@@ -314,6 +314,13 @@ def test_netcdf_with_NameConstraint(self):
         self.assertEqual(len(cubes), 1)
         self.assertEqual(cubes[0].var_name, "cdf_temp_dmax_tmean_abs")
 
+    def test_netcdf_with_2_NameConstraints(self):
+        var_names = ["cdf_temp_dmax_tmean_abs", "temp_dmax_tmean_abs"]
+        constrs = [iris.NameConstraint(var_name=var_name) for var_name in var_names]
+        cubes = iris.load(self.filename, constrs)
+        self.assertEqual(len(cubes), 2)
+        self.assertEqual(sorted([cube.var_name for cube in cubes]), var_names)
+
     def test_netcdf_with_no_constraint(self):
         cubes = iris.load(self.filename)
         self.assertEqual(len(cubes), 3)

lib/iris/tests/unit/fileformats/netcdf/loader/test__translate_constraints_to_var_callback.py

@@ -33,11 +33,35 @@ class Test(tests.IrisTest):
     def test_multiple_constraints(self):
         constrs = [
             iris.NameConstraint(standard_name="x_wind"),
-            iris.NameConstraint(var_name="var1"),
+            iris.NameConstraint(var_name="var2"),
+        ]
+        callback = _translate_constraints_to_var_callback(constrs)
+        result = [callback(var) for var in self.data_variables]
+        self.assertArrayEqual(result, [True, True, False, True, False])
+
+    def test_multiple_constraints_invalid(self):
+        constrs = [
+            iris.NameConstraint(standard_name="x_wind"),
+            iris.NameConstraint(var_name="var1", STASH="m01s00i024"),
         ]
         result = _translate_constraints_to_var_callback(constrs)
         self.assertIsNone(result)
 
+    def test_multiple_constraints__multiname(self):
+        # Modify the first constraint to require BOTH var-name and std-name match
+        constrs = [
+            iris.NameConstraint(standard_name="x_wind", var_name="var1"),
+            iris.NameConstraint(var_name="var2"),
+        ]
+        callback = _translate_constraints_to_var_callback(constrs)
+        # Add 2 extra vars: one passes both name checks, and the other does not
+        vars = self.data_variables + [
+            CFDataVariable("var1", MagicMock(standard_name="x_wind")),
+            CFDataVariable("var1", MagicMock(standard_name="air_pressure")),
+        ]
+        result = [callback(var) for var in vars]
+        self.assertArrayEqual(result, [True, True, False, True, False, True, False])
+
     def test_non_NameConstraint(self):
         constr = iris.AttributeConstraint(STASH="m01s00i002")
         result = _translate_constraints_to_var_callback(constr)
@@ -91,6 +115,11 @@ def test_NameConstraint_with_STASH(self):
         result = _translate_constraints_to_var_callback(constr)
         self.assertIsNone(result)
 
+    def test_no_constraints(self):
+        constrs = []
+        result = _translate_constraints_to_var_callback(constrs)
+        self.assertIsNone(result)
+
 
 if __name__ == "__main__":
     tests.main()