ci: Fix benchmark version overrides crudely

misc/python/materialize/git.py

@@ -148,7 +148,7 @@ def get_latest_version(
 
 def get_tags_of_current_commit(include_tags: YesNoOnce = YesNoOnce.ONCE) -> list[str]:
     if include_tags:
-        fetch(include_tags=include_tags, only_tags=True)
+        fetch(get_remote(), include_tags=include_tags, only_tags=True)
 
     result = spawn.capture(["git", "tag", "--points-at", "HEAD"])
 

misc/python/materialize/scalability/endpoint/endpoints.py

@@ -126,7 +126,7 @@ def __init__(
         self,
         composition: Composition,
         specified_target: str,
-        resolved_target: str,
+        resolved_target: str | None,
         use_balancerd: bool,
         image: str | None = None,
         alternative_image: str | None = None,
@@ -141,7 +141,7 @@ def __init__(
         self.use_balancerd = use_balancerd
         super().__init__(specified_target)
 
-    def resolved_target(self) -> str:
+    def resolved_target(self) -> str | None:
         return self._resolved_target
 
     def port(self) -> int:

misc/python/materialize/version_list.py

@@ -130,7 +130,7 @@ def get_supported_self_managed_versions() -> list[MzVersion]:
 _SKIP_IMAGE_CHECK_BELOW_THIS_VERSION = MzVersion.parse_mz("v0.77.0")
 
 
-def resolve_ancestor_image_tag(ancestor_overrides: dict[str, MzVersion]) -> str:
+def resolve_ancestor_image_tag(ancestor_overrides: dict[str, MzVersion]) -> str | None:
     """
     Resolve the ancestor image tag.
     :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS
@@ -148,7 +148,10 @@ def resolve_ancestor_image_tag(ancestor_overrides: dict[str, MzVersion]) -> str:
         return image_tag
 
     ancestor_image_resolution = _create_ancestor_image_resolution(ancestor_overrides)
-    image_tag, context = ancestor_image_resolution.resolve_image_tag()
+    result = ancestor_image_resolution.resolve_image_tag()
+    if result is None:
+        return None
+    image_tag, context = result
     print(f"Using {image_tag} as image tag for ancestor (context: {context})")
     return image_tag
 
@@ -201,7 +204,7 @@ def _get_override_commit_instead_of_version(
 
     def _resolve_image_tag_of_previous_release(
         self, context_prefix: str, previous_minor: bool
-    ) -> tuple[str, str]:
+    ) -> tuple[str, str] | None:
         tagged_release_version = git.get_tagged_release_version(version_type=MzVersion)
         assert tagged_release_version is not None
         previous_release_version = get_previous_published_version(
@@ -213,11 +216,13 @@ def _resolve_image_tag_of_previous_release(
         )
 
         if override_commit is not None:
+            # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use
             # use the commit instead of the previous release
-            return (
-                commit_to_image_tag(override_commit),
-                f"commit override instead of previous release ({previous_release_version})",
-            )
+            # return (
+            #     commit_to_image_tag(override_commit),
+            #     f"commit override instead of previous release ({previous_release_version})",
+            # )
+            return None
 
         return (
             release_version_to_image_tag(previous_release_version),
@@ -226,7 +231,7 @@ def _resolve_image_tag_of_previous_release(
 
     def _resolve_image_tag_of_previous_release_from_current(
         self, context: str
-    ) -> tuple[str, str]:
+    ) -> tuple[str, str] | None:
         # Even though we are on main we might be in an older state, pick the
         # latest release that was before our current version.
         current_version = MzVersion.parse_cargo()
@@ -239,11 +244,13 @@ def _resolve_image_tag_of_previous_release_from_current(
         )
 
         if override_commit is not None:
+            # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use
             # use the commit instead of the latest release
-            return (
-                commit_to_image_tag(override_commit),
-                f"commit override instead of latest release ({previous_published_version})",
-            )
+            # return (
+            #     commit_to_image_tag(override_commit),
+            #     f"commit override instead of latest release ({previous_published_version})",
+            # )
+            return None
 
         return (
             release_version_to_image_tag(previous_published_version),
@@ -254,7 +261,7 @@ def _resolve_image_tag_of_merge_base(
         self,
         context_when_image_of_commit_exists: str,
         context_when_falling_back_to_latest: str,
-    ) -> tuple[str, str]:
+    ) -> tuple[str, str] | None:
         # If the current PR has a known and accepted regression, don't compare
         # against merge base of it
         override_commit = self._get_override_commit_instead_of_version(
@@ -263,10 +270,12 @@ def _resolve_image_tag_of_merge_base(
         common_ancestor_commit = buildkite.get_merge_base()
 
         if override_commit is not None:
-            return (
-                commit_to_image_tag(override_commit),
-                f"commit override instead of merge base ({common_ancestor_commit})",
-            )
+            # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use
+            # return (
+            #     commit_to_image_tag(override_commit),
+            #     f"commit override instead of merge base ({common_ancestor_commit})",
+            # )
+            return None
 
         if image_of_commit_exists(common_ancestor_commit):
             return (
@@ -281,7 +290,7 @@ def _resolve_image_tag_of_merge_base(
 
 
 class AncestorImageResolutionLocal(AncestorImageResolutionBase):
-    def resolve_image_tag(self) -> tuple[str, str]:
+    def resolve_image_tag(self) -> tuple[str, str] | None:
         if build_context.is_on_release_version():
             return self._resolve_image_tag_of_previous_release(
                 "previous minor release because on local release branch",
@@ -299,7 +308,7 @@ def resolve_image_tag(self) -> tuple[str, str]:
 
 
 class AncestorImageResolutionInBuildkite(AncestorImageResolutionBase):
-    def resolve_image_tag(self) -> tuple[str, str]:
+    def resolve_image_tag(self) -> tuple[str, str] | None:
         if buildkite.is_in_pull_request():
             return self._resolve_image_tag_of_merge_base(
                 "merge base of pull request",

test/feature-benchmark/mzcompose.py

@@ -319,10 +319,12 @@ def run_one_scenario(
     return result
 
 
-resolved_tags: dict[tuple[str, frozenset[tuple[str, MzVersion]]], str] = {}
+resolved_tags: dict[tuple[str, frozenset[tuple[str, MzVersion]]], str | None] = {}
 
 
-def resolve_tag(tag: str, scenario_class: type[Scenario], scale: str | None) -> str:
+def resolve_tag(
+    tag: str, scenario_class: type[Scenario], scale: str | None
+) -> str | None:
     if tag == "common-ancestor":
         overrides = get_ancestor_overrides_for_performance_regressions(
             scenario_class, scale

test/parallel-benchmark/mzcompose.py

@@ -70,7 +70,7 @@
 PARALLEL_BENCHMARK_FRAMEWORK_VERSION = "1.2.0"
 
 
-def known_regression(scenario: str, other_tag: str) -> bool:
+def known_regression(scenario: str, other_tag: str | None) -> bool:
     return False
 
 
@@ -567,7 +567,7 @@ def less_than_is_regression(stat: str) -> bool:
 def check_regressions(
     this_stats: dict[Scenario, dict[str, Statistics]],
     other_stats: dict[Scenario, dict[str, Statistics]],
-    other_tag: str,
+    other_tag: str | None,
 ) -> list[TestFailureDetails]:
     failures: list[TestFailureDetails] = []
 
@@ -641,7 +641,7 @@ def check_regressions(
     return failures
 
 
-def resolve_tag(tag: str) -> str:
+def resolve_tag(tag: str) -> str | None:
     if tag == "common-ancestor":
         # TODO: We probably will need overrides too
         return resolve_ancestor_image_tag({})

test/version-consistency/mzcompose.py

@@ -138,7 +138,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
         raise FailedTestExecutionError(errors=test_summary.failures)
 
 
-def resolve_tag(tag: str) -> str:
+def resolve_tag(tag: str) -> str | None:
     if tag == "common-ancestor":
         return resolve_ancestor_image_tag(
             ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS