Maintain bottle coverage during macOS bootstrap

Signed-off-by: Patrick Linnane <patrick@linnane.io>

Author: p-linnane

Library/Homebrew/bottle_transition.rb

@@ -0,0 +1,79 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "formula"
+require "formulary"
+require "test_runner_formula"
+
+# Preserve bottles while bootstrapping a macOS release.
+# TODO: remove this class and its callers when `HOMEBREW_MACOS_NEWEST_SUPPORTED` is `27`.
+class BottleTransition
+  MACOS = :golden_gate
+
+  sig { returns(Utils::Bottles::Tag) }
+  def self.tag
+    Utils::Bottles::Tag.new(system: MACOS, arch: :arm64)
+  end
+
+  sig { returns(T::Boolean) }
+  def self.active?
+    MacOSVersion.from_symbol(MACOS) > HOMEBREW_MACOS_NEWEST_SUPPORTED
+  end
+
+  sig { params(base_ref: T.nilable(String)).void }
+  def initialize(base_ref: nil)
+    if ENV.fetch("GITHUB_EVENT_NAME", nil) == "merge_group"
+      event = JSON.parse(File.read(ENV.fetch("GITHUB_EVENT_PATH")))
+      base_ref = event.dig("merge_group", "base_sha")
+      if !base_ref.is_a?(String) || !/\A[0-9a-f]{40}\z/.match?(base_ref)
+        raise UsageError, "Missing immutable merge-group base SHA."
+      end
+    end
+    base_branch = ENV.fetch("GITHUB_BASE_REF", nil).presence
+    @base_ref = T.let(base_ref || (base_branch ? "origin/#{base_branch}" : "origin/HEAD"), String)
+    @base_revision = T.let(nil, T.nilable(String))
+    @required = T.let({}, T::Hash[String, T::Boolean])
+  end
+
+  sig { params(formula: Formula).returns(T::Boolean) }
+  def required?(formula)
+    return false unless self.class.active?
+    return false unless formula.tap&.core_tap?
+    return false if formula.disabled?
+
+    compatible = Homebrew::SimulateSystem.with(os: MACOS, arch: :arm) do
+      current = TestRunnerFormula.new(Formulary.factory(formula.path))
+      current.macos_compatible? && current.arm64_compatible? &&
+        current.compatible_with?(MacOSVersion.from_symbol(MACOS))
+    end
+    return false unless compatible
+
+    @required.fetch(formula.name) do
+      repository = CoreTap.instance.path
+      @base_revision ||= Utils.safe_popen_read("git", "-C", repository, "rev-parse", "--verify",
+                                               "#{@base_ref}^{commit}").strip
+      path = formula.tap_path.relative_path_from(repository)
+      files = Utils.safe_popen_read("git", "-C", repository, "ls-tree", "--name-only", "-z",
+                                    @base_revision, "--", path)
+      return @required[formula.name] = false if files.empty?
+
+      contents = Utils.safe_popen_read("git", "-C", repository, "show", "#{@base_revision}:#{path}")
+      @required[formula.name] = Homebrew::SimulateSystem.with(os: MACOS, arch: :arm) do
+        previous = Formulary.from_contents(formula.name, formula.tap_path, contents)
+        previous.bottle_specification.collector.tags.include?(self.class.tag)
+      end
+    end
+  end
+
+  sig { params(formula: Formula, tags: T::Array[Utils::Bottles::Tag]).void }
+  def check!(formula, tags:)
+    return unless required?(formula)
+    return if tags.include?(self.class.tag) || tags.include?(Utils::Bottles.tag(:all))
+
+    raise UsageError, <<~EOS
+      #{formula.full_name} already has #{self.class.tag} bottle coverage on #{@base_ref}.
+      Build an #{self.class.tag} bottle for #{formula.pkg_version} before publishing or merging.
+      Re-run CI against the updated base branch if mass bottling finished after CI started.
+    EOS
+  end
+end

Library/Homebrew/dev-cmd/pr-upload.rb

@@ -8,6 +8,7 @@
 require "github_packages"
 require "github_releases"
 require "extend/hash/deep_merge"
+require "bottle_transition"
 
 module Homebrew
   module DevCmd
@@ -108,6 +109,8 @@ def run
           end
         end
 
+        check_transition_bottles!(bottles_hash)
+
         if github_releases?(bottles_hash)
           github_releases = GitHubReleases.new
           github_releases.upload_bottles(bottles_hash)
@@ -122,6 +125,44 @@ def run
         end
       end
 
+      # Verify transition bottles before upload.
+      sig { params(bottles_hash: T::Hash[String, T.untyped]).void }
+      def check_transition_bottles!(bottles_hash)
+        # Reload the bottle block written by `brew bottle --merge`.
+        Formulary.clear_cache
+        transition = BottleTransition.new
+        bottles_hash.each_value do |bottle_hash|
+          formula_path = HOMEBREW_REPOSITORY/bottle_hash.fetch("formula").fetch("path")
+          formula = Formulary.factory(formula_path)
+          next unless transition.required?(formula)
+
+          spec = formula.bottle_specification
+          transition.check!(formula, tags: spec.collector.tags)
+          bottle = bottle_hash.fetch("bottle")
+          tags = bottle.fetch("tags").keys.map { |tag| Utils::Bottles.tag(tag.to_sym) }
+          if !args.keep_old? && tags.exclude?(BottleTransition.tag) && tags.exclude?(Utils::Bottles.tag(:all))
+            raise UsageError, <<~EOS
+              #{formula.full_name} #{formula.pkg_version}: upload set is missing
+              #{BottleTransition.tag} or `all` bottle artifacts.
+              Restore the matching bottle artifacts and JSON files before retrying.
+            EOS
+          end
+
+          root_url = bottle.fetch("root_url")
+          metadata_matches = bottle_hash.fetch("formula").fetch("pkg_version") == formula.pkg_version.to_s &&
+                             bottle.fetch("rebuild", 0).to_i == spec.rebuild &&
+                             (GitHubPackages.root_url_if_match(root_url) || root_url) == spec.root_url &&
+                             bottle.fetch("tags").all? do |tag, tag_hash|
+                               tag_spec = spec.collector.specification_for(Utils::Bottles.tag(tag.to_sym),
+                                                                           no_older_versions: true)
+                               tag_spec && tag_spec.checksum.hexdigest == tag_hash.fetch("sha256")
+                             end
+          next if metadata_matches
+
+          raise UsageError, "#{formula.full_name}: bottle metadata does not match the committed formula."
+        end
+      end
+
       private
 
       sig { params(bottles_hash: T::Hash[String, T.untyped]).void }
@@ -154,12 +195,28 @@ def github_packages?(bottles_hash)
         end, T.nilable(T::Boolean))
       end
 
+      public
+
+      # Merge compatible bottle metadata.
       sig { params(json_files: T::Array[String], args: T.untyped).returns(T::Hash[String, T.untyped]) }
       def bottles_hash_from_json_files(json_files, args)
         puts "Reading JSON files: #{json_files.join(", ")}" if args.verbose?
 
         bottles_hash = json_files.reduce({}) do |hash, json_file|
-          hash.deep_merge(JSON.parse(File.read(json_file)))
+          incoming = JSON.parse(File.read(json_file))
+          incoming.each do |name, bottle_hash|
+            bottle = bottle_hash.fetch("bottle")
+            bottle["root_url"] = GitHubPackages.root_url_if_match(bottle["root_url"]) || bottle["root_url"]
+            previous = hash[name]
+            next unless previous
+            next if previous.fetch("formula").slice("path", "pkg_version") ==
+                    bottle_hash.fetch("formula").slice("path", "pkg_version") &&
+                    previous.fetch("bottle").slice("root_url", "rebuild") ==
+                    bottle_hash.fetch("bottle").slice("root_url", "rebuild")
+
+            raise UsageError, "Inconsistent bottle metadata for #{name} in #{json_file}."
+          end
+          hash.deep_merge(incoming)
         end
 
         if args.root_url

Library/Homebrew/github_runner_matrix.rb

@@ -3,8 +3,12 @@
 
 require "test_runner_formula"
 require "github_runner"
+require "bottle_transition"
+require "utils/output"
 
 class GitHubRunnerMatrix
+  include Utils::Output::Mixin
+
   # When bumping newest runner, run e.g. `git log -p --reverse -G "sha256 tahoe"`
   # on homebrew/core and tag the first commit with a bottle e.g.
   # `git tag 15-sequoia f42c4a659e4da887fc714f8f41cc26794a4bb320`
@@ -209,7 +213,57 @@ def create_runner(platform, arch, spec, macos_version = nil)
 
   sig { params(macos_version: MacOSVersion).returns(T::Boolean) }
   def runner_enabled?(macos_version)
-    macos_version.between?(OLDEST_HOMEBREW_CORE_MACOS_RUNNER, NEWEST_HOMEBREW_CORE_MACOS_RUNNER)
+    return true if macos_version.between?(OLDEST_HOMEBREW_CORE_MACOS_RUNNER, NEWEST_HOMEBREW_CORE_MACOS_RUNNER)
+    return false if @all_supported || @dependent_matrix
+
+    macos_version.to_sym == BottleTransition::MACOS && transition_formulae.present?
+  end
+
+  sig { returns(T::Array[TestRunnerFormula]) }
+  def transition_formulae
+    @transition_formulae ||= T.let(begin
+      transition = BottleTransition.new
+      covered = @testing_formulae.select { |formula| transition.required?(formula.formula) }
+      needed_names = []
+      testing_names = @testing_formulae.map(&:name)
+
+      Homebrew::SimulateSystem.with(os: BottleTransition::MACOS, arch: :arm) do
+        covered.each do |formula|
+          dependencies = Formulary.factory(formula.name).recursive_dependencies do |dependent, dependency|
+            next Dependable::PRUNE if dependency.optional?
+
+            # Let the runner resolve older bottles for test-only dependencies.
+            if dependency.test? && !dependency.build? && testing_names.exclude?(dependency.name)
+              next Dependable::PRUNE
+            end
+            if dependency.is_a?(UsesFromMacOSDependency) && dependency.use_macos_install?
+              next Dependable::PRUNE
+            end
+            next unless dependent.is_a?(Formula)
+            next unless dependency.build?
+            next if testing_names.include?(dependent.name)
+
+            Dependable::PRUNE unless dependent.bottle_specification.tag?(Utils::Bottles.tag(:all))
+          end
+          missing = dependencies.reject do |dependency|
+            testing_names.include?(dependency.name) ||
+              dependency.to_formula.bottle_specification.tag?(BottleTransition.tag, no_older_versions: true)
+          end
+          if missing.present?
+            opoo <<~EOS
+              Skipping #{formula.name}'s #{BottleTransition.tag} build: missing bottles for #{missing.map(&:name).join(", ")}.
+              Bootstrap these dependencies and retry CI before publishing.
+            EOS
+            next
+          end
+
+          needed_names << formula.name
+          needed_names.concat(dependencies.map(&:name) & testing_names)
+        end
+      end
+
+      @testing_formulae.select { |formula| needed_names.include?(formula.name) }
+    end, T.nilable(T::Array[TestRunnerFormula]))
   end
 
   sig { returns(String) }
@@ -254,8 +308,12 @@ def compatible_testing_formulae(runner)
       arch = runner.arch
       macos_version = runner.macos_version
 
-      @testing_formulae.select do |formula|
-        Homebrew::SimulateSystem.with(os: platform, arch: Homebrew::SimulateSystem.arch_symbols.fetch(arch)) do
+      transition_runner = BottleTransition.active? && macos_version&.to_sym == BottleTransition::MACOS
+      testing_formulae = transition_runner ? transition_formulae : @testing_formulae
+      os = transition_runner ? BottleTransition::MACOS : platform
+
+      testing_formulae.select do |formula|
+        Homebrew::SimulateSystem.with(os:, arch: Homebrew::SimulateSystem.arch_symbols.fetch(arch)) do
           simulated_formula = TestRunnerFormula.new(Formulary.factory(formula.name))
           next false if macos_version && !simulated_formula.compatible_with?(macos_version)