Content addressable prototype compact index

app/controllers/api/compact_index_controller.rb

@@ -11,6 +11,13 @@ class Api::CompactIndexController < Api::BaseController
       info_prefix: "v2/info",
       versions_file_location_key: "versions_file_location_v2",
       versions_surrogate_key: "v2/versions"
+    }.freeze,
+    # The public "/v2" endpoint serves the content-addressable index, which is
+    # internally the v3 format (v3/* storage prefixes, versions_file_location_v3).
+    3 => {
+      info_prefix: "v3/info",
+      versions_file_location_key: "versions_file_location_v3",
+      versions_surrogate_key: "v3/versions"
     }.freeze
   }.freeze
 
@@ -45,8 +52,17 @@ def info
   private
 
   def compact_index_serving_version
-    # Compact index responses are cached by URL, so this flag must only be toggled globally.
-    @compact_index_serving_version ||= FeatureFlag.enabled?(FeatureFlag::SERVE_COMPACT_INDEX_V2) ? 2 : 1
+    # The public /v2/* routes serve the content-addressable index (internally v3),
+    # pinned via a route default. The legacy routes fall back to the global flag.
+    # Responses are cached by URL, so the served version is stable per URL.
+    @compact_index_serving_version ||=
+      if params.key?(:compact_index_version)
+        params[:compact_index_version].to_i
+      elsif FeatureFlag.enabled?(FeatureFlag::SERVE_COMPACT_INDEX_V2)
+        2
+      else
+        1
+      end
   end
 
   def compact_index_config

app/jobs/after_version_write_job.rb

@@ -20,6 +20,7 @@ def perform(version:)
 
       version.info_checksum = gem_info.info_checksum(version: 1)
       version.info_checksum_v2 = gem_info.info_checksum(version: 2)
+      version.info_checksum_v3 = gem_info.info_checksum(version: 3)
       version.save(validate: false)
 
       SetLinksetHomeJob.perform_later(version:)

app/jobs/upload_info_file_job.rb

@@ -20,7 +20,7 @@ class InvalidBackfillVersion < ArgumentError; end
 
   discard_on InvalidBackfillVersion
 
-  PATH_PREFIXES = { 1 => "info", 2 => "v2/info" }.freeze
+  PATH_PREFIXES = { 1 => "info", 2 => "v2/info", 3 => "v3/info" }.freeze
 
   def perform(rubygem_name:, backfill_only_version: nil)
     unless backfill_only_version.nil? || PATH_PREFIXES.key?(backfill_only_version)
@@ -32,10 +32,13 @@ def perform(rubygem_name:, backfill_only_version: nil)
 
     if backfill_only_version
       response_body = upload_info_file(gem_info, rubygem_name, version: backfill_only_version, purge: false)
-      persist_backfill_checksum(rubygem_name, checksum: Digest::MD5.hexdigest(response_body)) if backfill_only_version == 2
+      # v1 checksums are already populated for every version; only the newer
+      # formats (v2, v3, ...) need their last-version checksum backfilled.
+      persist_backfill_checksum(rubygem_name, version: backfill_only_version, checksum: Digest::MD5.hexdigest(response_body)) if backfill_only_version >= 2
     else
       upload_info_file(gem_info, rubygem_name, version: 1, purge: true)
       upload_info_file(gem_info, rubygem_name, version: 2, purge: true)
+      upload_info_file(gem_info, rubygem_name, version: 3, purge: true)
     end
   end
 
@@ -76,7 +79,11 @@ def upload_info_file(gem_info, rubygem_name, version:, purge:)
     response_body
   end
 
-  def persist_backfill_checksum(rubygem_name, checksum:)
+  def persist_backfill_checksum(rubygem_name, version:, checksum:)
+    config = GemInfo::VERSIONS.fetch(version)
+    checksum_column = config[:checksum_column]
+    yanked_checksum_column = config[:yanked_checksum_column]
+
     rubygem = Rubygem.find_by(name: rubygem_name)
     return unless rubygem
 
@@ -87,9 +94,9 @@ def persist_backfill_checksum(rubygem_name, checksum:)
 
     scope = Version.where(id: last_version.id)
     if last_version.indexed
-      scope.where(indexed: true, info_checksum_v2: nil).update_all(info_checksum_v2: checksum)
+      scope.where(indexed: true, checksum_column => nil).update_all(checksum_column => checksum)
     else
-      scope.where(indexed: false, yanked_info_checksum_v2: nil).update_all(yanked_info_checksum_v2: checksum)
+      scope.where(indexed: false, yanked_checksum_column => nil).update_all(yanked_checksum_column => checksum)
     end
   end
 end

app/jobs/upload_names_file_job.rb

@@ -22,6 +22,7 @@ def perform
 
     upload_names_file(response_body, key: "names", surrogate_key: "names s3-compact-index s3-names")
     upload_names_file(response_body, key: "v2/names", surrogate_key: "v2/names s3-compact-index s3-v2/names")
+    upload_names_file(response_body, key: "v3/names", surrogate_key: "v3/names s3-compact-index s3-v3/names")
   end
 
   private

app/jobs/upload_versions_file_job.rb

@@ -34,6 +34,17 @@ def perform
       key: "v2/versions",
       surrogate_key: "v2/versions s3-compact-index s3-v2/versions"
     )
+
+    # V3 dual-write: only update v3/versions once the bootstrap rake task has seeded the local file.
+    v3_path = Rails.application.config.rubygems["versions_file_location_v3"]
+    return unless v3_path && File.exist?(v3_path)
+
+    upload_versions_file(
+      versions_file_location: v3_path,
+      extra_gems: GemInfo.compact_index_versions(versions_file_updated_at("versions_file_location_v3"), version: 3),
+      key: "v3/versions",
+      surrogate_key: "v3/versions s3-compact-index s3-v3/versions"
+    )
   end
 
   private

app/models/gem_info.rb

@@ -5,7 +5,9 @@ class GemInfo
     1 => { cache_prefix: "info", stats_prefix: "compact_index.memcached.info", klass: CompactIndex::GemVersion,
            checksum_column: "info_checksum", yanked_checksum_column: "yanked_info_checksum" },
     2 => { cache_prefix: "info_v2", stats_prefix: "compact_index.memcached.info_v2", klass: CompactIndex::GemVersionV2,
-           checksum_column: "info_checksum_v2", yanked_checksum_column: "yanked_info_checksum_v2" }
+           checksum_column: "info_checksum_v2", yanked_checksum_column: "yanked_info_checksum_v2" },
+    3 => { cache_prefix: "info_v3", stats_prefix: "compact_index.memcached.info_v3", klass: CompactIndex::GemVersionV3,
+           checksum_column: "info_checksum_v3", yanked_checksum_column: "yanked_info_checksum_v3" }
   }.freeze
 
   def initialize(rubygem_name, cached: true)
@@ -50,19 +52,19 @@ def self.compact_index_versions(date, version: 1)
     checksum_column = config[:checksum_column]
     yanked_checksum_column = config[:yanked_checksum_column]
 
-    query = ["(SELECT r.name, v.created_at as date, v.#{checksum_column} as info_checksum, v.number, v.platform
+    query = ["(SELECT r.name, v.created_at as date, v.#{checksum_column} as info_checksum, v.number, v.platform, v.sha256, v.ruby_minor
               FROM rubygems AS r, versions AS v
               WHERE v.rubygem_id = r.id AND
                     v.created_at > ?)
               UNION
-              (SELECT r.name, v.yanked_at as date, v.#{yanked_checksum_column} as info_checksum, '-'||v.number, v.platform
+              (SELECT r.name, v.yanked_at as date, v.#{yanked_checksum_column} as info_checksum, '-'||v.number, v.platform, v.sha256, v.ruby_minor
               FROM rubygems AS r, versions AS v
               WHERE v.rubygem_id = r.id AND
                     v.indexed is false AND
                     v.yanked_at > ?)
               ORDER BY date, number, platform, name", date, date]
 
-    map_gem_versions(execute_raw_sql(query).map { |v| [v["name"], [v]] })
+    map_gem_versions(execute_raw_sql(query).map { |v| [v["name"], [v]] }, version)
   end
 
   def self.compact_index_public_versions(updated_at, version: 1)
@@ -72,7 +74,7 @@ def self.compact_index_public_versions(updated_at, version: 1)
 
     query = ["SELECT r.name, v.indexed, COALESCE(v.yanked_at, v.created_at) as stamp,
                      v.sha256, COALESCE(v.#{yanked_checksum_column}, v.#{checksum_column}) as info_checksum,
-                     v.number, v.platform
+                     v.number, v.platform, v.ruby_minor
               FROM rubygems AS r, versions AS v
               WHERE v.rubygem_id = r.id AND
                     (v.created_at <= ? OR v.yanked_at <= ?)
@@ -87,22 +89,45 @@ def self.compact_index_public_versions(updated_at, version: 1)
     end
     versions_by_gem.reject! { |_, versions| versions.empty? }
 
-    map_gem_versions(versions_by_gem)
+    map_gem_versions(versions_by_gem, version)
   end
 
   def self.execute_raw_sql(query)
     sanitized_sql = ActiveRecord::Base.send(:sanitize_sql_array, query)
     ActiveRecord::Base.connection.execute(sanitized_sql)
   end
 
-  def self.map_gem_versions(versions_by_gem)
-    versions_by_gem.map do |gem_name, versions|
-      compact_index_versions = versions.map do |version|
-        CompactIndex::GemVersion.new(version["number"],
-          version["platform"],
-          version["sha256"],
-          version["info_checksum"])
+  def self.map_gem_versions(versions_by_gem, serving_version = 1)
+    # v3 lists "skinny" binaries by content address, so the version tokens in the
+    # versions file match the v3 info files. GemVersionV3#number_and_platform reads
+    # ruby_minor (skinny marker) and derives the address from the hex checksum.
+    content_addressable = serving_version >= 3
+
+    versions_by_gem.filter_map do |gem_name, versions|
+      compact_index_versions = versions.filter_map do |version|
+        skinny = version["platform"].present? && version["platform"] != "ruby" && version["ruby_minor"].present?
+
+        if content_addressable
+          sha256 = version["sha256"]
+          CompactIndex::GemVersionV3.new(
+            version["number"],
+            version["platform"],
+            sha256 && Version._sha256_hex(sha256),
+            version["info_checksum"],
+            nil, nil, nil, nil,
+            version["ruby_minor"]
+          )
+        elsif !skinny
+          # Skinny binaries only appear in the content-addressable (v3) index.
+          CompactIndex::GemVersion.new(
+            version["number"],
+            version["platform"],
+            version["sha256"],
+            version["info_checksum"]
+          )
+        end
       end
+      next if compact_index_versions.empty?
       CompactIndex::Gem.new(gem_name, compact_index_versions)
     end
   end
@@ -111,9 +136,9 @@ def self.map_gem_versions(versions_by_gem)
 
   private
 
-  DEPENDENCY_NAMES_INDEX = 8
+  DEPENDENCY_NAMES_INDEX = 9
 
-  DEPENDENCY_REQUIREMENTS_INDEX = 7
+  DEPENDENCY_REQUIREMENTS_INDEX = 8
 
   # Marshal.load of pre-deploy cache entries fails when GemVersion grows a Struct field.
   def read_cache(cache_key)
@@ -123,7 +148,15 @@ def read_cache(cache_key)
   end
 
   def compute_compact_index_info(version:)
-    requirements_and_dependencies.map do |row|
+    version_class = VERSIONS.dig(version, :klass)
+    requirements_and_dependencies.filter_map do |row|
+      number, platform, checksum, info_checksum, ruby_version, rubygems_version, created_at, ruby_minor, = row
+
+      # Skinny (content-addressable) binaries are served only from the v3
+      # (content-addressable) index. The legacy v1/v2 indexes carry source and
+      # fat binaries only, so older clients never see content-addressed entries.
+      next if skinny?(platform, ruby_minor) && version < 3
+
       dependencies = []
       if row[DEPENDENCY_REQUIREMENTS_INDEX]
         reqs = row[DEPENDENCY_REQUIREMENTS_INDEX].split("@")
@@ -134,22 +167,26 @@ def compute_compact_index_info(version:)
         end
       end
 
-      number, platform, checksum, info_checksum, ruby_version, rubygems_version, created_at, = row
-      version_class = VERSIONS.dig(version, :klass)
       checksum = Version._sha256_hex(checksum)
       created_at = created_at&.utc&.iso8601
-      args = { number:, platform:, checksum:, info_checksum:, dependencies:, ruby_version:, rubygems_version:, created_at: }
+      args = { number:, platform:, checksum:, info_checksum:, dependencies:, ruby_version:, rubygems_version:, created_at:, ruby_minor: }
       args = args.slice(*version_class.members)
       version_class.new(**args)
     end
   end
 
+  # A "skinny" (content-addressable) binary is a platformed gem with a pinned
+  # Ruby minor (ruby_minor is "" for source and fat binaries).
+  def skinny?(platform, ruby_minor)
+    platform.present? && platform != "ruby" && ruby_minor.present?
+  end
+
   def requirements_and_dependencies
     @requirements_and_dependencies ||= fetch_requirements_and_dependencies
   end
 
   def fetch_requirements_and_dependencies
-    group_by_columns = "number, platform, sha256, info_checksum, required_ruby_version, required_rubygems_version, versions.created_at"
+    group_by_columns = "number, platform, sha256, info_checksum, required_ruby_version, required_rubygems_version, versions.created_at, ruby_minor"
 
     dep_req_agg = "string_agg(dependencies.requirements, '@' ORDER BY rubygems_dependencies.name, dependencies.id)"
 

app/tasks/maintenance/backfill_compact_index_v3_task.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class Maintenance::BackfillCompactIndexV3Task < MaintenanceTasks::Task
+  attribute :min_rubygem_id, :integer
+  attribute :max_rubygem_id, :integer
+  attribute :force_upload_info_file_job, :boolean, default: false
+
+  def collection
+    scope = Rubygem.with_versions
+    scope = scope.where(rubygems: { id: min_rubygem_id.. }) if min_rubygem_id.present?
+    scope = scope.where(rubygems: { id: ..max_rubygem_id }) if max_rubygem_id.present?
+    scope
+  end
+
+  def process(rubygem)
+    last_version = rubygem.versions
+      .order(Arel.sql("COALESCE(yanked_at, created_at) DESC, number DESC, platform DESC"))
+      .first
+
+    return unless last_version
+    return if !force_upload_info_file_job && (last_version.info_checksum_v3.present? || last_version.yanked_info_checksum_v3.present?)
+
+    UploadInfoFileJob.perform_later(rubygem_name: rubygem.name, backfill_only_version: 3)
+  end
+end

config/routes.rb

@@ -137,6 +137,14 @@
   get '/info/:gem_name' => 'api/compact_index#info', as: :info,
       constraints: { gem_name: Patterns::ROUTE_PATTERN }
   get '/names' => 'api/compact_index#names'
+
+  # Content-addressable compact index (internally the v3 format). Serves "skinny"
+  # binaries addressed by content; the legacy endpoints above stay classic so
+  # older clients are unaffected.
+  get '/v2/versions' => 'api/compact_index#versions', defaults: { compact_index_version: 3 }
+  get '/v2/info/:gem_name' => 'api/compact_index#info', as: :info_v2,
+      constraints: { gem_name: Patterns::ROUTE_PATTERN }, defaults: { compact_index_version: 3 }
+  get '/v2/names' => 'api/compact_index#names', defaults: { compact_index_version: 3 }
   ################################################################################
   # API v0
 

config/rubygems.yml

@@ -9,6 +9,7 @@ development:
   s3_endpoint: s3.amazonaws.com
   versions_file_location: "./config/versions.list"
   versions_file_location_v2: "./config/versions_v2.list"
+  versions_file_location_v3: "./config/versions_v3.list"
 
 test:
   protocol: http
@@ -21,6 +22,7 @@ test:
   s3_endpoint: s3.amazonaws.com
   versions_file_location: "./test/helpers/versions.list"
   versions_file_location_v2: "./test/helpers/versions_v2.list"
+  versions_file_location_v3: "./test/helpers/versions_v3.list"
 
 staging:
   protocol: https
@@ -33,6 +35,7 @@ staging:
   s3_compact_index_bucket: compact-index.oregon.staging.s3.rubygems.org
   versions_file_location: "./config/versions.list"
   versions_file_location_v2: "./config/versions_v2.list"
+  versions_file_location_v3: "./config/versions_v3.list"
 
 production:
   protocol: https
@@ -45,4 +48,5 @@ production:
   s3_compact_index_bucket: compact-index.oregon.production.s3.rubygems.org
   versions_file_location: "./config/versions.list"
   versions_file_location_v2: "./config/versions_v2.list"
+  versions_file_location_v3: "./config/versions_v3.list"
   separate_admin_host: rubygems.team

db/migrate/20260605120000_add_info_checksum_v3_and_yanked_info_checksum_v3_to_versions.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class AddInfoChecksumV3AndYankedInfoChecksumV3ToVersions < ActiveRecord::Migration[8.1]
+  def change
+    safety_assured do
+      change_table :versions, bulk: true do |t|
+        t.string :info_checksum_v3
+        t.string :yanked_info_checksum_v3
+      end
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.1].define(version: 2026_06_04_133906) do
+ActiveRecord::Schema[8.1].define(version: 2026_06_05_120000) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "hstore"
   enable_extension "pg_catalog.plpgsql"
@@ -702,6 +702,7 @@
     t.boolean "indexed", default: true
     t.string "info_checksum"
     t.string "info_checksum_v2"
+    t.string "info_checksum_v3"
     t.boolean "latest"
     t.string "licenses"
     t.hstore "metadata", default: {}, null: false
@@ -724,6 +725,7 @@
     t.datetime "yanked_at", precision: nil
     t.string "yanked_info_checksum"
     t.string "yanked_info_checksum_v2"
+    t.string "yanked_info_checksum_v3"
     t.index "lower((full_name)::text)", name: "index_versions_on_lower_full_name"
     t.index "lower((gem_full_name)::text)", name: "index_versions_on_lower_gem_full_name"
     t.index ["built_at"], name: "index_versions_on_built_at"