test(integration): catch on-disk storage format breaks pre-merge (#4003)

Author: hubcio

.config/nextest.toml

@@ -15,6 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# Floor for the flags CI and scripts/ci/storage-compat.sh pass: --no-tests
+# (0.9.75), --run-ignored only (0.9.76) and --ignore-default-filter (0.9.77).
+# nextest refuses to run below it, and setup-rust-with-cache replaces a
+# cache-restored binary that fails `cargo nextest show-config version`.
+nextest-version = { required = "0.9.77" }
+
 [[profile.default.overrides]]
 # This is a solution (or actually a workaround) for the problem that nextest does not support
 # #[serial] macro which shall enforce sequential execution of the test case.

.github/actions/rust/pre-merge/action.yml

@@ -20,7 +20,7 @@ description: Rust pre-merge testing and linting github iggy actions
 
 inputs:
   task:
-    description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-1, test-2, test-3, miri)"
+    description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-1, test-2, test-3, test-storage-compat, miri)"
     required: true
   component:
     description: "Component name (for context)"
@@ -84,19 +84,22 @@ runs:
     # a subset of crates changed.
     # Safety: cargo check/clippy run on the full workspace separately, catching all
     # compilation errors. This only scopes test BUILD and EXECUTION.
+    # test-storage-compat is excluded from the DAG and coverage setup below:
+    # storage-compat.sh builds two explicit binaries and drives its own nextest
+    # run, and reads none of the /tmp plan files or the llvm-cov environment.
     - name: Fetch base branch for DAG analysis
-      if: startsWith(inputs.task, 'test-')
+      if: startsWith(inputs.task, 'test-') && inputs.task != 'test-storage-compat'
       run: git fetch origin master --depth=1 2>/dev/null || true
       shell: bash
 
     - name: Install cargo-rail
-      if: startsWith(inputs.task, 'test-')
+      if: startsWith(inputs.task, 'test-') && inputs.task != 'test-storage-compat'
       uses: taiki-e/install-action@v2
       with:
         tool: cargo-rail
 
     - name: Compute affected crates (cargo-rail)
-      if: startsWith(inputs.task, 'test-')
+      if: startsWith(inputs.task, 'test-') && inputs.task != 'test-storage-compat'
       run: |
         METADATA_JSON=$(cargo metadata --format-version 1 --no-deps 2>/dev/null || echo "{}")
         TOTAL_CRATES=$(echo "$METADATA_JSON" | jq '.workspace_members | length' 2>/dev/null || echo "?")
@@ -202,18 +205,25 @@ runs:
       shell: bash
 
     - name: Install cargo-llvm-cov
-      if: startsWith(inputs.task, 'test-')
+      if: startsWith(inputs.task, 'test-') && inputs.task != 'test-storage-compat'
       uses: taiki-e/install-action@v2
       with:
         tool: cargo-llvm-cov
 
     - name: Build and test with coverage
-      if: startsWith(inputs.task, 'test-')
+      # test-storage-compat drives its own nextest invocation from
+      # scripts/ci/storage-compat.sh. Without this exclusion it would also run
+      # here with an empty PARTITION_FLAG, i.e. a fourth *unpartitioned* copy
+      # of the whole suite on top of test-1/2/3.
+      if: startsWith(inputs.task, 'test-') && inputs.task != 'test-storage-compat'
       run: |
         # Parse partition index from task name (test-1 -> hash:1/3, test-2 -> hash:2/3, ...).
         # TEST_PARTITIONS must match the number of test-N tasks in
         # .github/config/components.yml. Cluster bootstrap makes each test
         # CPU-heavy, so partitions stay small.
+        # The regex is numeric-only on purpose: non-numeric test-* tasks
+        # (test-storage-compat) opt out of partitioning rather than producing
+        # an out-of-range hash:N/3.
         TEST_PARTITIONS=3
         TASK="${{ inputs.task }}"
         PARTITION_FLAG=""
@@ -383,6 +393,17 @@ runs:
         ls -la codecov.json
       shell: bash
 
+    # On-disk format backwards compatibility. The script builds an iggy-server
+    # from the master tip under the checked-out merge ref (origin/master off a
+    # pull_request run) and one from HEAD, seeds a data directory with the
+    # former and reads it back with the latter. It does its own shallow fetch
+    # of the baseline commit and exits non-zero if it cannot resolve one, so
+    # the default depth-1 checkout is sufficient here.
+    - name: Storage format backwards compatibility
+      if: inputs.task == 'test-storage-compat'
+      run: ./scripts/ci/storage-compat.sh
+      shell: bash
+
     # Miri (UB detector) on the unsafe-heavy crates that don't pull tokio /
     # compio. Pinned nightly so MIRIFLAGS behavior is stable across CI runs;
     # bump the date quarterly. Tree-borrows is the future-default aliasing

.github/actions/utils/setup-rust-with-cache/action.yml

@@ -125,9 +125,12 @@ runs:
     - name: Install cargo-nextest
       if: runner.os == 'Linux' && inputs.install-nextest == 'true'
       run: |
-        if command -v cargo-nextest &> /dev/null; then
+        # A binary restored from the cargo cache can predate the floor in
+        # .config/nextest.toml. `show-config version` exits non-zero below it,
+        # and on a nextest too old to know the subcommand at all, so both fall
+        # through to a fresh install instead of aborting the lane later.
+        if command -v cargo-nextest &> /dev/null && cargo nextest show-config version; then
           echo "cargo-nextest already installed"
-          cargo nextest --version
           exit 0
         fi
 

.github/config/components.yml

@@ -135,6 +135,43 @@ components:
       - "core/message_bus/**"
       - "core/partitions/**"
 
+  # On-disk format backwards compatibility. Boots a server built from the
+  # master tip the PR merges onto, seeds a data directory, then swaps in a
+  # HEAD-built binary and asserts it reads everything back. Paths are the
+  # crates that own the on-disk representation plus the boot path that reads
+  # it back; a change to any of them can silently break rollforward for
+  # existing deployments, which no other check covers. Matching is literal
+  # glob with no crate-graph expansion, so every crate has to be spelled out:
+  # `core/server/**` does not cover `core/server_common/**`.
+  # `breaking:storage` on the PR skips it (see .github/workflows/_test.yml).
+  rust-storage-compat:
+    depends_on:
+      - "rust-workspace"
+      - "rust-configs" # partition/metadata on-disk schema knobs and their defaults
+      - "ci-infrastructure" # action.yml/workflow edits should re-run the check
+    paths:
+      - "core/server/**"
+      - "core/server_common/**" # per-message segment record + segment readers/writers
+      - "core/binary_protocol/**"
+      - "core/common/**"
+      - "core/consensus/**" # VsrState is the superblock payload
+      - "core/journal/**"
+      - "core/metadata/**"
+      - "core/partitions/**"
+      - "core/shard/**" # drives the shutdown flush and boot-time partition recovery
+      - "scripts/ci/storage-compat.sh"
+      # The check itself, so weakening it re-runs it. Scoped to the test, the
+      # module wiring that keeps it compiled in, the crate manifest and the
+      # harness it drives rather than `core/integration/**`, which would fire
+      # this lane on every unrelated integration test change.
+      - "core/integration/Cargo.toml"
+      - "core/integration/tests/mod.rs"
+      - "core/integration/tests/data_integrity/mod.rs"
+      - "core/integration/tests/data_integrity/storage_compat.rs"
+      - "core/integration/src/harness/**"
+    tasks:
+      - "test-storage-compat"
+
   # Standalone simulation tool, does NOT affect server binary or foreign SDKs.
   # Split from rust-cluster to avoid triggering SDK tests on simulator-only changes.
   rust-simulator:

.github/workflows/_test.yml

@@ -33,6 +33,7 @@ on:
 
 permissions:
   contents: read
+  issues: read # listLabelsOnIssue, for the breaking:storage escape hatch
   security-events: write
   pull-requests: write
 
@@ -55,15 +56,62 @@ jobs:
         run: echo "No changes detected, skipping tests"
 
       # Rust
+      # `breaking:storage` is the documented escape hatch for a deliberate
+      # on-disk format change, which the compat check cannot pass by
+      # construction. The label has to be read live: pre-merge.yml does not
+      # subscribe to `labeled` (pr-triage-apply.yml writes S-* labels, so every
+      # triage write would re-run the whole gate), and a re-run replays the
+      # webhook-frozen `pull_request.labels` from the last push. The operator
+      # flow is "check fails, apply label, re-run", which the frozen payload
+      # cannot see. Paginated so a PR with more than 100 labels still resolves.
+      - name: Resolve breaking:storage escape hatch
+        id: storage_hatch
+        if: >-
+          startsWith(inputs.component, 'rust') &&
+          inputs.task == 'test-storage-compat' &&
+          github.event.pull_request.number
+        uses: actions/github-script@v9
+        with:
+          script: |
+            let skip = false;
+            try {
+              const labels = await github.paginate(
+                github.rest.issues.listLabelsOnIssue,
+                {
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.payload.pull_request.number,
+                  per_page: 100,
+                },
+              );
+              skip = labels.some(l => l.name === 'breaking:storage');
+            } catch (e) {
+              // Fail toward running. A rate limit or a narrowed token must not
+              // wave an on-disk format break through as a green check.
+              core.warning(`listLabelsOnIssue failed (${e.status ?? e.code ?? 'network'}), running the check`);
+            }
+            core.info(`breaking:storage escape hatch: skip=${skip}`);
+            core.setOutput('skip', String(skip));
+
       - name: Run Rust task
-        if: startsWith(inputs.component, 'rust')
+        # Skipping the step (not the job) keeps the leg green so the pre-merge
+        # status roll-up stays reportable instead of pending.
+        if: >-
+          startsWith(inputs.component, 'rust') &&
+          steps.storage_hatch.outputs.skip != 'true'
         uses: ./.github/actions/rust/pre-merge
         with:
           task: ${{ inputs.task }}
           component: ${{ inputs.component }}
 
       - name: Upload coverage to Codecov
-        if: startsWith(inputs.component, 'rust') && startsWith(inputs.task, 'test-')
+        # test-storage-compat runs its own nextest invocation without llvm-cov,
+        # so it emits no codecov.json. Uploading an empty report under the
+        # `rust` flag would read as a coverage drop on the PR.
+        if: >-
+          startsWith(inputs.component, 'rust') &&
+          startsWith(inputs.task, 'test-') &&
+          inputs.task != 'test-storage-compat'
         uses: codecov/codecov-action@v7.0.0
         with:
           token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/pre-merge.yml

@@ -31,6 +31,8 @@ concurrency:
 
 permissions:
   contents: read
+  # listLabelsOnIssue in _test.yml, for the breaking:storage escape hatch
+  issues: read
   security-events: write
   pull-requests: write
 

core/integration/src/harness/handle/server.rs

@@ -726,6 +726,17 @@ impl ServerHandle {
         self.test_transport = Some(transport);
     }
 
+    /// Point the next `start()` at a different server binary.
+    ///
+    /// `start()` re-reads `config.executable_path` on every call, so a test
+    /// can boot one build, then restart the SAME data directory under another
+    /// one. `None` restores the cargo-built binary of the crate under test,
+    /// which is why this takes an explicit `Option` rather than
+    /// `impl Into<String>`.
+    pub fn set_executable_path(&mut self, path: Option<String>) {
+        self.config.executable_path = path;
+    }
+
     /// Configure MCP server for this iggy server.
     pub fn set_mcp_config(&mut self, config: McpConfig) {
         self.mcp = Some(McpHandle::with_server_id(

core/integration/tests/data_integrity/mod.rs

@@ -39,3 +39,8 @@ mod verify_cluster_replica_data_identical;
 // Auto-commit offset replication is inherently a multi-node (VSR) property: the
 // backup only holds the offset if the poll's auto-commit rode consensus.
 mod verify_auto_commit_offset_replicates;
+
+// On-disk format compatibility across a binary swap. `#[ignore]`d: it needs a
+// baseline `iggy-server` built from the merge base, which only
+// `scripts/ci/storage-compat.sh` provides.
+mod storage_compat;