Merge pull request #2713 from hyperledger-labs/cocreature/canton-3.4-main

Merge remote-tracking branch 'origin/main' into canton-3.4

Author: moritzkiefer-da

.github/actions/nix/run_bash_command_in_nix/action.yml

@@ -35,6 +35,8 @@ runs:
           target="default"
         fi
 
+        echo "Running command with nix version $(nix --version)"
+
         # Attempt to build the nix env with retries to work around transient download failures
         echo "building nix env"
         MAX_RETRY_NIX_ENV=5
@@ -48,6 +50,7 @@ runs:
           fi
           nix develop \
             path:nix#$target \
+            --verbose \
             --ignore-environment \
             --keep USER \
             --keep HOME \
@@ -142,6 +145,7 @@ runs:
         # Run the command for real within the nix environment
         nix develop \
           path:nix#$target \
+          --verbose \
           --ignore-environment \
           --keep USER \
           --keep HOME \

.github/actions/nix/setup_nix/action.yml

@@ -38,9 +38,14 @@ runs:
       shell: bash
       run: |
         set -euxo pipefail
+        NIX_BINARY_VERSION=2.32.0
+        echo "NIX_BINARY_VERSION=$NIX_BINARY_VERSION" >> $GITHUB_ENV
+        cat nix/canton-sources.json
         git ls-files nix/ | grep -v '[.]md$' | LC_ALL=C sort | xargs sha256sum -b > /tmp/nix-cache-key
         uname -m >> /tmp/nix-cache-key # Add architecture to the cache key
         echo "gh_cache_version: ${{ inputs.cache_version }}" >> /tmp/nix-cache-key # Add cache version to the cache key
+        echo "home: $HOME" >> /tmp/nix-cache-key # important when restoring simlinks from cache, apparently
+        echo "nix binary version: $NIX_BINARY_VERSION" >> /tmp/nix-cache-key # different nix versions might behave differently and corrupt the caches
         if [ "${{ inputs.oss_only }}" == true ]; then
           echo "Using OSS only dependencies"
           echo "oss_only: ${{ inputs.oss_only }}" >> /tmp/nix-cache-key
@@ -86,8 +91,9 @@ runs:
           # we use rsync here because it's simply faster to install
           rsync -avi /cache/nix/$cache_key/.nix-* $HOME/
           rsync -avi "/cache/nix/$cache_key/nix" $HOME/.config/
-          rsync -avi "/cache/nix/$cache_key/nix_store/var/" /nix/var
-          sudo mount --bind /cache/nix/$cache_key/nix_store/store /nix/store
+          # TODO (#2663): fix & uncomment these two lines
+          # rsync -avi "/cache/nix/$cache_key/nix_store/var/" /nix/var
+          # sudo mount --bind /cache/nix/$cache_key/nix_store/store /nix/store
         else
           sudo mkdir -p "/cache/nix/$cache_key"
           sudo chown $(whoami):$(whoami) "/cache/nix/$cache_key"
@@ -126,7 +132,7 @@ runs:
             max-jobs = 16
         EOF
           fi
-          sh <(curl -fsSL --retry 8 https://releases.nixos.org/nix/nix-2.13.3/install) --no-daemon
+          sh <(curl -fsSL --retry 8 "https://releases.nixos.org/nix/nix-$NIX_BINARY_VERSION/install") --no-daemon
           sudo mkdir -p /etc/nix
           sudo chmod a+rw /etc/nix
           if [[ "${{ inputs.oss_only }}" == true ]]; then
@@ -147,8 +153,6 @@ runs:
             target="default"
           fi
           nix develop path:nix#${target} -v --profile "$HOME/.nix-shell" --command echo "Done loading packages"
-          echo "Garbage collecting to reduce cache size"
-          nix-store --gc
         fi
 
     - name: Invoke nix before saving cache
@@ -176,6 +180,9 @@ runs:
           export USER=$(whoami)
           . ~/.nix-profile/etc/profile.d/nix.sh
 
+          echo "Garbage collecting to reduce cache size"
+          nix-store --gc
+
           nix copy --all --to 'file:///cache/nix/binary_cache?trusted=1' -v
 
           CLONE_COMMAND="rclone --no-update-dir-modtime --no-update-modtime --size-only --multi-thread-streams=32 --transfers=32 --ignore-existing --links --create-empty-src-dirs --fast-list --metadata --order-by name,mixed --retries 10 copy"
@@ -187,7 +194,8 @@ runs:
 
           #requires to preserve read only during clone
           sudo ${CLONE_COMMAND} /nix/store/ /cache/nix/$cache_key/nix_store/store
-          sudo ${CLONE_COMMAND} /nix/var/ "/cache/nix/$cache_key/nix_store/var"
+          # TODO (#2663): fix & uncomment this line
+          # sudo ${CLONE_COMMAND} /nix/var/ "/cache/nix/$cache_key/nix_store/var"
 
           echo "done" > "/cache/nix/$cache_key/cached"
         fi

.github/actions/tests/common_test_setup/action.yml

@@ -64,7 +64,7 @@ runs:
       with:
         artifactory_user: ${{ inputs.artifactory_user }}
         artifactory_password: ${{ inputs.artifactory_password }}
-        cache_version: 5
+        cache_version: 7
         should_save: ${{ inputs.save_nix_cache }}
         should_save_gcp: ${{ inputs.save_nix_cache_to_gcp }}
         oss_only: ${{ inputs.oss_only }}
@@ -76,4 +76,4 @@ runs:
       id: setup_sbt
       uses: ./.github/actions/sbt/setup_sbt
       with:
-        cache_version: 5
+        cache_version: 7

.github/actions/tests/scala_test/action.yml

@@ -141,7 +141,7 @@ runs:
       id: setup_sbt
       uses: ./.github/actions/sbt/setup_sbt
       with:
-        cache_version: 5
+        cache_version: 7
 
     - name: Set Daml package versions
       uses: ./.github/actions/nix/run_bash_command_in_nix
@@ -285,7 +285,7 @@ runs:
       if: ${{ !cancelled() }}
       uses: ./.github/actions/sbt/post_sbt
       with:
-        cache_version: 5
+        cache_version: 7
         setup_sbt_cache_hits: ${{ steps.setup_sbt.outputs.cache_hits }}
         # Save caches only from one runner, to reduce conflicts on the save
         save_caches: ${{ inputs.runner_index == 0 && inputs.test_suite_name == 'wall-clock-time' }}

.github/workflows/build.docs.yml

@@ -48,7 +48,7 @@ jobs:
       - name: Post-SBT job
         uses: ./.github/actions/sbt/post_sbt
         with:
-          cache_version: 5
+          cache_version: 7
           setup_sbt_cache_hits: ${{ steps.setup.outputs.sbt_cache_hits }}
 
       - name: Report Failures on Slack & Github

.github/workflows/build.static_tests.yml

@@ -9,6 +9,10 @@ on:
       self_hosted:
         type: boolean
         required: true
+      skip_todo_check:
+        type: boolean
+        required: false
+        default: false
 
 jobs:
   static_tests:
@@ -53,6 +57,7 @@ jobs:
 
       - name: Checking TODOs
         uses: ./.github/actions/nix/run_bash_command_in_nix
+        if: ${{ inputs.skip_todo_check == false }}
         with:
           cmd: |
             echo "PR number: $CIRCLE_PULL_REQUEST"

.github/workflows/build.ts_cli_tests.yml

@@ -20,18 +20,18 @@ jobs:
           # Checkout the PR head commit to get the commit message first
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Check out repository code
-        uses: actions/checkout@v4
-        if: inputs.commit_sha != ''
-        with:
-          ref: ${{ inputs.commit_sha }}
-
       - name: Check if static only
         uses: ./.github/actions/tests/skip_on_static
         id: skip
         with:
           gh_token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        if: inputs.commit_sha != ''
+        with:
+          ref: ${{ inputs.commit_sha }}
+
       - name: Setup
         id: setup
         if: steps.skip.outputs.skip != 'true'

.github/workflows/build.ui_tests.yml

@@ -58,7 +58,7 @@ jobs:
         if: steps.skip.outputs.skip != 'true'
         uses: ./.github/actions/sbt/post_sbt
         with:
-          cache_version: 5
+          cache_version: 7
           setup_sbt_cache_hits: ${{ steps.setup.outputs.sbt_cache_hits }}
 
       - name: Upload logs

.github/workflows/pr_non_contributors.yml

@@ -32,17 +32,3 @@ jobs:
     needs: env_hold
     with:
       commit_sha: ${{ github.event.pull_request.head.sha }}
-
-  # Note: unapproved runs must not be granted access to secrets, and must not run on self-hosted runners
-  no_approval_static_tests:
-    name: Static Tests (No Approval Required)
-    uses: ./.github/workflows/build.static_tests.yml
-    if: github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
-    secrets: {} # Explcitly do not grant access to secrets
-    permissions:
-      contents: read
-      pull-requests: read # Required for the static tests
-      issues: read # Required for the static tests
-    with:
-      self_hosted: false
-      commit_sha: ${{ github.event.pull_request.head.sha }}

.github/workflows/pr_static_checks.yml

@@ -0,0 +1,23 @@
+name: Static checks for PRs from forks (No Approval Required)
+on:
+  # unapproved runs must be on: pull_request, to prevent cache pollution on main
+  pull_request:
+    types: [ opened, synchronize, reopened ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.head.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Note: unapproved runs must not be granted access to secrets, and must not run on self-hosted runners
+  no_approval_static_tests:
+    name: Static Tests
+    uses: ./.github/workflows/build.static_tests.yml
+    if: github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+    secrets: {} # Explcitly do not grant access to secrets
+    permissions:
+      contents: read
+    with:
+      self_hosted: false
+      commit_sha: ${{ github.event.pull_request.head.sha }}
+      skip_todo_check: true # runs from forks with on: pull_request run in context of the fork, so issue references will be broken