Merge branch 'main' into update/scopedown-github-token

Author: kalyazin

.buildkite/common.py

@@ -76,12 +76,6 @@ def group(label, command, instances, platforms, **kwargs):
 
     https://buildkite.com/docs/pipelines/group-step
     """
-    # Use the 1st character of the group name (should be an emoji)
-    label1 = label[0]
-    # if the emoji is in the form ":emoji:", pick the entire slug
-    if label.startswith(":") and ":" in label[1:]:
-        label1 = label[: label.index(":", 1) + 1]
-
     steps = []
     commands = command
     if isinstance(command, str):
@@ -92,7 +86,7 @@ def group(label, command, instances, platforms, **kwargs):
             args = {"instance": instance, "os": os_, "kv": kv}
             step = {
                 "command": [cmd.format(**args) for cmd in commands],
-                "label": f"{label1} {instance} {os_} {kv}",
+                "label": f"{label}-{instance}-{os_}-{kv}",
                 "agents": args,
             }
             step_kwargs = dict_fmt(kwargs, args)

.buildkite/pipeline_cpu_template.py

@@ -23,7 +23,7 @@ class BkStep(str, Enum):
 cpu_template_test = {
     "rdmsr": {
         BkStep.COMMAND: [
-            "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_rdmsr' "
+            "tools/devtool -y test --no-build -- -m no_block_pr -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_rdmsr' "
         ],
         BkStep.LABEL: "📖 rdmsr",
         "instances": [

.buildkite/pipeline_perf.py

@@ -17,54 +17,74 @@
 # has to be the node associated with the NUMA node from which we picked CPUs.
 perf_test = {
     "virtio-block-sync": {
-        "label": "💿 Virtio Sync Block Performance",
+        "label": "virtio-block-sync",
         "tests": "integration_tests/performance/test_block.py::test_block_performance -k 'not Async'",
         "devtool_opts": "-c 1-10 -m 0",
     },
     "virtio-block-async": {
-        "label": "💿 Virtio Async Block Performance",
+        "label": "virtio-block-async",
         "tests": "integration_tests/performance/test_block.py::test_block_performance -k Async",
         "devtool_opts": "-c 1-10 -m 0",
     },
     "vhost-user-block": {
-        "label": "💿 vhost-user Block Performance",
+        "label": "vhost-user-block",
         "tests": "integration_tests/performance/test_block.py::test_block_vhost_user_performance",
         "devtool_opts": "-c 1-10 -m 0",
         "ab_opts": "--noise-threshold 0.1",
     },
     "network": {
-        "label": "📠 Network Latency and Throughput",
+        "label": "network",
         "tests": "integration_tests/performance/test_network.py",
         "devtool_opts": "-c 1-10 -m 0",
         # Triggers if delta is > 0.01ms (10µs) or default relative threshold (5%)
         # only relevant for latency test, throughput test will always be magnitudes above this anyway
         "ab_opts": "--absolute-strength 0.010",
     },
     "snapshot-latency": {
-        "label": "📸 Snapshot Latency",
+        "label": "snapshot-latency",
         "tests": "integration_tests/performance/test_snapshot.py::test_restore_latency integration_tests/performance/test_snapshot.py::test_post_restore_latency integration_tests/performance/test_snapshot.py::test_snapshot_create_latency",
         "devtool_opts": "-c 1-12 -m 0",
     },
     "population-latency": {
-        "label": "📸 Memory Population Latency",
+        "label": "population-latency",
         "tests": "integration_tests/performance/test_snapshot.py::test_population_latency",
         "devtool_opts": "-c 1-12 -m 0",
     },
     "vsock-throughput": {
-        "label": "🧦 Vsock Throughput",
+        "label": "vsock-throughput",
         "tests": "integration_tests/performance/test_vsock.py",
         "devtool_opts": "-c 1-10 -m 0",
     },
     "memory-overhead": {
-        "label": "💾 Memory Overhead and 👢 Boottime",
-        "tests": "integration_tests/performance/test_memory_overhead.py integration_tests/performance/test_boottime.py::test_boottime",
+        "label": "memory-overhead",
+        "tests": "integration_tests/performance/test_memory_overhead.py",
+        "devtool_opts": "-c 1-10 -m 0",
+    },
+    "boottime": {
+        "label": "boottime",
+        "tests": "integration_tests/performance/test_boottime.py::test_boottime",
         "devtool_opts": "-c 1-10 -m 0",
     },
     "jailer": {
-        "label": "⛓️ jailer",
+        "label": "jailer",
         "tests": "integration_tests/performance/test_jailer.py",
         "devtool_opts": "-c 1-10 -m 0",
     },
+    "pmem": {
+        "label": "pmem",
+        "tests": "integration_tests/performance/test_pmem.py",
+        "devtool_opts": "-c 1-10 -m 0",
+    },
+    "mmds": {
+        "label": "mmds",
+        "tests": "integration_tests/performance/test_mmds.py",
+        "devtool_opts": "-c 1-10 -m 0",
+    },
+    "memory-hotplug": {
+        "label": "memory-hotplug",
+        "tests": "integration_tests/performance/test_hotplug_memory.py",
+        "devtool_opts": "-c 1-10 -m 0",
+    },
 }
 
 REVISION_A = os.environ.get("REVISION_A")

.buildkite/pipeline_pr_no_block.py

@@ -22,7 +22,7 @@
     "❓ Optional",
     pipeline.devtool_test(
         devtool_opts="--performance -c 1-10 -m 0",
-        pytest_opts="integration_tests/ -m 'no_block_pr and not nonci' --log-cli-level=INFO",
+        pytest_opts="integration_tests/ -m no_block_pr --log-cli-level=INFO",
     ),
 )
 if not run_all_tests(get_changed_files()):

.cargo/config.toml

@@ -9,3 +9,10 @@ git-fetch-with-cli = true
 
 [env]
 AWS_LC_SYS_NO_JITTER_ENTROPY = "1"
+# disable AVX512 as it adds 600k of binary size
+# this was only used for MMDS token generation
+# Note: due to a bug in aws-lc [1] the AWS_LC_SYS_CFLAGS only work with
+# the cmake compiler.
+# [1]: https://github.com/aws/aws-lc-rs/issues/965
+AWS_LC_SYS_CMAKE_BUILDER = "1"
+AWS_LC_SYS_CFLAGS = "-DMY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX"

.github/workflows/monitor_libseccomp_releases.yml

@@ -0,0 +1,69 @@
+name: Monitor libseccomp Releases
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Daily at midnight UTC
+  workflow_dispatch:  # Allow manual trigger
+
+permissions:
+  issues: write
+  contents: read
+
+jobs:
+  check-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+          
+      - name: Get current libseccomp version from Dockerfile
+        id: current
+        run: |
+          CURRENT=$(grep 'LIBSECCOMP_VER' tools/devctr/Dockerfile | grep -oP "v[0-9.]+")
+          echo "version=$CURRENT" >> $GITHUB_OUTPUT
+          
+      - name: Check for new libseccomp release
+        id: latest
+        run: |
+          LATEST=$(curl -s https://api.github.com/repos/seccomp/libseccomp/releases/latest | jq -r '.tag_name')
+          echo "version=$LATEST" >> $GITHUB_OUTPUT
+
+      - name: Check latest version is newer
+        id: semver_check
+        run: |
+          CURRENT=$(echo ${{ steps.current.outputs.version }} | grep -oP "[0-9.]+")
+          LATEST=$(echo ${{ steps.latest.outputs.version }} | grep -oP "[0-9.]+") 
+          if ! printf '%s\n%s' "$LATEST" "$CURRENT" | sort -VC && [ "$CURRENT" != "$LATEST" ]; then
+            echo "is_newer=true" >> $GITHUB_OUTPUT;
+          else
+            echo "is_newer=false" >> $GITHUB_OUTPUT;
+          fi
+ 
+      - name: Check if issue exists
+        if: steps.semver_check.outputs.is_newer == 'true' # New release has higher semantic version
+        id: issue_check
+        run: |
+          ISSUES=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/${{ github.repository }}/issues?state=open&labels=dependencies" | \
+          jq -r --arg tag "${{ steps.latest.outputs.version }}" '[.[] | select(.title | contains("chore(deps): update libseccomp to \($tag)"))] | length')
+          echo "exists=$ISSUES" >> $GITHUB_OUTPUT
+
+      - name: Create issue for new release
+        id: create_issue
+        if: steps.semver_check.outputs.is_newer == 'true' && steps.issue_check.outputs.exists == '0' # No existing issue for new version
+        run: |
+          gh issue create \
+            --title "chore(deps): update libseccomp to ${{ steps.latest.outputs.version }}" \
+            --body "$(cat <<EOF
+          A new version of libseccomp has been released: **${{ steps.latest.outputs.version }}**
+
+          Current version in Dockerfile: **${{ steps.current.outputs.version }}**
+
+          Repository: https://github.com/seccomp/libseccomp/releases/tag/${{ steps.latest.outputs.version }}
+
+          Please review and consider updating Firecracker's dependency in \`tools/devctr/Dockerfile\`.
+          EOF
+          )" \
+            --label "dependencies"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CHANGELOG.md

@@ -10,12 +10,41 @@ and this project adheres to
 
 ### Added
 
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+## [v1.14.0]
+
+### Added
+
 - [#5463](https://github.com/firecracker-microvm/firecracker/pull/5463): Added
   support for `virtio-pmem` devices. See [documentation](docs/pmem.md) for more
   information.
+- [#5534](https://github.com/firecracker-microvm/firecracker/pull/5534): Added
+  support for memory hot-plugging through the `virtio-mem` device. See
+  [documentation](docs/memory-hotplug.md) for more information.
+- [#5491](https://github.com/firecracker-microvm/firecracker/pull/5491): Added
+  support for `virtio-balloon` free page reporting and hinting. Free page
+  reporting is a developer preview not for production feature. See
+  [documentation](docs/ballooning.md) for more information.
 
 ### Changed
 
+- [#4028](https://github.com/firecracker-microvm/firecracker/pull/4028):
+  Firecracker now creates the log and metrics files if they do not exist,
+  simplifying the launch of Firecracker by removing a manual step.
+- [#5516](https://github.com/firecracker-microvm/firecracker/pull/5516): Balloon
+  stats now supports guest kernel >= 6.12, adding metrics on guest OOM kills,
+  memory allocation stalls, and memory scan/reclaim info.
+- [#5526](https://github.com/firecracker-microvm/firecracker/pull/5526): Specify
+  IA32_MTRRdefType MSR on VM boot to allow it to set page attributes for memory
+  regions.
+
 ### Deprecated
 
 ### Removed
@@ -35,6 +64,16 @@ and this project adheres to
   Intel AMX enabling for kernels that support dynamic XSTATE features for
   userspace applications but not for KVM guests (e.g. kernel versions >= 5.16
   and < 5.17).
+- [#5485](https://github.com/firecracker-microvm/firecracker/pull/5485): Fixed a
+  bug causing a read/write from an iovec to be duplicated when receiving an
+  error on an iovec other than the first. This caused a data corruption issue in
+  the vsock device starting from guest kernel 6.17.
+- [#5494](https://github.com/firecracker-microvm/firecracker/pull/5494): Fixed a
+  watchdog soft lockup bug on microVMs restored from snapshots by calling
+  KVM_KVMCLOCK_CTRL ioctl before resuming.
+- [#5538](https://github.com/firecracker-microvm/firecracker/pull/5538): Fixed a
+  cache coherency issue on non-FWB aarch64 platforms by adding `dma-coherent`
+  property to virtio-mmio nodes in the FDT.
 
 ## [1.13.0]