From e8b3bcc9cf29fef07c3376f7e1fa8b603d1d65ce Mon Sep 17 00:00:00 2001 From: Chizkiyahu Raful Date: Wed, 22 Oct 2025 12:23:50 +0300 Subject: [PATCH 1/4] ci_build script 1. refactor "clone repositories and build" from `build-with-manifest.yml` and `build-with-manifest-overrides.yml` to `ci_build.sh` 2. run `ci_build.sh` in docker to cache the build and reduce build time by ~50% Signed-off-by: Chizkiyahu Raful --- .../build-with-manifest-overrides.yml | 56 ++------ .github/workflows/build-with-manifest.yml | 74 +++------- .github/workflows/scripts/ci_build.sh | 127 ++++++++++++++++++ docker/DOCKER.md | 10 +- docker/Dockerfile | 19 +++ 5 files changed, 185 insertions(+), 101 deletions(-) create mode 100644 .github/workflows/scripts/ci_build.sh diff --git a/.github/workflows/build-with-manifest-overrides.yml b/.github/workflows/build-with-manifest-overrides.yml index 263ab8f..f7a7330 100644 --- a/.github/workflows/build-with-manifest-overrides.yml +++ b/.github/workflows/build-with-manifest-overrides.yml @@ -17,56 +17,18 @@ jobs: build: runs-on: ubuntu-latest container: - image: ghcr.io/arm/ml-sdk-linux-amd64:latest + image: ghcr.io/${{ github.repository_owner }}/ml-sdk-linux-amd64:latest credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} options: --user root - steps: - - name: Clone repositories and build - env: - MANIFEST_URL: https://github.com/arm/ai-ml-sdk-manifest.git - REPO_DIR: ${{ github.workspace }}/sdk - INSTALL_DIR: ${{ github.workspace }}/install - OVERRIDES: ${{ inputs.overrides_json }} - run: | - mkdir -p $REPO_DIR && cd $REPO_DIR - - # Initialize the default manifest state - repo init -u $MANIFEST_URL - repo sync --no-clone-bundle -j $(nproc) - - mkdir -p .repo/local_manifests - echo '' > .repo/local_manifests/override.xml - echo "$OVERRIDES" | jq -r 'to_entries[] | ""' >> .repo/local_manifests/override.xml - echo '' >> .repo/local_manifests/override.xml - - # Resolve each project's path from the active manifest and re-sync it - for NAME in $(echo "$OVERRIDES" | jq -r 'keys[]'); do - PROJECT_PATH=$(repo manifest -r | xmlstarlet sel -t -v "//project[@name='${NAME}']/@path") - if [ -z "$PROJECT_PATH" ]; then - echo "ERROR: project path for $NAME not found in manifest" - exit 1 - fi - echo "Syncing $NAME ($PROJECT_PATH)" - repo sync -j"$(nproc)" --force-sync "$PROJECT_PATH" - done - - echo "Build VGF-Lib" - ./sw/vgf-lib/scripts/build.py -j $(nproc) --doc --test + env: + REPO_DIR: /home/mlsdkuser/sdk + INSTALL_DIR: /home/mlsdkuser/install + MANIFEST_URL: https://github.com/${{ github.repository_owner }}/ai-ml-sdk-manifest.git + OVERRIDES: ${{ inputs.overrides_json }} - echo "Build Model Converter" - ./sw/model-converter/scripts/build.py -j $(nproc) --doc --test - - echo "Build Emulation Layer" - export VK_LAYER_PATH=$INSTALL_DIR/share/vulkan/explicit_layer.d - export LD_LIBRARY_PATH=$INSTALL_DIR/lib - ./sw/emulation-layer/scripts/build.py -j $(nproc) --doc --test --install $INSTALL_DIR - - echo "Build Scenario Runner" - export VK_INSTANCE_LAYERS=VK_LAYER_ML_Graph_Emulation:VK_LAYER_ML_Tensor_Emulation - ./sw/scenario-runner/scripts/build.py -j $(nproc) --doc --test - - echo "Build SDK Root" - ./scripts/build.py -j $(nproc) --doc + steps: + - name: update repositories and build + run: /home/mlsdkuser/ci_build.sh \ No newline at end of file diff --git a/.github/workflows/build-with-manifest.yml b/.github/workflows/build-with-manifest.yml index dc098ed..16926c5 100644 --- a/.github/workflows/build-with-manifest.yml +++ b/.github/workflows/build-with-manifest.yml @@ -6,10 +6,15 @@ name: Build ML SDK on: workflow_dispatch: + inputs: + changed_repo: { type: string, required: false, default: "", description: "org/repo-a" } + changed_sha: { type: string, required: false, default: "", description: "40-char sha" } + clean: { type: boolean, required: false, default: false , description: "without cache" } workflow_call: inputs: - changed_repo: { type: string, required: true } # e.g., org/repo-a - changed_sha: { type: string, required: true } # e.g., 40-char sha + changed_repo: { type: string, required: true, description: "org/repo-a" } + changed_sha: { type: string, required: true, description: "40-char sha" } + clean: { type: boolean, required: false, default: false , description: "without cache" } permissions: contents: read @@ -19,62 +24,25 @@ jobs: build: runs-on: ubuntu-latest container: - image: ghcr.io/arm/ml-sdk-linux-amd64:latest + image: ghcr.io/${{ github.repository_owner }}/ml-sdk-linux-amd64:latest credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} options: --user root + env: + REPO_DIR: /home/mlsdkuser/sdk + INSTALL_DIR: /home/mlsdkuser/install + MANIFEST_URL: https://github.com/${{ github.repository_owner }}/ai-ml-sdk-manifest.git + CHANGED_REPO: ${{ inputs.changed_repo }} + CHANGED_SHA: ${{ inputs.changed_sha }} + steps: - - name: Clone repositories and build - env: - MANIFEST_URL: https://github.com/arm/ai-ml-sdk-manifest.git - REPO_DIR: ${{ github.workspace }}/sdk - INSTALL_DIR: ${{ github.workspace }}/install - CHANGED_REPO: ${{ inputs.changed_repo }} - CHANGED_SHA: ${{ inputs.changed_sha }} + - name: clean + if: ${{ inputs.clean == true }} run: | - mkdir -p $REPO_DIR && cd $REPO_DIR - - # Initialize the default manifest state - repo init -u $MANIFEST_URL - repo sync --no-clone-bundle -j $(nproc) - - if [ -n "$CHANGED_REPO" ]; then - # Find project path for changed repo - PROJECT_PATH=$(repo manifest -r | xmlstarlet sel -t -v "//project[@name='${CHANGED_REPO}']/@path") - if [ -z "$PROJECT_PATH" ]; then - echo "Could not find project path for ${CHANGED_REPO} in manifest" - exit 1 - fi - echo "Changed project path: $PROJECT_PATH" - - # Create a local manifest override to pin the changed repo to the exact SHA - mkdir -p .repo/local_manifests - cat > .repo/local_manifests/override.xml < - - - EOF - - # Re-sync the changed project to the specified SHA - repo sync -j $(nproc) --force-sync "$PROJECT_PATH" - fi - - echo "Build VGF-Lib" - ./sw/vgf-lib/scripts/build.py -j $(nproc) --doc --test - - echo "Build Model Converter" - ./sw/model-converter/scripts/build.py -j $(nproc) --doc --test - - echo "Build Emulation Layer" - export VK_LAYER_PATH=$INSTALL_DIR/share/vulkan/explicit_layer.d - export LD_LIBRARY_PATH=$INSTALL_DIR/lib - ./sw/emulation-layer/scripts/build.py -j $(nproc) --doc --test --install $INSTALL_DIR - - echo "Build Scenario Runner" - export VK_INSTANCE_LAYERS=VK_LAYER_ML_Graph_Emulation:VK_LAYER_ML_Tensor_Emulation - ./sw/scenario-runner/scripts/build.py -j $(nproc) --doc --test + rm -rf $REPO_DIR + rm -rf $INSTALL_DIR - echo "Build SDK Root" - ./scripts/build.py -j $(nproc) --doc + - name: Update repositories and build + run: /home/mlsdkuser/ci_build.sh diff --git a/.github/workflows/scripts/ci_build.sh b/.github/workflows/scripts/ci_build.sh new file mode 100644 index 0000000..5d0f43c --- /dev/null +++ b/.github/workflows/scripts/ci_build.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates +# SPDX-License-Identifier: Apache-2.0 + + +set -euo pipefail + +usage() { + echo "Usage: $(basename "$0")" + echo + echo "Environment:" + echo " MANIFEST_URL (optional) default: https://github.com/arm/ai-ml-sdk-manifest.git" + echo " REPO_DIR (optional) default: ./sdk" + echo " INSTALL_DIR (optional) default: ./install" + echo " CHANGED_REPO (optional) manifest project name to pin and resync" + echo " CHANGED_SHA (optional) commit SHA to pin CHANGED_REPO to (required if CHANGED_REPO is set)" + echo " OVERRIDES (optional) JSON object: { \"org/repo\": \"40-char-sha\", ... }" +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +MANIFEST_URL="${MANIFEST_URL:-https://github.com/arm/ai-ml-sdk-manifest.git}" +REPO_DIR="${REPO_DIR:-$PWD/sdk}" +INSTALL_DIR="${INSTALL_DIR:-$PWD/install}" +CHANGED_REPO="${CHANGED_REPO:-}" +CHANGED_SHA="${CHANGED_SHA:-}" +OVERRIDES="${OVERRIDES:-}" + +echo "Using manifest URL: $MANIFEST_URL" +echo "Using repo directory: $REPO_DIR" +echo "Using install directory: $INSTALL_DIR" +echo "find CHANGED_REPO: $CHANGED_REPO" +echo "find CHANGED_SHA: $CHANGED_SHA" +echo "find OVERRIDES: $OVERRIDES" + +# for macOS compatibility +if ! command -v nproc >/dev/null 2>&1; then + nproc() { sysctl -n hw.ncpu; } +fi + +mkdir -p $REPO_DIR +REPO_DIR="$(realpath "$REPO_DIR")" +INSTALL_DIR="$(realpath "$INSTALL_DIR")" +pushd $REPO_DIR + +repo init -u $MANIFEST_URL +# --force-sync to ensure we get latest even if there are local changes when re-running +repo sync --no-clone-bundle -j $(nproc) --force-sync + +if [ -n "$OVERRIDES" ]; then + mkdir -p .repo/local_manifests + echo '' > .repo/local_manifests/override.xml + echo "$OVERRIDES" | jq -r 'to_entries[] | ""' >> .repo/local_manifests/override.xml + echo '' >> .repo/local_manifests/override.xml + + # Resolve each project's path from the active manifest and re-sync it + for NAME in $(echo "$OVERRIDES" | jq -r 'keys[]'); do + PROJECT_PATH=$(repo manifest -r | xmlstarlet sel -t -v "//project[@name='${NAME}']/@path") + if [ -z "$PROJECT_PATH" ]; then + echo "ERROR: project path for $NAME not found in manifest" + exit 1 + fi + echo "Syncing $NAME ($PROJECT_PATH)" + repo sync -j"$(nproc)" --force-sync "$PROJECT_PATH" + done + +elif [ -n "$CHANGED_REPO" ]; then + if [ -z "$CHANGED_SHA" ]; then + echo "CHANGED_REPO is set but CHANGED_SHA is empty" + exit 1 + fi + + # Find project path for changed repo + PROJECT_PATH=$(repo manifest -r | xmlstarlet sel -t -v "//project[@name='${CHANGED_REPO}']/@path") + if [ -z "$PROJECT_PATH" ]; then + echo "Could not find project path for ${CHANGED_REPO} in manifest" + exit 1 + fi + echo "Changed project path: $PROJECT_PATH" + + # Create a local manifest override to pin the changed repo to the exact SHA + mkdir -p .repo/local_manifests + cat > .repo/local_manifests/override.xml < + + +EOF + + # Re-sync the changed project to the specified SHA + repo sync -j $(nproc) --force-sync "$PROJECT_PATH" +fi + +echo "Build VGF-Lib" +./sw/vgf-lib/scripts/build.py -j $(nproc) --doc --test + +echo "Build Model Converter" +./sw/model-converter/scripts/build.py -j $(nproc) --doc --test + +echo "Build Emulation Layer" +export VK_LAYER_PATH=$INSTALL_DIR/share/vulkan/explicit_layer.d +export LD_LIBRARY_PATH=$INSTALL_DIR/lib + +if [ "$(uname)" = "Darwin" ]; then + echo "macOS detected, skipping Emulation Layer tests" + ./sw/emulation-layer/scripts/build.py -j $(nproc) --doc --install $INSTALL_DIR +else + ./sw/emulation-layer/scripts/build.py -j $(nproc) --doc --test --install $INSTALL_DIR +fi + +echo "Build Scenario Runner" +export VK_INSTANCE_LAYERS=VK_LAYER_ML_Graph_Emulation:VK_LAYER_ML_Tensor_Emulation + +if [ "$(uname)" = "Darwin" ]; then + echo "macOS detected, skipping Scenario Runner tests" + ./sw/scenario-runner/scripts/build.py -j $(nproc) --doc +else + ./sw/scenario-runner/scripts/build.py -j $(nproc) --doc --test +fi + +echo "Build SDK Root" +./scripts/build.py -j $(nproc) --doc + +popd diff --git a/docker/DOCKER.md b/docker/DOCKER.md index 46d168d..264170f 100644 --- a/docker/DOCKER.md +++ b/docker/DOCKER.md @@ -4,5 +4,13 @@ To build the docker image run the following command from the folder containing the `Dockerfile`: ```sh -docker build --tag ml-sdk-image --file Dockerfile --build-arg user=$(whoami) --build-arg uid=$(id -u) .. +docker buildx build --tag ml-sdk-image --file Dockerfile --build-arg user=$(whoami) --build-arg uid=$(id -u) .. ``` + +# Troubleshooting +## Error `did not complete successfully: cannot allocate memory` +increase the memory by running + +```sh +docker buildx build --memory=64g --tag ml-sdk-image --file Dockerfile --build-arg user=$(whoami) --build-arg uid=$(id -u) .. +``` \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 8f77f14..69b074f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -92,4 +92,23 @@ WORKDIR /home/$user RUN chown -R $user:$user . USER $user + +# Configure git to avoid warnings and issues with permissions +RUN git config --global user.name "github-actions[bot]" && \ + git config --global user.email "github-actions[bot]@users.noreply.github.com" && \ + git config --global color.ui false && \ + git config --system --add safe.directory '*' && \ + git config --global --add safe.directory '*' + +ARG REPO_OWNER=arm +ARG MANIFEST_URL=https://github.com/$REPO_OWNER/ai-ml-sdk-manifest.git +ARG REPO_DIR=./sdk +ARG INSTALL_DIR=./install +ARG CHANGED_REPO +ARG CHANGED_SHA + +ENV MANIFEST_URL=$MANIFEST_URL REPO_DIR=$REPO_DIR INSTALL_DIR=$INSTALL_DIR CHANGED_REPO=$CHANGED_REPO CHANGED_SHA=$CHANGED_SHA +COPY --chown=$user --chmod=755 ../.github/workflows/scripts/ci_build.sh ./ +RUN ./ci_build.sh + CMD ["/bin/bash"] From be9f9adc3ce815fb8470ecfdbac946649b2f72b7 Mon Sep 17 00:00:00 2001 From: Chizkiyahu Raful Date: Wed, 22 Oct 2025 12:52:38 +0300 Subject: [PATCH 2/4] publish-linux-docker-image.yml - fix missing file path Signed-off-by: Chizkiyahu Raful --- .github/workflows/publish-linux-docker-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-linux-docker-image.yml b/.github/workflows/publish-linux-docker-image.yml index a7b5f4d..345a79c 100644 --- a/.github/workflows/publish-linux-docker-image.yml +++ b/.github/workflows/publish-linux-docker-image.yml @@ -9,6 +9,7 @@ on: branches: [ main ] paths: - 'docker/Dockerfile' + - '.github/workflows/scripts/ci_build.sh' workflow_dispatch: jobs: From 43b20354bb5959a846ede1b8dd2ca6d689aa418e Mon Sep 17 00:00:00 2001 From: Chizkiyahu Raful Date: Wed, 22 Oct 2025 13:03:00 +0300 Subject: [PATCH 3/4] Remove redundant safe.directory configuration in Dockerfile Signed-off-by: Chizkiyahu Raful --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 69b074f..bde742f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -97,7 +97,6 @@ USER $user RUN git config --global user.name "github-actions[bot]" && \ git config --global user.email "github-actions[bot]@users.noreply.github.com" && \ git config --global color.ui false && \ - git config --system --add safe.directory '*' && \ git config --global --add safe.directory '*' ARG REPO_OWNER=arm From a053fa1c1747eae97db2bb7af21d40f0af4fa117 Mon Sep 17 00:00:00 2001 From: Chizkiyahu Raful Date: Wed, 22 Oct 2025 16:49:40 +0300 Subject: [PATCH 4/4] remove `ci_build.sh` from docker layers Signed-off-by: Chizkiyahu Raful --- .../build-with-manifest-overrides.yml | 13 ++++---- .github/workflows/build-with-manifest.yml | 30 ++++++------------- docker/DOCKER.md | 8 ----- docker/Dockerfile | 9 ------ 4 files changed, 15 insertions(+), 45 deletions(-) diff --git a/.github/workflows/build-with-manifest-overrides.yml b/.github/workflows/build-with-manifest-overrides.yml index f7a7330..99df8cb 100644 --- a/.github/workflows/build-with-manifest-overrides.yml +++ b/.github/workflows/build-with-manifest-overrides.yml @@ -23,12 +23,11 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} options: --user root - env: - REPO_DIR: /home/mlsdkuser/sdk - INSTALL_DIR: /home/mlsdkuser/install - MANIFEST_URL: https://github.com/${{ github.repository_owner }}/ai-ml-sdk-manifest.git - OVERRIDES: ${{ inputs.overrides_json }} - steps: - - name: update repositories and build + - name: Clone repositories and build + env: + MANIFEST_URL: https://github.com/${{ github.repository_owner }}/ai-ml-sdk-manifest.git + REPO_DIR: ${{ github.workspace }}/sdk + INSTALL_DIR: ${{ github.workspace }}/install + OVERRIDES: ${{ inputs.overrides_json }} run: /home/mlsdkuser/ci_build.sh \ No newline at end of file diff --git a/.github/workflows/build-with-manifest.yml b/.github/workflows/build-with-manifest.yml index 16926c5..8f75bb7 100644 --- a/.github/workflows/build-with-manifest.yml +++ b/.github/workflows/build-with-manifest.yml @@ -6,15 +6,10 @@ name: Build ML SDK on: workflow_dispatch: - inputs: - changed_repo: { type: string, required: false, default: "", description: "org/repo-a" } - changed_sha: { type: string, required: false, default: "", description: "40-char sha" } - clean: { type: boolean, required: false, default: false , description: "without cache" } workflow_call: inputs: - changed_repo: { type: string, required: true, description: "org/repo-a" } - changed_sha: { type: string, required: true, description: "40-char sha" } - clean: { type: boolean, required: false, default: false , description: "without cache" } + changed_repo: { type: string, required: true } # e.g., org/repo-a + changed_sha: { type: string, required: true } # e.g., 40-char sha permissions: contents: read @@ -30,19 +25,12 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} options: --user root - env: - REPO_DIR: /home/mlsdkuser/sdk - INSTALL_DIR: /home/mlsdkuser/install - MANIFEST_URL: https://github.com/${{ github.repository_owner }}/ai-ml-sdk-manifest.git - CHANGED_REPO: ${{ inputs.changed_repo }} - CHANGED_SHA: ${{ inputs.changed_sha }} - steps: - - name: clean - if: ${{ inputs.clean == true }} - run: | - rm -rf $REPO_DIR - rm -rf $INSTALL_DIR - - - name: Update repositories and build + - name: Clone repositories and build + env: + MANIFEST_URL: https://github.com/${{ github.repository_owner }}/ai-ml-sdk-manifest.git + REPO_DIR: ${{ github.workspace }}/sdk + INSTALL_DIR: ${{ github.workspace }}/install + CHANGED_REPO: ${{ inputs.changed_repo }} + CHANGED_SHA: ${{ inputs.changed_sha }} run: /home/mlsdkuser/ci_build.sh diff --git a/docker/DOCKER.md b/docker/DOCKER.md index 264170f..4d0a795 100644 --- a/docker/DOCKER.md +++ b/docker/DOCKER.md @@ -6,11 +6,3 @@ the `Dockerfile`: ```sh docker buildx build --tag ml-sdk-image --file Dockerfile --build-arg user=$(whoami) --build-arg uid=$(id -u) .. ``` - -# Troubleshooting -## Error `did not complete successfully: cannot allocate memory` -increase the memory by running - -```sh -docker buildx build --memory=64g --tag ml-sdk-image --file Dockerfile --build-arg user=$(whoami) --build-arg uid=$(id -u) .. -``` \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index bde742f..b869455 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -99,15 +99,6 @@ RUN git config --global user.name "github-actions[bot]" && \ git config --global color.ui false && \ git config --global --add safe.directory '*' -ARG REPO_OWNER=arm -ARG MANIFEST_URL=https://github.com/$REPO_OWNER/ai-ml-sdk-manifest.git -ARG REPO_DIR=./sdk -ARG INSTALL_DIR=./install -ARG CHANGED_REPO -ARG CHANGED_SHA - -ENV MANIFEST_URL=$MANIFEST_URL REPO_DIR=$REPO_DIR INSTALL_DIR=$INSTALL_DIR CHANGED_REPO=$CHANGED_REPO CHANGED_SHA=$CHANGED_SHA COPY --chown=$user --chmod=755 ../.github/workflows/scripts/ci_build.sh ./ -RUN ./ci_build.sh CMD ["/bin/bash"]