diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index 9fa6d962df3..82a45591c56 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -27,7 +27,7 @@ jobs: # Run once for each of the platforms strategy: matrix: - platform: ["generic", "orinnx"] + platform: ["generic_k1", "orinnx"] # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -65,6 +65,10 @@ jobs: - name: Enforce file permissions run: chmod -Rv g-w,o-w docker/ + - name: Get CPU count + id: cpu-count + run: echo "count=$(nproc)" >> $GITHUB_OUTPUT + # Build the docker image - name: 🐳 Build and push the docker image 🐳 uses: docker/build-push-action@v4 @@ -73,7 +77,9 @@ jobs: tags: "nubots/nubots:${{ matrix.platform }}" file: docker/Dockerfile context: docker - build-args: platform=${{ matrix.platform }} + build-args: | + platform=${{ matrix.platform == 'generic_k1' && 'generic' || matrix.platform }} + jobs=${{ steps.cpu-count.outputs.count }} cache-from: type=registry,ref=nubots/nubots:${{ matrix.platform }} cache-to: type=inline push: true diff --git a/.github/workflows/nubots.yaml b/.github/workflows/nubots.yaml index bf48e7fa165..570b1e785fe 100644 --- a/.github/workflows/nubots.yaml +++ b/.github/workflows/nubots.yaml @@ -117,6 +117,10 @@ jobs: - name: Enforce file permissions run: chmod -Rv g-w,o-w docker/ + - name: Get CPU count + id: cpu-count + run: echo "count=$(nproc)" >> $GITHUB_OUTPUT + # Build and push the docker image - name: 🐳 Build the docker image 🐳 uses: docker/build-push-action@v4 @@ -129,7 +133,9 @@ jobs: type=registry,ref=nubots/nubots:${{ env.DOCKER_TAG }} type=registry,ref=nubots/nubots:generic cache-to: type=inline - build-args: platform=generic + build-args: | + platform=generic + jobs=${{ steps.cpu-count.outputs.count }} push: true - id: image_output diff --git a/.github/workflows/nusight.yaml b/.github/workflows/nusight.yaml index 714c6b3b52b..24b54c9b47b 100644 --- a/.github/workflows/nusight.yaml +++ b/.github/workflows/nusight.yaml @@ -121,6 +121,10 @@ jobs: - name: Enforce file permissions run: chmod -Rv g-w,o-w docker/ + - name: Get CPU count + id: cpu-count + run: echo "count=$(nproc)" >> $GITHUB_OUTPUT + # Build and push the docker image - name: 🐳 Build the docker image 🐳 uses: docker/build-push-action@v4 @@ -133,7 +137,9 @@ jobs: type=registry,ref=nubots/nubots:${{ env.DOCKER_TAG }} type=registry,ref=nubots/nubots:generic cache-to: type=inline - build-args: platform=generic + build-args: | + platform=generic + jobs=${{ steps.cpu-count.outputs.count }} push: true - id: image_output diff --git a/docker/Dockerfile b/docker/Dockerfile index 7626eee1e69..f84996525a8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -107,6 +107,16 @@ RUN --mount=type=bind,source=usr/local/toolchain,target=/usr/local/toolchain \ # |_|\___/ \___/|_|\___|_| |_|\__,_|_|_| |_| # ################################################ ARG prefix=/usr/local +ARG jobs=1 + +# Install prefix/toolchain-dir for source-built dependencies below, computed once per platform +RUN if [ "${platform}" = "orinnx" ]; then \ + echo "DEP_PREFIX=/l4t/targetfs${prefix}" > /root/.dep_env && \ + echo "DEP_TOOLCHAIN='--toolchain-dir /usr/local'" >> /root/.dep_env; \ + else \ + echo "DEP_PREFIX=/usr/local" > /root/.dep_env && \ + echo "DEP_TOOLCHAIN=''" >> /root/.dep_env; \ + fi # Create a python virtual environment at ${PREFIX} to decouple python packages from the system RUN python3 -m venv ${prefix} @@ -133,16 +143,20 @@ RUN install-package -p ${platform} libncurses-dev libncurses6 libtinfo6 libc6-de RUN pip3 install pyyaml mako # Install OpenCL C Headers -RUN install-from-source https://github.com/KhronosGroup/OpenCL-Headers/archive/refs/tags/v2022.05.18.tar.gz \ - --prefix /l4t/targetfs/${prefix} \ - --toolchain-dir /usr/local +RUN . /root/.dep_env && \ + install-from-source https://github.com/KhronosGroup/OpenCL-Headers/archive/refs/tags/v2022.05.18.tar.gz \ + --jobs ${jobs} \ + --prefix ${DEP_PREFIX} \ + ${DEP_TOOLCHAIN} # OpenCL loader library RUN install-package ruby autotools-dev automake libtool -RUN install-from-source https://github.com/OCL-dev/ocl-icd/archive/refs/tags/v2.3.1.tar.gz \ - --prefix /l4t/targetfs/${prefix} \ - --toolchain-dir /usr/local +RUN . /root/.dep_env && \ + install-from-source https://github.com/OCL-dev/ocl-icd/archive/refs/tags/v2.3.1.tar.gz \ + --jobs ${jobs} \ + --prefix ${DEP_PREFIX} \ + ${DEP_TOOLCHAIN} RUN if [ "${platform}" != "generic" ]; then \ ln -sf libm.so.6 /l4t/targetfs/usr/lib/aarch64-linux-gnu/libm.so && \ @@ -152,7 +166,8 @@ RUN if [ "${platform}" != "generic" ]; then \ fi # OpenCV -RUN install-from-source https://github.com/opencv/opencv/archive/refs/tags/4.9.0.tar.gz +RUN install-from-source https://github.com/opencv/opencv/archive/refs/tags/4.9.0.tar.gz \ + --jobs ${jobs} RUN wget -nv https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protoc-21.4-linux-x86_64.zip -O protoc-21.4-linux-x86_64.zip && \ unzip protoc-21.4-linux-x86_64.zip -d /usr/ && \ @@ -160,6 +175,7 @@ RUN wget -nv https://github.com/protocolbuffers/protobuf/releases/download/v21.4 # Protobuf RUN install-from-source https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-cpp-3.21.4.tar.gz \ + --jobs ${jobs} \ -Dprotobuf_PROTOC_EXECUTABLE="/usr/bin/protoc" \ -Dprotobuf_BUILD_TESTS=OFF \ -Dprotobuf_BUILD_LIBPROTOC=ON \ @@ -171,6 +187,7 @@ RUN if [ "${platform}" != "generic" ]; then \ unzip protoc-21.4-linux-aarch_64.zip -d /l4t/targetfs/usr/ && \ rm protoc-21.4-linux-aarch_64.zip && \ install-from-source https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-cpp-3.21.4.tar.gz \ + --jobs ${jobs} \ --toolchain-dir /usr \ -Dprotobuf_PROTOC_EXECUTABLE="/usr/bin/protoc" \ -Dprotobuf_BUILD_TESTS=OFF \ @@ -192,6 +209,7 @@ RUN install-package opencl-c-headers \ RUN install-from-source https://github.com/uxlfoundation/oneTBB/archive/refs/tags/v2021.13.0.tar.gz \ + --jobs ${jobs} \ --build-system cmake \ -DCMAKE_BUILD_TYPE=Release \ -DTBB_TEST:BOOL='OFF' \ @@ -219,13 +237,14 @@ RUN mkdir build && cd build && \ -DENABLE_SYSTEM_PROTOBUF:BOOL='OFF' \ -DENABLE_SYSTEM_FLATBUFFERS:BOOL='OFF' \ -DENABLE_INTEL_GPU:BOOL='OFF' \ - -DENABLE_INTEL_CPU:BOOL='OFF' \ - -DENABLE_ARM_COMPUTE_CMAKE:BOOL='ON' \ + -DENABLE_INTEL_CPU:BOOL='ON' \ + -DENABLE_ARM_COMPUTE_CMAKE:BOOL='OFF' \ + -DARM_COMPUTE_SCONS_JOBS="${jobs}" \ -DENABLE_CPPLINT:BOOL='OFF' \ -DCMAKE_CXX_STANDARD='17' \ -Wno-dev \ -GNinja && \ - ninja -j$(nproc) && ninja -j$(nproc) install + ninja -j${jobs} && ninja -j${jobs} install # Move OpenVino libraries to our library path RUN ARCH=$([ "${platform}" = "orinnx" ] && echo "aarch64" || echo "intel64") && \ @@ -260,19 +279,23 @@ RUN install-package -p ${platform} \ libnvinfer-vc-plugin-dev # Eigen3 -RUN install-from-source https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz +RUN install-from-source https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz \ + --jobs ${jobs} # tcmalloc -RUN install-from-source https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz \ +RUN . /root/.dep_env && \ + install-from-source https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz \ + --jobs ${jobs} \ --build-system autotools \ - --prefix /l4t/targetfs/${prefix} \ - --toolchain-dir /usr/local \ + --prefix ${DEP_PREFIX} \ + ${DEP_TOOLCHAIN} \ --with-tcmalloc-pagesize=64 \ --enable-minimal # Libjpeg RUN install-package yasm RUN install-from-source https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/2.0.7-esr.tar.gz \ + --jobs ${jobs} \ -DWITH_SIMD=ON \ -DFORCE_INLINE=ON \ -DINLINE_WORKS=1 \ @@ -286,16 +309,19 @@ RUN install-from-source https://github.com/libjpeg-turbo/libjpeg-turbo/archive/r # yaml-cpp RUN install-from-source https://github.com/jbeder/yaml-cpp/archive/refs/tags/yaml-cpp-0.7.0.tar.gz \ + --jobs ${jobs} \ -DYAML_CPP_BUILD_TESTS=OFF \ -DBUILD_SHARED_LIBS=OFF # fmt formatting library RUN install-from-source https://github.com/fmtlib/fmt/archive/refs/tags/9.0.0.tar.gz \ + --jobs ${jobs} \ -DFMT_DOC=OFF \ -DFMT_TEST=OFF # Catch unit testing library RUN install-from-source https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz \ + --jobs ${jobs} \ -DCATCH_CONFIG_CONSOLE_WIDTH=120 \ -DCATCH_CONFIG_CPP11_TO_STRING=ON \ -DCATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS=ON \ @@ -306,69 +332,83 @@ RUN install-from-source https://github.com/catchorg/Catch2/archive/refs/tags/v3. # LibUV RUN install-from-source https://github.com/libuv/libuv/archive/refs/tags/v1.44.2.tar.gz \ + --jobs ${jobs} \ -Dlibuv_buildtests=OFF \ -DBUILD_TESTING=OFF # NUClear! RUN install-from-source https://github.com/Fastcode/NUClear/archive/925dca0f31484a7df64fd335de5a6c9335483c7f.tar.gz \ + --jobs ${jobs} \ -DBUILD_TESTS=OFF # Backtrace -RUN install-from-source https://github.com/ianlancetaylor/libbacktrace/archive/8602fda64e78f1f46563220f2ee9f7e70819c51d.tar.gz \ - --prefix /l4t/targetfs/${prefix} \ - --toolchain-dir /usr/local \ +RUN . /root/.dep_env && \ + install-from-source https://github.com/ianlancetaylor/libbacktrace/archive/8602fda64e78f1f46563220f2ee9f7e70819c51d.tar.gz \ + --jobs ${jobs} \ + --prefix ${DEP_PREFIX} \ + ${DEP_TOOLCHAIN} \ --without-system-libunwind \ --enable-shared \ --enable-static # Alsa and espeak -RUN install-from-source https://github.com/alsa-project/alsa-lib/archive/refs/tags/v1.2.7.2.tar.gz \ - --prefix /l4t/targetfs/${prefix} \ - --toolchain-dir /usr/local \ +RUN . /root/.dep_env && \ + install-from-source https://github.com/alsa-project/alsa-lib/archive/refs/tags/v1.2.7.2.tar.gz \ + --jobs ${jobs} \ + --prefix ${DEP_PREFIX} \ + ${DEP_TOOLCHAIN} \ --without-debug RUN install-package -p ${platform} alsa-utils # mio memory mapping library RUN install-from-source https://github.com/mandreyel/mio/archive/3f86a95c0784d73ce6815237ec33ed25f233b643.tar.gz \ + --jobs ${jobs} \ --patch https://patch-diff.githubusercontent.com/raw/mandreyel/mio/pull/93.patch # zstr header to have file streams that do zlib compression -RUN INCLUDE_PREFIX=$([ "${platform}" = "orinnx" ] && echo "/l4t/targetfs/usr/local" || echo "/usr/local") && \ +RUN . /root/.dep_env && \ install-from-source https://raw.githubusercontent.com/mateidavid/zstr/v1.0.6/src/zstr.hpp \ - --prefix ${INCLUDE_PREFIX} --no-toolchain \ + --prefix ${DEP_PREFIX} --no-toolchain \ --header-path include/zstr && \ install-from-source https://raw.githubusercontent.com/mateidavid/zstr/v1.0.6/src/strict_fstream.hpp \ - --prefix ${INCLUDE_PREFIX} --no-toolchain \ + --prefix ${DEP_PREFIX} --no-toolchain \ --header-path include/zstr # Visual Mesh RUN install-from-source https://github.com/NUbots/VisualMesh/archive/64c49fce3a17bbee73339eaee556f47a80b93c40.tar.gz \ + --jobs ${jobs} \ -DBUILD_TENSORFLOW_OP=OFF \ -DBUILD_EXAMPLES=OFF \ -DBUILD_OPENCL_ENGINE=ON # JSON for modern C++ (https://github.com/nlohmann/json) -RUN INCLUDE_PREFIX=$([ "${platform}" = "orinnx" ] && echo "/l4t/targetfs/usr/local" || echo "/usr/local") && \ +RUN . /root/.dep_env && \ install-from-source https://github.com/nlohmann/json/releases/download/v3.10.5/json.hpp \ - --prefix ${INCLUDE_PREFIX} --no-toolchain + --prefix ${DEP_PREFIX} --no-toolchain # TinyXML -RUN install-from-source https://github.com/leethomason/tinyxml2/archive/refs/tags/9.0.0.tar.gz +RUN install-from-source https://github.com/leethomason/tinyxml2/archive/refs/tags/9.0.0.tar.gz \ + --jobs ${jobs} # NLopt -RUN install-from-source https://github.com/stevengj/nlopt/archive/refs/tags/v2.7.1.tar.gz +RUN install-from-source https://github.com/stevengj/nlopt/archive/refs/tags/v2.7.1.tar.gz \ + --jobs ${jobs} # tinyrobotics RUN install-from-source https://github.com/Tom0Brien/tinyrobotics/archive/refs/tags/0.0.1.tar.gz \ + --jobs ${jobs} \ -DBUILD_TESTS=OFF # lame -RUN install-from-source https://downloads.sourceforge.net/lame/lame-3.100.tar.gz \ - --prefix /l4t/targetfs/${prefix} \ - --toolchain-dir /usr/local +RUN . /root/.dep_env && \ + install-from-source https://downloads.sourceforge.net/lame/lame-3.100.tar.gz \ + --jobs ${jobs} \ + --prefix ${DEP_PREFIX} \ + ${DEP_TOOLCHAIN} # Nanopb for generating messages for NUSense -RUN install-from-source https://github.com/nanopb/nanopb/archive/refs/tags/0.4.9.1.tar.gz +RUN install-from-source https://github.com/nanopb/nanopb/archive/refs/tags/0.4.9.1.tar.gz \ + --jobs ${jobs} # Install Booster Robotics SDK @@ -401,6 +441,9 @@ RUN install-package \ gcc-12 \ g++-12 +# qemu lets ctest run the cross compiled test binaries, see CMAKE_CROSSCOMPILING_EMULATOR in the toolchain +RUN if [ "${platform}" != "generic" ]; then install-package qemu-user-static; fi + # Node tools RUN wget -nv https://nodejs.org/dist/v20.13.1/node-v20.13.1-linux-x64.tar.xz -O - \ @@ -427,6 +470,10 @@ RUN install-package clinfo # Copy the nvidia OpenCL vendorfile into the container so we can also use OpenCL on nvidia GPUs COPY --chown=root:root etc/OpenCL/vendors/nvidia.icd /etc/OpenCL/vendors/nvidia.icd +# Creates symlink for librt since CUDAToolkit CMake module can't find it +# See: https://notes.rdu.im/howtos/fix-librt-not-found-in-jammy/ +RUN ln -s /lib/x86_64-linux-gnu/librt.so.1 /usr/lib/x86_64-linux-gnu/librt.so + # Re-run ldconfig to make sure the database is up to date RUN ldconfig diff --git a/docker/usr/local/bin/install-package b/docker/usr/local/bin/install-package index f5ad5be551f..1a5972ef2bb 100755 --- a/docker/usr/local/bin/install-package +++ b/docker/usr/local/bin/install-package @@ -1,6 +1,11 @@ #!/bin/sh set -e +# If we are not root, rerun using sudo before the arguments are consumed by the parser below +if ! [ $(id -u) = 0 ]; then + exec sudo "$0" "$@" +fi + # Parse arguments platform="" packages="" @@ -19,66 +24,57 @@ while [ $# -gt 0 ]; do esac done -# If we are not root, rerun using sudo -if ! [ $(id -u) = 0 ]; then - sudo "$0" "$@" -else - if [ -n "$platform" ]; then - # -p flag was provided with a value - if [ "$platform" = "orinnx" ]; then - # Install amd64 packages on host AND extract arm64 to sysroot - apt-get update +if [ -z "$packages" ]; then + echo "Error: No packages given to install" >&2 + exit 1 +fi - # Install amd64 versions on host - apt-get install -y --no-install-recommends $packages +if [ "$platform" = "orinnx" ]; then + # Install amd64 packages on host AND extract arm64 to sysroot + apt-get update - # Download arm64 packages and their dependencies to sysroot - mkdir -p /tmp/debs - chmod 777 /tmp/debs - cd /tmp/debs + # Install amd64 versions on host + apt-get install -y --no-install-recommends $packages - # Get recursive dependencies for arm64 and download them - for pkg in $packages; do - # Download the requested package itself so -dev artifacts (e.g. libudev.so) - # are present in the sysroot, not just its runtime dependencies. - apt-get download "$pkg:arm64" || { echo "Error: Failed to download $pkg:arm64" >&2; exit 1; } + # Download arm64 packages and their dependencies to sysroot + mkdir -p /tmp/debs + chmod 777 /tmp/debs + cd /tmp/debs - deps=$(apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances "$pkg:arm64" 2>/dev/null | grep "^\w" | sort -u) - if [ -z "$deps" ]; then - echo "Error: No arm64 package found for $pkg" >&2 - exit 1 - fi - for dep in $deps; do - apt-get download "$dep" || { echo "Error: Failed to download $dep:arm64" >&2; exit 1; } - done - done + # Get recursive dependencies for arm64 and download them + for pkg in $packages; do + # Download the requested package itself so -dev artifacts (e.g. libudev.so) + # are present in the sysroot, not just its runtime dependencies. + apt-get download "$pkg:arm64" || { echo "Error: Failed to download $pkg:arm64" >&2; exit 1; } - # Extract all downloaded debs to sysroot - for deb in /tmp/debs/*.deb; do - [ -f "$deb" ] && dpkg -x "$deb" /l4t/targetfs - done + deps=$(apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances "$pkg:arm64" 2>/dev/null | grep "^\w" | sort -u) + if [ -z "$deps" ]; then + echo "Error: No arm64 package found for $pkg" >&2 + exit 1 + fi + for dep in $deps; do + apt-get download "$dep" || { echo "Error: Failed to download $dep:arm64" >&2; exit 1; } + done + done - # dpkg -x extracts absolute symlinks verbatim (e.g. libm.so -> /lib/aarch64-linux-gnu/libm.so.6), - # which resolve against the container root instead of the sysroot. Broken .so symlinks make the - # cross linker silently fall back to non-PIC static archives, so rewrite them as relative links. - find /l4t/targetfs -lname '/*' | while read -r link; do - target=$(readlink "$link") - ln -sfn "$(realpath -m --relative-to="$(dirname "$link")" "/l4t/targetfs$target")" "$link" - done + # Extract all downloaded debs to sysroot + for deb in /tmp/debs/*.deb; do + [ -f "$deb" ] && dpkg -x "$deb" /l4t/targetfs + done - cd / - rm -rf /tmp/debs - rm -rf /var/cache/apt /var/lib/apt/lists/* - else - # Normal install for other platforms - apt-get update - apt-get install -y --no-install-recommends $packages - rm -rf /var/cache/apt /var/lib/apt/lists/* - fi - else - # No -p flag, just install the packages - apt-get update - apt-get install -y --no-install-recommends $packages - rm -rf /var/cache/apt /var/lib/apt/lists/* - fi + # dpkg -x extracts absolute symlinks verbatim (e.g. libm.so -> /lib/aarch64-linux-gnu/libm.so.6), + # which resolve against the container root instead of the sysroot. Broken .so symlinks make the + # cross linker silently fall back to non-PIC static archives, so rewrite them as relative links. + find /l4t/targetfs -lname '/*' | while read -r link; do + target=$(readlink "$link") + ln -sfn "$(realpath -m --relative-to="$(dirname "$link")" "/l4t/targetfs$target")" "$link" + done + + cd / + rm -rf /tmp/debs + rm -rf /var/cache/apt /var/lib/apt/lists/* +else + apt-get update + apt-get install -y --no-install-recommends $packages + rm -rf /var/cache/apt /var/lib/apt/lists/* fi diff --git a/docker/usr/local/toolchain/generate_orinnx_toolchain.py b/docker/usr/local/toolchain/generate_orinnx_toolchain.py index 0949203b9ab..4eeadb9e5a9 100644 --- a/docker/usr/local/toolchain/generate_orinnx_toolchain.py +++ b/docker/usr/local/toolchain/generate_orinnx_toolchain.py @@ -63,6 +63,21 @@ "-Wl,--allow-shlib-undefined", "-Wl,-Bdynamic", ], + "emulator": [ + "/usr/bin/qemu-aarch64-static", + "-L", + _TARGETFS_DIR, + "-E", + "LD_LIBRARY_PATH={}".format( + ":".join( + [ + f"{_TARGETFS_DIR}/usr/local/lib", + f"{_TARGETFS_DIR}/usr/lib/{_CROSS_PREFIX}", + f"{_TARGETFS_DIR}/lib/{_CROSS_PREFIX}", + ] + ) + ), + ], "asm_object": "elf64", "arch": "aarch64", "prefix": _CROSS_PREFIX, diff --git a/docker/usr/local/toolchain/generate_toolchains.py b/docker/usr/local/toolchain/generate_toolchains.py index 98b82b8078d..dfaa20146c4 100755 --- a/docker/usr/local/toolchain/generate_toolchains.py +++ b/docker/usr/local/toolchain/generate_toolchains.py @@ -51,6 +51,11 @@ def generate_cmake_toolchain(target, prefix): else "\n" ) + emulator = target.get("emulator", []) + emulator_block = ( + 'set(CMAKE_CROSSCOMPILING_EMULATOR "{}")\n'.format(";".join(emulator)) if emulator else "\n" + ) + linker_flags = target.get("linker_flags", []) linker_flags_block = ( 'set(CMAKE_EXE_LINKER_FLAGS_INIT "{} " CACHE STRING "Flags used by the linker during all built types.")\n'.format( @@ -73,6 +78,7 @@ def generate_cmake_toolchain(target, prefix): set(CMAKE_C_COMPILER {c_compiler}) set(CMAKE_CXX_COMPILER {cxx_compiler}) + {emulator_block} {sysroot_block} set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) @@ -118,6 +124,7 @@ def generate_cmake_toolchain(target, prefix): arch=target["arch"], c_compiler=target.get("c_compiler", "/usr/bin/gcc"), cxx_compiler=target.get("cxx_compiler", "/usr/bin/g++"), + emulator_block=emulator_block, sysroot_block=sysroot_block, sysroot=sysroot, linker_flags_block=linker_flags_block, diff --git a/shared/utility/vision/TensorRT.cpp b/shared/utility/vision/TensorRT.cpp index fcef96f71ee..e6970f1afca 100644 --- a/shared/utility/vision/TensorRT.cpp +++ b/shared/utility/vision/TensorRT.cpp @@ -71,9 +71,19 @@ namespace utility::vision { } auto config = std::unique_ptr(builder->createBuilderConfig()); +#if NV_TENSORRT_MAJOR >= 11 + // kFP16 removed in TensorRT 11; precision follows model types instead + (void) fp16; +#elif NV_TENSORRT_MAJOR >= 10 + // platformHasFastFp16 removed in TensorRT 10; fast FP16 is now universal + if (fp16) { + config->setFlag(BuilderFlag::kFP16); + } +#else if (fp16 && builder->platformHasFastFp16()) { config->setFlag(BuilderFlag::kFP16); } +#endif auto plan = std::unique_ptr(builder->buildSerializedNetwork(*network, *config)); if (plan == nullptr) { diff --git a/tools/target.py b/tools/target.py index deee30dc65d..7bd492b3750 100755 --- a/tools/target.py +++ b/tools/target.py @@ -46,15 +46,21 @@ def register(command): command.add_argument( "-r", "--reset", default=False, action="store_true", dest="reset", help="Reset buildx instance" ) + command.add_argument( + "-j", "--jobs", default=defaults.jobs, help="the number of jobs to use when building the platform image" + ) -def run(target, username, uid, reset, **kwargs): +def run(target, username, uid, reset, jobs, **kwargs): if target is None: target = platform.selected(defaults.image, username) print(f"Currently selected platform is {target}") else: # Ensure the platform image is built - platform.build(defaults.image, target, username, uid, reset) + platform.build(defaults.image, target, username, uid, reset, jobs) + + if target == "generic": + target += "_k1" # Tag the built platform image as the selected image err = subprocess.call( diff --git a/tools/tests.py b/tools/tests.py index b72316395b2..8b53e6d4d91 100644 --- a/tools/tests.py +++ b/tools/tests.py @@ -38,7 +38,7 @@ TESTS_OUTPUT_DIR = "tests_output" -@run_on_docker(image="nubots:generic") +@run_on_docker def register(command): command.help = "Run and list tests" @@ -96,7 +96,7 @@ def register(command): ) -@run_on_docker(image="nubots:generic") +@run_on_docker def run(sub_command, num_jobs=0, test=None, given_ctest_args=[], **kwargs): # Change into the build directory diff --git a/tools/utility/dockerise/defaults.py b/tools/utility/dockerise/defaults.py index 1a760615bab..f40ca3e4d89 100644 --- a/tools/utility/dockerise/defaults.py +++ b/tools/utility/dockerise/defaults.py @@ -34,6 +34,7 @@ image = "nubots" image_user = "nubots" directory = "NUbots" +jobs = os.cpu_count() or 1 # This can fail if the local user id doesn't exist (e.g. you're in docker and set a uid) # In those cases, we really don't care so we just use the local userid instead diff --git a/tools/utility/dockerise/platform.py b/tools/utility/dockerise/platform.py index abe33bb8e22..c992fcfbcb0 100644 --- a/tools/utility/dockerise/platform.py +++ b/tools/utility/dockerise/platform.py @@ -80,12 +80,12 @@ def list(): ] -def build(image, platform, username, uid, reset): +def build(image, platform, username, uid, reset, jobs): pty = WrapPty() # If we are building the selected platform we need to work out what that refers to _selected = platform == "selected_k1" - platform = selected(image) if _selected else platform + platform = selected(image, username) if _selected else platform # Temporary workaround to prevent conflict with NUgus if platform == "generic": @@ -140,6 +140,9 @@ def build(image, platform, username, uid, reset): # Make sure buildx will use the correct buildx instance subprocess.run(["docker", "buildx", "use", builder_name]) + platform_arg = platform + if platform.endswith("_k1"): + platform_arg = platform[:-3] # Build the image! err = pty.spawn( [ @@ -150,7 +153,11 @@ def build(image, platform, username, uid, reset): local_tag, "--pull", "--build-arg", - f"platform={platform}", + f"platform={platform_arg}", + "--build-arg", + f"jobs={jobs}", + "--platform", + "linux/amd64", "--build-arg", f"user_uid={uid}", "--output=type=docker", diff --git a/tools/utility/dockerise/run.py b/tools/utility/dockerise/run.py index 8020c919455..497246fd7e6 100644 --- a/tools/utility/dockerise/run.py +++ b/tools/utility/dockerise/run.py @@ -98,7 +98,7 @@ def _setup_internal_image(image, rebuild, clean_volume, clean_uv_cache): # Rebuild the image if we are rebuilding if rebuild: - platform.build(repository, target) + platform.build(repository, target, defaults.local_user, os.getuid(), False, defaults.jobs) # Find the current target for this platform target = target if target != "selected_k1" else platform.selected(repository, defaults.local_user)