diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml new file mode 100644 index 00000000000..c4561d3ada3 --- /dev/null +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -0,0 +1,362 @@ +name: Build Linux(musllinux) ARM64 + +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' + +jobs: + build_musllinux_wheels: + name: Build musllinux wheels (Alpine Linux aarch64) + runs-on: GH-Linux-ARM64 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Configure git safe directory + run: | + git config --global --add safe.directory '*' + + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 + + - name: Build chdb wheels in container + uses: addnab/docker-run-action@v3 + with: + image: quay.io/pypa/musllinux_1_2_aarch64 + options: -v ${{ github.workspace }}:/workspace --privileged + run: | + cd /workspace + + # Configure git safe directory in container + apk update + apk add --no-cache git python3 py3-pip py3-setuptools + echo "=== Configure git safe directory ===" + git config --global --add safe.directory /workspace + git describe --tags + python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" + + # 1. Check system info + echo "=== Container System Info ===" + echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + if [ -f /lib/ld-musl-aarch64.so.1 ]; then + echo "musl libc aarch64" + elif [ -f /lib/libc.musl-aarch64.so.1 ]; then + echo "musl libc aarch64" + else + echo "Not musl libc" + fi + echo "Workspace mounted at: /workspace" + ls -la /workspace + + # 2. Install build dependencies + echo "=== Installing build dependencies ===" + apk add --no-cache make build-base openssl-dev zlib-dev \ + bzip2-dev readline-dev sqlite-dev wget curl llvm \ + ncurses-dev xz-dev tk-dev libxml2-dev \ + libffi-dev linux-headers + apk add --no-cache make cmake ccache ninja yasm gawk + apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + + # 3. Scan SQLite vulnerabilities + echo "=== Scanning SQLite vulnerabilities ===" + # Install grype + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + grype db update + + # Check SQLite vulnerabilities + echo "Scanning SQLite packages for vulnerabilities..." + GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) + echo "Raw grype output:" + echo "$GRYPE_RAW_OUTPUT" + + SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" + else + echo "No SQLite vulnerabilities found" + fi + + # 4. Setup Python environments + echo "=== Setting up Python environments ===" + # Setup pyenv + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + + # Install Python versions + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Installing Python $version" + pyenv install $version:latest + done + pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + + # Verify installations + echo "Installed versions:" + pyenv versions + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "Python $version is installed" + done + echo "All Python versions verified successfully!" + + # Install Rust + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source $HOME/.cargo/env + rustup toolchain install nightly-2025-07-07 + rustup component add --toolchain nightly-2025-07-07 rust-src + + # Install Python dependencies + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + if [ "$version" = "3.8" ]; then + python -m pip install setuptools tox twine psutil wheel + else + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + fi + pyenv shell --unset + done + + # 5. Build chdb + echo "=== Building chdb ===" + echo "Timestamp: $(date)" + echo "Current directory: $(pwd)" + echo "Available disk space: $(df -h .)" + + # Setup clang + echo "Setting up clang compiler..." + ln -sf /usr/bin/clang-20 /usr/bin/clang + ln -sf /usr/bin/clang++-20 /usr/bin/clang++ + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + echo "Compiler versions:" + $CC --version + $CXX --version + + # Build + echo "Starting chdb build with Python 3.8..." + pyenv shell 3.8 + python --version + echo "Build start time: $(date)" + bash ./chdb/build-musl.sh + echo "Build end time: $(date)" + + # Test + echo "Running smoke test with Python 3.9..." + pyenv shell 3.9 + python --version + echo "Test start time: $(date)" + bash -x ./chdb/test_smoke.sh + echo "Test end time: $(date)" + + # Check build results + echo "Build results summary:" + ccache -s + echo "chdb directory contents:" + ls -lh chdb + echo "Build artifacts size:" + du -sh chdb + + # 6. Create and audit wheels + echo "=== Creating and auditing wheels ===" + echo "Wheel creation start time: $(date)" + echo "Available disk space before wheel build: $(df -h .)" + + # Build wheels + echo "Building wheels with Python 3.8..." + pyenv shell 3.8 + python --version + echo "Running make wheel..." + make wheel + echo "Wheel build completed at: $(date)" + echo "Initial wheel files:" + ls -lh dist/ || echo "No dist directory yet" + + # Install patchelf + echo "Installing patchelf for wheel auditing..." + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + cp bin/patchelf /usr/bin/ + chmod +x /usr/bin/patchelf + echo "patchelf version: $(patchelf --version)" + + # Audit wheels + echo "Auditing wheels with Python 3.13..." + pyenv shell 3.13 + python --version + python -m pip install auditwheel + echo "auditwheel version: $(auditwheel --version)" + echo "Starting wheel audit at: $(date)" + auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl + echo "Wheel audit completed at: $(date)" + + # Clean up non-musllinux wheels + echo "Cleaning up non-musllinux wheels..." + echo "Before cleanup:" + ls -lh dist/ + rm -f dist/*-linux_aarch64.whl + echo "After cleanup:" + ls -lh dist/ + echo "Final wheel sizes:" + du -sh dist/* + + # 7. Test wheels + echo "=== Testing wheels ===" + echo "Wheel testing start time: $(date)" + echo "Available wheels for testing:" + ls -lh dist/*.whl + echo "Wheel file details:" + file dist/*.whl + + TOTAL_TESTS=5 + CURRENT_TEST=0 + TEST_FAILED=false + + for version in 3.9 3.10 3.11 3.12 3.13; do + CURRENT_TEST=$((CURRENT_TEST + 1)) + echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" + echo "Test start time: $(date)" + + echo "Switching to Python $version..." + pyenv shell $version + python --version + echo "pip version: $(python -m pip --version)" + + echo "Installing chdb wheel..." + python -m pip install dist/*.whl --force-reinstall + echo "Installation completed at: $(date)" + + echo "Running basic query test..." + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + + echo "Running full test suite..." + if make test; then + echo "Test suite PASSED for Python $version at: $(date)" + else + echo "Test suite FAILED for Python $version at: $(date)" + TEST_FAILED=true + break + fi + + pyenv shell --unset + echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" + echo "" + done + + echo "All wheel tests completed at: $(date)" + + # Check if any tests failed + if [ "$TEST_FAILED" = true ]; then + echo "ERROR: One or more test suites failed!" + echo "Test failure detected - aborting build process" + exit 1 + fi + + # Create test success marker file only if all tests passed + echo "All tests passed successfully!" + echo "Creating test success marker..." + touch /workspace/.test_success_marker + echo "Test success marker created at: $(date)" + + # 8. Scan chdb libraries + echo "=== Scanning chdb libraries ===" + FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" + SQLITE_VULNERABILITIES_FOUND=false + + for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi + done + + if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "SQLite vulnerabilities detected in chdb libraries!" + else + echo "No SQLite vulnerabilities found in chdb libraries" + fi + + # Show final results + echo "=== Final wheel files ===" + ls -la ./dist/ + continue-on-error: false + # Check test success before upload + - name: Verify test completion + run: | + echo "=== Verifying test completion ===" + if [ ! -f ".test_success_marker" ]; then + echo "ERROR: Test success marker file not found!" + echo "This indicates that the wheel testing did not complete successfully." + echo "Aborting upload process." + exit 1 + fi + echo "Test success marker found. All tests completed successfully." + echo "Proceeding with wheel upload..." + continue-on-error: false + # Upload wheels to release + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + echo "=== Uploading wheels to release ===" + ls -la ./dist/ + gh release upload ${{ github.ref_name }} ./dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + + # Upload to PyPI + - name: Upload to PyPI + if: startsWith(github.ref, 'refs/tags/v') + uses: addnab/docker-run-action@v3 + with: + image: quay.io/pypa/musllinux_1_2_aarch64 + options: -v ${{ github.workspace }}:/workspace + run: | + cd /workspace + echo "=== Uploading to PyPI ===" + + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.13 + python -m twine upload ./dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + + # Upload artifacts + - name: Upload build artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: chdb-artifacts-musllinux-aarch64 + path: | + ./dist/*.whl + overwrite: true diff --git a/.github/workflows/build_musllinux_x86_wheels.yml b/.github/workflows/build_musllinux_x86_wheels.yml new file mode 100644 index 00000000000..3797060cefd --- /dev/null +++ b/.github/workflows/build_musllinux_x86_wheels.yml @@ -0,0 +1,270 @@ +name: Build Linux(musllinux) x86 + +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' + + +jobs: + build_musllinux_wheels: + name: Build musllinux wheels (Alpine Linux x86_64) + runs-on: gh-64c + container: + image: quay.io/pypa/musllinux_1_2_x86_64 + options: --privileged + steps: + - name: Check system info + run: | + echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + if [ -f /lib/ld-musl-x86_64.so.1 ]; then + echo "musl libc x86_64" + elif [ -f /lib/libc.musl-x86_64.so.1 ]; then + echo "musl libc x86_64" + else + echo "Not musl libc" + fi + + echo "=== CPU Information ===" + cat /proc/cpuinfo + echo "" + echo "=== Checking CPU requirements ===" + if grep -q "ssse3" /proc/cpuinfo && grep -q "sse4_1" /proc/cpuinfo && grep -q "sse4_2" /proc/cpuinfo; then + echo "CPU meets minimum requirements" + else + echo "CPU does not meet minimum requirements" + fi + - name: Install Python build dependencies + run: | + apk update + apk add --no-cache make build-base openssl-dev zlib-dev \ + bzip2-dev readline-dev sqlite-dev wget curl llvm \ + ncurses-dev xz-dev tk-dev libxml2-dev \ + libffi-dev linux-headers + - name: Scan SQLite vulnerabilities with grype + run: | + # Install grype and required tools + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + + # Update grype vulnerability database + grype db update + + # Check SQLite vulnerabilities in installed packages + echo "Scanning SQLite packages for vulnerabilities..." + GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) + echo "Raw grype output:" + echo "$GRYPE_RAW_OUTPUT" + + SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + + if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" + else + echo "No SQLite vulnerabilities found" + fi + continue-on-error: false + - name: Setup pyenv + run: | + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv install 3.8:latest + pyenv install 3.9:latest + pyenv install 3.10:latest + pyenv install 3.11:latest + pyenv install 3.12:latest + pyenv install 3.13:latest + pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + + # Verify installations + echo "Installed versions:" + pyenv versions + - name: Verify pyenv installations + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + echo "Verifying all required Python versions are available:" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "Python $version is installed" + done + echo "All Python versions verified successfully!" + - name: Install dependencies for all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + if [ "$version" = "3.8" ]; then + python -m pip install setuptools tox twine psutil wheel + else + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + fi + pyenv shell --unset + done + - name: Install clang++ for Alpine + run: | + apk add --no-cache make cmake ccache ninja yasm gawk wget + apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + # Install Rust toolchain via rustup for proper target management + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source $HOME/.cargo/env + rustup toolchain install nightly-2025-07-07 + rustup component add --toolchain nightly-2025-07-07 rust-src + rustc --version + cargo --version + ccache -s + - name: Update git + run: | + apk add --no-cache git + git --version + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Configure git safe directory + run: | + git config --global --add safe.directory '*' + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: musllinux-1-2-x86_64 + max-size: 5G + append-timestamp: true + - name: setup clang and link clang-20 to clang + run: | + ln -sf /usr/bin/clang-20 /usr/bin/clang + ln -sf /usr/bin/clang++-20 /usr/bin/clang++ + which clang++ + clang++ --version + - name: Run chdb/build-musl.sh + timeout-minutes: 600 + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + source $HOME/.cargo/env + pyenv shell 3.8 + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + bash ./chdb/build-musl.sh + pyenv shell 3.9 + bash -x ./chdb/test_smoke.sh + continue-on-error: false + - name: Scan chdb libraries with grype + run: | + echo "Scanning chdb libraries for vulnerabilities..." + + FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" + + SQLITE_VULNERABILITIES_FOUND=false + + for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi + done + + if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "SQLite vulnerabilities detected in chdb libraries!" + else + echo "No SQLite vulnerabilities found in chdb libraries" + fi + continue-on-error: false + - name: Check ccache statistics + run: | + ccache -s + ls -lh chdb + df -h + - name: Build wheels + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + pyenv shell 3.8 + make wheel + - name: Install patchelf from github + run: | + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + cp bin/patchelf /usr/bin/ + chmod +x /usr/bin/patchelf + patchelf --version + - name: Audit wheels + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.13 + python -m pip install auditwheel + auditwheel -v repair -w dist/ --plat musllinux_1_2_x86_64 dist/*.whl + continue-on-error: false + - name: Show files + run: | + rm -f dist/*-linux_x86_64.whl + ls -lh dist + shell: bash + - name: Test wheel on all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.9 3.10 3.11 3.12 3.13; do + echo "Testing chdb on Python $version" + pyenv shell $version + python -m pip install dist/*.whl --force-reinstall + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + make test + pyenv shell --unset + done + continue-on-error: false + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - uses: actions/upload-artifact@v4 + with: + name: chdb-artifacts-musllinux-x86_64 + path: | + ./dist/*.whl + overwrite: true + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.13 + python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 099ff907e9a..f798d19698c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,6 +163,12 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWIT set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing") endif() +if (USE_MUSL) + add_definitions(-DUSE_MUSL=1 -D__MUSL__=1) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftls-model=global-dynamic") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftls-model=global-dynamic") +endif() + option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON) option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF) option(ENABLE_BENCHMARKS "Build all benchmark programs in 'benchmarks' subdirectories" OFF) @@ -396,7 +402,7 @@ option(ENABLE_LIBRARIES "Enable all external libraries by default" ON) # Increase stack size on Musl. We need big stack for our recursive-descend parser. if (USE_MUSL) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-stack_size,2097152") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,stack-size=2097152") endif () include(cmake/dbms_glob_sources.cmake) diff --git a/PreLoad.cmake b/PreLoad.cmake index 2124b7c0e9b..4672f0cb5c6 100644 --- a/PreLoad.cmake +++ b/PreLoad.cmake @@ -91,9 +91,13 @@ if (OS MATCHES "Linux" # - compile musl with debug and -fasynchronous-unwind-tables # # But none of this changes anything so far. - set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "") + if ("$ENV{USE_MUSL}" STREQUAL "") + set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "") + endif() elseif (ARCH MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)") - set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "") + if ("$ENV{USE_MUSL}" STREQUAL "") + set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "") + endif() elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)") set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "") elseif (ARCH MATCHES "^(s390x.*|S390X.*)") diff --git a/chdb/build-musl.sh b/chdb/build-musl.sh new file mode 100755 index 00000000000..208a491e692 --- /dev/null +++ b/chdb/build-musl.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +set -e + +export USE_MUSL=1 + +build_type=${1:-Release} + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +. ${DIR}/vars.sh + +BUILD_DIR=${PROJ_DIR}/buildlib + +HDFS="-DENABLE_HDFS=1 -DENABLE_GSASL_LIBRARY=1 -DENABLE_KRB5=1" +MYSQL="-DENABLE_MYSQL=1" +RUST_FEATURES="-DENABLE_RUST=0" +if [ "$(uname)" == "Linux" ]; then + GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=0" + UNWIND="-DUSE_UNWIND=1" + JEMALLOC="-DENABLE_JEMALLOC=0" + PYINIT_ENTRY="-Wl,-ePyInit_${CHDB_PY_MOD}" + ICU="-DENABLE_ICU=1" + SED_INPLACE="sed -i" + # only x86_64, enable AVX, enable embedded compiler + if [ "$(uname -m)" == "x86_64" ]; then + CPU_FEATURES="-DENABLE_AVX=1 -DENABLE_AVX2=0" + LLVM="-DENABLE_EMBEDDED_COMPILER=1 -DENABLE_DWARF_PARSER=1" + RUST_FEATURES="-DENABLE_RUST=1 -DENABLE_DELTA_KERNEL_RS=1" + CORROSION_CMAKE_FILE="${PROJ_DIR}/contrib/corrosion-cmake/CMakeLists.txt" + if [ -f "${CORROSION_CMAKE_FILE}" ]; then + if ! grep -q 'OPENSSL_NO_DEPRECATED_3_0' "${CORROSION_CMAKE_FILE}"; then + echo "Modifying corrosion CMakeLists.txt for Linux x86_64..." + ${SED_INPLACE} 's/corrosion_set_env_vars(${target_name} "RUSTFLAGS=${RUSTFLAGS}")/corrosion_set_env_vars(${target_name} "RUSTFLAGS=${RUSTFLAGS} --cfg osslconf=\\\"OPENSSL_NO_DEPRECATED_3_0\\\"")/g' "${CORROSION_CMAKE_FILE}" + else + echo "corrosion CMakeLists.txt already modified, skipping..." + fi + else + echo "Warning: corrosion CMakeLists.txt not found at ${CORROSION_CMAKE_FILE}" + fi + else + CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0 -DNO_ARMV81_OR_HIGHER=1" + LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0" + fi +else + echo "OS not supported" + exit 1 +fi + +if [ ! -d $BUILD_DIR ]; then + mkdir $BUILD_DIR +fi + +cd ${BUILD_DIR} +CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 -DENABLE_CLICKHOUSE_SERVER=0 -DENABLE_CLICKHOUSE_CLIENT=0 \ + -DENABLE_CLICKHOUSE_KEEPER=0 -DENABLE_CLICKHOUSE_KEEPER_CONVERTER=0 -DENABLE_CLICKHOUSE_LOCAL=1 -DENABLE_CLICKHOUSE_SU=0 -DENABLE_CLICKHOUSE_BENCHMARK=0 \ + -DENABLE_AZURE_BLOB_STORAGE=1 -DENABLE_CLICKHOUSE_COPIER=0 -DENABLE_CLICKHOUSE_DISKS=0 -DENABLE_CLICKHOUSE_FORMAT=0 -DENABLE_CLICKHOUSE_GIT_IMPORT=0 \ + -DENABLE_AWS_S3=1 -DENABLE_HIVE=0 -DENABLE_AVRO=1 \ + -DENABLE_CLICKHOUSE_OBFUSCATOR=0 -DENABLE_CLICKHOUSE_ODBC_BRIDGE=0 -DENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER=0 \ + -DENABLE_KAFKA=1 -DENABLE_LIBPQXX=1 -DENABLE_NATS=0 -DENABLE_AMQPCPP=0 -DENABLE_NURAFT=0 \ + -DENABLE_CASSANDRA=0 -DENABLE_ODBC=0 -DENABLE_NLP=0 \ + -DENABLE_LDAP=0 \ + -DUSE_MUSL=1 \ + -DRust_RUSTUP_INSTALL_MISSING_TARGET=ON \ + ${MYSQL} \ + ${HDFS} \ + -DENABLE_LIBRARIES=0 ${RUST_FEATURES} \ + ${GLIBC_COMPATIBILITY} \ + -DENABLE_UTILS=0 ${LLVM} ${UNWIND} \ + ${ICU} -DENABLE_UTF8PROC=1 ${JEMALLOC} \ + -DENABLE_PARQUET=1 -DENABLE_ROCKSDB=1 -DENABLE_SQLITE=1 -DENABLE_VECTORSCAN=1 \ + -DENABLE_PROTOBUF=1 -DENABLE_THRIFT=1 -DENABLE_MSGPACK=1 \ + -DENABLE_BROTLI=1 -DENABLE_H3=1 -DENABLE_CURL=1 \ + -DENABLE_CLICKHOUSE_ALL=0 -DUSE_STATIC_LIBRARIES=1 -DSPLIT_SHARED_LIBRARIES=0 \ + -DENABLE_SIMDJSON=1 -DENABLE_RAPIDJSON=1 \ + ${CPU_FEATURES} \ + -DENABLE_AVX512=0 -DENABLE_AVX512_VBMI=0 \ + -DENABLE_LIBFIU=1 \ + ${COMPILER_CACHE} \ + -DCHDB_VERSION=${CHDB_VERSION} \ + " + +BINARY=${BUILD_DIR}/programs/clickhouse + +# build chdb python module +py_version="3.8" +current_py_version=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") +if [ "$current_py_version" != "$py_version" ]; then + echo "Error: Current Python version is $current_py_version, but required version is $py_version" + echo "Please switch to Python $py_version using: pyenv shell $py_version" + exit 1 +fi +echo "Using Python version: $current_py_version" +cmake ${CMAKE_ARGS} -DENABLE_PYTHON=1 -DPYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION=${py_version} .. +ninja -d keeprsp || true + +# del the binary and run ninja -v again to capture the command, then modify it to generate CHDB_PY_MODULE +/bin/rm -f ${BINARY} +cd ${BUILD_DIR} +ninja -d keeprsp -v > build.log || true + +USING_RESPONSE_FILE=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log | grep '@CMakeFiles/clickhouse.rsp' || true) + +if [ ! "${USING_RESPONSE_FILE}" == "" ]; then + if [ -f CMakeFiles/clickhouse.rsp ]; then + cp -a CMakeFiles/clickhouse.rsp CMakeFiles/pychdb.rsp + else + echo "CMakeFiles/clickhouse.rsp not found" + exit 1 + fi +fi + +# extract the command to generate CHDB_PY_MODULE +PYCHDB_CMD=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log \ + | sed "s/-o programs\/clickhouse/-fPIC -Wl,-undefined,dynamic_lookup -shared ${PYINIT_ENTRY} -o ${CHDB_PY_MODULE}/" \ + | sed 's/^[^&]*&& //' | sed 's/&&.*//' \ + | sed 's/ -Wl,-undefined,error/ -Wl,-undefined,dynamic_lookup/g' \ + | sed 's/ -Xlinker --no-undefined//g' \ + | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/pychdb.rsp/g' \ + ) + +PYCHDB_CMD=$(echo ${PYCHDB_CMD} | sed 's/ src\/CMakeFiles\/clickhouse_malloc.dir\/Common\/stubFree.c.o//g') +if [ ! "${USING_RESPONSE_FILE}" == "" ]; then + ${SED_INPLACE} 's/ src\/CMakeFiles\/clickhouse_malloc.dir\/Common\/stubFree.c.o//g' CMakeFiles/pychdb.rsp +fi + +PYCHDB_CMD=$(echo ${PYCHDB_CMD} | sed 's|-Wl,-rpath,/[^[:space:]]*/pybind11-cmake|-Wl,-rpath,\$ORIGIN|g') + +echo ${PYCHDB_CMD} > pychdb_cmd.sh + +${PYCHDB_CMD} + +ls -lh ${CHDB_PY_MODULE} + +PYCHDB=${BUILD_DIR}/${CHDB_PY_MODULE} + +if [ ${build_type} == "Debug" ]; then + echo -e "\nDebug build, skip strip" +else + echo -e "\nStrip the binary:" + ${STRIP} --remove-section=.comment --remove-section=.note ${PYCHDB} +fi +echo -e "\nStripe the binary:" + +echo -e "\nPYCHDB: ${PYCHDB}" +ls -lh ${PYCHDB} +echo -e "\nldd ${PYCHDB}" +${LDD} ${PYCHDB} || echo "Binary is statically linked (not a dynamic executable)" +echo -e "\nfile info of ${PYCHDB}" +file ${PYCHDB} + +rm -f ${CHDB_DIR}/*.so +cp -a ${PYCHDB} ${CHDB_DIR}/${CHDB_PY_MODULE} + +echo -e "\nSymbols:" +echo -e "\nPyInit in PYCHDB: ${PYCHDB}" +${NM} ${PYCHDB} | grep PyInit || true +echo -e "\nquery_stable in PYCHDB: ${PYCHDB}" +${NM} ${PYCHDB} | grep query_stable || true + +echo -e "\nAfter copy:" +cd ${PROJ_DIR} && pwd + +ccache -s || true + +CMAKE_ARGS="${CMAKE_ARGS}" bash ${DIR}/build_pybind11.sh --all diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index fa58a739611..264157aa86c 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -98,7 +98,7 @@ elseif (ARCH_AARCH64) # and the build machine is too old, i.e. doesn't satisfy above modern profile, then these intermediate binaries will not run (dump # SIGILL). Even if they could run, the build machine wouldn't be able to run the ClickHouse binary. In that case, suggest to run the # build with the compat profile. - if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)" AND NOT NO_ARMV81_OR_HIGHER) + if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)" AND NOT NO_ARMV81_OR_HIGHER AND NOT USE_MUSL) # CPU features in /proc/cpuinfo and compiler flags don't align :( ... pick some obvious flags contained in the modern but not in the # legacy profile (full Graviton 3 /proc/cpuinfo is "fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm # jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm ssbs paca pacg dcpodp svei8mm svebf16 i8mm @@ -156,7 +156,7 @@ elseif (ARCH_AMD64) endif() # Same best-effort check for x86 as above for ARM. - if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER) + if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER AND NOT USE_MUSL) # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse # sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf diff --git a/contrib/pybind11 b/contrib/pybind11 index e4b03842254..b1c2263dc37 160000 --- a/contrib/pybind11 +++ b/contrib/pybind11 @@ -1 +1 @@ -Subproject commit e4b03842254775bbb7b8567376787e6eb04ebfd3 +Subproject commit b1c2263dc37c7b3aec9c210bf1beb8b83dbbb995 diff --git a/contrib/pybind11-cmake/CMakeLists.txt b/contrib/pybind11-cmake/CMakeLists.txt index 42d577b8205..0427cf47b71 100644 --- a/contrib/pybind11-cmake/CMakeLists.txt +++ b/contrib/pybind11-cmake/CMakeLists.txt @@ -23,7 +23,11 @@ set(STUBS_SOURCES "${LIBRARY_DIR}/source/non_limited_api/non_limited_api_stubs.cpp" ) -add_library(pybind11_stubs SHARED ${STUBS_SOURCES}) +if (NOT USE_MUSL) + add_library(pybind11_stubs SHARED ${STUBS_SOURCES}) +else() + add_library(pybind11_stubs STATIC ${STUBS_SOURCES}) +endif() target_include_directories(pybind11_stubs PRIVATE ${PYBIND11_INCLUDE_DIR}) target_include_directories(pybind11_stubs PRIVATE ${Python_INCLUDE_DIRS}) target_compile_definitions(pybind11_stubs PUBLIC @@ -43,6 +47,9 @@ if (APPLE) target_link_options(pybind11_stubs PRIVATE -undefined dynamic_lookup) else() target_link_options(pybind11_stubs PRIVATE -Wl,--unresolved-symbols=ignore-all) + if (USE_MUSL) + target_link_options(pybind11_stubs PRIVATE -nostartfiles) + endif() endif() add_library(ch_contrib::pybind11_stubs ALIAS pybind11_stubs) @@ -69,10 +76,19 @@ if(Python_FOUND) target_include_directories(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE ${PYBIND11_INCLUDE_DIR}) target_include_directories(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE ${Python_INCLUDE_DIRS}) target_link_libraries(${PYBIND11_NONLIMITEDAPI_LIBNAME} PUBLIC ch_contrib::pybind11_stubs) + + if (USE_MUSL AND CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + target_link_libraries(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -Wl,--whole-archive -lgcc -Wl,--no-whole-archive) + message(STATUS "Linking whole libgcc archive for ARM64+musl to provide all compiler runtime symbols") + endif() + if (APPLE) target_link_options(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -undefined dynamic_lookup) else() target_link_options(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -Wl,--unresolved-symbols=ignore-all) + if (USE_MUSL) + target_link_options(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -nostartfiles) + endif() endif() else() message(FATAL_ERROR "Python ${PY_VER} not found, aborting build") diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 94c747cc7c4..f3d869e10d2 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -71,7 +71,6 @@ # include #endif - namespace fs = std::filesystem; namespace CurrentMetrics diff --git a/programs/local/chdb.cpp b/programs/local/chdb.cpp index b9ac4b6437f..889b61c85ea 100644 --- a/programs/local/chdb.cpp +++ b/programs/local/chdb.cpp @@ -7,6 +7,17 @@ #include "QueryResult.h" #include "chdb-internal.h" +#if defined(USE_MUSL) && defined(__aarch64__) +void chdb_musl_compile_stub(int arg) +{ + jmp_buf buf1; + sigjmp_buf buf2; + + setjmp(buf1); + sigsetjmp(buf2, arg); +} +#endif + #if USE_PYTHON # include "FormatHelper.h" # include "PythonTableCache.h" diff --git a/programs/main.cpp b/programs/main.cpp index b2bc03a5928..67d270540d2 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -64,6 +64,17 @@ const char * __ubsan_default_options() #pragma clang diagnostic pop #endif +#if defined(USE_MUSL) && defined(__aarch64__) +void main_musl_compile_stub(int arg) +{ + jmp_buf buf1; + sigjmp_buf buf2; + + setjmp(buf1); + sigsetjmp(buf2, arg); +} +#endif + /// Universal executable for various clickhouse applications int mainEntryClickHouseBenchmark(int argc, char ** argv); int mainEntryClickHouseCheckMarks(int argc, char ** argv); diff --git a/src/Common/AllocationInterceptors.h b/src/Common/AllocationInterceptors.h index 55b364f2643..db16aa2bd46 100644 --- a/src/Common/AllocationInterceptors.h +++ b/src/Common/AllocationInterceptors.h @@ -8,7 +8,7 @@ // NOLINTBEGIN -#if defined(SANITIZER) || defined(SANITIZE_COVERAGE) || defined(OS_DARWIN) || defined(OS_FREEBSD) || !USE_JEMALLOC +#if defined(SANITIZER) || defined(SANITIZE_COVERAGE) || defined(OS_DARWIN) || defined(OS_FREEBSD) || (!USE_JEMALLOC && !defined(USE_MUSL)) #define __real_malloc(size) ::malloc(size) #define __real_calloc(nmemb, size) ::calloc(nmemb, size) diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index 3f59a8c48a1..a5b53d3a8a7 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -7,7 +7,6 @@ #include #include - namespace ProfileEvents { extern const Event TableFunctionExecute; diff --git a/tests/test_open_session_after_failure.py b/tests/test_open_session_after_failure.py index 4935585b668..2c26bd0eed8 100644 --- a/tests/test_open_session_after_failure.py +++ b/tests/test_open_session_after_failure.py @@ -2,6 +2,7 @@ import unittest import shutil +import os from chdb import session @@ -20,8 +21,9 @@ def tearDown(self) -> None: def test_path(self): # Test that creating session with invalid path (read-only directory) raises exception - with self.assertRaises(Exception): - sess = session.Session(test_dir2) + if not os.access(test_dir2, os.W_OK): + with self.assertRaises(Exception): + sess = session.Session(test_dir2) # Test that creating session with valid path works after failure sess = session.Session(test_dir1)