feat(algo): add GDS_PAGE_RANK backed by icebug (NetworKit, zero-copy Arrow) #68
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| permissions: | |
| actions: read | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # pg_client requires libpq (PostgreSQL) at build time and a running | |
| # PostgreSQL instance at test time (provided by run_pgembed_fixture.py). | |
| EXTENSION_LIST: adbc;azure;delta;duckdb;fts;httpfs;iceberg;json;llm;neo4j;pg_client;postgres;sqlite;unity_catalog;vector;algo | |
| jobs: | |
| # ───────────────────────────────────────────────────────────────── | |
| # Resolve the latest successful ladybug build-and-deploy run and | |
| # download prebuilt liblbug + CLI artifacts. Runs on the host | |
| # runner (no container) so gh CLI is available natively. | |
| # ───────────────────────────────────────────────────────────────── | |
| resolve-run: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: write | |
| contents: read | |
| outputs: | |
| run-id: ${{ steps.resolve.outputs.run-id }} | |
| steps: | |
| - name: Resolve latest successful build-and-deploy run | |
| id: resolve | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| RUN_ID="$( | |
| curl -fsSL \ | |
| -H "Authorization: Bearer $GITHUB_TOKEN" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| "https://api.github.com/repos/LadybugDB/ladybug/actions/workflows/build-and-deploy.yml/runs?branch=main&status=success&per_page=1" \ | |
| | python3 -c 'import json,sys; data=json.load(sys.stdin); runs=data.get("workflow_runs") or []; print(runs[0]["id"] if runs else "")' | |
| )" | |
| if [ -z "$RUN_ID" ]; then | |
| echo "Could not find a successful LadybugDB/ladybug build-and-deploy run." >&2 | |
| exit 1 | |
| fi | |
| echo "run-id=$RUN_ID" >> "$GITHUB_OUTPUT" | |
| - name: Download prebuilt liblbug and CLI | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| mkdir -p prebuilt | |
| # 1) Shared library + headers | |
| gh run download "${{ steps.resolve.outputs.run-id }}" \ | |
| --repo LadybugDB/ladybug \ | |
| --name "liblbug-linux-x86_64" \ | |
| --dir prebuilt | |
| # 2) CLI binary | |
| gh run download "${{ steps.resolve.outputs.run-id }}" \ | |
| --repo LadybugDB/ladybug \ | |
| --name "lbug_cli-linux-x86_64" \ | |
| --dir prebuilt | |
| ls -la prebuilt/ | |
| - name: Upload prebuilt artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: lbug-prebuilts | |
| path: prebuilt/* | |
| retention-days: 1 | |
| # ───────────────────────────────────────────────────────────────── | |
| # Build all extensions inside the manylinux container (glibc compat) | |
| # and run the extension integration tests. | |
| # ───────────────────────────────────────────────────────────────── | |
| build-and-test: | |
| needs: resolve-run | |
| strategy: | |
| matrix: | |
| include: | |
| # ── Linux ──────────────────────────────────────────────── | |
| - os: ubuntu-latest | |
| os_name: linux | |
| arch: x86_64 | |
| container: quay.io/pypa/manylinux_2_28_x86_64 | |
| # - os: ubuntu-24.04-arm | |
| # os_name: linux | |
| # arch: aarch64 | |
| # container: quay.io/pypa/manylinux_2_28_aarch64 | |
| # ── macOS ──────────────────────────────────────────────── | |
| # - os: macos-latest | |
| # os_name: osx | |
| # arch: arm64 | |
| # - os: macos-15-intel | |
| # os_name: osx | |
| # arch: x86_64 | |
| # ── Windows ────────────────────────────────────────────── | |
| # - os: windows-2022 | |
| # os_name: win | |
| # arch: x86_64 | |
| runs-on: ${{ matrix.os }} | |
| container: ${{ matrix.container }} | |
| steps: | |
| # ── Source checkouts ────────────────────────────────────────── | |
| - name: Checkout ladybug | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: LadybugDB/ladybug | |
| fetch-depth: 1 | |
| path: ladybug | |
| # The following three checkouts place the sub-repos directly into | |
| # the ladybug tree at the same paths the in-tree submodules would | |
| # occupy — no symlinks, no path gymnastics. | |
| - name: Checkout extension into ladybug/extension | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| path: ladybug/extension | |
| - name: Checkout dataset into ladybug/dataset | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: LadybugDB/dataset | |
| fetch-depth: 1 | |
| path: ladybug/dataset | |
| - name: Checkout benchmark into ladybug/benchmark | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: LadybugDB/benchmarks | |
| fetch-depth: 1 | |
| path: ladybug/benchmark | |
| - name: Mark workspace safe for git (Linux container) | |
| if: runner.os == 'Linux' | |
| run: git config --global --add safe.directory '*' | |
| # ── Compute platform variables ─────────────────────────── | |
| # Derive DuckDB download params and ladybug artifact arch from | |
| # runner.os / runner.arch instead of cluttering the matrix. | |
| - name: Compute platform variables | |
| run: | | |
| # --- DuckDB archive --- | |
| case "${{ runner.os }}" in | |
| Linux) | |
| echo "DUCKDB_LIB=libduckdb.so" >> "$GITHUB_ENV" | |
| case "${{ runner.arch }}" in | |
| X64) echo "DUCKDB_ARCH=linux-amd64" >> "$GITHUB_ENV" ;; | |
| ARM64) echo "DUCKDB_ARCH=linux-arm64" >> "$GITHUB_ENV" ;; | |
| esac | |
| ;; | |
| macOS) | |
| echo "DUCKDB_LIB=libduckdb.dylib" >> "$GITHUB_ENV" | |
| echo "DUCKDB_ARCH=osx-universal" >> "$GITHUB_ENV" | |
| ;; | |
| Windows) | |
| echo "DUCKDB_LIB=duckdb.dll" >> "$GITHUB_ENV" | |
| case "${{ runner.arch }}" in | |
| X64|AMD64) echo "DUCKDB_ARCH=windows-amd64" >> "$GITHUB_ENV" ;; | |
| ARM64) echo "DUCKDB_ARCH=windows-arm64" >> "$GITHUB_ENV" ;; | |
| esac | |
| ;; | |
| esac | |
| # --- Ladybug artifact arch (maps runner arch to release names) --- | |
| case "${{ runner.arch }}" in | |
| X64|AMD64) echo "LBUG_ARTIFACT_ARCH=x86_64" >> "$GITHUB_ENV" ;; | |
| ARM64) echo "LBUG_ARTIFACT_ARCH=aarch64" >> "$GITHUB_ENV" ;; | |
| esac | |
| # --- Upstream run ID for prebuilt artifacts — echoed by resolve job --- | |
| echo "LBUG_PRECOMPILED_RUN_ID=${{ needs.resolve-run.outputs.run-id }}" >> "$GITHUB_ENV" | |
| - name: Setup ccache | |
| uses: hendrikmuhs/ccache-action@v1.2 | |
| with: | |
| key: extension-ci-${{ runner.os }}-${{ runner.arch }}-${{ github.ref }} | |
| max-size: 2G | |
| create-symlink: true | |
| restore-keys: | | |
| extension-ci-${{ runner.os }}-${{ runner.arch }}-refs/heads/main | |
| extension-ci-${{ runner.os }}-${{ runner.arch }}- | |
| # ── Build dependencies ──────────────────────────────────────── | |
| - name: Install build dependencies (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| dnf install -y 'dnf-command(config-manager)' | |
| dnf config-manager --set-enabled powertools | |
| dnf install -y cmake ninja-build python3 python3-pip git wget unzip \ | |
| gcc-toolset-13 openssl3 openssl3-devel pkg-config \ | |
| libpq libpq-devel libomp-devel | |
| - name: Install uv | |
| working-directory: ladybug | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| uv venv | |
| - name: Install test Python dependencies | |
| working-directory: ladybug | |
| run: | | |
| uv pip install duckdb rangehttpserver requests | |
| # ── Setup pixi + ADBC runtime ─────────────────────────── | |
| # The ADBC extension requires libadbc-driver-manager and libarrow, | |
| # which the ladybug repo's pixi.toml pins to conda-forge. These | |
| # are needed both to *build* the ADBC extension and to run its | |
| # tests. | |
| - name: Setup pixi | |
| if: runner.os != 'Windows' | |
| uses: prefix-dev/setup-pixi@v0.9.4 | |
| with: | |
| pixi-version: v0.68.1 | |
| working-directory: ladybug | |
| cache: false | |
| activate-environment: true | |
| - name: Configure ADBC runtime | |
| if: runner.os != 'Windows' | |
| run: | | |
| curl -LsSf https://dbc.columnar.tech/install.sh | sh | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| echo "LD_LIBRARY_PATH=${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" >> "$GITHUB_ENV" | |
| "$HOME/.local/bin/dbc" install --level user duckdb || true | |
| # ── Install DuckDB ──────────────────────────────────────────── | |
| - name: Install DuckDB (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| wget "https://github.com/duckdb/duckdb/releases/latest/download/libduckdb-${{ env.DUCKDB_ARCH }}.zip" | |
| unzip "libduckdb-${{ env.DUCKDB_ARCH }}.zip" -d duckdb | |
| cp duckdb/duckdb.h /usr/local/include/ | |
| cp duckdb/duckdb.hpp /usr/local/include/ | |
| cp duckdb/${{ env.DUCKDB_LIB }} /usr/local/lib/ | |
| ldconfig | |
| # CMake config so find_package(DuckDB) works | |
| mkdir -p /usr/local/lib/cmake/DuckDB | |
| tee /usr/local/lib/cmake/DuckDB/DuckDBConfig.cmake > /dev/null << 'CMEOF' | |
| if(NOT TARGET DuckDB::duckdb) | |
| add_library(DuckDB::duckdb SHARED IMPORTED) | |
| set_target_properties(DuckDB::duckdb PROPERTIES | |
| INTERFACE_INCLUDE_DIRECTORIES "/usr/local/include" | |
| IMPORTED_LOCATION "/usr/local/lib/${{ env.DUCKDB_LIB }}" | |
| ) | |
| endif() | |
| set(DuckDB_LIBRARIES DuckDB::duckdb) | |
| set(DuckDB_INCLUDE_DIRS "/usr/local/include") | |
| set(DuckDB_FOUND TRUE) | |
| CMEOF | |
| # ── Download prebuilt liblbug + CLI from the resolve-run job ── | |
| # The resolve-run job (runs on host runner, has gh CLI) downloaded | |
| # and archived these; we extract them into the ladybug tree. | |
| - name: Download prebuilt artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: lbug-prebuilts | |
| path: prebuilt-tmp | |
| - name: Extract prebuilt liblbug and CLI | |
| run: | | |
| cd prebuilt-tmp | |
| mkdir -p "${{ github.workspace }}/ladybug/src/include" | |
| mkdir -p "${{ github.workspace }}/ladybug/tools/shell" | |
| # Shared library + headers | |
| tar xzf "liblbug-linux-${{ env.LBUG_ARTIFACT_ARCH }}.tar.gz" -C "${{ github.workspace }}/ladybug/src/include/" | |
| # CLI binary | |
| tar xzf "lbug_cli-linux-${{ env.LBUG_ARTIFACT_ARCH }}.tar.gz" -C "${{ github.workspace }}/ladybug/tools/shell/" | |
| chmod +x "${{ github.workspace }}/ladybug/tools/shell/lbug" | |
| # ── Test fixtures (before the build, like minimal-linux-extension-test) ── | |
| - name: Create ADBC DuckDB fixture | |
| working-directory: ladybug | |
| run: | | |
| mkdir -p extension/adbc/test | |
| uv run python - <<'PY' | |
| import duckdb | |
| con = duckdb.connect("extension/adbc/test/adbc.duckdb") | |
| con.execute("CREATE OR REPLACE TABLE games(id BIGINT, title VARCHAR, score BIGINT)") | |
| con.execute(""" | |
| INSERT INTO games VALUES | |
| (1, 'Portal', 95), | |
| (2, 'Celeste', 94), | |
| (3, 'Hades', 93) | |
| """) | |
| con.close() | |
| PY | |
| # ── Build extensions, test infrastructure, and extension tests ── | |
| # Single release build with ccache — replaces what was previously 3 | |
| # separate cmake invocations (extension-release + relwithdebinfo + | |
| # extension-test-build). | |
| - name: Build extensions and tests | |
| working-directory: ladybug | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| source /opt/rh/gcc-toolset-13/enable | |
| export CC=gcc | |
| export CXX=g++ | |
| export EXTRA_CMAKE_FLAGS="\ | |
| -DOPENSSL_ROOT_DIR=/usr/lib64/openssl3 \ | |
| -DOPENSSL_INCLUDE_DIR=/usr/include/openssl3 \ | |
| -DOPENSSL_CRYPTO_LIBRARY=/usr/lib64/openssl3/libcrypto.so \ | |
| -DOPENSSL_SSL_LIBRARY=/usr/lib64/openssl3/libssl.so" | |
| cmake -B build/release \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DBUILD_EXTENSIONS="${EXTENSION_LIST}" \ | |
| -DBUILD_EXTENSION_TESTS=TRUE \ | |
| -DBUILD_TESTS=TRUE \ | |
| -DBUILD_SHELL=TRUE \ | |
| -DENABLE_BACKTRACES=TRUE \ | |
| ${EXTRA_CMAKE_FLAGS} . | |
| cmake --build build/release -j "$(nproc)" | |
| cp build/release/tools/shell/lbug lbug.prod | |
| # ── Test fixtures ───────────────────────────────────────────── | |
| - name: Generate test dataset | |
| working-directory: ladybug | |
| run: uv run python3 scripts/generate-tinysnb.py release | |
| - name: Start extension repo server (for remote-load tests) | |
| working-directory: ladybug | |
| run: | | |
| uv run python3 scripts/setup-extension-repo.py & | |
| sleep 3 | |
| # ── Run tests ───────────────────────────────────────────────── | |
| # Runs both the e2e test runner (via pgembed Postgres fixture) and | |
| # extension unit tests — matching what `make extension-test` does. | |
| - name: Run extension tests | |
| working-directory: ladybug | |
| env: | |
| E2E_TEST_FILES_DIRECTORY: extension | |
| run: | | |
| source /opt/rh/gcc-toolset-13/enable | |
| cp lbug.prod build/release/tools/shell/lbug | |
| uv run --python 3.12 python3 scripts/run_pgembed_fixture.py -- \ | |
| ctest --test-dir build/release/test/runner --output-on-failure -j 1 --exclude-regex "" && \ | |
| ctest --test-dir build/release/extension --output-on-failure -j "$(nproc)" --exclude-regex "" | |
| # ── Collect & upload artifacts ──────────────────────────────── | |
| - name: Collect built artifacts | |
| working-directory: ladybug | |
| run: python3 scripts/collect-extensions.py | |
| - name: Upload built artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: lbug-extensions-${{ matrix.os_name }}-${{ matrix.arch }} | |
| path: ladybug/extension-artifacts |