From 0470616600a5ff0ebac0ded039ede1d48531c0d8 Mon Sep 17 00:00:00 2001 From: Shashank Date: Tue, 3 Nov 2026 17:22:21 +0530 Subject: [PATCH] running with curio --- .config/forest.dic | 3 +- .github/workflows/lotus-api-bump.yml | 2 +- .github/workflows/miner-tests.yml | 46 ++++- docs/docs/users/guides/running_with_curio.md | 11 +- mise.toml | 4 + scripts/devnet-forest-curio/.env | 32 ++++ scripts/devnet-forest-curio/README.md | 17 ++ scripts/devnet-forest-curio/build_image.sh | 21 +++ scripts/devnet-forest-curio/check.sh | 39 ++++ scripts/devnet-forest-curio/curio.dockerfile | 58 ++++++ .../devnet-forest-curio/curio_entrypoint.sh | 36 ++++ scripts/devnet-forest-curio/devnet-layer.toml | 7 + .../devnet-forest-curio/docker-compose.yml | 149 +++++++++++++++ scripts/devnet-forest-curio/setup.sh | 48 +++++ scripts/devnet-forest-miner/README.md | 3 +- scripts/devnet/test_harness.sh | 16 +- src/dev/subcommands/devnet_cmd/miner.rs | 27 ++- src/dev/subcommands/devnet_cmd/miner/api.rs | 38 +++- src/dev/subcommands/devnet_cmd/miner/curio.rs | 173 ++++++++++++++++++ .../devnet_cmd/miner/onboarding.rs | 49 +++-- 20 files changed, 726 insertions(+), 53 deletions(-) create mode 100644 scripts/devnet-forest-curio/.env create mode 100644 scripts/devnet-forest-curio/README.md create mode 100755 scripts/devnet-forest-curio/build_image.sh create mode 100755 scripts/devnet-forest-curio/check.sh create mode 100644 scripts/devnet-forest-curio/curio.dockerfile create mode 100755 scripts/devnet-forest-curio/curio_entrypoint.sh create mode 100644 scripts/devnet-forest-curio/devnet-layer.toml create mode 100644 scripts/devnet-forest-curio/docker-compose.yml create mode 100755 scripts/devnet-forest-curio/setup.sh create mode 100644 src/dev/subcommands/devnet_cmd/miner/curio.rs diff --git a/.config/forest.dic b/.config/forest.dic index 47a8acc2cc4..6bc3fc6bd79 100644 --- a/.config/forest.dic +++ b/.config/forest.dic @@ -1,4 +1,4 @@ -303 +304 ABI Algorand/M API's @@ -64,6 +64,7 @@ coreos CPUs cron crypto +Curio daemon daemonize Datacap diff --git a/.github/workflows/lotus-api-bump.yml b/.github/workflows/lotus-api-bump.yml index 015510f53a6..f0c04817452 100644 --- a/.github/workflows/lotus-api-bump.yml +++ b/.github/workflows/lotus-api-bump.yml @@ -26,7 +26,7 @@ jobs: for env in ./scripts/tests/api_compare/.env ./scripts/tests/bootstrapper/.env ./scripts/tests/snapshot_parity/.env; do sed -i "s/\(LOTUS_IMAGE=filecoin.*\):.*/\1:$TAG-calibnet/" "$env" done - for env in ./scripts/devnet/.env ./scripts/devnet-forest-miner/.env; do + for env in ./scripts/devnet/.env ./scripts/devnet-forest-miner/.env ./scripts/devnet-forest-curio/.env; do sed -i "s/\(LOTUS_IMAGE=filecoin.*\):.*/\1:$TAG-2k/" "$env" done else diff --git a/.github/workflows/miner-tests.yml b/.github/workflows/miner-tests.yml index 3cbe924663f..437841896df 100644 --- a/.github/workflows/miner-tests.yml +++ b/.github/workflows/miner-tests.yml @@ -77,18 +77,20 @@ jobs: if-no-files-found: error miner-tests: - name: Miner integration tests (${{ matrix.group }}) + name: Miner integration tests (${{ matrix.devnet }}, ${{ matrix.group }}) needs: build runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: - # The `forest-dev devnet miner --group` values. Add groups as more Lotus itests are ported. + # Add groups as more Lotus itests are ported; both stacks run every group. group: [onboarding] + devnet: [devnet-forest-miner, devnet-forest-curio] env: # We use a custom Dockerfile for CI to speed up the build process. FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile MINER_TEST_GROUP: ${{ matrix.group }} + DEVNET_DIR: scripts/${{ matrix.devnet }} steps: - uses: actions/cache@v6 id: cache @@ -98,23 +100,51 @@ jobs: - name: Load cache into volume if: steps.cache.outputs.cache-hit == 'true' run: | - docker volume create devnet-forest-miner_filecoin-proofs + docker volume create "${{ matrix.devnet }}_filecoin-proofs" docker run --rm \ - -v devnet-forest-miner_filecoin-proofs:/proofs \ + -v "${{ matrix.devnet }}_filecoin-proofs":/proofs \ -v "$FIL_PROOFS_PARAMETER_CACHE":/cache \ "$SHELL_IMAGE" \ sh -c "cp /cache/* /proofs" - uses: actions/checkout@v7 with: persist-credentials: false + - name: Read Curio commit + if: matrix.devnet == 'devnet-forest-curio' + run: | + CURIO_COMMIT=$(grep -E '^CURIO_COMMIT=' scripts/devnet-forest-curio/.env | cut -d= -f2) + echo "CURIO_COMMIT=${CURIO_COMMIT}" >> "$GITHUB_ENV" + - name: Restore Curio image cache + if: matrix.devnet == 'devnet-forest-curio' + id: curio-cache + uses: actions/cache/restore@v6 + with: + path: /tmp/curio-image + key: curio-image-${{ env.CURIO_COMMIT }}-${{ hashFiles('scripts/devnet-forest-curio/curio.dockerfile') }} + - name: Load Curio image + if: matrix.devnet == 'devnet-forest-curio' && steps.curio-cache.outputs.cache-hit == 'true' + run: docker load -i /tmp/curio-image/curio.tar + - name: Build Curio image + if: matrix.devnet == 'devnet-forest-curio' && steps.curio-cache.outputs.cache-hit != 'true' + run: | + mkdir -p /tmp/curio-image + ./scripts/devnet-forest-curio/build_image.sh + docker save "forest/curio-devnet:${CURIO_COMMIT}" -o /tmp/curio-image/curio.tar + timeout-minutes: 45 + - name: Save Curio image cache + if: matrix.devnet == 'devnet-forest-curio' && steps.curio-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: /tmp/curio-image + key: curio-image-${{ env.CURIO_COMMIT }}-${{ hashFiles('scripts/devnet-forest-curio/curio.dockerfile') }} - uses: actions/download-artifact@v8 with: name: "forest-${{ runner.os }}" - name: Devnet setup - run: ./scripts/devnet-forest-miner/setup.sh + run: ./${{ env.DEVNET_DIR }}/setup.sh timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }} - name: Devnet check - run: ./scripts/devnet-forest-miner/check.sh + run: ./${{ env.DEVNET_DIR }}/check.sh timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }} - name: Add forest binaries to PATH run: | @@ -123,7 +153,7 @@ jobs: - name: Miner tests run: | set -euo pipefail - export DEVNET_DIR=scripts/devnet-forest-miner + export DEVNET_DIR="${{ env.DEVNET_DIR }}" source ./scripts/devnet/test_harness.sh devnet_test_env_init forest-dev devnet miner --group "$MINER_TEST_GROUP" @@ -135,7 +165,7 @@ jobs: if: steps.cache.outputs.cache-hit != 'true' run: | docker run --rm \ - -v devnet-forest-miner_filecoin-proofs:/proofs \ + -v "${{ matrix.devnet }}_filecoin-proofs":/proofs \ -v "$FIL_PROOFS_PARAMETER_CACHE":/cache \ "$SHELL_IMAGE" \ sh -c "cp /proofs/* /cache" diff --git a/docs/docs/users/guides/running_with_curio.md b/docs/docs/users/guides/running_with_curio.md index 2b54b4cac4c..9233a1725d4 100644 --- a/docs/docs/users/guides/running_with_curio.md +++ b/docs/docs/users/guides/running_with_curio.md @@ -3,8 +3,13 @@ title: Running Forest With Curio sidebar_position: 99 --- -:::note +[Curio](https://curiostorage.org/) is the successor to Lotus Miner. Forest can act as Curio's chain node (the same "Forest, Curio, YugabyteDB" layout Curio documents). -[Curio](https://curiostorage.org/) is the successor to Lotus-miner under active development by Curio Storage Inc. Compatibility with Forest is not yet confirmed. +Compatibility was checked against **Curio v1.29.0** on the local stack in [`scripts/devnet-forest-curio`](https://github.com/ChainSafe/forest/tree/main/scripts/devnet-forest-curio): -::: +- Put an admin token in `FULLNODE_API_INFO` and keep the miner worker key in Forest's wallet (`MinerCreateBlock` and message signing use it). +- On a private network, the genesis name must start with `local` or `test` (Curio drops other chain nodes). The stack uses `localnet`. + +```shell +mise run test:devnet:miner:curio +``` diff --git a/mise.toml b/mise.toml index 528841152d8..a6318d523c2 100644 --- a/mise.toml +++ b/mise.toml @@ -253,6 +253,10 @@ run = "scripts/devnet/with_devnet.sh scripts/devnet-forest-miner devnet_integrat description = "Run the Lotus Miner integration tests against the Forest-produced local docker devnet." run = "scripts/devnet/with_devnet.sh scripts/devnet-forest-miner forest-dev devnet miner" +[tasks."test:devnet:miner:curio"] +description = "Run the miner integration tests against the Forest-produced Curio docker devnet." +run = "scripts/devnet/with_devnet.sh scripts/devnet-forest-curio forest-dev devnet miner" + [tasks."codecov:nextest"] description = "Run codecov with nextest" run = ''' diff --git a/scripts/devnet-forest-curio/.env b/scripts/devnet-forest-curio/.env new file mode 100644 index 00000000000..22e806372fe --- /dev/null +++ b/scripts/devnet-forest-curio/.env @@ -0,0 +1,32 @@ +LOTUS_IMAGE=filecoin/lotus-all-in-one:v1.38.3-2k +# Curio rejects chain nodes whose network name does not start with local or test. +NETWORK_NAME=localnet +FOREST_DATA_DIR=/forest_data +LOTUS_DATA_DIR=/lotus_data +FIL_PROOFS_PARAMETER_CACHE=/var/tmp/filecoin-proof-parameters +MINER_ACTOR_ADDRESS=t01000 +LOTUS_RPC_PORT=1234 +LOTUS_P2P_PORT=1235 +# Unused here; compose interpolates the extended forest-miner file which references it. +MINER_RPC_PORT=2345 +FOREST_RPC_PORT=3456 +FOREST_P2P_PORT=4903 +# Curio v1.29.0 +CURIO_COMMIT=409a2ab3510d65f22ae31ba1afddc911f4b315d7 +CURIO_REPO_PATH=/var/lib/curio +YUGABYTE_IMAGE=yugabytedb/yugabyte:2025.2.2.2-b11@sha256:1fdda2ae928facf0b7b25b7c1183c540567c33730cce10a557c81c33a8e18422 +GENESIS_NETWORK_VERSION=18 +SHARK_HEIGHT=-10 +HYGGE_HEIGHT=-9 +LIGHTNING_HEIGHT=3 +THUNDER_HEIGHT=6 +WATERMELON_HEIGHT=9 +DRAGON_HEIGHT=12 +WAFFLE_HEIGHT=18 +TUKTUK_HEIGHT=20 +TEEP_HEIGHT=22 +TOCK_HEIGHT=24 +GOLDEN_WEEK_HEIGHT=26 +FIREHORSE_HEIGHT=28 +SOLSTICE_HEIGHT=30 +TARGET_HEIGHT=34 diff --git a/scripts/devnet-forest-curio/README.md b/scripts/devnet-forest-curio/README.md new file mode 100644 index 00000000000..fe6de869316 --- /dev/null +++ b/scripts/devnet-forest-curio/README.md @@ -0,0 +1,17 @@ +# Forest-produced devnet (Curio seals and mines) + +Same chain topology as [`../devnet-forest-miner`](../devnet-forest-miner), with Curio in place of Lotus Miner: + +| | this stack | +| --------------- | ---------------------- | +| Block producer | **Forest** + **Curio** | +| Validating node | Lotus | +| HarmonyDB | Yugabyte | + +`NETWORK_NAME=localnet` because Curio rejects a chain node whose name does not start with `local` or `test`. The Curio container gets the same `LOTUS_*` upgrade-height variables as Lotus; a `2k` Curio binary otherwise assumes a different genesis schedule. + +`setup.sh` builds `forest/curio-devnet:` once (about 15 minutes) and reuses it afterward. + +```shell +mise run test:devnet:miner:curio +``` diff --git a/scripts/devnet-forest-curio/build_image.sh b/scripts/devnet-forest-curio/build_image.sh new file mode 100755 index 00000000000..88f4f8cc511 --- /dev/null +++ b/scripts/devnet-forest-curio/build_image.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Build forest/curio-devnet:${CURIO_COMMIT} when that tag is missing. +set -euo pipefail + +PARENT_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=/dev/null +source "${PARENT_PATH}/.env" + +IMAGE="forest/curio-devnet:${CURIO_COMMIT}" +if docker image inspect "${IMAGE}" >/dev/null 2>&1; then + echo "Using existing ${IMAGE}" + exit 0 +fi + +echo "Building ${IMAGE} (linux/amd64) ..." +docker build \ + --platform linux/amd64 \ + --build-arg "CURIO_COMMIT=${CURIO_COMMIT}" \ + -f "${PARENT_PATH}/curio.dockerfile" \ + -t "${IMAGE}" \ + "${PARENT_PATH}" diff --git a/scripts/devnet-forest-curio/check.sh b/scripts/devnet-forest-curio/check.sh new file mode 100755 index 00000000000..eda584d66a2 --- /dev/null +++ b/scripts/devnet-forest-curio/check.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Assert Forest and Lotus both climb past TARGET_HEIGHT (Curio is mining through Forest). + +set -eux + +PARENT_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +pushd "${PARENT_PATH}" +# shellcheck source=/dev/null +source .env + +function get_sync_height { + local port=$1 + curl --silent -X POST -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","id":2,"method":"Filecoin.ChainHead","param":"null"}' \ + "http://127.0.0.1:${port}/rpc/v1" | jq '.result.Height' +} + +start_time=$(date +%s) +timeout=$((start_time + 900)) # 15 minutes for Yugabyte + Curio cold start +target_height=$TARGET_HEIGHT + +while true; do + forest_height=$(get_sync_height "${FOREST_RPC_PORT}") + lotus_height=$(get_sync_height "${LOTUS_RPC_PORT}") + if [ "$forest_height" -gt "$target_height" ] && [ "$lotus_height" -gt "$target_height" ]; then + echo "Both nodes past $target_height (Forest: $forest_height, Lotus: $lotus_height): Lotus validated Forest/Curio-produced blocks" + break + fi + + current_time=$(date +%s) + if [ "$current_time" -gt "$timeout" ]; then + echo "Timeout reached (Forest: $forest_height, Lotus: $lotus_height), target $target_height not reached by both" + exit 1 + fi + + sleep 1 +done + +popd diff --git a/scripts/devnet-forest-curio/curio.dockerfile b/scripts/devnet-forest-curio/curio.dockerfile new file mode 100644 index 00000000000..ca83587f7b5 --- /dev/null +++ b/scripts/devnet-forest-curio/curio.dockerfile @@ -0,0 +1,58 @@ +# Curio 2k image for the Forest-produced Curio devnet. Tags and GOAMD64 must be +# set explicitly: Curio's Dockerfile clears CURIO_TAGS, and its Makefile hardcodes +# GOAMD64=v3 which SIGILLs under QEMU. +ARG CURIO_COMMIT +FROM rust:1.86.0-slim-bookworm AS rust-toolchain + +FROM golang:1.26-trixie AS curio-builder +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ARG CURIO_COMMIT +RUN test -n "$CURIO_COMMIT" + +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates build-essential clang \ + ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev git && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=rust-toolchain /usr/local/cargo /usr/local/cargo +COPY --from=rust-toolchain /usr/local/rustup /usr/local/rustup +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH + +WORKDIR /opt/curio +RUN git clone https://github.com/filecoin-project/curio.git . && \ + git checkout "$CURIO_COMMIT" && \ + git submodule update --init + +# Portable BLST so the binary runs under QEMU; rewrite GOAMD64=v3 -> v1. +RUN sed -i 's/GOAMD64=v3/GOAMD64=v1/g' scripts/makefiles/30-build.mk + +ENV FFI_USE_OPENCL=1 \ + CURIO_TAGS="cunative nosupraseal 2k" \ + CGO_CFLAGS_ALLOW="-D__BLST_PORTABLE__" \ + CGO_CFLAGS="-D__BLST_PORTABLE__" \ + FFI_USE_BLST_PORTABLE=1 \ + FFI_USE_GPU=0 + +RUN make clean deps && make build && strip curio sptool + +FROM ubuntu:24.04 +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates curl jq dnsutils && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=curio-builder /etc/ssl/certs /etc/ssl/certs +COPY --from=curio-builder /lib/*/libdl.so.2 /lib/ +COPY --from=curio-builder /lib/*/librt.so.1 /lib/ +COPY --from=curio-builder /lib/*/libgcc_s.so.1 /lib/ +COPY --from=curio-builder /lib/*/libutil.so.1 /lib/ +COPY --from=curio-builder /usr/lib/*/libltdl.so.7 /lib/ +COPY --from=curio-builder /usr/lib/*/libnuma.so.1 /lib/ +COPY --from=curio-builder /usr/lib/*/libhwloc.so.* /lib/ +COPY --from=curio-builder /usr/lib/*/libOpenCL.so.1 /lib/ +COPY --from=curio-builder /opt/curio/curio /opt/curio/sptool /usr/local/bin/ + +RUN curio --version && ! command -v lotus-miner + +CMD ["/bin/bash"] diff --git a/scripts/devnet-forest-curio/curio_entrypoint.sh b/scripts/devnet-forest-curio/curio_entrypoint.sh new file mode 100755 index 00000000000..9ae917f7071 --- /dev/null +++ b/scripts/devnet-forest-curio/curio_entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Bootstrap Curio against Forest for genesis ${MINER_ACTOR_ADDRESS}, then run. +set -euo pipefail + +set +x +export FULLNODE_API_INFO="$(cat "${FOREST_DATA_DIR}/token.jwt"):/dns/forest/tcp/${FOREST_RPC_PORT}/http" +set -x + +MACHINE="$(hostname -i | awk '{print $1}'):12300" + +if [ ! -f "${CURIO_REPO_PATH}/.init.config" ]; then + curio config new-cluster "${MINER_ACTOR_ADDRESS}" + curio config create --title devnet < /devnet-layer.toml + touch "${CURIO_REPO_PATH}/.init.config" +fi + +curio run --nosync --name forest-devnet --layers seal,post,devnet & +CURIO_PID=$! + +until curio cli --machine "${MACHINE}" wait-api; do + if ! kill -0 "${CURIO_PID}" 2>/dev/null; then + echo "curio exited before its API came up" >&2 + wait "${CURIO_PID}" + exit 1 + fi + echo "waiting for curio API on ${MACHINE} ..." + sleep 2 +done + +if [ ! -f "${CURIO_REPO_PATH}/.init.storage" ]; then + curio cli --machine "${MACHINE}" storage attach "${LOTUS_DATA_DIR}/genesis-sectors" + curio cli --machine "${MACHINE}" storage attach --init --seal --store "${CURIO_REPO_PATH}/storage" + touch "${CURIO_REPO_PATH}/.init.storage" +fi + +wait "${CURIO_PID}" diff --git a/scripts/devnet-forest-curio/devnet-layer.toml b/scripts/devnet-forest-curio/devnet-layer.toml new file mode 100644 index 00000000000..28492f5a774 --- /dev/null +++ b/scripts/devnet-forest-curio/devnet-layer.toml @@ -0,0 +1,7 @@ +[Batching.PreCommit] +Timeout = "0h0m5s" +Slack = "6h0m0s" + +[Batching.Commit] +Timeout = "0h0m5s" +Slack = "1h0m0s" diff --git a/scripts/devnet-forest-curio/docker-compose.yml b/scripts/devnet-forest-curio/docker-compose.yml new file mode 100644 index 00000000000..191f03aa55b --- /dev/null +++ b/scripts/devnet-forest-curio/docker-compose.yml @@ -0,0 +1,149 @@ +# Forest-produced chain with Curio as the only block producer and sealer. +# Chain services are reused from ../devnet-forest-miner; there is no lotus-miner. + +x-lotus-heights: &lotus-heights + LOTUS_GENESIS_NETWORK_VERSION: ${GENESIS_NETWORK_VERSION} + LOTUS_SHARK_HEIGHT: ${SHARK_HEIGHT} + LOTUS_HYGGE_HEIGHT: ${HYGGE_HEIGHT} + LOTUS_LIGHTNING_HEIGHT: ${LIGHTNING_HEIGHT} + LOTUS_THUNDER_HEIGHT: ${THUNDER_HEIGHT} + LOTUS_WATERMELON_HEIGHT: ${WATERMELON_HEIGHT} + LOTUS_DRAGON_HEIGHT: ${DRAGON_HEIGHT} + LOTUS_WAFFLE_HEIGHT: ${WAFFLE_HEIGHT} + LOTUS_TUKTUK_HEIGHT: ${TUKTUK_HEIGHT} + LOTUS_TEEP_HEIGHT: ${TEEP_HEIGHT} + LOTUS_TOCK_HEIGHT: ${TOCK_HEIGHT} + LOTUS_GOLDENWEEK_HEIGHT: ${GOLDEN_WEEK_HEIGHT} + LOTUS_FIREHORSE_HEIGHT: ${FIREHORSE_HEIGHT} + LOTUS_SOLSTICE_HEIGHT: ${SOLSTICE_HEIGHT} + +x-forest-heights: &forest-heights + FOREST_GENESIS_NETWORK_VERSION: ${GENESIS_NETWORK_VERSION} + FOREST_SHARK_HEIGHT: ${SHARK_HEIGHT} + FOREST_HYGGE_HEIGHT: ${HYGGE_HEIGHT} + FOREST_LIGHTNING_HEIGHT: ${LIGHTNING_HEIGHT} + FOREST_THUNDER_HEIGHT: ${THUNDER_HEIGHT} + FOREST_WATERMELON_HEIGHT: ${WATERMELON_HEIGHT} + FOREST_DRAGON_HEIGHT: ${DRAGON_HEIGHT} + FOREST_WAFFLE_HEIGHT: ${WAFFLE_HEIGHT} + FOREST_TUKTUK_HEIGHT: ${TUKTUK_HEIGHT} + FOREST_TEEP_HEIGHT: ${TEEP_HEIGHT} + FOREST_TOCK_HEIGHT: ${TOCK_HEIGHT} + FOREST_GOLDEN_WEEK_HEIGHT: ${GOLDEN_WEEK_HEIGHT} + FOREST_FIREHORSE_HEIGHT: ${FIREHORSE_HEIGHT} + FOREST_SOLSTICE_HEIGHT: ${SOLSTICE_HEIGHT} + +services: + lotus_init: + extends: + file: ../devnet-forest-miner/docker-compose.yml + service: lotus_init + env_file: + - ../devnet-forest-miner/lotus.env + + lotus_node: + extends: + file: ../devnet-forest-miner/docker-compose.yml + service: lotus_node + env_file: + - ../devnet-forest-miner/lotus.env + + lotus_config: + extends: + file: ../devnet-forest-miner/docker-compose.yml + service: lotus_config + env_file: + - ../devnet-forest-miner/lotus.env + + forest: + extends: + file: ../devnet-forest-miner/docker-compose.yml + service: forest + volumes: + - lotus-data:${LOTUS_DATA_DIR} + - filecoin-proofs:${FIL_PROOFS_PARAMETER_CACHE} + - forest-data:${FOREST_DATA_DIR} + - ../devnet-forest-miner/forest_config.toml.tpl:/forest/forest_config.toml.tpl + + forest_wallet_import: + extends: + file: ../devnet-forest-miner/docker-compose.yml + service: forest_wallet_import + + yugabyte: + container_name: yugabyte + image: ${YUGABYTE_IMAGE} + init: true + environment: + TINI_SUBREAPER: "true" + volumes: + - yugabyte-data:/home/yugabyte/yb_data + command: + [ + "bin/yugabyted", + "start", + "--base_dir=/home/yugabyte/yb_data", + "--background=false", + ] + healthcheck: + test: + [ + "CMD", + "/home/yugabyte/bin/ysqlsh", + "-h", + "127.0.0.1", + "-U", + "yugabyte", + "-c", + "SELECT 1", + ] + interval: 5s + timeout: 5s + retries: 30 + start_period: 20s + networks: + - devnet + + curio: + container_name: curio + image: forest/curio-devnet:${CURIO_COMMIT} + pull_policy: never + init: true + restart: on-failure + depends_on: + forest_wallet_import: + condition: service_completed_successfully + yugabyte: + condition: service_healthy + environment: + <<: *lotus-heights + CURIO_REPO_PATH: ${CURIO_REPO_PATH} + CURIO_HARMONYDB_HOSTS: yugabyte + FIL_PROOFS_PARAMETER_CACHE: ${FIL_PROOFS_PARAMETER_CACHE} + FIL_PROOFS_USE_GPU_COLUMN_BUILDER: "0" + FIL_PROOFS_USE_GPU_TREE_BUILDER: "0" + BELLMAN_NO_GPU: "1" + FOREST_DATA_DIR: ${FOREST_DATA_DIR} + FOREST_RPC_PORT: ${FOREST_RPC_PORT} + LOTUS_DATA_DIR: ${LOTUS_DATA_DIR} + MINER_ACTOR_ADDRESS: ${MINER_ACTOR_ADDRESS} + volumes: + - curio-data:${CURIO_REPO_PATH} + - lotus-data:${LOTUS_DATA_DIR} + - forest-data:${FOREST_DATA_DIR}:ro + - filecoin-proofs:${FIL_PROOFS_PARAMETER_CACHE} + - ./curio_entrypoint.sh:/curio_entrypoint.sh:ro + - ./devnet-layer.toml:/devnet-layer.toml:ro + networks: + - devnet + entrypoint: ["/bin/bash", "/curio_entrypoint.sh"] + +volumes: + filecoin-proofs: + lotus-data: + forest-data: + curio-data: + yugabyte-data: + +networks: + devnet: diff --git a/scripts/devnet-forest-curio/setup.sh b/scripts/devnet-forest-curio/setup.sh new file mode 100755 index 00000000000..554ae2f91bc --- /dev/null +++ b/scripts/devnet-forest-curio/setup.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Build the Curio image if needed, bring up the stack, wait for Forest's RPC. + +set -euxo pipefail + +PARENT_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +pushd "${PARENT_PATH}" +# shellcheck source=/dev/null +source .env + +# Upgrade heights must stay in lockstep with the Lotus Miner stack. +miner_env="${PARENT_PATH}/../devnet-forest-miner/.env" +while IFS='=' read -r key value; do + case "${key}" in + GENESIS_NETWORK_VERSION|*_HEIGHT) + ours="${!key-}" + if [ "${ours}" != "${value}" ]; then + echo "ERROR: ${key}=${ours} differs from ${miner_env} (${value})" >&2 + exit 1 + fi + ;; + esac +done < <(grep -E '^(GENESIS_NETWORK_VERSION|[A-Z_]+_HEIGHT)=' "${miner_env}") + +./build_image.sh + +docker compose down --remove-orphans +docker compose rm -f +docker volume rm -f \ + devnet-forest-curio_lotus-data \ + devnet-forest-curio_forest-data \ + devnet-forest-curio_curio-data \ + devnet-forest-curio_yugabyte-data + +docker compose up --build --force-recreate --detach + +function call_forest_chain_head { + curl --silent -X POST -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","id":2,"method":"Filecoin.ChainHead","param":"null"}' \ + "http://127.0.0.1:${FOREST_RPC_PORT}/rpc/v1" +} + +until call_forest_chain_head; do + echo "Forest is unavailable - sleeping for 1s" + sleep 1 +done + +popd diff --git a/scripts/devnet-forest-miner/README.md b/scripts/devnet-forest-miner/README.md index 91f7c905e56..d2209298d7e 100644 --- a/scripts/devnet-forest-miner/README.md +++ b/scripts/devnet-forest-miner/README.md @@ -17,8 +17,9 @@ validates and follows. mise run test:devnet:forest-miner ``` -The Lotus Miner suites (`forest-dev devnet miner`) run only against this devnet: +The miner suites (`forest-dev devnet miner`) run against this Lotus Miner stack and against [`../devnet-forest-curio`](../devnet-forest-curio): ```shell mise run test:devnet:miner +mise run test:devnet:miner:curio ``` diff --git a/scripts/devnet/test_harness.sh b/scripts/devnet/test_harness.sh index d975741b819..78015882d0d 100644 --- a/scripts/devnet/test_harness.sh +++ b/scripts/devnet/test_harness.sh @@ -22,10 +22,14 @@ function devnet_test_env_init { local token token=$(docker exec forest cat "${FOREST_DATA_DIR}/token.jwt") export FULLNODE_API_INFO="${token}:/ip4/127.0.0.1/tcp/${FOREST_RPC_PORT}/http" - # The repo token carries all permissions (https://github.com/filecoin-project/lotus/blob/v1.38.3/node/modules/core.go#L165-L173). - local miner_token - miner_token=$(docker exec lotus-miner sh -c 'cat "$LOTUS_MINER_PATH/token"') - export MINER_API_INFO="${miner_token}:/ip4/127.0.0.1/tcp/${MINER_RPC_PORT}/http" + + # Lotus Miner only. Curio is driven by docker exec (CLI + Yugabyte), not MINER_API_INFO. + if [[ "${DEVNET_DIR}" != *devnet-forest-curio ]]; then + # The repo token carries all permissions (https://github.com/filecoin-project/lotus/blob/v1.38.3/node/modules/core.go#L165-L173). + local miner_token + miner_token=$(docker exec lotus-miner sh -c 'cat "$LOTUS_MINER_PATH/token"') + export MINER_API_INFO="${miner_token}:/ip4/127.0.0.1/tcp/${MINER_RPC_PORT}/http" + fi local tmp tmp="$(mktemp --directory)" @@ -47,7 +51,9 @@ function devnet_test_env_init { echo "Devnet test env initialised:" echo " FULLNODE_API_INFO=:/ip4/127.0.0.1/tcp/${FOREST_RPC_PORT}/http" - echo " MINER_API_INFO=:/ip4/127.0.0.1/tcp/${MINER_RPC_PORT}/http" + if [[ -n "${MINER_API_INFO:-}" ]]; then + echo " MINER_API_INFO=:/ip4/127.0.0.1/tcp/${MINER_RPC_PORT}/http" + fi echo " FOREST_TEST_PRELOADED_ADDRESS=${FOREST_TEST_PRELOADED_ADDRESS}" echo " Funding ${test_addr} with ${DEVNET_TEST_FUND_AMT} from ${genesis_addr}..." diff --git a/src/dev/subcommands/devnet_cmd/miner.rs b/src/dev/subcommands/devnet_cmd/miner.rs index e1e0718ab90..85343ca291f 100644 --- a/src/dev/subcommands/devnet_cmd/miner.rs +++ b/src/dev/subcommands/devnet_cmd/miner.rs @@ -1,10 +1,10 @@ // Copyright 2019-2026 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -//! Lotus Miner integration tests, ported from Lotus `itests`. They need -//! `scripts/devnet-forest-miner`, where Lotus Miner seals and mines through Forest's RPC. +//! Miner integration tests on Forest, against Lotus Miner or Curio. mod api; +mod curio; mod devnet; mod onboarding; @@ -17,7 +17,7 @@ use libtest_mimic::{Arguments, Trial}; use std::fmt::Debug; use std::path::Path; -/// Lotus Miner integration tests on Forest (`scripts/devnet-forest-miner` only) +/// Miner integration tests on Forest (`scripts/devnet-forest-miner` or `scripts/devnet-forest-curio`) #[derive(Debug, clap::Args)] pub struct MinerTestCommand { /// Run only this group; all groups run by default @@ -35,11 +35,7 @@ enum Group { impl MinerTestCommand { pub async fn run(self) -> anyhow::Result<()> { - let devnet_dir = harness_env("DEVNET_DIR")?; - ensure!( - Path::new(&devnet_dir).ends_with("devnet-forest-miner"), - "`devnet miner` drives lotus-miner over Forest's RPC, which only scripts/devnet-forest-miner does; DEVNET_DIR is `{devnet_dir}`" - ); + let _ = on_curio()?; let groups = match self.group { Some(group) => vec![group], None => Group::value_variants().to_vec(), @@ -59,6 +55,21 @@ fn tests(group: Group) -> Vec { } } +/// Whether `DEVNET_DIR` is the Curio stack (`devnet-forest-curio`); errors on any other directory. +pub(super) fn on_curio() -> anyhow::Result { + let devnet_dir = harness_env("DEVNET_DIR")?; + let path = Path::new(&devnet_dir); + if path.ends_with("devnet-forest-curio") { + Ok(true) + } else { + ensure!( + path.ends_with("devnet-forest-miner"), + "`devnet miner` needs scripts/devnet-forest-miner or scripts/devnet-forest-curio; DEVNET_DIR is `{devnet_dir}`" + ); + Ok(false) + } +} + /// Calls `request` on `node`, naming the node and the method in the error. async fn call( client: &Client, diff --git a/src/dev/subcommands/devnet_cmd/miner/api.rs b/src/dev/subcommands/devnet_cmd/miner/api.rs index 8094adf30ed..5d0f2980131 100644 --- a/src/dev/subcommands/devnet_cmd/miner/api.rs +++ b/src/dev/subcommands/devnet_cmd/miner/api.rs @@ -1,9 +1,10 @@ // Copyright 2019-2026 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -//! The miner the trials drive: a `lotus-miner` whose full node is Forest, reached through the -//! Lotus Miner JSON-RPC API, which Forest's RPC does not define. +//! Genesis miner driver: Lotus Miner JSON-RPC or Curio CLI/SQL, selected by `DEVNET_DIR`. +use super::curio; +use super::on_curio; use crate::dev::subcommands::tests_cmd::helpers::{harness_env, poll}; use crate::rpc::{ApiPaths, Client, Request}; use crate::shim::address::Address; @@ -16,10 +17,15 @@ use serde_json::json; use std::str::FromStr as _; use std::sync::atomic::{AtomicU8, Ordering}; -/// The miner's on-chain actor and its API. +pub(super) enum Producer { + Lotus(MinerApi), + Curio, +} + +/// The miner's on-chain actor and its producer API. pub(super) struct TestMiner { pub actor: Address, - pub api: MinerApi, + pub producer: Producer, } impl TestMiner { @@ -28,10 +34,26 @@ impl TestMiner { let raw = harness_env("MINER_ACTOR_ADDRESS")?; let actor = Address::from_str(&raw) .with_context(|| format!("invalid MINER_ACTOR_ADDRESS `{raw}`"))?; - Ok(Self { - actor, - api: MinerApi::connect()?, - }) + let producer = if on_curio()? { + Producer::Curio + } else { + Producer::Lotus(MinerApi::connect()?) + }; + Ok(Self { actor, producer }) + } + + pub async fn pledge_sector(&self) -> anyhow::Result { + match &self.producer { + Producer::Lotus(api) => api.pledge_sector().await, + Producer::Curio => curio::pledge_sector(self.actor).await, + } + } + + pub async fn wait_until_proving(&self, sector: u64) -> anyhow::Result<()> { + match &self.producer { + Producer::Lotus(api) => api.wait_until_proving(sector).await, + Producer::Curio => curio::wait_until_proving(self.actor, sector).await, + } } } diff --git a/src/dev/subcommands/devnet_cmd/miner/curio.rs b/src/dev/subcommands/devnet_cmd/miner/curio.rs new file mode 100644 index 00000000000..f06b8bdb88d --- /dev/null +++ b/src/dev/subcommands/devnet_cmd/miner/curio.rs @@ -0,0 +1,173 @@ +// Copyright 2019-2026 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +//! Curio seal/status/create-miner helpers for the Forest-produced Curio stack. + +use crate::dev::subcommands::tests_cmd::helpers::{docker, harness_env, poll}; +use crate::shim::address::Address; +use anyhow::{Context as _, bail}; +use serde::Deserialize; +use serde_json::json; + +/// Starts one CC sector via `curio seal start` and returns its number. +pub(super) async fn pledge_sector(actor: Address) -> anyhow::Result { + let out = docker(&[ + "exec", + "curio", + "curio", + "seal", + "start", + "--now", + "--cc", + "--count", + "1", + "--actor", + &actor.to_string(), + ]) + .context("`curio seal start` failed")?; + let number = out + .lines() + .rev() + .find_map(|l| l.trim().parse::().ok()) + .with_context(|| format!("no sector number in `curio seal start` output:\n{out}"))?; + Ok(number) +} + +/// Polls Harmony DB until the sector is committed. +pub(super) async fn wait_until_proving(actor: Address, sector: u64) -> anyhow::Result<()> { + let sp_id = actor.id().context("miner actor is not an ID address")?; + let waited = poll(&format!("Curio sector {sector} to finish sealing"), || async { + let status = sector_status(sp_id, sector)?; + match status { + SectorStatus::Done => Ok(Some(())), + SectorStatus::Failed { + reason, + reason_msg, + } => { + let events = seal_events(actor, sector).unwrap_or_default(); + bail!( + "Curio sector {sector} failed: {reason}: {reason_msg}\nseal events:\n{events}" + ); + } + SectorStatus::Unknown => bail!("sector {sector} unknown to Curio"), + SectorStatus::InProgress => Ok(None), + } + }) + .await; + if let Err(e) = waited { + let events = seal_events(actor, sector).unwrap_or_default(); + return Err(e.context(format!("seal events:\n{events}"))); + } + Ok(()) +} + +/// Creates a miner with `sptool` against Forest (empty peer id). +pub(super) fn create_miner(owner: Address, worker: Address) -> anyhow::Result<()> { + let port = harness_env("FOREST_RPC_PORT")?; + let data_dir = harness_env("FOREST_DATA_DIR")?; + let script = format!( + "export FULLNODE_API_INFO=\"$(cat '{data_dir}/token.jwt'):/dns/forest/tcp/{port}/http\" && \ + exec sptool --actor t01000 actor new-miner \ + --owner='{owner}' --worker='{worker}' --from='{owner}' \ + --sector-size 2KiB --confidence 1" + ); + docker(&["exec", "curio", "sh", "-c", &script]).context("`sptool actor new-miner` failed")?; + Ok(()) +} + +#[derive(Debug)] +enum SectorStatus { + Done, + Failed { reason: String, reason_msg: String }, + InProgress, + Unknown, +} + +#[derive(Debug, Deserialize)] +struct StatusRow { + after_commit_msg_success: bool, + failed: bool, + failed_reason: String, + failed_reason_msg: String, + in_meta: bool, + in_pipeline: bool, +} + +fn sector_status(sp_id: u64, sector: u64) -> anyhow::Result { + let sql = format!( + "SELECT json_build_object(\ + 'after_commit_msg_success', COALESCE(p.after_commit_msg_success IS NOT NULL, false),\ + 'failed', COALESCE(p.failed, false),\ + 'failed_reason', COALESCE(p.failed_reason, ''),\ + 'failed_reason_msg', COALESCE(p.failed_reason_msg, ''),\ + 'in_meta', EXISTS (SELECT 1 FROM curio.sectors_meta m \ + WHERE m.sp_id = {sp_id} AND m.sector_num = {sector}),\ + 'in_pipeline', p.sp_id IS NOT NULL\ + ) FROM (SELECT {sp_id}::bigint AS sp_id, {sector}::bigint AS sector_number) s \ + LEFT JOIN curio.sectors_sdr_pipeline p \ + ON p.sp_id = s.sp_id AND p.sector_number = s.sector_number;" + ); + let out = docker(&[ + "exec", + "yugabyte", + "bin/ysqlsh", + "-h", + "yugabyte", + "-U", + "yugabyte", + "-At", + "-c", + &sql, + ]) + .context("querying Curio sector status")?; + let row: StatusRow = serde_json::from_str(out.trim()) + .with_context(|| format!("decoding Curio status JSON: {out}"))?; + if row.failed { + return Ok(SectorStatus::Failed { + reason: row.failed_reason, + reason_msg: row.failed_reason_msg, + }); + } + if row.after_commit_msg_success || (!row.in_pipeline && row.in_meta) { + return Ok(SectorStatus::Done); + } + if !row.in_pipeline && !row.in_meta { + return Ok(SectorStatus::Unknown); + } + Ok(SectorStatus::InProgress) +} + +fn seal_events(actor: Address, sector: u64) -> anyhow::Result { + docker(&[ + "exec", + "curio", + "curio", + "seal", + "events", + "--actor", + &actor.to_string(), + "--sector", + §or.to_string(), + ]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_row_decodes() { + let row: StatusRow = serde_json::from_value(json!({ + "after_commit_msg_success": true, + "failed": false, + "failed_reason": "", + "failed_reason_msg": "", + "in_meta": true, + "in_pipeline": true, + })) + .unwrap(); + assert!(row.after_commit_msg_success); + assert!(row.in_meta); + assert!(!row.failed); + } +} diff --git a/src/dev/subcommands/devnet_cmd/miner/onboarding.rs b/src/dev/subcommands/devnet_cmd/miner/onboarding.rs index 434934eccc5..9be99f506bc 100644 --- a/src/dev/subcommands/devnet_cmd/miner/onboarding.rs +++ b/src/dev/subcommands/devnet_cmd/miner/onboarding.rs @@ -1,12 +1,12 @@ // Copyright 2019-2026 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -//! Lotus Miner creating a miner and pledging a sector through Forest's RPC. Mirrors the creation -//! step of Lotus `testNonGenesisMiner` (which also starts the new miner) and `TestPledgeSectors` -//! (single sector, real 2 KiB proofs). +//! Creating a miner and pledging a sector through Forest's RPC (Lotus Miner or Curio). use super::api::TestMiner; +use super::curio; use super::devnet::Devnet; +use super::on_curio; use crate::dev::subcommands::tests_cmd::helpers::*; use crate::key_management::generate_key; use crate::rpc::Client; @@ -46,13 +46,18 @@ async fn create_miner() -> anyhow::Result<()> { .await?; let before = list_miners(&net.forest).await?; - let repo = format!( - "/tmp/forest-onboarding-{}", - SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis() - ); - let init = init_miner(&repo, owner_key, worker_key); - let _ = docker(&["exec", "lotus-miner", "rm", "-rf", &repo]); - init?; + let curio = on_curio()?; + if curio { + curio::create_miner(owner_key, worker_key)?; + } else { + let repo = format!( + "/tmp/forest-onboarding-{}", + SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis() + ); + let init = init_lotus_miner(&repo, owner_key, worker_key); + let _ = docker(&["exec", "lotus-miner", "rm", "-rf", &repo]); + init?; + } let created: Vec
= list_miners(&net.forest) .await? @@ -60,7 +65,7 @@ async fn create_miner() -> anyhow::Result<()> { .copied() .collect(); let [miner] = created.as_slice() else { - bail!("expected exactly one new miner after `lotus-miner init`, found {created:?}"); + bail!("expected exactly one new miner, found {created:?}"); }; let worker_id = net @@ -83,10 +88,18 @@ async fn create_miner() -> anyhow::Result<()> { "miner {miner}: sector size {:?}, expected 2 KiB", info.sector_size ); - ensure!( - !info.peer_id.is_empty(), - "miner {miner}: no peer ID recorded" - ); + if curio { + ensure!( + info.peer_id.is_empty(), + "miner {miner}: Curio should leave peer_id empty, got {}", + info.peer_id + ); + } else { + ensure!( + !info.peer_id.is_empty(), + "miner {miner}: no peer ID recorded" + ); + } Ok(()) } @@ -94,8 +107,8 @@ async fn create_miner() -> anyhow::Result<()> { async fn pledge_sector() -> anyhow::Result<()> { let net = Devnet::connect()?; let miner = TestMiner::genesis()?; - let sector = miner.api.pledge_sector().await?; - miner.api.wait_until_proving(sector).await?; + let sector = miner.pledge_sector().await?; + miner.wait_until_proving(sector).await?; net.synced() .await? .same(|tsk| StateSectorGetInfo::request((miner.actor, sector, tsk))) @@ -112,7 +125,7 @@ async fn list_miners(client: &Client) -> anyhow::Result> { .collect()) } -fn init_miner(repo: &str, owner: Address, worker: Address) -> anyhow::Result<()> { +fn init_lotus_miner(repo: &str, owner: Address, worker: Address) -> anyhow::Result<()> { let port = harness_env("FOREST_RPC_PORT")?; let data_dir = harness_env("FOREST_DATA_DIR")?; let script = format!(