From 9b527137cd5af1fabacb374fe13415c9e11ed0c0 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 26 Oct 2026 07:00:42 -0400 Subject: [PATCH 01/14] feat: add `--target` option to `forest-cli sync wait` --- scripts/devnet-forest-miner/check.sh | 18 +++++++++++----- scripts/devnet/check.sh | 31 +++++++++++++--------------- src/cli/subcommands/sync_cmd.rs | 22 +++++++++++++++++--- 3 files changed, 46 insertions(+), 25 deletions(-) diff --git a/scripts/devnet-forest-miner/check.sh b/scripts/devnet-forest-miner/check.sh index 20942dbbfea..a8864ab9356 100755 --- a/scripts/devnet-forest-miner/check.sh +++ b/scripts/devnet-forest-miner/check.sh @@ -18,22 +18,30 @@ function get_sync_height { } start_time=$(date +%s) -timeout=$((start_time + 360)) # 6 minutes +timeout_secs=360 +timeout=$((start_time + timeout_secs)) # 6 minutes # Target height chosen so that every migration in .env is crossed. target_height=$TARGET_HEIGHT +# forest waits for $target_height +docker exec \ + -e FULLNODE_API_INFO="/ip4/127.0.0.1/tcp/${FOREST_RPC_PORT}/http" \ + forest \ + timeout "${timeout_secs}s" \ + forest-cli sync wait --target "$target_height" + +# lotus waits for $target_height while true; do - forest_height=$(get_sync_height "${FOREST_RPC_PORT}") lotus_height=$(get_sync_height "${LOTUS_RPC_PORT}") - if [ "$forest_height" -gt "$target_height" ] && [ "$lotus_height" -gt "$target_height" ]; then - echo "Both nodes past $target_height (Forest: $forest_height, Lotus: $lotus_height): Lotus validated Forest-produced blocks" + if [ "$lotus_height" -gt "$target_height" ]; then + echo "Lotus node past $target_height ($lotus_height): Lotus validated Forest-produced blocks" break fi current_time=$(date +%s) if [ "$current_time" -gt "$timeout" ]; then - echo "Timeout reached (Forest: $forest_height, Lotus: $lotus_height), target $target_height not reached by both" + echo "Timeout reached (Lotus: $lotus_height), target $target_height not reached by Lotus node" exit 1 fi diff --git a/scripts/devnet/check.sh b/scripts/devnet/check.sh index b7682133807..e8734d1966f 100755 --- a/scripts/devnet/check.sh +++ b/scripts/devnet/check.sh @@ -29,23 +29,20 @@ start_time=$(date +%s) timeout=$((start_time + 360)) # Set timeout to 6 minutes # Target height set so that all migrations are applied. -target_height=$TARGET_HEIGHT - -while true; do - height=$(get_sync_height ${FOREST_RPC_PORT}) - if [ "$height" -gt "$target_height" ]; then - echo "Height is larger than $target_height: $height" - break - fi - - current_time=$(date +%s) - if [ "$current_time" -gt "$timeout" ]; then - echo "Timeout reached, height is still not larger than $target_height: $height" - exit 1 - fi - - sleep 1 -done +target_height=$((TARGET_HEIGHT + 1)) + +# waits node to reach $target_height +docker exec \ + -e FULLNODE_API_INFO="/ip4/127.0.0.1/tcp/${FOREST_RPC_PORT}/http" \ + forest \ + timeout 20m \ + forest-cli sync wait --target "$target_height" + +height=$(get_sync_height "${FOREST_RPC_PORT}") +if [ "$height" -le "$TARGET_HEIGHT" ]; then + echo "sync wait --target exited early: head is $height, target is $target_height" + exit 1 +fi # Check the offline RPC, which should be initialized at that point. It should be at the genesis height, so 0. height=$(get_sync_height ${FOREST_OFFLINE_RPC_PORT}) diff --git a/src/cli/subcommands/sync_cmd.rs b/src/cli/subcommands/sync_cmd.rs index 6ac0fc5ca3a..2814daa692e 100644 --- a/src/cli/subcommands/sync_cmd.rs +++ b/src/cli/subcommands/sync_cmd.rs @@ -5,6 +5,7 @@ use crate::blocks::TipsetKey; use crate::chain_sync::{NodeSyncStatus, SyncStatusReport}; use crate::rpc::sync::{SnapshotProgressState, SyncStatus}; use crate::rpc::{self, prelude::*}; +use crate::shim::clock::ChainEpoch; use anyhow::Context; use cid::Cid; use clap::Subcommand; @@ -23,6 +24,10 @@ pub enum SyncCommands { /// Don't exit after node is synced #[arg(short)] watch: bool, + + /// Exit once the node reaches the given chain epoch + #[arg(long, conflicts_with = "watch")] + target: Option, }, /// Check sync status Status, @@ -43,7 +48,7 @@ pub enum SyncCommands { impl SyncCommands { pub async fn run(self, client: rpc::Client) -> anyhow::Result<()> { match self { - Self::Wait { watch } => { + Self::Wait { watch, target } => { let mut term = Term::buffered_stdout(); let mut last_term_frame: Option<(usize, (u16, u16))> = None; @@ -70,9 +75,20 @@ impl SyncCommands { last_term_frame = Some((rows, size)); term.flush()?; + let is_wait_done = if let Some(target) = target { + // Exit if we reach a given target + report.current_head_epoch >= target + } else { + // Exit if synced and not in watch mode. + !watch && report.status == NodeSyncStatus::Synced + }; + // Exit if synced and not in watch mode. - if !watch && report.status == NodeSyncStatus::Synced { - writeln!(term, "\nSync complete!")?; + if is_wait_done { + match target { + Some(target) => writeln!(term, "\nReached target epoch {target}!")?, + None => writeln!(term, "\nSync complete!")?, + } term.flush()?; break; } From 84e6b64ac0a7ca67678e3244a9f4762e6991172c Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 26 Oct 2026 07:48:37 -0400 Subject: [PATCH 02/14] chore: add CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d6af7265e5..40b7be0e306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ ### Added +- [#7802](https://github.com/ChainSafe/forest/pull/7802): Added `--target ` to `forest-cli sync wait`. The command exits once the node head reaches the given epoch. + ### Changed ### Removed From 78a353f122516b5896d3c2e1c813112557bd34a9 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 26 Oct 2026 07:53:03 -0400 Subject: [PATCH 03/14] chore: waits target_height + 1 --- scripts/devnet-forest-miner/check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/devnet-forest-miner/check.sh b/scripts/devnet-forest-miner/check.sh index a8864ab9356..136648238fb 100755 --- a/scripts/devnet-forest-miner/check.sh +++ b/scripts/devnet-forest-miner/check.sh @@ -29,7 +29,7 @@ docker exec \ -e FULLNODE_API_INFO="/ip4/127.0.0.1/tcp/${FOREST_RPC_PORT}/http" \ forest \ timeout "${timeout_secs}s" \ - forest-cli sync wait --target "$target_height" + forest-cli sync wait --target "$((target_height + 1))" # lotus waits for $target_height while true; do From 303c1cd34d19d87fa79ca7ab8718c46cb905bc47 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 21 Oct 2026 23:03:56 -0400 Subject: [PATCH 04/14] fix: `external-rpc-checks` checks last 24h epochs --- .github/workflows/external-rpc-checks.yml | 2 +- scripts/tests/external-rpc-checks/.env | 1 - .../external-rpc-checks/docker-compose.yaml | 25 +++++++--- scripts/tests/external-rpc-checks/resolve.rb | 41 +++++++++++------ scripts/tests/external-rpc-checks/setup.sh | 42 +++++++++++------ scripts/tests/external-rpc-checks/verify.rb | 3 +- scripts/tests/external-rpc-checks/wait.rb | 46 +++++++++++++++++++ 7 files changed, 121 insertions(+), 39 deletions(-) create mode 100644 scripts/tests/external-rpc-checks/wait.rb diff --git a/.github/workflows/external-rpc-checks.yml b/.github/workflows/external-rpc-checks.yml index f9f1b5c699f..4d54b10bab4 100644 --- a/.github/workflows/external-rpc-checks.yml +++ b/.github/workflows/external-rpc-checks.yml @@ -130,7 +130,7 @@ jobs: run: | echo "FOREST_IMAGE=${FOREST_IMAGE_INPUT}" >> "$GITHUB_ENV" - name: Run the RPC checks - # Mostly buffer around snapshot download, node processing should take ~10 minutes. + # Mostly buffer around snapshot download and syncing. timeout-minutes: 90 run: ./setup.sh - name: Dump docker logs diff --git a/scripts/tests/external-rpc-checks/.env b/scripts/tests/external-rpc-checks/.env index c4eef04913a..84ddafc20d0 100644 --- a/scripts/tests/external-rpc-checks/.env +++ b/scripts/tests/external-rpc-checks/.env @@ -3,4 +3,3 @@ FOREST_IMAGE=ghcr.io/chainsafe/forest:edge-fat FOREST_RPC_PORT=2345 FOREST_HEALTHZ_RPC_PORT=2346 FOREST_CHAIN=calibnet -FOREST_CHECK_EPOCHS=1000 diff --git a/scripts/tests/external-rpc-checks/docker-compose.yaml b/scripts/tests/external-rpc-checks/docker-compose.yaml index 35e9c5d6cd2..12d1a2c4e3a 100644 --- a/scripts/tests/external-rpc-checks/docker-compose.yaml +++ b/scripts/tests/external-rpc-checks/docker-compose.yaml @@ -9,6 +9,15 @@ x-forest-service: &forest-service - FOREST_PATH=/data user: 0:0 +# Talks to the node from inside the compose network, so its RPC port is never published. +x-rpc-client: &rpc-client + image: ghcr.io/chainsafe/forest-rpc-checks:latest + depends_on: + forest: + condition: service_healthy + environment: + FOREST_RPC_URL: forest:${FOREST_RPC_PORT}/rpc/v1 + services: resolve: image: ghcr.io/chainsafe/forest-rpc-checks:latest @@ -18,7 +27,6 @@ services: entrypoint: ["ruby", "/resolve.rb"] environment: - FOREST_CHAIN=${FOREST_CHAIN} - - FOREST_CHECK_EPOCHS=${FOREST_CHECK_EPOCHS} init: <<: *forest-service @@ -56,18 +64,21 @@ services: start_period: 5m start_interval: 5s + # Waits for the node to sync past the day the checks cover. + wait: + <<: *rpc-client + volumes: + - ./wait.rb:/wait.rb:ro + entrypoint: ["ruby", "/wait.rb"] + # Probes the node for the epochs the checks are about to query. verify: - image: ghcr.io/chainsafe/forest-rpc-checks:latest - depends_on: - forest: - condition: service_healthy + <<: *rpc-client volumes: - ./verify.rb:/verify.rb:ro entrypoint: ["ruby", "/verify.rb"] - environment: - FOREST_RPC_URL: forest:${FOREST_RPC_PORT}/rpc/v1 + # Not an rpc-client: ./setup.sh probes the dataset with it before the node is up. rpc-checks: image: ghcr.io/chainsafe/forest-rpc-checks:latest environment: diff --git a/scripts/tests/external-rpc-checks/resolve.rb b/scripts/tests/external-rpc-checks/resolve.rb index 1efd8f5a86e..70e68ce2f97 100644 --- a/scripts/tests/external-rpc-checks/resolve.rb +++ b/scripts/tests/external-rpc-checks/resolve.rb @@ -1,15 +1,21 @@ # frozen_string_literal: true -# Picks the ${FOREST_CHAIN} snapshot published DAYS_AGO days ago (UTC), by the date in its name, -# and records under /data what the rest of the harness needs: its URL, its head epoch, and the -# chain plus the ${FOREST_CHECK_EPOCHS}-epoch range the checks cover. -# Runs in the checks image, which has Ruby, so the Forest image needs no curl. +# Picks the ${FOREST_CHAIN} snapshot the checks start from and records under /data its URL, +# its head epoch, and the chain plus the epoch range of the UTC day DAYS_AGO days ago, which +# is what the dataset's daily archives cover. +# +# The archive's snapshots carry the state of their last 2000 epochs and the index backfill +# re-executes every epoch down and the node syncs the rest upwards (see ./wait.rb). require 'date' require 'json' require 'net/http' -# Compose passes an unset host variable through as an empty string, which `ENV.fetch` accepts. +SECONDS_IN_EPOCH = 30 +EPOCHS_IN_DAY = 2880 +SNAPSHOT_DEPTH = 2000 +GENESIS = { 'calibnet' => 1_667_326_380, 'mainnet' => 1_598_306_400 }.freeze + def env(name) value = ENV[name].to_s abort "#{name} is not set" if value.empty? @@ -24,8 +30,8 @@ def env_int(name) end chain = env('FOREST_CHAIN') -epochs = env_int('FOREST_CHECK_EPOCHS') days_ago = env_int('DAYS_AGO') +genesis = GENESIS.fetch(chain) { abort "no genesis timestamp for #{chain}" } LIST = URI("https://forest-archive.chainsafe.dev/list/#{chain}/latest-v2?format=json") def fetch(uri, attempts: 3) @@ -42,13 +48,22 @@ def fetch(uri, attempts: 3) abort "failed to fetch #{uri} after #{attempts} attempts: #{reason}" end -day = (Time.now.utc.to_date - days_ago).iso8601 -urls = JSON.parse(fetch(LIST))['items'].map { |item| item['url'] } -url = urls.find { |candidate| candidate.include?("_#{day}_") } -abort "no #{chain} snapshot published for #{day}" if url.nil? -epoch = Integer(url[/_height_(\d+)/, 1]) +day = Time.now.utc.to_date - days_ago +start = (Time.utc(day.year, day.month, day.day).to_i - genesis) / SECONDS_IN_EPOCH +finish = start + EPOCHS_IN_DAY - 1 + +snapshots = JSON.parse(fetch(LIST))['items'].filter_map do |item| + height = item['url'][/_height_(\d+)/, 1] + [item['url'], Integer(height)] if height +end +reachable = start...start + SNAPSHOT_DEPTH +url, epoch = snapshots.select { |_, height| reachable.cover?(height) }.max_by(&:last) +if url.nil? + warn "no #{chain} snapshot heads within the first #{SNAPSHOT_DEPTH} epochs of #{day.iso8601}" + exit 2 +end File.write('/data/snapshot-url', "#{url}\n") File.write('/data/snapshot-epoch', "#{epoch}\n") -File.write('/data/check-target', "#{chain} #{epoch - epochs} #{epoch - 1}\n") -puts "#{chain} snapshot for #{day}: #{url}" +File.write('/data/check-target', "#{chain} #{start} #{finish}\n") +puts "#{chain} #{day.iso8601} is epochs #{start}..#{finish}; snapshot at #{epoch}: #{url}" diff --git a/scripts/tests/external-rpc-checks/setup.sh b/scripts/tests/external-rpc-checks/setup.sh index 340f7ff5ca3..59078732161 100755 --- a/scripts/tests/external-rpc-checks/setup.sh +++ b/scripts/tests/external-rpc-checks/setup.sh @@ -10,11 +10,17 @@ pushd "${PARENT_PATH}" # This should not be needed in GH. It is useful for running locally. docker compose down --remove-orphans --volumes -# The dataset publishes each UTC day's archives some time the next morning, so -# yesterday's may not exist yet. Try yesterday's snapshot first, and the day -# before if the dataset has no data for it. -for days_ago in 1 2; do - docker compose run --rm --env DAYS_AGO="${days_ago}" resolve +# Resolves the snapshot and probes the dataset for DAYS_AGO. Sets CHAIN, +# START and END on success; returns 2 after saying what that day lacks. +check_day() { + local status=0 + docker compose run --rm --env DAYS_AGO="$1" resolve || status=$? + case ${status} in + 0) ;; + 2) return 2 ;; # resolve said why + *) exit "${status}" ;; + esac + CHECK_TARGET="$(docker compose run --rm --no-TTY --entrypoint cat resolve /data/check-target)" read -r CHAIN START END <<< "${CHECK_TARGET}" [[ -n ${CHAIN} && ${START} =~ ^[0-9]+$ && ${END} =~ ^[0-9]+$ ]] || { @@ -22,24 +28,30 @@ for days_ago in 1 2; do exit 1 } - probe=0 - docker compose run --rm rpc-checks --probe --network "${CHAIN}" "${START}" "${END}" || probe=$? - case ${probe} in - 0) break ;; - 2) echo "the dataset has not published that day yet" ;; - *) exit "${probe}" ;; + docker compose run --rm rpc-checks --probe --network "${CHAIN}" "${START}" "${END}" || status=$? + case ${status} in + 0) ;; + 2) + echo "the dataset has not published that day yet" + return 2 + ;; + *) exit "${status}" ;; esac -done -[[ ${probe} -eq 0 ]] || { - echo "the dataset has published neither yesterday nor the day before; raise it with its maintainer" - exit 1 } +# Tries to fetch latest available archive dataset (from yesterday) +# if not available lookup an archive dataset before yesterday +# if nothing is available than fail +check_day 1 || check_day 2 + # Imports the snapshot and back-fills the index. docker compose run --rm init docker compose up --detach --wait forest +# Syncs the rest till reach END. +docker compose run --rm wait "${END}" + docker compose run --rm verify "${START}" "${END}" docker compose run --rm rpc-checks "${START}" "${END}" diff --git a/scripts/tests/external-rpc-checks/verify.rb b/scripts/tests/external-rpc-checks/verify.rb index 567ba308c91..57987db1c5f 100644 --- a/scripts/tests/external-rpc-checks/verify.rb +++ b/scripts/tests/external-rpc-checks/verify.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -# Confirms the back-fill indexed the epochs the checks query. Runs in a container -# so the node's RPC port never has to be published. +# Confirms the node indexed the epochs the checks query. require 'json' require 'net/http' diff --git a/scripts/tests/external-rpc-checks/wait.rb b/scripts/tests/external-rpc-checks/wait.rb new file mode 100644 index 00000000000..edb890a5946 --- /dev/null +++ b/scripts/tests/external-rpc-checks/wait.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +# Blocks until the node's head is past the given epoch: the snapshot heads mid-day, and the +# checks need every epoch of the day executed, with the last one's receipts stored by its child. +# Waits for that epoch rather than for the network head (`forest-cli sync wait`, /readyz) +# because the scheduled run starts hours after the day ended, and catching up to the network +# head first would roughly triple the sync before the checks. +# Gives up when the head stops advancing for STALL_SECONDS, which must outlast the cold start +# (peer discovery and header fetching took ~5 minutes locally); the workflow timeout is the +# hard cap. + +require 'json' +require 'net/http' + +POLL_SECONDS = 15 +STALL_SECONDS = 20 * 60 + +$stdout.sync = true + +abort "usage: #{File.basename($PROGRAM_NAME)} " unless ARGV.size == 1 +url = URI("http://#{ENV.fetch('FOREST_RPC_URL')}") +target = Integer(ARGV.first) + 1 + +# The head epoch, or nil if the node did not answer. +def head(url) + request = Net::HTTP::Post.new(url, 'Content-Type' => 'application/json') + request.body = { jsonrpc: '2.0', id: 1, method: 'Filecoin.ChainHead', params: [] }.to_json + response = Net::HTTP.start(url.hostname, url.port) { |http| http.request(request) } + JSON.parse(response.body).dig('result', 'Height') or raise "unexpected response: #{response.body}" +rescue StandardError => e + puts "head unavailable: #{e.message}" + nil +end + +seen = nil +advanced_at = Time.now +until (epoch = head(url)) && epoch >= target + if epoch + puts "head at #{epoch}, #{target - epoch} epochs to go" + advanced_at = Time.now if epoch != seen + seen = epoch + end + abort "the head has not advanced for #{STALL_SECONDS}s" if Time.now - advanced_at > STALL_SECONDS + sleep POLL_SECONDS +end +puts "head at #{epoch} is past #{target - 1}" From d981241ec945bf09c9e80df39070806bed7536c6 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 21 Oct 2026 23:24:14 -0400 Subject: [PATCH 05/14] chore: addressing small points --- .../tests/external-rpc-checks/docker-compose.yaml | 1 - scripts/tests/external-rpc-checks/wait.rb | 13 ++++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/scripts/tests/external-rpc-checks/docker-compose.yaml b/scripts/tests/external-rpc-checks/docker-compose.yaml index 12d1a2c4e3a..a7f90dd8da3 100644 --- a/scripts/tests/external-rpc-checks/docker-compose.yaml +++ b/scripts/tests/external-rpc-checks/docker-compose.yaml @@ -78,7 +78,6 @@ services: - ./verify.rb:/verify.rb:ro entrypoint: ["ruby", "/verify.rb"] - # Not an rpc-client: ./setup.sh probes the dataset with it before the node is up. rpc-checks: image: ghcr.io/chainsafe/forest-rpc-checks:latest environment: diff --git a/scripts/tests/external-rpc-checks/wait.rb b/scripts/tests/external-rpc-checks/wait.rb index edb890a5946..04f58be7d56 100644 --- a/scripts/tests/external-rpc-checks/wait.rb +++ b/scripts/tests/external-rpc-checks/wait.rb @@ -1,19 +1,14 @@ # frozen_string_literal: true -# Blocks until the node's head is past the given epoch: the snapshot heads mid-day, and the -# checks need every epoch of the day executed, with the last one's receipts stored by its child. -# Waits for that epoch rather than for the network head (`forest-cli sync wait`, /readyz) -# because the scheduled run starts hours after the day ended, and catching up to the network -# head first would roughly triple the sync before the checks. -# Gives up when the head stops advancing for STALL_SECONDS, which must outlast the cold start -# (peer discovery and header fetching took ~5 minutes locally); the workflow timeout is the -# hard cap. +# Blocks until the node's head is past the given epoch. +# Gives up when the head stops advancing for STALL_SECONDS +# place a buffer for peer discovery and header fetching require 'json' require 'net/http' POLL_SECONDS = 15 -STALL_SECONDS = 20 * 60 +STALL_SECONDS = 600 $stdout.sync = true From 0910df4e87e72ff76b7fc59b62a4a4e25de457ef Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 26 Oct 2026 08:02:18 -0400 Subject: [PATCH 06/14] chore: remove unneeded changes --- .../external-rpc-checks/docker-compose.yaml | 21 ++++--------------- scripts/tests/external-rpc-checks/resolve.rb | 10 ++++----- scripts/tests/external-rpc-checks/setup.sh | 7 ++++--- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/scripts/tests/external-rpc-checks/docker-compose.yaml b/scripts/tests/external-rpc-checks/docker-compose.yaml index a7f90dd8da3..3b912d390ff 100644 --- a/scripts/tests/external-rpc-checks/docker-compose.yaml +++ b/scripts/tests/external-rpc-checks/docker-compose.yaml @@ -9,15 +9,6 @@ x-forest-service: &forest-service - FOREST_PATH=/data user: 0:0 -# Talks to the node from inside the compose network, so its RPC port is never published. -x-rpc-client: &rpc-client - image: ghcr.io/chainsafe/forest-rpc-checks:latest - depends_on: - forest: - condition: service_healthy - environment: - FOREST_RPC_URL: forest:${FOREST_RPC_PORT}/rpc/v1 - services: resolve: image: ghcr.io/chainsafe/forest-rpc-checks:latest @@ -64,16 +55,12 @@ services: start_period: 5m start_interval: 5s - # Waits for the node to sync past the day the checks cover. - wait: - <<: *rpc-client - volumes: - - ./wait.rb:/wait.rb:ro - entrypoint: ["ruby", "/wait.rb"] - # Probes the node for the epochs the checks are about to query. verify: - <<: *rpc-client + image: ghcr.io/chainsafe/forest-rpc-checks:latest + depends_on: + forest: + condition: service_healthy volumes: - ./verify.rb:/verify.rb:ro entrypoint: ["ruby", "/verify.rb"] diff --git a/scripts/tests/external-rpc-checks/resolve.rb b/scripts/tests/external-rpc-checks/resolve.rb index 70e68ce2f97..57a9b9ea96a 100644 --- a/scripts/tests/external-rpc-checks/resolve.rb +++ b/scripts/tests/external-rpc-checks/resolve.rb @@ -1,11 +1,9 @@ # frozen_string_literal: true -# Picks the ${FOREST_CHAIN} snapshot the checks start from and records under /data its URL, -# its head epoch, and the chain plus the epoch range of the UTC day DAYS_AGO days ago, which -# is what the dataset's daily archives cover. -# -# The archive's snapshots carry the state of their last 2000 epochs and the index backfill -# re-executes every epoch down and the node syncs the rest upwards (see ./wait.rb). +# Picks the ${FOREST_CHAIN} snapshot published DAYS_AGO days ago (UTC), by the date in its name, +# and records under /data what the rest of the harness needs: its URL, its head epoch, and the +# chain plus the ${FOREST_CHECK_EPOCHS}-epoch range the checks cover. +# Runs in the checks image, which has Ruby, so the Forest image needs no curl. require 'date' require 'json' diff --git a/scripts/tests/external-rpc-checks/setup.sh b/scripts/tests/external-rpc-checks/setup.sh index 59078732161..286074b4e25 100755 --- a/scripts/tests/external-rpc-checks/setup.sh +++ b/scripts/tests/external-rpc-checks/setup.sh @@ -10,14 +10,15 @@ pushd "${PARENT_PATH}" # This should not be needed in GH. It is useful for running locally. docker compose down --remove-orphans --volumes -# Resolves the snapshot and probes the dataset for DAYS_AGO. Sets CHAIN, -# START and END on success; returns 2 after saying what that day lacks. +# Resolves the snapshot and probes the dataset for DAYS_AGO. +# Sets CHAIN, START and END on success. +# Returns 2 on failure check_day() { local status=0 docker compose run --rm --env DAYS_AGO="$1" resolve || status=$? case ${status} in 0) ;; - 2) return 2 ;; # resolve said why + 2) return 2 ;; *) exit "${status}" ;; esac From 56141dcaa1350452aebcc029cd4526138346ac30 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 26 Oct 2026 08:38:15 -0400 Subject: [PATCH 07/14] chore: use sync wait --target --- .../external-rpc-checks/docker-compose.yaml | 4 ++ scripts/tests/external-rpc-checks/setup.sh | 5 ++- scripts/tests/external-rpc-checks/wait.rb | 41 ------------------- 3 files changed, 7 insertions(+), 43 deletions(-) delete mode 100644 scripts/tests/external-rpc-checks/wait.rb diff --git a/scripts/tests/external-rpc-checks/docker-compose.yaml b/scripts/tests/external-rpc-checks/docker-compose.yaml index 3b912d390ff..a5091dd9021 100644 --- a/scripts/tests/external-rpc-checks/docker-compose.yaml +++ b/scripts/tests/external-rpc-checks/docker-compose.yaml @@ -32,6 +32,8 @@ services: environment: - FOREST_PATH=/data - FOREST_CHAIN_INDEXER_ENABLED=1 + # for the `forest-cli` that ./setup.sh runs inside this container + - FULLNODE_API_INFO=/ip4/127.0.0.1/tcp/${FOREST_RPC_PORT}/http command: - --chain=${FOREST_CHAIN} - --encrypt-keystore=false @@ -64,6 +66,8 @@ services: volumes: - ./verify.rb:/verify.rb:ro entrypoint: ["ruby", "/verify.rb"] + environment: + FOREST_RPC_URL: forest:${FOREST_RPC_PORT}/rpc/v1 rpc-checks: image: ghcr.io/chainsafe/forest-rpc-checks:latest diff --git a/scripts/tests/external-rpc-checks/setup.sh b/scripts/tests/external-rpc-checks/setup.sh index 286074b4e25..244a2301fc6 100755 --- a/scripts/tests/external-rpc-checks/setup.sh +++ b/scripts/tests/external-rpc-checks/setup.sh @@ -10,7 +10,7 @@ pushd "${PARENT_PATH}" # This should not be needed in GH. It is useful for running locally. docker compose down --remove-orphans --volumes -# Resolves the snapshot and probes the dataset for DAYS_AGO. +# Resolves the snapshot and probes the dataset for DAYS_AGO. # Sets CHAIN, START and END on success. # Returns 2 on failure check_day() { @@ -51,7 +51,8 @@ docker compose run --rm init docker compose up --detach --wait forest # Syncs the rest till reach END. -docker compose run --rm wait "${END}" +docker compose exec --no-TTY forest \ + timeout 30m forest-cli sync wait --target "$((END + 1))" docker compose run --rm verify "${START}" "${END}" diff --git a/scripts/tests/external-rpc-checks/wait.rb b/scripts/tests/external-rpc-checks/wait.rb deleted file mode 100644 index 04f58be7d56..00000000000 --- a/scripts/tests/external-rpc-checks/wait.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -# Blocks until the node's head is past the given epoch. -# Gives up when the head stops advancing for STALL_SECONDS -# place a buffer for peer discovery and header fetching - -require 'json' -require 'net/http' - -POLL_SECONDS = 15 -STALL_SECONDS = 600 - -$stdout.sync = true - -abort "usage: #{File.basename($PROGRAM_NAME)} " unless ARGV.size == 1 -url = URI("http://#{ENV.fetch('FOREST_RPC_URL')}") -target = Integer(ARGV.first) + 1 - -# The head epoch, or nil if the node did not answer. -def head(url) - request = Net::HTTP::Post.new(url, 'Content-Type' => 'application/json') - request.body = { jsonrpc: '2.0', id: 1, method: 'Filecoin.ChainHead', params: [] }.to_json - response = Net::HTTP.start(url.hostname, url.port) { |http| http.request(request) } - JSON.parse(response.body).dig('result', 'Height') or raise "unexpected response: #{response.body}" -rescue StandardError => e - puts "head unavailable: #{e.message}" - nil -end - -seen = nil -advanced_at = Time.now -until (epoch = head(url)) && epoch >= target - if epoch - puts "head at #{epoch}, #{target - epoch} epochs to go" - advanced_at = Time.now if epoch != seen - seen = epoch - end - abort "the head has not advanced for #{STALL_SECONDS}s" if Time.now - advanced_at > STALL_SECONDS - sleep POLL_SECONDS -end -puts "head at #{epoch} is past #{target - 1}" From 4afc5b3e8289878877bf85e28a54642268b7daab Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 28 Oct 2026 10:32:21 -0400 Subject: [PATCH 08/14] chore: remove uneeded change --- scripts/devnet-forest-miner/check.sh | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/scripts/devnet-forest-miner/check.sh b/scripts/devnet-forest-miner/check.sh index 136648238fb..20942dbbfea 100755 --- a/scripts/devnet-forest-miner/check.sh +++ b/scripts/devnet-forest-miner/check.sh @@ -18,30 +18,22 @@ function get_sync_height { } start_time=$(date +%s) -timeout_secs=360 -timeout=$((start_time + timeout_secs)) # 6 minutes +timeout=$((start_time + 360)) # 6 minutes # Target height chosen so that every migration in .env is crossed. target_height=$TARGET_HEIGHT -# forest waits for $target_height -docker exec \ - -e FULLNODE_API_INFO="/ip4/127.0.0.1/tcp/${FOREST_RPC_PORT}/http" \ - forest \ - timeout "${timeout_secs}s" \ - forest-cli sync wait --target "$((target_height + 1))" - -# lotus waits for $target_height while true; do + forest_height=$(get_sync_height "${FOREST_RPC_PORT}") lotus_height=$(get_sync_height "${LOTUS_RPC_PORT}") - if [ "$lotus_height" -gt "$target_height" ]; then - echo "Lotus node past $target_height ($lotus_height): Lotus validated Forest-produced blocks" + if [ "$forest_height" -gt "$target_height" ] && [ "$lotus_height" -gt "$target_height" ]; then + echo "Both nodes past $target_height (Forest: $forest_height, Lotus: $lotus_height): Lotus validated Forest-produced blocks" break fi current_time=$(date +%s) if [ "$current_time" -gt "$timeout" ]; then - echo "Timeout reached (Lotus: $lotus_height), target $target_height not reached by Lotus node" + echo "Timeout reached (Forest: $forest_height, Lotus: $lotus_height), target $target_height not reached by both" exit 1 fi From f3c2e701ec60a2e92c9fc7c54ed481a927577ae1 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 28 Oct 2026 10:33:20 -0400 Subject: [PATCH 09/14] chore: remove unneeded changes --- scripts/devnet/check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/devnet/check.sh b/scripts/devnet/check.sh index e74fb75dfb4..2327484fd49 100755 --- a/scripts/devnet/check.sh +++ b/scripts/devnet/check.sh @@ -26,7 +26,7 @@ function get_f3_latest_cert_instance { } # Target height set so that all migrations are applied. -target_height=$((TARGET_HEIGHT + 1)) +target_height=$TARGET_HEIGHT # waits node to reach $target_height, timeout docker exec \ From ccb54d556dec69d3e51d9d8c9436da15f9873f6b Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 28 Oct 2026 11:23:21 -0400 Subject: [PATCH 10/14] chore: dont polute stdout with sync wait logs --- scripts/tests/external-rpc-checks/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/external-rpc-checks/setup.sh b/scripts/tests/external-rpc-checks/setup.sh index 244a2301fc6..b39212da6df 100755 --- a/scripts/tests/external-rpc-checks/setup.sh +++ b/scripts/tests/external-rpc-checks/setup.sh @@ -52,7 +52,7 @@ docker compose up --detach --wait forest # Syncs the rest till reach END. docker compose exec --no-TTY forest \ - timeout 30m forest-cli sync wait --target "$((END + 1))" + timeout 30m forest-cli sync wait --target "$((END + 1))" > /dev/null docker compose run --rm verify "${START}" "${END}" From 7ed3c6836327eb1b459b2023cfab79ecc628905b Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 28 Oct 2026 11:33:42 -0400 Subject: [PATCH 11/14] chore: include --wait to forest healthcheck --- scripts/tests/external-rpc-checks/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/tests/external-rpc-checks/docker-compose.yaml b/scripts/tests/external-rpc-checks/docker-compose.yaml index a5091dd9021..40254a9eb79 100644 --- a/scripts/tests/external-rpc-checks/docker-compose.yaml +++ b/scripts/tests/external-rpc-checks/docker-compose.yaml @@ -48,6 +48,7 @@ services: "forest-cli", "healthcheck", "live", + "--wait", "--healthcheck-port", "${FOREST_HEALTHZ_RPC_PORT}", ] From 52b7bdae30fe95c64aaa082b6bf782e5754abbb0 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 28 Oct 2026 14:47:57 -0400 Subject: [PATCH 12/14] chore: improve fail messages on setup.sh --- .github/workflows/external-rpc-checks.yml | 2 +- .../external-rpc-checks/docker-compose.yaml | 1 - scripts/tests/external-rpc-checks/setup.sh | 21 +++++++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/external-rpc-checks.yml b/.github/workflows/external-rpc-checks.yml index 4d54b10bab4..f9f1b5c699f 100644 --- a/.github/workflows/external-rpc-checks.yml +++ b/.github/workflows/external-rpc-checks.yml @@ -130,7 +130,7 @@ jobs: run: | echo "FOREST_IMAGE=${FOREST_IMAGE_INPUT}" >> "$GITHUB_ENV" - name: Run the RPC checks - # Mostly buffer around snapshot download and syncing. + # Mostly buffer around snapshot download, node processing should take ~10 minutes. timeout-minutes: 90 run: ./setup.sh - name: Dump docker logs diff --git a/scripts/tests/external-rpc-checks/docker-compose.yaml b/scripts/tests/external-rpc-checks/docker-compose.yaml index 40254a9eb79..ef54472d46f 100644 --- a/scripts/tests/external-rpc-checks/docker-compose.yaml +++ b/scripts/tests/external-rpc-checks/docker-compose.yaml @@ -32,7 +32,6 @@ services: environment: - FOREST_PATH=/data - FOREST_CHAIN_INDEXER_ENABLED=1 - # for the `forest-cli` that ./setup.sh runs inside this container - FULLNODE_API_INFO=/ip4/127.0.0.1/tcp/${FOREST_RPC_PORT}/http command: - --chain=${FOREST_CHAIN} diff --git a/scripts/tests/external-rpc-checks/setup.sh b/scripts/tests/external-rpc-checks/setup.sh index b39212da6df..270dab5379a 100755 --- a/scripts/tests/external-rpc-checks/setup.sh +++ b/scripts/tests/external-rpc-checks/setup.sh @@ -18,11 +18,17 @@ check_day() { docker compose run --rm --env DAYS_AGO="$1" resolve || status=$? case ${status} in 0) ;; - 2) return 2 ;; - *) exit "${status}" ;; + 2) + echo "no snapshot covers the day $1 day(s) ago" + return 2 + ;; + *) + echo "resolve failed for the day $1 day(s) ago with status ${status}" + exit "${status}" + ;; esac - CHECK_TARGET="$(docker compose run --rm --no-TTY --entrypoint cat resolve /data/check-target)" + CHECK_TARGET="$(docker compose run --rm --entrypoint cat resolve /data/check-target)" read -r CHAIN START END <<< "${CHECK_TARGET}" [[ -n ${CHAIN} && ${START} =~ ^[0-9]+$ && ${END} =~ ^[0-9]+$ ]] || { echo "resolve did not report a usable chain and epoch range: ${CHECK_TARGET}" @@ -33,10 +39,13 @@ check_day() { case ${status} in 0) ;; 2) - echo "the dataset has not published that day yet" + echo "the dataset has not published the day $1 day(s) ago yet" return 2 ;; - *) exit "${status}" ;; + *) + echo "the dataset probe failed for ${CHAIN} ${START}..${END}" + exit "${status}" + ;; esac } @@ -51,7 +60,7 @@ docker compose run --rm init docker compose up --detach --wait forest # Syncs the rest till reach END. -docker compose exec --no-TTY forest \ +docker compose exec forest \ timeout 30m forest-cli sync wait --target "$((END + 1))" > /dev/null docker compose run --rm verify "${START}" "${END}" From a8d3d118322ae7c4919d442497971ea151c4f06c Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 28 Oct 2026 15:07:16 -0400 Subject: [PATCH 13/14] chore: update resolve init comment --- scripts/tests/external-rpc-checks/resolve.rb | 32 ++++++-------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/scripts/tests/external-rpc-checks/resolve.rb b/scripts/tests/external-rpc-checks/resolve.rb index 57a9b9ea96a..de5121039af 100644 --- a/scripts/tests/external-rpc-checks/resolve.rb +++ b/scripts/tests/external-rpc-checks/resolve.rb @@ -1,9 +1,8 @@ # frozen_string_literal: true -# Picks the ${FOREST_CHAIN} snapshot published DAYS_AGO days ago (UTC), by the date in its name, -# and records under /data what the rest of the harness needs: its URL, its head epoch, and the -# chain plus the ${FOREST_CHECK_EPOCHS}-epoch range the checks cover. -# Runs in the checks image, which has Ruby, so the Forest image needs no curl. +# Picks the newest ${FOREST_CHAIN} snapshot whose head lies within the first 2000 epochs of +# the UTC day: it holds the state the backfill needs to reach the day's start, +# and the node syncs the rest. Writes its URL, head epoch and the day's epoch range under /data. require 'date' require 'json' @@ -14,24 +13,6 @@ SNAPSHOT_DEPTH = 2000 GENESIS = { 'calibnet' => 1_667_326_380, 'mainnet' => 1_598_306_400 }.freeze -def env(name) - value = ENV[name].to_s - abort "#{name} is not set" if value.empty? - - value -end - -def env_int(name) - Integer(env(name)) -rescue ArgumentError => e - abort e.message -end - -chain = env('FOREST_CHAIN') -days_ago = env_int('DAYS_AGO') -genesis = GENESIS.fetch(chain) { abort "no genesis timestamp for #{chain}" } -LIST = URI("https://forest-archive.chainsafe.dev/list/#{chain}/latest-v2?format=json") - def fetch(uri, attempts: 3) reason = nil attempts.times do |i| @@ -46,11 +27,16 @@ def fetch(uri, attempts: 3) abort "failed to fetch #{uri} after #{attempts} attempts: #{reason}" end +chain = ENV.fetch('FOREST_CHAIN') +days_ago = Integer(ENV.fetch('DAYS_AGO')) +genesis = GENESIS.fetch(chain) { abort "no genesis timestamp for #{chain}" } + day = Time.now.utc.to_date - days_ago start = (Time.utc(day.year, day.month, day.day).to_i - genesis) / SECONDS_IN_EPOCH finish = start + EPOCHS_IN_DAY - 1 -snapshots = JSON.parse(fetch(LIST))['items'].filter_map do |item| +list = URI("https://forest-archive.chainsafe.dev/list/#{chain}/latest-v2?format=json") +snapshots = JSON.parse(fetch(list))['items'].filter_map do |item| height = item['url'][/_height_(\d+)/, 1] [item['url'], Integer(height)] if height end From 7d0c4d198de6daaadc02440650d02663a8ece1b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20Junior?= Date: Thu, 29 Oct 2026 08:46:14 -0400 Subject: [PATCH 14/14] Update scripts/tests/external-rpc-checks/setup.sh Co-authored-by: Hubert --- scripts/tests/external-rpc-checks/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/external-rpc-checks/setup.sh b/scripts/tests/external-rpc-checks/setup.sh index 270dab5379a..24e4df838eb 100755 --- a/scripts/tests/external-rpc-checks/setup.sh +++ b/scripts/tests/external-rpc-checks/setup.sh @@ -51,7 +51,7 @@ check_day() { # Tries to fetch latest available archive dataset (from yesterday) # if not available lookup an archive dataset before yesterday -# if nothing is available than fail +# if nothing is available then fail check_day 1 || check_day 2 # Imports the snapshot and back-fills the index.