diff --git a/.github/workflows/zilkworm-benchmark.yml b/.github/workflows/zilkworm-benchmark.yml new file mode 100644 index 000000000..b07113f97 --- /dev/null +++ b/.github/workflows/zilkworm-benchmark.yml @@ -0,0 +1,871 @@ +name: "Zilkworm Benchmark" +run-name: "Zilkworm Benchmark (${{ inputs.mode || github.event.inputs.mode }}, ${{ inputs.input_source || github.event.inputs.input_source }})" + +on: + workflow_dispatch: + inputs: + instance_family: + type: string + required: false + description: The family of the instance, can be multiple ones concat with "+" e.g. g6+g5 + default: g7e.2xlarge + memory_allocator: + type: choice + required: false + description: Memory allocator to use + options: + - jemalloc + - mimalloc + default: jemalloc + zilkworm_ref: + type: string + required: false + description: Git ref of axiom-crypto/zilkworm-stateless to build the guest from + default: feat/openvm-rv64 + input_source: + type: string + required: false + description: Guest input; "mock", "block:" (mainnet block via RPC witness pipeline), or an s3:// / https:// URL to an eth-act SSZ StatelessInput .bin + default: mock + mode: + type: choice + required: false + description: Running mode + options: + - execute + - execute-metered + - prove-app + - prove-stark + default: prove-stark + profiling: + type: choice + required: false + description: Profiling mode + options: + - none + - host # host profiling (samply, flamegraphs, etc) + - guest # guest profiling (circuit flamegraphs) + - nsys # nsys profiling (cuda) + default: none + app_log_blowup: + type: number + required: false + description: Application level log blowup + leaf_log_blowup: + type: number + required: false + description: Aggregation (leaf) level log blowup + internal_log_blowup: + type: number + required: false + description: Internal level log blowup + segment_max_memory: + type: string + required: false + description: GPU memory soft limit override for segmentation + app_l_skip: + type: number + required: false + description: Log of univariate skip domain size override + exec_mode: + type: choice + required: false + description: OpenVM execution backend + options: + - interpreter + - rvr + - tco + default: rvr + num_children_leaf: + type: number + required: false + description: Number of app proofs that leaf verifier aggregates + num_children_internal: + type: number + required: false + description: Number of proofs that internal verifier aggregates + workflow_call: + inputs: + ref: + type: string + required: false + description: Git ref to checkout + instance_family: + type: string + required: false + description: The family of the instance, can be multiple ones concat with "+" e.g. g6+g5 + default: g7e.2xlarge + memory_allocator: + type: string + required: false + description: Memory allocator to use (mimalloc or jemalloc) + default: jemalloc + zilkworm_ref: + type: string + required: false + description: Git ref of axiom-crypto/zilkworm-stateless to build the guest from + default: feat/openvm-rv64 + input_source: + type: string + required: false + description: Guest input; "mock", "block:", or an s3:// / https:// URL to an eth-act SSZ StatelessInput .bin + default: mock + mode: + type: string + required: false + description: Running mode, one of {execute, execute-metered, prove-app, prove-stark} + default: prove-stark + profiling: + type: string + required: false + description: Profiling mode (none, host, guest, nsys) + default: none + app_log_blowup: + type: number + required: false + description: Application level log blowup + leaf_log_blowup: + type: number + required: false + description: Aggregation (leaf) level log blowup + internal_log_blowup: + type: number + required: false + description: Internal level log blowup + segment_max_memory: + type: string + required: false + description: GPU memory soft limit override for segmentation + app_l_skip: + type: number + required: false + description: Log of univariate skip domain size override + exec_mode: + type: string + required: false + description: OpenVM execution backend (interpreter, rvr, tco) + default: rvr + num_children_leaf: + type: number + required: false + description: Number of app proofs that leaf verifier aggregates + num_children_internal: + type: number + required: false + description: Number of proofs that internal verifier aggregates + tag: + type: string + required: false + description: Tag for cache keys (default is commit hash) + secrets: + GH_ACTIONS_DEPLOY_PRIVATE_KEY: + required: true + RPC_URL_1: + required: false + outputs: + metric_name: + description: "Name of the metric" + value: ${{ jobs.run-zilkworm.outputs.metric_name }} + gpu_peak_memory_mib: + description: "Peak GPU memory usage in MiB" + value: ${{ jobs.run-zilkworm.outputs.gpu_peak_memory_mib }} + num_segments: + description: "Number of segments" + value: ${{ jobs.run-zilkworm.outputs.num_segments }} + +env: + OPENVM_BRANCH: develop-v2.1.0 + ZILKWORM_REPO: axiom-crypto/zilkworm-stateless + RISCV_XPACK_VERSION: "15.2.0-1.1" + S3_FIXTURE_READ_ENABLED: ${{ vars.S3_PUBLIC_DATA_BUCKET != '' }} + S3_PUBLIC_UPLOAD_ENABLED: ${{ vars.S3_PUBLIC_DATA_BUCKET != '' && vars.S3_REGION != '' }} + S3_WORKFLOW_UPLOAD_ENABLED: ${{ vars.S3_WORKFLOW_DATA_BUCKET != '' }} + S3_MD_PATH: s3://${{ vars.S3_WORKFLOW_DATA_BUCKET }}/benchmark/github/results + S3_FIXTURE_PATH: s3://${{ vars.S3_PUBLIC_DATA_BUCKET }}/benchmark/github/fixtures + S3_METRICS_PATH: s3://${{ vars.S3_WORKFLOW_DATA_BUCKET }}/benchmark/github/metrics + S3_PUBLIC_PATH_BASE: s3://${{ vars.S3_PUBLIC_DATA_BUCKET }}/benchmark/github + S3_PUBLIC_URL_BASE: https://${{ vars.S3_PUBLIC_DATA_BUCKET }}.s3.${{ vars.S3_REGION }}.amazonaws.com/benchmark/github + S3_SAMPLY_PROFILE_PATH: s3://${{ vars.S3_PUBLIC_DATA_BUCKET }}/benchmark/github/samply + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + BIN_NAME: openvm-zilkworm-benchmark + +jobs: + run-zilkworm: + name: Run Zilkworm benchmark + permissions: + contents: write + runs-on: + - runs-on=${{ github.run_id }}/family=${{ inputs.instance_family || github.event.inputs.instance_family }}/volume=80gb/tag=bench-zilkworm-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}/extras=s3-cache/spot=false/image=${{ startsWith(github.event.inputs.instance_family || inputs.instance_family, 'g') && 'ubuntu24-gpu-x64' || contains(github.event.inputs.instance_family || inputs.instance_family, 'g.') && 'ubuntu24-full-arm64' || 'ubuntu24-full-x64' }} + outputs: + metric_name: ${{ steps.set-metric-name.outputs.name }} + gpu_peak_memory_mib: ${{ steps.report-gpu-memory.outputs.peak_memory_mib }} + num_segments: ${{ steps.report-segments.outputs.num_segments }} + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: mozilla-actions/sccache-action@v0.0.9 + - uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref || github.head_ref || github.ref }} + - uses: dtolnay/rust-toolchain@nightly + env: + RUSTUP_PERMIT_COPY_RENAME: "1" + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Add workflow inputs to summary + run: | + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Workflow Inputs" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + echo '${{ toJSON(inputs || github.event.inputs) }}' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + - name: Report S3-disabled mode + if: ${{ env.S3_PUBLIC_UPLOAD_ENABLED != 'true' && env.S3_WORKFLOW_UPLOAD_ENABLED != 'true' }} + run: | + echo "S3 uploads disabled; using GitHub artifacts and gh-pages." + echo "S3 uploads disabled; using GitHub artifacts and gh-pages." >> "$GITHUB_STEP_SUMMARY" + + - name: Get current commit hash + run: | + echo "current_sha=$(git rev-parse HEAD)" >> $GITHUB_ENV + # for add_metadata function + echo "CURRENT_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Set metric name + id: set-metric-name + run: | + input_json_str="${{ toJSON(inputs || github.event.inputs) }}" + input_hash=$(echo $input_json_str | sha256sum | cut -d' ' -f1) + METRIC_NAME=zilkworm-$current_sha-${input_hash}-${{ github.run_id }} + echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV + METRIC_PATH=".bench_metrics/${METRIC_NAME}.json" + echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV + echo "name=${METRIC_NAME}" >> $GITHUB_OUTPUT + + - name: Load SSH key + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: | + ${{ secrets.GH_ACTIONS_DEPLOY_PRIVATE_KEY }} + - name: Ensure GitHub.com is a known host + run: | + mkdir -p ~/.ssh + ssh-keyscan github.com >> ~/.ssh/known_hosts + + - name: Checkout zilkworm-stateless + run: | + ZILKWORM_REF="${{ inputs.zilkworm_ref || github.event.inputs.zilkworm_ref || 'feat/openvm-rv64' }}" + # Clone OUTSIDE the workspace so its cargo crates are not adopted + # by the openvm-eth workspace manifest. + ZILKWORM_DIR="${RUNNER_TEMP}/zilkworm-stateless" + git clone --branch "$ZILKWORM_REF" "https://github.com/${ZILKWORM_REPO}.git" "$ZILKWORM_DIR" + ZILKWORM_SHA=$(git -C "$ZILKWORM_DIR" rev-parse HEAD) + echo "ZILKWORM_DIR=${ZILKWORM_DIR}" >> $GITHUB_ENV + echo "ZILKWORM_SHA=${ZILKWORM_SHA}" >> $GITHUB_ENV + echo "Zilkworm commit: ${ZILKWORM_SHA}" + + - name: Checkout openvm (for scripts) + run: git clone --depth=1 --branch="$OPENVM_BRANCH" https://github.com/openvm-org/openvm.git + - name: Build openvm-prof + working-directory: openvm + run: | + cargo build -p openvm-prof + echo "${{ github.workspace }}/openvm/target/debug" >> $GITHUB_PATH + - name: Install architecture specific tools + run: | + source openvm/ci/scripts/utils.sh + if [[ "${S3_FIXTURE_READ_ENABLED}" == "true" || "${S3_PUBLIC_UPLOAD_ENABLED}" == "true" || "${S3_WORKFLOW_UPLOAD_ENABLED}" == "true" ]]; then + install_s5cmd + fi + sudo apt update && sudo apt install -y gnuplot cmake + sudo apt install -y linux-tools-common linux-tools-generic linux-tools-$(uname -r) || \ + sudo apt install -y linux-tools-aws linux-cloud-tools-aws || true + + # Workaround for https://bugs.launchpad.net/ubuntu/+source/linux-hwe-6.14/+bug/2117159 + if ! command -v perf &> /dev/null || ! perf --version &> /dev/null; then + echo "Applying workaround for missing perf in HWE 6.14 packages..." + PERF_PATH=$(find /usr/lib/linux-tools* -name perf -executable 2>/dev/null | head -n1) + if [ -n "$PERF_PATH" ]; then + KERNEL_VERSION=$(uname -r) + TARGET_DIR="/usr/lib/linux-tools/${KERNEL_VERSION}" + sudo mkdir -p "$TARGET_DIR" + sudo ln -sf "$PERF_PATH" "$TARGET_DIR/perf" + echo "Symlinked $PERF_PATH to $TARGET_DIR/perf" + else + echo "Warning: Could not find any perf binary to symlink" + fi + fi + + # TODO: remove once https://github.com/runs-on/runner-images-for-aws/issues/55 is fixed + - name: Setup GPU driver + if: ${{ startsWith(inputs.instance_family || github.event.inputs.instance_family, 'g') }} + run: bash ./scripts/gpu_driver_setup.sh + + - name: Install clang-22 and lld + if: ${{ (inputs.exec_mode || github.event.inputs.exec_mode || 'rvr') == 'rvr' }} + run: | + sudo apt-get update + sudo apt-get install -y lsb-release wget software-properties-common gnupg + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 22 + sudo apt-get install -y lld + + - name: Read Rust toolchain channel + run: | + RUST_TOOLCHAIN=$(sed -n 's/^channel = "\(.*\)"/\1/p' rust-toolchain.toml) + [[ -n "$RUST_TOOLCHAIN" ]] || { echo "No channel in rust-toolchain.toml" >&2; exit 1; } + echo "RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" >> $GITHUB_ENV + + - name: Set build profiles + id: set-build-profiles + run: | + HOST_PROFILE="maxperf" + if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" || "${{ inputs.profiling || github.event.inputs.profiling }}" == "nsys" ]]; then + HOST_PROFILE="profiling" + fi + echo "host_profile=${HOST_PROFILE}" >> $GITHUB_OUTPUT + + - name: Set build args + id: set-build-args + run: | + FEATURES="parallel,metrics,unprotected" + FEATURES="${FEATURES},${{ inputs.memory_allocator || github.event.inputs.memory_allocator || 'jemalloc' }}" + if [[ "${{ inputs.instance_family || github.event.inputs.instance_family }}" =~ ^g ]]; then + FEATURES="${FEATURES},cuda" + fi + if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "nsys" ]]; then + FEATURES="${FEATURES},nvtx" + fi + if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then + # Emits per-opcode metrics labeled with the guest function stack during preflight + # execution; openvm's flamegraph.py renders them as circuit flamegraphs. Requires a + # prove mode (preflight only runs during trace generation). + FEATURES="${FEATURES},perf-metrics" + fi + + EXEC_MODE="${{ inputs.exec_mode || github.event.inputs.exec_mode || 'rvr' }}" + + # Resolve "native" to the host CPU name (e.g. znver3) so sccache keys and + # cargo fingerprints differ across instance families (else SIGILL from reuse) + TARGET_CPU=$(rustc "+${RUST_TOOLCHAIN}" --print target-cpus | sed -n 's/^ *native.*(currently \([^)]*\)).*/\1/p') + TARGET_CPU="${TARGET_CPU:-native}" + echo "Resolved target CPU: ${TARGET_CPU}" + RUSTFLAGS="-Ctarget-cpu=${TARGET_CPU}" + + case "$EXEC_MODE" in + interpreter) ;; # default interpreted execution; no extra backend feature + tco) FEATURES="${FEATURES},tco" ;; + rvr) FEATURES="${FEATURES},rvr" ;; + *) echo "Unsupported exec_mode: $EXEC_MODE"; exit 1 ;; + esac + + if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" || "${{ inputs.profiling || github.event.inputs.profiling }}" == "nsys" ]]; then + RUSTFLAGS="${RUSTFLAGS} -C force-frame-pointers=yes" + fi + + if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" != "nsys" ]]; then + # Jemalloc sometimes causes nsys to hang forever (unknown reason) + JEMALLOC_SYS_WITH_MALLOC_CONF="retain:true,background_thread:true,metadata_thp:always,thp:always,dirty_decay_ms:10000,muzzy_decay_ms:10000,abort_conf:true" + echo "JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF}" >> $GITHUB_ENV + fi + echo "RUSTFLAGS=${RUSTFLAGS}" >> $GITHUB_ENV + echo "FEATURES=${FEATURES}" >> $GITHUB_ENV + + - name: Set cache keys + id: set-cache-keys + run: | + arch=$(uname -m) + # Use inputs.tag if defined, otherwise use current commit SHA + TAG="${{ inputs.tag || env.current_sha }}" + echo "elf_cache_key=zilkworm-elf-${ZILKWORM_SHA}-xpack${RISCV_XPACK_VERSION}" >> $GITHUB_OUTPUT + RUSTFLAGS_HASH=$(echo "${RUSTFLAGS}" | sha256sum | cut -d' ' -f1 | head -c8) + JEMALLOC_CONF_HASH=$(echo "${JEMALLOC_SYS_WITH_MALLOC_CONF}" | sha256sum | cut -d' ' -f1 | head -c8) + FEATURES_HASH=$(echo "${FEATURES}" | sha256sum | cut -d' ' -f1 | head -c8) + HOST_PROFILE=${{ steps.set-build-profiles.outputs.host_profile }} + echo "host_cache_key=zilkworm-host-${TAG}-${arch}-${HOST_PROFILE}-${RUSTFLAGS_HASH}-${JEMALLOC_CONF_HASH}-${FEATURES_HASH}" >> $GITHUB_OUTPUT + + - name: Restore Guest ELF from cache + id: cache-guest-elf-restore + uses: runs-on/cache/restore@v4 + with: + path: guest-elf/z6m_guest.elf + key: ${{ steps.set-cache-keys.outputs.elf_cache_key }} + + - name: Install xPack riscv-none-elf-gcc (pinned) + if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true' + run: | + if ! command -v npm &> /dev/null; then + sudo apt-get update && sudo apt-get install -y npm + fi + sudo npm install --location=global xpm@latest + xpm install "@xpack-dev-tools/riscv-none-elf-gcc@${RISCV_XPACK_VERSION}" --global + "$HOME/.local/xPacks/@xpack-dev-tools/riscv-none-elf-gcc/${RISCV_XPACK_VERSION}/.content/bin/riscv-none-elf-gcc" --version + + - name: Build Guest ELF + if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true' + run: | + make -C "$ZILKWORM_DIR" guest_openvm + mkdir -p guest-elf + cp "$ZILKWORM_DIR/build/openvm/z6m_guest.elf" guest-elf/ + + - name: Save Guest ELF to cache + if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true' + uses: runs-on/cache/save@v4 + with: + path: guest-elf/z6m_guest.elf + key: ${{ steps.cache-guest-elf-restore.outputs.cache-primary-key }} + + - name: Prepare guest input + id: prepare-input + run: | + INPUT_SOURCE="${{ inputs.input_source || github.event.inputs.input_source || 'mock' }}" + EXPECTED_ARGS="" + mkdir -p guest-input + if [[ "$INPUT_SOURCE" == "mock" ]]; then + (cd "$ZILKWORM_DIR/conformance" && cargo run --release -- mock --out-dir "$GITHUB_WORKSPACE/guest-input" --name mock) + INPUT_PATH="guest-input/mock_input.bin" + # The synthetic mock block cannot pass stateless validation (empty + # witness), so conformance harnesses assert the failure result. + EXPECTED_ARGS="--expected-output guest-input/mock_expected_failure.bin" + elif [[ "$INPUT_SOURCE" == block:* ]]; then + BLOCK_NUMBER="${INPUT_SOURCE#block:}" + NAME="block${BLOCK_NUMBER}" + S3_INPUT_BASE="${S3_FIXTURE_PATH}/zilkworm-input/v1/mainnet/${BLOCK_NUMBER}" + RESTORED=false + if [[ "${S3_FIXTURE_READ_ENABLED}" == "true" ]]; then + if s5cmd cp "${S3_INPUT_BASE}.input.bin" "guest-input/${NAME}_input.bin" && s5cmd cp "${S3_INPUT_BASE}.expected_success.bin" "guest-input/${NAME}_expected_success.bin"; then + echo "Restored zilkworm input for block ${BLOCK_NUMBER} from S3" + RESTORED=true + else + rm -f "guest-input/${NAME}_input.bin" "guest-input/${NAME}_expected_success.bin" + fi + fi + if [[ "$RESTORED" != "true" ]]; then + # Generate: Reth witness pipeline -> payload/witness JSONs -> + # eth-act SSZ StatelessInput assembly in the zilkworm repo. + cargo build -p openvm-rpc-proxy --bin zilkworm-input + ./target/debug/zilkworm-input --rpc-url "${{ secrets.RPC_URL_1 }}" --block-number "${BLOCK_NUMBER}" --out-dir guest-input + (cd "$ZILKWORM_DIR/conformance" && cargo run --release -- reth --payload-json "$GITHUB_WORKSPACE/guest-input/payload.json" --witness-json "$GITHUB_WORKSPACE/guest-input/witness.json" --out-dir "$GITHUB_WORKSPACE/guest-input" --name "${NAME}") + if [[ "${S3_PUBLIC_UPLOAD_ENABLED}" == "true" ]]; then + s5cmd cp "guest-input/${NAME}_input.bin" "${S3_INPUT_BASE}.input.bin" + s5cmd cp "guest-input/${NAME}_expected_success.bin" "${S3_INPUT_BASE}.expected_success.bin" + fi + fi + INPUT_PATH="guest-input/${NAME}_input.bin" + EXPECTED_ARGS="--expected-output guest-input/${NAME}_expected_success.bin" + elif [[ "$INPUT_SOURCE" == s3://* ]]; then + s5cmd cp "$INPUT_SOURCE" guest-input/input.bin + INPUT_PATH="guest-input/input.bin" + elif [[ "$INPUT_SOURCE" == http://* || "$INPUT_SOURCE" == https://* ]]; then + curl -fL "$INPUT_SOURCE" -o guest-input/input.bin + INPUT_PATH="guest-input/input.bin" + else + echo "Unsupported input_source: $INPUT_SOURCE" >&2 + exit 1 + fi + MODE="${{ inputs.mode || github.event.inputs.mode }}" + if [[ "$MODE" != "execute" && "$MODE" != "execute-metered" ]]; then + # Public values are only surfaced by the execute modes. + EXPECTED_ARGS="" + fi + ls -la guest-input/ + echo "input_path=${INPUT_PATH}" >> "$GITHUB_OUTPUT" + echo "expected_args=${EXPECTED_ARGS}" >> "$GITHUB_OUTPUT" + + - name: Restore Host Binary from cache + id: cache-host-binary-restore + uses: runs-on/cache/restore@v4 + with: + path: target/${{ steps.set-build-profiles.outputs.host_profile }}/${{ env.BIN_NAME }} + key: ${{ steps.set-cache-keys.outputs.host_cache_key }} + + - name: Build Host Binary + if: steps.cache-host-binary-restore.outputs.cache-hit != 'true' + working-directory: bin/zilkworm-benchmark + run: | + export JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF} + HOST_PROFILE=${{ steps.set-build-profiles.outputs.host_profile }} + TOOLCHAIN="+${RUST_TOOLCHAIN}" + RUSTFLAGS=$RUSTFLAGS cargo $TOOLCHAIN build --bin ${{ env.BIN_NAME }} --profile=$HOST_PROFILE --no-default-features --features=$FEATURES + + - name: Save Host Binary to cache + if: steps.cache-host-binary-restore.outputs.cache-hit != 'true' + uses: runs-on/cache/save@v4 + with: + path: target/${{ steps.set-build-profiles.outputs.host_profile }}/${{ env.BIN_NAME }} + key: ${{ steps.cache-host-binary-restore.outputs.cache-primary-key }} + + - name: Set up run benchmark script + run: | + mkdir -p .bench_metrics + MODE=${{ inputs.mode || github.event.inputs.mode }} + HOST_PROFILE=${{ steps.set-build-profiles.outputs.host_profile }} + INPUT_PATH="${{ steps.prepare-input.outputs.input_path }}" + EXPECTED_ARGS="${{ steps.prepare-input.outputs.expected_args }}" + OPTIONAL_ARGS="" + APP_LOG_BLOWUP="${{ inputs.app_log_blowup || github.event.inputs.app_log_blowup }}" + LEAF_LOG_BLOWUP="${{ inputs.leaf_log_blowup || github.event.inputs.leaf_log_blowup }}" + INTERNAL_LOG_BLOWUP="${{ inputs.internal_log_blowup || github.event.inputs.internal_log_blowup }}" + SEGMENT_MAX_MEMORY="${{ inputs.segment_max_memory || github.event.inputs.segment_max_memory }}" + APP_L_SKIP="${{ inputs.app_l_skip || github.event.inputs.app_l_skip }}" + NUM_CHILDREN_LEAF="${{ inputs.num_children_leaf || github.event.inputs.num_children_leaf }}" + NUM_CHILDREN_INTERNAL="${{ inputs.num_children_internal || github.event.inputs.num_children_internal }}" + HELP_OUTPUT="$(COLUMNS=240 ./target/$HOST_PROFILE/${{ env.BIN_NAME }} --help)" + read_default() { + awk -v arg="$1" '$0 ~ arg { in_arg = 1; next } in_arg && /\[default:/ { gsub(/.*\[default: |].*/, ""); print; exit }' <<< "$HELP_OUTPUT" + } + DEFAULT_SEGMENT_MAX_MEMORY="$(read_default "--segment-max-memory ")" + if [[ -z "$DEFAULT_SEGMENT_MAX_MEMORY" ]]; then + echo "Failed to read segmentation defaults from ${{ env.BIN_NAME }} --help" >&2 + exit 1 + fi + RESOLVED_SEGMENT_MAX_MEMORY="${SEGMENT_MAX_MEMORY:-$DEFAULT_SEGMENT_MAX_MEMORY}" + echo "RESOLVED_SEGMENT_MAX_MEMORY=$RESOLVED_SEGMENT_MAX_MEMORY" >> "$GITHUB_ENV" + if [[ -n $APP_LOG_BLOWUP ]]; then + OPTIONAL_ARGS="$OPTIONAL_ARGS --app-log-blowup $APP_LOG_BLOWUP" + fi + if [[ -n $LEAF_LOG_BLOWUP ]]; then + OPTIONAL_ARGS="$OPTIONAL_ARGS --leaf-log-blowup $LEAF_LOG_BLOWUP" + fi + if [[ -n $INTERNAL_LOG_BLOWUP ]]; then + OPTIONAL_ARGS="$OPTIONAL_ARGS --internal-log-blowup $INTERNAL_LOG_BLOWUP" + fi + if [[ -n $SEGMENT_MAX_MEMORY ]]; then + OPTIONAL_ARGS="$OPTIONAL_ARGS --segment-max-memory $SEGMENT_MAX_MEMORY" + fi + if [[ -n $APP_L_SKIP ]]; then + OPTIONAL_ARGS="$OPTIONAL_ARGS --app-l-skip $APP_L_SKIP" + fi + if [[ -n $NUM_CHILDREN_LEAF ]]; then + OPTIONAL_ARGS="$OPTIONAL_ARGS --num-children-leaf $NUM_CHILDREN_LEAF" + fi + if [[ -n $NUM_CHILDREN_INTERNAL ]]; then + OPTIONAL_ARGS="$OPTIONAL_ARGS --num-children-internal $NUM_CHILDREN_INTERNAL" + fi + cat > run_benchmark.sh <> $GITHUB_ENV + elif [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "nsys" ]]; then + nsys profile --trace=cuda,osrt,nvtx \ + --sample=cpu \ + --cpuctxsw=process-tree \ + --cuda-memory-usage=true \ + --force-overwrite=true \ + -o zilkworm.nsys-rep \ + ./run_benchmark.sh + echo "=== CUDA GPU Kernel Summary ===" + nsys stats --report cuda_gpu_kern_sum zilkworm.nsys-rep + # nsys has its own memory tracking, set peak to 0 + echo "GPU_PEAK_MEMORY_MIB=0" >> $GITHUB_ENV + else + # Start GPU memory monitoring + start_gpu_monitor "$GPU_LOG_FILE" 5 + + set -o pipefail + ./ci/monitor_memory.sh ./run_benchmark.sh 2>&1 | tee benchmark_output.log + + # Finalize GPU monitoring and capture peak + finalize_gpu_monitor + echo "GPU_PEAK_MEMORY_MIB=${GPU_PEAK_MEMORY}" >> $GITHUB_ENV + echo "GPU_LOG_FILE=${GPU_LOG_FILE}" >> $GITHUB_ENV + + echo "MEM_USAGE_PATH=memory_usage.png" >> $GITHUB_ENV + fi + + - name: Report GPU peak memory + id: report-gpu-memory + run: | + echo "peak_memory_mib=${GPU_PEAK_MEMORY_MIB}" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_STEP_SUMMARY + echo "### GPU Memory" >> $GITHUB_STEP_SUMMARY + echo "Peak GPU memory usage: **$(awk "BEGIN{printf \"%.2f\",${GPU_PEAK_MEMORY_MIB}/1024}") GB**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + - name: Report segments + id: report-segments + run: | + NUM_SEGMENTS=$(jq '[.gauge[] | .labels[] | select(.[0] == "segment") | .[1]] | unique | length' "${METRIC_PATH}" 2>/dev/null || echo "") + echo "num_segments=${NUM_SEGMENTS}" >> $GITHUB_OUTPUT + if [[ -n "${NUM_SEGMENTS}" && "${NUM_SEGMENTS}" -gt 0 ]]; then + echo "### Segments" >> $GITHUB_STEP_SUMMARY + echo "Number of segments: **${NUM_SEGMENTS}**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi + + - name: Upload memory usage graph + if: ${{ (inputs.profiling || github.event.inputs.profiling) == 'none' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.set-metric-name.outputs.name }}-memory-usage + path: | + ${{ env.MEM_USAGE_PATH }} + ${{ env.GPU_LOG_FILE }} + retention-days: 1 + + - name: Upload nsys report + if: ${{ (inputs.profiling || github.event.inputs.profiling) == 'nsys' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.set-metric-name.outputs.name }}-nsys-report + path: ./zilkworm.nsys-rep + retention-days: 1 + + - name: Upload Benchmark Metrics + if: ${{ env.S3_WORKFLOW_UPLOAD_ENABLED == 'true' }} + run: s5cmd cp ${METRIC_PATH} ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}.json + + - name: Generate markdown + run: | + openvm-prof --json-paths $METRIC_PATH --num-devices 16 --proofs-per-device 1 + MD_PATH=${METRIC_PATH%.json}.md + DETAILED_MD_PATH=${MD_PATH%.md}.detailed.md + GPU_MEMORY_SVG_PATH=${MD_PATH%.md}.memory.svg + echo "MD_PATH=${MD_PATH}" >> $GITHUB_ENV + echo "DETAILED_MD_PATH=${DETAILED_MD_PATH}" >> $GITHUB_ENV + echo "GPU_MEMORY_SVG_PATH=${GPU_MEMORY_SVG_PATH}" >> $GITHUB_ENV + + - name: Add benchmark metadata to markdown + run: | + COMMIT_URL=https://github.com/${{ github.repository }}/commit/${current_sha} + BENCHMARK_WORKFLOW_URL=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + + source openvm/ci/scripts/utils.sh + + if [[ "${S3_PUBLIC_UPLOAD_ENABLED}" == "true" ]]; then + add_metadata $MD_PATH \ + ${{ inputs.instance_family || github.event.inputs.instance_family }} \ + ${{ inputs.memory_allocator || github.event.inputs.memory_allocator || 'jemalloc' }} \ + $COMMIT_URL \ + $BENCHMARK_WORKFLOW_URL + else + { + echo "" + echo "Commit: ${COMMIT_URL}" + echo "" + echo "Zilkworm commit: https://github.com/${ZILKWORM_REPO}/commit/${ZILKWORM_SHA}" + echo "" + echo "Instance Type: ${{ inputs.instance_family || github.event.inputs.instance_family }}" + echo "" + echo "Memory Allocator: ${{ inputs.memory_allocator || github.event.inputs.memory_allocator || 'jemalloc' }}" + echo "" + echo "[Benchmark Workflow](${BENCHMARK_WORKFLOW_URL})" + } >> "$MD_PATH" + fi + + # Add GPU peak memory to the markdown if available + if [[ -n "${GPU_PEAK_MEMORY_MIB}" && "${GPU_PEAK_MEMORY_MIB}" != "0" ]]; then + echo "" >> $MD_PATH + echo "**Peak GPU Memory (nvidia-smi):** $(awk "BEGIN{printf \"%.2f\",${GPU_PEAK_MEMORY_MIB}/1024}") GB" >> $MD_PATH + fi + + - name: Generate circuit flamegraphs + if: ${{ (inputs.profiling || github.event.inputs.profiling) == 'guest' }} + run: | + cargo install inferno + GUEST_SYMBOLS_PATH="${METRIC_PATH%.json}.syms" + if [[ -f $GUEST_SYMBOLS_PATH ]]; then + python3 openvm/ci/scripts/metric_unify/flamegraph.py $METRIC_PATH --guest-symbols $GUEST_SYMBOLS_PATH + else + python3 openvm/ci/scripts/metric_unify/flamegraph.py $METRIC_PATH + fi + + - name: Upload circuit flamegraphs + if: ${{ (inputs.profiling || github.event.inputs.profiling) == 'guest' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.set-metric-name.outputs.name }}-circuit-flamegraphs + path: .bench_metrics/flamegraphs/*.svg + retention-days: 3 + + - name: Generate samply profile artifacts + if: ${{ inputs.profiling == 'host' || github.event.inputs.profiling == 'host' }} + run: | + cargo install --git https://github.com/mstange/samply.git samply --force + + mkdir -p samply_profile + SAMPLY_PROFILE_PATH=samply_profile + + samply import perf.data --presymbolicate --save-only --output $SAMPLY_PROFILE_PATH/profile.json.gz + echo "Saved profile: $SAMPLY_PROFILE_PATH/profile.json.gz" + echo "SAMPLY_PROFILE_PATH=${SAMPLY_PROFILE_PATH}" >> $GITHUB_ENV + + - name: Publish samply profile + if: ${{ (inputs.profiling == 'host' || github.event.inputs.profiling == 'host') && env.S3_PUBLIC_UPLOAD_ENABLED == 'true' }} + run: | + s5cmd cp $SAMPLY_PROFILE_PATH/profile.json.gz ${{ env.S3_SAMPLY_PROFILE_PATH }}/${METRIC_NAME}/profile.json.gz + + FIREFOX_PROFILER_URL=$(python3 "$GITHUB_WORKSPACE/scripts/upload_firefox_profile.py" "$SAMPLY_PROFILE_PATH/profile.json.gz") || true + + if [ -n "$FIREFOX_PROFILER_URL" ]; then + echo "Firefox Profiler URL: $FIREFOX_PROFILER_URL" + echo "FIREFOX_PROFILER_URL=$FIREFOX_PROFILER_URL" >> $GITHUB_ENV + echo "" >> $MD_PATH + echo "**[Firefox Profiler]($FIREFOX_PROFILER_URL)**" >> $MD_PATH + else + echo "Warning: failed to upload profile to Firefox Profiler" + fi + + - name: Upload samply profile artifacts + if: ${{ inputs.profiling == 'host' || github.event.inputs.profiling == 'host' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.set-metric-name.outputs.name }}-samply-profile + path: ${{ env.SAMPLY_PROFILE_PATH }} + retention-days: 1 + + - name: Upload markdown + if: ${{ env.S3_WORKFLOW_UPLOAD_ENABLED == 'true' }} + run: | + S3_MD_PATH="${{ env.S3_MD_PATH }}/${METRIC_NAME}.md" + s5cmd cp $MD_PATH $S3_MD_PATH + + s5cmd cp $DETAILED_MD_PATH "${{ env.S3_MD_PATH }}/${METRIC_NAME}.detailed.md" + + echo "S3_MD_PATH=${S3_MD_PATH}" >> $GITHUB_ENV + + - name: Upload metric artifacts + id: upload-metric-artifact + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.set-metric-name.outputs.name }} + path: | + ${{ env.METRIC_PATH }} + ${{ env.MD_PATH }} + ${{ env.DETAILED_MD_PATH }} + ${{ env.GPU_MEMORY_SVG_PATH }} + if-no-files-found: warn + retention-days: 1 + + - name: sccache stats + if: always() + run: sccache --show-stats || true + + ### Update gh-pages + - name: Check for gh-pages branch + id: check-gh-pages + run: | + if git ls-remote --exit-code --heads origin gh-pages > /dev/null; then + echo "exists=true" >> "$GITHUB_OUTPUT" + echo "ref=gh-pages" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + echo "ref=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout gh-pages + uses: actions/checkout@v4 + with: + ref: ${{ steps.check-gh-pages.outputs.ref }} + path: gh-pages + + - name: Initialize gh-pages branch + if: ${{ steps.check-gh-pages.outputs.exists != 'true' }} + working-directory: gh-pages + run: | + git checkout --orphan gh-pages + git rm -rf . + + - name: Set up git + working-directory: gh-pages + run: | + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + + - name: Commit to gh-pages branch + id: publish-gh-pages + continue-on-error: true + working-directory: gh-pages + env: + REF_NAME: ${{ github.head_ref || github.ref }} + run: | + GH_PAGES_PATH="benchmarks-dispatch/${REF_NAME}" + echo "GH_PAGES_PATH=${GH_PAGES_PATH}" >> $GITHUB_ENV + mkdir -p ${GH_PAGES_PATH} + cp "$GITHUB_WORKSPACE/$MD_PATH" "${GH_PAGES_PATH}/${METRIC_NAME}.md" + + git add ${GH_PAGES_PATH}/${METRIC_NAME}.md + git commit --allow-empty -m "Update benchmark result at ${GH_PAGES_PATH}/${METRIC_NAME}.md" + MAX_RETRIES=10 + RETRY_DELAY=5 + ATTEMPT=0 + SUCCESS=false + while [ $ATTEMPT -lt $MAX_RETRIES ]; do + echo "Attempt $((ATTEMPT + 1)) to push of $MAX_RETRIES..." + if git fetch origin gh-pages; then + git merge origin/gh-pages --no-edit + fi + if git push origin HEAD:gh-pages; then + SUCCESS=true + break + else + echo "Push failed. Retrying in $RETRY_DELAY seconds..." + sleep $RETRY_DELAY + ATTEMPT=$((ATTEMPT + 1)) + fi + done + if [ "$SUCCESS" = false ]; then + echo "PUSH_FAILED" + exit 1 + fi + + - name: Update summary with results + if: ${{ steps.publish-gh-pages.outcome == 'success' }} + working-directory: gh-pages + run: | + RESULT_URL="https://github.com/${{ github.repository }}/blob/gh-pages/${GH_PAGES_PATH}/${METRIC_NAME}.md" + echo "### Results" >> $GITHUB_STEP_SUMMARY + echo "[${METRIC_NAME}](${RESULT_URL})" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + cat "${GH_PAGES_PATH}/${METRIC_NAME}.md" >> $GITHUB_STEP_SUMMARY + + - name: Path to result + if: ${{ steps.publish-gh-pages.outcome == 'success' }} + working-directory: gh-pages + run: | + echo "https://github.com/${{ github.repository }}/blob/gh-pages/${GH_PAGES_PATH}/${METRIC_NAME}.md" diff --git a/Cargo.lock b/Cargo.lock index 37971507d..d22a848d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7847,6 +7847,7 @@ dependencies = [ "tracing", "tracing-actix-web", "tracing-subscriber 0.3.23", + "url", ] [[package]] @@ -8212,6 +8213,26 @@ dependencies = [ "zstd", ] +[[package]] +name = "openvm-zilkworm-benchmark" +version = "0.4.0" +dependencies = [ + "clap", + "dotenv", + "eyre", + "hex", + "openvm-circuit", + "openvm-cuda-backend", + "openvm-sdk", + "openvm-sdk-config", + "openvm-stark-sdk", + "openvm-transpiler", + "openvm-verify-stark-host", + "tracing", + "tracing-subscriber 0.3.23", + "zstd", +] + [[package]] name = "ordered-float" version = "4.6.0" diff --git a/Cargo.toml b/Cargo.toml index 2135ceaaf..aa217fc5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "bin/reth-benchmark", "bin/reth-verify", + "bin/zilkworm-benchmark", "crates/mpt", "crates/revm-crypto", "crates/stateless-executor", diff --git a/bin/zilkworm-benchmark/Cargo.toml b/bin/zilkworm-benchmark/Cargo.toml new file mode 100644 index 000000000..3231e3b98 --- /dev/null +++ b/bin/zilkworm-benchmark/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "openvm-zilkworm-benchmark" +edition.workspace = true +repository.workspace = true +homepage.workspace = true +exclude.workspace = true +version.workspace = true + +[lib] +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +tracing-subscriber.workspace = true +tracing.workspace = true +dotenv = "0.15.0" +clap = { version = "4.5.7", features = ["derive", "env"] } +hex = "0.4.3" +eyre.workspace = true +zstd.workspace = true + +# Note: the openvm-sdk commit does not need to be exactly the same as the `openvm` commit used in the guest program: as long as +# the openvm-sdk commit doesn't change any guest libraries, they are compatible +openvm-stark-sdk = { git = "https://github.com/openvm-org/stark-backend.git", branch = "develop-v2.1.0", default-features = false } +openvm-cuda-backend = { git = "https://github.com/openvm-org/stark-backend.git", branch = "develop-v2.1.0", default-features = false, optional = true } +openvm-transpiler = { git = "https://github.com/openvm-org/openvm.git", branch = "develop-v2.1.0", default-features = false } +openvm-circuit = { git = "https://github.com/openvm-org/openvm.git", branch = "develop-v2.1.0", default-features = false } +openvm-verify-stark-host = { git = "https://github.com/openvm-org/openvm.git", branch = "develop-v2.1.0", default-features = false } +openvm-sdk = { git = "https://github.com/openvm-org/openvm.git", branch = "develop-v2.1.0", default-features = false } +openvm-sdk-config = { git = "https://github.com/openvm-org/openvm.git", branch = "develop-v2.1.0", default-features = false } + +[features] +default = ["parallel", "jemalloc", "metrics"] +parallel = ["openvm-sdk/parallel"] +metrics = ["openvm-sdk/metrics", "openvm-circuit/metrics", "openvm-stark-sdk/metrics"] +tco = ["openvm-sdk/tco"] +rvr = ["openvm-sdk/rvr"] +perf-metrics = ["openvm-sdk/perf-metrics"] +mimalloc = ["openvm-sdk/mimalloc"] +jemalloc = ["openvm-sdk/jemalloc"] +cuda = ["dep:openvm-cuda-backend", "openvm-sdk/cuda"] +unprotected = ["openvm-sdk/unprotected"] +nvtx = ["openvm-stark-sdk/nvtx"] diff --git a/bin/zilkworm-benchmark/README.md b/bin/zilkworm-benchmark/README.md new file mode 100644 index 000000000..87dee0b94 --- /dev/null +++ b/bin/zilkworm-benchmark/README.md @@ -0,0 +1,67 @@ +# OpenVM Zilkworm Benchmark + +Benchmark host for [Zilkworm](https://github.com/eth-act/zilkworm-stateless) +(mirrored at [axiom-crypto/zilkworm-stateless](https://github.com/axiom-crypto/zilkworm-stateless)), a bare-metal C++ +stateless Ethereum block validator compiled to an RV64IM ELF. The guest runs +on OpenVM's RV64 target (`rv64i + rv64m + io + keccak + sha2` extensions) and +goes through the same execute / prove pipeline as the Reth benchmark. + +## Guest + +Build the guest ELF from the `feat/openvm-rv64` branch of +`axiom-crypto/zilkworm-stateless` (requires cmake >= 3.28 and the xPack +`riscv-none-elf-gcc` toolchain, see that repo's README): + +```bash +git clone https://github.com/axiom-crypto/zilkworm-stateless.git +cd zilkworm-stateless +git checkout feat/openvm-rv64 +make guest_openvm # produces build/openvm/z6m_guest.elf +``` + +## Input + +The guest consumes one eth-act SSZ `StatelessInput` byte vector (the +`*_input.bin` format produced by the zilkworm `conformance` vector +generator) and reveals the SSZ `StatelessValidationResult` +(`root[32] || success[1] || offset[4] || chain_config`) as public values. + +Generate the synthetic mock vector (note: the mock block intentionally +*fails* validation — assert `mock_expected_failure.bin`): + +```bash +cd zilkworm-stateless/conformance +cargo run --release -- mock --out-dir vectors --name mock +``` + +EEST stateless fixtures can be converted with the same tool's `eest` +subcommand; real-block vectors with `real` (see `conformance/README.md`). + +## Run + +```bash +cargo run --bin openvm-zilkworm-benchmark --release -- \ + --mode execute \ + --guest-elf path/to/z6m_guest.elf \ + --input-path path/to/mock_input.bin \ + --expected-output path/to/mock_expected_failure.bin +``` + +Modes: `execute`, `execute-metered`, `prove-app`, `prove-stark`, `keygen`, +`generate-vm-vkey`. Proving parameters (`--app-log-blowup`, `--app-l-skip`, +`--leaf-log-blowup`, `--internal-log-blowup`, `--segment-max-memory`, +`--num-children-leaf`, `--num-children-internal`) match the Reth benchmark +host. + +Feature flags also mirror the Reth host: `rvr` (recompile the guest to +native C for execution; needs `clang-22` and `lld`), `tco`, `cuda`, +`jemalloc`/`mimalloc`, `metrics`, `perf-metrics`, `unprotected`, `nvtx`. + +## CI + +The `Zilkworm Benchmark` workflow +(`.github/workflows/zilkworm-benchmark.yml`) runs parallel to the Reth +Benchmark: it builds the guest ELF from `axiom-crypto/zilkworm-stateless` with the +pinned xPack toolchain, prepares the input (`mock` or an `s3://` / `https://` +URL to a `StatelessInput` .bin), and uploads metrics / markdown to S3 and +gh-pages using the same tooling. diff --git a/bin/zilkworm-benchmark/openvm.toml b/bin/zilkworm-benchmark/openvm.toml new file mode 100644 index 000000000..8d8e6d7ce --- /dev/null +++ b/bin/zilkworm-benchmark/openvm.toml @@ -0,0 +1,50 @@ +# VM extension config for the Zilkworm RV64IM guest. +# +# The order of supported_moduli / fp2 moduli / ecc curves is ABI: it defines +# the funct7 indices of the modular / Fp2 / short-Weierstrass custom +# instructions the guest emits. Keep in sync with zilkworm-stateless's +# zkvm/openvm/src/include/openvm_ecc.hpp (MOD_*/FP2_*/CURVE_* constants). + +[app_vm_config.rv64i] +[app_vm_config.rv64m] +[app_vm_config.io] + +[app_vm_config.keccak] +[app_vm_config.sha2] + +[app_vm_config.modular] +supported_moduli = [ + # 0: secp256k1 coordinate field + "115792089237316195423570985008687907853269984665640564039457584007908834671663", + # 1: secp256k1 scalar field + "115792089237316195423570985008687907852837564279074904382605163141518161494337", + # 2: bn254 (alt bn128) coordinate field + "21888242871839275222246405745257275088696311157297823662689037894645226208583", + # 3: bn254 scalar field + "21888242871839275222246405745257275088548364400416034343698204186575808495617", +] + +[app_vm_config.fp2] +supported_moduli = [ + # 0: bn254 Fp2 + [ + "Bn254Fp2", + "21888242871839275222246405745257275088696311157297823662689037894645226208583", + ], +] + +# 0: secp256k1 +[[app_vm_config.ecc.supported_curves]] +struct_name = "Secp256k1Point" +modulus = "115792089237316195423570985008687907853269984665640564039457584007908834671663" +scalar = "115792089237316195423570985008687907852837564279074904382605163141518161494337" +a = "0" +b = "7" + +# 1: bn254 G1 +[[app_vm_config.ecc.supported_curves]] +struct_name = "Bn254G1Affine" +modulus = "21888242871839275222246405745257275088696311157297823662689037894645226208583" +scalar = "21888242871839275222246405745257275088548364400416034343698204186575808495617" +a = "0" +b = "3" diff --git a/bin/zilkworm-benchmark/src/lib.rs b/bin/zilkworm-benchmark/src/lib.rs new file mode 100644 index 000000000..c3602e75f --- /dev/null +++ b/bin/zilkworm-benchmark/src/lib.rs @@ -0,0 +1,429 @@ +//! Benchmark host for the Zilkworm stateless Ethereum validator guest. +//! +//! Zilkworm (github.com/eth-act/zilkworm-stateless) is a bare-metal C++ +//! stateless block validator compiled to an RV64IM ELF. This host loads the +//! prebuilt guest ELF, feeds it an eth-act SSZ `StatelessInput` byte vector +//! (an EEST / conformance `*_input.bin`), and runs it through the same +//! execute / prove pipeline as the Reth benchmark. +//! +//! The guest reads exactly one length-prefixed input vector from the hint +//! stream and reveals the SSZ `StatelessValidationResult` +//! (`root[32] || success[1] || offset[4] || chain_config`) as public values. + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use clap::Parser; +use eyre::Result; +use openvm_circuit::arch::{ + execution_mode::metered::segment_ctx::DEFAULT_MAX_MEMORY, instructions::exe::VmExe, + verify_segments, VmCircuitConfig, +}; +use openvm_sdk::{ + config::{ + AggregationSystemParams, AggregationTreeConfig, AppConfig, DEFAULT_APP_LOG_BLOWUP, + DEFAULT_APP_L_SKIP, DEFAULT_INTERNAL_LOG_BLOWUP, DEFAULT_LEAF_LOG_BLOWUP, + }, + fs::{read_object_from_file, write_object_to_file}, + Sdk, StdIn, SC, +}; +use openvm_sdk_config::{SdkVmConfig, TranspilerConfig}; +use openvm_stark_sdk::{ + bench::run_with_metric_collection, + config::{ + app_params_with_100_bits_security, baby_bear_poseidon2::F, + internal_params_with_100_bits_security, leaf_params_with_100_bits_security, + MAX_APP_LOG_STACKED_HEIGHT, SECURITY_BITS_TARGET, + }, + openvm_stark_backend::{codec::Encode, SystemParams}, +}; +use openvm_transpiler::{elf::Elf, openvm_platform::memory::MEM_SIZE, FromElf}; +use openvm_verify_stark_host::{ + verify_vm_stark_proof_decoded, + vk::{write_vk_to_file, VmStarkVerifyingKey}, +}; +use tracing::{info, info_span}; + +const VM_MAX_CONSTRAINT_DEGREE: usize = 4; + +/// Capacity of the user public-values address space in bytes. Must be at +/// least `z6m::MAX_PUBLIC_VALUES_SIZE` (256) so the guest's largest SSZ +/// `StatelessValidationResult` fits. +const PUBLIC_VALUES_BYTES: usize = 256; + +/// Enum representing the execution mode of the host executable. +#[derive(Debug, Clone, clap::ValueEnum)] +pub enum BenchMode { + /// Execute the VM without generating a proof. + Execute, + /// Execute the VM with metering to get segments information. + ExecuteMetered, + /// Generate sequence of app proofs for continuation segments. + ProveApp, + /// Generate a full end-to-end STARK proof with aggregation. + ProveStark, + /// Generate proving and verifying keys for app and aggregation circuits. + Keygen, + /// Generate VM verifying key baseline artifact and write it to a local file. + GenerateVmVkey, +} + +impl std::fmt::Display for BenchMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Execute => write!(f, "execute"), + Self::ExecuteMetered => write!(f, "execute_metered"), + Self::ProveApp => write!(f, "prove_app"), + Self::ProveStark => write!(f, "prove_stark"), + Self::Keygen => write!(f, "keygen"), + Self::GenerateVmVkey => write!(f, "generate_vm_vkey"), + } + } +} + +#[derive(Parser, Debug)] +#[command(allow_external_subcommands = true)] +pub struct BenchmarkCli { + /// Application level log blowup + #[arg(long, default_value_t = DEFAULT_APP_LOG_BLOWUP)] + pub app_log_blowup: usize, + + /// Log of univariate skip domain size + #[arg(long, default_value_t = DEFAULT_APP_L_SKIP)] + pub app_l_skip: usize, + + /// Aggregation (leaf) level log blowup + #[arg(long, default_value_t = DEFAULT_LEAF_LOG_BLOWUP)] + pub leaf_log_blowup: usize, + + /// Internal level log blowup + #[arg(long, default_value_t = DEFAULT_INTERNAL_LOG_BLOWUP)] + pub internal_log_blowup: usize, + + #[command(flatten)] + pub agg_tree_config: AggregationTreeConfig, + + /// Estimated proving-memory cap per VM segment, in bytes + #[arg(long, default_value_t = DEFAULT_MAX_MEMORY)] + pub segment_max_memory: usize, +} + +/// The arguments for the host executable. +#[derive(Debug, Parser)] +pub struct HostArgs { + /// The execution mode. + #[clap(long, value_enum)] + mode: BenchMode, + + /// Path to the prebuilt Zilkworm RV64IM guest ELF (z6m_guest.elf). + #[clap(long, env = "ZILKWORM_GUEST_ELF")] + guest_elf: PathBuf, + + /// Path to the guest input: raw eth-act SSZ `StatelessInput` bytes + /// (a conformance `*_input.bin` / EEST `*.input.bin` file). + /// Required for all modes except keygen / generate-vm-vkey. + #[clap(long)] + input_path: Option, + + /// Optional path to the expected public values (an SSZ + /// `StatelessValidationResult`, e.g. a conformance + /// `*_expected_success.bin`). When provided, execution output is + /// checked against it. + #[clap(long)] + expected_output: Option, + + /// Label used for metric grouping; defaults to the input file stem. + #[clap(long)] + program_label: Option, + + #[clap(flatten)] + benchmark: BenchmarkCli, + + /// If specified, the proof and other output is written to this dir. + #[arg(long, default_value = "output")] + pub output_dir: PathBuf, + + /// If specified, loads the app proving key from this path. + #[arg(long)] + pub app_pk_path: Option, + + /// Path to save the app verifying key (overrides output_dir) + #[arg(long)] + pub app_vk_path: Option, + + /// If specified, loads the agg proving key from this path. + #[arg(long)] + pub agg_pk_path: Option, +} + +/// VM extension config matching the instructions the Zilkworm guest issues: +/// RV64IM + hint/reveal IO, the keccak-f[1600] and SHA-256 accelerators, and +/// the modular / Fp2 / short-Weierstrass extensions used by the guest's ECC +/// precompile hooks (secp256k1 ecrecover, bn254 field + curve ops). The +/// moduli/curve order in openvm.toml is ABI — it must match the funct7 +/// indices in zilkworm's zkvm/openvm/src/include/openvm_ecc.hpp. +pub fn zilkworm_vm_config() -> SdkVmConfig { + let mut config = SdkVmConfig::from_toml(include_str!("../openvm.toml")) + .expect("invalid embedded openvm.toml") + .optimize(); + config.system.config = config + .system + .config + .clone() + .with_max_constraint_degree(VM_MAX_CONSTRAINT_DEGREE) + .with_public_values_bytes(PUBLIC_VALUES_BYTES); + config +} + +pub fn build_zilkworm_exe(vm_config: &SdkVmConfig, guest_elf: &[u8]) -> Result> { + let transpiler = vm_config.transpiler().clone(); + let elf = Elf::decode(guest_elf, MEM_SIZE as u32)?; + Ok(VmExe::from_elf(elf, transpiler)?) +} + +fn override_system_params( + params: SystemParams, + log_blowup: usize, + l_skip: usize, +) -> Result { + let log_stacked_height = params.log_stacked_height(); + if l_skip > log_stacked_height { + eyre::bail!("l_skip ({l_skip}) must be <= log_stacked_height ({log_stacked_height})"); + } + + let whir = params.whir().clone(); + Ok(SystemParams::new( + log_blowup, + l_skip, + log_stacked_height - l_skip, + params.w_stack, + whir.log_final_poly_len(log_stacked_height), + whir.folding_pow_bits, + whir.mu_pow_bits, + whir.proximity, + SECURITY_BITS_TARGET, + params.logup, + params.max_constraint_degree, + whir.query_phase_pow_bits, + whir.k, + )) +} + +fn override_log_blowup(params: SystemParams, log_blowup: usize) -> Result { + let l_skip = params.l_skip; + override_system_params(params, log_blowup, l_skip) +} + +/// Check the guest's revealed public values against an expected SSZ +/// `StatelessValidationResult`. The public-values buffer is fixed-size and +/// zero-padded past the SSZ payload, so compare the expected prefix and +/// require the tail to be zero. +fn check_expected_output(public_values: &[u8], expected_path: &Path) -> Result<()> { + let expected = fs::read(expected_path)?; + if expected.len() > public_values.len() { + eyre::bail!( + "expected output ({} bytes) exceeds public values capacity ({} bytes)", + expected.len(), + public_values.len() + ); + } + if public_values[..expected.len()] != expected[..] { + eyre::bail!( + "public values mismatch:\n expected: {}\n actual: {}", + hex::encode(&expected), + hex::encode(&public_values[..expected.len()]) + ); + } + if public_values[expected.len()..].iter().any(|&b| b != 0) { + eyre::bail!("public values have non-zero bytes past the expected output"); + } + info!("Public values match expected output ({} bytes)", expected.len()); + Ok(()) +} + +fn report_public_values(public_values: &[u8]) { + // SSZ StatelessValidationResult: root[32] || success[1] || offset[4] || chain_config + if public_values.len() >= 33 { + let root = hex::encode(&public_values[..32]); + let success = public_values[32] == 1; + println!("BENCH_STATE_ROOT=0x{root}"); + println!("BENCH_SUCCESS={success}"); + } + println!("BENCH_PUBLIC_VALUES={}", hex::encode(public_values)); +} + +pub fn run_zilkworm_benchmark(args: HostArgs) -> Result<()> { + // Initialize the environment variables. + dotenv::dotenv().ok(); + + #[cfg(feature = "cuda")] + eprintln!("CUDA Backend Enabled"); + + let mut vm_config = zilkworm_vm_config(); + vm_config.as_mut().set_segmentation_max_memory(args.benchmark.segment_max_memory); + + for (air_idx, air) in VmCircuitConfig::::create_airs(&vm_config)?.into_airs().enumerate() { + tracing::debug!("air_idx={air_idx} | {}", air.name()); + } + + let app_params = override_system_params( + app_params_with_100_bits_security(MAX_APP_LOG_STACKED_HEIGHT), + args.benchmark.app_log_blowup, + args.benchmark.app_l_skip, + )?; + + // Setup: this can all be done once before receiving proof input + let app_config = AppConfig::new(vm_config.clone(), app_params); + let agg_params = AggregationSystemParams { + leaf: override_log_blowup( + leaf_params_with_100_bits_security(), + args.benchmark.leaf_log_blowup, + )?, + internal: override_log_blowup( + internal_params_with_100_bits_security(), + args.benchmark.internal_log_blowup, + )?, + }; + + // Resolve key paths: explicit flag wins; otherwise fall back to /.pk + // if the file exists. + let app_pk_path = args.app_pk_path.clone().or_else(|| { + let p = args.output_dir.join("app.pk"); + p.exists().then_some(p) + }); + let agg_pk_path = args.agg_pk_path.clone().or_else(|| { + let p = args.output_dir.join("agg.pk"); + p.exists().then_some(p) + }); + + let mut sdk_builder = Sdk::builder().agg_tree_config(args.benchmark.agg_tree_config); + + if let Some(p) = app_pk_path { + info!("Loading app proving key from {}", p.display()); + let app_pk = read_object_from_file(&p)?; + sdk_builder = sdk_builder.app_pk(app_pk); + } else { + sdk_builder = sdk_builder.app_config(app_config); + } + + if let Some(p) = agg_pk_path { + info!("Loading agg proving key from {}", p.display()); + let agg_pk = read_object_from_file(&p)?; + sdk_builder = sdk_builder.agg_pk(agg_pk); + } else { + sdk_builder = sdk_builder.agg_params(agg_params); + } + + let sdk = sdk_builder.build()?; + + let guest_elf = fs::read(&args.guest_elf)?; + let exe = build_zilkworm_exe(&vm_config, &guest_elf)?; + + if matches!(args.mode, BenchMode::GenerateVmVkey) { + let prover = sdk.prover(exe)?; + let vk = VmStarkVerifyingKey { + mvk: (*sdk.agg_vk()).clone(), + baseline: prover.generate_baseline(), + }; + let vk_path = PathBuf::from("zilkworm.vm.vk"); + write_vk_to_file(&vk_path, &vk)?; + info!("VM verifying key written to {}", vk_path.display()); + return Ok(()); + } + + if matches!(args.mode, BenchMode::Keygen) { + fs::create_dir_all(&args.output_dir)?; + + let app_pk_path = args.app_pk_path.unwrap_or_else(|| args.output_dir.join("app.pk")); + let app_vk_path = args.app_vk_path.unwrap_or_else(|| args.output_dir.join("app.vk")); + let agg_pk_path = args.agg_pk_path.unwrap_or_else(|| args.output_dir.join("agg.pk")); + + info!("Generating app proving key..."); + let (app_pk, app_vk) = sdk.app_keygen(); + + info!("Saving app proving key to: {}", app_pk_path.display()); + write_object_to_file(&app_pk_path, &app_pk)?; + + info!("Saving app verifying key to: {}", app_vk_path.display()); + write_object_to_file(&app_vk_path, &app_vk)?; + + info!("Generating aggregation proving key..."); + let agg_pk = sdk.agg_pk(); + + info!("Saving agg proving key to: {}", agg_pk_path.display()); + write_object_to_file(&agg_pk_path, &agg_pk)?; + + info!("Keygen completed successfully!"); + return Ok(()); + } + + let input_path = args + .input_path + .as_ref() + .ok_or_else(|| eyre::eyre!("--input-path is required for mode {}", args.mode))?; + let input_bytes = fs::read(input_path)?; + info!("Loaded {} input bytes from {}", input_bytes.len(), input_path.display()); + + let label = args.program_label.clone().unwrap_or_else(|| { + input_path + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| "input".to_string()) + }); + let program_name = format!("zilkworm.{}.{}", args.mode, label); + + let stdin: StdIn = vec![input_bytes].into(); + + run_with_metric_collection("OUTPUT_PATH", move || { + info_span!("zilkworm-block", label = label).in_scope(|| -> Result<()> { + match args.mode { + BenchMode::Execute => { + let compiled = sdk.compile(exe)?; + let public_values = sdk.execute(&compiled, stdin)?; + info!("Execute completed"); + report_public_values(&public_values); + if let Some(expected) = &args.expected_output { + check_expected_output(&public_values, expected)?; + } + } + BenchMode::ExecuteMetered => { + let compiled = sdk.compile_metered(exe)?; + let (public_values, segments) = sdk.execute_metered(&compiled, stdin)?; + info!("Execute metered completed, {} segments", segments.len()); + report_public_values(&public_values); + if let Some(expected) = &args.expected_output { + check_expected_output(&public_values, expected)?; + } + } + BenchMode::ProveApp => { + let mut prover = sdk.app_prover(exe)?; + prover.set_program_name(program_name); + let app_proof = prover.prove(stdin)?; + let (_, app_vk) = sdk.app_keygen(); + verify_segments(&prover.vm().engine, &app_vk.vk, &app_proof.per_segment)?; + } + BenchMode::ProveStark => { + let (proof, baseline) = sdk.prove(exe, stdin, &[])?; + let vk = VmStarkVerifyingKey { mvk: (*sdk.agg_vk()).clone(), baseline }; + let encoded = proof.encode_to_vec()?; + let compressed = zstd::encode_all(&encoded[..], 19)?; + tracing::info!( + "Proof Size (bytes): {}, Compressed Size: {}", + encoded.len(), + compressed.len() + ); + verify_vm_stark_proof_decoded(&vk, &proof)?; + } + _ => { + // Keygen, GenerateVmVkey handled earlier + unreachable!(); + } + } + + Ok(()) + }) + })?; + Ok(()) +} diff --git a/bin/zilkworm-benchmark/src/main.rs b/bin/zilkworm-benchmark/src/main.rs new file mode 100644 index 000000000..cd62a6df1 --- /dev/null +++ b/bin/zilkworm-benchmark/src/main.rs @@ -0,0 +1,7 @@ +use clap::Parser; +use openvm_zilkworm_benchmark::{run_zilkworm_benchmark, HostArgs}; + +fn main() -> eyre::Result<()> { + let args = HostArgs::parse(); + run_zilkworm_benchmark(args) +} diff --git a/crates/rpc-proxy/Cargo.toml b/crates/rpc-proxy/Cargo.toml index 74eab098a..22000f19c 100644 --- a/crates/rpc-proxy/Cargo.toml +++ b/crates/rpc-proxy/Cargo.toml @@ -31,6 +31,7 @@ revm = { workspace = true, features = ["c-kzg", "blst"] } risc0-ethereum-trie = { git = "https://github.com/risc0/risc0-ethereum.git", tag = "v3.0.1", features = ["orphan"] } clap = { workspace = true, features = ["derive", "env"] } +url.workspace = true itertools = { workspace = true } rayon.workspace = true eyre.workspace = true diff --git a/crates/rpc-proxy/src/bin/zilkworm-input.rs b/crates/rpc-proxy/src/bin/zilkworm-input.rs new file mode 100644 index 000000000..e1198d822 --- /dev/null +++ b/crates/rpc-proxy/src/bin/zilkworm-input.rs @@ -0,0 +1,121 @@ +//! Generates the Zilkworm (eth-act) guest-input ingredients for a mainnet +//! block: a `debug_executionWitness`-style witness JSON (built through the +//! same preflight pipeline the Reth benchmark uses) and a payload JSON with +//! the raw block data. Both files are consumed by zilkworm-stateless's +//! `conformance reth` subcommand, which assembles the SSZ `StatelessInput`. + +use std::{fs, path::PathBuf, sync::Arc}; + +use alloy::eips::eip2718::Encodable2718; +use alloy_consensus::{transaction::Transaction as _, TxEnvelope}; +use alloy_provider::{network::Ethereum, Provider, RootProvider}; +use alloy_rpc_types::BlockNumberOrTag; +use clap::Parser; +use eyre::OptionExt; +use openvm_rpc_proxy::{execution_witness, PreimageLookup, DEFAULT_PREIMAGE_CACHE_NIBBLES}; +use reth_chainspec::MAINNET; +use reth_evm_ethereum::EthEvmConfig; +use serde_json::json; + +#[derive(Parser, Debug)] +struct Args { + /// The RPC URL used to fetch block data and build the witness. + #[clap(long, env = "RPC_1")] + rpc_url: url::Url, + + /// The block number to generate input for. + #[clap(long)] + block_number: u64, + + /// Output directory for payload.json / witness.json. + #[clap(long, default_value = ".")] + out_dir: PathBuf, + + /// Number of nibbles to precompute for the preimage lookup table. + #[clap(long, default_value_t = DEFAULT_PREIMAGE_CACHE_NIBBLES, value_parser = clap::value_parser!(u8).range(..=8))] + preimage_cache_nibbles: u8, +} + +#[tokio::main] +async fn main() -> eyre::Result<()> { + tracing_subscriber::fmt::init(); + let args = Args::parse(); + + let provider: RootProvider = RootProvider::new_http(args.rpc_url.clone()); + let block_id = BlockNumberOrTag::Number(args.block_number); + + // Witness via the same preflight pipeline the Reth benchmark uses; the + // result has the debug_executionWitness wire shape (state/codes/keys/ + // headers hex arrays) that zilkworm's `conformance reth` ingests. + let evm_config = Arc::new(EthEvmConfig::ethereum(MAINNET.clone())); + let lookup = Arc::new(PreimageLookup::new(args.preimage_cache_nibbles)); + let witness = execution_witness(evm_config, &provider, block_id, &lookup).await?; + + let block = + provider.get_block_by_number(block_id).full().await?.ok_or_eyre("block not found")?; + let header = &block.header; + + // Raw 2718-encoded transactions in payload order, and the concatenated + // blob versioned hashes (newPayload's versioned_hashes parameter). + let mut transactions = Vec::new(); + let mut versioned_hashes = Vec::new(); + for tx in block.transactions.clone().into_transactions() { + let envelope = TxEnvelope::from(tx); + if let Some(hashes) = envelope.blob_versioned_hashes() { + versioned_hashes.extend(hashes.iter().map(|h| format!("{h:#x}"))); + } + transactions.push(format!("0x{}", hex_encode(envelope.encoded_2718()))); + } + + let withdrawals = block + .withdrawals + .as_ref() + .map(|ws| { + ws.iter() + .map(|w| { + json!({ + "index": w.index, + "validatorIndex": w.validator_index, + "address": format!("{:#x}", w.address), + "amount": w.amount, + }) + }) + .collect::>() + }) + .unwrap_or_default(); + + let payload = json!({ + "parentHash": format!("{:#x}", header.parent_hash), + "feeRecipient": format!("{:#x}", header.beneficiary), + "stateRoot": format!("{:#x}", header.state_root), + "receiptsRoot": format!("{:#x}", header.receipts_root), + "logsBloom": format!("{:#x}", header.logs_bloom), + "prevRandao": format!("{:#x}", header.mix_hash), + "blockNumber": header.number, + "gasLimit": header.gas_limit, + "gasUsed": header.gas_used, + "timestamp": header.timestamp, + "extraData": format!("{:#x}", header.extra_data), + "baseFeePerGas": format!("{:#x}", header.base_fee_per_gas.ok_or_eyre("missing base fee")?), + "blockHash": format!("{:#x}", header.hash), + "transactions": transactions, + "withdrawals": withdrawals, + "blobGasUsed": header.blob_gas_used.ok_or_eyre("missing blobGasUsed")?, + "excessBlobGas": header.excess_blob_gas.ok_or_eyre("missing excessBlobGas")?, + "versionedHashes": versioned_hashes, + "parentBeaconBlockRoot": format!("{:#x}", header.parent_beacon_block_root.ok_or_eyre("missing parentBeaconBlockRoot")?), + }); + + fs::create_dir_all(&args.out_dir)?; + let payload_path = args.out_dir.join("payload.json"); + let witness_path = args.out_dir.join("witness.json"); + fs::write(&payload_path, serde_json::to_string(&payload)?)?; + fs::write(&witness_path, serde_json::to_string(&witness)?)?; + println!("payload: {}", payload_path.display()); + println!("witness: {}", witness_path.display()); + Ok(()) +} + +fn hex_encode(bytes: Vec) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect::>().join("") +}