Reth Benchmark #114
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Reth Benchmark" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| # Default runner is not big enough for this | |
| # https://aws.amazon.com/ec2/instance-types/ | |
| instance_family: | |
| type: string | |
| required: false | |
| description: The family of the instance, can be multiple ones concat with "+" e.g. r8g+r7g | |
| default: m7a.48xlarge | |
| memory_allocator: | |
| type: string | |
| required: false | |
| description: Memory allocator to use (mimalloc or jemalloc) | |
| default: mimalloc | |
| block_number: | |
| type: number | |
| required: false | |
| description: Block number to run the benchmark on | |
| default: 21000000 | |
| app_log_blowup: | |
| type: number | |
| required: false | |
| description: Application level log blowup | |
| default: 2 | |
| agg_log_blowup: | |
| type: number | |
| required: false | |
| description: Aggregation (leaf) level log blowup | |
| default: 2 | |
| root_log_blowup: | |
| type: number | |
| required: false | |
| description: Root level log blowup (only for e2e) | |
| default: 3 | |
| internal_log_blowup: | |
| type: number | |
| required: false | |
| description: Internal level log blowup (only for e2e) | |
| default: 2 | |
| mode: | |
| type: choice | |
| required: false | |
| description: Running mode, one of execute, prove, or prove-e2e | |
| options: | |
| - execute | |
| - prove | |
| - prove-e2e | |
| default: prove | |
| collect_metrics: | |
| type: boolean | |
| required: false | |
| description: Whether to collect metrics for flamegraphs | |
| default: false | |
| max_segment_length: | |
| type: number | |
| required: false | |
| description: Max segment length for continuations, must be larger than 524288 | |
| default: 8388508 # 2**23 - 100 | |
| env: | |
| S3_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/results | |
| S3_METRICS_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/metrics | |
| PUBLIC_S3_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/flamegraphs | |
| FEATURE_FLAGS: "bench-metrics,parallel,nightly-features" | |
| CMD_ARGS: "" | |
| INPUT_ARGS: "" | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| jobs: | |
| run-reth: | |
| name: Run Reth benchmark | |
| runs-on: | |
| - runs-on | |
| - run-id=${{ github.run_id }} | |
| - family=${{ inputs.instance_family }} | |
| - disk=large | |
| - tag=bench-reth-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref || github.ref }} | |
| - uses: dtolnay/rust-toolchain@nightly | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| cache-on-failure: true | |
| - name: Display workflow inputs | |
| run: echo "${{ toJSON(github.event.inputs) }}" | |
| # - name: Load SSH key | |
| # uses: webfactory/[email protected] | |
| # with: | |
| # ssh-private-key: | | |
| # ${{ secrets.GH_ACTIONS_DEPLOY_PRIVATE_KEY }} | |
| - name: Install cargo-openvm | |
| run: | | |
| cargo install --git https://github.com/openvm-org/openvm.git cargo-openvm | |
| - name: Install openvm-prof | |
| run: | | |
| cargo install --git https://github.com/openvm-org/openvm.git --profile=dev openvm-prof | |
| - name: Checkout openvm (for scripts) | |
| run: | | |
| git clone https://github.com/openvm-org/openvm.git | |
| - name: Install architecture specific tools | |
| run: | | |
| rustup install nightly-2024-10-30 | |
| source openvm/ci/scripts/utils.sh | |
| install_s5cmd | |
| - name: Setup halo2 | |
| if: ${{ github.event.inputs.mode == 'prove-e2e' }} | |
| run: | | |
| bash openvm/extensions/native/recursion/trusted_setup_s3.sh | |
| export PARAMS_DIR=$(pwd)/params | |
| - name: Set metric name | |
| run: | | |
| input_json_str="${{ toJSON(github.event.inputs) }}" | |
| input_hash=$(echo $input_json_str | sha256sum | cut -d' ' -f1) | |
| METRIC_NAME=reth-${input_hash} | |
| echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV | |
| METRIC_PATH=".bench_metrics/${METRIC_NAME}.json" | |
| echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV | |
| - name: Build ELF | |
| working-directory: bin/client-eth | |
| run: | | |
| PROFILE="release" | |
| if [[ "${{ inputs.collect_metrics }}" == "true" ]]; then | |
| PROFILE="profiling" | |
| fi | |
| cargo openvm build --no-transpile --profile=$PROFILE | |
| mkdir -p ../host/elf | |
| cp target/riscv32im-risc0-zkvm-elf/release/openvm-client-eth ../host/elf/ | |
| - name: Run Reth | |
| run: | | |
| mkdir -p rpc-cache | |
| mkdir -p .bench_metrics # TODO: remove this | |
| RPC_1=${{ secrets.RPC_URL_1 }} | |
| MODE=${{ inputs.mode }} | |
| BLOCK_NUMBER=${{ inputs.block_number }} | |
| OPTIONAL_ARGS="" | |
| PROFILE="release" | |
| FEATURES="bench-metrics,${{ inputs.memory_allocator }}" | |
| if [[ "${{ inputs.collect_metrics }}" == "true" ]]; then | |
| OPTIONAL_ARGS="--collect-metrics" | |
| PROFILE="profiling" | |
| FEATURES="${FEATURES},profiling" | |
| fi | |
| arch=$(uname -m) | |
| case $arch in | |
| arm64|aarch64) | |
| RUSTFLAGS="-Ctarget-cpu=native" | |
| ;; | |
| x86_64|amd64) | |
| RUSTFLAGS="-Ctarget-cpu=native -C target-feature=+avx512f" | |
| ;; | |
| *) | |
| echo "Unsupported architecture: $arch" | |
| exit 1 | |
| ;; | |
| esac | |
| RUSTFLAGS=$RUSTFLAGS cargo build --bin openvm-reth-benchmark --profile=$PROFILE --no-default-features --features=$FEATURES | |
| OUTPUT_PATH=${METRIC_PATH} ./target/$PROFILE/openvm-reth-benchmark \ | |
| --$MODE --block-number $BLOCK_NUMBER --rpc-url $RPC_1 --cache-dir rpc-cache \ | |
| --app-log-blowup ${{ inputs.app_log_blowup }} \ | |
| --agg-log-blowup ${{ inputs.agg_log_blowup }} \ | |
| --root-log-blowup ${{ inputs.root_log_blowup }} \ | |
| --max-segment-length ${{ inputs.max_segment_length }} $OPTIONAL_ARGS | |
| - name: Upload Benchmark Metrics | |
| run: | | |
| current_sha=$(git rev-parse HEAD) | |
| echo "Current SHA: $current_sha" | |
| echo "current_sha=${current_sha}" >> $GITHUB_ENV | |
| s5cmd cp ${METRIC_PATH} ${{ env.S3_METRICS_PATH }}/${current_sha}-${METRIC_NAME}.json | |
| - name: Generate markdown # result path is hardcoded results.md | |
| run: | | |
| openvm-prof --json-paths $METRIC_PATH | |
| MD_PATH=${METRIC_PATH%.json}.md | |
| echo "MD_PATH=${MD_PATH}" >> $GITHUB_ENV | |
| # - name: Install inferno-flamegraph | |
| # run: cargo install inferno | |
| # - name: Generate flamegraphs | |
| # run: | | |
| # if [[ -f $METRIC_PATH ]]; then | |
| # python3 openvm/ci/scripts/metric_unify/flamegraph.py $METRIC_PATH | |
| # s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.PUBLIC_S3_PATH }}/${current_sha}/" | |
| # echo "UPLOAD_FLAMEGRAPHS=1" >> $GITHUB_ENV | |
| # fi | |
| - name: Add benchmark metadata to markdown, and upload | |
| run: | | |
| COMMIT_URL=https://github.com/${{ github.repository }}/commit/${current_sha} | |
| BENCHMARK_WORKFLOW_URL=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| source openvm/ci/scripts/utils.sh | |
| add_metadata $MD_PATH \ | |
| ${{ inputs.max_segment_length }} \ | |
| ${{ inputs.instance_family }} \ | |
| ${{ inputs.memory_allocator }} \ | |
| $COMMIT_URL \ | |
| $BENCHMARK_WORKFLOW_URL | |
| - name: Upload markdown | |
| run: | | |
| s5cmd cp $MD_PATH "${{ env.S3_PATH }}/${current_sha}-${METRIC_NAME}.md" | |
| ### Update gh-pages | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| - name: Set up git | |
| run: | | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --global user.name "github-actions[bot]" | |
| - name: Commit to gh-pages branch | |
| run: | | |
| GH_PAGES_PATH="benchmarks-dispatch/${{ github.head_ref || github.ref }}" | |
| echo "GH_PAGES_PATH=${GH_PAGES_PATH}" >> $GITHUB_ENV | |
| mkdir -p ${GH_PAGES_PATH} | |
| s5cmd cp "${{ env.S3_PATH }}/${current_sha}-${METRIC_NAME}.md" "${GH_PAGES_PATH}/${METRIC_NAME}.md" | |
| git add ${GH_PAGES_PATH}/${METRIC_NAME}.md | |
| git commit --allow-empty -m "Update benchmark result at ${GH_PAGES_PATH}/${METRIC_NAME}.md" | |
| MAX_RETRIES=10 | |
| RETRY_DELAY=5 | |
| ATTEMPT=0 | |
| SUCCESS=false | |
| while [ $ATTEMPT -lt $MAX_RETRIES ]; do | |
| echo "Attempt $((ATTEMPT + 1)) to push of $MAX_RETRIES..." | |
| git fetch origin gh-pages | |
| git merge origin/gh-pages --no-edit | |
| if git push origin gh-pages; then | |
| SUCCESS=true | |
| break | |
| else | |
| echo "Push failed. Retrying in $RETRY_DELAY seconds..." | |
| sleep $RETRY_DELAY | |
| ATTEMPT=$((ATTEMPT + 1)) | |
| fi | |
| done | |
| if [ "$SUCCESS" = false ]; then | |
| echo "PUSH_FAILED" | |
| exit 1 | |
| fi | |
| - name: Path to result | |
| run: | | |
| echo "https://github.com/axiom-crypto/openvm-reth-benchmark/blob/gh-pages/${GH_PAGES_PATH}/${METRIC_NAME}.md" |