reth-benchmark.yml

name: "Reth Benchmark"

on:
  workflow_dispatch:
    inputs:
      # Default runner is not big enough for this
      # https://aws.amazon.com/ec2/instance-types/
      instance_family:
        type: string
        required: false
        description: The family of the instance, can be multiple ones concat with "+" e.g. r8g+r7g
        default: m7a.48xlarge
      memory_allocator:
        type: string
        required: false
        description: Memory allocator to use (mimalloc or jemalloc)
        default: mimalloc
      block_number:
        type: number
        required: false
        description: Block number to run the benchmark on
        default: 21000000
      app_log_blowup:
        type: number
        required: false
        description: Application level log blowup
        default: 2
      agg_log_blowup:
        type: number
        required: false
        description: Aggregation (leaf) level log blowup
        default: 2
      root_log_blowup:
        type: number
        required: false
        description: Root level log blowup (only for e2e)
        default: 3
      internal_log_blowup:
        type: number
        required: false
        description: Internal level log blowup (only for e2e)
        default: 2
      mode:
        type: choice
        required: false
        description: Running mode, one of execute, prove, or prove-e2e
        options:
          - execute
          - prove
          - prove-e2e
        default: prove
      collect_metrics:
        type: boolean
        required: false
        description: Whether to collect metrics for flamegraphs
        default: false
      max_segment_length:
        type: number
        required: false
        description: Max segment length for continuations, must be larger than 524288
        default: 8388508 # 2**23 - 100

env:
  S3_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/results
  S3_METRICS_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/metrics
  PUBLIC_S3_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/flamegraphs
  FEATURE_FLAGS: "bench-metrics,parallel,nightly-features"
  CMD_ARGS: ""
  INPUT_ARGS: ""
  CARGO_NET_GIT_FETCH_WITH_CLI: "true"

jobs:
  run-reth:
    name: Run Reth benchmark
    runs-on:
      - runs-on
      - run-id=${{ github.run_id }}
      - family=${{ inputs.instance_family }}
      - disk=large
      - tag=bench-reth-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref || github.ref }}
      - uses: dtolnay/rust-toolchain@nightly
      - uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true
      - name: Display workflow inputs
        run: echo "${{ toJSON(github.event.inputs) }}"
      # - name: Load SSH key
      #   uses: webfactory/ssh-agent@v0.9.0
      #   with:
      #     ssh-private-key: |
      #       ${{ secrets.GH_ACTIONS_DEPLOY_PRIVATE_KEY }}

      - name: Install cargo-openvm
        run: |
          cargo install --git https://github.com/openvm-org/openvm.git cargo-openvm
      - name: Install openvm-prof
        run: |
          cargo install --git https://github.com/openvm-org/openvm.git --profile=dev openvm-prof
      - name: Checkout openvm (for scripts)
        run: |
          git clone https://github.com/openvm-org/openvm.git

      - name: Install architecture specific tools
        run: |
          rustup install nightly-2024-10-30
          source openvm/ci/scripts/utils.sh
          install_s5cmd

      - name: Setup halo2
        if: ${{ github.event.inputs.mode == 'prove-e2e' }}
        run: |
          bash openvm/extensions/native/recursion/trusted_setup_s3.sh
          export PARAMS_DIR=$(pwd)/params

      - name: Set metric name
        run: |
          input_json_str="${{ toJSON(github.event.inputs) }}"
          input_hash=$(echo $input_json_str | sha256sum | cut -d' ' -f1)
          METRIC_NAME=reth-${input_hash}
          echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV
          METRIC_PATH=".bench_metrics/${METRIC_NAME}.json"
          echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV

      - name: Build ELF
        working-directory: bin/client-eth
        run: |
          PROFILE="release"
          if [[ "${{ inputs.collect_metrics }}" == "true" ]]; then
            PROFILE="profiling"
          fi
          cargo openvm build --no-transpile --profile=$PROFILE
          mkdir -p ../host/elf
          cp target/riscv32im-risc0-zkvm-elf/release/openvm-client-eth ../host/elf/

      - name: Run Reth
        run: |
          mkdir -p rpc-cache
          mkdir -p .bench_metrics # TODO: remove this
          RPC_1=${{ secrets.RPC_URL_1 }}
          MODE=${{ inputs.mode }}
          BLOCK_NUMBER=${{ inputs.block_number }}
          OPTIONAL_ARGS=""
          PROFILE="release"
          FEATURES="bench-metrics,${{ inputs.memory_allocator }}"
          if [[ "${{ inputs.collect_metrics }}" == "true" ]]; then
            OPTIONAL_ARGS="--collect-metrics"
            PROFILE="profiling"
            FEATURES="${FEATURES},profiling"
          fi
          arch=$(uname -m)
          case $arch in
          arm64|aarch64)
              RUSTFLAGS="-Ctarget-cpu=native"
              ;;
          x86_64|amd64)
              RUSTFLAGS="-Ctarget-cpu=native -C target-feature=+avx512f"
              ;;
          *)
          echo "Unsupported architecture: $arch"
          exit 1
          ;;
          esac
          RUSTFLAGS=$RUSTFLAGS cargo build --bin openvm-reth-benchmark --profile=$PROFILE --no-default-features --features=$FEATURES
          OUTPUT_PATH=${METRIC_PATH} ./target/$PROFILE/openvm-reth-benchmark \
            --$MODE --block-number $BLOCK_NUMBER --rpc-url $RPC_1 --cache-dir rpc-cache \
            --app-log-blowup ${{ inputs.app_log_blowup }} \
            --agg-log-blowup ${{ inputs.agg_log_blowup }} \
            --root-log-blowup ${{ inputs.root_log_blowup }} \
            --max-segment-length ${{ inputs.max_segment_length }} $OPTIONAL_ARGS

      - name: Upload Benchmark Metrics
        run: |
          current_sha=$(git rev-parse HEAD)
          echo "Current SHA: $current_sha"
          echo "current_sha=${current_sha}" >> $GITHUB_ENV
          s5cmd cp ${METRIC_PATH} ${{ env.S3_METRICS_PATH }}/${current_sha}-${METRIC_NAME}.json

      - name: Generate markdown # result path is hardcoded results.md
        run: |
          openvm-prof --json-paths $METRIC_PATH
          MD_PATH=${METRIC_PATH%.json}.md
          echo "MD_PATH=${MD_PATH}" >> $GITHUB_ENV

      # - name: Install inferno-flamegraph
      #   run: cargo install inferno

      # - name: Generate flamegraphs
      #   run: |
      #     if [[ -f $METRIC_PATH ]]; then
      #       python3 openvm/ci/scripts/metric_unify/flamegraph.py $METRIC_PATH
      #       s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.PUBLIC_S3_PATH }}/${current_sha}/"
      #       echo "UPLOAD_FLAMEGRAPHS=1" >> $GITHUB_ENV
      #     fi

      - name: Add benchmark metadata to markdown, and upload
        run: |
          COMMIT_URL=https://github.com/${{ github.repository }}/commit/${current_sha}
          BENCHMARK_WORKFLOW_URL=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

          source openvm/ci/scripts/utils.sh
          add_metadata $MD_PATH \
            ${{ inputs.max_segment_length }} \
            ${{ inputs.instance_family }} \
            ${{ inputs.memory_allocator }} \
            $COMMIT_URL \
            $BENCHMARK_WORKFLOW_URL

      - name: Upload markdown
        run: |
          s5cmd cp $MD_PATH "${{ env.S3_PATH }}/${current_sha}-${METRIC_NAME}.md"

      ### Update gh-pages
      - uses: actions/checkout@v4
        with:
          ref: gh-pages

      - name: Set up git
        run: |
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global user.name "github-actions[bot]"

      - name: Commit to gh-pages branch
        run: |
          GH_PAGES_PATH="benchmarks-dispatch/${{ github.head_ref || github.ref }}"
          echo "GH_PAGES_PATH=${GH_PAGES_PATH}" >> $GITHUB_ENV
          mkdir -p ${GH_PAGES_PATH}
          s5cmd cp "${{ env.S3_PATH }}/${current_sha}-${METRIC_NAME}.md" "${GH_PAGES_PATH}/${METRIC_NAME}.md"

          git add ${GH_PAGES_PATH}/${METRIC_NAME}.md
          git commit --allow-empty -m "Update benchmark result at ${GH_PAGES_PATH}/${METRIC_NAME}.md"
          MAX_RETRIES=10
          RETRY_DELAY=5
          ATTEMPT=0
          SUCCESS=false
          while [ $ATTEMPT -lt $MAX_RETRIES ]; do
              echo "Attempt $((ATTEMPT + 1)) to push of $MAX_RETRIES..."
              git fetch origin gh-pages
              git merge origin/gh-pages --no-edit
              if git push origin gh-pages; then
                  SUCCESS=true
                  break
              else
                  echo "Push failed. Retrying in $RETRY_DELAY seconds..."
                  sleep $RETRY_DELAY
                  ATTEMPT=$((ATTEMPT + 1))
              fi
          done
          if [ "$SUCCESS" = false ]; then
              echo "PUSH_FAILED"
              exit 1
          fi
      - name: Path to result
        run: |
          echo "https://github.com/axiom-crypto/openvm-reth-benchmark/blob/gh-pages/${GH_PAGES_PATH}/${METRIC_NAME}.md"