From 5493438a5dd78d2710f1b66a89573019c2250edd Mon Sep 17 00:00:00 2001 From: Elias Tazartes <66871571+Eikix@users.noreply.github.com> Date: Wed, 6 Mar 2024 12:33:15 +0100 Subject: [PATCH] add hive in CI (#809) * add hive in ci * fix hive_chain copy in dockerfile * load hive image into docker * build hive chain binary * add logs * fix build copy * try absolute path for lib kakarot * fix copy * copy kakarot build * fix build copy * try to fix again * fix artifacts * add workdir in copy * relative path kakarot * copying the project for compiler * try * fix * fix * only run rpc test * fix rpc dockerfile * fix --- .github/workflows/kakarot_rpc.yml | 11 +++- .github/workflows/test.yml | 55 +++++++++++++--- .gitignore | 2 + docker/hive/Dockerfile | 77 ++++++++++++++++++++--- docker/rpc/Dockerfile | 25 +------- scripts/extract_abi.sh | 30 +++++++-- src/eth_provider/starknet/kakarot_core.rs | 6 +- src/eth_provider/starknet/mod.rs | 2 +- 8 files changed, 157 insertions(+), 51 deletions(-) diff --git a/.github/workflows/kakarot_rpc.yml b/.github/workflows/kakarot_rpc.yml index 230fb38b1..a398435e5 100644 --- a/.github/workflows/kakarot_rpc.yml +++ b/.github/workflows/kakarot_rpc.yml @@ -24,7 +24,7 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push + - name: Build and push RPC uses: docker/build-push-action@v4 with: push: true @@ -33,3 +33,12 @@ jobs: file: ./docker/rpc/Dockerfile platforms: linux/amd64,linux/arm64 build-args: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + - name: Build and push Hive + uses: docker/build-push-action@v4 + with: + push: true + tags: ghcr.io/${{ github.repository }}/hive:${{ github.ref_name }}, ghcr.io/${{ github.repository }}/hive:latest + context: . + file: ./docker/hive/Dockerfile + platforms: linux/amd64,linux/arm64 + build-args: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fdf9e3ef2..541fa108a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,13 +22,6 @@ jobs: uses: taiki-e/install-action@cargo-llvm-cov # nextest setup - uses: taiki-e/install-action@nextest - # Load lib submodules - - name: Load submodules - id: cached-submodules - uses: actions/cache/restore@v3 - with: - path: ./lib/ - key: ${{ runner.os }}-submodules # Install Foundry - name: install foundry uses: foundry-rs/foundry-toolchain@v1 @@ -77,3 +70,51 @@ jobs: with: files: lcov.info fail_ci_if_error: false + hive: + runs-on: ubuntu-latest-16-cores + timeout-minutes: 45 + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build Hive + uses: docker/build-push-action@v4 + with: + push: false + tags: hive + load: true + context: . + file: ./docker/hive/Dockerfile + platforms: linux/amd64 + build-args: | + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + APIBARA_STARKNET_BIN_DIR=dsrifc349xgnwsv661shqljxb8pbxx2j + APIBARA_SINK_BIN_DIR=vr6f0crii7571q19j122dfqb1ha1m4i9 + - name: Clone kkrt-labs hive fork + run: git clone https://github.com/kkrt-labs/hive + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.22" + - name: Compile hive + run: | + cd hive + go build hive.go + - name: Run hive + run: | + cd hive + ./hive --client kakarot --sim=ethereum/rpc$ --docker.endpoint=unix:///var/run/docker.sock + - name: Print logs + if: always() + run: | + cd hive + echo "Logs:" + cat workspace/logs/*.log + echo "Kakarot logs:" + cat workspace/logs/kakarot/*.log + echo "Details logs:" + cat workspace/logs/details/*.log diff --git a/.gitignore b/.gitignore index 63317a701..35ea8ebcb 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ broadcast .hive artifacts/ +build/ +.kakarot/ diff --git a/docker/hive/Dockerfile b/docker/hive/Dockerfile index 67487d41f..388c7cdbe 100644 --- a/docker/hive/Dockerfile +++ b/docker/hive/Dockerfile @@ -5,13 +5,19 @@ FROM ghcr.io/dojoengine/dojo:v0.6.0-alpha.2 as katana # Indexer service ### Apibara DNA indexer and indexer -FROM quay.io/apibara/starknet:1.4.0 as apibara +FROM quay.io/apibara/starknet:1.4.1 as apibara FROM quay.io/apibara/sink-mongo:0.5.3 as indexer FROM debian:bookworm as apibara-build RUN apt-get update && apt-get install -y patchelf && rm -rf /var/lib/apt/lists/* -# Run `docker image inspect apibara/starknet:1.4.0` to get the exact path -COPY --from=apibara /nix/store/nkskc1iii2s658r2mx8bsgk11p6ibz82-apibara-starknet-1.4.0/bin/apibara-starknet /usr/local/bin/starknet -COPY --from=indexer /nix/store/3iqnrcirqpg4s7zdy1wdh0dq17jwzmlc-apibara-sink-mongo-0.5.3/bin/apibara-sink-mongo /usr/local/bin/sink-mongo + +# Apibara Starknet and Sink are built with Nix, so we need to extract the binaries from the Nix store +# Depending on the platform, the path to the binaries will be different +ARG APIBARA_STARKNET_BIN_DIR +ARG APIBARA_SINK_BIN_DIR +# Run `docker image inspect apibara/starknet:1.4.1-x86-64` to get the exact path +# ⚠️ This path is subject to change, so it's important to check it before building the image ⚠️ +COPY --from=apibara /nix/store/${APIBARA_STARKNET_BIN_DIR}-apibara-starknet-1.4.1/bin/apibara-starknet /usr/local/bin/starknet +COPY --from=indexer /nix/store/${APIBARA_SINK_BIN_DIR}-apibara-sink-mongo-0.5.3/bin/apibara-sink-mongo /usr/local/bin/sink-mongo # Change the interpreter path. ARG BUILDPLATFORM RUN case $BUILDPLATFORM in \ @@ -39,11 +45,62 @@ RUN git clone -v "https://github.com/kkrt-labs/kakarot-indexer.git" FROM mongo:6.0.8 as mongo # Ethereum RPC Server -FROM ghcr.io/kkrt-labs/kakarot-rpc/node:v0.2.6 as rpc +### Build the Cairo programs artifacts +FROM python:3.9.13 as compiler +# install poetry +ENV POETRY_VERSION=1.7.1 +ENV PATH="$PATH:/root/.local/bin:/root/.foundry/bin" +ARG GITHUB_TOKEN +ENV GITHUB_TOKEN=$GITHUB_TOKEN + +RUN curl -sSL https://install.python-poetry.org | python3 - +RUN poetry config virtualenvs.create false + +WORKDIR /usr/src/compiler + +# Install dependencies +RUN curl -L https://foundry.paradigm.xyz -o foundry.sh \ + && chmod +x foundry.sh \ + && ./foundry.sh \ + && foundryup \ + && apt-get update && apt-get install -y \ + jq \ + wget \ + tar \ + unzip \ + zip + +COPY . . + +RUN make setup + +COPY . . + +### Build the RPC server +# Define ARG for build platform +FROM rust:1.75 as rpc + +RUN apt-get update && apt-get install -y \ + gcc-aarch64-linux-gnu libssl-dev clang libclang-dev + +# Set working directory +WORKDIR /usr/src/rpc + +COPY . . + +COPY --from=compiler /usr/src/compiler/.kakarot/artifacts /usr/src/rpc/.kakarot/artifacts + + +RUN cargo build --release \ + --features hive --target-dir /usr/src/rpc/target && \ + cargo build --release \ + --bin hive_genesis --features testing --target-dir /usr/src/rpc/target && \ + cargo build --release \ + --bin hive_chain --features hive-chain --target-dir /usr/src/rpc/target FROM debian:bookworm-slim as base # Install any necessary dependencies -RUN apt-get update && apt-get install -y tini curl jq && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y tini curl jq libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/* # Environment ############# @@ -97,11 +154,11 @@ RUN mkdir -p /data/db # Generate the genesis -COPY --from=rpc /usr/local/bin/kakarot-rpc-hive /usr/local/bin/kakarot-rpc -COPY --from=rpc /usr/local/bin/hive_genesis /usr/local/bin -COPY --from=rpc /usr/local/bin/hive_chain /usr/local/bin +COPY --from=rpc /usr/src/rpc/target/release/kakarot-rpc /usr/local/bin/kakarot-rpc +COPY --from=rpc /usr/src/rpc/target/release/hive_genesis /usr/local/bin/hive_genesis +COPY --from=rpc /usr/src/rpc/target/release/hive_chain /usr/local/bin/hive_chain RUN mkdir -p /genesis/contracts -COPY ./lib/kakarot/build /genesis/contracts +COPY --from=compiler /usr/src/compiler/.kakarot/build /genesis/contracts # Copy start script COPY docker/hive/start.sh /start.sh diff --git a/docker/rpc/Dockerfile b/docker/rpc/Dockerfile index 863807c03..3ff52dfa2 100644 --- a/docker/rpc/Dockerfile +++ b/docker/rpc/Dockerfile @@ -22,10 +22,7 @@ RUN curl -L https://foundry.paradigm.xyz -o foundry.sh \ unzip \ zip -COPY .git ./.git -COPY .gitmodules .gitmodules -COPY Makefile Makefile -COPY scripts scripts +COPY . . RUN make setup # Define ARG for build platform @@ -38,7 +35,7 @@ ARG TARGETPLATFORM # Set working directory WORKDIR /usr/src/rpc -COPY --from=compiler /usr/src/compiler/artifacts /usr/src/rpc/artifacts +COPY --from=compiler /usr/src/compiler/.kakarot/artifacts /usr/src/rpc/.kakarot/artifacts # Copy source code COPY . . @@ -65,21 +62,6 @@ RUN build_platform() { \ --target=$ARCH; \ # Copy the built binary to a common release directory cp /usr/src/rpc/target/$ARCH/release/kakarot-rpc /usr/src/rpc/target/release/; \ - # Build the Rust application for the specified target for hive - BINDGEN_EXTRA_CLANG_ARGS=$BINDGEN_EXTRA_CLANG_ARGS cargo build --all --release \ - --features hive --target=$ARCH; \ - # Copy the built binary to a common release directory - cp /usr/src/rpc/target/$ARCH/release/kakarot-rpc /usr/src/rpc/target/release/kakarot-rpc-hive; \ - # Build the hive genesis binary - BINDGEN_EXTRA_CLANG_ARGS=$BINDGEN_EXTRA_CLANG_ARGS cargo build --all --release \ - --bin hive_genesis --features testing --target=$ARCH; \ - # Copy the genesis binary to a common release directory - cp /usr/src/rpc/target/$ARCH/release/hive_genesis /usr/src/rpc/target/release/; \ - # Build the hive chain binary - BINDGEN_EXTRA_CLANG_ARGS=$BINDGEN_EXTRA_CLANG_ARGS cargo build --all --release \ - --bin hive_chain --features hive-chain --target=$ARCH; \ - # Copy the genesis binary to a common release directory - cp /usr/src/rpc/target/$ARCH/release/hive_chain /usr/src/rpc/target/release/; \ } \ && rustup self update \ && case "$TARGETPLATFORM" in \ @@ -106,9 +88,6 @@ WORKDIR /usr/src/app # Copy the built binary from the previous container COPY --from=builder /usr/src/rpc/target/release/kakarot-rpc /usr/local/bin -COPY --from=builder /usr/src/rpc/target/release/kakarot-rpc-hive /usr/local/bin -COPY --from=builder /usr/src/rpc/target/release/hive_genesis /usr/local/bin -COPY --from=builder /usr/src/rpc/target/release/hive_chain /usr/local/bin # Expose the port that the RPC server will run on EXPOSE 9545 diff --git a/scripts/extract_abi.sh b/scripts/extract_abi.sh index db9ed47de..f4d62b6c4 100755 --- a/scripts/extract_abi.sh +++ b/scripts/extract_abi.sh @@ -4,24 +4,42 @@ set -o pipefail # Directory containing the original JSON files src_dir="./lib/kakarot/build" +# Root build directory +root_dir="./.kakarot" # Directory to write the new JSON files to -dest_dir="./artifacts" +dest_dir="/artifacts" +# Directory to write the entire build to +build_dir="/build" + +# Start logging +echo "Starting build process" # Ensure the destination directory exists -rm -rf "${dest_dir}" -mkdir -p "${dest_dir}" +echo "Removing existing build directory ${root_dir}" +rm -rf "${root_dir}" +echo "Creating new build directory ${root_dir}" +mkdir -p "${root_dir}" -# Find and process each JSON file +# Start processing JSON files +echo "Finding JSON files in ${src_dir} to process..." find "${src_dir}" -type f -name '*.json' | while read -r src_file; do # Extract the filename without the extension filename=$(basename -- "${src_file}") filename="${filename%.*}" + echo "Processing file: ${src_file}" # Check and create a subdirectory structure in destination if needed sub_dir=$(dirname "${src_file}") sub_dir=${sub_dir#"${src_dir}"} - mkdir -p "${dest_dir}${sub_dir}" + echo "Creating subdirectories in ${root_dir}${dest_dir}${sub_dir} and ${root_dir}${build_dir}${sub_dir}" + mkdir -p "${root_dir}${dest_dir}${sub_dir}" + mkdir -p "${root_dir}${build_dir}${sub_dir}" # Use jq to extract the 'abi' field and write it to a new JSON file - jq '.abi' "${src_file}" >"${dest_dir}${sub_dir}/${filename}.json" + jq '.abi' "${src_file}" >"${root_dir}${dest_dir}${sub_dir}/${filename}.json" + + # Copy the original JSON file to the build directory + cp "${src_file}" "${root_dir}${build_dir}${sub_dir}/${filename}.json" done + +echo "Build process complete." diff --git a/src/eth_provider/starknet/kakarot_core.rs b/src/eth_provider/starknet/kakarot_core.rs index 0956cf65e..2d3796a75 100644 --- a/src/eth_provider/starknet/kakarot_core.rs +++ b/src/eth_provider/starknet/kakarot_core.rs @@ -20,18 +20,18 @@ use crate::{ pub mod proxy { use super::*; - abigen_legacy!(Proxy, "./artifacts/proxy.json"); + abigen_legacy!(Proxy, "./.kakarot/artifacts/proxy.json"); } pub mod contract_account { use super::*; - abigen_legacy!(ContractAccount, "./artifacts/contract_account.json"); + abigen_legacy!(ContractAccount, "./.kakarot/artifacts/contract_account.json"); } #[allow(clippy::too_many_arguments)] pub mod core { use super::*; - abigen_legacy!(KakarotCore, "./artifacts/kakarot.json"); + abigen_legacy!(KakarotCore, "./.kakarot/artifacts/kakarot.json"); } fn env_var_to_field_element(var_name: &str) -> FieldElement { diff --git a/src/eth_provider/starknet/mod.rs b/src/eth_provider/starknet/mod.rs index 9b3516a34..1005f3108 100644 --- a/src/eth_provider/starknet/mod.rs +++ b/src/eth_provider/starknet/mod.rs @@ -5,7 +5,7 @@ use cainome::rs::abigen_legacy; use lazy_static::lazy_static; use starknet_crypto::FieldElement; -abigen_legacy!(ERC20, "./artifacts/fixtures/ERC20.json"); +abigen_legacy!(ERC20, "./.kakarot/artifacts/fixtures/ERC20.json"); lazy_static! { pub static ref STARKNET_NATIVE_TOKEN: FieldElement =