From d02e15093d4f4120f9527af8c7ca97eba4d40fc9 Mon Sep 17 00:00:00 2001 From: 0xaatif <169152398+0xaatif@users.noreply.github.com> Date: Mon, 8 Jul 2024 16:44:26 +0100 Subject: [PATCH] refactor!: docker builds (#357) * mark: 0xaatif/docker * run: docker init * run: rm compose.yaml * refactor: docker builds * feat: smoke test executables * refactor(ci): use new docker file * feat: xargs --verbose * refactor: one image * fix: one image * wibble: change id meta_leader -> meta * fix: censor minor version * fix: restore target-cpu=native --- .cargo/config.toml | 4 + .dockerignore | 28 +++++++ .github/workflows/docker_build.yml | 23 ++---- .github/workflows/docker_build_push.yml | 40 ++-------- Cargo.lock | 36 +++++++++ Cargo.toml | 24 +++--- Dockerfile | 100 ++++++++++++++++++++++++ leader.Dockerfile | 60 -------------- worker.Dockerfile | 40 ---------- zero_bin/.cargo/config.toml | 8 +- zero_bin/leader/Cargo.toml | 6 +- zero_bin/leader/build.rs | 19 +++++ zero_bin/leader/src/main.rs | 31 +++----- zero_bin/leader/src/utils.rs | 72 ----------------- zero_bin/tools/prove_rpc.sh | 5 +- zero_bin/tools/prove_stdio.sh | 5 +- 16 files changed, 236 insertions(+), 265 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 .dockerignore create mode 100644 Dockerfile delete mode 100644 leader.Dockerfile delete mode 100644 worker.Dockerfile create mode 100644 zero_bin/leader/build.rs delete mode 100644 zero_bin/leader/src/utils.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..6340ce34a --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,4 @@ +[build] +# https://github.com/rust-lang/rust/pull/124129 +# https://github.com/dtolnay/linkme/pull/88 +rustflags = ["-Z", "linker-features=-lld"] diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..1562ab5b3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,28 @@ +# this is loosely based on `docker init`'s rust template. + +**/.DS_Store +**/.classpath +**/.dockerignore +# **/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/charts +**/docker-compose* +**/compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/secrets.dev.yaml +**/values.dev.yaml +/bin +/target +LICENSE +README.md diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml index 8d74a2fa4..c907bbf39 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/docker_build.yml @@ -12,24 +12,15 @@ on: jobs: docker: - name: Build and run leader and worker docker images for regression check + name: Regression test docker images runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - - name: Build leader docker container - run: | - docker build --progress plain -t leader:${{ github.ref_name }} -f leader.Dockerfile . - - - name: Run leader docker container - run: | - docker run --rm leader:${{ github.ref_name }} --help - - - name: Build worker docker container - run: | - docker build --progress plain -t worker:${{ github.ref_name }} -f worker.Dockerfile . - - - name: Run worker docker container - run: | - docker run --rm worker:${{ github.ref_name }} --help + - run: | + docker build --progress=plain --build-arg=PROFILE=dev --tag scratch . + docker run --rm --init --entrypoint leader scratch --help + docker run --rm --init --entrypoint worker scratch --help + docker run --rm --init --entrypoint rpc scratch --help + docker run --rm --init --entrypoint verifier scratch --help diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push.yml index 112ea158e..0c64efecd 100644 --- a/.github/workflows/docker_build_push.yml +++ b/.github/workflows/docker_build_push.yml @@ -8,8 +8,6 @@ on: env: REGISTRY: ghcr.io - IMAGE_NAME_LEADER: ${{ github.repository }}-leader - IMAGE_NAME_WORKER: ${{ github.repository }}-worker jobs: docker: @@ -34,50 +32,24 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Leader Docker - id: meta_leader - uses: docker/metadata-action@v5 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LEADER }} - tags: | - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - - name: Push to GitHub Container Registry - Leader - uses: docker/build-push-action@v3 - with: - context: . - file: ./leader.Dockerfile - push: true - # platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta_leader.outputs.tags }} - labels: ${{ steps.meta_leader.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Extract metadata (tags, labels) for Worker Docker - id: meta_worker + - name: Extract metadata (tags, labels) + id: meta uses: docker/metadata-action@v5 with: images: | - name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_WORKER }} + name=${{ env.REGISTRY }}/${{ github.repository }} tags: | type=ref,event=branch type=ref,event=pr type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} - - name: Push to GitHub Container Registry - Worker + - name: Push to GitHub Container Registry uses: docker/build-push-action@v3 with: - context: . - file: ./worker.Dockerfile push: true # platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta_worker.outputs.tags }} - labels: ${{ steps.meta_worker.outputs.labels }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/Cargo.lock b/Cargo.lock index a2b0f7e02..b4daeaaad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1161,6 +1161,38 @@ dependencies = [ "serde", ] +[[package]] +name = "camino" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.23", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cast" version = "0.3.0" @@ -2814,6 +2846,7 @@ dependencies = [ "alloy", "anyhow", "axum", + "cargo_metadata", "clap", "dotenvy", "futures", @@ -4373,6 +4406,9 @@ name = "semver" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +dependencies = [ + "serde", +] [[package]] name = "semver-parser" diff --git a/Cargo.toml b/Cargo.toml index b64e627fb..1ff61b81d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,18 +1,20 @@ [workspace] -members = ["mpt_trie", - "smt_trie", - "proof_gen", - "trace_decoder", +members = [ + "compat", "evm_arithmetization", + "mpt_trie", "proc_macro", - "zero_bin/leader", - "zero_bin/worker", + "proof_gen", + "smt_trie", + "trace_decoder", "zero_bin/common", + "zero_bin/leader", "zero_bin/ops", - "zero_bin/verifier", + "zero_bin/prover", "zero_bin/rpc", - "zero_bin/prover", - "compat"] + "zero_bin/verifier", + "zero_bin/worker", +] resolver = "2" [workspace.package] @@ -24,7 +26,7 @@ keywords = ["cryptography", "STARK", "plonky2", "ethereum", "zk"] categories = ["cryptography::cryptocurrencies"] [workspace.dependencies] -alloy = { git = "https://github.com/alloy-rs/alloy", tag='v0.1.1', default-features = false, features = [ +alloy = { git = "https://github.com/alloy-rs/alloy", tag = 'v0.1.1', default-features = false, features = [ "consensus", "reqwest", "json-rpc", @@ -36,7 +38,7 @@ alloy = { git = "https://github.com/alloy-rs/alloy", tag='v0.1.1', default-featu "providers", "transports", "transport-http", - "rpc-types-debug" + "rpc-types-debug", ] } anyhow = "1.0.86" async-stream = "0.3.5" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..18158356f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,100 @@ +# syntax=docker/dockerfile:1 +# This is loosely based on `docker init`'s rust template. +# For a completely clean build, run something like this: +# ``` +# docker build --build-arg=PROFILE=dev --no-cache +# ``` + +############# +# Build stage +############# +# - `/src` is the repo directory. +# - `/artifacts` is $CARGO_TARGET_DIR. +# - `/output` is where the binaries go. + +ARG BUILD_BASE=rustlang/rust:nightly-bullseye-slim +FROM ${BUILD_BASE} AS build + +# Install build dependencies. +RUN apt-get update && apt-get install -y \ + # for jemalloc + libjemalloc-dev \ + libjemalloc2 \ + make \ + # for openssl + libssl-dev \ + pkg-config \ + # clean the image + && rm -rf /var/lib/apt/lists/* + +ARG PROFILE=release +# forward the docker argument so that the script below can read it +ENV PROFILE=${PROFILE} + +# Build the application. +RUN \ + # mount the repository so we don't have to COPY it in + --mount=type=bind,source=.,target=/src \ + # cache artifacts and the cargo registry to speed up subsequent builds + --mount=type=cache,target=/artifacts \ + --mount=type=cache,target=/usr/local/cargo/registry/ \ + # run the build + < anyhow::Result<()> { + let meta = cargo_metadata::MetadataCommand::new() + .exec() + .context("failed to probe cargo-metadata")?; + let version = &meta + .packages + .iter() + .find(|it| it.name == "evm_arithmetization") + .context("couldn't find evm_arithmetization package")? + .version; + println!( + "cargo::rustc-env=EVM_ARITHMETIZATION_PACKAGE_VERSION={}.{}.x", + // patch version change should not prompt circuits regeneration + version.major, + version.minor + ); + Ok(()) +} diff --git a/zero_bin/leader/src/main.rs b/zero_bin/leader/src/main.rs index a2c7020f1..94b2abb94 100644 --- a/zero_bin/leader/src/main.rs +++ b/zero_bin/leader/src/main.rs @@ -13,14 +13,14 @@ use tracing::{info, warn}; use zero_bin_common::block_interval::BlockInterval; use crate::client::{client_main, ProofParams}; -use crate::utils::get_package_version; mod cli; mod client; mod http; mod init; mod stdio; -mod utils; + +const EVM_ARITH_VER_KEY: &str = "EVM_ARITHMETIZATION_PKG_VER"; fn get_previous_proof(path: Option) -> Result> { if path.is_none() { @@ -39,24 +39,17 @@ async fn main() -> Result<()> { load_dotenvy_vars_if_present(); init::tracing(); - if env::var("EVM_ARITHMETIZATION_PKG_VER").is_err() { - let pkg_ver = get_package_version("evm_arithmetization")?; - // Extract the major and minor version parts and append 'x' as the patch version - if let Some((major_minor, _)) = pkg_ver.as_ref().and_then(|s| s.rsplit_once('.')) { - let circuits_version = format!("{}.x", major_minor); - // Set the environment variable for the evm_arithmetization package version - #[allow(unused_unsafe)] - unsafe { - env::set_var("EVM_ARITHMETIZATION_PKG_VER", circuits_version); - } - } else { - // Set to "NA" if version extraction fails - #[allow(unused_unsafe)] - unsafe { - env::set_var("EVM_ARITHMETIZATION_PKG_VER", "NA"); - } + if env::var_os(EVM_ARITH_VER_KEY).is_none() { + // Safety: + // - we're early enough in main that nothing else should race + unsafe { + env::set_var( + EVM_ARITH_VER_KEY, + // see build.rs + env!("EVM_ARITHMETIZATION_PACKAGE_VERSION"), + ); } - } + }; let args = cli::Cli::parse(); if let paladin::config::Runtime::InMemory = args.paladin.runtime { diff --git a/zero_bin/leader/src/utils.rs b/zero_bin/leader/src/utils.rs deleted file mode 100644 index e2c7d5f10..000000000 --- a/zero_bin/leader/src/utils.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::fs::File; -use std::io::{BufReader, Read}; -use std::path::Path; - -use anyhow::Result; - -/// Retrieves the version of a specified package from the `Cargo.lock` file. -/// -/// This function attempts to find the version of a package specified by -/// `package_name` by reading and parsing the `Cargo.lock` file. The -/// `Cargo.lock` file is expected to be located one directory level up from the -/// directory specified by the `CARGO_MANIFEST_DIR` environment variable. The -/// path may need adjustment depending on the structure of the project. -/// -/// # Parameters -/// - `package_name`: The name of the package for which the version is being -/// retrieved. -/// -/// # Returns -/// - `Ok(Some(String))`: If the package is found in the `Cargo.lock` file, -/// returns the version of the package. -/// - `Ok(None)`: If the package is not found in the `Cargo.lock` file, or if -/// the `Cargo.lock` file does not exist. -/// - `Err(_)`: If any error occurs during the execution, such as issues with -/// file paths, file access, reading, or parsing the `Cargo.lock` file. -/// -/// # Examples -/// ```no_run -/// let version = get_package_version("my_package"); -/// match version { -/// Ok(Some(ver)) => println!("Found version: {}", ver), -/// Ok(None) => println!("Package not found."), -/// Err(e) => println!("Error occurred: {}", e), -/// } -/// ``` -/// -/// # Errors -/// This function can return an `Err` result if: -/// - There is a problem finding, opening, or reading the `Cargo.lock` file. -/// - There is a failure in parsing the `Cargo.lock` file as TOML. -/// -/// The function uses `?` to propagate errors upwards, so the exact nature of -/// the error will be indicated by the error value returned in the `Err` variant -/// of the `Result`. -pub(crate) fn get_package_version(package_name: &str) -> Result> { - let manifest_dir = env!("CARGO_MANIFEST_DIR"); - let zero_bin_path = Path::new(manifest_dir) - .join("../") // Adjust the path according to your workspace structure - .canonicalize()?; - - let cargo_lock_path = zero_bin_path.join("Cargo.lock"); - let cargo_lock_file = File::open(cargo_lock_path); - if cargo_lock_file.is_err() { - return Ok(None); - } - - let mut cargo_lock_contents = String::new(); - BufReader::new(cargo_lock_file?).read_to_string(&mut cargo_lock_contents)?; - - let lockfile: toml::Value = toml::from_str(&cargo_lock_contents)?; - if let Some(package) = lockfile["package"] - .as_array() - .unwrap() - .iter() - .find(|&p| p["name"].as_str() == Some(package_name)) - { - let version = package["version"].as_str().unwrap(); - return Ok(Some(version.to_string())); - } - - Ok(None) -} diff --git a/zero_bin/tools/prove_rpc.sh b/zero_bin/tools/prove_rpc.sh index 9a0700f22..363e3dfd4 100755 --- a/zero_bin/tools/prove_rpc.sh +++ b/zero_bin/tools/prove_rpc.sh @@ -13,9 +13,8 @@ export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=1 export RUST_LOG=info -# Disable the lld linker for now, as it's causing issues with the linkme package. -# https://github.com/rust-lang/rust/pull/124129 -# https://github.com/dtolnay/linkme/pull/88 +# Script users are running locally, and might benefit from extra perf. +# See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' if [[ $8 == "test_only" ]]; then diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index 43f62dd59..23b4b5948 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -29,9 +29,8 @@ export TOKIO_WORKER_THREADS=$num_procs export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full export RUST_LOG=info -# Disable the lld linker for now, as it's causing issues with the linkme package. -# https://github.com/rust-lang/rust/pull/124129 -# https://github.com/dtolnay/linkme/pull/88 +# Script users are running locally, and might benefit from extra perf. +# See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' INPUT_FILE=$1