probably port CI to toolbox #8
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Workflow template for CI jobs for Core Components | |
on: | |
workflow_call: | |
inputs: | |
compilers: | |
description: "JSON of required compilers and their versions" | |
type: string | |
required: false | |
default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18", "1.3.19", "1.3.21"] } , { "zkvyper": ["1.3.13"] }]' | |
jobs: | |
# lint: | |
# name: lint | |
# uses: ./.github/workflows/ci-core-lint-reusable.yml | |
unit-tests: | |
runs-on: [ matterlabs-ci-runner-highmem-long ] | |
steps: | |
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 | |
with: | |
submodules: "recursive" | |
fetch-depth: 0 | |
- name: Setup environment | |
run: | | |
echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV | |
echo $(pwd)/bin >> $GITHUB_PATH | |
echo IN_DOCKER=1 >> .env | |
# TODO: Remove when we after upgrade of hardhat-plugins | |
- name: pre-download compilers | |
run: | | |
# Download needed versions of vyper compiler | |
# Not sanitized due to unconventional path and tags | |
mkdir -p ./hardhat-nodejs/compilers-v2/vyper/linux | |
wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10 https://github.com/vyperlang/vyper/releases/download/v0.3.10/vyper.0.3.10+commit.91361694.linux | |
wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3+commit.48e326f0.linux | |
chmod +x ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10 | |
chmod +x ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3 | |
COMPILERS_JSON='${{ inputs.compilers }}' | |
echo "$COMPILERS_JSON" | jq -r '.[] | to_entries[] | .key as $compiler | .value[] | "\(.),\($compiler)"' | while IFS=, read -r version compiler; do | |
mkdir -p "./hardhat-nodejs/compilers-v2/$compiler" | |
wget -nv -O "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}" "https://github.com/matter-labs/${compiler}-bin/releases/download/v${version}/${compiler}-linux-amd64-musl-v${version}" | |
chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}" | |
done | |
- name: Start services | |
run: | | |
ci_localnet_up | |
ci_run sccache --start-server | |
- name: Init | |
run: | | |
ci_run run_retried rustup show | |
ci_run ./bin/zkt | |
ci_run zk_supervisor contracts | |
# FIXME: enable contract tests once tehy are stable | |
#- name: Contracts unit tests | |
# run: ci_run yarn l1-contracts test | |
- name: Rust unit tests | |
run: | | |
ci_run zk_supervisor test rust | |
# Benchmarks are not tested by `cargo nextest` unless specified explicitly, and even then `criterion` harness is incompatible | |
# with how `cargo nextest` runs tests. Thus, we run criterion-based benchmark tests manually. | |
ci_run cargo test --release -p vm-benchmark --bench oneshot --bench batch | |
# FIXME: support loadtest together with sync layer. | |
# loadtest: | |
# runs-on: [ matterlabs-ci-runner-high-performance ] | |
# strategy: | |
# fail-fast: false | |
# matrix: | |
# # FIXME: support new VM mode | |
# vm_mode: ["OLD"] | |
# steps: | |
# - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 | |
# with: | |
# submodules: "recursive" | |
# fetch-depth: 0 | |
# - name: Setup environment | |
# run: | | |
# echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV | |
# echo $(pwd)/bin >> $GITHUB_PATH | |
# echo IN_DOCKER=1 >> .env | |
# - name: Loadtest configuration | |
# run: | | |
# echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'NEW' && 21000 || 16000 }} >> .env | |
# echo ACCOUNTS_AMOUNT="100" >> .env | |
# echo MAX_INFLIGHT_TXS="10" >> .env | |
# echo SYNC_API_REQUESTS_LIMIT="15" >> .env | |
# echo FAIL_FAST=true >> .env | |
# echo IN_DOCKER=1 >> .env | |
# - name: Start services | |
# run: | | |
# ci_localnet_up | |
# ci_run sccache --start-server | |
# - name: Init | |
# run: | | |
# ci_run git config --global --add safe.directory /usr/src/zksync | |
# ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen | |
# ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts | |
# ci_run git config --global --add safe.directory /usr/src/zksync/contracts | |
# ci_run ./bin/zkt | |
# ci_run zk_inception chain create \ | |
# --chain-name legacy \ | |
# --chain-id sequential \ | |
# --prover-mode no-proofs \ | |
# --wallet-creation localhost \ | |
# --l1-batch-commit-data-generator-mode rollup \ | |
# --base-token-address 0x0000000000000000000000000000000000000001 \ | |
# --base-token-price-nominator 1 \ | |
# --base-token-price-denominator 1 \ | |
# --set-as-default false \ | |
# --ignore-prerequisites \ | |
# --legacy-bridge | |
# ci_run zk_inception ecosystem init --dev --verbose | |
# ci_run zk_supervisor contracts --test-contracts | |
# # `sleep 60` because we need to wait until server added all the tokens | |
# - name: Run server | |
# run: | | |
# ci_run sed -i -e 's/mode: FULL/mode: LIGHTWEIGHT/' chains/legacy/configs/general.yaml | |
# ci_run sed -i -e 's/state_keeper_fast_vm_mode:.*/state_keeper_fast_vm_mode: ${{ matrix.vm_mode }}/' chains/legacy/configs/general.yaml | |
# ci_run sed -i -e 's/delay_interval:.*/delay_interval: 50/' chains/legacy/configs/general.yaml | |
# ci_run zk_inception server --uring --chain=legacy --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log & | |
# ci_run sleep 60 | |
# - name: Perform loadtest | |
# run: ci_run zk_supervisor t loadtest -v --chain=legacy | |
# - name: Show server.log logs | |
# if: always() | |
# run: ci_run cat server.log || true | |
# - name: Show sccache logs | |
# if: always() | |
# run: | | |
# ci_run sccache --show-stats || true | |
# ci_run cat /tmp/sccache_log.txt || true | |
integration: | |
name: Integration (consensus=${{ matrix.consensus }}, base_token=${{ matrix.base_token }}, gateway = ${{ matrix.gateway }}) | |
strategy: | |
# In matrix jobs, fail-fast is true by default. | |
# To be consistent with the rest of the workflow we disable it explicitly. | |
fail-fast: false | |
matrix: | |
# FIXME: support consensus tests for SL migration | |
consensus: [false] | |
# FIXME: support custom tokens | |
base_token: ["Eth"] | |
# FIXME: run tests even when not using synclayer | |
gateway: [true] | |
env: | |
SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher${{ matrix.consensus && ',consensus' || '' }}${{ matrix.base_token == 'Custom' && ',base_token_ratio_persister' || '' }}" | |
runs-on: [ matterlabs-ci-runner-highmem-long ] | |
steps: | |
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 | |
with: | |
submodules: "recursive" | |
fetch-depth: 0 | |
- name: Setup environment | |
run: | | |
echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV | |
echo $(pwd)/bin >> $GITHUB_PATH | |
echo IN_DOCKER=1 >> .env | |
echo RUN_CONTRACT_VERIFICATION_TEST=true >> .env | |
echo ZKSYNC_DEBUG_LOGS=true >> .env | |
- name: Download zksolc/solc and zkvyper/vyper | |
run: | | |
sudo apt update && sudo apt install wget -y | |
mkdir -p $(pwd)/etc/solc-bin/0.8.23 | |
wget https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/solc-linux-amd64-v0.8.23%2Bcommit.f704f362 | |
mv solc-linux-amd64-v0.8.23+commit.f704f362 $(pwd)/etc/solc-bin/0.8.23/solc | |
chmod +x $(pwd)/etc/solc-bin/0.8.23/solc | |
mkdir -p $(pwd)/etc/solc-bin/zkVM-0.8.23-1.0.0 | |
wget https://github.com/matter-labs/era-solidity/releases/download/0.8.23-1.0.0/solc-linux-amd64-0.8.23-1.0.0 -O $(pwd)/etc/solc-bin/zkVM-0.8.23-1.0.0/solc | |
chmod +x $(pwd)/etc/solc-bin/zkVM-0.8.23-1.0.0/solc | |
mkdir -p $(pwd)/etc/zksolc-bin/v1.3.21 | |
wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.21 | |
mv zksolc-linux-amd64-musl-v1.3.21 $(pwd)/etc/zksolc-bin/v1.3.21/zksolc | |
chmod +x $(pwd)/etc/zksolc-bin/v1.3.21/zksolc | |
mkdir -p $(pwd)/etc/vyper-bin/0.3.10 | |
wget -O vyper0.3.10 https://github.com/vyperlang/vyper/releases/download/v0.3.10/vyper.0.3.10%2Bcommit.91361694.linux | |
mv vyper0.3.10 $(pwd)/etc/vyper-bin/0.3.10/vyper | |
chmod +x $(pwd)/etc/vyper-bin/0.3.10/vyper | |
mkdir -p $(pwd)/etc/zkvyper-bin/v1.3.13 | |
wget https://github.com/matter-labs/zkvyper-bin/raw/main/linux-amd64/zkvyper-linux-amd64-musl-v1.3.13 | |
mv zkvyper-linux-amd64-musl-v1.3.13 $(pwd)/etc/zkvyper-bin/v1.3.13/zkvyper | |
chmod +x $(pwd)/etc/zkvyper-bin/v1.3.13/zkvyper | |
- name: Start services | |
run: | | |
ci_localnet_up | |
ci_run pre_download_compilers.sh | |
ci_run sccache --start-server | |
- name: Build zk_toolbox | |
run: ci_run bash -c "./bin/zkt" | |
- name: Create log directories | |
run: | | |
SERVER_LOGS_DIR=logs/server | |
INTEGRATION_TESTS_LOGS_DIR=logs/integration_tests | |
INTEGRATION_TESTS_EN_LOGS_DIR=logs/integration_tests/en | |
SNAPSHOT_RECOVERY_LOGS_DIR=logs/snapshot_recovery/ | |
GENESIS_RECOVERY_LOGS_DIR=logs/genesis_recovery/ | |
EXTERNAL_NODE_LOGS_DIR=logs/external_node | |
REVERT_LOGS_DIR=logs/revert | |
mkdir -p $SERVER_LOGS_DIR | |
mkdir -p $INTEGRATION_TESTS_LOGS_DIR | |
mkdir -p $INTEGRATION_TESTS_EN_LOGS_DIR | |
mkdir -p $SNAPSHOT_RECOVERY_LOGS_DIR | |
mkdir -p $GENESIS_RECOVERY_LOGS_DIR | |
mkdir -p $EXTERNAL_NODE_LOGS_DIR | |
mkdir -p $REVERT_LOGS_DIR | |
echo "SERVER_LOGS_DIR=$SERVER_LOGS_DIR" >> $GITHUB_ENV | |
echo "INTEGRATION_TESTS_LOGS_DIR=$INTEGRATION_TESTS_LOGS_DIR" >> $GITHUB_ENV | |
echo "INTEGRATION_TESTS_EN_LOGS_DIR=$INTEGRATION_TESTS_EN_LOGS_DIR" >> $GITHUB_ENV | |
echo "SNAPSHOT_RECOVERY_LOGS_DIR=$SNAPSHOT_RECOVERY_LOGS_DIR" >> $GITHUB_ENV | |
echo "GENESIS_RECOVERY_LOGS_DIR=$GENESIS_RECOVERY_LOGS_DIR" >> $GITHUB_ENV | |
echo "EXTERNAL_NODE_LOGS_DIR=$EXTERNAL_NODE_LOGS_DIR" >> $GITHUB_ENV | |
echo "REVERT_LOGS_DIR=$REVERT_LOGS_DIR" >> $GITHUB_ENV | |
- name: Initialize ecosystem | |
run: | | |
ci_run git config --global --add safe.directory /usr/src/zksync | |
ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts | |
ci_run git config --global --add safe.directory /usr/src/zksync/contracts | |
ci_run zk_inception ecosystem init --deploy-paymaster --deploy-erc20 \ | |
--deploy-ecosystem --l1-rpc-url=http://localhost:8545 \ | |
--server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ | |
--server-db-name=zksync_server_localhost_era \ | |
--prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ | |
--prover-db-name=zksync_prover_localhost_era \ | |
--ignore-prerequisites --verbose \ | |
--observability=false | |
- name: Initialize gateway chain | |
run: | | |
ci_run zk_inception chain create \ | |
--chain-name gateway \ | |
--chain-id 505 \ | |
--prover-mode no-proofs \ | |
--wallet-creation localhost \ | |
--l1-batch-commit-data-generator-mode rollup \ | |
--base-token-address 0x0000000000000000000000000000000000000001 \ | |
--base-token-price-nominator 1 \ | |
--base-token-price-denominator 1 \ | |
--set-as-default false \ | |
--ignore-prerequisites | |
ci_run zk_inception chain init \ | |
--deploy-paymaster \ | |
--l1-rpc-url=http://localhost:8545 \ | |
--server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ | |
--server-db-name=zksync_server_localhost_gateway \ | |
--prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ | |
--prover-db-name=zksync_prover_localhost_gateway \ | |
--port-offset 2000 \ | |
--chain gateway \ | |
ci_run zk_inception chain convert-to-gateway --chain gateway --ignore-prerequisites | |
- name: Build test dependencies | |
run: | | |
ci_run zk_supervisor test build | |
- name: Run gateway | |
run: | | |
ci_run zk_inception server --ignore-prerequisites --chain gateway &> ${{ env.SERVER_LOGS_DIR }}/gateway.log & | |
ci_run sleep 5 | |
- name: Migrate chain to gateway | |
run: | | |
ci_run zk_inception chain migrate-to-gateway --chain era --gateway-chain-name gateway | |
- name: Run era chain | |
ci_run zk_inception server --ignore-prerequisites --chain era &> ${{ env.SERVER_LOGS_DIR }}/rollup.log & | |
- name: Run integration tests | |
run: | | |
ci_run zk_supervisor test integration --no-deps --ignore-prerequisites --chain era &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/rollup.log & | |
PID1=$! | |
wait $PID1 | |
- name: Upload logs | |
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 | |
if: always() | |
with: | |
name: logs | |
path: logs | |
# FIXME: restore snapshot test | |
# - name: Snapshot recovery test | |
# # We use `yarn` directly because the test launches `zk` commands in both server and EN envs. | |
# # An empty topmost environment helps avoid a mess when redefining env vars shared between both envs | |
# # (e.g., DATABASE_URL). | |
# # | |
# # Since `base_token` doesn't meaningfully influence the test, we use it as a flag for | |
# # enabling / disabling tree during pruning. | |
# run: | | |
# if [[ "${{ matrix.deployment_mode }}" == "Validium" ]]; then | |
# ci_run zk config compile ext-node-validium | |
# ci_run zk config compile ext-node-validium-docker | |
# fi | |
# ENABLE_CONSENSUS=${{ matrix.consensus }} \ | |
# DEPLOYMENT_MODE=${{ matrix.deployment_mode }} \ | |
# SNAPSHOTS_CREATOR_VERSION=${{ matrix.deployment_mode == 'Validium' && '0' || '1' }} \ | |
# DISABLE_TREE_DURING_PRUNING=${{ matrix.base_token == 'Eth' }} \ | |
# ETH_CLIENT_WEB3_URL="http://localhost:8545" \ | |
# PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE,DISABLE_TREE_DURING_PRUNING,SNAPSHOTS_CREATOR_VERSION,ETH_CLIENT_WEB3_URL" \ | |
# ci_run yarn recovery-test snapshot-recovery-test | |
# - name: Genesis recovery test | |
# run: | | |
# ENABLE_CONSENSUS=${{ matrix.consensus }} \ | |
# DEPLOYMENT_MODE=${{ matrix.deployment_mode }} \ | |
# ETH_CLIENT_WEB3_URL="http://localhost:8545" \ | |
# PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE,ETH_CLIENT_WEB3_URL" \ | |
# ci_run yarn recovery-test genesis-recovery-test | |
# FIXME: restore fee proj tests | |
# - name: Fee projection tests | |
# run: ci_run zk test i fees | |
# FIXME: restore revert tests | |
# - name: Run revert test | |
# run: | | |
# ci_run pkill zksync_server || true | |
# ci_run sleep 2 | |
# ENABLE_CONSENSUS=${{ matrix.consensus }} DEPLOYMENT_MODE=Rollup PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE" ci_run zk test i revert | |
# FIXME: restore upgrade test for SL | |
# # This test should be the last one as soon as it | |
# # finished bootloader will be different | |
# - name: Run upgrade test | |
# run: | | |
# ci_run pkill zksync_server || true | |
# ci_run sleep 10 | |
# ci_run zk test i upgrade | |
# FIXME: restore tests with EN and sync layer | |
# external-node: | |
# name: External node (consensus=${{ matrix.consensus }}, base_token=${{ matrix.base_token }}, deployment_mode=${{ matrix.deployment_mode }}) | |
# strategy: | |
# fail-fast: false | |
# matrix: | |
# consensus: [ false, true ] | |
# base_token: [ "Eth", "Custom" ] | |
# deployment_mode: [ "Rollup", "Validium" ] | |
# runs-on: [ matterlabs-ci-runner-highmem-long ] | |
# env: | |
# SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,da_dispatcher${{ matrix.consensus && ',consensus' || '' }}${{ matrix.base_token == 'Custom' && ',base_token_ratio_persister' || '' }}" | |
# EXT_NODE_FLAGS: "${{ matrix.consensus && '-- --enable-consensus' || '' }}" | |
# steps: | |
# - name: Checkout code # Checks out the repository under $GITHUB_WORKSPACE, so the job can access it. | |
# uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 | |
# with: | |
# submodules: "recursive" | |
# fetch-depth: 0 | |
# - name: Setup environment | |
# run: | | |
# echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV | |
# echo $(pwd)/bin >> $GITHUB_PATH | |
# echo IN_DOCKER=1 >> .env | |
# echo RUN_CONTRACT_VERIFICATION_TEST=true >> .env | |
# echo ZKSYNC_DEBUG_LOGS=true >> .env | |
# - name: Start services | |
# run: | | |
# ci_localnet_up | |
# ci_run pre_download_compilers.sh | |
# ci_run sccache --start-server | |
# - name: Init | |
# run: | | |
# ci_run git config --global --add safe.directory /usr/src/zksync | |
# ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen | |
# ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts | |
# ci_run git config --global --add safe.directory /usr/src/zksync/contracts | |
# ci_run zk | |
# ci_run run_retried rustup show | |
# if [[ "${{ matrix.deployment_mode }}" == "Rollup" ]]; then | |
# ci_run zk config compile | |
# elif [[ "${{ matrix.deployment_mode }}" == "Validium" ]]; then | |
# ci_run zk env dev_validium_docker | |
# ci_run zk config compile dev_validium_docker | |
# fi | |
# ci_run zk init ${{ matrix.base_token == 'Custom' && '--base-token-name BAT' || ''}} ${{ matrix.deployment_mode == 'Validium' && '--validium-mode' || ''}} | |
# # `sleep 30` because we need to wait until server started properly | |
# - name: Run server | |
# run: | | |
# ci_run zk server --components=$SERVER_COMPONENTS &>>server.log & | |
# ci_run sleep 30 | |
# - name: Run external node | |
# run: | | |
# ci_run zk env ext-node-docker | |
# ci_run zk db setup | |
# ci_run zk external-node $EXT_NODE_FLAGS &>>ext-node.log & | |
# ci_run sleep 30 | |
# - name: Integration tests | |
# run: ci_run zk test i server --testPathIgnorePatterns 'contract-verification|snapshots-creator' | |
# - name: Run revert test | |
# run: | | |
# ENABLE_CONSENSUS=${{ matrix.consensus }} \ | |
# DEPLOYMENT_MODE=${{ matrix.deployment_mode }} \ | |
# PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE" \ | |
# ci_run zk test i revert-en | |
# # test terminates the nodes, so we restart them. | |
# if [[ "${{ matrix.deployment_mode }}" == "Rollup" ]]; then | |
# ZKSYNC_ENV=docker ci_run zk server --components=$SERVER_COMPONENTS &>>server.log & | |
# ZKSYNC_ENV=ext-node-docker ci_run zk external-node $EXT_NODE_FLAGS &>>ext-node.log & | |
# elif [[ "${{ matrix.deployment_mode }}" == "Validium" ]]; then | |
# ZKSYNC_ENV=dev_validium_docker ci_run zk server --components=$SERVER_COMPONENTS &>>server.log & | |
# ZKSYNC_ENV=ext-node-validium-docker ci_run zk external-node $EXT_NODE_FLAGS &>>ext-node.log & | |
# fi | |
# ci_run sleep 30 | |
# - name: Run upgrade test | |
# run: | | |
# ci_run zk env docker | |
# CHECK_EN_URL="http://0.0.0.0:3060" ci_run zk test i upgrade | |
# - name: Show server.log logs | |
# if: always() | |
# run: ci_run cat server.log || true | |
# - name: Show ext-node.log logs | |
# if: always() | |
# run: ci_run cat ext-node.log || true | |
# - name: Show contract_verifier.log logs | |
# if: always() | |
# run: ci_run cat ext-node.log || true | |
# - name: Show revert logs (main node) | |
# if: always() | |
# run: ci_run cat logs/revert/en/default/server.log || true | |
# - name: Show revert logs (EN) | |
# if: always() | |
# run: ci_run cat logs/revert/en/default/external_node.log || true | |
# - name: Show upgrade.log logs | |
# if: always() | |
# run: ci_run cat core/tests/upgrade-test/upgrade.log || true | |
# - name: Show sccache logs | |
# if: always() | |
# run: | | |
# ci_run sccache --show-stats || true | |
# ci_run cat /tmp/sccache_log.txt || true | |