Skip to content

probably port CI to toolbox #8

probably port CI to toolbox

probably port CI to toolbox #8

name: Workflow template for CI jobs for Core Components
on:
workflow_call:
inputs:
compilers:
description: "JSON of required compilers and their versions"
type: string
required: false
default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18", "1.3.19", "1.3.21"] } , { "zkvyper": ["1.3.13"] }]'
jobs:
# lint:
# name: lint
# uses: ./.github/workflows/ci-core-lint-reusable.yml
unit-tests:
runs-on: [ matterlabs-ci-runner-highmem-long ]
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
with:
submodules: "recursive"
fetch-depth: 0
- name: Setup environment
run: |
echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
echo $(pwd)/bin >> $GITHUB_PATH
echo IN_DOCKER=1 >> .env
# TODO: Remove when we after upgrade of hardhat-plugins
- name: pre-download compilers
run: |
# Download needed versions of vyper compiler
# Not sanitized due to unconventional path and tags
mkdir -p ./hardhat-nodejs/compilers-v2/vyper/linux
wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10 https://github.com/vyperlang/vyper/releases/download/v0.3.10/vyper.0.3.10+commit.91361694.linux
wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3+commit.48e326f0.linux
chmod +x ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10
chmod +x ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3
COMPILERS_JSON='${{ inputs.compilers }}'
echo "$COMPILERS_JSON" | jq -r '.[] | to_entries[] | .key as $compiler | .value[] | "\(.),\($compiler)"' | while IFS=, read -r version compiler; do
mkdir -p "./hardhat-nodejs/compilers-v2/$compiler"
wget -nv -O "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}" "https://github.com/matter-labs/${compiler}-bin/releases/download/v${version}/${compiler}-linux-amd64-musl-v${version}"
chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}"
done
- name: Start services
run: |
ci_localnet_up
ci_run sccache --start-server
- name: Init
run: |
ci_run run_retried rustup show
ci_run ./bin/zkt
ci_run zk_supervisor contracts
# FIXME: enable contract tests once tehy are stable
#- name: Contracts unit tests
# run: ci_run yarn l1-contracts test
- name: Rust unit tests
run: |
ci_run zk_supervisor test rust
# Benchmarks are not tested by `cargo nextest` unless specified explicitly, and even then `criterion` harness is incompatible
# with how `cargo nextest` runs tests. Thus, we run criterion-based benchmark tests manually.
ci_run cargo test --release -p vm-benchmark --bench oneshot --bench batch
# FIXME: support loadtest together with sync layer.
# loadtest:
# runs-on: [ matterlabs-ci-runner-high-performance ]
# strategy:
# fail-fast: false
# matrix:
# # FIXME: support new VM mode
# vm_mode: ["OLD"]
# steps:
# - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
# with:
# submodules: "recursive"
# fetch-depth: 0
# - name: Setup environment
# run: |
# echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
# echo $(pwd)/bin >> $GITHUB_PATH
# echo IN_DOCKER=1 >> .env
# - name: Loadtest configuration
# run: |
# echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'NEW' && 21000 || 16000 }} >> .env
# echo ACCOUNTS_AMOUNT="100" >> .env
# echo MAX_INFLIGHT_TXS="10" >> .env
# echo SYNC_API_REQUESTS_LIMIT="15" >> .env
# echo FAIL_FAST=true >> .env
# echo IN_DOCKER=1 >> .env
# - name: Start services
# run: |
# ci_localnet_up
# ci_run sccache --start-server
# - name: Init
# run: |
# ci_run git config --global --add safe.directory /usr/src/zksync
# ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen
# ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
# ci_run git config --global --add safe.directory /usr/src/zksync/contracts
# ci_run ./bin/zkt
# ci_run zk_inception chain create \
# --chain-name legacy \
# --chain-id sequential \
# --prover-mode no-proofs \
# --wallet-creation localhost \
# --l1-batch-commit-data-generator-mode rollup \
# --base-token-address 0x0000000000000000000000000000000000000001 \
# --base-token-price-nominator 1 \
# --base-token-price-denominator 1 \
# --set-as-default false \
# --ignore-prerequisites \
# --legacy-bridge
# ci_run zk_inception ecosystem init --dev --verbose
# ci_run zk_supervisor contracts --test-contracts
# # `sleep 60` because we need to wait until server added all the tokens
# - name: Run server
# run: |
# ci_run sed -i -e 's/mode: FULL/mode: LIGHTWEIGHT/' chains/legacy/configs/general.yaml
# ci_run sed -i -e 's/state_keeper_fast_vm_mode:.*/state_keeper_fast_vm_mode: ${{ matrix.vm_mode }}/' chains/legacy/configs/general.yaml
# ci_run sed -i -e 's/delay_interval:.*/delay_interval: 50/' chains/legacy/configs/general.yaml
# ci_run zk_inception server --uring --chain=legacy --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
# ci_run sleep 60
# - name: Perform loadtest
# run: ci_run zk_supervisor t loadtest -v --chain=legacy
# - name: Show server.log logs
# if: always()
# run: ci_run cat server.log || true
# - name: Show sccache logs
# if: always()
# run: |
# ci_run sccache --show-stats || true
# ci_run cat /tmp/sccache_log.txt || true
integration:
name: Integration (consensus=${{ matrix.consensus }}, base_token=${{ matrix.base_token }}, gateway = ${{ matrix.gateway }})
strategy:
# In matrix jobs, fail-fast is true by default.
# To be consistent with the rest of the workflow we disable it explicitly.
fail-fast: false
matrix:
# FIXME: support consensus tests for SL migration
consensus: [false]
# FIXME: support custom tokens
base_token: ["Eth"]
# FIXME: run tests even when not using synclayer
gateway: [true]
env:
SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher${{ matrix.consensus && ',consensus' || '' }}${{ matrix.base_token == 'Custom' && ',base_token_ratio_persister' || '' }}"
runs-on: [ matterlabs-ci-runner-highmem-long ]
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
with:
submodules: "recursive"
fetch-depth: 0
- name: Setup environment
run: |
echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
echo $(pwd)/bin >> $GITHUB_PATH
echo IN_DOCKER=1 >> .env
echo RUN_CONTRACT_VERIFICATION_TEST=true >> .env
echo ZKSYNC_DEBUG_LOGS=true >> .env
- name: Download zksolc/solc and zkvyper/vyper
run: |
sudo apt update && sudo apt install wget -y
mkdir -p $(pwd)/etc/solc-bin/0.8.23
wget https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/solc-linux-amd64-v0.8.23%2Bcommit.f704f362
mv solc-linux-amd64-v0.8.23+commit.f704f362 $(pwd)/etc/solc-bin/0.8.23/solc
chmod +x $(pwd)/etc/solc-bin/0.8.23/solc
mkdir -p $(pwd)/etc/solc-bin/zkVM-0.8.23-1.0.0
wget https://github.com/matter-labs/era-solidity/releases/download/0.8.23-1.0.0/solc-linux-amd64-0.8.23-1.0.0 -O $(pwd)/etc/solc-bin/zkVM-0.8.23-1.0.0/solc
chmod +x $(pwd)/etc/solc-bin/zkVM-0.8.23-1.0.0/solc
mkdir -p $(pwd)/etc/zksolc-bin/v1.3.21
wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.21
mv zksolc-linux-amd64-musl-v1.3.21 $(pwd)/etc/zksolc-bin/v1.3.21/zksolc
chmod +x $(pwd)/etc/zksolc-bin/v1.3.21/zksolc
mkdir -p $(pwd)/etc/vyper-bin/0.3.10
wget -O vyper0.3.10 https://github.com/vyperlang/vyper/releases/download/v0.3.10/vyper.0.3.10%2Bcommit.91361694.linux
mv vyper0.3.10 $(pwd)/etc/vyper-bin/0.3.10/vyper
chmod +x $(pwd)/etc/vyper-bin/0.3.10/vyper
mkdir -p $(pwd)/etc/zkvyper-bin/v1.3.13
wget https://github.com/matter-labs/zkvyper-bin/raw/main/linux-amd64/zkvyper-linux-amd64-musl-v1.3.13
mv zkvyper-linux-amd64-musl-v1.3.13 $(pwd)/etc/zkvyper-bin/v1.3.13/zkvyper
chmod +x $(pwd)/etc/zkvyper-bin/v1.3.13/zkvyper
- name: Start services
run: |
ci_localnet_up
ci_run pre_download_compilers.sh
ci_run sccache --start-server
- name: Build zk_toolbox
run: ci_run bash -c "./bin/zkt"
- name: Create log directories
run: |
SERVER_LOGS_DIR=logs/server
INTEGRATION_TESTS_LOGS_DIR=logs/integration_tests
INTEGRATION_TESTS_EN_LOGS_DIR=logs/integration_tests/en
SNAPSHOT_RECOVERY_LOGS_DIR=logs/snapshot_recovery/
GENESIS_RECOVERY_LOGS_DIR=logs/genesis_recovery/
EXTERNAL_NODE_LOGS_DIR=logs/external_node
REVERT_LOGS_DIR=logs/revert
mkdir -p $SERVER_LOGS_DIR
mkdir -p $INTEGRATION_TESTS_LOGS_DIR
mkdir -p $INTEGRATION_TESTS_EN_LOGS_DIR
mkdir -p $SNAPSHOT_RECOVERY_LOGS_DIR
mkdir -p $GENESIS_RECOVERY_LOGS_DIR
mkdir -p $EXTERNAL_NODE_LOGS_DIR
mkdir -p $REVERT_LOGS_DIR
echo "SERVER_LOGS_DIR=$SERVER_LOGS_DIR" >> $GITHUB_ENV
echo "INTEGRATION_TESTS_LOGS_DIR=$INTEGRATION_TESTS_LOGS_DIR" >> $GITHUB_ENV
echo "INTEGRATION_TESTS_EN_LOGS_DIR=$INTEGRATION_TESTS_EN_LOGS_DIR" >> $GITHUB_ENV
echo "SNAPSHOT_RECOVERY_LOGS_DIR=$SNAPSHOT_RECOVERY_LOGS_DIR" >> $GITHUB_ENV
echo "GENESIS_RECOVERY_LOGS_DIR=$GENESIS_RECOVERY_LOGS_DIR" >> $GITHUB_ENV
echo "EXTERNAL_NODE_LOGS_DIR=$EXTERNAL_NODE_LOGS_DIR" >> $GITHUB_ENV
echo "REVERT_LOGS_DIR=$REVERT_LOGS_DIR" >> $GITHUB_ENV
- name: Initialize ecosystem
run: |
ci_run git config --global --add safe.directory /usr/src/zksync
ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
ci_run git config --global --add safe.directory /usr/src/zksync/contracts
ci_run zk_inception ecosystem init --deploy-paymaster --deploy-erc20 \
--deploy-ecosystem --l1-rpc-url=http://localhost:8545 \
--server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
--server-db-name=zksync_server_localhost_era \
--prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
--prover-db-name=zksync_prover_localhost_era \
--ignore-prerequisites --verbose \
--observability=false
- name: Initialize gateway chain
run: |
ci_run zk_inception chain create \
--chain-name gateway \
--chain-id 505 \
--prover-mode no-proofs \
--wallet-creation localhost \
--l1-batch-commit-data-generator-mode rollup \
--base-token-address 0x0000000000000000000000000000000000000001 \
--base-token-price-nominator 1 \
--base-token-price-denominator 1 \
--set-as-default false \
--ignore-prerequisites
ci_run zk_inception chain init \
--deploy-paymaster \
--l1-rpc-url=http://localhost:8545 \
--server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
--server-db-name=zksync_server_localhost_gateway \
--prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
--prover-db-name=zksync_prover_localhost_gateway \
--port-offset 2000 \
--chain gateway \
ci_run zk_inception chain convert-to-gateway --chain gateway --ignore-prerequisites
- name: Build test dependencies
run: |
ci_run zk_supervisor test build
- name: Run gateway
run: |
ci_run zk_inception server --ignore-prerequisites --chain gateway &> ${{ env.SERVER_LOGS_DIR }}/gateway.log &
ci_run sleep 5
- name: Migrate chain to gateway
run: |
ci_run zk_inception chain migrate-to-gateway --chain era --gateway-chain-name gateway
- name: Run era chain
ci_run zk_inception server --ignore-prerequisites --chain era &> ${{ env.SERVER_LOGS_DIR }}/rollup.log &
- name: Run integration tests
run: |
ci_run zk_supervisor test integration --no-deps --ignore-prerequisites --chain era &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/rollup.log &
PID1=$!
wait $PID1
- name: Upload logs
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
if: always()
with:
name: logs
path: logs
# FIXME: restore snapshot test
# - name: Snapshot recovery test
# # We use `yarn` directly because the test launches `zk` commands in both server and EN envs.
# # An empty topmost environment helps avoid a mess when redefining env vars shared between both envs
# # (e.g., DATABASE_URL).
# #
# # Since `base_token` doesn't meaningfully influence the test, we use it as a flag for
# # enabling / disabling tree during pruning.
# run: |
# if [[ "${{ matrix.deployment_mode }}" == "Validium" ]]; then
# ci_run zk config compile ext-node-validium
# ci_run zk config compile ext-node-validium-docker
# fi
# ENABLE_CONSENSUS=${{ matrix.consensus }} \
# DEPLOYMENT_MODE=${{ matrix.deployment_mode }} \
# SNAPSHOTS_CREATOR_VERSION=${{ matrix.deployment_mode == 'Validium' && '0' || '1' }} \
# DISABLE_TREE_DURING_PRUNING=${{ matrix.base_token == 'Eth' }} \
# ETH_CLIENT_WEB3_URL="http://localhost:8545" \
# PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE,DISABLE_TREE_DURING_PRUNING,SNAPSHOTS_CREATOR_VERSION,ETH_CLIENT_WEB3_URL" \
# ci_run yarn recovery-test snapshot-recovery-test
# - name: Genesis recovery test
# run: |
# ENABLE_CONSENSUS=${{ matrix.consensus }} \
# DEPLOYMENT_MODE=${{ matrix.deployment_mode }} \
# ETH_CLIENT_WEB3_URL="http://localhost:8545" \
# PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE,ETH_CLIENT_WEB3_URL" \
# ci_run yarn recovery-test genesis-recovery-test
# FIXME: restore fee proj tests
# - name: Fee projection tests
# run: ci_run zk test i fees
# FIXME: restore revert tests
# - name: Run revert test
# run: |
# ci_run pkill zksync_server || true
# ci_run sleep 2
# ENABLE_CONSENSUS=${{ matrix.consensus }} DEPLOYMENT_MODE=Rollup PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE" ci_run zk test i revert
# FIXME: restore upgrade test for SL
# # This test should be the last one as soon as it
# # finished bootloader will be different
# - name: Run upgrade test
# run: |
# ci_run pkill zksync_server || true
# ci_run sleep 10
# ci_run zk test i upgrade
# FIXME: restore tests with EN and sync layer
# external-node:
# name: External node (consensus=${{ matrix.consensus }}, base_token=${{ matrix.base_token }}, deployment_mode=${{ matrix.deployment_mode }})
# strategy:
# fail-fast: false
# matrix:
# consensus: [ false, true ]
# base_token: [ "Eth", "Custom" ]
# deployment_mode: [ "Rollup", "Validium" ]
# runs-on: [ matterlabs-ci-runner-highmem-long ]
# env:
# SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,da_dispatcher${{ matrix.consensus && ',consensus' || '' }}${{ matrix.base_token == 'Custom' && ',base_token_ratio_persister' || '' }}"
# EXT_NODE_FLAGS: "${{ matrix.consensus && '-- --enable-consensus' || '' }}"
# steps:
# - name: Checkout code # Checks out the repository under $GITHUB_WORKSPACE, so the job can access it.
# uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
# with:
# submodules: "recursive"
# fetch-depth: 0
# - name: Setup environment
# run: |
# echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
# echo $(pwd)/bin >> $GITHUB_PATH
# echo IN_DOCKER=1 >> .env
# echo RUN_CONTRACT_VERIFICATION_TEST=true >> .env
# echo ZKSYNC_DEBUG_LOGS=true >> .env
# - name: Start services
# run: |
# ci_localnet_up
# ci_run pre_download_compilers.sh
# ci_run sccache --start-server
# - name: Init
# run: |
# ci_run git config --global --add safe.directory /usr/src/zksync
# ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen
# ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
# ci_run git config --global --add safe.directory /usr/src/zksync/contracts
# ci_run zk
# ci_run run_retried rustup show
# if [[ "${{ matrix.deployment_mode }}" == "Rollup" ]]; then
# ci_run zk config compile
# elif [[ "${{ matrix.deployment_mode }}" == "Validium" ]]; then
# ci_run zk env dev_validium_docker
# ci_run zk config compile dev_validium_docker
# fi
# ci_run zk init ${{ matrix.base_token == 'Custom' && '--base-token-name BAT' || ''}} ${{ matrix.deployment_mode == 'Validium' && '--validium-mode' || ''}}
# # `sleep 30` because we need to wait until server started properly
# - name: Run server
# run: |
# ci_run zk server --components=$SERVER_COMPONENTS &>>server.log &
# ci_run sleep 30
# - name: Run external node
# run: |
# ci_run zk env ext-node-docker
# ci_run zk db setup
# ci_run zk external-node $EXT_NODE_FLAGS &>>ext-node.log &
# ci_run sleep 30
# - name: Integration tests
# run: ci_run zk test i server --testPathIgnorePatterns 'contract-verification|snapshots-creator'
# - name: Run revert test
# run: |
# ENABLE_CONSENSUS=${{ matrix.consensus }} \
# DEPLOYMENT_MODE=${{ matrix.deployment_mode }} \
# PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE" \
# ci_run zk test i revert-en
# # test terminates the nodes, so we restart them.
# if [[ "${{ matrix.deployment_mode }}" == "Rollup" ]]; then
# ZKSYNC_ENV=docker ci_run zk server --components=$SERVER_COMPONENTS &>>server.log &
# ZKSYNC_ENV=ext-node-docker ci_run zk external-node $EXT_NODE_FLAGS &>>ext-node.log &
# elif [[ "${{ matrix.deployment_mode }}" == "Validium" ]]; then
# ZKSYNC_ENV=dev_validium_docker ci_run zk server --components=$SERVER_COMPONENTS &>>server.log &
# ZKSYNC_ENV=ext-node-validium-docker ci_run zk external-node $EXT_NODE_FLAGS &>>ext-node.log &
# fi
# ci_run sleep 30
# - name: Run upgrade test
# run: |
# ci_run zk env docker
# CHECK_EN_URL="http://0.0.0.0:3060" ci_run zk test i upgrade
# - name: Show server.log logs
# if: always()
# run: ci_run cat server.log || true
# - name: Show ext-node.log logs
# if: always()
# run: ci_run cat ext-node.log || true
# - name: Show contract_verifier.log logs
# if: always()
# run: ci_run cat ext-node.log || true
# - name: Show revert logs (main node)
# if: always()
# run: ci_run cat logs/revert/en/default/server.log || true
# - name: Show revert logs (EN)
# if: always()
# run: ci_run cat logs/revert/en/default/external_node.log || true
# - name: Show upgrade.log logs
# if: always()
# run: ci_run cat core/tests/upgrade-test/upgrade.log || true
# - name: Show sccache logs
# if: always()
# run: |
# ci_run sccache --show-stats || true
# ci_run cat /tmp/sccache_log.txt || true