From 1ba362bfc653e4d8ca80d57d79fd5d3180f818b8 Mon Sep 17 00:00:00 2001 From: ctrlc03 <93448202+ctrlc03@users.noreply.github.com> Date: Mon, 30 Oct 2023 08:36:03 +0000 Subject: [PATCH] fix(ci) - remove unsupported node versions from workflows and speed things up (#775) * fix(ci) - remove unsupported node versions from workflows * fix(integration-tests) - fix poll deployment integration test * feat(ci) - do not generate zKeys if no changes on the smart contracts * fix(ci check) - change way of getting which files changed * fix(ci): change syntax to access variables * fix(ci check) - debug * fix(ci check) - debug * fix(ci check) - debug * fix(ci check) - debug * fix(ci check) - remove hnctl start * fix(ci tests) - run hardhat on localhost * fix(ci tests) - debug * fix(ci tests) - debug * fix(docs) revert node version * fix(docs) revert node version * fix(ci) - re enable e2e tests * fix(ci) - skip subsidy6 for debugging * fix(ci) - try to run with generate zkeys * fix(ci) - make avaiable updated zkeys and updated download script * fix(ci) - fix wrong path in workflow * fix(ci) - run only integration tests and increase poll timeout * fix(tests) - ensure time travel has the correct time * fix(integration-tests) - fix time travel * fix(ci) - re enable e2e tests * chore(docs) - specify node versions * Update Node.js mentions for consistency * Update README.md Co-authored-by: Sam Richards * chore(cleanup) - remove uncommented line --------- Co-authored-by: Sam Richards --- .github/scripts/hnctl.sh | 20 ----- .github/workflows/circuit-build.yml | 2 +- .github/workflows/contracts-build.yml | 2 +- .github/workflows/core-build.yml | 2 +- .github/workflows/crypto-build.yml | 2 +- .github/workflows/domainobjs-build.yml | 2 +- .github/workflows/e2e.yml | 4 +- .github/workflows/release.yml | 4 +- .github/workflows/reusable-e2e.yml | 33 +++++++- README.md | 4 +- cli/ts/deployPollWithSigner.ts | 25 ++++-- docker/Dockerfile | 2 +- docs/README.md | 3 +- docs/installation.md | 4 +- integrationTests/integrations.yml | 2 +- integrationTests/scripts/download_zkeys.sh | 21 ++--- .../deployPollWithRandomSigner.test.ts | 77 ++++++++++--------- integrationTests/ts/__tests__/suites.ts | 2 +- 18 files changed, 111 insertions(+), 100 deletions(-) delete mode 100755 .github/scripts/hnctl.sh diff --git a/.github/scripts/hnctl.sh b/.github/scripts/hnctl.sh deleted file mode 100755 index 8c4d86182f..0000000000 --- a/.github/scripts/hnctl.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -e - -start_hardhat_network() { - installed=$(ls -lh `npm root --global --loglevel=error` | grep hnctl | wc -l) - if [ "$installed" = "0" ]; then - echo 'aa' - npm install -g hnctl - fi - hnctl start -} - -destroy_hardhat_network() { - hnctl destroy -} - - -$1 - diff --git a/.github/workflows/circuit-build.yml b/.github/workflows/circuit-build.yml index 37d9f4b4d1..0a6552695a 100644 --- a/.github/workflows/circuit-build.yml +++ b/.github/workflows/circuit-build.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [14, 16, 18] + node-version: [18] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/contracts-build.yml b/.github/workflows/contracts-build.yml index f90be60928..999cc377ce 100644 --- a/.github/workflows/contracts-build.yml +++ b/.github/workflows/contracts-build.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [14, 16, 18] + node-version: [18] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/core-build.yml b/.github/workflows/core-build.yml index 43e367b66b..12f67a59e4 100644 --- a/.github/workflows/core-build.yml +++ b/.github/workflows/core-build.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [14, 16, 18] + node-version: [18] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/crypto-build.yml b/.github/workflows/crypto-build.yml index 46fd856580..d5b8c59514 100644 --- a/.github/workflows/crypto-build.yml +++ b/.github/workflows/crypto-build.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [14, 16, 18] + node-version: [18] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/domainobjs-build.yml b/.github/workflows/domainobjs-build.yml index b865767f95..4a62bec6bb 100644 --- a/.github/workflows/domainobjs-build.yml +++ b/.github/workflows/domainobjs-build.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [14, 16, 18] + node-version: [18] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 10a70278dd..33ef9488c3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -24,10 +24,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Use Node.js 16 + - name: Use Node.js 18 uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 registry-url: 'https://registry.npmjs.org' # To prevent `npm install` failure of circuit package, diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c6dcb02526..21b7f8078f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,10 +25,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Use Node.js 16 + - name: Use Node.js 18 uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 registry-url: 'https://registry.npmjs.org' # To prevent `npm install` failure of circuit package, diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml index e5d99534e8..5da18984c1 100644 --- a/.github/workflows/reusable-e2e.yml +++ b/.github/workflows/reusable-e2e.yml @@ -15,6 +15,25 @@ jobs: uses: actions/setup-node@v3 with: node-version: 16 + + # Check for changes in the 'circuit' folder + - name: Get changed files + id: get-changed-files + uses: jitterbit/get-changed-files@v1 + with: + format: 'csv' + + - name: Check for changes in 'circuit' folder + id: check_changes + run: | + CHANGED_FILES=${{ steps.get-changed-files.outputs.all }} + if echo "$CHANGED_FILES" | grep -q "circuit" && ! echo "$CHANGED_FILES" | grep -q ".github/workflows/circuit-build.yml"; then + echo "CHANGED=true" >> $GITHUB_ENV + echo "Circuits have changes." + else + echo "CHANGED=false" >> $GITHUB_ENV + echo "No changes on circuits." + fi - name: Install dependencies run: | @@ -47,6 +66,8 @@ jobs: run: | cd contracts npm run compileSol + npm run hardhat & + sleep 5 - name: Download rapidsnark (1c137) run: | @@ -61,6 +82,7 @@ jobs: chmod +x /home/runner/work/maci/.local/bin/circom - name: Generate zkeys + if: ${{ env.CHANGED == 'true' }} run: | cd cli mkdir -p zkeys @@ -68,8 +90,11 @@ jobs: npx zkey-manager compile -c ./zkeys.config.yml npx zkey-manager genZkeys -c ./zkeys.config.yml - - name: Run Hardhat Network - run: ./.github/scripts/hnctl.sh start_hardhat_network + - name: Download zkeys + if: ${{ env.CHANGED == 'false' }} + run: | + cd integrationTests + npm run download-zkeys - name: Bespoke Test run: ./.github/scripts/run-e2e-tests.sh @@ -79,5 +104,5 @@ jobs: cd integrationTests npm run test - - name: Destroy Hardhat Network - run: ./.github/scripts/hnctl.sh destroy_hardhat_network + - name: Stop Hardhat + run: kill $(lsof -t -i:8545) diff --git a/README.md b/README.md index 8cd6c54d82..d3962d6640 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Below you can find a list of the packages included in this repository. ### Requirements -You should have Node.js (version `>=14 and <=18`) installed. Use `nvm` to install it and manage versions. +You should have Node.js v18 installed. Use `nvm` to install it and manage versions. This code has been tested with earlier Node versions (14 and 16), however we do recommend using Node 18. You also need a Ubuntu/Debian Linux machine on an Intel CPU. @@ -48,7 +48,7 @@ Install dependencies: If you are missing the correct version of glibc see `circuits/scripts/installGlibc.sh` -Clone this repository, install NodeJS dependencies, and build the source code: +Clone this repository, install Node.js dependencies, and build the source code: ```bash git clone git@github.com:privacy-scaling-explorations/maci.git diff --git a/cli/ts/deployPollWithSigner.ts b/cli/ts/deployPollWithSigner.ts index 90c15b4c2f..49ede14259 100644 --- a/cli/ts/deployPollWithSigner.ts +++ b/cli/ts/deployPollWithSigner.ts @@ -2,8 +2,10 @@ const { ethers } = require('hardhat') import { parseArtifact, deployVerifier, - deployPpt, - getDefaultSigner, + getDefaultSigner, + deployMessageProcessor, + deployTally, + deploySubsidy } from 'maci-contracts' import { @@ -188,8 +190,15 @@ const deployPollWithSigner = async (args: any) => { // Deploy a PollProcessorAndTallyer contract const verifierContract = await deployVerifier(true) console.log('Verifier:', verifierContract.address) - const pptContract = await deployPpt(verifierContract.address, true) - await pptContract.deployTransaction.wait() + + const mpContract = await deployMessageProcessor(verifierContract.address, contractAddrs['PoseidonT3'],contractAddrs['PoseidonT4'],contractAddrs['PoseidonT5'],contractAddrs['PoseidonT6']) + await mpContract.deployTransaction.wait() + + const tallyContract = await deployTally(verifierContract.address, contractAddrs['PoseidonT3'],contractAddrs['PoseidonT4'],contractAddrs['PoseidonT5'],contractAddrs['PoseidonT6']) + await tallyContract.deployTransaction.wait() + + const subsidyContract = await deploySubsidy(verifierContract.address, contractAddrs['PoseidonT3'],contractAddrs['PoseidonT4'],contractAddrs['PoseidonT5'],contractAddrs['PoseidonT6']) + await subsidyContract.deployTransaction.wait() const [ maciAbi ] = parseArtifact('MACI') const maciContract = new ethers.Contract( @@ -223,9 +232,13 @@ const deployPollWithSigner = async (args: any) => { const pollAddr = log.args._pollAddr console.log('Poll ID:', pollId.toString()) console.log('Poll contract:', pollAddr) - console.log('PollProcessorAndTallyer contract:', pptContract.address) + console.log('MessageProcessor contract:', mpContract.address) + console.log('Tally contract:', tallyContract.address) + console.log('Subsidy contract:', subsidyContract.address) contractAddrs['Verifier-' + pollId.toString()] = verifierContract.address - contractAddrs['PollProcessorAndTally-' + pollId.toString()] = pptContract.address + contractAddrs['MessageProcessor-' + pollId.toString()] = mpContract.address + contractAddrs['Tally-' + pollId.toString()] = tallyContract.address + contractAddrs['Subsidy-' + pollId.toString()] = subsidyContract.address contractAddrs['Poll-' + pollId.toString()] = pollAddr writeJSONFile(contractFilepath, contractAddrs) diff --git a/docker/Dockerfile b/docker/Dockerfile index 57004001a0..117c05102c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,7 +11,7 @@ RUN apt-get update \ && apt-get install -y libgmp-dev nlohmann-json3-dev nasm g++ -# install nvm and nodejs +# install nvm and Node.js ENV NODE_VERSION=14.18.1 RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash ENV NVM_DIR=/root/.nvm diff --git a/docs/README.md b/docs/README.md index a945b03f5c..a722b4f1f0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,8 +10,7 @@ MACI offers the following guarantees: * **Collusion resistance**: no-one except a trusted coordinator should be certain of the validity of a vote, reducing the effectiveness of bribery. -* **Receipt-freeness**: no-one voter prove (besides to the coordinator) which - way they voted. +* **Receipt-freeness**: no voter should be able to prove (besides to the coordinator) which way they voted. * **Privacy**: no-one except a trusted coordinator should be able to decrypt a vote. * **Uncensorability**: no-one — not even the trusted coordinator — should be diff --git a/docs/installation.md b/docs/installation.md index fa6ac216f9..d4a146fc9e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -6,8 +6,8 @@ You need the following to use MACI: - An x86-64 system - A Linux system, preferably a Debian-based distribution like Ubuntu -- NodeJS. Use [`nvm`](https://github.com/nvm-sh/nvm) to install it. MACI has - been tested with Node 14 and 16. +- Node.js: use [`nvm`](https://github.com/nvm-sh/nvm) to install it. MACI has + been tested with Node 14, 16 and 18. We do however recommend to use Node 18 as Node 14 is deprecated and Node 16 will soon be deprecated too. - The `libgmp-dev` `nlohmann-json3-dev` `nasm` and `g++` Debian/Ubuntu packages. They are needed to run `circom-helper`, which in turn is used to develop and test zk-SNARK circuits. diff --git a/integrationTests/integrations.yml b/integrationTests/integrations.yml index 0031f948bf..a30cbd29e2 100644 --- a/integrationTests/integrations.yml +++ b/integrationTests/integrations.yml @@ -17,7 +17,7 @@ invalidVote: voteCreditBalance: 1 constants: poll: - duration: 120 + duration: 250 intStateTreeDepth: 1 messageTreeDepth: 2 messageBatchDepth: 1 diff --git a/integrationTests/scripts/download_zkeys.sh b/integrationTests/scripts/download_zkeys.sh index b019e1464c..5100109b28 100755 --- a/integrationTests/scripts/download_zkeys.sh +++ b/integrationTests/scripts/download_zkeys.sh @@ -5,19 +5,12 @@ cd .. mkdir -p ../cli/zkeys -PKGS="zkeys_10-2-1-2_glibc-211.tar.gz ProcessMessages_10-2-1-2_test.0.zkey TallyVotes_10-1-2_test.0.zkey SubsidyPerBatch_10-1-2_test.0.zkey" +URL=https://maci-develop-fra.s3.eu-central-1.amazonaws.com/v1.1.2/maci_keys_10-2-1-2_dev.tar.gz +DIR_NAME="maci_keys.tar.gz" +OUT_DIR=../cli/ + +echo "downloading $URL" +curl $URL -o "$OUT_DIR/$DIR_NAME" +tar -xvf "$OUT_DIR/$DIR_NAME" -C "$OUT_DIR" -BASE_URL=https://maci-develop-fra.s3.eu-central-1.amazonaws.com/v1.1.1-aa4ba27/10-2-1-2 -OUT_DIR=../cli/zkeys -for p in $PKGS -do - url="$BASE_URL/$p" - echo "downloading $url" - curl $url -o "$OUT_DIR/$p" - extension="${p##*.}" - if [ "$extension" == "gz" ] - then - tar -xvf "$OUT_DIR/$p" -C "$OUT_DIR" - fi -done diff --git a/integrationTests/ts/__tests__/deployPollWithRandomSigner.test.ts b/integrationTests/ts/__tests__/deployPollWithRandomSigner.test.ts index 210096cae5..951ed6be14 100644 --- a/integrationTests/ts/__tests__/deployPollWithRandomSigner.test.ts +++ b/integrationTests/ts/__tests__/deployPollWithRandomSigner.test.ts @@ -30,50 +30,51 @@ describe('Test deployPollWithSigner', () => { const test = data.suites[0] it(test.description, async () => { const result = await executeSuite(test, expect) - console.log(result) expect(result).toBeTruthy() - - let caughtException = false - try { + let caughtException = false + try { const config = loadYaml() - const tally = JSON.parse(fs.readFileSync(path.join(__dirname, '../../../cli/tally.json')).toString()) - const coordinatorKeypair = new Keypair() + const tally = JSON.parse(fs.readFileSync(path.join(__dirname, '../../../cli/tally.json')).toString()) + const coordinatorKeypair = new Keypair() + + // start another round by a random user + const wallet = Wallet.createRandom() - // start another round by a random user - const wallet = Wallet.createRandom() + // fund wallet with 1 ETH + const totalAmount = utils.parseEther("1") + const fundWalletCommand = `node build/index.js fundWallet` + + ` -w ${wallet.address}` + + ` -a ${totalAmount.toString()}` + execute(fundWalletCommand) - // fund wallet with 1 ETH - const totalAmount = utils.parseEther("1") - const fundWalletCommand = `node build/index.js fundWallet` + - ` -w ${wallet.address}` + - ` -a ${totalAmount.toString()}` - execute(fundWalletCommand) - - const duration = 999999 - const deployPollCommand = `node build/index.js deployPollWithSigner` + - ` -x ${tally.maci}` + - ` -s ${wallet.privateKey}` + - ` -pk ${coordinatorKeypair.pubKey.serialize()}` + - ` -t ${duration}` + - ` -g ${config.constants.maci.maxMessages}` + - ` -mv ${config.constants.maci.maxVoteOptions}` + - ` -i ${config.constants.poll.intStateTreeDepth}` + - ` -m ${config.constants.poll.messageTreeDepth}` + - ` -b ${config.constants.poll.messageBatchDepth}` + - ` -v ${config.constants.maci.voteOptionTreeDepth}` - execute(deployPollCommand) + // sleep for 5 seconds + await new Promise(resolve => setTimeout(resolve, 5000)) + + const duration = 999999 + const deployPollCommand = `node build/index.js deployPollWithSigner` + + ` -x ${tally.maci}` + + ` -s ${wallet.privateKey}` + + ` -pk ${coordinatorKeypair.pubKey.serialize()}` + + ` -t ${duration}` + + ` -g ${config.constants.maci.maxMessages}` + + ` -mv ${config.constants.maci.maxVoteOptions}` + + ` -i ${config.constants.poll.intStateTreeDepth}` + + ` -m ${config.constants.poll.messageTreeDepth}` + + ` -b ${config.constants.poll.messageBatchDepth}` + + ` -v ${config.constants.maci.voteOptionTreeDepth}` + execute(deployPollCommand) - // this is the second poll with pollId = 1 - const pollId = 1 - const pollDuration = await getPollDuration(tally.provider, tally.maci, pollId) - expect(pollDuration.toString()).toEqual(duration.toString()) - - } catch (e) { - console.log(e) - caughtException = true - expect(caughtException).toEqual(false) - } + // this is the second poll with pollId = 1 + const pollId = 1 + const pollDuration = await getPollDuration(tally.provider, tally.maci, pollId) + expect(pollDuration.toString()).toEqual(duration.toString()) + + } catch (e) { + console.log(e) + caughtException = true + } + expect(caughtException).toEqual(false) }) }) diff --git a/integrationTests/ts/__tests__/suites.ts b/integrationTests/ts/__tests__/suites.ts index c6e191f519..af0dd3b5d5 100644 --- a/integrationTests/ts/__tests__/suites.ts +++ b/integrationTests/ts/__tests__/suites.ts @@ -230,7 +230,7 @@ const executeSuite = async (data: any, expect: any) => { } } - const timeTravelCommand = `node build/index.js timeTravel -s ${config.constants.maci.votingDuration}` + const timeTravelCommand = `node build/index.js timeTravel -s ${config.constants.poll.duration}` execute(timeTravelCommand) const mergeMessagesCommand = `node build/index.js mergeMessages -x ${maciAddress} -o ${pollId}`