feat(genLocalState) - add e2e test, add sleep feature to prevent rate…

… limiting, change utility name, add better error handling
privacy-scaling-explorations · Nov 9, 2023 · 808ac9d · 808ac9d
1 parent 95ba802
commit 808ac9d
Show file tree

Hide file tree

Showing 7 changed files with 142 additions and 38 deletions.
diff --git a/cli/tests/prepare_test.sh b/cli/tests/prepare_test.sh
@@ -51,6 +51,55 @@ clean() {
 
 }
 
+gen_proofs_pre_fetch() {
+     echo "merge messages ..."
+    $MACI_CLI mergeMessages \
+        --poll-id $1
+
+    echo "merge signups ..."
+    $MACI_CLI mergeSignups \
+        --poll-id $1
+
+    echo "gen proofs ..."
+
+    # generate the local state
+    $MACI_CLI genLocalState \
+        --poll-id $1 \
+        --output localState.json \
+        --privkey macisk.49953af3585856f539d194b46c82f4ed54ec508fb9b882940cbe68bbc57e59e \
+        --num-blocks-per-request 50
+
+    ARCH=$(uname -m)
+    if [[ $ARCH == *"arm"* ]]; then
+        # ARM parameters
+        $MACI_CLI genProofs \
+            --privkey macisk.49953af3585856f539d194b46c82f4ed54ec508fb9b882940cbe68bbc57e59e \
+            --poll-id $1 \
+            --process-wasm ./zkeys/ProcessMessages_"$PROCESS_MESSAGES_PARAMS"_js/ProcessMessages_"$PROCESS_MESSAGES_PARAMS".wasm \
+            --tally-wasm ./zkeys/TallyVotes_"$TALLY_VOTES_PARAMS"_js/TallyVotes_"$TALLY_VOTES_PARAMS".wasm \
+            --process-zkey "$ZKEYS_DIR"/ProcessMessages_"$PROCESS_MESSAGES_PARAMS".0.zkey \
+            --tally-zkey "$ZKEYS_DIR"/TallyVotes_"$TALLY_VOTES_PARAMS".0.zkey \
+            --tally-file tally.json \
+            --output proofs/ \
+            --state-file localState.json \
+            $SUBSIDDY_OPTION_GEN_PROOFS
+    else
+        # Intel parameters
+        $MACI_CLI genProofs \
+            --privkey macisk.49953af3585856f539d194b46c82f4ed54ec508fb9b882940cbe68bbc57e59e \
+            --poll-id $1 \
+            --rapidsnark ~/rapidsnark/build/prover \
+            --process-witnessgen ./zkeys/ProcessMessages_"$PROCESS_MESSAGES_PARAMS" \
+            --tally-witnessgen ./zkeys/TallyVotes_"$TALLY_VOTES_PARAMS" \
+            --process-zkey "$ZKEYS_DIR"/ProcessMessages_"$PROCESS_MESSAGES_PARAMS".0.zkey \
+            --tally-zkey "$ZKEYS_DIR"/TallyVotes_"$TALLY_VOTES_PARAMS".0.zkey \
+            --tally-file tally.json \
+            --output proofs/ \
+            --state-file localState.json \
+            $SUBSIDDY_OPTION_GEN_PROOFS
+    fi
+}
+
 gen_proofs() {
     echo "merge messages ..."
     $MACI_CLI mergeMessages \

diff --git a/cli/tests/vanilla/testPreFetch.sh b/cli/tests/vanilla/testPreFetch.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+BASE_DIR="$(dirname "$BASH_SOURCE")"
+
+. "$BASE_DIR"/../prepare_test.sh
+
+# 1 signup and 1 message
+clean
+
+POLL_ID=0
+
+init_maci
+deploy_poll
+
+
+$MACI_CLI signup \
+    --pubkey macipk.3e7bb2d7f0a1b7e980f1b6f363d1e3b7a12b9ae354c2cd60a9cfa9fd12917391
+
+$MACI_CLI publish \
+    --pubkey macipk.3e7bb2d7f0a1b7e980f1b6f363d1e3b7a12b9ae354c2cd60a9cfa9fd12917391 \
+    --privkey macisk.fd7aa614ec4a82716ffc219c24fd7e7b52a2b63b5afb17e81c22fe21515539c \
+    --state-index 1 \
+    --vote-option-index 0 \
+    --new-vote-weight 9 \
+    --nonce 1 \
+    --poll-id "$POLL_ID"
+
+$MACI_CLI timeTravel \
+    --seconds 90
+
+gen_proofs_pre_fetch "$POLL_ID"
+
+prove_and_verify_on_chain "$POLL_ID"
diff --git a/cli/ts/fetchLogs.ts → cli/ts/genLocalState.ts b/cli/ts/fetchLogs.ts → cli/ts/genLocalState.ts
@@ -3,7 +3,6 @@ import * as fs from 'fs'
 
 import {
     genMaciStateFromContract,
-    parseArtifact,
 } from 'maci-contracts'
 
 import {
@@ -20,10 +19,9 @@ import {readJSONFile} from 'maci-common'
 import {contractFilepath} from './config'
 import { Keypair, PrivKey } from 'maci-domainobjs'
 
-
 const configureSubparser = (subparsers: any) => {
     const parser = subparsers.addParser(
-        'fetchLogs',
+        'genLocalState',
         { addHelp: true },
     )
 
@@ -116,9 +114,17 @@ const configureSubparser = (subparsers: any) => {
             help: 'The output file for signups and messages',
         }
     )
+
+    parser.addArgument(
+        ['-s', '--sleep'],
+        {
+            type: 'int',
+            help: 'The number of milliseconds to sleep between each RPC request to avoid getting rate limited',
+        }
+    )
 }
 
-const fetchLogs = async (args: any) => {
+const genLocalState = async (args: any) => {
     // read the contract addresses from the config file
     const contractAddrs = readJSONFile(contractFilepath)
 
@@ -127,7 +133,7 @@ const fetchLogs = async (args: any) => {
 
     // ensure we have at least one address
     if ((!contractAddrs||!contractAddrs["MACI"]||!contractAddrs[pollArtifactName]) && !args.maci_contract && !args.poll_contract) {
-        console.error('Error: MACI and Poll contract addresses are empty or this poll Id does not exist') 
+        console.error('Error: If no contract address is stored locally, please specify the poll id, the maci contract address and the poll address') 
         return 
     }
     // prioritize cli flag arg
@@ -136,12 +142,12 @@ const fetchLogs = async (args: any) => {
 
     // validate it's a valid eth address
     if (!validateEthAddress(maciAddress)) {
-        console.error('Error: invalid MACI contract address')
+        console.error("Error: invalid MACI contract address. Ensure the address starts with '0x' followed by the 40 hexadecimal characters")
         return
     }
 
     if (!validateEthAddress(pollAddress)) {
-        console.error("Error: invalid Poll contract address")
+        console.error("Error: invalid Poll contract address. Ensure the address starts with '0x' followed by the 40 hexadecimal characters")
         return 
     }
 
@@ -160,22 +166,6 @@ const fetchLogs = async (args: any) => {
         return
     }
 
-    // fetch abis
-    const [ maciContractAbi ] = parseArtifact('MACI')
-    const [ pollContractAbi ] = parseArtifact('Poll')
-
-    const maciContract = new ethers.Contract(
-        maciAddress,
-        maciContractAbi,
-        provider,
-    )
-
-    const pollContract = new ethers.Contract(
-        pollAddress,
-        pollContractAbi,
-        provider
-    )
-
     // The coordinator's MACI private key
     let serializedPrivkey: string 
     if (args.prompt_for_maci_privkey) {
@@ -194,7 +184,6 @@ const fetchLogs = async (args: any) => {
 
     // calculate the end block number 
     const endBlockNumber = args.end_block ? args.end_block : await provider.getBlockNumber()
-    console.log('Fetching logs till:', endBlockNumber)
 
     console.log('Fetching signup and publish message logs')
     // some rpc endpoint like bsc chain has limitation to retreive history logs
@@ -204,15 +193,17 @@ const fetchLogs = async (args: any) => {
         const txn = await provider.getTransaction(txHash);
         fromBlock = txn.blockNumber
     }
-    console.log(`fromBlock = ${fromBlock}`)
+    console.log(`Fetching logs from ${fromBlock} till ${endBlockNumber} and generating the offline maci state`)
+
     const maciState = await genMaciStateFromContract(
         provider,
         maciAddress,
         coordinatorKeypair,
         pollId,
         fromBlock,
         args.blocks_per_batch,
-        args.end_block
+        args.end_block,
+        args.sleep
     )
 
     // write the state to a file
@@ -221,6 +212,6 @@ const fetchLogs = async (args: any) => {
 }
 
 export {
-    fetchLogs,
+    genLocalState,
     configureSubparser,
 }
diff --git a/cli/ts/genProofs.ts b/cli/ts/genProofs.ts
@@ -418,9 +418,13 @@ const genProofs = async (args: any) => {
     let maciState: MaciState
     // Build an off-chain representation of the MACI contract using data in the contract storage
     if (args.state_file) {
-        // @todo actually read the file first 
         const content = JSON.parse(fs.readFileSync(args.state_file, 'utf-8').toString())
-        maciState = MaciState.fromJSON(content)
+        try {
+            maciState = MaciState.fromJSON(content)
+        } catch (error: any) {
+            console.log('The provided state file is invalid. Please check that the correct path was provided and try again.')
+            return 
+        }
         // ensure we merge all messages
         maciState.polls.forEach((poll: Poll) => poll.mergeAllMessages()) 
     } else {

diff --git a/cli/ts/index.ts b/cli/ts/index.ts
@@ -130,9 +130,9 @@ import {
     configureSubparser as configureSubparserForCheckVerifyKey,
 } from './checkVerifyingKey'
 import {
-    fetchLogs,
-    configureSubparser as configureSubparserForFetchLogs,
-} from './fetchLogs'
+    genLocalState,
+    configureSubparser as configureSubparserForGenerateLocalState,
+} from './genLocalState'
 
 const main = async () => {
     const parser = new argparse.ArgumentParser({ 
@@ -201,8 +201,8 @@ const main = async () => {
     // Subcommand: checkVerifyKey
     configureSubparserForCheckVerifyKey(subparsers)
 
-    // Subcommand: fetchlogs
-    configureSubparserForFetchLogs(subparsers)
+    // Subcommand: genLocalState
+    configureSubparserForGenerateLocalState(subparsers)
 
     const args = parser.parseArgs()
 
@@ -245,8 +245,8 @@ const main = async () => {
         await verify(args)
     } else if (args.subcommand === 'checkVerifyingKey') {
         await checkVerifyingKey(args)
-    } else if (args.subcommand === 'fetchLogs') {
-        await fetchLogs(args)
+    } else if (args.subcommand === 'genLocalState') {
+        await genLocalState(args)
     }
 }
 

diff --git a/contracts/ts/genMaciState.ts b/contracts/ts/genMaciState.ts
@@ -15,6 +15,7 @@ import {
 import { Contract, providers, utils } from 'ethers'
 // import { assert } from 'assert'
 import assert = require("assert")
+import { sleep } from './utils'
 
 interface Action {
     type: string;
@@ -30,7 +31,8 @@ const genMaciStateFromContract = async (
     pollId: number,
     fromBlock: number = 0,
     blocksPerRequest: number = 50,
-    endBlock?: number
+    endBlock?: number,
+    sleepAmount?: number
 ): Promise<MaciState> => {
     pollId = Number(pollId)
     // Verify and sort pollIds
@@ -73,6 +75,9 @@ const genMaciStateFromContract = async (
             address: address
         })
 
+        // sleep to avoid rate limiting
+        if (sleepAmount) await sleep(sleepAmount)
+
         initLogs = initLogs.concat(tmpInitLogs)
 
         const tmpSignUpLogs = await provider.getLogs({
@@ -82,6 +87,8 @@ const genMaciStateFromContract = async (
             address: address
         })
 
+        if (sleepAmount) await sleep(sleepAmount)
+
         signUpLogs = signUpLogs.concat(tmpSignUpLogs)
 
         const tmpMergeStateAqSubRootsLogs = await provider.getLogs({
@@ -91,6 +98,8 @@ const genMaciStateFromContract = async (
             address: address
         })
 
+        if (sleepAmount) await sleep(sleepAmount)
+
         mergeStateAqSubRootsLogs = mergeStateAqSubRootsLogs.concat(tmpMergeStateAqSubRootsLogs)
 
         const tmpMergeStateAqLogs = await provider.getLogs({
@@ -100,6 +109,8 @@ const genMaciStateFromContract = async (
             address: address
         })
 
+        if (sleepAmount) await sleep(sleepAmount)
+
         mergeStateAqLogs = mergeStateAqLogs.concat(tmpMergeStateAqLogs)
 
         const tmpDeployPollLogs = await provider.getLogs({
@@ -109,6 +120,8 @@ const genMaciStateFromContract = async (
             address: address
         })
 
+        if (sleepAmount) await sleep(sleepAmount)
+
         deployPollLogs = deployPollLogs.concat(tmpDeployPollLogs)
     }
 
@@ -289,6 +302,7 @@ const genMaciStateFromContract = async (
             fromBlock: i,
             toBlock
         })
+        if (sleepAmount) await sleep(sleepAmount)
 
         publishMessageLogs = publishMessageLogs.concat(tmpPublishMessageLogs)
 
@@ -298,6 +312,7 @@ const genMaciStateFromContract = async (
             toBlock,
             address: pollContract.address
         })
+        if (sleepAmount) await sleep(sleepAmount)
 
         topupLogs = topupLogs.concat(tmpTopupLogs)    
 
@@ -307,6 +322,7 @@ const genMaciStateFromContract = async (
             toBlock,
             address: pollContract.address
         })
+        if (sleepAmount) await sleep(sleepAmount)
 
         mergeMaciStateAqSubRootsLogs = mergeMaciStateAqSubRootsLogs.concat(tmpMergeMaciStateAqSubRootsLogs)
 
@@ -316,6 +332,7 @@ const genMaciStateFromContract = async (
             toBlock,
             address: pollContract.address
         })
+        if (sleepAmount) await sleep(sleepAmount)
 
         mergeMaciStateAqLogs = mergeMaciStateAqLogs.concat(tmpMergeMaciStateAqLogs)
 
@@ -325,6 +342,7 @@ const genMaciStateFromContract = async (
             toBlock,
             address: pollContract.address
         })
+        if (sleepAmount) await sleep(sleepAmount)
 
         mergeMessageAqSubRootsLogs = mergeMessageAqSubRootsLogs.concat(tmpMergeMessageAqSubRootsLogs)
 
@@ -334,6 +352,7 @@ const genMaciStateFromContract = async (
             toBlock,
             address: pollContract.address
         })
+        if (sleepAmount) await sleep(sleepAmount)
 
         mergeMessageAqLogs = mergeMessageAqLogs.concat(tmpMergeMessageAqLogs)
     }
@@ -355,7 +374,6 @@ const genMaciStateFromContract = async (
             event.args._encPubKey.map((x) => BigInt(x.toString()))
         )
 
-
         actions.push({
             type: 'PublishMessage',
             // @ts-ignore