chore: optimize message processor and tally

- [x] Remove sha256 hashing
- [x] Support multiple params for verifier

.github/workflows/coordinator-build.yml

@@ -75,12 +75,6 @@ jobs:
         run: |
           pnpm run build
 
-      - name: Run hardhat
-        run: |
-          pnpm run hardhat &
-          sleep 5
-        working-directory: coordinator
-
       - name: Download rapidsnark (1c137)
         run: |
           mkdir -p ~/rapidsnark/build
@@ -109,6 +103,12 @@ jobs:
           pnpm generate-keypair
         working-directory: coordinator
 
+      - name: Run hardhat
+        run: |
+          pnpm run hardhat &
+          sleep 5
+        working-directory: coordinator
+
       - name: Test
         run: pnpm run test
         working-directory: coordinator

.github/workflows/e2e.yml

@@ -10,11 +10,15 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  e2e:
-    uses: ./.github/workflows/reusable-e2e.yml
+  e2e-cli:
+    uses: ./.github/workflows/reusable-e2e-cli.yml
+
+  integration:
+    needs: e2e-cli
+    uses: ./.github/workflows/reusable-integration.yml
 
   npm-publish:
-    needs: e2e
+    needs: [e2e-cli, integration]
     if: ${{ github.event_name == 'push' }}
     runs-on: ubuntu-22.04
 

.github/workflows/reusable-e2e.yml → .github/workflows/reusable-e2e-cli.yml

@@ -1,14 +1,14 @@
-name: E2E
+name: E2E cli
 
 on:
   workflow_call:
 
 jobs:
-  e2e:
+  e2e-cli:
     strategy:
       fail-fast: false
       matrix:
-        command: ["test:cli", "test:integration"]
+        command: ["test:cli"]
 
     runs-on: ubuntu-22.04
 
@@ -61,12 +61,6 @@ jobs:
         run: |
           pnpm run build
 
-      - name: Run hardhat fork
-        run: |
-          cd contracts
-          pnpm run hardhat &
-          sleep 5
-
       - name: Download rapidsnark (1c137)
         run: |
           mkdir -p ~/rapidsnark/build
@@ -90,6 +84,12 @@ jobs:
         run: |
           pnpm download-zkeys:test
 
+      - name: Run hardhat fork
+        run: |
+          cd contracts
+          pnpm run hardhat &
+          sleep 5
+
       - name: ${{ matrix.command }}
         run: pnpm run ${{ matrix.command }}
 

.github/workflows/reusable-integration.yml

@@ -0,0 +1,98 @@
+name: Integration tests
+
+on:
+  workflow_call:
+
+jobs:
+  integration:
+    strategy:
+      fail-fast: false
+      matrix:
+        command: ["test:integration"]
+
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 9
+
+      - name: Use Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "pnpm"
+
+      # Check for changes in the 'circuit' folder
+      - name: Get changed files
+        id: get-changed-files
+        uses: jitterbit/get-changed-files@v1
+        with:
+          format: "csv"
+
+      - name: Check for changes in 'circuit' folder
+        id: check_changes
+        run: |
+          CHANGED_FILES=${{ steps.get-changed-files.outputs.all }}
+          if echo "$CHANGED_FILES" | grep -q "\.circom"; then
+            echo "CHANGED=true" >> $GITHUB_ENV
+            echo "Circuits have changes."
+          else
+            echo "CHANGED=false" >> $GITHUB_ENV
+            echo "No changes on circuits."
+          fi
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes \
+            build-essential \
+            libgmp-dev \
+            libsodium-dev \
+            nasm \
+            nlohmann-json3-dev
+
+      - name: Install
+        run: |
+          pnpm install --frozen-lockfile --prefer-offline
+
+      - name: Build
+        run: |
+          pnpm run build
+
+      - name: Download rapidsnark (1c137)
+        run: |
+          mkdir -p ~/rapidsnark/build
+          wget -qO ~/rapidsnark/build/prover https://maci-devops-zkeys.s3.ap-northeast-2.amazonaws.com/rapidsnark-linux-amd64-1c137
+          chmod +x ~/rapidsnark/build/prover
+
+      - name: Download circom Binary v2.1.6
+        run: |
+          wget -qO ${{ github.workspace }}/circom https://github.com/iden3/circom/releases/download/v2.1.6/circom-linux-amd64
+          chmod +x ${{ github.workspace }}/circom
+          sudo mv ${{ github.workspace }}/circom /bin/circom
+
+      - name: Compile Circuits And Generate zkeys
+        if: ${{ env.CHANGED == 'true' }}
+        run: |
+          pnpm build:circuits-c -- --outPath ../cli/zkeys
+          pnpm setup:zkeys -- --outPath ../cli/zkeys
+
+      - name: Download zkeys
+        if: ${{ env.CHANGED == 'false' }}
+        run: |
+          pnpm download-zkeys:test
+
+      - name: Run hardhat fork
+        run: |
+          cd contracts
+          pnpm run hardhat &
+          sleep 5
+
+      - name: ${{ matrix.command }}
+        run: pnpm run ${{ matrix.command }}
+
+      - name: Stop Hardhat
+        if: always()
+        run: kill $(lsof -t -i:8545)

circuits/.gitignore

@@ -2,3 +2,5 @@ ptau
 input.json
 circom/main
 circom/test
+zkeys/
+

circuits/circom/circuits.json

@@ -3,24 +3,42 @@
     "file": "./core/qv/processMessages",
     "template": "ProcessMessages",
     "params": [10, 2, 1, 2],
-    "pubs": ["inputHash"]
+    "pubs": [
+      "numSignUps",
+      "index",
+      "batchEndIndex",
+      "msgRoot",
+      "currentSbCommitment",
+      "newSbCommitment",
+      "pollEndTimestamp",
+      "actualStateTreeDepth"
+    ]
   },
   "ProcessMessagesNonQv_10-2-1-2_test": {
     "file": "./core/non-qv/processMessages",
     "template": "ProcessMessagesNonQv",
     "params": [10, 2, 1, 2],
-    "pubs": ["inputHash"]
+    "pubs": [
+      "numSignUps",
+      "index",
+      "batchEndIndex",
+      "msgRoot",
+      "currentSbCommitment",
+      "newSbCommitment",
+      "pollEndTimestamp",
+      "actualStateTreeDepth"
+    ]
   },
   "TallyVotes_10-1-2_test": {
     "file": "./core/qv/tallyVotes",
     "template": "TallyVotes",
     "params": [10, 1, 2],
-    "pubs": ["inputHash"]
+    "pubs": ["index", "numSignUps", "sbCommitment", "currentTallyCommitment", "newTallyCommitment"]
   },
   "TallyVotesNonQv_10-1-2_test": {
     "file": "./core/non-qv/tallyVotes",
     "template": "TallyVotesNonQv",
     "params": [10, 1, 2],
-    "pubs": ["inputHash"]
+    "pubs": ["index", "numSignUps", "sbCommitment", "currentTallyCommitment", "newTallyCommitment"]
   }
 }

circuits/circom/core/non-qv/processMessages.circom

@@ -8,7 +8,6 @@ include "./safe-comparators.circom";
 include "../../utils/hashers.circom";
 include "../../utils/messageToCommand.circom";
 include "../../utils/privToPubKey.circom";
-include "../../utils/processMessagesInputHasher.circom";
 include "../../utils/non-qv/stateLeafAndBallotTransformer.circom";
 include "../../trees/incrementalMerkleTree.circom";
 include "../../trees/incrementalQuinaryTree.circom";
@@ -36,6 +35,7 @@ include "../../trees/incrementalQuinaryTree.circom";
     // Default for Binary trees.
     var STATE_TREE_ARITY = 2;
     var batchSize = MESSAGE_TREE_ARITY ** msgBatchDepth;
+    var maxVoteOptions = MESSAGE_TREE_ARITY ** voteOptionTreeDepth;
     var MSG_LENGTH = 10;
     var PACKED_CMD_LENGTH = 4;
     var STATE_LEAF_LENGTH = 4;
@@ -48,17 +48,8 @@ include "../../trees/incrementalQuinaryTree.circom";
     var STATE_LEAF_TIMESTAMP_IDX = 3;
     var msgTreeZeroValue = 8370432830353022751713833565135785980866757267633941821328460903436894336785;
 
-    // nb. The usage of SHA-256 hash is necessary to save some gas costs at verification time
-    // at the cost of more constraints for the prover.
-    // Basically, some values from the contract are passed as private inputs and the hash as a public input.
-
-    // The SHA-256 hash of values provided by the contract.
-    signal input inputHash;
-    signal input packedVals;
     // Number of users that have completed the sign up.
-    signal numSignUps;
-    // Number of options for this poll.
-    signal maxVoteOptions;
+    signal input numSignUps;
     // Time when the poll ends.
     signal input pollEndTimestamp;
     // The existing message tree root.
@@ -79,6 +70,10 @@ include "../../trees/incrementalQuinaryTree.circom";
     // @note it is a public input to ensure fair processing from 
     // the coordinator (no censoring)
     signal input actualStateTreeDepth;
+    // The last batch index
+    signal input batchEndIndex;
+    // The batch index of current message batch
+    signal input index;
 
     // The state leaves upon which messages are applied.
     //    transform(currentStateLeaf[4], message5) => newStateLeaf4
@@ -110,14 +105,6 @@ include "../../trees/incrementalQuinaryTree.circom";
     // Therefore, the index of the first message to process does not match the index of the
     // first message (e.g., [msg1, msg2, msg3] => first message to process has index 3).
 
-    // The index of the first message leaf in the batch, inclusive.
-    signal batchStartIndex;
-
-    // The index of the last message leaf in the batch to process, exclusive.
-    // This value may be less than batchStartIndex + batchSize if this batch is
-    // the last batch and the total number of messages is not a multiple of the batch size.
-    signal batchEndIndex;
-
     // The history of state and ballot roots and temporary intermediate
     // signals (for processing purposes).
     signal stateRoots[batchSize + 1];
@@ -131,31 +118,6 @@ include "../../trees/incrementalQuinaryTree.circom";
     var computedCurrentSbCommitment = PoseidonHasher(3)([currentStateRoot, currentBallotRoot, currentSbSalt]);
     computedCurrentSbCommitment === currentSbCommitment;
 
-    // Verify public inputs and assign unpacked values.
-    var (
-        computedMaxVoteOptions, 
-        computedNumSignUps, 
-        computedBatchStartIndex, 
-        computedBatchEndIndex, 
-        computedHash
-    ) = ProcessMessagesInputHasher()(
-        packedVals,
-        coordPubKey,
-        msgRoot,
-        currentSbCommitment,
-        newSbCommitment,
-        pollEndTimestamp,
-        actualStateTreeDepth
-    );
-
-    // The unpacked values from packedVals.
-    computedMaxVoteOptions ==> maxVoteOptions;
-    computedNumSignUps ==> numSignUps;
-    computedBatchStartIndex ==> batchStartIndex;
-    computedBatchEndIndex ==> batchEndIndex;
-    // Matching constraints.
-    computedHash === inputHash;
-
     //  ----------------------------------------------------------------------- 
     // 0. Ensure that the maximum vote options signal is valid and if
     // the maximum users signal is valid.
@@ -173,7 +135,7 @@ include "../../trees/incrementalQuinaryTree.circom";
         computedMessageHashers[i] = MessageHasher()(msgs[i], encPubKeys[i]);
     }
 
-    // If batchEndIndex - batchStartIndex < batchSize, the remaining
+    // If endIndex - startIndex < batchSize, the remaining
     // message hashes should be the zero value.
     // e.g. [m, z, z, z, z] if there is only 1 real message in the batch
     // This makes possible to have a batch of messages which is only partially full.
@@ -182,7 +144,7 @@ include "../../trees/incrementalQuinaryTree.circom";
     var computedPathIndex[msgTreeDepth - msgBatchDepth];
 
     for (var i = 0; i < batchSize; i++) {
-        var batchStartIndexValid = SafeLessThan(32)([batchStartIndex + i, batchEndIndex]);
+        var batchStartIndexValid = SafeLessThan(32)([index + i, batchEndIndex]);
         computedLeaves[i] = Mux1()([msgTreeZeroValue, computedMessageHashers[i]], batchStartIndexValid);
     }
 
@@ -195,8 +157,8 @@ include "../../trees/incrementalQuinaryTree.circom";
     // Computing the path_index values. Since msgBatchLeavesExists tests
     // the existence of a subroot, the length of the proof correspond to the last 
     // n elements of a proof from the root to a leaf, where n = msgTreeDepth - msgBatchDepth.
-    // e.g. if batchStartIndex = 25, msgTreeDepth = 4, msgBatchDepth = 2, then path_index = [1, 0].
-    var computedMsgBatchPathIndices[msgTreeDepth] = QuinGeneratePathIndices(msgTreeDepth)(batchStartIndex);
+    // e.g. if startIndex = 25, msgTreeDepth = 4, msgBatchDepth = 2, then path_index = [1, 0].
+    var computedMsgBatchPathIndices[msgTreeDepth] = QuinGeneratePathIndices(msgTreeDepth)(index);
 
     for (var i = msgBatchDepth; i < msgTreeDepth; i++) {
         computedPathIndex[i - msgBatchDepth] = computedMsgBatchPathIndices[i];
@@ -286,7 +248,6 @@ include "../../trees/incrementalQuinaryTree.circom";
 
         (computedNewVoteStateRoot[i], computedNewVoteBallotRoot[i]) = ProcessOneNonQv(stateTreeDepth, voteOptionTreeDepth)(
             numSignUps,
-            maxVoteOptions,
             pollEndTimestamp,
             stateRoots[i + 1],
             ballotRoots[i + 1],
@@ -336,6 +297,7 @@ template ProcessOneNonQv(stateTreeDepth, voteOptionTreeDepth) {
     var BALLOT_NONCE_IDX = 0;
     // Ballot vote option (VO) root index.
     var BALLOT_VO_ROOT_IDX = 1;
+    var maxVoteOptions = MESSAGE_TREE_ARITY ** voteOptionTreeDepth;
 
     // Indices for elements within a state leaf.
     // Public key.
@@ -348,7 +310,6 @@ template ProcessOneNonQv(stateTreeDepth, voteOptionTreeDepth) {
 
     // Inputs representing the message and the current state.
     signal input numSignUps;
-    signal input maxVoteOptions;
     signal input pollEndTimestamp;
 
     // The current value of the state tree root.