refactor: zero copy vec (#1469)

* stash

* cleanup

* stash processor refactor, plus cleanups, pre zero copy lifetime

* chore: fix sigkill issue

* zero copy vec: add lifetim

* stash

* refactor: use zerocopy crate instead of pointers

* refactor: zero copy vec

* fix: system cpi test

* cleanup

cleanup

* chore: remove unwraps, unify padding handling, unify naming

* chore: remove dead code

* cleanup

* chore: workflow distribute light-batched-merkle-tree tests between ci tests

.github/workflows/rust.yml

@@ -36,11 +36,12 @@ jobs:
       matrix:
         group:
           - name: concurrent-merkle-tree
-            packages: light-concurrent-merkle-tree
+            packages: >-
+              light-concurrent-merkle-tree
+              light-batched-merkle-tree
           - name: program-libs
             packages: >-
               aligned-sized
-              light-batched-merkle-tree
               light-bloom-filter
               light-hasher
               light-utils
@@ -49,21 +50,15 @@ jobs:
               light-zero-copy
               light-hash-set
               light-indexed-merkle-tree
+              light-batched-merkle-tree
           - name: sdk-libs
             packages: >-
               light-macros
               light-sdk
               light-program-test
               light-client
-          - name: program-tests
-            packages: >-
-              account-compression-test
-              compressed-token-test
-              e2e-test
-              registry-test
-              system-cpi-test
-              system-test
-              sdk-test
+              light-batched-merkle-tree
+              
       fail-fast: false
     name: Test ${{ matrix.group.name }}
 
@@ -108,8 +103,23 @@ jobs:
               echo "::group::Testing ${pkg}"
               start=$(date +%s)
               
-              cargo test -p "${pkg}" || exit 1
-              
+
+              if [ "${pkg}" == "light-batched-merkle-tree" ]; then
+                if [ "${name}" == "sdk-libs" ]; then
+                  # execute simulate transactions test
+                  cargo test -p "${pkg}" -- --test test_simulate_transactions || exit 1
+                elif [ "${name}" == "program-libs" ]; then
+                  # execute e2e test
+                  cargo test -p "${pkg}" -- --test test_e2e || exit 1
+                else
+                  # execute all tests except test_simulate_transactions and test_e2e
+                  cargo test -p "${pkg}" -- --skip test_simulate_transactions --skip test_e2e || exit 1
+                fi
+              else {
+                cargo test -p "${pkg}" || exit 1
+                }
+              fi
+                            
               end=$(date +%s)
               duration=$((end - start))
               formatted_time=$(format_duration "$duration")

Cargo.toml

@@ -116,6 +116,7 @@ bytemuck = { version = "1.17" }
 # Math and crypto
 num-bigint = "0.4.6"
 num-traits = "0.2.19"
+zerocopy = { version = "0.8.14"}
 
 # HTTP client
 reqwest = "0.11.26"

forester-utils/src/address_merkle_tree_config.rs

@@ -3,7 +3,7 @@ use account_compression::{
     QueueAccount, StateMerkleTreeAccount, StateMerkleTreeConfig,
 };
 use anchor_lang::Discriminator;
-use light_batched_merkle_tree::merkle_tree::BatchedMerkleTreeMetadata;
+use light_batched_merkle_tree::merkle_tree::BatchedMerkleTreeAccount;
 use light_client::rpc::RpcConnection;
 use light_hasher::{Discriminator as LightDiscriminator, Poseidon};
 use num_traits::Zero;
@@ -151,7 +151,7 @@ pub async fn state_tree_ready_for_rollover<R: RpcConnection>(
     rpc: &mut R,
     merkle_tree: Pubkey,
 ) -> bool {
-    let account = rpc.get_account(merkle_tree).await.unwrap().unwrap();
+    let mut account = rpc.get_account(merkle_tree).await.unwrap().unwrap();
     let rent_exemption = rpc
         .get_minimum_balance_for_rent_exemption(account.data.len())
         .await
@@ -169,15 +169,15 @@ pub async fn state_tree_ready_for_rollover<R: RpcConnection>(
             .await;
             (tree.next_index(), tree_meta_data, 26)
         }
-        BatchedMerkleTreeMetadata::DISCRIMINATOR => {
-            let account = AccountZeroCopy::<BatchedMerkleTreeMetadata>::new(rpc, merkle_tree).await;
-
-            let tree_meta_data = account.deserialized();
+        BatchedMerkleTreeAccount::DISCRIMINATOR => {
+            let tree_meta_data =
+                BatchedMerkleTreeAccount::state_tree_from_bytes_mut(account.data.as_mut_slice())
+                    .unwrap();
 
             (
-                tree_meta_data.next_index as usize,
-                tree_meta_data.metadata,
-                tree_meta_data.height,
+                tree_meta_data.get_metadata().next_index as usize,
+                tree_meta_data.get_metadata().metadata,
+                tree_meta_data.get_metadata().height,
             )
         }
         _ => panic!("Invalid discriminator"),

forester/src/rollover/operations.rs

@@ -82,7 +82,7 @@ pub async fn get_tree_fullness<R: RpcConnection>(
                 .await?
                 .unwrap();
             let queue_account = rpc
-                .get_anchor_account::<QueueAccount>(&account.metadata.associated_queue)
+                .get_anchor_account::<QueueAccount>(&account.metadata.associated_queue.into())
                 .await?
                 .unwrap();
             let merkle_tree =

forester/src/tree_data_sync.rs

@@ -82,7 +82,7 @@ fn create_tree_accounts(
 ) -> TreeAccounts {
     let tree_accounts = TreeAccounts::new(
         pubkey,
-        metadata.associated_queue,
+        metadata.associated_queue.into(),
         tree_type,
         metadata.rollover_metadata.rolledover_slot != u64::MAX,
     );

js/compressed-token/rollup.config.js

@@ -79,8 +79,4 @@ const typesConfig = {
     plugins: [dts()],
 };
 
-export default [
-    rolls('cjs', 'browser'),
-    rolls('cjs', 'node'),
-    typesConfig,
-];
+export default [rolls('cjs', 'browser'), rolls('cjs', 'node'), typesConfig];