feat: ✨ Implement Forest root changes catch up in Blockchain Service

.gitignore

@@ -33,4 +33,4 @@ res/benchmarking
 test/dist/*
 
 # Ignore cSpell config files
-cSpell.json
+cspell.json

api-augment/dist/types/interfaces/augment-api-events.d.ts

@@ -2071,14 +2071,35 @@ declare module "@polkadot/api-base/types/events" {
         }
       >;
       /**
-       * A set of mutations has been applied to the Forest.
+       * A set of mutations has been applied to a given Forest.
+       * This is the generic version of [`MutationsAppliedForProvider`](Event::MutationsAppliedForProvider)
+       * when [`generic_apply_delta`](ProofsDealerInterface::generic_apply_delta) is used
+       * and the root is not necessarily linked to a specific Provider.
        **/
       MutationsApplied: AugmentedEvent<
         ApiType,
-        [provider: H256, mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>, newRoot: H256],
+        [mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>, oldRoot: H256, newRoot: H256],
         {
-          provider: H256;
           mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>;
+          oldRoot: H256;
+          newRoot: H256;
+        }
+      >;
+      /**
+       * A set of mutations has been applied to the Forest of a given Provider.
+       **/
+      MutationsAppliedForProvider: AugmentedEvent<
+        ApiType,
+        [
+          providerId: H256,
+          mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>,
+          oldRoot: H256,
+          newRoot: H256
+        ],
+        {
+          providerId: H256;
+          mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>;
+          oldRoot: H256;
           newRoot: H256;
         }
       >;

api-augment/dist/types/interfaces/lookup.d.ts

@@ -1650,9 +1650,15 @@ declare const _default: {
         provider: string;
         maybeProviderAccount: string;
       };
+      MutationsAppliedForProvider: {
+        providerId: string;
+        mutations: string;
+        oldRoot: string;
+        newRoot: string;
+      };
       MutationsApplied: {
-        provider: string;
         mutations: string;
+        oldRoot: string;
         newRoot: string;
       };
       ChallengesTickerSet: {

api-augment/dist/types/interfaces/types-lookup.d.ts

@@ -2132,10 +2132,17 @@ declare module "@polkadot/types/lookup" {
       readonly provider: H256;
       readonly maybeProviderAccount: Option<AccountId32>;
     } & Struct;
+    readonly isMutationsAppliedForProvider: boolean;
+    readonly asMutationsAppliedForProvider: {
+      readonly providerId: H256;
+      readonly mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>;
+      readonly oldRoot: H256;
+      readonly newRoot: H256;
+    } & Struct;
     readonly isMutationsApplied: boolean;
     readonly asMutationsApplied: {
-      readonly provider: H256;
       readonly mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>;
+      readonly oldRoot: H256;
       readonly newRoot: H256;
     } & Struct;
     readonly isChallengesTickerSet: boolean;
@@ -2150,6 +2157,7 @@ declare module "@polkadot/types/lookup" {
       | "SlashableProvider"
       | "NoRecordOfLastSubmittedProof"
       | "NewChallengeCycleInitialised"
+      | "MutationsAppliedForProvider"
       | "MutationsApplied"
       | "ChallengesTickerSet";
   }

api-augment/src/interfaces/augment-api-events.ts

@@ -1601,12 +1601,33 @@ declare module "@polkadot/api-base/types/events" {
        **/
       ChallengesTickerSet: AugmentedEvent<ApiType, [paused: bool], { paused: bool }>;
       /**
-       * A set of mutations has been applied to the Forest.
+       * A set of mutations has been applied to a given Forest.
+       * This is the generic version of [`MutationsAppliedForProvider`](Event::MutationsAppliedForProvider)
+       * when [`generic_apply_delta`](ProofsDealerInterface::generic_apply_delta) is used
+       * and the root is not necessarily linked to a specific Provider.
        **/
       MutationsApplied: AugmentedEvent<
         ApiType,
-        [provider: H256, mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>, newRoot: H256],
-        { provider: H256; mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>; newRoot: H256 }
+        [mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>, oldRoot: H256, newRoot: H256],
+        { mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>; oldRoot: H256; newRoot: H256 }
+      >;
+      /**
+       * A set of mutations has been applied to the Forest of a given Provider.
+       **/
+      MutationsAppliedForProvider: AugmentedEvent<
+        ApiType,
+        [
+          providerId: H256,
+          mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>,
+          oldRoot: H256,
+          newRoot: H256
+        ],
+        {
+          providerId: H256;
+          mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>;
+          oldRoot: H256;
+          newRoot: H256;
+        }
       >;
       /**
        * A manual challenge was submitted.

api-augment/src/interfaces/lookup.ts

@@ -1672,9 +1672,15 @@ export default {
         provider: "H256",
         maybeProviderAccount: "Option<AccountId32>"
       },
+      MutationsAppliedForProvider: {
+        providerId: "H256",
+        mutations: "Vec<(H256,ShpTraitsTrieMutation)>",
+        oldRoot: "H256",
+        newRoot: "H256"
+      },
       MutationsApplied: {
-        provider: "H256",
         mutations: "Vec<(H256,ShpTraitsTrieMutation)>",
+        oldRoot: "H256",
         newRoot: "H256"
       },
       ChallengesTickerSet: {

api-augment/src/interfaces/types-lookup.ts

@@ -2223,10 +2223,17 @@ declare module "@polkadot/types/lookup" {
       readonly provider: H256;
       readonly maybeProviderAccount: Option<AccountId32>;
     } & Struct;
+    readonly isMutationsAppliedForProvider: boolean;
+    readonly asMutationsAppliedForProvider: {
+      readonly providerId: H256;
+      readonly mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>;
+      readonly oldRoot: H256;
+      readonly newRoot: H256;
+    } & Struct;
     readonly isMutationsApplied: boolean;
     readonly asMutationsApplied: {
-      readonly provider: H256;
       readonly mutations: Vec<ITuple<[H256, ShpTraitsTrieMutation]>>;
+      readonly oldRoot: H256;
       readonly newRoot: H256;
     } & Struct;
     readonly isChallengesTickerSet: boolean;
@@ -2241,6 +2248,7 @@ declare module "@polkadot/types/lookup" {
       | "SlashableProvider"
       | "NoRecordOfLastSubmittedProof"
       | "NewChallengeCycleInitialised"
+      | "MutationsAppliedForProvider"
       | "MutationsApplied"
       | "ChallengesTickerSet";
   }

client/blockchain-service/Cargo.toml

@@ -38,6 +38,7 @@ sc-network = { workspace = true }
 sc-utils = { workspace = true }
 sc-tracing = { workspace = true }
 sp-api = { workspace = true }
+sp-blockchain = { workspace = true }
 sp-core = { workspace = true, default-features = true }
 sp-keystore = { workspace = true }
 sp-runtime = { workspace = true, default-features = true }

client/blockchain-service/src/commands.rs

@@ -31,7 +31,7 @@ use super::{
     handler::BlockchainService,
     transaction::SubmittedTransaction,
     types::{
-        BestBlockInfo, ConfirmStoringRequest, Extrinsic, ExtrinsicResult, RespondStorageRequest,
+        ConfirmStoringRequest, Extrinsic, ExtrinsicResult, MinimalBlockInfo, RespondStorageRequest,
         RetryStrategy, StopStoringForInsolventUserRequest, SubmitProofRequest, Tip,
     },
 };
@@ -55,7 +55,7 @@ pub enum BlockchainServiceCommand {
         callback: tokio::sync::oneshot::Sender<Result<()>>,
     },
     GetBestBlockInfo {
-        callback: tokio::sync::oneshot::Sender<BestBlockInfo>,
+        callback: tokio::sync::oneshot::Sender<MinimalBlockInfo>,
     },
     WaitForBlock {
         block_number: BlockNumber,

client/blockchain-service/src/handler.rs

@@ -14,6 +14,7 @@ use sc_service::RpcHandlers;
 use sc_tracing::tracing::{debug, error, info, trace, warn};
 use shc_forest_manager::traits::ForestStorageHandler;
 use sp_api::{ApiError, ProvideRuntimeApi};
+use sp_blockchain::TreeRoute;
 use sp_core::H256;
 use sp_keystore::{Keystore, KeystorePtr};
 use sp_runtime::{
@@ -63,7 +64,7 @@ use crate::{
     transaction::SubmittedTransaction,
     typed_store::{CFDequeAPI, ProvidesTypedDbSingleAccess},
     types::{
-        BestBlockInfo, ForestStorageSnapshotInfo, NewBlockNotificationKind,
+        ForestStorageSnapshotInfo, MinimalBlockInfo, NewBlockNotificationKind,
         StopStoringForInsolventUserRequest, SubmitProofRequest,
     },
 };
@@ -103,12 +104,12 @@ where
     ///
     /// This is used to manage Forest Storage instances and update their roots when there are
     /// Forest-root-changing events on-chain, for the Storage Provider managed by this service.
-    pub(crate) _forest_storage_handler: FSH,
+    pub(crate) forest_storage_handler: FSH,
     /// The hash and number of the last best block processed by the BlockchainService.
     ///
     /// This is used to detect when the BlockchainService gets out of syncing mode and should therefore
     /// run some initialisation tasks. Also used to detect reorgs.
-    pub(crate) best_block: BestBlockInfo,
+    pub(crate) best_block: MinimalBlockInfo,
     /// Nonce counter for the extrinsics.
     pub(crate) nonce_counter: u32,
     /// A registry of waiters for a block number.
@@ -1022,8 +1023,8 @@ where
             client,
             keystore,
             rpc_handlers,
-            _forest_storage_handler: forest_storage_handler,
-            best_block: BestBlockInfo::default(),
+            forest_storage_handler,
+            best_block: MinimalBlockInfo::default(),
             nonce_counter: 0,
             wait_for_block_request_by_number: BTreeMap::new(),
             wait_for_tick_request_by_number: BTreeMap::new(),
@@ -1042,22 +1043,37 @@ where
     ) where
         Block: cumulus_primitives_core::BlockT<Hash = H256>,
     {
-        let last_block_processed = self.best_block.number;
+        let last_block_processed = self.best_block;
+
+        // Check if this new imported block is the new best, and if it causes a reorg.
         let new_block_notification_kind = self.register_best_block_and_check_reorg(&notification);
-        let BestBlockInfo {
-            number: block_number,
-            hash: block_hash,
-        } = match new_block_notification_kind {
-            NewBlockNotificationKind::NewBestBlock(new_best_block_info) => new_best_block_info,
+
+        // Get the new best block info, and the `TreeRoute`, i.e. the blocks from the old best block to the new best block.
+        // A new non-best block is ignored and not processed.
+        let (block_info, tree_route) = match new_block_notification_kind {
+            NewBlockNotificationKind::NewBestBlock(new_best_block_info) => {
+                // Making up a tree route with the new best block, and the old best block as the pivot.
+                let tree_route = TreeRoute::new(
+                    vec![
+                        last_block_processed.clone().into(),
+                        new_best_block_info.clone().into(),
+                    ],
+                    0,
+                )
+                .expect("`TreeRoute` with `pivot` 0 should be valid; qed");
+                (new_best_block_info, tree_route)
+            }
             NewBlockNotificationKind::NewNonBestBlock(_) => return,
             NewBlockNotificationKind::Reorg {
                 old_best_block: _,
                 new_best_block,
-            } => {
-                // TODO: Handle catch up of reorgs.
-                new_best_block
-            }
+                tree_route,
+            } => (new_best_block, tree_route),
         };
+        let MinimalBlockInfo {
+            number: block_number,
+            hash: block_hash,
+        } = block_info;
 
         info!(target: LOG_TARGET, "📥 Block import notification (#{}): {}", block_number, block_hash);
 
@@ -1068,11 +1084,12 @@ where
         // Check if we just came out of syncing mode.
         // We use saturating_sub because in a reorg, there is a potential scenario where the last
         // block processed is higher than the current block number.
-        if block_number.saturating_sub(last_block_processed) > SYNC_MODE_MIN_BLOCKS_BEHIND {
+        if block_number.saturating_sub(last_block_processed.number) > SYNC_MODE_MIN_BLOCKS_BEHIND {
             self.handle_initial_sync(notification).await;
         }
 
-        self.process_block_import(&block_hash, &block_number).await;
+        self.process_block_import(&block_hash, &block_number, tree_route)
+            .await;
     }
 
     fn pre_block_processing_checks(&mut self, block_hash: &H256) {
@@ -1166,9 +1183,19 @@ where
         }
     }
 
-    async fn process_block_import(&mut self, block_hash: &H256, block_number: &BlockNumber) {
+    async fn process_block_import<Block>(
+        &mut self,
+        block_hash: &H256,
+        block_number: &BlockNumber,
+        tree_route: TreeRoute<Block>,
+    ) where
+        Block: cumulus_primitives_core::BlockT<Hash = H256>,
+    {
         trace!(target: LOG_TARGET, "📠 Processing block import #{}: {}", block_number, block_hash);
 
+        // Before triggering any task, we make sure to be caught up to the Forest roots on-chain.
+        self.forest_root_changes_catchup(&tree_route).await;
+
         // Trigger catch up of proofs if the block is a multiple of `CHECK_FOR_PENDING_PROOFS_PERIOD`.
         // This is only relevant if this node is managing a BSP.
         if let Some(StorageProviderId::BackupStorageProvider(bsp_id)) = &self.provider_id {
@@ -1502,11 +1529,11 @@ where
                 // Process the events.
                 for ev in block_events {
                     match ev.event.clone() {
-                        // New storage request event coming from pallet-file-system.
                         RuntimeEvent::ProofsDealer(
-                            pallet_proofs_dealer::Event::MutationsApplied {
-                                provider: provider_id,
+                            pallet_proofs_dealer::Event::MutationsAppliedForProvider {
+                                provider_id,
                                 mutations,
+                                old_root: _,
                                 new_root,
                             },
                         ) => {