ledger-tool: verify: add --verify-slot-hashes

This adds:

    --verify-slot-hashes <FILENAME>
        If the file does not exist, write the slot hashes to this file;
        if the file does exist, verify slot hashes against this file.

The first case can be used to dump a list of (slot, hash) to a json file
during a replay. The second case can be used to check slot hashes against
previously recorded values.

This is useful for debugging consensus failures, eg:

    # on good commit/branch
    ledger-tool verify --verify-slot-hashes good.json ...

    # on bad commit or potentially consensus breaking branch
    ledger-tool verify --verify-slot-hashes good.json ...

On a hash mismatch an error will be logged with the expected hash vs the
computed hash.

ledger-tool/src/main.rs

@@ -1068,6 +1068,11 @@ fn get_access_type(process_options: &ProcessOptions) -> AccessType {
 
 #[cfg(not(target_env = "msvc"))]
 use jemallocator::Jemalloc;
+use {
+    serde::Deserialize,
+    solana_ledger::blockstore_processor::ProcessSlotCallback,
+    std::{collections::VecDeque, sync::Mutex},
+};
 
 #[cfg(not(target_env = "msvc"))]
 #[global_allocator]
@@ -1716,7 +1721,14 @@ fn main() {
                              information that went into computing the completed bank's bank hash. \
                              The file will be written within <LEDGER_DIR>/bank_hash_details/",
                         ),
-                ),
+                )
+                .arg(
+                    Arg::with_name("verify_slot_hashes")
+                        .long("verify-slot-hashes")
+                        .takes_value(true)
+                        .value_name("FILENAME")
+                        .help("Record slot hashes to new file or verify slot hashes match contents of existing file.")
+                )
         )
         .subcommand(
             SubCommand::with_name("graph")
@@ -2722,6 +2734,74 @@ fn main() {
                     );
                 }
 
+                #[derive(Serialize, Deserialize)]
+                struct SlotHash {
+                    slot: Slot,
+                    hash: String,
+                }
+
+                let (slot_callback, record_slot_hashes_file, recorded_slot_hashes) = if let Some(
+                    filename,
+                ) =
+                    arg_matches.value_of_os("verify_slot_hashes")
+                {
+                    let filename = Path::new(filename);
+
+                    if filename.exists() {
+                        let file = File::open(filename).unwrap_or_else(|err| {
+                            eprintln!("Unable to read file: {}: {err:#}", filename.display());
+                            exit(1);
+                        });
+
+                        let slot_hashes: Arc<Mutex<VecDeque<SlotHash>>> = Arc::new(Mutex::new(
+                            serde_json::from_reader(file).unwrap_or_else(|err| {
+                                eprintln!("Error loading slot hashes file: {err:#}");
+                                exit(1);
+                            }),
+                        ));
+
+                        let slot_callback = Arc::new(move |bank: &Bank| {
+                            let SlotHash {
+                                slot: expected_slot,
+                                hash: expected_hash,
+                            } = slot_hashes.lock().unwrap().pop_front().unwrap();
+                            if bank.slot() != expected_slot
+                                || bank.hash().to_string() != expected_hash
+                            {
+                                error!("Expected slot: {expected_slot} hash: {expected_hash} got slot: {} hash: {}",
+                                    bank.slot(), bank.hash());
+                            }
+                        });
+
+                        (Some(slot_callback as ProcessSlotCallback), None, None)
+                    } else {
+                        let file = File::create(filename).unwrap_or_else(|err| {
+                            eprintln!("Unable to write to file: {}: {:#}", filename.display(), err);
+                            exit(1);
+                        });
+
+                        let slot_hashes = Arc::new(Mutex::new(Vec::new()));
+
+                        let slot_callback = Arc::new({
+                            let slot_hashes = Arc::clone(&slot_hashes);
+                            move |bank: &Bank| {
+                                slot_hashes.lock().unwrap().push(SlotHash {
+                                    slot: bank.slot(),
+                                    hash: bank.hash().to_string(),
+                                });
+                            }
+                        });
+
+                        (
+                            Some(slot_callback as ProcessSlotCallback),
+                            Some(file),
+                            Some(slot_hashes),
+                        )
+                    }
+                } else {
+                    (None, None, None)
+                };
+
                 let process_options = ProcessOptions {
                     new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
                     run_verification: !(arg_matches.is_present("skip_poh_verify")
@@ -2749,6 +2829,7 @@ fn main() {
                         use_snapshot_archives_at_startup::cli::NAME,
                         UseSnapshotArchivesAtStartup
                     ),
+                    slot_callback,
                     ..ProcessOptions::default()
                 };
                 let print_accounts_stats = arg_matches.is_present("print_accounts_stats");
@@ -2787,6 +2868,15 @@ fn main() {
                         })
                         .ok();
                 }
+
+                if let Some(recorded_slot_hashes_file) = record_slot_hashes_file {
+                    serde_json::to_writer(
+                        recorded_slot_hashes_file,
+                        &recorded_slot_hashes.unwrap(),
+                    )
+                    .unwrap();
+                }
+
                 exit_signal.store(true, Ordering::Relaxed);
                 system_monitor_service.join().unwrap();
             }

ledger/src/blockstore_processor.rs

@@ -674,15 +674,17 @@ pub enum BlockstoreProcessorError {
     RootBankWithMismatchedCapitalization(Slot),
 }
 
-/// Callback for accessing bank state while processing the blockstore
-pub type ProcessCallback = Arc<dyn Fn(&Bank) + Sync + Send>;
+/// Callback for accessing bank state after each slot is confirmed while
+/// processing the blockstore
+pub type ProcessSlotCallback = Arc<dyn Fn(&Bank) + Sync + Send>;
 
 #[derive(Default, Clone)]
 pub struct ProcessOptions {
     /// Run PoH, transaction signature and other transaction verifications on the entries.
     pub run_verification: bool,
     pub full_leader_cache: bool,
     pub halt_at_slot: Option<Slot>,
+    pub slot_callback: Option<ProcessSlotCallback>,
     pub new_hard_forks: Option<Vec<Slot>>,
     pub debug_keys: Option<Arc<HashSet<Pubkey>>>,
     pub account_indexes: AccountSecondaryIndexes,
@@ -1808,6 +1810,11 @@ fn process_single_slot(
         result?
     }
     bank.freeze(); // all banks handled by this routine are created from complete slots
+
+    if let Some(slot_callback) = &opts.slot_callback {
+        slot_callback(bank);
+    }
+
     if blockstore.is_primary_access() {
         blockstore.insert_bank_hash(bank.slot(), bank.hash(), false);
     }