ledger-tool: verify: add --verify-slots and --verify-slots-details

This adds:

    --verify-slots <FILENAME>
        If the file does not exist, write the slot hashes to this file;
        if the file does exist, verify slot hashes against this file.

    --verify-slots-details none|bank
	Store the bank (=accounts) json file, or not.

The first case can be used to dump a list of (slot, hash) to a json file
during a replay. The second case can be used to check slot hashes against
previously recorded values.

This is useful for debugging consensus failures, eg:

    # on good commit/branch
    ledger-tool verify --verify-slots good.json --verify-slots-details=bank

    # on bad commit or potentially consensus breaking branch
    ledger-tool verify --verify-slots good.json

On a hash mismatch an error will be logged with the expected hash vs the
computed hash.

accounts-db/src/transaction_results.rs

@@ -74,7 +74,7 @@ impl TransactionExecutionResult {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct TransactionExecutionDetails {
     pub status: transaction::Result<()>,
     pub log_messages: Option<Vec<String>>,
@@ -87,7 +87,7 @@ pub struct TransactionExecutionDetails {
     pub accounts_data_len_delta: i64,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum DurableNonceFee {
     Valid(u64),
     Invalid,

ledger-tool/src/main.rs

@@ -40,7 +40,7 @@ use {
         blockstore::{create_new_ledger, Blockstore, PurgeType},
         blockstore_db::{self, columns as cf, Column, ColumnName, Database},
         blockstore_options::{AccessType, LedgerColumnOptions, BLOCKSTORE_DIRECTORY_ROCKS_FIFO},
-        blockstore_processor::ProcessOptions,
+        blockstore_processor::{ProcessOptions, ProcessSlotCallback},
         shred::Shred,
         use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup},
     },
@@ -90,7 +90,7 @@ use {
         str::FromStr,
         sync::{
             atomic::{AtomicBool, Ordering},
-            Arc, RwLock,
+            Arc, Mutex, RwLock,
         },
         time::{Duration, UNIX_EPOCH},
     },
@@ -1461,7 +1461,22 @@ fn main() {
                              information that went into computing the completed bank's bank hash. \
                              The file will be written within <LEDGER_DIR>/bank_hash_details/",
                         ),
-                ),
+                )
+                .arg(
+                    Arg::with_name("verify_slots")
+                        .long("verify-slots")
+                        .takes_value(true)
+                        .value_name("FILENAME")
+                        .help("Record slots to new file or verify slots match contents of existing file.")
+                )
+                .arg(
+                    Arg::with_name("verify_slots_details")
+                        .long("verify-slots-details")
+                        .possible_values(&["none", "bank"])
+                        .default_value("none")
+                        .takes_value(true)
+                        .help("In the slot recording, include bank details or not")
+                )
         )
         .subcommand(
             SubCommand::with_name("graph")
@@ -2387,6 +2402,86 @@ fn main() {
                         );
                     }
 
+                    let include_bank = match arg_matches.value_of("verify_slots_details").unwrap() {
+                        "none" => false,
+                        "bank" => true,
+                        _ => unreachable!(),
+                    };
+
+                    let (slot_callback, record_slots_file, recorded_slots) = if let Some(filename) =
+                        arg_matches.value_of_os("verify_slots")
+                    {
+                        let filename = Path::new(filename);
+
+                        if filename.exists() {
+                            let file = File::open(filename).unwrap_or_else(|err| {
+                                eprintln!("Unable to read file: {}: {err:#}", filename.display());
+                                exit(1);
+                            });
+
+                            let details: bank_hash_details::BankHashDetails =
+                                serde_json::from_reader(file).unwrap_or_else(|err| {
+                                    eprintln!("Error loading slots file: {err:#}");
+                                    exit(1);
+                                });
+
+                            let slots = Arc::new(Mutex::new(details.bank_hash_details));
+
+                            let slot_callback = Arc::new(move |bank: &Bank| {
+                                let bank_hash_details::BankHashSlotDetails {
+                                    slot: expected_slot,
+                                    bank_hash: expected_hash,
+                                    ..
+                                } = slots.lock().unwrap().remove(0);
+                                if bank.slot() != expected_slot
+                                    || bank.hash().to_string() != expected_hash
+                                {
+                                    error!("Expected slot: {expected_slot} hash: {expected_hash} got slot: {} hash: {}",
+                                        bank.slot(), bank.hash());
+                                }
+                            });
+
+                            (Some(slot_callback as ProcessSlotCallback), None, None)
+                        } else {
+                            let file = File::create(filename).unwrap_or_else(|err| {
+                                eprintln!(
+                                    "Unable to write to file: {}: {:#}",
+                                    filename.display(),
+                                    err
+                                );
+                                exit(1);
+                            });
+
+                            let slot_hashes = Arc::new(Mutex::new(Vec::new()));
+
+                            let slot_callback = Arc::new({
+                                let slots = Arc::clone(&slot_hashes);
+                                move |bank: &Bank| {
+                                    let slot_details = if include_bank {
+                                        bank_hash_details::BankHashSlotDetails::try_from(bank)
+                                            .unwrap()
+                                    } else {
+                                        bank_hash_details::BankHashSlotDetails {
+                                            slot: bank.slot(),
+                                            bank_hash: bank.hash().to_string(),
+                                            ..Default::default()
+                                        }
+                                    };
+
+                                    slots.lock().unwrap().push(slot_details);
+                                }
+                            });
+
+                            (
+                                Some(slot_callback as ProcessSlotCallback),
+                                Some(file),
+                                Some(slot_hashes),
+                            )
+                        }
+                    } else {
+                        (None, None, None)
+                    };
+
                     let process_options = ProcessOptions {
                         new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
                         run_verification: !(arg_matches.is_present("skip_poh_verify")
@@ -2414,6 +2509,7 @@ fn main() {
                             use_snapshot_archives_at_startup::cli::NAME,
                             UseSnapshotArchivesAtStartup
                         ),
+                        slot_callback,
                         ..ProcessOptions::default()
                     };
                     let print_accounts_stats = arg_matches.is_present("print_accounts_stats");
@@ -2447,6 +2543,21 @@ fn main() {
                             })
                             .ok();
                     }
+
+                    if let Some(recorded_slots_file) = record_slots_file {
+                        if let Ok(recorded_slots) = recorded_slots.clone().unwrap().lock() {
+                            let bank_hashes =
+                                bank_hash_details::BankHashDetails::new(recorded_slots.to_vec());
+
+                            // writing the json file ends up with a syscall for each number, comma, indentation etc.
+                            // use BufWriter to speed things up
+
+                            let writer = std::io::BufWriter::new(recorded_slots_file);
+
+                            serde_json::to_writer_pretty(writer, &bank_hashes).unwrap();
+                        }
+                    }
+
                     exit_signal.store(true, Ordering::Relaxed);
                     system_monitor_service.join().unwrap();
                 }

ledger/src/blockstore_processor.rs

@@ -674,15 +674,17 @@ pub enum BlockstoreProcessorError {
     RootBankWithMismatchedCapitalization(Slot),
 }
 
-/// Callback for accessing bank state while processing the blockstore
-pub type ProcessCallback = Arc<dyn Fn(&Bank) + Sync + Send>;
+/// Callback for accessing bank state after each slot is confirmed while
+/// processing the blockstore
+pub type ProcessSlotCallback = Arc<dyn Fn(&Bank) + Sync + Send>;
 
 #[derive(Default, Clone)]
 pub struct ProcessOptions {
     /// Run PoH, transaction signature and other transaction verifications on the entries.
     pub run_verification: bool,
     pub full_leader_cache: bool,
     pub halt_at_slot: Option<Slot>,
+    pub slot_callback: Option<ProcessSlotCallback>,
     pub new_hard_forks: Option<Vec<Slot>>,
     pub debug_keys: Option<Arc<HashSet<Pubkey>>>,
     pub account_indexes: AccountSecondaryIndexes,
@@ -1808,6 +1810,11 @@ fn process_single_slot(
         result?
     }
     bank.freeze(); // all banks handled by this routine are created from complete slots
+
+    if let Some(slot_callback) = &opts.slot_callback {
+        slot_callback(bank);
+    }
+
     if blockstore.is_primary_access() {
         blockstore.insert_bank_hash(bank.slot(), bank.hash(), false);
     }

runtime/src/bank/bank_hash_details.rs

@@ -22,7 +22,7 @@ use {
 };
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-pub(crate) struct BankHashDetails {
+pub struct BankHashDetails {
     /// The client version
     pub version: String,
     /// The encoding format for account data buffers
@@ -66,8 +66,8 @@ impl BankHashDetails {
 }
 
 /// The components that go into a bank hash calculation for a single bank/slot.
-#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-pub(crate) struct BankHashSlotDetails {
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, Default)]
+pub struct BankHashSlotDetails {
     pub slot: Slot,
     pub bank_hash: String,
     pub parent_bank_hash: String,
@@ -141,8 +141,8 @@ impl TryFrom<&Bank> for BankHashSlotDetails {
 
 /// Wrapper around a Vec<_> to facilitate custom Serialize/Deserialize trait
 /// implementations.
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub(crate) struct BankHashAccounts {
+#[derive(Clone, Debug, Eq, PartialEq, Default)]
+pub struct BankHashAccounts {
     pub accounts: Vec<PubkeyHashAccount>,
 }