ledger-tool: verify: add --verify-slots and --verify-slots-details

This adds:

    --verify-slots <FILENAME>
        If the file does not exist, write the slot hashes to this file;
        if the file does exist, verify slot hashes against this file.

    --verify-slots-details none|bank|tx|bank-tx
	Store the bank (=accounts) and/or transaction batches in the
	json file. For each tx batch, the bank hash is included.

The first case can be used to dump a list of (slot, hash) to a json file
during a replay. The second case can be used to check slot hashes against
previously recorded values.

This is useful for debugging consensus failures, eg:

    # on good commit/branch
    ledger-tool verify --verify-slots good.json --verify-slots-details=tx

    # on bad commit or potentially consensus breaking branch
    ledger-tool verify --verify-slots good.json --verify-slots-details=tx

On a hash mismatch an error will be logged with the expected hash vs the
computed hash.

accounts-db/src/transaction_results.rs

@@ -68,7 +68,7 @@ impl TransactionExecutionResult {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct TransactionExecutionDetails {
     pub status: transaction::Result<()>,
     pub log_messages: Option<Vec<String>>,
@@ -81,7 +81,7 @@ pub struct TransactionExecutionDetails {
     pub accounts_data_len_delta: i64,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum DurableNonceFee {
     Valid(u64),
     Invalid,
@@ -109,7 +109,7 @@ impl DurableNonceFee {
 /// transaction instruction
 pub type InnerInstructions = Vec<InnerInstruction>;
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct InnerInstruction {
     pub instruction: CompiledInstruction,
     /// Invocation stack height of this instruction. Instruction stack height

core/src/validator.rs

@@ -2168,6 +2168,7 @@ fn initialize_rpc_transaction_history_services(
     let max_complete_transaction_status_slot = Arc::new(AtomicU64::new(blockstore.max_root()));
     let (transaction_status_sender, transaction_status_receiver) = unbounded();
     let transaction_status_sender = Some(TransactionStatusSender {
+        bank_hash: false,
         sender: transaction_status_sender,
     });
     let transaction_status_service = Some(TransactionStatusService::new(

ledger-tool/Cargo.toml

@@ -18,6 +18,7 @@ csv = { workspace = true }
 dashmap = { workspace = true }
 futures = { workspace = true }
 histogram = { workspace = true }
+hex = { workspace = true }
 itertools = { workspace = true }
 log = { workspace = true }
 num_cpus = { workspace = true }

ledger-tool/src/ledger_utils.rs

@@ -68,6 +68,7 @@ pub fn load_and_process_ledger(
     blockstore: Arc<Blockstore>,
     process_options: ProcessOptions,
     snapshot_archive_path: Option<PathBuf>,
+    transaction_status_sender: Option<TransactionStatusSender>,
     incremental_snapshot_archive_path: Option<PathBuf>,
 ) -> Result<(Arc<RwLock<BankForks>>, Option<StartingSnapshotHashes>), String> {
     let bank_snapshots_dir = if blockstore.is_primary_access() {
@@ -326,12 +327,13 @@ pub fn load_and_process_ledger(
             );
             (
                 Some(TransactionStatusSender {
+                    bank_hash: false,
                     sender: transaction_status_sender,
                 }),
                 Some(transaction_status_service),
             )
         } else {
-            (None, None)
+            (transaction_status_sender, None)
         };
 
     let result = blockstore_processor::process_blockstore_from_root(
@@ -352,6 +354,8 @@ pub fn load_and_process_ledger(
     accounts_hash_verifier.join().unwrap();
     if let Some(service) = transaction_status_service {
         service.join().unwrap();
+    } else if let Some(transaction_status_sender) = transaction_status_sender {
+        drop(transaction_status_sender.sender);
     }
 
     result