From 46c3aa9076934b26de7e243d1a06112ed5bd2443 Mon Sep 17 00:00:00 2001 From: Alessandro Decina Date: Fri, 9 Dec 2022 10:50:08 +0000 Subject: [PATCH] ledger-tool: verify: add --verify-slot-hashes This adds: --verify-slot-hashes If the file does not exist, write the slot hashes to this file; if the file does exist, verify slot hashes against this file. The first case can be used to dump a list of (slot, hash) to a json file during a replay. The second case can be used to check slot hashes against previously recorded values. This is useful for debugging consensus failures, eg: # on good commit/branch ledger-tool verify --verify-slot-hashes good.json ... # on bad commit or potentially consensus breaking branch ledger-tool verify --verify-slot-hashes good.json ... On a hash mismatch an error will be logged with the expected hash vs the computed hash. --- ledger-tool/src/main.rs | 92 +++++++++++++++++++++++++++++- ledger/src/blockstore_processor.rs | 11 +++- 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index ac8404edd53dbb..4cc52bc9594520 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1068,6 +1068,11 @@ fn get_access_type(process_options: &ProcessOptions) -> AccessType { #[cfg(not(target_env = "msvc"))] use jemallocator::Jemalloc; +use { + serde::Deserialize, + solana_ledger::blockstore_processor::ProcessSlotCallback, + std::{collections::VecDeque, sync::Mutex}, +}; #[cfg(not(target_env = "msvc"))] #[global_allocator] @@ -1716,7 +1721,14 @@ fn main() { information that went into computing the completed bank's bank hash. \ The file will be written within /bank_hash_details/", ), - ), + ) + .arg( + Arg::with_name("verify_slot_hashes") + .long("verify-slot-hashes") + .takes_value(true) + .value_name("FILENAME") + .help("Record slot hashes to new file or verify slot hashes match contents of existing file.") + ) ) .subcommand( SubCommand::with_name("graph") @@ -2722,6 +2734,74 @@ fn main() { ); } + #[derive(Serialize, Deserialize)] + struct SlotHash { + slot: Slot, + hash: String, + } + + let (slot_callback, record_slot_hashes_file, recorded_slot_hashes) = if let Some( + filename, + ) = + arg_matches.value_of_os("verify_slot_hashes") + { + let filename = Path::new(filename); + + if filename.exists() { + let file = File::open(filename).unwrap_or_else(|err| { + eprintln!("Unable to read file: {}: {err:#}", filename.display()); + exit(1); + }); + + let slot_hashes: Arc>> = Arc::new(Mutex::new( + serde_json::from_reader(file).unwrap_or_else(|err| { + eprintln!("Error loading slot hashes file: {err:#}"); + exit(1); + }), + )); + + let slot_callback = Arc::new(move |bank: &Bank| { + let SlotHash { + slot: expected_slot, + hash: expected_hash, + } = slot_hashes.lock().unwrap().pop_front().unwrap(); + if bank.slot() != expected_slot + || bank.hash().to_string() != expected_hash + { + error!("Expected slot: {expected_slot} hash: {expected_hash} got slot: {} hash: {}", + bank.slot(), bank.hash()); + } + }); + + (Some(slot_callback as ProcessSlotCallback), None, None) + } else { + let file = File::create(filename).unwrap_or_else(|err| { + eprintln!("Unable to write to file: {}: {:#}", filename.display(), err); + exit(1); + }); + + let slot_hashes = Arc::new(Mutex::new(Vec::new())); + + let slot_callback = Arc::new({ + let slot_hashes = Arc::clone(&slot_hashes); + move |bank: &Bank| { + slot_hashes.lock().unwrap().push(SlotHash { + slot: bank.slot(), + hash: bank.hash().to_string(), + }); + } + }); + + ( + Some(slot_callback as ProcessSlotCallback), + Some(file), + Some(slot_hashes), + ) + } + } else { + (None, None, None) + }; + let process_options = ProcessOptions { new_hard_forks: hardforks_of(arg_matches, "hard_forks"), run_verification: !(arg_matches.is_present("skip_poh_verify") @@ -2749,6 +2829,7 @@ fn main() { use_snapshot_archives_at_startup::cli::NAME, UseSnapshotArchivesAtStartup ), + slot_callback, ..ProcessOptions::default() }; let print_accounts_stats = arg_matches.is_present("print_accounts_stats"); @@ -2787,6 +2868,15 @@ fn main() { }) .ok(); } + + if let Some(recorded_slot_hashes_file) = record_slot_hashes_file { + serde_json::to_writer( + recorded_slot_hashes_file, + &recorded_slot_hashes.unwrap(), + ) + .unwrap(); + } + exit_signal.store(true, Ordering::Relaxed); system_monitor_service.join().unwrap(); } diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index e2208ce557e12d..df341de081f6e3 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -674,8 +674,9 @@ pub enum BlockstoreProcessorError { RootBankWithMismatchedCapitalization(Slot), } -/// Callback for accessing bank state while processing the blockstore -pub type ProcessCallback = Arc; +/// Callback for accessing bank state after each slot is confirmed while +/// processing the blockstore +pub type ProcessSlotCallback = Arc; #[derive(Default, Clone)] pub struct ProcessOptions { @@ -683,6 +684,7 @@ pub struct ProcessOptions { pub run_verification: bool, pub full_leader_cache: bool, pub halt_at_slot: Option, + pub slot_callback: Option, pub new_hard_forks: Option>, pub debug_keys: Option>>, pub account_indexes: AccountSecondaryIndexes, @@ -1808,6 +1810,11 @@ fn process_single_slot( result? } bank.freeze(); // all banks handled by this routine are created from complete slots + + if let Some(slot_callback) = &opts.slot_callback { + slot_callback(bank); + } + if blockstore.is_primary_access() { blockstore.insert_bank_hash(bank.slot(), bank.hash(), false); }