From 8013ba64e693925d959b16910d5540f1974da4b9 Mon Sep 17 00:00:00 2001 From: Sean Young Date: Mon, 18 Dec 2023 12:50:48 +0000 Subject: [PATCH] ledger-tool: verify: add --verify-slots and --verify-slots-details This adds: --verify-slots If the file does not exist, write the slot hashes to this file; if the file does exist, verify slot hashes against this file. --verify-slots-details none|bank Store the bank (=accounts) json file, or not. The first case can be used to dump a list of (slot, hash) to a json file during a replay. The second case can be used to check slot hashes against previously recorded values. This is useful for debugging consensus failures, eg: # on good commit/branch ledger-tool verify --verify-slots good.json --verify-slots-details=bank # on bad commit or potentially consensus breaking branch ledger-tool verify --verify-slots good.json On a hash mismatch an error will be logged with the expected hash vs the computed hash. --- accounts-db/src/transaction_results.rs | 4 +- ledger-tool/src/main.rs | 117 ++++++++++++++++++++++++- ledger/src/blockstore_processor.rs | 11 ++- runtime/src/bank/bank_hash_details.rs | 10 +-- 4 files changed, 130 insertions(+), 12 deletions(-) diff --git a/accounts-db/src/transaction_results.rs b/accounts-db/src/transaction_results.rs index bcfe185856ace4..5aa311acea9103 100644 --- a/accounts-db/src/transaction_results.rs +++ b/accounts-db/src/transaction_results.rs @@ -74,7 +74,7 @@ impl TransactionExecutionResult { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct TransactionExecutionDetails { pub status: transaction::Result<()>, pub log_messages: Option>, @@ -87,7 +87,7 @@ pub struct TransactionExecutionDetails { pub accounts_data_len_delta: i64, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum DurableNonceFee { Valid(u64), Invalid, diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 47b5cc0024400d..98f6a28ed1e2f1 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -40,7 +40,7 @@ use { blockstore::{create_new_ledger, Blockstore, PurgeType}, blockstore_db::{self, columns as cf, Column, ColumnName, Database}, blockstore_options::{AccessType, LedgerColumnOptions, BLOCKSTORE_DIRECTORY_ROCKS_FIFO}, - blockstore_processor::ProcessOptions, + blockstore_processor::{ProcessOptions, ProcessSlotCallback}, shred::Shred, use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup}, }, @@ -90,7 +90,7 @@ use { str::FromStr, sync::{ atomic::{AtomicBool, Ordering}, - Arc, RwLock, + Arc, Mutex, RwLock, }, time::{Duration, UNIX_EPOCH}, }, @@ -1461,7 +1461,22 @@ fn main() { information that went into computing the completed bank's bank hash. \ The file will be written within /bank_hash_details/", ), - ), + ) + .arg( + Arg::with_name("verify_slots") + .long("verify-slots") + .takes_value(true) + .value_name("FILENAME") + .help("Record slots to new file or verify slots match contents of existing file.") + ) + .arg( + Arg::with_name("verify_slots_details") + .long("verify-slots-details") + .possible_values(&["none", "bank"]) + .default_value("none") + .takes_value(true) + .help("In the slot recording, include bank details or not") + ) ) .subcommand( SubCommand::with_name("graph") @@ -2387,6 +2402,86 @@ fn main() { ); } + let include_bank = match arg_matches.value_of("verify_slots_details").unwrap() { + "none" => false, + "bank" => true, + _ => unreachable!(), + }; + + let (slot_callback, record_slots_file, recorded_slots) = if let Some(filename) = + arg_matches.value_of_os("verify_slots") + { + let filename = Path::new(filename); + + if filename.exists() { + let file = File::open(filename).unwrap_or_else(|err| { + eprintln!("Unable to read file: {}: {err:#}", filename.display()); + exit(1); + }); + + let details: bank_hash_details::BankHashDetails = + serde_json::from_reader(file).unwrap_or_else(|err| { + eprintln!("Error loading slots file: {err:#}"); + exit(1); + }); + + let slots = Arc::new(Mutex::new(details.bank_hash_details)); + + let slot_callback = Arc::new(move |bank: &Bank| { + let bank_hash_details::BankHashSlotDetails { + slot: expected_slot, + bank_hash: expected_hash, + .. + } = slots.lock().unwrap().remove(0); + if bank.slot() != expected_slot + || bank.hash().to_string() != expected_hash + { + error!("Expected slot: {expected_slot} hash: {expected_hash} got slot: {} hash: {}", + bank.slot(), bank.hash()); + } + }); + + (Some(slot_callback as ProcessSlotCallback), None, None) + } else { + let file = File::create(filename).unwrap_or_else(|err| { + eprintln!( + "Unable to write to file: {}: {:#}", + filename.display(), + err + ); + exit(1); + }); + + let slot_hashes = Arc::new(Mutex::new(Vec::new())); + + let slot_callback = Arc::new({ + let slots = Arc::clone(&slot_hashes); + move |bank: &Bank| { + let slot_details = if include_bank { + bank_hash_details::BankHashSlotDetails::try_from(bank) + .unwrap() + } else { + bank_hash_details::BankHashSlotDetails { + slot: bank.slot(), + bank_hash: bank.hash().to_string(), + ..Default::default() + } + }; + + slots.lock().unwrap().push(slot_details); + } + }); + + ( + Some(slot_callback as ProcessSlotCallback), + Some(file), + Some(slot_hashes), + ) + } + } else { + (None, None, None) + }; + let process_options = ProcessOptions { new_hard_forks: hardforks_of(arg_matches, "hard_forks"), run_verification: !(arg_matches.is_present("skip_poh_verify") @@ -2414,6 +2509,7 @@ fn main() { use_snapshot_archives_at_startup::cli::NAME, UseSnapshotArchivesAtStartup ), + slot_callback, ..ProcessOptions::default() }; let print_accounts_stats = arg_matches.is_present("print_accounts_stats"); @@ -2447,6 +2543,21 @@ fn main() { }) .ok(); } + + if let Some(recorded_slots_file) = record_slots_file { + if let Ok(recorded_slots) = recorded_slots.clone().unwrap().lock() { + let bank_hashes = + bank_hash_details::BankHashDetails::new(recorded_slots.to_vec()); + + // writing the json file ends up with a syscall for each number, comma, indentation etc. + // use BufWriter to speed things up + + let writer = std::io::BufWriter::new(recorded_slots_file); + + serde_json::to_writer_pretty(writer, &bank_hashes).unwrap(); + } + } + exit_signal.store(true, Ordering::Relaxed); system_monitor_service.join().unwrap(); } diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index cc8a4e5cb607ac..82286aab010dee 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -674,8 +674,9 @@ pub enum BlockstoreProcessorError { RootBankWithMismatchedCapitalization(Slot), } -/// Callback for accessing bank state while processing the blockstore -pub type ProcessCallback = Arc; +/// Callback for accessing bank state after each slot is confirmed while +/// processing the blockstore +pub type ProcessSlotCallback = Arc; #[derive(Default, Clone)] pub struct ProcessOptions { @@ -683,6 +684,7 @@ pub struct ProcessOptions { pub run_verification: bool, pub full_leader_cache: bool, pub halt_at_slot: Option, + pub slot_callback: Option, pub new_hard_forks: Option>, pub debug_keys: Option>>, pub account_indexes: AccountSecondaryIndexes, @@ -1808,6 +1810,11 @@ fn process_single_slot( result? } bank.freeze(); // all banks handled by this routine are created from complete slots + + if let Some(slot_callback) = &opts.slot_callback { + slot_callback(bank); + } + if blockstore.is_primary_access() { blockstore.insert_bank_hash(bank.slot(), bank.hash(), false); } diff --git a/runtime/src/bank/bank_hash_details.rs b/runtime/src/bank/bank_hash_details.rs index 9072f6a12f1496..b705d5876e3493 100644 --- a/runtime/src/bank/bank_hash_details.rs +++ b/runtime/src/bank/bank_hash_details.rs @@ -22,7 +22,7 @@ use { }; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub(crate) struct BankHashDetails { +pub struct BankHashDetails { /// The client version pub version: String, /// The encoding format for account data buffers @@ -66,8 +66,8 @@ impl BankHashDetails { } /// The components that go into a bank hash calculation for a single bank/slot. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub(crate) struct BankHashSlotDetails { +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, Default)] +pub struct BankHashSlotDetails { pub slot: Slot, pub bank_hash: String, pub parent_bank_hash: String, @@ -141,8 +141,8 @@ impl TryFrom<&Bank> for BankHashSlotDetails { /// Wrapper around a Vec<_> to facilitate custom Serialize/Deserialize trait /// implementations. -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct BankHashAccounts { +#[derive(Clone, Debug, Eq, PartialEq, Default)] +pub struct BankHashAccounts { pub accounts: Vec, }