feat(zebra-checkpoints): make zebra-checkpoints work for zebrad bac…

…kend (#5894)

* make `zebra-checkpoint` util work with zebra as the backend

* update snapshots

* update documentation

* applies suggestions from code review

* irefactor zebra-checkpoints to work with zebra using deserialization of the raw block

* fix imports and derives

* rename mode to backend

* remove old stuff

* fix docs

Co-authored-by: arya2 <[email protected]>

book/src/dev/zebra-checkpoints.md

@@ -1,6 +1,6 @@
 # zebra-checkpoints
 
-`zebra-checkpoints` uses a local `zcashd` instance to generate a list of checkpoints for Zebra's checkpoint verifier.
+`zebra-checkpoints` uses a local `zebrad` or `zcashd` instance to generate a list of checkpoints for Zebra's checkpoint verifier.
 
 Developers should run this tool every few months to add new checkpoints to Zebra.
 (By default, Zebra uses these checkpoints to sync to the chain tip.)

zebra-consensus/src/checkpoint/README.md

@@ -22,14 +22,16 @@ Or jump straight to [the exact commands for updating the lists](https://github.c
 
 ### Use the `zebra-checkpoints` utility
 
-`zebra-checkpoints` is the program we use to collect checkpoints. Currently this program uses `zcash-cli` to get the hashes. `zcash-cli` must be available in your machine and it must be connected to a synchronized (Mainnet or Testnet) instance of `zcashd` to get the most recent hashes.
+`zebra-checkpoints` is the program we use to collect checkpoints. Currently this program uses `zcash-cli` to get the hashes. `zcash-cli` must be available in your machine and it must be connected to a synchronized (Mainnet or Testnet) instance of `zebrad` or `zcashd` to get the most recent hashes.
 
 First, [build the `zebra-checkpoints` binary](https://github.com/ZcashFoundation/zebra/tree/main/zebra-utils/README.md#zebra-checkpoints).
 
 When updating the lists there is no need to start from the genesis block. The program option `--last-checkpoint` will let you specify at what block height you want to start. In general the maintainer will copy the last height from each list and use the mentioned option to start from there.
 
 It is easier if `zcash-cli` is in your execution path however you can specify the location of it anywhere in the filesystem with option `--cli`.
 
+By default, `zebra-checkpoints` will use a `zebrad` backend. If the running instance is `zcashd`, please add `-b zcashd` to your command. 
+
 To update the checkpoint list, run:
 
 ```sh
@@ -43,7 +45,7 @@ $ zebra-checkpoints --last-checkpoint $(tail -1 zebra-consensus/src/checkpoint/m
 ...
 ```
 
-If we are looking to update the testnet hashes we must make sure the cli is connected with a testnet chain. If we have our `zcashd` running locally we can make this by starting with `zcashd -testnet`.
+If we are looking to update the testnet hashes we must make sure the cli is connected with a testnet chain. If we are using `zcashd` as the backend and this is running locally, we can make this by starting with `zcashd -testnet`. If we are using `zebrad` as the backend, then we must start with a configuration file where the `network` field of the `[network]` section is `Testnet`.
 
 Anything we add after `--` will pass through into the `zcash-cli` program so we can specify the testnet here.
 

zebra-utils/Cargo.toml

@@ -39,6 +39,7 @@ hex = "0.4.3"
 serde_json = "1.0.91"
 tracing-error = "0.2.0"
 tracing-subscriber = "0.3.16"
+thiserror = "1.0.38"
 
 zebra-node-services = { path = "../zebra-node-services" }
 zebra-chain = { path = "../zebra-chain" }

zebra-utils/README.md

@@ -15,7 +15,7 @@ Binaries are easier to use if they are located in your system execution path.
 
 This command generates a list of zebra checkpoints, and writes them to standard output. Each checkpoint consists of a block height and hash.
 
-To create checkpoints, you need a synchronized instance of `zcashd`, and the `zcash-cli` RPC client.
+To create checkpoints, you need a synchronized instance of `zebrad` or `zcashd`, and the `zcash-cli` RPC client.
 
 `zebra-checkpoints` is a standalone rust binary, you can compile it using:
 

zebra-utils/src/bin/zebra-checkpoints/args.rs

@@ -3,10 +3,40 @@
 //! For usage please refer to the program help: `zebra-checkpoints --help`
 
 use structopt::StructOpt;
+use thiserror::Error;
+
+use std::str::FromStr;
+
+/// The backend type the zebra-checkpoints utility will use to get data from.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Backend {
+    Zebrad,
+    Zcashd,
+}
+
+impl FromStr for Backend {
+    type Err = InvalidModeError;
+
+    fn from_str(string: &str) -> Result<Self, Self::Err> {
+        match string.to_lowercase().as_str() {
+            "zebrad" => Ok(Backend::Zebrad),
+            "zcashd" => Ok(Backend::Zcashd),
+            _ => Err(InvalidModeError(string.to_owned())),
+        }
+    }
+}
+
+#[derive(Debug, Error)]
+#[error("Invalid mode: {0}")]
+pub struct InvalidModeError(String);
 
 /// zebra-checkpoints arguments
 #[derive(Clone, Debug, Eq, PartialEq, StructOpt)]
 pub struct Args {
+    /// Backend type
+    #[structopt(default_value = "zebrad", short, long)]
+    pub backend: Backend,
+
     /// Path to zcash-cli command
     #[structopt(default_value = "zcash-cli", short, long)]
     pub cli: String,

zebra-utils/src/bin/zebra-checkpoints/main.rs

@@ -14,10 +14,13 @@ use std::process::Stdio;
 use std::os::unix::process::ExitStatusExt;
 
 use color_eyre::eyre::{ensure, Result};
+use hex::FromHex;
 use serde_json::Value;
 use structopt::StructOpt;
 
-use zebra_chain::{block, transparent::MIN_TRANSPARENT_COINBASE_MATURITY};
+use zebra_chain::{
+    block, serialization::ZcashDeserializeInto, transparent::MIN_TRANSPARENT_COINBASE_MATURITY,
+};
 use zebra_node_services::constants::{MAX_CHECKPOINT_BYTE_COUNT, MAX_CHECKPOINT_HEIGHT_GAP};
 use zebra_utils::init_tracing;
 
@@ -69,9 +72,14 @@ fn main() -> Result<()> {
 
     // get the current block count
     let mut cmd = passthrough_cmd();
-    cmd.arg("getblockcount");
+    cmd.arg("getblockchaininfo");
+
+    let output = cmd_output(&mut cmd)?;
+    let get_block_chain_info: Value = serde_json::from_str(&output)?;
+
     // calculate the maximum height
-    let height_limit: block::Height = cmd_output(&mut cmd)?.trim().parse()?;
+    let height_limit = block::Height(get_block_chain_info["blocks"].as_u64().unwrap() as u32);
+
     assert!(height_limit <= block::Height::MAX);
     // Checkpoints must be on the main chain, so we skip blocks that are within the
     // Zcash reorg limit.
@@ -104,18 +112,46 @@ fn main() -> Result<()> {
         // unfortunately we need to create a process for each block
         let mut cmd = passthrough_cmd();
 
-        // get block data
-        cmd.args(["getblock", &x.to_string()]);
-        let output = cmd_output(&mut cmd)?;
-        // parse json
-        let v: Value = serde_json::from_str(&output)?;
+        let (hash, height, size) = match args::Args::from_args().backend {
+            args::Backend::Zcashd => {
+                // get block data from zcashd using verbose=1
+                cmd.args(["getblock", &x.to_string(), "1"]);
+                let output = cmd_output(&mut cmd)?;
+
+                // parse json
+                let v: Value = serde_json::from_str(&output)?;
+
+                // get the values we are interested in
+                let hash: block::Hash = v["hash"].as_str().unwrap().parse()?;
+                let height = block::Height(v["height"].as_u64().unwrap() as u32);
+
+                let size = v["size"].as_u64().unwrap();
+
+                (hash, height, size)
+            }
+            args::Backend::Zebrad => {
+                // get block data from zebrad by deserializing the raw block
+                cmd.args(["getblock", &x.to_string(), "0"]);
+                let output = cmd_output(&mut cmd)?;
+
+                let block_bytes = <Vec<u8>>::from_hex(output.trim_end_matches('\n'))?;
+
+                let block = block_bytes
+                    .zcash_deserialize_into::<block::Block>()
+                    .expect("obtained block should deserialize");
+
+                (
+                    block.hash(),
+                    block
+                        .coinbase_height()
+                        .expect("block has always a coinbase height"),
+                    block_bytes.len().try_into()?,
+                )
+            }
+        };
 
-        // get the values we are interested in
-        let hash: block::Hash = v["hash"].as_str().unwrap().parse()?;
-        let height = block::Height(v["height"].as_u64().unwrap() as u32);
         assert!(height <= block::Height::MAX);
         assert_eq!(x, height.0);
-        let size = v["size"].as_u64().unwrap();
 
         // compute
         cumulative_bytes += size;