diff --git a/Cargo.lock b/Cargo.lock index b150e24..1bdc4af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -450,6 +450,20 @@ dependencies = [ "constant_time_eq", ] +[[package]] +name = "blobscan-client" +version = "0.1.0" +dependencies = [ + "eyre", + "hex", + "reqwest", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "block-buffer" version = "0.7.3" @@ -4455,6 +4469,7 @@ dependencies = [ "async-trait", "bincode", "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "blobscan-client", "bytes", "chrono", "clap", @@ -4486,6 +4501,7 @@ version = "0.1.0" dependencies = [ "bincode", "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "blobscan-client", "chrono", "ethers", "eyre", diff --git a/Cargo.toml b/Cargo.toml index c0a84a3..e0eab2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,12 +5,13 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] -members = ["state-reconstruct-fetcher"] +members = ["state-reconstruct-fetcher", "state-reconstruct-fetcher/blobscan-client"] [dependencies] async-trait = "0.1.74" bincode = "1" blake2 = "0.10.6" +blobscan-client = { path = "./state-reconstruct-fetcher/blobscan-client" } bytes = "1.5" chrono = "0.4.31" clap = { version = "4.4.7", features = ["derive", "env"] } diff --git a/state-reconstruct-fetcher/Cargo.toml b/state-reconstruct-fetcher/Cargo.toml index 4df31de..a1d7eb9 100644 --- a/state-reconstruct-fetcher/Cargo.toml +++ b/state-reconstruct-fetcher/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] bincode = "1.3.3" blake2 = "0.10.6" +blobscan-client = { path = "./blobscan-client" } ethers = "1.0.2" eyre = "0.6.8" indexmap = { version = "2.0.2", features = ["serde"] } diff --git a/state-reconstruct-fetcher/blobscan-client/Cargo.toml b/state-reconstruct-fetcher/blobscan-client/Cargo.toml new file mode 100644 index 0000000..97f1832 --- /dev/null +++ b/state-reconstruct-fetcher/blobscan-client/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "blobscan-client" +version = "0.1.0" +edition = "2021" + +[dependencies] +eyre = "0.6.8" +hex = "0.4.3" +reqwest = "0.11.24" +serde = { version = "1.0.189", features = ["derive"] } +serde_json = { version = "1.0.107", features = ["std"] } +thiserror = "1.0.50" +tokio = { version = "1.33.0", features = ["signal"] } +tracing = "0.1.40" diff --git a/state-reconstruct-fetcher/blobscan-client/src/blob_support.rs b/state-reconstruct-fetcher/blobscan-client/src/blob_support.rs new file mode 100644 index 0000000..83a0aac --- /dev/null +++ b/state-reconstruct-fetcher/blobscan-client/src/blob_support.rs @@ -0,0 +1,17 @@ +use thiserror::Error; + +#[allow(clippy::enum_variant_names)] +#[derive(Error, Debug)] +pub enum BlobError { + #[error("blob storage error: {0}")] + StorageError(String), + + #[error("blob format error")] + FormatError(String, String), +} + +pub trait BlobSupport { + fn format_url(&self, kzg_commitment: &[u8]) -> String; + + fn get_blob_data(&self, json_str: &str) -> Result; +} diff --git a/state-reconstruct-fetcher/blobscan-client/src/lib.rs b/state-reconstruct-fetcher/blobscan-client/src/lib.rs new file mode 100644 index 0000000..9afad61 --- /dev/null +++ b/state-reconstruct-fetcher/blobscan-client/src/lib.rs @@ -0,0 +1,4 @@ +mod blob_support; +pub use crate::blob_support::{BlobError, BlobSupport}; +mod scraping_support; +pub use crate::scraping_support::ScrapingSupport; diff --git a/state-reconstruct-fetcher/blobscan-client/src/scraping_support.rs b/state-reconstruct-fetcher/blobscan-client/src/scraping_support.rs new file mode 100644 index 0000000..d272488 --- /dev/null +++ b/state-reconstruct-fetcher/blobscan-client/src/scraping_support.rs @@ -0,0 +1,52 @@ +use serde::Deserialize; +use serde_json::json; + +use crate::blob_support::{BlobError, BlobSupport}; + +#[derive(Deserialize)] +struct JsonResponse { + result: JsonResponseResult, +} + +#[derive(Deserialize)] +struct JsonResponseResult { + data: JsonResponseData, +} + +#[derive(Deserialize)] +struct JsonResponseData { + json: JsonResponseJson, +} + +#[derive(Deserialize)] +struct JsonResponseJson { + data: String, +} + +#[derive(Default)] +pub struct ScrapingSupport {} + +impl BlobSupport for ScrapingSupport { + fn format_url(&self, kzg_commitment: &[u8]) -> String { + let id = format!("0x{}", hex::encode(kzg_commitment)); + let json_source = json!({ + "json": { + "id": id + } + }); + let json_string = json_source.to_string(); + let url = reqwest::Url::parse_with_params( + "https://blobscan.com/api/trpc/blob.getByBlobIdFull", + &[("input", &json_string)], + ) + .unwrap(); + url.to_string() + } + + fn get_blob_data(&self, json_str: &str) -> Result { + match serde_json::from_str::(json_str) { + Ok(rsp) => Ok(rsp.result.data.json.data), + Err(e) => Err(BlobError::FormatError(json_str.to_string(), e.to_string())), + } + } +} diff --git a/state-reconstruct-fetcher/src/api_support.rs b/state-reconstruct-fetcher/src/api_support.rs new file mode 100644 index 0000000..5eedfda --- /dev/null +++ b/state-reconstruct-fetcher/src/api_support.rs @@ -0,0 +1,30 @@ +use blobscan_client::{BlobError, BlobSupport}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct JsonResponse { + data: String, +} + +pub struct ApiSupport { + url_base: String, +} + +impl ApiSupport { + pub fn new(blob_url: String) -> Self { + Self { url_base: blob_url } + } +} + +impl BlobSupport for ApiSupport { + fn format_url(&self, kzg_commitment: &[u8]) -> String { + format!("{}0x{}", self.url_base, hex::encode(kzg_commitment)) + } + + fn get_blob_data(&self, json_str: &str) -> Result { + match serde_json::from_str::(json_str) { + Ok(data) => Ok(data.data), + Err(e) => Err(BlobError::FormatError(json_str.to_string(), e.to_string())), + } + } +} diff --git a/state-reconstruct-fetcher/src/blob_http_client.rs b/state-reconstruct-fetcher/src/blob_http_client.rs index 8bbbeee..8671998 100644 --- a/state-reconstruct-fetcher/src/blob_http_client.rs +++ b/state-reconstruct-fetcher/src/blob_http_client.rs @@ -1,25 +1,18 @@ -use serde::Deserialize; +use blobscan_client::{BlobError, BlobSupport}; use tokio::time::{sleep, Duration}; -use crate::types::ParseError; - /// `MAX_RETRIES` is the maximum number of retries on failed blob retrieval. const MAX_RETRIES: u8 = 5; /// The interval in seconds to wait before retrying to fetch a blob. const FAILED_FETCH_RETRY_INTERVAL_S: u64 = 10; -#[derive(Deserialize)] -struct JsonResponse { - data: String, -} - pub struct BlobHttpClient { client: reqwest::Client, - url_base: String, + support: Box, } impl BlobHttpClient { - pub fn new(blob_url: String) -> eyre::Result { + pub fn new(support: Box) -> eyre::Result { let mut headers = reqwest::header::HeaderMap::new(); headers.insert( "Accept", @@ -28,18 +21,15 @@ impl BlobHttpClient { let client = reqwest::Client::builder() .default_headers(headers) .build()?; - Ok(Self { - client, - url_base: blob_url, - }) + Ok(Self { client, support }) } - pub async fn get_blob(&self, kzg_commitment: &[u8]) -> Result, ParseError> { - let url = self.format_url(kzg_commitment); + pub async fn get_blob(&self, kzg_commitment: &[u8]) -> Result, BlobError> { + let url = self.support.format_url(kzg_commitment); for attempt in 1..=MAX_RETRIES { - match self.retrieve_url(&url).await { + match self.client.get(&url).send().await { Ok(response) => match response.text().await { - Ok(text) => match get_blob_data(&text) { + Ok(text) => match self.support.get_blob_data(&text) { Ok(data) => { let plain = if let Some(p) = data.strip_prefix("0x") { p @@ -47,7 +37,7 @@ impl BlobHttpClient { &data }; return hex::decode(plain).map_err(|e| { - ParseError::BlobFormatError(plain.to_string(), e.to_string()) + BlobError::FormatError(plain.to_string(), e.to_string()) }); } Err(e) => { @@ -66,25 +56,6 @@ impl BlobHttpClient { } } } - Err(ParseError::BlobStorageError(url)) - } - - fn format_url(&self, kzg_commitment: &[u8]) -> String { - format!("{}0x{}", self.url_base, hex::encode(kzg_commitment)) - } - - async fn retrieve_url(&self, url: &str) -> eyre::Result { - let result = self.client.get(url).send().await?; - Ok(result) - } -} - -fn get_blob_data(json_str: &str) -> Result { - match serde_json::from_str::(json_str) { - Ok(data) => Ok(data.data), - Err(e) => Err(ParseError::BlobFormatError( - json_str.to_string(), - e.to_string(), - )), + Err(BlobError::StorageError(url)) } } diff --git a/state-reconstruct-fetcher/src/l1_fetcher.rs b/state-reconstruct-fetcher/src/l1_fetcher.rs index ec78dcb..a3c77bd 100644 --- a/state-reconstruct-fetcher/src/l1_fetcher.rs +++ b/state-reconstruct-fetcher/src/l1_fetcher.rs @@ -1,5 +1,6 @@ use std::{cmp, fs::File, future::Future, sync::Arc}; +use blobscan_client::{BlobSupport, ScrapingSupport}; use ethers::{ abi::{Contract, Function}, prelude::*, @@ -14,6 +15,7 @@ use tokio::{ use tokio_util::sync::CancellationToken; use crate::{ + api_support::ApiSupport, blob_http_client::BlobHttpClient, constants::ethereum::{BLOB_BLOCK, BLOCK_STEP, BOOJUM_BLOCK, GENESIS_BLOCK, ZK_SYNC_ADDR}, database::InnerDB, @@ -450,7 +452,7 @@ impl L1Fetcher { ) -> Result>> { let metrics = self.metrics.clone(); let contracts = self.contracts.clone(); - let client = BlobHttpClient::new(self.config.blobs_url.clone())?; + let client = BlobHttpClient::new(make_support(self.config.blobs_url.clone()))?; Ok(tokio::spawn({ async move { let mut boojum_mode = false; @@ -546,6 +548,14 @@ impl L1Fetcher { } } +fn make_support(url: String) -> Box { + if url.starts_with("https://blobscan.com") { + Box::::default() + } else { + Box::new(ApiSupport::new(url)) + } +} + pub async fn parse_calldata( l1_block_number: u64, commit_blocks_fn: &Function, diff --git a/state-reconstruct-fetcher/src/lib.rs b/state-reconstruct-fetcher/src/lib.rs index e116019..180f81a 100644 --- a/state-reconstruct-fetcher/src/lib.rs +++ b/state-reconstruct-fetcher/src/lib.rs @@ -1,5 +1,6 @@ #![feature(array_chunks)] #![feature(iter_next_chunk)] +pub mod api_support; pub mod blob_http_client; pub mod constants; pub mod database; diff --git a/state-reconstruct-fetcher/src/types/mod.rs b/state-reconstruct-fetcher/src/types/mod.rs index 1a77494..b0ff5a4 100644 --- a/state-reconstruct-fetcher/src/types/mod.rs +++ b/state-reconstruct-fetcher/src/types/mod.rs @@ -1,3 +1,4 @@ +use blobscan_client::BlobError; use ethers::{abi, types::U256}; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; @@ -42,6 +43,15 @@ pub enum ParseError { BlobFormatError(String, String), } +impl From for ParseError { + fn from(opt: BlobError) -> Self { + match opt { + BlobError::StorageError(code) => ParseError::BlobStorageError(code), + BlobError::FormatError(data, msg) => ParseError::BlobFormatError(data, msg), + } + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub enum PackingType { Add(U256),