using blobscan API or GUI, depending on blobs-url config param

Cargo.toml

@@ -5,12 +5,13 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [workspace]
-members = ["state-reconstruct-fetcher"]
+members = ["state-reconstruct-fetcher", "state-reconstruct-fetcher/blobscan-client"]
 
 [dependencies]
 async-trait = "0.1.74"
 bincode = "1"
 blake2 = "0.10.6"
+blobscan-client = { path = "./state-reconstruct-fetcher/blobscan-client" }
 bytes = "1.5"
 chrono = "0.4.31"
 clap = { version = "4.4.7", features = ["derive", "env"] }

state-reconstruct-fetcher/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 bincode = "1.3.3"
 blake2 = "0.10.6"
+blobscan-client = { path = "./blobscan-client" }
 ethers = "1.0.2"
 eyre = "0.6.8"
 indexmap = { version = "2.0.2", features = ["serde"] }

state-reconstruct-fetcher/blobscan-client/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "blobscan-client"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+eyre = "0.6.8"
+hex = "0.4.3"
+reqwest = "0.11.24"
+serde = { version = "1.0.189", features = ["derive"] }
+serde_json = { version = "1.0.107", features = ["std"] }
+thiserror = "1.0.50"
+tokio = { version = "1.33.0", features = ["signal"] }
+tracing = "0.1.40"

state-reconstruct-fetcher/blobscan-client/src/blob_support.rs

@@ -0,0 +1,17 @@
+use thiserror::Error;
+
+#[allow(clippy::enum_variant_names)]
+#[derive(Error, Debug)]
+pub enum BlobError {
+    #[error("blob storage error: {0}")]
+    StorageError(String),
+
+    #[error("blob format error")]
+    FormatError(String, String),
+}
+
+pub trait BlobSupport {
+    fn format_url(&self, kzg_commitment: &[u8]) -> String;
+
+    fn get_blob_data(&self, json_str: &str) -> Result<String, BlobError>;
+}

state-reconstruct-fetcher/blobscan-client/src/lib.rs

@@ -0,0 +1,4 @@
+mod blob_support;
+pub use crate::blob_support::{BlobError, BlobSupport};
+mod scraping_support;
+pub use crate::scraping_support::ScrapingSupport;

state-reconstruct-fetcher/blobscan-client/src/scraping_support.rs

@@ -0,0 +1,52 @@
+use serde::Deserialize;
+use serde_json::json;
+
+use crate::blob_support::{BlobError, BlobSupport};
+
+#[derive(Deserialize)]
+struct JsonResponse {
+    result: JsonResponseResult,
+}
+
+#[derive(Deserialize)]
+struct JsonResponseResult {
+    data: JsonResponseData,
+}
+
+#[derive(Deserialize)]
+struct JsonResponseData {
+    json: JsonResponseJson,
+}
+
+#[derive(Deserialize)]
+struct JsonResponseJson {
+    data: String,
+}
+
+#[derive(Default)]
+pub struct ScrapingSupport {}
+
+impl BlobSupport for ScrapingSupport {
+    fn format_url(&self, kzg_commitment: &[u8]) -> String {
+        let id = format!("0x{}", hex::encode(kzg_commitment));
+        let json_source = json!({
+            "json": {
+                "id": id
+            }
+        });
+        let json_string = json_source.to_string();
+        let url = reqwest::Url::parse_with_params(
+            "https://blobscan.com/api/trpc/blob.getByBlobIdFull",
+            &[("input", &json_string)],
+        )
+        .unwrap();
+        url.to_string()
+    }
+
+    fn get_blob_data(&self, json_str: &str) -> Result<String, BlobError> {
+        match serde_json::from_str::<JsonResponse>(json_str) {
+            Ok(rsp) => Ok(rsp.result.data.json.data),
+            Err(e) => Err(BlobError::FormatError(json_str.to_string(), e.to_string())),
+        }
+    }
+}

state-reconstruct-fetcher/src/api_support.rs

@@ -0,0 +1,30 @@
+use blobscan_client::{BlobError, BlobSupport};
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+struct JsonResponse {
+    data: String,
+}
+
+pub struct ApiSupport {
+    url_base: String,
+}
+
+impl ApiSupport {
+    pub fn new(blob_url: String) -> Self {
+        Self { url_base: blob_url }
+    }
+}
+
+impl BlobSupport for ApiSupport {
+    fn format_url(&self, kzg_commitment: &[u8]) -> String {
+        format!("{}0x{}", self.url_base, hex::encode(kzg_commitment))
+    }
+
+    fn get_blob_data(&self, json_str: &str) -> Result<String, BlobError> {
+        match serde_json::from_str::<JsonResponse>(json_str) {
+            Ok(data) => Ok(data.data),
+            Err(e) => Err(BlobError::FormatError(json_str.to_string(), e.to_string())),
+        }
+    }
+}

state-reconstruct-fetcher/src/blob_http_client.rs

@@ -1,25 +1,18 @@
-use serde::Deserialize;
+use blobscan_client::{BlobError, BlobSupport};
 use tokio::time::{sleep, Duration};
 
-use crate::types::ParseError;
-
 /// `MAX_RETRIES` is the maximum number of retries on failed blob retrieval.
 const MAX_RETRIES: u8 = 5;
 /// The interval in seconds to wait before retrying to fetch a blob.
 const FAILED_FETCH_RETRY_INTERVAL_S: u64 = 10;
 
-#[derive(Deserialize)]
-struct JsonResponse {
-    data: String,
-}
-
 pub struct BlobHttpClient {
     client: reqwest::Client,
-    url_base: String,
+    support: Box<dyn BlobSupport + Send + Sync>,
 }
 
 impl BlobHttpClient {
-    pub fn new(blob_url: String) -> eyre::Result<Self> {
+    pub fn new(support: Box<dyn BlobSupport + Send + Sync>) -> eyre::Result<Self> {
         let mut headers = reqwest::header::HeaderMap::new();
         headers.insert(
             "Accept",
@@ -28,26 +21,23 @@ impl BlobHttpClient {
         let client = reqwest::Client::builder()
             .default_headers(headers)
             .build()?;
-        Ok(Self {
-            client,
-            url_base: blob_url,
-        })
+        Ok(Self { client, support })
     }
 
-    pub async fn get_blob(&self, kzg_commitment: &[u8]) -> Result<Vec<u8>, ParseError> {
-        let url = self.format_url(kzg_commitment);
+    pub async fn get_blob(&self, kzg_commitment: &[u8]) -> Result<Vec<u8>, BlobError> {
+        let url = self.support.format_url(kzg_commitment);
         for attempt in 1..=MAX_RETRIES {
-            match self.retrieve_url(&url).await {
+            match self.client.get(&url).send().await {
                 Ok(response) => match response.text().await {
-                    Ok(text) => match get_blob_data(&text) {
+                    Ok(text) => match self.support.get_blob_data(&text) {
                         Ok(data) => {
                             let plain = if let Some(p) = data.strip_prefix("0x") {
                                 p
                             } else {
                                 &data
                             };
                             return hex::decode(plain).map_err(|e| {
-                                ParseError::BlobFormatError(plain.to_string(), e.to_string())
+                                BlobError::FormatError(plain.to_string(), e.to_string())
                             });
                         }
                         Err(e) => {
@@ -66,25 +56,6 @@ impl BlobHttpClient {
                 }
             }
         }
-        Err(ParseError::BlobStorageError(url))
-    }
-
-    fn format_url(&self, kzg_commitment: &[u8]) -> String {
-        format!("{}0x{}", self.url_base, hex::encode(kzg_commitment))
-    }
-
-    async fn retrieve_url(&self, url: &str) -> eyre::Result<reqwest::Response> {
-        let result = self.client.get(url).send().await?;
-        Ok(result)
-    }
-}
-
-fn get_blob_data(json_str: &str) -> Result<String, ParseError> {
-    match serde_json::from_str::<JsonResponse>(json_str) {
-        Ok(data) => Ok(data.data),
-        Err(e) => Err(ParseError::BlobFormatError(
-            json_str.to_string(),
-            e.to_string(),
-        )),
+        Err(BlobError::StorageError(url))
     }
 }

state-reconstruct-fetcher/src/l1_fetcher.rs

@@ -1,5 +1,6 @@
 use std::{cmp, fs::File, future::Future, sync::Arc};
 
+use blobscan_client::{BlobSupport, ScrapingSupport};
 use ethers::{
     abi::{Contract, Function},
     prelude::*,
@@ -14,6 +15,7 @@ use tokio::{
 use tokio_util::sync::CancellationToken;
 
 use crate::{
+    api_support::ApiSupport,
     blob_http_client::BlobHttpClient,
     constants::ethereum::{BLOB_BLOCK, BLOCK_STEP, BOOJUM_BLOCK, GENESIS_BLOCK, ZK_SYNC_ADDR},
     database::InnerDB,
@@ -450,7 +452,7 @@ impl L1Fetcher {
     ) -> Result<tokio::task::JoinHandle<Option<u64>>> {
         let metrics = self.metrics.clone();
         let contracts = self.contracts.clone();
-        let client = BlobHttpClient::new(self.config.blobs_url.clone())?;
+        let client = BlobHttpClient::new(make_support(self.config.blobs_url.clone()))?;
         Ok(tokio::spawn({
             async move {
                 let mut boojum_mode = false;
@@ -546,6 +548,14 @@ impl L1Fetcher {
     }
 }
 
+fn make_support(url: String) -> Box<dyn BlobSupport + Send + Sync> {
+    if url.starts_with("https://blobscan.com") {
+        Box::<ScrapingSupport>::default()
+    } else {
+        Box::new(ApiSupport::new(url))
+    }
+}
+
 pub async fn parse_calldata(
     l1_block_number: u64,
     commit_blocks_fn: &Function,

state-reconstruct-fetcher/src/lib.rs

@@ -1,5 +1,6 @@
 #![feature(array_chunks)]
 #![feature(iter_next_chunk)]
+pub mod api_support;
 pub mod blob_http_client;
 pub mod constants;
 pub mod database;

state-reconstruct-fetcher/src/types/mod.rs

@@ -1,3 +1,4 @@
+use blobscan_client::BlobError;
 use ethers::{abi, types::U256};
 use indexmap::IndexMap;
 use serde::{Deserialize, Serialize};
@@ -42,6 +43,15 @@ pub enum ParseError {
     BlobFormatError(String, String),
 }
 
+impl From<BlobError> for ParseError {
+    fn from(opt: BlobError) -> Self {
+        match opt {
+            BlobError::StorageError(code) => ParseError::BlobStorageError(code),
+            BlobError::FormatError(data, msg) => ParseError::BlobFormatError(data, msg),
+        }
+    }
+}
+
 #[derive(Debug, Clone, Copy, Serialize, Deserialize)]
 pub enum PackingType {
     Add(U256),