From 385c4c46bd678774a81c83251847cc48652a167d Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 21:42:06 +0000 Subject: [PATCH 01/13] can read eigenda v1 cert --- Cargo.lock | 2 + bin/client/justfile | 4 +- bin/host/Cargo.toml | 1 + bin/host/src/eigenda_fetcher/mod.rs | 9 ++++ crates/eigenda/Cargo.toml | 1 + crates/eigenda/src/certificate.rs | 76 +++++++++++++++++++++++++++++ crates/eigenda/src/lib.rs | 4 ++ 7 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 crates/eigenda/src/certificate.rs diff --git a/Cargo.lock b/Cargo.lock index 171dfe4..80614e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1723,6 +1723,7 @@ name = "hokulea-eigenda" version = "0.1.0" dependencies = [ "alloy-primitives", + "alloy-rlp", "async-trait", "kona-derive", "op-alloy-protocol", @@ -1740,6 +1741,7 @@ dependencies = [ "async-trait", "clap", "hokulea-client", + "hokulea-eigenda", "hokulea-proof", "kona-host", "kona-preimage", diff --git a/bin/client/justfile b/bin/client/justfile index 9ac832d..de993ed 100644 --- a/bin/client/justfile +++ b/bin/client/justfile @@ -69,7 +69,7 @@ run-client-native-against-devnet verbosity='' block_number='' rollup_config_path L1_BEACON_RPC="http://127.0.0.1:5052" L2_RPC="http://127.0.0.1:9545" ROLLUP_NODE_RPC="http://127.0.0.1:7545" - ROLLUP_CONFIG_PATH="../../../optimism/.devnet/rollup.json" + ROLLUP_CONFIG_PATH="/home/ubuntu/op-main-repo/.devnet/rollup.json" if [ -z "{{block_number}}" ]; then BLOCK_NUMBER=$(cast block finalized --json --rpc-url $L2_RPC | jq -r .number | cast 2d) @@ -120,7 +120,7 @@ run-client-native block_number l1_rpc l1_beacon_rpc l2_rpc rollup_node_rpc rollu cd $(git rev-parse --show-toplevel) echo "Running host program with native client program..." - cargo r --bin hokulea-host --release -- \ + cargo r --bin hokulea-host -- \ --l1-head $L1_HEAD \ --agreed-l2-head-hash $AGREED_L2_HEAD_HASH \ --claimed-l2-output-root $CLAIMED_L2_OUTPUT_ROOT \ diff --git a/bin/host/Cargo.toml b/bin/host/Cargo.toml index e286988..69820d0 100644 --- a/bin/host/Cargo.toml +++ b/bin/host/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" # Workspace hokulea-proof.workspace = true hokulea-client.workspace = true +hokulea-eigenda.workspace = true # Kona kona-preimage = { workspace = true, features = ["std"] } diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 643785d..976e78c 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -12,6 +12,8 @@ use kona_preimage::{PreimageKey, PreimageKeyType}; use std::sync::Arc; use tokio::sync::RwLock; use tracing::{error, info, trace, warn}; +use hokulea_eigenda::BlobInfo; +use alloy_rlp::{Decodable}; /// The [FetcherWithEigenDASupport] struct wraps and extends kona's [Fetcher] struct with the ability /// to fetch preimages from EigenDA. @@ -136,6 +138,13 @@ where trace!(target: "fetcher_with_eigenda_support", "Fetching hint: {hint_type} {hint_data}"); if hint_type == ExtendedHintType::EigenDACommitment { + + let item_slice= hint_data.as_ref(); + + // the fourth because 0x01010000 in the beginnin is metadata + let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..])?; + info!("cert_blob_info {:?}", cert_blob_info); + let cert = hint_data; info!(target: "fetcher_with_eigenda_support", "Fetching AltDACommitment cert: {:?}", cert); // Fetch the blob sidecar from the blob provider. diff --git a/crates/eigenda/Cargo.toml b/crates/eigenda/Cargo.toml index 258342d..baa1f5f 100644 --- a/crates/eigenda/Cargo.toml +++ b/crates/eigenda/Cargo.toml @@ -9,6 +9,7 @@ kona-derive.workspace = true # Op Alloy op-alloy-protocol.workspace = true alloy-primitives.workspace = true +alloy-rlp.workspace = true tracing.workspace = true async-trait.workspace = true diff --git a/crates/eigenda/src/certificate.rs b/crates/eigenda/src/certificate.rs new file mode 100644 index 0000000..1ecdcaa --- /dev/null +++ b/crates/eigenda/src/certificate.rs @@ -0,0 +1,76 @@ +// data struct copied from https://github.com/Layr-Labs/eigenda-client-rs/blob/3ac1f62ae3d99aedf3de7a2fe827fab17db7b874/src/blob_info.rs +use core::fmt; + +use alloy_primitives::Bytes; +use alloy_rlp::{RlpDecodable, RlpEncodable}; + +use alloc::vec::Vec; + + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct G1Commitment { + pub x: [u8; 32], + pub y: [u8; 32], +} + + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BlobQuorumParam { + pub quorum_number: u32, + pub adversary_threshold_percentage: u32, + pub confirmation_threshold_percentage: u32, + pub chunk_length: u32, +} + +impl BlobQuorumParam { + pub fn to_bytes(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.extend(&self.quorum_number.to_be_bytes()); + bytes.extend(&self.adversary_threshold_percentage.to_be_bytes()); + bytes.extend(&self.confirmation_threshold_percentage.to_be_bytes()); + bytes.extend(&self.chunk_length.to_be_bytes()); + + bytes + } +} + + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BlobHeader { + pub commitment: G1Commitment, + pub data_length: u32, + pub blob_quorum_params: Vec, +} + + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BatchHeader { + pub batch_root: Bytes, + pub quorum_numbers: Bytes, + pub quorum_signed_percentages: Bytes, + pub reference_block_number: u32, +} + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BatchMetadata { + pub batch_header: BatchHeader, + pub signatory_record_hash: Bytes, + pub fee: Bytes, + pub confirmation_block_number: u32, + pub batch_header_hash: Bytes, +} + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BlobVerificationProof { + pub batch_id: u32, + pub blob_index: u32, + pub batch_medatada: BatchMetadata, + pub inclusion_proof: Bytes, + pub quorum_indexes: Bytes, +} + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BlobInfo { + pub blob_header: BlobHeader, + pub blob_verification_proof: BlobVerificationProof, +} diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index b62f3f5..e457bbf 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -26,3 +26,7 @@ pub use eigenda_blobs::EigenDABlobSource; mod eigenda_data; pub use eigenda_data::EigenDABlobData; + +mod certificate; +pub use certificate::BlobInfo; + From cf99d402de59e33f9e52c5be4c4f9e052564903e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 21:45:35 +0000 Subject: [PATCH 02/13] small fix --- README.md | 19 ++++++++++++++++++- bin/client/justfile | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 32b65b7..01df249 100644 --- a/README.md +++ b/README.md @@ -14,4 +14,21 @@ Then run hokulea: ```bash cd bin/client just run-client-native-against-devnet -``` \ No newline at end of file +``` + +To use eigenda proxy within optimism devnet, modify ops-bedrock/docker-compose.yaml + +``` + da-server: + image: ghcr.io/layr-labs/eigenda-proxy:v1.6.1 + environment: + EIGENDA_PROXY_ADDR: 0.0.0.0 + EIGENDA_PROXY_PORT: 3100 + EIGENDA_PROXY_METRICS_ENABLED: true + EIGENDA_PROXY_METRICS_PORT: 7300 + EIGENDA_PROXY_MEMSTORE_ENABLED: true + EIGENDA_PROXY_MEMSTORE_EXPIRATION: 45m + EIGENDA_PROXY_MEMSTORE_PUT_LATENCY: 0s + EIGENDA_PROXY_MEMSTORE_GET_LATENCY: 0s + EIGENDA_PROXY_EIGENDA_CERT_VERIFICATION_DISABLED: true +``` diff --git a/bin/client/justfile b/bin/client/justfile index de993ed..fb05bd2 100644 --- a/bin/client/justfile +++ b/bin/client/justfile @@ -69,7 +69,7 @@ run-client-native-against-devnet verbosity='' block_number='' rollup_config_path L1_BEACON_RPC="http://127.0.0.1:5052" L2_RPC="http://127.0.0.1:9545" ROLLUP_NODE_RPC="http://127.0.0.1:7545" - ROLLUP_CONFIG_PATH="/home/ubuntu/op-main-repo/.devnet/rollup.json" + ROLLUP_CONFIG_PATH="../../../optimism/.devnet/rollup.json" if [ -z "{{block_number}}" ]; then BLOCK_NUMBER=$(cast block finalized --json --rpc-url $L2_RPC | jq -r .number | cast 2d) From 0ff56150ddc783be0368f40b8b9c87e62c8675f3 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 22:03:08 +0000 Subject: [PATCH 03/13] fix lint --- bin/host/src/eigenda_fetcher/mod.rs | 4 ++-- crates/eigenda/src/certificate.rs | 22 +++++----------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 976e78c..c3a735a 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -13,7 +13,7 @@ use std::sync::Arc; use tokio::sync::RwLock; use tracing::{error, info, trace, warn}; use hokulea_eigenda::BlobInfo; -use alloy_rlp::{Decodable}; +use alloy_rlp::Decodable; /// The [FetcherWithEigenDASupport] struct wraps and extends kona's [Fetcher] struct with the ability /// to fetch preimages from EigenDA. @@ -139,7 +139,7 @@ where if hint_type == ExtendedHintType::EigenDACommitment { - let item_slice= hint_data.as_ref(); + let item_slice = hint_data.as_ref(); // the fourth because 0x01010000 in the beginnin is metadata let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..])?; diff --git a/crates/eigenda/src/certificate.rs b/crates/eigenda/src/certificate.rs index 1ecdcaa..eedfd65 100644 --- a/crates/eigenda/src/certificate.rs +++ b/crates/eigenda/src/certificate.rs @@ -1,6 +1,3 @@ -// data struct copied from https://github.com/Layr-Labs/eigenda-client-rs/blob/3ac1f62ae3d99aedf3de7a2fe827fab17db7b874/src/blob_info.rs -use core::fmt; - use alloy_primitives::Bytes; use alloy_rlp::{RlpDecodable, RlpEncodable}; @@ -13,7 +10,6 @@ pub struct G1Commitment { pub y: [u8; 32], } - #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct BlobQuorumParam { pub quorum_number: u32, @@ -22,19 +18,7 @@ pub struct BlobQuorumParam { pub chunk_length: u32, } -impl BlobQuorumParam { - pub fn to_bytes(&self) -> Vec { - let mut bytes = Vec::new(); - bytes.extend(&self.quorum_number.to_be_bytes()); - bytes.extend(&self.adversary_threshold_percentage.to_be_bytes()); - bytes.extend(&self.confirmation_threshold_percentage.to_be_bytes()); - bytes.extend(&self.chunk_length.to_be_bytes()); - - bytes - } -} - - +/// eigenda v1 blob header #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct BlobHeader { pub commitment: G1Commitment, @@ -60,6 +44,7 @@ pub struct BatchMetadata { pub batch_header_hash: Bytes, } +/// eigenda v1 blob verification proof #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct BlobVerificationProof { pub batch_id: u32, @@ -69,8 +54,11 @@ pub struct BlobVerificationProof { pub quorum_indexes: Bytes, } +/// eigenda v1 certificate #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct BlobInfo { + /// v1 blob header pub blob_header: BlobHeader, + /// v1 blob verification proof with merkle tree pub blob_verification_proof: BlobVerificationProof, } From 60c9879e8135dd02941709ced1bed9aecd21b825 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 22:08:03 +0000 Subject: [PATCH 04/13] fix link for real --- bin/host/src/eigenda_fetcher/mod.rs | 7 +++---- crates/eigenda/src/certificate.rs | 2 -- crates/eigenda/src/lib.rs | 1 - 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index c3a735a..e5b36c7 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -4,16 +4,16 @@ use crate::eigenda_blobs::OnlineEigenDABlobProvider; use alloy_primitives::{keccak256, B256}; use alloy_provider::ReqwestProvider; +use alloy_rlp::Decodable; use anyhow::{anyhow, Result}; use core::panic; +use hokulea_eigenda::BlobInfo; use hokulea_proof::hint::{ExtendedHint, ExtendedHintType}; use kona_host::{blobs::OnlineBlobProvider, fetcher::Fetcher, kv::KeyValueStore}; use kona_preimage::{PreimageKey, PreimageKeyType}; use std::sync::Arc; use tokio::sync::RwLock; use tracing::{error, info, trace, warn}; -use hokulea_eigenda::BlobInfo; -use alloy_rlp::Decodable; /// The [FetcherWithEigenDASupport] struct wraps and extends kona's [Fetcher] struct with the ability /// to fetch preimages from EigenDA. @@ -138,13 +138,12 @@ where trace!(target: "fetcher_with_eigenda_support", "Fetching hint: {hint_type} {hint_data}"); if hint_type == ExtendedHintType::EigenDACommitment { - let item_slice = hint_data.as_ref(); // the fourth because 0x01010000 in the beginnin is metadata let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..])?; info!("cert_blob_info {:?}", cert_blob_info); - + let cert = hint_data; info!(target: "fetcher_with_eigenda_support", "Fetching AltDACommitment cert: {:?}", cert); // Fetch the blob sidecar from the blob provider. diff --git a/crates/eigenda/src/certificate.rs b/crates/eigenda/src/certificate.rs index eedfd65..afcb198 100644 --- a/crates/eigenda/src/certificate.rs +++ b/crates/eigenda/src/certificate.rs @@ -3,7 +3,6 @@ use alloy_rlp::{RlpDecodable, RlpEncodable}; use alloc::vec::Vec; - #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct G1Commitment { pub x: [u8; 32], @@ -26,7 +25,6 @@ pub struct BlobHeader { pub blob_quorum_params: Vec, } - #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct BatchHeader { pub batch_root: Bytes, diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index e457bbf..68e0d5d 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -29,4 +29,3 @@ pub use eigenda_data::EigenDABlobData; mod certificate; pub use certificate::BlobInfo; - From 968ebcb7be288211a18e75a2147adbe656508190 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 22:23:44 +0000 Subject: [PATCH 05/13] make it non eigenda-proxy compatible --- bin/host/src/eigenda_fetcher/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index e5b36c7..84671c1 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -141,8 +141,11 @@ where let item_slice = hint_data.as_ref(); // the fourth because 0x01010000 in the beginnin is metadata - let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..])?; - info!("cert_blob_info {:?}", cert_blob_info); + match BlobInfo::decode(&mut &item_slice[4..]) { + Ok( cert_blob_info) => info!("cert_blob_info {:?}", cert_blob_info), + Err(e) => info!("cannot decode cert_blob_info {:?}", e), + } + let cert = hint_data; info!(target: "fetcher_with_eigenda_support", "Fetching AltDACommitment cert: {:?}", cert); From a22354b58fa353c9e267d96b0d3c6c2d03db57ae Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 22:25:25 +0000 Subject: [PATCH 06/13] fix lint --- bin/host/src/eigenda_fetcher/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 84671c1..74b9e9c 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -142,10 +142,9 @@ where // the fourth because 0x01010000 in the beginnin is metadata match BlobInfo::decode(&mut &item_slice[4..]) { - Ok( cert_blob_info) => info!("cert_blob_info {:?}", cert_blob_info), + Ok(cert_blob_info) => info!("cert_blob_info {:?}", cert_blob_info), Err(e) => info!("cannot decode cert_blob_info {:?}", e), } - let cert = hint_data; info!(target: "fetcher_with_eigenda_support", "Fetching AltDACommitment cert: {:?}", cert); From 0455f02ff32f3ab0270b27e0f6fe62b98d00a38b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 02:09:48 +0000 Subject: [PATCH 07/13] wip --- bin/host/src/eigenda_fetcher/mod.rs | 47 +++++++++++++++++++++++++++- crates/proof/src/eigenda_provider.rs | 43 ++++++++++++++++++++----- 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 74b9e9c..f89da31 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -159,11 +159,56 @@ where // Acquire a lock on the key-value store and set the preimages. let mut kv_write_lock = self.kv_store.write().await; - // Set the preimage for the blob commitment. + // ToDo - remove it once cert is actually correct kv_write_lock.set( PreimageKey::new(*keccak256(cert), PreimageKeyType::GlobalGeneric).into(), eigenda_blob.to_vec(), )?; + + // fake a commitment + let t1 = cert.clone(); + let mut kzg_commitment = [0u8; 32]; + let mut a = 32; + if a > t1.len() { + a = t1.len() + } + kzg_commitment[..a].copy_from_slice(t1.as_ref()); + let blob_length = (eigenda_blob.len() + 32 - 1) / 32; // in term of field element + let kzg_proof = cert.clone(); + // end of fake + + // Write all the field elements to the key-value store. + // The preimage oracle key for each field element is the keccak256 hash of + // `abi.encodePacked(cert.KZGCommitment, uint256(i))` + let mut blob_key = [0u8; 80]; + blob_key[..32].copy_from_slice(kzg_commitment.as_ref()); + for i in 0..blob_length { + blob_key[72..].copy_from_slice(i.to_be_bytes().as_ref()); + let blob_key_hash = keccak256(blob_key.as_ref()); + + kv_write_lock.set( + PreimageKey::new(*blob_key_hash, PreimageKeyType::Keccak256).into(), + blob_key.into(), + )?; + kv_write_lock.set( + PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), + eigenda_blob[(i as usize) << 5..(i as usize + 1) << 5].to_vec(), + )?; + } + + // Write the KZG Proof as the last element. + blob_key[72..].copy_from_slice((blob_length).to_be_bytes().as_ref()); + let blob_key_hash = keccak256(blob_key.as_ref()); + + kv_write_lock.set( + PreimageKey::new(*blob_key_hash, PreimageKeyType::Keccak256).into(), + blob_key.into(), + )?; + kv_write_lock.set( + PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), + kzg_proof.to_vec(), + )?; + } else { panic!("Invalid hint type: {hint_type}. FetcherWithEigenDASupport.prefetch only supports EigenDACommitment hints."); } diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index 10e01e3..601693c 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -32,14 +32,43 @@ impl EigenDABlobProvider for OracleEigenDAProvider .write(&ExtendedHintType::EigenDACommitment.encode_with(&[cert])) .await .map_err(OracleProviderError::Preimage)?; + + // hack - remove later, when cert actually contain length let data = self - .oracle - .get(PreimageKey::new( - *keccak256(cert), - PreimageKeyType::GlobalGeneric, - )) - .await - .map_err(OracleProviderError::Preimage)?; + .oracle + .get(PreimageKey::new( + *keccak256(cert), + PreimageKeyType::GlobalGeneric, + )) + .await + .map_err(OracleProviderError::Preimage)?; + let blob_size = data.len(); + // + + let mut blob = vec![0, blob_size]; + let mut field_element_key = [0u8; 80]; + + + + field_element_key[..48].copy_from_slice(commitment.as_ref()); + for i in 0..FIELD_ELEMENTS_PER_BLOB { + field_element_key[72..].copy_from_slice(i.to_be_bytes().as_ref()); + + let mut field_element = [0u8; 32]; + self.oracle + .get_exact( + PreimageKey::new(*keccak256(field_element_key), PreimageKeyType::Blob), + &mut field_element, + ) + .await + .map_err(OracleProviderError::Preimage)?; + blob[(i as usize) << 5..(i as usize + 1) << 5].copy_from_slice(field_element.as_ref()); + } + + tracing::info!(target: "client_oracle", "Retrieved blob {blob_hash:?} from the oracle."); + + + Ok(data.into()) } From a5b38bc0fe4129c5e05bc71c98d762639640ff3e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 07:27:22 +0000 Subject: [PATCH 08/13] add partial decoding logic to be add to read raw blob --- Cargo.lock | 500 +++++++++++++++++++++++++-- Cargo.toml | 2 + bin/client/justfile | 2 +- bin/host/src/eigenda_blobs.rs | 2 +- bin/host/src/eigenda_fetcher/mod.rs | 44 ++- crates/eigenda/Cargo.toml | 3 + crates/eigenda/src/eigenda.rs | 43 ++- crates/eigenda/src/errors.rs | 7 + crates/eigenda/src/lib.rs | 3 + crates/proof/Cargo.toml | 4 + crates/proof/src/eigenda_provider.rs | 54 +-- 11 files changed, 584 insertions(+), 80 deletions(-) create mode 100644 crates/eigenda/src/errors.rs diff --git a/Cargo.lock b/Cargo.lock index 80614e6..189cdb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -167,7 +167,7 @@ dependencies = [ "alloy-sol-types", "serde", "serde_json", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -193,7 +193,7 @@ dependencies = [ "futures-utils-wasm", "serde", "serde_json", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -267,7 +267,7 @@ dependencies = [ "schnellru", "serde", "serde_json", - "thiserror", + "thiserror 2.0.7", "tokio", "tracing", "url", @@ -354,7 +354,7 @@ dependencies = [ "alloy-rpc-types-engine", "serde", "serde_with", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -425,7 +425,7 @@ dependencies = [ "auto_impl", "elliptic-curve", "k256", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -498,7 +498,7 @@ dependencies = [ "futures-utils-wasm", "serde", "serde_json", - "thiserror", + "thiserror 2.0.7", "tokio", "tower", "tracing", @@ -592,6 +592,39 @@ version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +[[package]] +name = "ark-bn254" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d69eab57e8d2663efa5c63135b2af4f396d66424f88954c21104125ab6b3e6bc" +dependencies = [ + "ark-ec", + "ark-ff 0.5.0", + "ark-std 0.5.0", +] + +[[package]] +name = "ark-ec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d68f2d516162846c1238e755a7c4d131b892b70cc70c471a8e3ca3ed818fce" +dependencies = [ + "ahash", + "ark-ff 0.5.0", + "ark-poly", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "educe", + "fnv", + "hashbrown 0.15.2", + "itertools 0.13.0", + "num-bigint", + "num-integer", + "num-traits", + "rayon", + "zeroize", +] + [[package]] name = "ark-ff" version = "0.3.0" @@ -630,6 +663,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ark-ff" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a177aba0ed1e0fbb62aa9f6d0502e9b46dad8c2eab04c14258a1212d2557ea70" +dependencies = [ + "ark-ff-asm 0.5.0", + "ark-ff-macros 0.5.0", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "arrayvec", + "digest 0.10.7", + "educe", + "itertools 0.13.0", + "num-bigint", + "num-traits", + "paste", + "rayon", + "zeroize", +] + [[package]] name = "ark-ff-asm" version = "0.3.0" @@ -650,6 +704,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ark-ff-asm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" +dependencies = [ + "quote", + "syn 2.0.90", +] + [[package]] name = "ark-ff-macros" version = "0.3.0" @@ -675,6 +739,35 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ark-ff-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09be120733ee33f7693ceaa202ca41accd5653b779563608f1234f78ae07c4b3" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "ark-poly" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579305839da207f02b89cd1679e50e67b4331e2f9294a57693e5051b7703fe27" +dependencies = [ + "ahash", + "ark-ff 0.5.0", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "educe", + "fnv", + "hashbrown 0.15.2", + "rayon", +] + [[package]] name = "ark-serialize" version = "0.3.0" @@ -696,6 +789,31 @@ dependencies = [ "num-bigint", ] +[[package]] +name = "ark-serialize" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7" +dependencies = [ + "ark-serialize-derive", + "ark-std 0.5.0", + "arrayvec", + "digest 0.10.7", + "num-bigint", + "rayon", +] + +[[package]] +name = "ark-serialize-derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "ark-std" version = "0.3.0" @@ -716,6 +834,17 @@ dependencies = [ "rand", ] +[[package]] +name = "ark-std" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" +dependencies = [ + "num-traits", + "rand", + "rayon", +] + [[package]] name = "arrayvec" version = "0.7.6" @@ -815,7 +944,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1130,6 +1259,43 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1285,6 +1451,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "directories" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1322,6 +1509,18 @@ dependencies = [ "spki", ] +[[package]] +name = "educe" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "either" version = "1.13.0" @@ -1356,6 +1555,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-ordinalize" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea0dcfa4e54eeb516fe454635a95753ddd39acda650ce703031c6973e315dd5" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "enumn" version = "0.1.14" @@ -1443,6 +1662,16 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "flate2" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1725,8 +1954,11 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "async-trait", + "bytes", "kona-derive", "op-alloy-protocol", + "rust-kzg-bn254", + "thiserror 2.0.7", "tracing", ] @@ -1756,6 +1988,7 @@ name = "hokulea-proof" version = "0.1.0" dependencies = [ "alloy-primitives", + "alloy-rlp", "async-trait", "hokulea-eigenda", "kona-derive", @@ -1765,6 +1998,7 @@ dependencies = [ "op-alloy-genesis", "op-alloy-protocol", "op-alloy-rpc-types-engine", + "tracing", ] [[package]] @@ -2180,7 +2414,7 @@ dependencies = [ "serde", "serde_json", "spin", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -2199,7 +2433,7 @@ dependencies = [ "op-alloy-genesis", "op-alloy-protocol", "op-alloy-rpc-types-engine", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -2217,7 +2451,7 @@ dependencies = [ "op-alloy-genesis", "op-alloy-protocol", "op-alloy-rpc-types-engine", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -2235,7 +2469,7 @@ dependencies = [ "op-alloy-genesis", "op-alloy-rpc-types-engine", "revm", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -2284,7 +2518,7 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "alloy-trie", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -2295,7 +2529,7 @@ dependencies = [ "alloy-primitives", "async-channel", "async-trait", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -2323,7 +2557,7 @@ dependencies = [ "serde", "serde_json", "spin", - "thiserror", + "thiserror 2.0.7", "tokio", "tracing", ] @@ -2337,7 +2571,7 @@ dependencies = [ "cfg-if", "kona-preimage", "linked_list_allocator", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -2381,7 +2615,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags", + "libc", ] [[package]] @@ -2678,7 +2922,7 @@ dependencies = [ "alloy-serde", "derive_more", "serde", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -2693,7 +2937,7 @@ dependencies = [ "alloy-sol-types", "serde", "serde_repr", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -2715,7 +2959,7 @@ dependencies = [ "op-alloy-consensus", "op-alloy-genesis", "serde", - "thiserror", + "thiserror 2.0.7", "tracing", "unsigned-varint", ] @@ -2748,7 +2992,7 @@ dependencies = [ "op-alloy-genesis", "op-alloy-protocol", "serde", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -2795,6 +3039,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "overload" version = "0.1.1" @@ -2865,7 +3115,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -2887,7 +3137,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror", + "thiserror 2.0.7", "ucd-trie", ] @@ -3095,6 +3345,26 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.8" @@ -3104,6 +3374,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "regex" version = "1.11.1" @@ -3325,6 +3606,32 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" +[[package]] +name = "rust-kzg-bn254" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdae4058a9f604acf7023d99d931d6f30261fff93787bcfd1f1ccfc725b701c" +dependencies = [ + "ark-bn254", + "ark-ec", + "ark-ff 0.5.0", + "ark-poly", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "byteorder", + "crossbeam-channel", + "directories", + "hex-literal", + "num-bigint", + "num-traits", + "num_cpus", + "rand", + "rayon", + "sha2", + "sys-info", + "ureq", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -3386,7 +3693,9 @@ version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3867,6 +4176,16 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "sys-info" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -3907,13 +4226,33 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.7", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] @@ -4218,6 +4557,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots", +] + [[package]] name = "url" version = "2.5.4" @@ -4380,6 +4735,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -4410,7 +4774,7 @@ checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ "windows-result", "windows-strings", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -4419,7 +4783,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -4429,7 +4793,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ "windows-result", - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", ] [[package]] @@ -4438,7 +4811,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -4447,7 +4820,22 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -4456,28 +4844,46 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -4490,24 +4896,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index 130a138..f793a27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,12 +62,14 @@ cfg-if = "1.0.0" reqwest = "0.12.9" async-trait = "0.1.83" linked_list_allocator = "0.10.5" +bytes = "1.9.0" # General sha2 = { version = "0.10.8", default-features = false } c-kzg = { version = "2.0.0", default-features = false } anyhow = { version = "1.0.93", default-features = false } thiserror = { version = "2.0.4", default-features = false } +rust-kzg-bn254 = { version = "0.2.1", default-features = false } # Tracing tracing-loki = "0.2.5" diff --git a/bin/client/justfile b/bin/client/justfile index fb05bd2..2773ded 100644 --- a/bin/client/justfile +++ b/bin/client/justfile @@ -69,7 +69,7 @@ run-client-native-against-devnet verbosity='' block_number='' rollup_config_path L1_BEACON_RPC="http://127.0.0.1:5052" L2_RPC="http://127.0.0.1:9545" ROLLUP_NODE_RPC="http://127.0.0.1:7545" - ROLLUP_CONFIG_PATH="../../../optimism/.devnet/rollup.json" + ROLLUP_CONFIG_PATH="~/op-main-repo/.devnet/rollup.json" if [ -z "{{block_number}}" ]; then BLOCK_NUMBER=$(cast block finalized --json --rpc-url $L2_RPC | jq -r .number | cast 2d) diff --git a/bin/host/src/eigenda_blobs.rs b/bin/host/src/eigenda_blobs.rs index f9c675b..36ce0f4 100644 --- a/bin/host/src/eigenda_blobs.rs +++ b/bin/host/src/eigenda_blobs.rs @@ -30,7 +30,7 @@ impl OnlineEigenDABlobProvider { ) -> Result { let url = format!("{}/{}/{}", self.base, GET_METHOD, cert.slice(1..)); - let raw_response = self.inner.get(url).send().await?; + let raw_response = self.inner.get(url).header("raw", "true").send().await?; raw_response.bytes().await } diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index f89da31..5f303ad 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -161,29 +161,35 @@ where // ToDo - remove it once cert is actually correct kv_write_lock.set( - PreimageKey::new(*keccak256(cert), PreimageKeyType::GlobalGeneric).into(), + PreimageKey::new(*keccak256(cert.clone()), PreimageKeyType::GlobalGeneric).into(), eigenda_blob.to_vec(), )?; - // fake a commitment - let t1 = cert.clone(); - let mut kzg_commitment = [0u8; 32]; - let mut a = 32; - if a > t1.len() { - a = t1.len() - } - kzg_commitment[..a].copy_from_slice(t1.as_ref()); - let blob_length = (eigenda_blob.len() + 32 - 1) / 32; // in term of field element - let kzg_proof = cert.clone(); - // end of fake + // data + let item_slice = cert.as_ref(); + let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); + info!("cert_blob_info {:?}", cert_blob_info); // Write all the field elements to the key-value store. // The preimage oracle key for each field element is the keccak256 hash of // `abi.encodePacked(cert.KZGCommitment, uint256(i))` - let mut blob_key = [0u8; 80]; - blob_key[..32].copy_from_slice(kzg_commitment.as_ref()); + + let mut blob_key = [0u8; 96]; + blob_key[..32].copy_from_slice(cert_blob_info.blob_header.commitment.x.as_ref()); + blob_key[32..64].copy_from_slice(cert_blob_info.blob_header.commitment.y.as_ref()); + + // Todo ensure data_length is always power of 2. Proxy made mistake + let data_size = cert_blob_info.blob_header.data_length as u64; + let blob_length: u64 = data_size / 32; + + // proxy could just return the original blob + let mut padded_eigenda_blob = vec![0u8; data_size as usize]; + padded_eigenda_blob[..eigenda_blob.len()].copy_from_slice(eigenda_blob.as_ref()); + + info!("cert_blob_info blob_length {:?}", blob_length); + for i in 0..blob_length { - blob_key[72..].copy_from_slice(i.to_be_bytes().as_ref()); + blob_key[88..].copy_from_slice(i.to_be_bytes().as_ref()); let blob_key_hash = keccak256(blob_key.as_ref()); kv_write_lock.set( @@ -192,22 +198,24 @@ where )?; kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), - eigenda_blob[(i as usize) << 5..(i as usize + 1) << 5].to_vec(), + padded_eigenda_blob[(i as usize) << 5..(i as usize + 1) << 5].to_vec(), )?; } // Write the KZG Proof as the last element. - blob_key[72..].copy_from_slice((blob_length).to_be_bytes().as_ref()); + blob_key[88..].copy_from_slice((blob_length).to_be_bytes().as_ref()); let blob_key_hash = keccak256(blob_key.as_ref()); kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::Keccak256).into(), blob_key.into(), )?; + // proof to be done kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), - kzg_proof.to_vec(), + [1,2,3].to_vec(), )?; + } else { panic!("Invalid hint type: {hint_type}. FetcherWithEigenDASupport.prefetch only supports EigenDACommitment hints."); diff --git a/crates/eigenda/Cargo.toml b/crates/eigenda/Cargo.toml index baa1f5f..c6ba39e 100644 --- a/crates/eigenda/Cargo.toml +++ b/crates/eigenda/Cargo.toml @@ -12,6 +12,9 @@ alloy-primitives.workspace = true alloy-rlp.workspace = true tracing.workspace = true async-trait.workspace = true +thiserror.workspace = true +bytes.workspace = true +rust-kzg-bn254.workspace = true [features] default = [] diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index 69e5d32..df23eeb 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -1,17 +1,22 @@ //! Contains the [EigenDADataSource], which is a concrete implementation of the //! [DataAvailabilityProvider] trait for the EigenDA protocol. + use crate::eigenda_blobs::EigenDABlobSource; use crate::traits::EigenDABlobProvider; +use crate::errors::CodecError; use alloc::{boxed::Box, fmt::Debug}; use alloy_primitives::Bytes; +use bytes::buf::Buf; use async_trait::async_trait; use kona_derive::{ sources::EthereumDataSource, traits::{BlobProvider, ChainProvider, DataAvailabilityProvider}, types::PipelineResult, + errors::{PipelineErrorKind, PipelineError, PipelineEncodingError}, }; use op_alloy_protocol::BlockInfo; +use rust_kzg_bn254::helpers::remove_empty_byte_from_padded_bytes_unchecked; /// A factory for creating an Ethereum data source provider. #[derive(Debug, Clone)] @@ -61,9 +66,13 @@ where // just dump all the data out info!(target: "eth-datasource", "next item {:?}", item); - let eigenda_source_result = self.eigenda_source.next(&item).await; - info!(target: "eigenda-datasource", "eigenda_source_result {:?}", eigenda_source_result); - eigenda_source_result + let padded_eigenda_blob = self.eigenda_source.next(&item).await?; + info!(target: "eigenda-datasource", "eigenda_source_result {:?}", padded_eigenda_blob); + + // get the actual blob as encoded inside blob + let eigenda_blob = self.default_decode_blob(padded_eigenda_blob)?; + + Ok(eigenda_blob) } fn clear(&mut self) { @@ -71,3 +80,31 @@ where self.ethereum_source.clear(); } } + +impl EigenDADataSource +where + C: ChainProvider + Send + Sync + Clone + Debug, + B: BlobProvider + Send + Sync + Clone + Debug, + A: EigenDABlobProvider + Send + Sync + Clone + Debug, +{ + // https://github.com/Layr-Labs/eigenda/blob/1345e77c8a91fed8e5e33f02c3e32c9ed9921670/api/clients/codecs/default_blob_codec.go#L44 + fn default_decode_blob(&self, padded_eigenda_blob: Bytes) -> PipelineResult { + if padded_eigenda_blob.len() < 32 { + // ToDo format error better + //return Err(PipelineErrorKind::Temporary(PipelineError::BadEncoding(PipelineEncodingError::SpanBatchError(())))); + unimplemented!() + } + + info!(target: "eigenda-datasource", "padded_eigenda_blob {:?}", padded_eigenda_blob); + + let content_size = padded_eigenda_blob.slice(2..6).get_u32(); + info!(target: "eigenda-datasource", "content_size {:?}", content_size); + let codec_data = padded_eigenda_blob.slice(32..); + + let blob_content = remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); + let blob_content: Bytes = blob_content.into(); + + Ok(blob_content.slice(..content_size as usize)) + } + +} \ No newline at end of file diff --git a/crates/eigenda/src/errors.rs b/crates/eigenda/src/errors.rs new file mode 100644 index 0000000..c9c2867 --- /dev/null +++ b/crates/eigenda/src/errors.rs @@ -0,0 +1,7 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum CodecError { + #[error("blob does not contain 32 header bytes, meaning it is malformed")] + BlobTooShort, +} diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index 68e0d5d..231a598 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -29,3 +29,6 @@ pub use eigenda_data::EigenDABlobData; mod certificate; pub use certificate::BlobInfo; + +mod errors; +pub use errors::CodecError; \ No newline at end of file diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml index b03c361..4d6fc6b 100644 --- a/crates/proof/Cargo.toml +++ b/crates/proof/Cargo.toml @@ -19,6 +19,10 @@ alloy-primitives.workspace = true op-alloy-protocol.workspace = true op-alloy-rpc-types-engine.workspace = true op-alloy-genesis = { workspace = true, features = ["serde"] } +alloy-rlp.workspace = true + + +tracing.workspace = true # General async-trait.workspace = true diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index 601693c..f335216 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -2,12 +2,14 @@ use alloc::boxed::Box; use alloc::sync::Arc; use alloy_primitives::{keccak256, Bytes}; use async_trait::async_trait; -use hokulea_eigenda::EigenDABlobProvider; +use hokulea_eigenda::{EigenDABlobProvider, BlobInfo}; use kona_preimage::{CommsClient, PreimageKey, PreimageKeyType}; use kona_proof::errors::OracleProviderError; use crate::hint::ExtendedHintType; +use alloy_rlp::Decodable; +use tracing::info; /// The oracle-backed EigenDA provider for the client program. #[derive(Debug, Clone)] @@ -33,43 +35,51 @@ impl EigenDABlobProvider for OracleEigenDAProvider .await .map_err(OracleProviderError::Preimage)?; + // the fourth because 0x01010000 in the beginnin is metadata + let item_slice = cert.as_ref(); + let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); + info!("cert_blob_info {:?}", cert_blob_info); + // hack - remove later, when cert actually contain length let data = self - .oracle - .get(PreimageKey::new( - *keccak256(cert), - PreimageKeyType::GlobalGeneric, - )) - .await - .map_err(OracleProviderError::Preimage)?; - let blob_size = data.len(); - // - - let mut blob = vec![0, blob_size]; - let mut field_element_key = [0u8; 80]; + .oracle + .get(PreimageKey::new( + *keccak256(cert), + PreimageKeyType::GlobalGeneric, + )) + .await + .map_err(OracleProviderError::Preimage)?; + + let mut blob: Vec = vec![0; cert_blob_info.blob_header.data_length as usize]; + // 96 because our g1 commitment has 64 bytes in v1 + let mut field_element_key = [0u8; 96]; - field_element_key[..48].copy_from_slice(commitment.as_ref()); - for i in 0..FIELD_ELEMENTS_PER_BLOB { - field_element_key[72..].copy_from_slice(i.to_be_bytes().as_ref()); + // ToDo data_length should be power of 2, proxy should have returned it with dividing 32 + let data_length = cert_blob_info.blob_header.data_length as u64 / 32; + + info!("cert_blob_info.blob_header.data_length {:?}", data_length); + + field_element_key[..32].copy_from_slice(&cert_blob_info.blob_header.commitment.x); + field_element_key[32..64].copy_from_slice(&cert_blob_info.blob_header.commitment.y); + for i in 0..data_length { + field_element_key[88..].copy_from_slice(i.to_be_bytes().as_ref()); let mut field_element = [0u8; 32]; self.oracle .get_exact( - PreimageKey::new(*keccak256(field_element_key), PreimageKeyType::Blob), + PreimageKey::new(*keccak256(field_element_key), PreimageKeyType::GlobalGeneric), &mut field_element, ) .await .map_err(OracleProviderError::Preimage)?; blob[(i as usize) << 5..(i as usize + 1) << 5].copy_from_slice(field_element.as_ref()); } + + info!("cert_blob_info blob {:?}", blob); - tracing::info!(target: "client_oracle", "Retrieved blob {blob_hash:?} from the oracle."); - - - - Ok(data.into()) + Ok(blob.into()) } async fn get_element(&mut self, cert: &Bytes, element: &Bytes) -> Result { From 86182b6725af0e007e08442a6577817ebfaa7373 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 17:41:07 +0000 Subject: [PATCH 09/13] move decode logic --- .gitignore | 1 + bin/host/src/eigenda_fetcher/mod.rs | 8 ++---- crates/eigenda/src/eigenda.rs | 41 ++-------------------------- crates/eigenda/src/eigenda_data.rs | 28 +++++++++++++++++-- crates/eigenda/src/lib.rs | 2 +- crates/proof/src/eigenda_provider.rs | 14 ++++++---- 6 files changed, 41 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index 5d1f033..b9dd3f1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ target/ data/ +optimism/ diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 5f303ad..736f070 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -178,14 +178,14 @@ where blob_key[..32].copy_from_slice(cert_blob_info.blob_header.commitment.x.as_ref()); blob_key[32..64].copy_from_slice(cert_blob_info.blob_header.commitment.y.as_ref()); - // Todo ensure data_length is always power of 2. Proxy made mistake + // Todo ensure data_length is always power of 2. Proxy made mistake let data_size = cert_blob_info.blob_header.data_length as u64; let blob_length: u64 = data_size / 32; // proxy could just return the original blob let mut padded_eigenda_blob = vec![0u8; data_size as usize]; padded_eigenda_blob[..eigenda_blob.len()].copy_from_slice(eigenda_blob.as_ref()); - + info!("cert_blob_info blob_length {:?}", blob_length); for i in 0..blob_length { @@ -213,10 +213,8 @@ where // proof to be done kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), - [1,2,3].to_vec(), + [1, 2, 3].to_vec(), )?; - - } else { panic!("Invalid hint type: {hint_type}. FetcherWithEigenDASupport.prefetch only supports EigenDACommitment hints."); } diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index df23eeb..9653c36 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -2,21 +2,19 @@ //! [DataAvailabilityProvider] trait for the EigenDA protocol. use crate::eigenda_blobs::EigenDABlobSource; -use crate::traits::EigenDABlobProvider; use crate::errors::CodecError; +use crate::traits::EigenDABlobProvider; use alloc::{boxed::Box, fmt::Debug}; use alloy_primitives::Bytes; -use bytes::buf::Buf; use async_trait::async_trait; use kona_derive::{ + errors::{PipelineEncodingError, PipelineError, PipelineErrorKind}, sources::EthereumDataSource, traits::{BlobProvider, ChainProvider, DataAvailabilityProvider}, types::PipelineResult, - errors::{PipelineErrorKind, PipelineError, PipelineEncodingError}, }; use op_alloy_protocol::BlockInfo; -use rust_kzg_bn254::helpers::remove_empty_byte_from_padded_bytes_unchecked; /// A factory for creating an Ethereum data source provider. #[derive(Debug, Clone)] @@ -66,12 +64,7 @@ where // just dump all the data out info!(target: "eth-datasource", "next item {:?}", item); - let padded_eigenda_blob = self.eigenda_source.next(&item).await?; - info!(target: "eigenda-datasource", "eigenda_source_result {:?}", padded_eigenda_blob); - - // get the actual blob as encoded inside blob - let eigenda_blob = self.default_decode_blob(padded_eigenda_blob)?; - + let eigenda_blob = self.eigenda_source.next(&item).await?; Ok(eigenda_blob) } @@ -80,31 +73,3 @@ where self.ethereum_source.clear(); } } - -impl EigenDADataSource -where - C: ChainProvider + Send + Sync + Clone + Debug, - B: BlobProvider + Send + Sync + Clone + Debug, - A: EigenDABlobProvider + Send + Sync + Clone + Debug, -{ - // https://github.com/Layr-Labs/eigenda/blob/1345e77c8a91fed8e5e33f02c3e32c9ed9921670/api/clients/codecs/default_blob_codec.go#L44 - fn default_decode_blob(&self, padded_eigenda_blob: Bytes) -> PipelineResult { - if padded_eigenda_blob.len() < 32 { - // ToDo format error better - //return Err(PipelineErrorKind::Temporary(PipelineError::BadEncoding(PipelineEncodingError::SpanBatchError(())))); - unimplemented!() - } - - info!(target: "eigenda-datasource", "padded_eigenda_blob {:?}", padded_eigenda_blob); - - let content_size = padded_eigenda_blob.slice(2..6).get_u32(); - info!(target: "eigenda-datasource", "content_size {:?}", content_size); - let codec_data = padded_eigenda_blob.slice(32..); - - let blob_content = remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); - let blob_content: Bytes = blob_content.into(); - - Ok(blob_content.slice(..content_size as usize)) - } - -} \ No newline at end of file diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 7a60e9d..80f895e 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -1,7 +1,10 @@ use alloy_primitives::Bytes; +use bytes::buf::Buf; use kona_derive::errors::BlobDecodingError; +use rust_kzg_bn254::helpers; + #[derive(Default, Clone, Debug)] /// Represents the data structure for EigenDA Blob. pub struct EigenDABlobData { @@ -13,9 +16,28 @@ impl EigenDABlobData { /// Decodes the blob into raw byte data. /// Returns a [BlobDecodingError] if the blob is invalid. pub(crate) fn decode(&self) -> Result { - // where we can implement zero bytes etc. - info!(target: "eigenda-blobdata", "decode {} {:?}", self.blob.len(), self.blob.clone()); - Ok(self.blob.clone()) + if self.blob.len() < 32 { + // ToDo format error better + //return Err(PipelineErrorKind::Temporary(PipelineError::BadEncoding(PipelineEncodingError::SpanBatchError(())))); + unimplemented!() + } + + info!(target: "eigenda-datasource", "padded_eigenda_blob {:?}", self.blob); + + // see https://github.com/Layr-Labs/eigenda/blob/f8b0d31d65b29e60172507074922668f4ca89420/api/clients/codecs/default_blob_codec.go#L44 + let content_size = self.blob.slice(2..6).get_u32(); + info!(target: "eigenda-datasource", "content_size {:?}", content_size); + + // the first 32 Bytes are reserved as the header field element + let codec_data = self.blob.slice(32..); + + // rust kzg bn254 impl already + let blob_content = + helpers::remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); + let blob_content: Bytes = blob_content.into(); + + // take data + Ok(blob_content.slice(..content_size as usize)) } } diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index 231a598..bfc282a 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -31,4 +31,4 @@ mod certificate; pub use certificate::BlobInfo; mod errors; -pub use errors::CodecError; \ No newline at end of file +pub use errors::CodecError; diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index f335216..704aa88 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -2,7 +2,7 @@ use alloc::boxed::Box; use alloc::sync::Arc; use alloy_primitives::{keccak256, Bytes}; use async_trait::async_trait; -use hokulea_eigenda::{EigenDABlobProvider, BlobInfo}; +use hokulea_eigenda::{BlobInfo, EigenDABlobProvider}; use kona_preimage::{CommsClient, PreimageKey, PreimageKeyType}; use kona_proof::errors::OracleProviderError; @@ -50,9 +50,8 @@ impl EigenDABlobProvider for OracleEigenDAProvider .await .map_err(OracleProviderError::Preimage)?; - let mut blob: Vec = vec![0; cert_blob_info.blob_header.data_length as usize]; - + // 96 because our g1 commitment has 64 bytes in v1 let mut field_element_key = [0u8; 96]; @@ -60,7 +59,7 @@ impl EigenDABlobProvider for OracleEigenDAProvider let data_length = cert_blob_info.blob_header.data_length as u64 / 32; info!("cert_blob_info.blob_header.data_length {:?}", data_length); - + field_element_key[..32].copy_from_slice(&cert_blob_info.blob_header.commitment.x); field_element_key[32..64].copy_from_slice(&cert_blob_info.blob_header.commitment.y); for i in 0..data_length { @@ -69,14 +68,17 @@ impl EigenDABlobProvider for OracleEigenDAProvider let mut field_element = [0u8; 32]; self.oracle .get_exact( - PreimageKey::new(*keccak256(field_element_key), PreimageKeyType::GlobalGeneric), + PreimageKey::new( + *keccak256(field_element_key), + PreimageKeyType::GlobalGeneric, + ), &mut field_element, ) .await .map_err(OracleProviderError::Preimage)?; blob[(i as usize) << 5..(i as usize + 1) << 5].copy_from_slice(field_element.as_ref()); } - + info!("cert_blob_info blob {:?}", blob); Ok(blob.into()) From fc05129e676fec39e123fd2a3d974552f075760e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 17:42:20 +0000 Subject: [PATCH 10/13] remove debug log --- crates/eigenda/src/eigenda_blobs.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/crates/eigenda/src/eigenda_blobs.rs b/crates/eigenda/src/eigenda_blobs.rs index 1fb3957..d2994a6 100644 --- a/crates/eigenda/src/eigenda_blobs.rs +++ b/crates/eigenda/src/eigenda_blobs.rs @@ -39,21 +39,15 @@ where /// Fetches the next blob from the source. pub async fn next(&mut self, altda_commitment: &Bytes) -> PipelineResult { - info!(target: "eigenda-blobsource", "next"); self.load_blobs(altda_commitment).await?; - info!(target: "eigenda-blobsource", "next 1"); let next_data = match self.next_data() { Ok(d) => d, Err(e) => return e, }; - info!(target: "eigenda-blobsource", "next 2"); // Decode the blob data to raw bytes. // Otherwise, ignore blob and recurse next. match next_data.decode() { - Ok(d) => { - info!(target: "eigenda-blobsource", "next 3"); - Ok(d) - } + Ok(d) => Ok(d), Err(_) => { warn!(target: "blob-source", "Failed to decode blob data, skipping"); panic!() From 78b9b584fd915a8d1a86726e3ad536f29f9abbb8 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 17:45:30 +0000 Subject: [PATCH 11/13] change altda to eigenda --- bin/host/src/eigenda_fetcher/mod.rs | 4 ++-- crates/eigenda/src/eigenda_blobs.rs | 19 ++++++++----------- crates/proof/src/hint.rs | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 736f070..6c02b31 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -96,7 +96,7 @@ where /// Fetch the preimage for the given key. The requested is routed to the appropriate fetcher /// based on the last hint that was received (see hint() above). - /// FetcherWithEigenDASupport -> get_preimage_altda -> prefetch that only understands altda hints + /// FetcherWithEigenDASupport -> get_preimage_eigenda -> prefetch that only understands eigenda hints /// \-> Fetcher -> get_preimage -> prefetch that understands all other hints pub async fn get_preimage(&self, key: B256) -> Result> { match self.last_eigenda_hint.as_ref() { @@ -147,7 +147,7 @@ where } let cert = hint_data; - info!(target: "fetcher_with_eigenda_support", "Fetching AltDACommitment cert: {:?}", cert); + info!(target: "fetcher_with_eigenda_support", "Fetching eigenda commitment cert: {:?}", cert); // Fetch the blob sidecar from the blob provider. let eigenda_blob = self .eigenda_blob_provider diff --git a/crates/eigenda/src/eigenda_blobs.rs b/crates/eigenda/src/eigenda_blobs.rs index d2994a6..741176c 100644 --- a/crates/eigenda/src/eigenda_blobs.rs +++ b/crates/eigenda/src/eigenda_blobs.rs @@ -17,7 +17,7 @@ where B: EigenDABlobProvider + Send, { /// Fetches blobs. - pub altda_fetcher: B, + pub eigenda_fetcher: B, /// EigenDA blobs. pub data: Vec, /// Whether the source is open. @@ -29,17 +29,17 @@ where B: EigenDABlobProvider + Send, { /// Creates a new blob source. - pub const fn new(altda_fetcher: B) -> Self { + pub const fn new(eigenda_fetcher: B) -> Self { Self { - altda_fetcher, + eigenda_fetcher, data: Vec::new(), open: false, } } /// Fetches the next blob from the source. - pub async fn next(&mut self, altda_commitment: &Bytes) -> PipelineResult { - self.load_blobs(altda_commitment).await?; + pub async fn next(&mut self, eigenda_commitment: &Bytes) -> PipelineResult { + self.load_blobs(eigenda_commitment).await?; let next_data = match self.next_data() { Ok(d) => d, Err(e) => return e, @@ -51,8 +51,6 @@ where Err(_) => { warn!(target: "blob-source", "Failed to decode blob data, skipping"); panic!() - // todo need to add recursion - // self.next(altda_commitment).await } } } @@ -64,14 +62,13 @@ where } /// Loads blob data into the source if it is not open. - async fn load_blobs(&mut self, altda_commitment: &Bytes) -> Result<(), BlobProviderError> { + async fn load_blobs(&mut self, eigenda_commitment: &Bytes) -> Result<(), BlobProviderError> { if self.open { return Ok(()); } - info!(target: "eigenda-blobsource", "going to fetch through altda fetcher"); - // it should use self.altda_fetcher to get the blob - let data = self.altda_fetcher.get_blob(altda_commitment).await; + info!(target: "eigenda-blobsource", "going to fetch through eigenda fetcher"); + let data = self.eigenda_fetcher.get_blob(eigenda_commitment).await; match data { Ok(data) => { self.open = true; diff --git a/crates/proof/src/hint.rs b/crates/proof/src/hint.rs index 0e388ad..af453b3 100644 --- a/crates/proof/src/hint.rs +++ b/crates/proof/src/hint.rs @@ -1,4 +1,4 @@ -//! This module contains the [ExtendedHintType], which adds an AltDACommitment case to kona's [HintType] enum. +//! This module contains the [ExtendedHintType], which adds an EigenDACommitment case to kona's [HintType] enum. use alloc::{ string::{String, ToString}, From 0d3c376e38c8801865b05b751d7444e80cb6cab4 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 19:12:24 +0000 Subject: [PATCH 12/13] add comments and add staleness check and empty bytes check --- crates/eigenda/src/eigenda.rs | 26 ++++++++++-- crates/eigenda/src/eigenda_data.rs | 2 + crates/proof/src/eigenda_provider.rs | 59 ++++++++++++++++++---------- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index 9653c36..5168275 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -4,6 +4,8 @@ use crate::eigenda_blobs::EigenDABlobSource; use crate::errors::CodecError; use crate::traits::EigenDABlobProvider; +use crate::BlobInfo; +use alloy_rlp::Decodable; use alloc::{boxed::Box, fmt::Debug}; use alloy_primitives::Bytes; @@ -59,12 +61,28 @@ where async fn next(&mut self, block_ref: &BlockInfo) -> PipelineResult { // then acutally use ethereum da to fetch. items are Bytes - let item = self.ethereum_source.next(block_ref).await?; + let cert = self.ethereum_source.next(block_ref).await?; - // just dump all the data out - info!(target: "eth-datasource", "next item {:?}", item); + // verify if cert is too stale + let cert_blob_info = BlobInfo::decode(&mut &cert.as_ref()[4..]).unwrap(); + info!("cert_blob_info {:?}", cert_blob_info); + let rbn = cert_blob_info + .blob_verification_proof + .batch_medatada + .batch_header + .reference_block_number as u64; + let l1_block_number = block_ref.number; - let eigenda_blob = self.eigenda_source.next(&item).await?; + // ToDo make it part of rollup config + let stale_gap = 100 as u64; + + // check staleness + if rbn + stale_gap < l1_block_number { + // return error + unimplemented!() + } + + let eigenda_blob = self.eigenda_source.next(&cert).await?; Ok(eigenda_blob) } diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 80f895e..3f12c30 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -36,6 +36,8 @@ impl EigenDABlobData { helpers::remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); let blob_content: Bytes = blob_content.into(); + // might insert a FFT here, + // take data Ok(blob_content.slice(..content_size as usize)) } diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index 704aa88..c72458f 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -3,7 +3,7 @@ use alloc::sync::Arc; use alloy_primitives::{keccak256, Bytes}; use async_trait::async_trait; use hokulea_eigenda::{BlobInfo, EigenDABlobProvider}; -use kona_preimage::{CommsClient, PreimageKey, PreimageKeyType}; +use kona_preimage::{errors::PreimageOracleError, CommsClient, PreimageKey, PreimageKeyType}; use kona_proof::errors::OracleProviderError; @@ -37,45 +37,62 @@ impl EigenDABlobProvider for OracleEigenDAProvider // the fourth because 0x01010000 in the beginnin is metadata let item_slice = cert.as_ref(); + + // cert should at least contain 32 bytes for header + 4 bytes for commitment type metadata + if item_slice.len() <= 32 + 4 { + return Err(OracleProviderError::Preimage(PreimageOracleError::Other( + "does not contain header".into(), + ))); + } + + // the first four bytes are metadata, like cert version, OP generic commitement + // see https://github.com/Layr-Labs/eigenda-proxy/blob/main/commitments/mode.go#L39 + // the first byte my guess is the OP let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); info!("cert_blob_info {:?}", cert_blob_info); - // hack - remove later, when cert actually contain length - let data = self - .oracle - .get(PreimageKey::new( - *keccak256(cert), - PreimageKeyType::GlobalGeneric, - )) - .await - .map_err(OracleProviderError::Preimage)?; - let mut blob: Vec = vec![0; cert_blob_info.blob_header.data_length as usize]; // 96 because our g1 commitment has 64 bytes in v1 - let mut field_element_key = [0u8; 96]; - - // ToDo data_length should be power of 2, proxy should have returned it with dividing 32 + // why 96, the original 4844 has bytes length of 80 (it has 48 bytes for commitment) + // even then, it is not that the entire 80 bytes are used. Some bytes are empty + // for solidity optimization, I remember. + // + // ToDo investigate later to decide a right size + let mut blob_key = [0u8; 96]; + + // In eigenDA terminology, length describes the number of field element, size describes + // number of bytes. In eigenda proxy memstore mode, the datalength is wronly assigned to + // be the bytes lenght. We need to resolve it later. + // For now, we internally divides 32. ToDo let data_length = cert_blob_info.blob_header.data_length as u64 / 32; info!("cert_blob_info.blob_header.data_length {:?}", data_length); - field_element_key[..32].copy_from_slice(&cert_blob_info.blob_header.commitment.x); - field_element_key[32..64].copy_from_slice(&cert_blob_info.blob_header.commitment.y); + // the common key + blob_key[..32].copy_from_slice(&cert_blob_info.blob_header.commitment.x); + blob_key[32..64].copy_from_slice(&cert_blob_info.blob_header.commitment.y); + for i in 0..data_length { - field_element_key[88..].copy_from_slice(i.to_be_bytes().as_ref()); + blob_key[88..].copy_from_slice(i.to_be_bytes().as_ref()); let mut field_element = [0u8; 32]; self.oracle .get_exact( - PreimageKey::new( - *keccak256(field_element_key), - PreimageKeyType::GlobalGeneric, - ), + PreimageKey::new(*keccak256(blob_key), PreimageKeyType::GlobalGeneric), &mut field_element, ) .await .map_err(OracleProviderError::Preimage)?; + + // if field element is 0, it means the host has identified that the data + // has breached eigenda invariant, i.e cert is valid + if field_element.len() == 0 { + return Err(OracleProviderError::Preimage(PreimageOracleError::Other( + "field elememnt is empty, breached eigenda invariant".into(), + ))); + } + blob[(i as usize) << 5..(i as usize + 1) << 5].copy_from_slice(field_element.as_ref()); } From 99d76f060fa30af11d5d5c165369297b36019a3d Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 21:38:52 +0000 Subject: [PATCH 13/13] add staleness comment --- crates/eigenda/src/eigenda.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index 5168275..dc08eb9 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -77,6 +77,9 @@ where let stale_gap = 100 as u64; // check staleness + // ToDo this would require the op-rollup to follow the same pattern + // but passing blockId to proxy which implement the logic, + // see https://github.com/ethereum-optimism/optimism/blob/0bb2ff57c8133f1e3983820c0bf238001eca119b/op-alt-da/damgr.go#L211 if rbn + stale_gap < l1_block_number { // return error unimplemented!()