diff --git a/Cargo.lock b/Cargo.lock index c0780e1ed..9a11d387c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2718,6 +2718,7 @@ dependencies = [ "ethrex-blockchain", "ethrex-core", "ethrex-dev", + "ethrex-metrics", "ethrex-rlp", "ethrex-rpc", "ethrex-sdk", diff --git a/cmd/ethrex/Cargo.toml b/cmd/ethrex/Cargo.toml index 01ba10bbb..bdabeee22 100644 --- a/cmd/ethrex/Cargo.toml +++ b/cmd/ethrex/Cargo.toml @@ -43,7 +43,7 @@ path = "./ethrex.rs" [features] default = ["dep:ethrex-storage", "libmdbx"] dev = ["dep:ethrex-dev"] -metrics = ["ethrex-blockchain/metrics"] +metrics = ["ethrex-blockchain/metrics", "ethrex-l2/metrics"] libmdbx = ["dep:libmdbx", "ethrex-storage/libmdbx"] redb = ["dep:redb", "ethrex-storage/redb"] l2 = ["dep:ethrex-l2", "ethrex-vm/l2"] diff --git a/crates/blockchain/Cargo.toml b/crates/blockchain/Cargo.toml index 6d0cb2f9a..5896f56a5 100644 --- a/crates/blockchain/Cargo.toml +++ b/crates/blockchain/Cargo.toml @@ -37,4 +37,4 @@ libmdbx = [ "ethrex-vm/libmdbx", ] c-kzg =["ethrex-core/c-kzg"] -metrics = ["ethrex-metrics/api"] +metrics = ["ethrex-metrics/transactions"] diff --git a/crates/blockchain/metrics/Cargo.toml b/crates/blockchain/metrics/Cargo.toml index df0267e30..b1c2b20b1 100644 --- a/crates/blockchain/metrics/Cargo.toml +++ b/crates/blockchain/metrics/Cargo.toml @@ -26,3 +26,5 @@ path = "./mod.rs" [features] default = ["api"] api = [] +transactions = [] +l2 = [] diff --git a/crates/blockchain/metrics/api.rs b/crates/blockchain/metrics/api.rs index 8f8689827..3b7ba6079 100644 --- a/crates/blockchain/metrics/api.rs +++ b/crates/blockchain/metrics/api.rs @@ -1,5 +1,8 @@ use axum::{routing::get, Router}; +#[cfg(feature = "l2")] +use crate::metrics_l2::METRICS_L2; + use crate::{metrics_transactions::METRICS_TX, MetricsApiError}; pub async fn start_prometheus_metrics_api(port: String) -> Result<(), MetricsApiError> { @@ -14,6 +17,27 @@ pub async fn start_prometheus_metrics_api(port: String) -> Result<(), MetricsApi Ok(()) } +#[allow(unused_mut)] async fn get_metrics() -> String { - METRICS_TX.gather_metrics() + let mut ret_string = match METRICS_TX.gather_metrics() { + Ok(string) => string, + Err(_) => { + tracing::error!("Failed to register METRICS_TX"); + String::new() + } + }; + + #[cfg(feature = "l2")] + { + ret_string.push('\n'); + match METRICS_L2.gather_metrics() { + Ok(string) => ret_string.push_str(&string), + Err(_) => { + tracing::error!("Failed to register METRICS_L2"); + return String::new(); + } + } + } + + ret_string } diff --git a/crates/blockchain/metrics/docker-compose-metrics-l1-dev.overrides.yaml b/crates/blockchain/metrics/docker-compose-metrics-l1-dev.overrides.yaml index 1bdafe6b5..28ed0c283 100644 --- a/crates/blockchain/metrics/docker-compose-metrics-l1-dev.overrides.yaml +++ b/crates/blockchain/metrics/docker-compose-metrics-l1-dev.overrides.yaml @@ -7,3 +7,7 @@ services: grafana: ports: - "3801:3000" + volumes: + - type: bind + source: ../metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l1.yaml + target: /etc/grafana/provisioning/dashboards/dashboard.yaml diff --git a/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml b/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml index 578142ec1..b72f11652 100644 --- a/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml +++ b/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml @@ -7,3 +7,6 @@ services: grafana: ports: - "3802:3000" + volumes: + - ../metrics/provisioning/grafana_provisioning/dashboards/l2_dashboards:/etc/grafana/provisioning/dashboards/l2_dashboards + - ../metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l2.yaml:/etc/grafana/provisioning/dashboards/dashboard.yaml diff --git a/crates/blockchain/metrics/docker-compose-metrics.yaml b/crates/blockchain/metrics/docker-compose-metrics.yaml index a7b3890cc..899772c9c 100644 --- a/crates/blockchain/metrics/docker-compose-metrics.yaml +++ b/crates/blockchain/metrics/docker-compose-metrics.yaml @@ -8,7 +8,7 @@ services: grafana: image: grafana/grafana volumes: - - ./provisioning/grafana_provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ./provisioning/grafana_provisioning/dashboards/common_dashboards:/etc/grafana/provisioning/dashboards/common_dashboards - ./provisioning/grafana_provisioning/datasources:/etc/grafana/provisioning/datasources #ports: defined in the .overrides file depends_on: diff --git a/crates/blockchain/metrics/metrics_l2.rs b/crates/blockchain/metrics/metrics_l2.rs new file mode 100644 index 000000000..dd6fc5c6b --- /dev/null +++ b/crates/blockchain/metrics/metrics_l2.rs @@ -0,0 +1,97 @@ +use prometheus::{Encoder, IntGaugeVec, Opts, Registry, TextEncoder}; +use std::sync::{Arc, LazyLock, Mutex}; + +use crate::MetricsError; + +pub static METRICS_L2: LazyLock = LazyLock::new(MetricsL2::default); + +pub struct MetricsL2 { + pub status_tracker: Arc>, +} + +impl Default for MetricsL2 { + fn default() -> Self { + Self::new() + } +} + +impl MetricsL2 { + pub fn new() -> Self { + MetricsL2 { + status_tracker: Arc::new(Mutex::new( + IntGaugeVec::new( + Opts::new( + "l2_blocks_tracker", + "Keeps track of the L2's status based on the L1's contracts", + ), + &["block_type"], + ) + .unwrap(), + )), + } + } + + pub fn set_block_type_and_block_number( + &self, + block_type: MetricsL2BlockType, + block_number: u64, + ) -> Result<(), MetricsError> { + let clone = self.status_tracker.clone(); + + let lock = clone + .lock() + .map_err(|e| MetricsError::MutexLockError(e.to_string()))?; + + let builder = lock + .get_metric_with_label_values(&[block_type.to_str()]) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + let block_number_as_i64: i64 = block_number.try_into()?; + + builder.set(block_number_as_i64); + + Ok(()) + } + + pub fn gather_metrics(&self) -> Result { + let r = Registry::new(); + + let clone = self.status_tracker.clone(); + + let lock = clone + .lock() + .map_err(|e| MetricsError::MutexLockError(e.to_string()))?; + + r.register(Box::new(lock.clone())) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + + let encoder = TextEncoder::new(); + let metric_families = r.gather(); + + let mut buffer = Vec::new(); + encoder + .encode(&metric_families, &mut buffer) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + + let res = String::from_utf8(buffer)?; + + Ok(res) + } +} + +/// [MetricsL2BlockType::LastCommittedBlock] and [MetricsL2BlockType::LastVerifiedBlock] Matche the crates/l2/contracts/src/l1/OnChainProposer.sol variables +/// [MetricsL2BlockType::LastFetchedL1Block] Matches the variable in crates/l2/contracts/src/l1/CommonBridge.sol +pub enum MetricsL2BlockType { + LastCommittedBlock, + LastVerifiedBlock, + LastFetchedL1Block, +} + +impl MetricsL2BlockType { + pub fn to_str(&self) -> &str { + match self { + MetricsL2BlockType::LastCommittedBlock => "lastCommittedBlock", + MetricsL2BlockType::LastVerifiedBlock => "lastVerifiedBlock", + MetricsL2BlockType::LastFetchedL1Block => "lastFetchedL1Block", + } + } +} diff --git a/crates/blockchain/metrics/metrics_transactions.rs b/crates/blockchain/metrics/metrics_transactions.rs index 1d2f97d60..143028cd8 100644 --- a/crates/blockchain/metrics/metrics_transactions.rs +++ b/crates/blockchain/metrics/metrics_transactions.rs @@ -2,6 +2,8 @@ use ethrex_core::types::TxType; use prometheus::{Encoder, IntCounter, IntCounterVec, Opts, Registry, TextEncoder}; use std::sync::{Arc, LazyLock, Mutex}; +use crate::MetricsError; + pub static METRICS_TX: LazyLock = LazyLock::new(MetricsTx::default); pub struct MetricsTx { @@ -71,49 +73,35 @@ impl MetricsTx { txs_lock.inc(); } - pub fn gather_metrics(&self) -> String { + pub fn gather_metrics(&self) -> Result { let r = Registry::new(); let txs_tracker = self.transactions_tracker.clone(); - let txs_tracker_lock = match txs_tracker.lock() { - Ok(lock) => lock, - Err(e) => { - tracing::error!("Failed to lock transactions_tracker mutex: {e}"); - return String::new(); - } - }; + let txs_tracker_lock = txs_tracker + .lock() + .map_err(|e| MetricsError::MutexLockError(e.to_string()))?; let txs_lock = self.transactions_total.clone(); - let txs_lock = match txs_lock.lock() { - Ok(lock) => lock, - Err(e) => { - tracing::error!("Failed to lock transactions_total mutex: {e}"); - return String::new(); - } - }; + let txs_lock = txs_lock + .lock() + .map_err(|e| MetricsError::MutexLockError(e.to_string()))?; - if r.register(Box::new(txs_lock.clone())).is_err() { - tracing::error!("Failed to register metric"); - return String::new(); - } - if r.register(Box::new(txs_tracker_lock.clone())).is_err() { - tracing::error!("Failed to register metric"); - return String::new(); - } + r.register(Box::new(txs_lock.clone())) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + r.register(Box::new(txs_tracker_lock.clone())) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; let encoder = TextEncoder::new(); let metric_families = r.gather(); let mut buffer = Vec::new(); - if encoder.encode(&metric_families, &mut buffer).is_err() { - tracing::error!("Failed to encode metrics"); - return String::new(); - } + encoder + .encode(&metric_families, &mut buffer) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + + let res = String::from_utf8(buffer)?; - String::from_utf8(buffer).unwrap_or_else(|e| { - tracing::error!("Failed to convert buffer to String: {e}"); - String::new() - }) + Ok(res) } } diff --git a/crates/blockchain/metrics/mod.rs b/crates/blockchain/metrics/mod.rs index ca78d93c7..bdcbff931 100644 --- a/crates/blockchain/metrics/mod.rs +++ b/crates/blockchain/metrics/mod.rs @@ -1,6 +1,8 @@ #[cfg(feature = "api")] pub mod api; -#[cfg(feature = "api")] +#[cfg(any(feature = "api", feature = "l2"))] +pub mod metrics_l2; +#[cfg(any(feature = "api", feature = "transactions"))] pub mod metrics_transactions; /// A macro to conditionally enable metrics-related code. @@ -22,7 +24,7 @@ pub mod metrics_transactions; /// ```toml /// ethrex-metrics = { path = "./metrics", default-features = false } /// [features] -/// metrics = ["ethrex-metrics/api"] +/// metrics = ["ethrex-metrics/transactions"] /// ``` /// /// In this way, when the `metrics` feature is enabled for that crate, the macro is triggered and the metrics_api is also used. @@ -52,3 +54,15 @@ pub enum MetricsApiError { #[error("{0}")] TcpError(#[from] std::io::Error), } + +#[derive(Debug, thiserror::Error)] +pub enum MetricsError { + #[error("MetricsL2Error: {0}")] + MutexLockError(String), + #[error("MetricsL2Error: {0}")] + PrometheusErr(String), + #[error("MetricsL2Error {0}")] + TryInto(#[from] std::num::TryFromIntError), + #[error("MetricsL2Error {0}")] + FromUtf8Error(#[from] std::string::FromUtf8Error), +} diff --git a/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/demo_dashboards/dashboard.json b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/common_dashboards/tx_dashboard.json similarity index 100% rename from crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/demo_dashboards/dashboard.json rename to crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/common_dashboards/tx_dashboard.json diff --git a/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard.yaml b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l1.yaml similarity index 54% rename from crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard.yaml rename to crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l1.yaml index 95021bafd..46dd9690c 100644 --- a/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard.yaml +++ b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l1.yaml @@ -1,11 +1,11 @@ apiVersion: 1 providers: - - name: "Custom Prometheus Dashboards" + - name: "Ethrex Common Dashboards" orgId: 1 folder: "" type: file disableDeletion: true editable: true options: - path: /etc/grafana/provisioning/dashboards/demo_dashboards + path: /etc/grafana/provisioning/dashboards/common_dashboards diff --git a/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l2.yaml b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l2.yaml new file mode 100644 index 000000000..27519c4c4 --- /dev/null +++ b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l2.yaml @@ -0,0 +1,20 @@ +apiVersion: 1 + +providers: + - name: "Ethrex Common Dashboards" + orgId: 1 + folder: "" + type: file + disableDeletion: true + editable: true + options: + path: /etc/grafana/provisioning/dashboards/common_dashboards + + - name: "EthrexL2 Dashboards" + orgId: 1 + folder: "" + type: file + disableDeletion: true + editable: true + options: + path: /etc/grafana/provisioning/dashboards/l2_dashboards diff --git a/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/l2_dashboards/blocks_dashboard.json b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/l2_dashboards/blocks_dashboard.json new file mode 100644 index 000000000..40048ccae --- /dev/null +++ b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/l2_dashboards/blocks_dashboard.json @@ -0,0 +1,110 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-001" + }, + "description": "Information coming from the contracts deployed on the L1.\n\nThe following condition should always be true lastVerifiedBlock <= lastCommittedBlock", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-001" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "l2_blocks_tracker", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{block_type}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "L2 status", + "transparent": true, + "type": "stat" + } + ], + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "EthrexL2 - Status", + "uid": "ce7ejkhjgdszkc", + "version": 2, + "weekStart": "" +} diff --git a/crates/l2/Cargo.toml b/crates/l2/Cargo.toml index 8bdd28b00..b5866928e 100644 --- a/crates/l2/Cargo.toml +++ b/crates/l2/Cargo.toml @@ -33,6 +33,8 @@ risc0-zkvm = { version = "1.2.0" } # sp1 sp1-sdk = { version = "3.4.0" } +ethrex-metrics = { path = "../blockchain/metrics", default-features = false } + [dev-dependencies] ethrex-sdk = { path = "./sdk" } rand = "0.8.5" @@ -48,3 +50,7 @@ indexing_slicing = "deny" as_conversions = "deny" unnecessary_cast = "warn" panic = "deny" + +[features] +default = [] +metrics = ["ethrex-metrics/l2"] diff --git a/crates/l2/proposer/errors.rs b/crates/l2/proposer/errors.rs index 624016c50..b10a728c9 100644 --- a/crates/l2/proposer/errors.rs +++ b/crates/l2/proposer/errors.rs @@ -136,3 +136,11 @@ pub enum StateDiffError { #[error("The length of the vector is too big to fit in u16: {0}")] LengthTooBig(#[from] core::num::TryFromIntError), } + +#[derive(Debug, thiserror::Error)] +pub enum MetricsGathererError { + #[error("MetricsGathererError: {0}")] + MetricsError(#[from] ethrex_metrics::MetricsError), + #[error("MetricsGatherer failed because of an EthClient error: {0}")] + EthClientError(#[from] EthClientError), +} diff --git a/crates/l2/proposer/metrics.rs b/crates/l2/proposer/metrics.rs new file mode 100644 index 000000000..553862e68 --- /dev/null +++ b/crates/l2/proposer/metrics.rs @@ -0,0 +1,97 @@ +use crate::{ + proposer::errors::MetricsGathererError, + utils::{ + config::{ + committer::CommitterConfig, errors::ConfigError, eth::EthConfig, + l1_watcher::L1WatcherConfig, + }, + eth_client::{errors::EthClientError, EthClient}, + }, +}; +use ethereum_types::Address; +use ethrex_metrics::metrics_l2::{MetricsL2BlockType, METRICS_L2}; +use std::time::Duration; +use tokio::time::sleep; +use tracing::{debug, error}; + +pub async fn start_metrics_gatherer() -> Result<(), ConfigError> { + let eth_config = EthConfig::from_env()?; + // Just for the CommonBridge address + let watcher_config = L1WatcherConfig::from_env()?; + // Just for the OnChainProposer Address + let committer_config = CommitterConfig::from_env()?; + let mut metrics_gatherer = + MetricsGatherer::new_from_config(watcher_config, committer_config, eth_config).await?; + metrics_gatherer.run().await; + Ok(()) +} + +pub struct MetricsGatherer { + eth_client: EthClient, + common_bridge_address: Address, + on_chain_proposer_address: Address, + check_interval: Duration, +} + +impl MetricsGatherer { + pub async fn new_from_config( + watcher_config: L1WatcherConfig, + committer_config: CommitterConfig, + eth_config: EthConfig, + ) -> Result { + let eth_client = EthClient::new_from_config(eth_config); + //let l2_client = EthClient::new("http://localhost:1729"); + Ok(Self { + eth_client, + common_bridge_address: watcher_config.bridge_address, + on_chain_proposer_address: committer_config.on_chain_proposer_address, + check_interval: Duration::from_millis(1000), + }) + } + + pub async fn run(&mut self) { + loop { + if let Err(err) = self.main_logic().await { + error!("Metrics Gatherer Error: {}", err); + } + + sleep(self.check_interval).await; + } + } + + async fn main_logic(&mut self) -> Result<(), MetricsGathererError> { + loop { + let last_fetched_l1_block = + EthClient::get_last_fetched_l1_block(&self.eth_client, self.common_bridge_address) + .await?; + + let last_committed_block = EthClient::get_last_committed_block( + &self.eth_client, + self.on_chain_proposer_address, + ) + .await?; + + let last_verified_block = EthClient::get_last_verified_block( + &self.eth_client, + self.on_chain_proposer_address, + ) + .await?; + + METRICS_L2.set_block_type_and_block_number( + MetricsL2BlockType::LastCommittedBlock, + last_committed_block, + )?; + METRICS_L2.set_block_type_and_block_number( + MetricsL2BlockType::LastVerifiedBlock, + last_verified_block, + )?; + METRICS_L2.set_block_type_and_block_number( + MetricsL2BlockType::LastFetchedL1Block, + last_fetched_l1_block, + )?; + + debug!("L2 Metrics Gathered"); + sleep(self.check_interval).await; + } + } +} diff --git a/crates/l2/proposer/mod.rs b/crates/l2/proposer/mod.rs index b11a0f41e..3930d4f4b 100644 --- a/crates/l2/proposer/mod.rs +++ b/crates/l2/proposer/mod.rs @@ -11,6 +11,8 @@ use tracing::{error, info}; pub mod l1_committer; pub mod l1_watcher; +#[cfg(feature = "metrics")] +pub mod metrics; pub mod prover_server; pub mod state_diff; @@ -36,6 +38,8 @@ pub async fn start_proposer(store: Store) { task_set.spawn(l1_committer::start_l1_commiter(store.clone())); task_set.spawn(prover_server::start_prover_server(store.clone())); task_set.spawn(start_proposer_server(store.clone())); + #[cfg(feature = "metrics")] + task_set.spawn(metrics::start_metrics_gatherer()); while let Some(res) = task_set.join_next().await { match res {