From 05ec6581ced4b6d68619e02b3a3e8ea44939ee25 Mon Sep 17 00:00:00 2001 From: fborello-lambda Date: Thu, 19 Dec 2024 13:41:40 -0300 Subject: [PATCH 1/4] feat: custom L2 dashboard --- Cargo.lock | 1 + cmd/ethrex/Cargo.toml | 2 +- crates/blockchain/Cargo.toml | 2 +- crates/blockchain/metrics/Cargo.toml | 2 + crates/blockchain/metrics/api.rs | 13 +- .../docker-compose-metrics-l2.overrides.yaml | 2 + .../metrics/docker-compose-metrics.yaml | 3 +- crates/blockchain/metrics/metrics_l2.rs | 116 ++++++++++++++++++ crates/blockchain/metrics/mod.rs | 6 +- .../tx_dashboard.json} | 0 .../dashboards/dashboard.yaml | 13 +- .../l2_dashboards/blocks_dashboard.json | 110 +++++++++++++++++ crates/l2/Cargo.toml | 6 + crates/l2/proposer/metrics.rs | 103 ++++++++++++++++ crates/l2/proposer/mod.rs | 4 + 15 files changed, 375 insertions(+), 8 deletions(-) create mode 100644 crates/blockchain/metrics/metrics_l2.rs rename crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/{demo_dashboards/dashboard.json => common_dashboards/tx_dashboard.json} (100%) create mode 100644 crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/l2_dashboards/blocks_dashboard.json create mode 100644 crates/l2/proposer/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 5e31430d2..bb451cea4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2160,6 +2160,7 @@ dependencies = [ "ethrex-blockchain", "ethrex-core", "ethrex-dev", + "ethrex-metrics", "ethrex-rlp", "ethrex-rpc", "ethrex-sdk", diff --git a/cmd/ethrex/Cargo.toml b/cmd/ethrex/Cargo.toml index 90d55fcd7..40360f3f2 100644 --- a/cmd/ethrex/Cargo.toml +++ b/cmd/ethrex/Cargo.toml @@ -43,7 +43,7 @@ path = "./ethrex.rs" [features] default = ["dep:ethrex-storage", "libmdbx"] dev = ["dep:ethrex-dev"] -metrics = ["ethrex-blockchain/metrics"] +metrics = ["ethrex-blockchain/metrics", "ethrex-l2/metrics"] libmdbx = ["dep:libmdbx", "ethrex-storage/libmdbx"] redb = ["dep:redb", "ethrex-storage/redb"] l2 = ["ethrex-vm/l2"] diff --git a/crates/blockchain/Cargo.toml b/crates/blockchain/Cargo.toml index d5e2a09c3..b4d5a12d3 100644 --- a/crates/blockchain/Cargo.toml +++ b/crates/blockchain/Cargo.toml @@ -33,4 +33,4 @@ default = ["c-kzg"] libmdbx = ["ethrex-core/libmdbx", "ethrex-storage/default", "ethrex-vm/libmdbx"] levm = ["ethrex-vm/levm"] c-kzg = ["ethrex-core/c-kzg"] -metrics = ["ethrex-metrics/api"] +metrics = ["ethrex-metrics/transactions"] diff --git a/crates/blockchain/metrics/Cargo.toml b/crates/blockchain/metrics/Cargo.toml index df0267e30..b1c2b20b1 100644 --- a/crates/blockchain/metrics/Cargo.toml +++ b/crates/blockchain/metrics/Cargo.toml @@ -26,3 +26,5 @@ path = "./mod.rs" [features] default = ["api"] api = [] +transactions = [] +l2 = [] diff --git a/crates/blockchain/metrics/api.rs b/crates/blockchain/metrics/api.rs index 8f8689827..c0db7ec29 100644 --- a/crates/blockchain/metrics/api.rs +++ b/crates/blockchain/metrics/api.rs @@ -1,5 +1,8 @@ use axum::{routing::get, Router}; +#[cfg(feature = "l2")] +use crate::metrics_l2::METRICS_L2; + use crate::{metrics_transactions::METRICS_TX, MetricsApiError}; pub async fn start_prometheus_metrics_api(port: String) -> Result<(), MetricsApiError> { @@ -14,6 +17,14 @@ pub async fn start_prometheus_metrics_api(port: String) -> Result<(), MetricsApi Ok(()) } +#[allow(unused_mut)] async fn get_metrics() -> String { - METRICS_TX.gather_metrics() + let mut ret_string = METRICS_TX.gather_metrics(); + #[cfg(feature = "l2")] + { + ret_string.push('\n'); + ret_string.push_str(&METRICS_L2.gather_metrics()) + } + + ret_string } diff --git a/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml b/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml index 578142ec1..c4d27c14c 100644 --- a/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml +++ b/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml @@ -7,3 +7,5 @@ services: grafana: ports: - "3802:3000" + volumes: + - ./provisioning/grafana_provisioning/dashboards/l2_dashboards:/etc/grafana/provisioning/dashboards/l2_dashboards diff --git a/crates/blockchain/metrics/docker-compose-metrics.yaml b/crates/blockchain/metrics/docker-compose-metrics.yaml index a7b3890cc..ca5e1b1e3 100644 --- a/crates/blockchain/metrics/docker-compose-metrics.yaml +++ b/crates/blockchain/metrics/docker-compose-metrics.yaml @@ -8,7 +8,8 @@ services: grafana: image: grafana/grafana volumes: - - ./provisioning/grafana_provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ./provisioning/grafana_provisioning/dashboards/common_dashboards:/etc/grafana/provisioning/dashboards/common_dashboards + - ./provisioning/grafana_provisioning/dashboards/dashboard.yaml:/etc/grafana/provisioning/dashboards/dashboard.yaml - ./provisioning/grafana_provisioning/datasources:/etc/grafana/provisioning/datasources #ports: defined in the .overrides file depends_on: diff --git a/crates/blockchain/metrics/metrics_l2.rs b/crates/blockchain/metrics/metrics_l2.rs new file mode 100644 index 000000000..af6c59b88 --- /dev/null +++ b/crates/blockchain/metrics/metrics_l2.rs @@ -0,0 +1,116 @@ +use prometheus::{Encoder, IntGaugeVec, Opts, Registry, TextEncoder}; +use std::sync::{Arc, LazyLock, Mutex}; + +pub static METRICS_L2: LazyLock = LazyLock::new(MetricsL2::default); + +pub struct MetricsL2 { + pub status_tracker: Arc>, +} + +impl Default for MetricsL2 { + fn default() -> Self { + Self::new() + } +} + +impl MetricsL2 { + pub fn new() -> Self { + MetricsL2 { + status_tracker: Arc::new(Mutex::new( + IntGaugeVec::new( + Opts::new( + "l2_blocks_tracker", + "Keeps track of the L2's status based on the L1's contracts", + ), + &["block_type"], + ) + .unwrap(), + )), + } + } + + pub fn set_block_type_and_block_number( + &self, + block_type: MetricsL2BlockType, + block_number: u64, + ) { + let clone = self.status_tracker.clone(); + + let lock = match clone.lock() { + Ok(lock) => lock, + Err(e) => { + tracing::error!("Failed to lock mutex: {e}"); + return; + } + }; + + let builder = match lock.get_metric_with_label_values(&[block_type.to_str()]) { + Ok(builder) => builder, + Err(e) => { + tracing::error!("Failed to build Metric: {e}"); + return; + } + }; + + let block_number_as_i64: i64 = match block_number.try_into() { + Ok(b) => b, + Err(e) => { + tracing::error!("Failed to convert block_number to i64: {e}"); + return; + } + }; + + builder.set(block_number_as_i64); + } + + pub fn gather_metrics(&self) -> String { + let r = Registry::new(); + + let clone = self.status_tracker.clone(); + + let lock = match clone.lock() { + Ok(lock) => lock, + Err(e) => { + tracing::error!("Failed to lock mutex: {e}"); + return String::new(); + } + }; + + if r.register(Box::new(lock.clone())).is_err() { + tracing::error!("Failed to register metric"); + return String::new(); + } + + let encoder = TextEncoder::new(); + let metric_families = r.gather(); + + let mut buffer = Vec::new(); + if encoder.encode(&metric_families, &mut buffer).is_err() { + tracing::error!("Failed to encode metrics"); + return String::new(); + } + + String::from_utf8(buffer).unwrap_or_else(|e| { + tracing::error!("Failed to convert buffer to String: {e}"); + String::new() + }) + } +} + +/// [MetricsL2BlockType::LastCommittedBlock] and [MetricsL2BlockType::LastVerifiedBlock] Matche the crates/l2/contracts/src/l1/OnChainProposer.sol variables +/// [MetricsL2BlockType::LastFetchedL1Block] Matches the variable in crates/l2/contracts/src/l1/CommonBridge.sol +pub enum MetricsL2BlockType { + LastCommittedBlock, + LastVerifiedBlock, + LastFetchedL1Block, +} + +impl MetricsL2BlockType { + pub fn to_str(&self) -> &str { + match self { + MetricsL2BlockType::LastCommittedBlock => "lastCommittedBlock", + MetricsL2BlockType::LastVerifiedBlock => "lastVerifiedBlock", + MetricsL2BlockType::LastFetchedL1Block => "lastFetchedL1Block", + } + } +} diff --git a/crates/blockchain/metrics/mod.rs b/crates/blockchain/metrics/mod.rs index ca78d93c7..e24983654 100644 --- a/crates/blockchain/metrics/mod.rs +++ b/crates/blockchain/metrics/mod.rs @@ -1,6 +1,8 @@ #[cfg(feature = "api")] pub mod api; -#[cfg(feature = "api")] +#[cfg(any(feature = "api", feature = "l2"))] +pub mod metrics_l2; +#[cfg(any(feature = "api", feature = "transactions"))] pub mod metrics_transactions; /// A macro to conditionally enable metrics-related code. @@ -22,7 +24,7 @@ pub mod metrics_transactions; /// ```toml /// ethrex-metrics = { path = "./metrics", default-features = false } /// [features] -/// metrics = ["ethrex-metrics/api"] +/// metrics = ["ethrex-metrics/transactions"] /// ``` /// /// In this way, when the `metrics` feature is enabled for that crate, the macro is triggered and the metrics_api is also used. diff --git a/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/demo_dashboards/dashboard.json b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/common_dashboards/tx_dashboard.json similarity index 100% rename from crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/demo_dashboards/dashboard.json rename to crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/common_dashboards/tx_dashboard.json diff --git a/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard.yaml b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard.yaml index 95021bafd..27519c4c4 100644 --- a/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard.yaml +++ b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard.yaml @@ -1,11 +1,20 @@ apiVersion: 1 providers: - - name: "Custom Prometheus Dashboards" + - name: "Ethrex Common Dashboards" orgId: 1 folder: "" type: file disableDeletion: true editable: true options: - path: /etc/grafana/provisioning/dashboards/demo_dashboards + path: /etc/grafana/provisioning/dashboards/common_dashboards + + - name: "EthrexL2 Dashboards" + orgId: 1 + folder: "" + type: file + disableDeletion: true + editable: true + options: + path: /etc/grafana/provisioning/dashboards/l2_dashboards diff --git a/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/l2_dashboards/blocks_dashboard.json b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/l2_dashboards/blocks_dashboard.json new file mode 100644 index 000000000..40048ccae --- /dev/null +++ b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/l2_dashboards/blocks_dashboard.json @@ -0,0 +1,110 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-001" + }, + "description": "Information coming from the contracts deployed on the L1.\n\nThe following condition should always be true lastVerifiedBlock <= lastCommittedBlock", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-001" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "l2_blocks_tracker", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{block_type}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "L2 status", + "transparent": true, + "type": "stat" + } + ], + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "EthrexL2 - Status", + "uid": "ce7ejkhjgdszkc", + "version": 2, + "weekStart": "" +} diff --git a/crates/l2/Cargo.toml b/crates/l2/Cargo.toml index 95d57fd95..dba1b90eb 100644 --- a/crates/l2/Cargo.toml +++ b/crates/l2/Cargo.toml @@ -31,6 +31,8 @@ zkvm_interface = { path = "./prover/zkvm/interface/", default-features = false } # risc0 risc0-zkvm = { version = "1.2.0" } +ethrex-metrics = { path = "../blockchain/metrics", default-features = false } + [dev-dependencies] ethrex-sdk = { path = "./sdk" } rand = "0.8.5" @@ -46,3 +48,7 @@ indexing_slicing = "deny" as_conversions = "deny" unnecessary_cast = "warn" panic = "deny" + +[features] +default = [] +metrics = ["ethrex-metrics/l2"] diff --git a/crates/l2/proposer/metrics.rs b/crates/l2/proposer/metrics.rs new file mode 100644 index 000000000..0f631da75 --- /dev/null +++ b/crates/l2/proposer/metrics.rs @@ -0,0 +1,103 @@ +use crate::{ + proposer::errors::L1WatcherError, + utils::{ + config::{ + committer::CommitterConfig, errors::ConfigError, eth::EthConfig, + l1_watcher::L1WatcherConfig, + }, + eth_client::{errors::EthClientError, EthClient}, + }, +}; +use ethereum_types::Address; +use ethrex_metrics::metrics_l2::{MetricsL2BlockType, METRICS_L2}; +use ethrex_storage::Store; +use std::time::Duration; +use tokio::time::sleep; +use tracing::{debug, error}; + +pub async fn start_metrics_gatherer(store: Store) -> Result<(), ConfigError> { + let eth_config = EthConfig::from_env()?; + // Just for the CommonBridge address + let watcher_config = L1WatcherConfig::from_env()?; + // Just for the OnChainProposer Address + let committer_config = CommitterConfig::from_env()?; + let mut metrics_gatherer = + MetricsGatherer::new_from_config(watcher_config, committer_config, eth_config).await?; + metrics_gatherer.run(&store).await; + Ok(()) +} + +pub struct MetricsGatherer { + eth_client: EthClient, + l2_client: EthClient, + common_bridge_address: Address, + on_chain_proposer_address: Address, + check_interval: Duration, +} + +impl MetricsGatherer { + pub async fn new_from_config( + watcher_config: L1WatcherConfig, + committer_config: CommitterConfig, + eth_config: EthConfig, + ) -> Result { + let eth_client = EthClient::new_from_config(eth_config); + let l2_client = EthClient::new("http://localhost:1729"); + Ok(Self { + eth_client, + l2_client, + common_bridge_address: watcher_config.bridge_address, + on_chain_proposer_address: committer_config.on_chain_proposer_address, + check_interval: Duration::from_millis(1000), + }) + } + + pub async fn run(&mut self, store: &Store) { + loop { + if let Err(err) = self.main_logic(store).await { + error!("Metrics Gatherer Error: {}", err); + } + + sleep(self.check_interval).await; + } + } + + async fn main_logic(&mut self, store: &Store) -> Result<(), L1WatcherError> { + loop { + let last_fetched_l1_block = + EthClient::get_last_fetched_l1_block(&self.eth_client, self.common_bridge_address) + .await? + .into(); + + let last_committed_block = EthClient::get_last_committed_block( + &self.eth_client, + self.on_chain_proposer_address, + ) + .await? + .into(); + + let last_verified_block = EthClient::get_last_verified_block( + &self.eth_client, + self.on_chain_proposer_address, + ) + .await? + .into(); + + METRICS_L2.set_block_type_and_block_number( + MetricsL2BlockType::LastCommittedBlock, + last_committed_block, + ); + METRICS_L2.set_block_type_and_block_number( + MetricsL2BlockType::LastVerifiedBlock, + last_verified_block, + ); + METRICS_L2.set_block_type_and_block_number( + MetricsL2BlockType::LastFetchedL1Block, + last_fetched_l1_block, + ); + + debug!("L2 Metrics Gathered"); + sleep(self.check_interval).await; + } + } +} diff --git a/crates/l2/proposer/mod.rs b/crates/l2/proposer/mod.rs index 63efc0dbb..56cbc93d0 100644 --- a/crates/l2/proposer/mod.rs +++ b/crates/l2/proposer/mod.rs @@ -13,6 +13,8 @@ pub mod l1_committer; pub mod l1_watcher; pub mod prover_server; pub mod state_diff; +#[cfg(feature = "metrics")] +pub mod metrics; pub mod errors; @@ -36,6 +38,8 @@ pub async fn start_proposer(store: Store) { task_set.spawn(l1_committer::start_l1_commiter(store.clone())); task_set.spawn(prover_server::start_prover_server(store.clone())); task_set.spawn(start_proposer_server(store.clone())); + #[cfg(feature = "metrics")] + task_set.spawn(metrics::start_metrics_gatherer(store.clone())); while let Some(res) = task_set.join_next().await { match res { From 55c6e76a2f446624664f83894695f17eada4a8cd Mon Sep 17 00:00:00 2001 From: fborello-lambda Date: Thu, 19 Dec 2024 13:52:20 -0300 Subject: [PATCH 2/4] chore: lint --- crates/l2/proposer/metrics.rs | 24 +++++++++--------------- crates/l2/proposer/mod.rs | 6 +++--- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/crates/l2/proposer/metrics.rs b/crates/l2/proposer/metrics.rs index 0f631da75..a66daec4d 100644 --- a/crates/l2/proposer/metrics.rs +++ b/crates/l2/proposer/metrics.rs @@ -10,12 +10,11 @@ use crate::{ }; use ethereum_types::Address; use ethrex_metrics::metrics_l2::{MetricsL2BlockType, METRICS_L2}; -use ethrex_storage::Store; use std::time::Duration; use tokio::time::sleep; use tracing::{debug, error}; -pub async fn start_metrics_gatherer(store: Store) -> Result<(), ConfigError> { +pub async fn start_metrics_gatherer() -> Result<(), ConfigError> { let eth_config = EthConfig::from_env()?; // Just for the CommonBridge address let watcher_config = L1WatcherConfig::from_env()?; @@ -23,13 +22,12 @@ pub async fn start_metrics_gatherer(store: Store) -> Result<(), ConfigError> { let committer_config = CommitterConfig::from_env()?; let mut metrics_gatherer = MetricsGatherer::new_from_config(watcher_config, committer_config, eth_config).await?; - metrics_gatherer.run(&store).await; + metrics_gatherer.run().await; Ok(()) } pub struct MetricsGatherer { eth_client: EthClient, - l2_client: EthClient, common_bridge_address: Address, on_chain_proposer_address: Address, check_interval: Duration, @@ -42,19 +40,18 @@ impl MetricsGatherer { eth_config: EthConfig, ) -> Result { let eth_client = EthClient::new_from_config(eth_config); - let l2_client = EthClient::new("http://localhost:1729"); + //let l2_client = EthClient::new("http://localhost:1729"); Ok(Self { eth_client, - l2_client, common_bridge_address: watcher_config.bridge_address, on_chain_proposer_address: committer_config.on_chain_proposer_address, check_interval: Duration::from_millis(1000), }) } - pub async fn run(&mut self, store: &Store) { + pub async fn run(&mut self) { loop { - if let Err(err) = self.main_logic(store).await { + if let Err(err) = self.main_logic().await { error!("Metrics Gatherer Error: {}", err); } @@ -62,26 +59,23 @@ impl MetricsGatherer { } } - async fn main_logic(&mut self, store: &Store) -> Result<(), L1WatcherError> { + async fn main_logic(&mut self) -> Result<(), L1WatcherError> { loop { let last_fetched_l1_block = EthClient::get_last_fetched_l1_block(&self.eth_client, self.common_bridge_address) - .await? - .into(); + .await?; let last_committed_block = EthClient::get_last_committed_block( &self.eth_client, self.on_chain_proposer_address, ) - .await? - .into(); + .await?; let last_verified_block = EthClient::get_last_verified_block( &self.eth_client, self.on_chain_proposer_address, ) - .await? - .into(); + .await?; METRICS_L2.set_block_type_and_block_number( MetricsL2BlockType::LastCommittedBlock, diff --git a/crates/l2/proposer/mod.rs b/crates/l2/proposer/mod.rs index 56cbc93d0..94bd31b4b 100644 --- a/crates/l2/proposer/mod.rs +++ b/crates/l2/proposer/mod.rs @@ -11,10 +11,10 @@ use tracing::{error, info}; pub mod l1_committer; pub mod l1_watcher; -pub mod prover_server; -pub mod state_diff; #[cfg(feature = "metrics")] pub mod metrics; +pub mod prover_server; +pub mod state_diff; pub mod errors; @@ -39,7 +39,7 @@ pub async fn start_proposer(store: Store) { task_set.spawn(prover_server::start_prover_server(store.clone())); task_set.spawn(start_proposer_server(store.clone())); #[cfg(feature = "metrics")] - task_set.spawn(metrics::start_metrics_gatherer(store.clone())); + task_set.spawn(metrics::start_metrics_gatherer()); while let Some(res) = task_set.join_next().await { match res { From c31a36567c665e8afe7aad51276975f47944402d Mon Sep 17 00:00:00 2001 From: fborello-lambda Date: Thu, 19 Dec 2024 14:25:11 -0300 Subject: [PATCH 3/4] refactor: error handling --- crates/blockchain/metrics/api.rs | 17 ++++- crates/blockchain/metrics/metrics_l2.rs | 71 +++++++------------ .../metrics/metrics_transactions.rs | 50 +++++-------- crates/blockchain/metrics/mod.rs | 12 ++++ crates/l2/proposer/errors.rs | 8 +++ crates/l2/proposer/metrics.rs | 10 +-- 6 files changed, 85 insertions(+), 83 deletions(-) diff --git a/crates/blockchain/metrics/api.rs b/crates/blockchain/metrics/api.rs index c0db7ec29..3b7ba6079 100644 --- a/crates/blockchain/metrics/api.rs +++ b/crates/blockchain/metrics/api.rs @@ -19,11 +19,24 @@ pub async fn start_prometheus_metrics_api(port: String) -> Result<(), MetricsApi #[allow(unused_mut)] async fn get_metrics() -> String { - let mut ret_string = METRICS_TX.gather_metrics(); + let mut ret_string = match METRICS_TX.gather_metrics() { + Ok(string) => string, + Err(_) => { + tracing::error!("Failed to register METRICS_TX"); + String::new() + } + }; + #[cfg(feature = "l2")] { ret_string.push('\n'); - ret_string.push_str(&METRICS_L2.gather_metrics()) + match METRICS_L2.gather_metrics() { + Ok(string) => ret_string.push_str(&string), + Err(_) => { + tracing::error!("Failed to register METRICS_L2"); + return String::new(); + } + } } ret_string diff --git a/crates/blockchain/metrics/metrics_l2.rs b/crates/blockchain/metrics/metrics_l2.rs index af6c59b88..dd6fc5c6b 100644 --- a/crates/blockchain/metrics/metrics_l2.rs +++ b/crates/blockchain/metrics/metrics_l2.rs @@ -1,6 +1,8 @@ use prometheus::{Encoder, IntGaugeVec, Opts, Registry, TextEncoder}; use std::sync::{Arc, LazyLock, Mutex}; +use crate::MetricsError; + pub static METRICS_L2: LazyLock = LazyLock::new(MetricsL2::default); pub struct MetricsL2 { @@ -33,67 +35,46 @@ impl MetricsL2 { &self, block_type: MetricsL2BlockType, block_number: u64, - ) { + ) -> Result<(), MetricsError> { let clone = self.status_tracker.clone(); - let lock = match clone.lock() { - Ok(lock) => lock, - Err(e) => { - tracing::error!("Failed to lock mutex: {e}"); - return; - } - }; - - let builder = match lock.get_metric_with_label_values(&[block_type.to_str()]) { - Ok(builder) => builder, - Err(e) => { - tracing::error!("Failed to build Metric: {e}"); - return; - } - }; - - let block_number_as_i64: i64 = match block_number.try_into() { - Ok(b) => b, - Err(e) => { - tracing::error!("Failed to convert block_number to i64: {e}"); - return; - } - }; + let lock = clone + .lock() + .map_err(|e| MetricsError::MutexLockError(e.to_string()))?; + + let builder = lock + .get_metric_with_label_values(&[block_type.to_str()]) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + let block_number_as_i64: i64 = block_number.try_into()?; builder.set(block_number_as_i64); + + Ok(()) } - pub fn gather_metrics(&self) -> String { + pub fn gather_metrics(&self) -> Result { let r = Registry::new(); let clone = self.status_tracker.clone(); - let lock = match clone.lock() { - Ok(lock) => lock, - Err(e) => { - tracing::error!("Failed to lock mutex: {e}"); - return String::new(); - } - }; - - if r.register(Box::new(lock.clone())).is_err() { - tracing::error!("Failed to register metric"); - return String::new(); - } + let lock = clone + .lock() + .map_err(|e| MetricsError::MutexLockError(e.to_string()))?; + + r.register(Box::new(lock.clone())) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; let encoder = TextEncoder::new(); let metric_families = r.gather(); let mut buffer = Vec::new(); - if encoder.encode(&metric_families, &mut buffer).is_err() { - tracing::error!("Failed to encode metrics"); - return String::new(); - } + encoder + .encode(&metric_families, &mut buffer) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + + let res = String::from_utf8(buffer)?; - String::from_utf8(buffer).unwrap_or_else(|e| { - tracing::error!("Failed to convert buffer to String: {e}"); - String::new() - }) + Ok(res) } } diff --git a/crates/blockchain/metrics/metrics_transactions.rs b/crates/blockchain/metrics/metrics_transactions.rs index 1d2f97d60..143028cd8 100644 --- a/crates/blockchain/metrics/metrics_transactions.rs +++ b/crates/blockchain/metrics/metrics_transactions.rs @@ -2,6 +2,8 @@ use ethrex_core::types::TxType; use prometheus::{Encoder, IntCounter, IntCounterVec, Opts, Registry, TextEncoder}; use std::sync::{Arc, LazyLock, Mutex}; +use crate::MetricsError; + pub static METRICS_TX: LazyLock = LazyLock::new(MetricsTx::default); pub struct MetricsTx { @@ -71,49 +73,35 @@ impl MetricsTx { txs_lock.inc(); } - pub fn gather_metrics(&self) -> String { + pub fn gather_metrics(&self) -> Result { let r = Registry::new(); let txs_tracker = self.transactions_tracker.clone(); - let txs_tracker_lock = match txs_tracker.lock() { - Ok(lock) => lock, - Err(e) => { - tracing::error!("Failed to lock transactions_tracker mutex: {e}"); - return String::new(); - } - }; + let txs_tracker_lock = txs_tracker + .lock() + .map_err(|e| MetricsError::MutexLockError(e.to_string()))?; let txs_lock = self.transactions_total.clone(); - let txs_lock = match txs_lock.lock() { - Ok(lock) => lock, - Err(e) => { - tracing::error!("Failed to lock transactions_total mutex: {e}"); - return String::new(); - } - }; + let txs_lock = txs_lock + .lock() + .map_err(|e| MetricsError::MutexLockError(e.to_string()))?; - if r.register(Box::new(txs_lock.clone())).is_err() { - tracing::error!("Failed to register metric"); - return String::new(); - } - if r.register(Box::new(txs_tracker_lock.clone())).is_err() { - tracing::error!("Failed to register metric"); - return String::new(); - } + r.register(Box::new(txs_lock.clone())) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + r.register(Box::new(txs_tracker_lock.clone())) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; let encoder = TextEncoder::new(); let metric_families = r.gather(); let mut buffer = Vec::new(); - if encoder.encode(&metric_families, &mut buffer).is_err() { - tracing::error!("Failed to encode metrics"); - return String::new(); - } + encoder + .encode(&metric_families, &mut buffer) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + + let res = String::from_utf8(buffer)?; - String::from_utf8(buffer).unwrap_or_else(|e| { - tracing::error!("Failed to convert buffer to String: {e}"); - String::new() - }) + Ok(res) } } diff --git a/crates/blockchain/metrics/mod.rs b/crates/blockchain/metrics/mod.rs index e24983654..bdcbff931 100644 --- a/crates/blockchain/metrics/mod.rs +++ b/crates/blockchain/metrics/mod.rs @@ -54,3 +54,15 @@ pub enum MetricsApiError { #[error("{0}")] TcpError(#[from] std::io::Error), } + +#[derive(Debug, thiserror::Error)] +pub enum MetricsError { + #[error("MetricsL2Error: {0}")] + MutexLockError(String), + #[error("MetricsL2Error: {0}")] + PrometheusErr(String), + #[error("MetricsL2Error {0}")] + TryInto(#[from] std::num::TryFromIntError), + #[error("MetricsL2Error {0}")] + FromUtf8Error(#[from] std::string::FromUtf8Error), +} diff --git a/crates/l2/proposer/errors.rs b/crates/l2/proposer/errors.rs index 624016c50..b10a728c9 100644 --- a/crates/l2/proposer/errors.rs +++ b/crates/l2/proposer/errors.rs @@ -136,3 +136,11 @@ pub enum StateDiffError { #[error("The length of the vector is too big to fit in u16: {0}")] LengthTooBig(#[from] core::num::TryFromIntError), } + +#[derive(Debug, thiserror::Error)] +pub enum MetricsGathererError { + #[error("MetricsGathererError: {0}")] + MetricsError(#[from] ethrex_metrics::MetricsError), + #[error("MetricsGatherer failed because of an EthClient error: {0}")] + EthClientError(#[from] EthClientError), +} diff --git a/crates/l2/proposer/metrics.rs b/crates/l2/proposer/metrics.rs index a66daec4d..553862e68 100644 --- a/crates/l2/proposer/metrics.rs +++ b/crates/l2/proposer/metrics.rs @@ -1,5 +1,5 @@ use crate::{ - proposer::errors::L1WatcherError, + proposer::errors::MetricsGathererError, utils::{ config::{ committer::CommitterConfig, errors::ConfigError, eth::EthConfig, @@ -59,7 +59,7 @@ impl MetricsGatherer { } } - async fn main_logic(&mut self) -> Result<(), L1WatcherError> { + async fn main_logic(&mut self) -> Result<(), MetricsGathererError> { loop { let last_fetched_l1_block = EthClient::get_last_fetched_l1_block(&self.eth_client, self.common_bridge_address) @@ -80,15 +80,15 @@ impl MetricsGatherer { METRICS_L2.set_block_type_and_block_number( MetricsL2BlockType::LastCommittedBlock, last_committed_block, - ); + )?; METRICS_L2.set_block_type_and_block_number( MetricsL2BlockType::LastVerifiedBlock, last_verified_block, - ); + )?; METRICS_L2.set_block_type_and_block_number( MetricsL2BlockType::LastFetchedL1Block, last_fetched_l1_block, - ); + )?; debug!("L2 Metrics Gathered"); sleep(self.check_interval).await; From c55a4e4be9d34475b2841308ca01384271db7199 Mon Sep 17 00:00:00 2001 From: fborello-lambda Date: Thu, 19 Dec 2024 18:05:29 -0300 Subject: [PATCH 4/4] fix: l1 grafana error msg --- .../docker-compose-metrics-l1-dev.overrides.yaml | 4 ++++ .../metrics/docker-compose-metrics-l2.overrides.yaml | 3 ++- crates/blockchain/metrics/docker-compose-metrics.yaml | 1 - .../dashboards/dashboard_config_l1.yaml | 11 +++++++++++ .../{dashboard.yaml => dashboard_config_l2.yaml} | 0 5 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l1.yaml rename crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/{dashboard.yaml => dashboard_config_l2.yaml} (100%) diff --git a/crates/blockchain/metrics/docker-compose-metrics-l1-dev.overrides.yaml b/crates/blockchain/metrics/docker-compose-metrics-l1-dev.overrides.yaml index 1bdafe6b5..28ed0c283 100644 --- a/crates/blockchain/metrics/docker-compose-metrics-l1-dev.overrides.yaml +++ b/crates/blockchain/metrics/docker-compose-metrics-l1-dev.overrides.yaml @@ -7,3 +7,7 @@ services: grafana: ports: - "3801:3000" + volumes: + - type: bind + source: ../metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l1.yaml + target: /etc/grafana/provisioning/dashboards/dashboard.yaml diff --git a/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml b/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml index c4d27c14c..b72f11652 100644 --- a/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml +++ b/crates/blockchain/metrics/docker-compose-metrics-l2.overrides.yaml @@ -8,4 +8,5 @@ services: ports: - "3802:3000" volumes: - - ./provisioning/grafana_provisioning/dashboards/l2_dashboards:/etc/grafana/provisioning/dashboards/l2_dashboards + - ../metrics/provisioning/grafana_provisioning/dashboards/l2_dashboards:/etc/grafana/provisioning/dashboards/l2_dashboards + - ../metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l2.yaml:/etc/grafana/provisioning/dashboards/dashboard.yaml diff --git a/crates/blockchain/metrics/docker-compose-metrics.yaml b/crates/blockchain/metrics/docker-compose-metrics.yaml index ca5e1b1e3..899772c9c 100644 --- a/crates/blockchain/metrics/docker-compose-metrics.yaml +++ b/crates/blockchain/metrics/docker-compose-metrics.yaml @@ -9,7 +9,6 @@ services: image: grafana/grafana volumes: - ./provisioning/grafana_provisioning/dashboards/common_dashboards:/etc/grafana/provisioning/dashboards/common_dashboards - - ./provisioning/grafana_provisioning/dashboards/dashboard.yaml:/etc/grafana/provisioning/dashboards/dashboard.yaml - ./provisioning/grafana_provisioning/datasources:/etc/grafana/provisioning/datasources #ports: defined in the .overrides file depends_on: diff --git a/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l1.yaml b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l1.yaml new file mode 100644 index 000000000..46dd9690c --- /dev/null +++ b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l1.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: "Ethrex Common Dashboards" + orgId: 1 + folder: "" + type: file + disableDeletion: true + editable: true + options: + path: /etc/grafana/provisioning/dashboards/common_dashboards diff --git a/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard.yaml b/crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l2.yaml similarity index 100% rename from crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard.yaml rename to crates/blockchain/metrics/provisioning/grafana_provisioning/dashboards/dashboard_config_l2.yaml