From 0ed28315f95cd88a01798a51908953447feb0937 Mon Sep 17 00:00:00 2001 From: Renato Dinhani <101204870+dinhani-cw@users.noreply.github.com> Date: Thu, 7 Mar 2024 01:03:17 -0300 Subject: [PATCH] feat: test metrics summary (#320) --- Cargo.lock | 21 ++++ Cargo.toml | 9 +- src/config.rs | 141 ++++++++++++++++++++------ src/eth/primitives/slot.rs | 11 ++ src/infra/docker.rs | 58 +++++++++++ src/infra/metrics.rs | 17 ++-- src/infra/mod.rs | 3 +- src/lib.rs | 6 +- tests/test_import_offline_snapshot.rs | 118 ++++++++++++--------- 9 files changed, 292 insertions(+), 92 deletions(-) create mode 100644 src/infra/docker.rs diff --git a/Cargo.lock b/Cargo.lock index 83e75ff3a..13324f592 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -553,6 +553,26 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const_format" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a214c7af3d04997541b18d432afaff4c455e79e2029079647e72fc2bd27673" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f6ff08fd20f4f299298a28e2dfa8a8ba1036e6cd2460ac1de7b425d76f2500" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -3697,6 +3717,7 @@ dependencies = [ "chrono", "clap", "const-hex", + "const_format", "crossbeam-channel", "derive-new", "derive_more", diff --git a/Cargo.toml b/Cargo.toml index 6c0a0cb43..be380a0f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ anyhow = "1.0.79" async-trait = "0.1.77" chrono = "0.4.31" clap = { version = "4.4.18", features = ["derive", "env"] } +const_format = "0.2.32" const-hex = "1.10.0" derive_more = "0.99.17" derive-new = "0.6.0" @@ -54,7 +55,7 @@ triehash = "0.8.4" # network jsonrpsee = { version = "0.21.0", features = ["server", "client"] } -reqwest = "0.11.24" +reqwest = { version = "0.11.24", features = ["json"] } tower = "0.4.13" # observability @@ -66,6 +67,10 @@ tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] } # storage sqlx = { version = "0.7.3", features = ["runtime-tokio", "postgres", "bigdecimal", "time"] } +# containers +testcontainers = "0.15.0" +testcontainers-modules = { version = "0.3.5", features = ["postgres"] } + # test fake = { version = "2.9.2", features = ["derive"] } @@ -73,8 +78,6 @@ fake = { version = "2.9.2", features = ["derive"] } binary_macros = "1.0.0" serial_test = "2.0.0" stringreader = "0.1.1" -testcontainers = "0.15.0" -testcontainers-modules = { version = "0.3.5", features = ["postgres"] } [build-dependencies] const-hex = "1.10.0" diff --git a/src/config.rs b/src/config.rs index 52e2214cc..4ce8969bc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -32,6 +32,14 @@ use crate::eth::EthExecutor; use crate::ext::not; use crate::infra::postgres::Postgres; +pub trait WithCommonConfig { + fn common(&self) -> &CommonConfig; +} + +// ----------------------------------------------------------------------------- +// Config: Stratus +// ----------------------------------------------------------------------------- + /// Configuration for main Stratus service. #[derive(Parser, Debug, derive_more::Deref)] pub struct StratusConfig { @@ -44,8 +52,18 @@ pub struct StratusConfig { pub common: CommonConfig, } +impl WithCommonConfig for StratusConfig { + fn common(&self) -> &CommonConfig { + &self.common + } +} + +// ----------------------------------------------------------------------------- +// Config: ImporterDownload +// ----------------------------------------------------------------------------- + /// Configuration for importer-download binary. -#[derive(Parser, Debug)] +#[derive(Parser, Debug, derive_more::Deref)] pub struct ImporterDownloadConfig { /// External RPC endpoint to sync blocks with Stratus. #[arg(short = 'r', long = "external-rpc", env = "EXTERNAL_RPC")] @@ -62,8 +80,22 @@ pub struct ImporterDownloadConfig { /// Accounts to retrieve initial balance information. #[arg(long = "initial-accounts", env = "INITIAL_ACCOUNTS", value_delimiter = ',')] pub initial_accounts: Vec
, + + #[deref] + #[clap(flatten)] + pub common: CommonConfig, +} + +impl WithCommonConfig for ImporterDownloadConfig { + fn common(&self) -> &CommonConfig { + &self.common + } } +// ----------------------------------------------------------------------------- +// Config: ImporterImport +// ----------------------------------------------------------------------------- + /// Configuration for importer-import binary. #[derive(Parser, Debug, derive_more::Deref)] pub struct ImporterImportConfig { @@ -80,6 +112,16 @@ pub struct ImporterImportConfig { pub common: CommonConfig, } +impl WithCommonConfig for ImporterImportConfig { + fn common(&self) -> &CommonConfig { + &self.common + } +} + +// ----------------------------------------------------------------------------- +// Config: RpcPoller +// ----------------------------------------------------------------------------- + /// Configuration for rpc-poller binary. #[derive(Parser, Debug, derive_more::Deref)] pub struct RpcPollerConfig { @@ -92,6 +134,16 @@ pub struct RpcPollerConfig { pub common: CommonConfig, } +impl WithCommonConfig for RpcPollerConfig { + fn common(&self) -> &CommonConfig { + &self.common + } +} + +// ----------------------------------------------------------------------------- +// Config: StateValidator +// ----------------------------------------------------------------------------- + /// Configuration for importer-import binary. #[derive(Parser, Debug, derive_more::Deref)] pub struct StateValidatorConfig { @@ -120,6 +172,16 @@ pub struct StateValidatorConfig { pub concurrent_tasks: u16, } +impl WithCommonConfig for StateValidatorConfig { + fn common(&self) -> &CommonConfig { + &self.common + } +} + +// ----------------------------------------------------------------------------- +// Config: Common +// ----------------------------------------------------------------------------- + /// Common configuration that can be used by any binary. #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -140,6 +202,9 @@ pub struct CommonConfig { #[arg(long = "blocking-threads", env = "BLOCKING_THREADS", default_value = "1")] pub num_blocking_threads: usize, + #[arg(long = "metrics-histogram-kind", env = "METRICS_HISTOGRAM_KIND", default_value = "summary")] + pub metrics_histogram_kind: MetricsHistogramKind, + /// Generates genesis block on startup when it does not exist. #[arg(long = "enable-genesis", env = "ENABLE_GENESIS", default_value = "false")] pub enable_genesis: bool, @@ -154,6 +219,12 @@ pub struct CommonConfig { pub nocapture: bool, } +impl WithCommonConfig for CommonConfig { + fn common(&self) -> &CommonConfig { + self + } +} + impl CommonConfig { /// Initializes storage. pub async fn init_storage(&self) -> anyhow::Result> { @@ -220,6 +291,10 @@ impl CommonConfig { } } +// ----------------------------------------------------------------------------- +// Enum: StorageConfig +// ----------------------------------------------------------------------------- + /// Storage configuration. #[derive(Clone, Debug, strum::Display)] pub enum StorageConfig { @@ -255,51 +330,53 @@ impl FromStr for StorageConfig { } } -#[derive(Clone, Debug, strum::Display)] -pub enum ValidatorMethodConfig { - Rpc { url: String }, - CompareTables, +// ----------------------------------------------------------------------------- +// Enum: MetricsHistogramKind +// ----------------------------------------------------------------------------- + +/// See: https://prometheus.io/docs/practices/histograms/ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MetricsHistogramKind { + /// Quantiles are calculated on client-side based on recent data kept in-memory. + /// + /// Client defines the quantiles to calculate. + Summary, + + /// Quantiles are calculated on server-side based on bucket counts. + /// + /// Cient defines buckets to group observations. + Histogram, } -impl FromStr for ValidatorMethodConfig { +impl FromStr for MetricsHistogramKind { type Err = anyhow::Error; fn from_str(s: &str) -> anyhow::Result { - match s { - "compare_tables" => Ok(Self::CompareTables), - s => Ok(Self::Rpc { url: s.to_string() }), + match s.to_lowercase().trim() { + "summary" => Ok(Self::Summary), + "histogram" => Ok(Self::Histogram), + s => Err(anyhow!("unknown metrics histogram kind: {}", s)), } } } -/// Enviroment where the application is running. -#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] -pub enum Environment { - Development, - Production, -} - -impl Environment { - /// Checks if the current environment is production. - pub fn is_production(&self) -> bool { - matches!(self, Self::Production) - } +// ----------------------------------------------------------------------------- +// Enum: ValidatorMethodConfig +// ----------------------------------------------------------------------------- - /// Checks if the current environment is development. - pub fn is_development(&self) -> bool { - matches!(self, Self::Development) - } +#[derive(Clone, Debug, strum::Display)] +pub enum ValidatorMethodConfig { + Rpc { url: String }, + CompareTables, } -impl FromStr for Environment { +impl FromStr for ValidatorMethodConfig { type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - let s = s.trim().to_lowercase(); - match s.as_str() { - "dev" | "development" => Ok(Self::Development), - "prod" | "production" => Ok(Self::Production), - s => Err(anyhow!("unknown environment: {}", s)), + fn from_str(s: &str) -> anyhow::Result { + match s { + "compare_tables" => Ok(Self::CompareTables), + s => Ok(Self::Rpc { url: s.to_string() }), } } } diff --git a/src/eth/primitives/slot.rs b/src/eth/primitives/slot.rs index a40ffdb9b..0c1b595a4 100644 --- a/src/eth/primitives/slot.rs +++ b/src/eth/primitives/slot.rs @@ -36,6 +36,11 @@ impl Slot { value: value.into(), } } + + /// Checks if the value is zero. + pub fn is_zero(&self) -> bool { + self.value.is_zero() + } } impl Display for Slot { @@ -131,6 +136,12 @@ impl From for [u8; 32] { pub struct SlotValue(U256); impl SlotValue { + /// Checks if the value is zero. + pub fn is_zero(&self) -> bool { + self.0.is_zero() + } + + /// Converts itself to [`U256`]. pub fn as_u256(&self) -> U256 { self.0 } diff --git a/src/infra/docker.rs b/src/infra/docker.rs new file mode 100644 index 000000000..4ce489754 --- /dev/null +++ b/src/infra/docker.rs @@ -0,0 +1,58 @@ +use testcontainers::clients::Cli; +use testcontainers::core::WaitFor; +use testcontainers::Container; +use testcontainers::GenericImage; +use testcontainers::RunnableImage; +use testcontainers_modules::postgres::Postgres as PostgresImage; + +#[derive(Default)] +pub struct Docker { + cli: Cli, +} + +impl Docker { + /// Starts PostgreSQL container for local development. + #[must_use] + pub fn start_postgres(&self) -> Container<'_, PostgresImage> { + tracing::info!("starting postgres container"); + + let image = RunnableImage::from(PostgresImage::default().with_user("postgres").with_password("123").with_db_name("stratus")) + .with_mapped_port((5432, 5432)) + .with_volume(("./static/schema/001-init.sql", "/docker-entrypoint-initdb.d/001-schema.sql")) + .with_volume(("./static/schema/002-schema-external-rpc.sql", "/docker-entrypoint-initdb.d/002-schema.sql")) + .with_tag("16.2"); + + self.cli.run(image) + } + + /// Starts Prometheus container for local development. + #[must_use] + pub fn start_prometheus(&self) -> Container<'_, GenericImage> { + tracing::info!("starting prometheus container"); + + let prometheus_image = GenericImage::new("prom/prometheus", "v2.50.1").with_wait_for(WaitFor::StdErrMessage { + message: "Starting rule manager...".to_string(), + }); + let prometheus_args: Vec = vec![ + "--config.file=/etc/prometheus/prometheus.yaml".into(), + "--storage.tsdb.path=/prometheus".into(), + "--log.level=debug".into(), + ]; + + let image = RunnableImage::from((prometheus_image, prometheus_args)) + .with_mapped_port((9090, 9090)) + .with_volume(("./static/prometheus.yaml", "/etc/prometheus/prometheus.yaml")); + + self.cli.run(image) + } + + /// Returns PostgreSQL container URL connection. + pub fn postgres_connection_url(&self) -> &'static str { + "postgres://postgres:123@localhost:5432/stratus" + } + + /// Returns Prometheus container API URL. + pub fn prometheus_api_url(&self) -> &'static str { + "http://localhost:9090/api/v1/query" + } +} diff --git a/src/infra/metrics.rs b/src/infra/metrics.rs index 30de609cc..264551827 100644 --- a/src/infra/metrics.rs +++ b/src/infra/metrics.rs @@ -11,6 +11,7 @@ use metrics_exporter_prometheus::Matcher; use metrics_exporter_prometheus::PrometheusBuilder; use paste::paste; +use crate::config::MetricsHistogramKind; use crate::ext::not; use crate::metrics; use crate::metrics_impl_fn_inc; @@ -27,7 +28,7 @@ const BUCKET_FOR_DURATION: [f64; 37] = [ /// Init application global metrics. /// /// Default configuration runs metrics exporter on port 9000. -pub fn init_metrics() { +pub fn init_metrics(histogram_kind: MetricsHistogramKind) { tracing::info!("starting metrics"); // get metric definitions @@ -42,10 +43,12 @@ pub fn init_metrics() { let mut builder = PrometheusBuilder::new(); // init buckets (comment to use summary) - builder = builder.set_buckets(&BUCKET_FOR_DURATION).unwrap(); - for metric in &metrics { - if metric.has_custom_buckets() { - builder = builder.set_buckets_for_metric(Matcher::Full(metric.name.to_string()), &metric.buckets).unwrap(); + if histogram_kind == MetricsHistogramKind::Histogram { + builder = builder.set_buckets(&BUCKET_FOR_DURATION).unwrap(); + for metric in &metrics { + if metric.has_custom_buckets() { + builder = builder.set_buckets_for_metric(Matcher::Full(metric.name.to_string()), &metric.buckets).unwrap(); + } } } @@ -76,7 +79,7 @@ metrics! { histogram_duration storage_read_current_block_number{success} [], "Time to execute storage read_account operation." - histogram_duration storage_read_account{kind, point_in_time, success} [], + histogram_duration storage_read_account{found_at, point_in_time, success} [], "Time to execute storage read_block operation." histogram_duration storage_read_block{success} [], @@ -85,7 +88,7 @@ metrics! { histogram_duration storage_read_logs{success} [], "Time to execute storage read_slot operation." - histogram_duration storage_read_slot{kind, point_in_time, success} [], + histogram_duration storage_read_slot{found_at, point_in_time, success} [], "Time to execute storage read_mined_transaction operation." histogram_duration storage_read_mined_transaction{success} [] diff --git a/src/infra/mod.rs b/src/infra/mod.rs index 223ffe5ba..9f3f99b3d 100644 --- a/src/infra/mod.rs +++ b/src/infra/mod.rs @@ -1,9 +1,10 @@ //! Shared infrastructure. pub mod blockchain_client; +pub mod docker; pub mod metrics; pub mod postgres; -pub mod tracing; // TODO: expose only struct, not module +pub mod tracing; pub use blockchain_client::BlockchainClient; pub use metrics::init_metrics; diff --git a/src/lib.rs b/src/lib.rs index 977579ed6..50eb6d996 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,5 @@ +use crate::config::WithCommonConfig; + pub mod config; pub mod eth; pub mod ext; @@ -6,10 +8,10 @@ pub mod infra; /// Executes global services initialization. pub fn init_global_services() -> T where - T: clap::Parser, + T: clap::Parser + WithCommonConfig, { let config = T::parse(); infra::init_tracing(); - infra::init_metrics(); + infra::init_metrics(config.common().metrics_histogram_kind); config } diff --git a/tests/test_import_offline_snapshot.rs b/tests/test_import_offline_snapshot.rs index b025ad12e..4154d0f70 100644 --- a/tests/test_import_offline_snapshot.rs +++ b/tests/test_import_offline_snapshot.rs @@ -1,6 +1,8 @@ use std::collections::HashMap; use std::sync::Arc; +use std::time::Duration; +use const_format::formatcp; use itertools::Itertools; use stratus::config::CommonConfig; use stratus::eth::primitives::ExternalBlock; @@ -10,42 +12,65 @@ use stratus::eth::storage::InMemoryPermanentStorageState; use stratus::eth::storage::InMemoryTemporaryStorage; use stratus::eth::storage::PermanentStorage; use stratus::eth::storage::StratusStorage; +use stratus::infra::docker::Docker; use stratus::infra::metrics::METRIC_EVM_EXECUTION; -use stratus::infra::metrics::METRIC_EVM_EXECUTION_ACCOUNT_READS; -use stratus::infra::metrics::METRIC_EVM_EXECUTION_SLOT_READS; -use stratus::infra::metrics::METRIC_EXECUTOR_IMPORT_OFFLINE_ACCOUNT_READS; -use stratus::infra::metrics::METRIC_EXECUTOR_IMPORT_OFFLINE_SLOT_READS; use stratus::infra::metrics::METRIC_STORAGE_COMMIT; use stratus::infra::metrics::METRIC_STORAGE_READ_ACCOUNT; use stratus::infra::metrics::METRIC_STORAGE_READ_SLOT; use stratus::infra::postgres::Postgres; use stratus::init_global_services; -use testcontainers::clients; -use testcontainers::Container; -use testcontainers::RunnableImage; -use testcontainers_modules::postgres::Postgres as PostgresImage; -const TRACKED_METRICS: [&str; 8] = [ - METRIC_EVM_EXECUTION, - METRIC_EVM_EXECUTION_SLOT_READS, - METRIC_EVM_EXECUTION_ACCOUNT_READS, - METRIC_EXECUTOR_IMPORT_OFFLINE_ACCOUNT_READS, - METRIC_EXECUTOR_IMPORT_OFFLINE_SLOT_READS, - METRIC_STORAGE_READ_ACCOUNT, - METRIC_STORAGE_READ_SLOT, - METRIC_STORAGE_COMMIT, +const METRIC_QUERIES: [&str; 30] = [ + // EVM + "", + formatcp!("{}_count", METRIC_EVM_EXECUTION), + formatcp!("{}_sum", METRIC_EVM_EXECUTION), + formatcp!("{}{{quantile='1'}}", METRIC_EVM_EXECUTION), + // STORAGE ACCOUNTS + "", + formatcp!("sum({}_count)", METRIC_STORAGE_READ_ACCOUNT), + formatcp!("{}_count{{found_at='temporary'}}", METRIC_STORAGE_READ_ACCOUNT), + formatcp!("{}_count{{found_at='permanent'}}", METRIC_STORAGE_READ_ACCOUNT), + formatcp!("{}_count{{found_at='default'}}", METRIC_STORAGE_READ_ACCOUNT), + formatcp!("sum({}_sum)", METRIC_STORAGE_READ_ACCOUNT), + formatcp!("{}_sum{{found_at='temporary'}}", METRIC_STORAGE_READ_ACCOUNT), + formatcp!("{}_sum{{found_at='permanent'}}", METRIC_STORAGE_READ_ACCOUNT), + formatcp!("{}_sum{{kifound_atnd='default'}}", METRIC_STORAGE_READ_ACCOUNT), + formatcp!("{}{{found_at='temporary', quantile='1'}}", METRIC_STORAGE_READ_ACCOUNT), + formatcp!("{}{{found_at='permanent', quantile='1'}}", METRIC_STORAGE_READ_ACCOUNT), + formatcp!("{}{{found_at='default', quantile='1'}}", METRIC_STORAGE_READ_ACCOUNT), + // STORAGE SLOTS + "", + formatcp!("sum({}_count)", METRIC_STORAGE_READ_SLOT), + formatcp!("{}_count{{found_at='temporary'}}", METRIC_STORAGE_READ_SLOT), + formatcp!("{}_count{{found_at='permanent'}}", METRIC_STORAGE_READ_SLOT), + formatcp!("{}_count{{found_at='default'}}", METRIC_STORAGE_READ_SLOT), + formatcp!("sum({}_sum)", METRIC_STORAGE_READ_SLOT), + formatcp!("{}_sum{{found_at='temporary'}}", METRIC_STORAGE_READ_SLOT), + formatcp!("{}_sum{{found_at='permanent'}}", METRIC_STORAGE_READ_SLOT), + formatcp!("{}_sum{{found_at='default'}}", METRIC_STORAGE_READ_SLOT), + formatcp!("{}{{found_at='temporary', quantile='1'}}", METRIC_STORAGE_READ_SLOT), + formatcp!("{}{{found_at='permanent', quantile='1'}}", METRIC_STORAGE_READ_SLOT), + formatcp!("{}{{found_at='default', quantile='1'}}", METRIC_STORAGE_READ_SLOT), + // STORAGE COMMIT + "", + formatcp!("{}{{quantile='1'}}", METRIC_STORAGE_COMMIT), ]; #[tokio::test] async fn test_import_offline_snapshot() { - let docker = clients::Cli::default(); let config = init_global_services::(); - // init block + // init containers + let docker = Docker::default(); + let _pg_guard = docker.start_postgres(); + let _prom_guard = docker.start_prometheus(); + + // init block data let block_json = include_str!("fixtures/block-292973/block.json"); let block: ExternalBlock = serde_json::from_str(block_json).unwrap(); - // init receipts + // init receipts data let receipts_json = include_str!("fixtures/block-292973/receipts.json"); let mut receipts = HashMap::new(); for receipt_json in receipts_json.lines() { @@ -53,42 +78,37 @@ async fn test_import_offline_snapshot() { receipts.insert(receipt.hash(), receipt); } - // init snapshot + // init snapshot data let snapshot_json = include_str!("fixtures/block-292973/snapshot.json"); let snapshot: InMemoryPermanentStorageState = serde_json::from_str(snapshot_json).unwrap(); - - // init postgres from snapshot - let (_pg_container, pg) = populate_postgres(&docker, snapshot).await; - let storage = Arc::new(StratusStorage::new(Arc::new(InMemoryTemporaryStorage::default()), Arc::new(pg))); + let pg = Postgres::new(docker.postgres_connection_url()).await.unwrap(); + populate_postgres(&pg, snapshot).await; // init executor and execute + let storage = Arc::new(StratusStorage::new(Arc::new(InMemoryTemporaryStorage::default()), Arc::new(pg))); let executor = config.init_executor(storage); executor.import_offline(block, &receipts).await.unwrap(); - // get metrics from prometheus-exporter page because there is no simple way to get them from the code - // for now just print them - let metrics = reqwest::get("http://localhost:9000/").await.unwrap().text().await.unwrap(); - for line in metrics.lines() { - for metric in TRACKED_METRICS { - if line.starts_with(metric) { - println!("{}", line); - } + // get metrics from prometheus (sleep to ensure prometheus collect metrics) + tokio::time::sleep(Duration::from_secs(2)).await; + for query in METRIC_QUERIES { + // formatting between query groups + if query.is_empty() { + println!("\n--------------------"); + continue; + } + + // get metrics and print + let url = format!("{}?query={}", docker.prometheus_api_url(), query); + let response = reqwest::get(url).await.unwrap().json::().await.unwrap(); + for result in response.get("data").unwrap().get("result").unwrap().as_array().unwrap() { + let value = result.get("value").unwrap().as_array().unwrap().last().unwrap().as_str().unwrap(); + println!("{:<64} = {}", query, value); } } } -async fn populate_postgres(docker: &clients::Cli, state: InMemoryPermanentStorageState) -> (Container<'_, PostgresImage>, Postgres) { - // init docker container (this should be extract to a reusable module if going to be used in other tests) - let pg_image = RunnableImage::from(PostgresImage::default().with_user("postgres").with_password("123").with_db_name("stratus")) - .with_mapped_port((5432, 5432)) - .with_volume(("./static/schema/001-init.sql", "/docker-entrypoint-initdb.d/001-schema.sql")) - .with_volume(("./static/schema/002-schema-external-rpc.sql", "/docker-entrypoint-initdb.d/002-schema.sql")) - .with_tag("16.1"); - let pg_container = docker.run(pg_image); - - // init postgres client - let pg = Postgres::new("postgres://postgres:123@localhost:5432/stratus").await.unwrap(); - +async fn populate_postgres(pg: &Postgres, state: InMemoryPermanentStorageState) { // save accounts let accounts = state.accounts.values().map(|a| a.to_account(&StoragePointInTime::Present)).collect_vec(); pg.save_accounts(accounts).await.unwrap(); @@ -98,6 +118,12 @@ async fn populate_postgres(docker: &clients::Cli, state: InMemoryPermanentStorag for account in state.accounts.values() { for slot_history in account.slots.values() { let slot = slot_history.get_current(); + + // we do not insert zero value slots because they were not really present in the storage when the snapshot was taken. + if slot.is_zero() { + continue; + } + sqlx::query("insert into account_slots(idx, value, account_address, creation_block) values($1, $2, $3, $4)") .bind(slot.index) .bind(slot.value) @@ -109,6 +135,4 @@ async fn populate_postgres(docker: &clients::Cli, state: InMemoryPermanentStorag } } tx.commit().await.unwrap(); - - (pg_container, pg) }