From df7b8dc51152a06511df1bced3dc5b2c87638da0 Mon Sep 17 00:00:00 2001 From: Sergii Mikhtoniuk Date: Thu, 12 Sep 2024 20:15:45 -0700 Subject: [PATCH] Observability improvements --- Cargo.lock | 280 +++++++++++---------- Cargo.toml | 87 ++++--- src/app/api-server/Cargo.toml | 20 +- src/app/api-server/src/app.rs | 45 ++-- src/app/api-server/src/cli.rs | 111 ++++++++ src/app/api-server/src/cli_parser.rs | 77 ------ src/app/api-server/src/flightsql_server.rs | 22 ++ src/app/api-server/src/http_server.rs | 8 +- src/app/api-server/src/lib.rs | 3 +- src/app/api-server/src/main.rs | 14 +- src/app/oracle-provider/Cargo.toml | 4 + src/utils/observability/Cargo.toml | 70 ------ src/utils/observability/src/axum.rs | 149 ----------- src/utils/observability/src/config.rs | 84 ------- src/utils/observability/src/health.rs | 87 ------- src/utils/observability/src/init.rs | 182 -------------- src/utils/observability/src/lib.rs | 17 -- src/utils/observability/src/metrics.rs | 23 -- src/utils/observability/src/tracing.rs | 63 ----- 19 files changed, 381 insertions(+), 965 deletions(-) create mode 100644 src/app/api-server/src/cli.rs delete mode 100644 src/app/api-server/src/cli_parser.rs delete mode 100644 src/utils/observability/Cargo.toml delete mode 100644 src/utils/observability/src/axum.rs delete mode 100644 src/utils/observability/src/config.rs delete mode 100644 src/utils/observability/src/health.rs delete mode 100644 src/utils/observability/src/init.rs delete mode 100644 src/utils/observability/src/lib.rs delete mode 100644 src/utils/observability/src/metrics.rs delete mode 100644 src/utils/observability/src/tracing.rs diff --git a/Cargo.lock b/Cargo.lock index f4277f24..45642ca0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,9 +133,9 @@ dependencies = [ [[package]] name = "alloy-chains" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4f201b0ac8f81315fbdc55269965a8ddadbc04ab47fa65a1a468f9a40f7a5f" +checksum = "b68b94c159bcc2ca5f758b8663d7b00fc7c5e40569984595ddf2221b0f7f7f6e" dependencies = [ "num_enum", "strum 0.26.3", @@ -629,7 +629,7 @@ dependencies = [ "alloy-transport", "futures", "http 1.1.0", - "rustls 0.23.12", + "rustls 0.23.13", "serde_json", "tokio", "tokio-tungstenite 0.23.1", @@ -703,9 +703,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.87" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8" +checksum = "4e1496f8fb1fbf272686b8d37f523dab3e4a7443300055e74cdaa449f3114356" [[package]] name = "approx" @@ -2332,8 +2332,8 @@ checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "container-runtime" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "cfg-if", @@ -2616,8 +2616,8 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "database-common" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "aws-config", @@ -2641,8 +2641,8 @@ dependencies = [ [[package]] name = "database-common-macros" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "quote", "syn 2.0.77", @@ -3281,11 +3281,11 @@ dependencies = [ [[package]] name = "enum-as-inner" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.77", @@ -3293,8 +3293,8 @@ dependencies = [ [[package]] name = "enum-variants" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" [[package]] name = "env_filter" @@ -3359,8 +3359,8 @@ dependencies = [ [[package]] name = "event-sourcing" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-stream", "async-trait", @@ -3374,8 +3374,8 @@ dependencies = [ [[package]] name = "event-sourcing-macros" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "quote", "syn 2.0.77", @@ -4007,8 +4007,8 @@ dependencies = [ [[package]] name = "http-common" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "axum", "http 0.2.12", @@ -4113,7 +4113,7 @@ dependencies = [ "http 1.1.0", "hyper 1.4.1", "hyper-util", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", @@ -4292,8 +4292,8 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "internal-error" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "thiserror", ] @@ -4414,8 +4414,8 @@ dependencies = [ [[package]] name = "kamu" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-recursion", "async-stream", @@ -4485,8 +4485,8 @@ dependencies = [ [[package]] name = "kamu-accounts" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "base32", @@ -4512,8 +4512,8 @@ dependencies = [ [[package]] name = "kamu-accounts-inmem" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -4529,8 +4529,8 @@ dependencies = [ [[package]] name = "kamu-accounts-postgres" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -4547,8 +4547,8 @@ dependencies = [ [[package]] name = "kamu-accounts-services" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "argon2", "async-trait", @@ -4572,8 +4572,8 @@ dependencies = [ [[package]] name = "kamu-accounts-sqlite" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -4590,8 +4590,8 @@ dependencies = [ [[package]] name = "kamu-adapter-auth-oso" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "dill", @@ -4607,8 +4607,8 @@ dependencies = [ [[package]] name = "kamu-adapter-flight-sql" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "arrow-flight", "async-trait", @@ -4625,8 +4625,8 @@ dependencies = [ [[package]] name = "kamu-adapter-graphql" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-graphql", "async-trait", @@ -4660,8 +4660,8 @@ dependencies = [ [[package]] name = "kamu-adapter-http" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "aws-sdk-s3", @@ -4707,8 +4707,8 @@ dependencies = [ [[package]] name = "kamu-adapter-oauth" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -4725,8 +4725,8 @@ dependencies = [ [[package]] name = "kamu-adapter-odata" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "axum", "chrono", @@ -4810,6 +4810,7 @@ dependencies = [ "messaging-outbox", "observability", "opendatafabric", + "prometheus", "rand", "secrecy", "serde", @@ -4833,8 +4834,8 @@ dependencies = [ [[package]] name = "kamu-auth-rebac" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "internal-error", @@ -4846,8 +4847,8 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-inmem" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "dill", @@ -4857,8 +4858,8 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-services" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "dill", @@ -4873,8 +4874,8 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-sqlite" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "database-common", @@ -4887,8 +4888,8 @@ dependencies = [ [[package]] name = "kamu-core" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-stream", "async-trait", @@ -4918,8 +4919,8 @@ dependencies = [ [[package]] name = "kamu-data-utils" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "arrow", "arrow-digest", @@ -4940,8 +4941,8 @@ dependencies = [ [[package]] name = "kamu-datasets" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "aes-gcm", "async-trait", @@ -4960,8 +4961,8 @@ dependencies = [ [[package]] name = "kamu-datasets-inmem" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -4979,8 +4980,8 @@ dependencies = [ [[package]] name = "kamu-datasets-postgres" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -4998,8 +4999,8 @@ dependencies = [ [[package]] name = "kamu-datasets-services" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -5020,8 +5021,8 @@ dependencies = [ [[package]] name = "kamu-datasets-sqlite" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -5039,8 +5040,8 @@ dependencies = [ [[package]] name = "kamu-flow-system" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -5067,8 +5068,8 @@ dependencies = [ [[package]] name = "kamu-flow-system-inmem" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-stream", "async-trait", @@ -5091,8 +5092,8 @@ dependencies = [ [[package]] name = "kamu-flow-system-postgres" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-stream", "async-trait", @@ -5109,13 +5110,14 @@ dependencies = [ [[package]] name = "kamu-flow-system-services" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-stream", "async-trait", "chrono", "database-common", + "database-common-macros", "dill", "futures", "internal-error", @@ -5138,8 +5140,8 @@ dependencies = [ [[package]] name = "kamu-flow-system-sqlite" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-stream", "async-trait", @@ -5156,8 +5158,8 @@ dependencies = [ [[package]] name = "kamu-ingest-datafusion" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -5186,8 +5188,8 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-inmem" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -5202,8 +5204,8 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-postgres" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-stream", "async-trait", @@ -5221,8 +5223,8 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-sqlite" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-stream", "async-trait", @@ -5287,8 +5289,8 @@ dependencies = [ [[package]] name = "kamu-task-system" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -5306,8 +5308,8 @@ dependencies = [ [[package]] name = "kamu-task-system-inmem" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -5320,8 +5322,8 @@ dependencies = [ [[package]] name = "kamu-task-system-postgres" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-stream", "async-trait", @@ -5338,12 +5340,13 @@ dependencies = [ [[package]] name = "kamu-task-system-services" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-stream", "async-trait", "database-common", + "database-common-macros", "dill", "futures", "internal-error", @@ -5351,6 +5354,7 @@ dependencies = [ "kamu-datasets", "kamu-task-system", "messaging-outbox", + "observability", "opendatafabric", "serde_json", "time-source", @@ -5360,8 +5364,8 @@ dependencies = [ [[package]] name = "kamu-task-system-sqlite" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-stream", "async-trait", @@ -5387,9 +5391,9 @@ dependencies = [ [[package]] name = "keccak-asm" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422fbc7ff2f2f5bdffeb07718e5a5324dca72b0c9293d50df4026652385e3314" +checksum = "505d1856a39b200489082f90d897c3f07c455563880bc5952e38eabf731c83b6" dependencies = [ "digest 0.10.7", "sha3-asm", @@ -5691,16 +5695,19 @@ dependencies = [ [[package]] name = "messaging-outbox" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", "database-common", + "database-common-macros", "dill", "futures", "internal-error", "mockall", + "observability", + "prometheus", "serde", "serde_json", "thiserror", @@ -5800,8 +5807,8 @@ dependencies = [ [[package]] name = "multiformats" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "base64 0.22.1", "bs58", @@ -6024,7 +6031,8 @@ dependencies = [ [[package]] name = "observability" -version = "0.36.0" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "axum", @@ -6048,9 +6056,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "33ea5043e58958ee56f3e15a90aee535795cd7dfd319846288d93c5b57d85cbe" [[package]] name = "opaque-debug" @@ -6060,8 +6068,8 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendatafabric" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "arrow", "base64 0.22.1", @@ -6864,7 +6872,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.12", + "rustls 0.23.13", "socket2", "thiserror", "tokio", @@ -6881,7 +6889,7 @@ dependencies = [ "rand", "ring", "rustc-hash", - "rustls 0.23.12", + "rustls 0.23.13", "slab", "thiserror", "tinyvec", @@ -6957,17 +6965,17 @@ dependencies = [ [[package]] name = "random-names" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "rand", ] [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags 2.6.0", ] @@ -7098,7 +7106,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-native-certs 0.7.3", "rustls-pemfile 2.1.3", "rustls-pki-types", @@ -7283,9 +7291,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.36" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno", @@ -7308,14 +7316,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.7", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] @@ -7395,9 +7403,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -7698,9 +7706,9 @@ dependencies = [ [[package]] name = "sha3-asm" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d79b758b7cb2085612b11a235055e485605a5103faccdd633f35bd7aee69dd" +checksum = "c28efc5e327c837aa837c59eae585fc250715ef939ac32881bcc11677cd02d46" dependencies = [ "cc", "cfg-if", @@ -7936,7 +7944,7 @@ dependencies = [ "once_cell", "paste", "percent-encoding", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pemfile 2.1.3", "serde", "serde_json", @@ -8444,8 +8452,8 @@ dependencies = [ [[package]] name = "time-source" -version = "0.199.2" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.199.2#b632ce11ffd05ed790a2fb7635cc70c3f880161b" +version = "0.200.0" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.200.0#cd2840bf2fb85749cb67da0b5af31cb9f831d702" dependencies = [ "async-trait", "chrono", @@ -8532,7 +8540,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pki-types", "tokio", ] @@ -8572,7 +8580,7 @@ checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" dependencies = [ "futures-util", "log", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -8913,7 +8921,7 @@ dependencies = [ "httparse", "log", "rand", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pki-types", "sha1", "thiserror", @@ -8986,9 +8994,9 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" @@ -9007,9 +9015,9 @@ checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" diff --git a/Cargo.toml b/Cargo.toml index fdb69864..3fec81c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,6 @@ members = [ # Utils "src/utils/graceful-shutdown", - "src/utils/observability", "src/utils/repo-tools", # Apps "src/app/api-server", @@ -14,52 +13,52 @@ resolver = "2" [workspace.dependencies] # Utils graceful-shutdown = { path = "src/utils/graceful-shutdown", version = "0.36.0", default-features = false } -observability = { path = "src/utils/observability", version = "0.36.0", default-features = false } # Utils (core) -container-runtime = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -database-common = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -database-common-macros = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -http-common = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -internal-error = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -messaging-outbox = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -random-names = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -time-source = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } +container-runtime = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +database-common = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +database-common-macros = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +http-common = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +internal-error = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +messaging-outbox = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +observability = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +random-names = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +time-source = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } # Domain -opendatafabric = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-task-system = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-task-system-services = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-flow-system = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-flow-system-services = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-accounts = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-datasets = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } +opendatafabric = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-task-system = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-task-system-services = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-flow-system = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-flow-system-services = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-accounts = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-datasets = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } # Infra -kamu = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-task-system-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-task-system-postgres = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-task-system-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-flow-system-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-flow-system-postgres = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-flow-system-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-adapter-graphql = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-adapter-http = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-adapter-oauth = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-adapter-odata = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-adapter-auth-oso = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-adapter-flight-sql = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-accounts-services = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-accounts-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-accounts-postgres = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-accounts-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-datasets-services = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-datasets-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-datasets-postgres = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-datasets-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-messaging-outbox-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-messaging-outbox-postgres = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-messaging-outbox-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-auth-rebac-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-auth-rebac-services = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } -kamu-auth-rebac-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.199.2", version = "0.199.2", default-features = false } +kamu = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-task-system-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-task-system-postgres = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-task-system-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-flow-system-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-flow-system-postgres = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-flow-system-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-adapter-graphql = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-adapter-http = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-adapter-oauth = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-adapter-odata = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-adapter-auth-oso = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-adapter-flight-sql = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-accounts-services = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-accounts-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-accounts-postgres = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-accounts-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-datasets-services = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-datasets-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-datasets-postgres = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-datasets-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-messaging-outbox-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-messaging-outbox-postgres = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-messaging-outbox-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-auth-rebac-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-auth-rebac-services = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } +kamu-auth-rebac-sqlite = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.200.0", version = "0.200.0", default-features = false } [workspace.package] diff --git a/src/app/api-server/Cargo.toml b/src/app/api-server/Cargo.toml index 95c84283..e895bf6b 100644 --- a/src/app/api-server/Cargo.toml +++ b/src/app/api-server/Cargo.toml @@ -13,6 +13,10 @@ edition = { workspace = true } publish = { workspace = true } +[lib] +doctest = false + + [dependencies] dill = "0.9" container-runtime = { workspace = true } @@ -78,7 +82,8 @@ tonic = { version = "0.11", default-features = false } tower = { version = "0.4" } tower-http = { version = "0.4", features = ["trace", "cors"] } -# Tracing / logging / telemetry +# Tracing / logging / telemetry / metrics +prometheus = { version = "0.13", default-features = false } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = [ "std", @@ -91,7 +96,18 @@ tracing-log = "0.2" # Utils async-trait = { version = "0.1", default-features = false } chrono = "0.4" -clap = "4" +clap = { version = "4", default-features = false, features = [ + "std", + "color", + "help", + "usage", + "error-context", + "suggestions", + "derive", + "cargo", + # "env", + "wrap_help", +] } datafusion = { version = "41", default-features = false, features = [ "crypto_expressions", "encoding_expressions", diff --git a/src/app/api-server/src/app.rs b/src/app/api-server/src/app.rs index a12d3f8e..34eb93bb 100644 --- a/src/app/api-server/src/app.rs +++ b/src/app/api-server/src/app.rs @@ -26,6 +26,7 @@ use crate::config::{ ACCOUNT_KAMU, }; use crate::{ + cli, configure_database_components, configure_in_memory_components, connect_database_initially, @@ -41,7 +42,7 @@ const DEFAULT_RUST_LOG: &str = "debug,"; ///////////////////////////////////////////////////////////////////////////////////////// -pub async fn run(matches: clap::ArgMatches, config: ApiServerConfig) -> Result<(), InternalError> { +pub async fn run(args: cli::Cli, config: ApiServerConfig) -> Result<(), InternalError> { let repo_url = if let Some(repo_url) = config.repo.repo_url.as_ref().cloned() { repo_url } else { @@ -55,7 +56,7 @@ pub async fn run(matches: clap::ArgMatches, config: ApiServerConfig) -> Result<( Url::from_directory_path(workspace_dir.join("datasets")).unwrap() }; - let multi_tenant = matches.get_flag("multi-tenant") || repo_url.scheme() != "file"; + let multi_tenant = args.multi_tenant || repo_url.scheme() != "file"; let local_dir = tempfile::tempdir().unwrap(); @@ -90,6 +91,9 @@ pub async fn run(matches: clap::ArgMatches, config: ApiServerConfig) -> Result<( .bind::() .build(); + // Register metrics + let metrics_registry = observability::metrics::register_all(&catalog); + // Database requires extra actions: let final_catalog = if db_config.needs_database() { // Connect database and obtain a connection pool @@ -105,30 +109,29 @@ pub async fn run(matches: clap::ArgMatches, config: ApiServerConfig) -> Result<( initialize_components(&final_catalog, server_account_subject.clone()).await; - match matches.subcommand() { - Some(("gql", sub)) => match sub.subcommand() { - Some(("schema", _)) => { + match args.command { + cli::Command::Gql(c) => match c.subcommand { + cli::Gql::Schema(_) => { println!("{}", kamu_adapter_graphql::schema().sdl()); Ok(()) } - Some(("query", qsub)) => { - let result = crate::gql_server::gql_query( - qsub.get_one("query").map(String::as_str).unwrap(), - qsub.get_flag("full"), - final_catalog, - ) - .await; + cli::Gql::Query(sc) => { + let result = crate::gql_server::gql_query(&sc.query, sc.full, final_catalog).await; print!("{}", result); Ok(()) } - _ => unimplemented!(), }, - Some(("run", sub)) => { + cli::Command::Metrics(_) => { + // TODO: Proper implementation is blocked by https://github.com/tikv/rust-prometheus/issues/526 + let metric_families = metrics_registry.gather(); + println!("{metric_families:#?}"); + Ok(()) + } + cli::Command::Run(c) => { let shutdown_requested = graceful_shutdown::trap_signals(); - let address = sub - .get_one::("address") - .copied() + let address = c + .address .unwrap_or(std::net::Ipv4Addr::new(127, 0, 0, 1).into()); // API servers are built from the regular catalog @@ -137,14 +140,14 @@ pub async fn run(matches: clap::ArgMatches, config: ApiServerConfig) -> Result<( // all processing in the user context. let http_server = crate::http_server::build_server( address, - sub.get_one("http-port").copied(), + c.http_port, final_catalog.clone(), multi_tenant, ); let flightsql_server = crate::flightsql_server::FlightSqlServer::new( address, - sub.get_one("flightsql-port").copied(), + c.flightsql_port, final_catalog.clone(), ) .await; @@ -203,7 +206,6 @@ pub async fn run(matches: clap::ArgMatches, config: ApiServerConfig) -> Result<( res = outbox_processor.run() => { res.int_err() }, } } - _ => unimplemented!(), } } @@ -293,6 +295,8 @@ pub async fn init_dependencies( b.add::(); + b.add_value(prometheus::Registry::new()); + b.add::(); b.add_value(container_runtime::ContainerRuntimeConfig { runtime: config.engine.runtime, @@ -369,6 +373,7 @@ pub async fn init_dependencies( b.add::(); b.bind::(); b.add::(); + b.add::(); messaging_outbox::register_message_dispatcher::( &mut b, diff --git a/src/app/api-server/src/cli.rs b/src/app/api-server/src/cli.rs new file mode 100644 index 00000000..507613a5 --- /dev/null +++ b/src/app/api-server/src/cli.rs @@ -0,0 +1,111 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::path::PathBuf; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, clap::Parser)] +#[command(name = crate::BINARY_NAME)] +#[command(version = crate::VERSION)] +#[command(after_help = r#" +To get help for individual commands use: + kamu -h + kamu -h +"#)] +pub struct Cli { + /// Path to the config file + #[arg(long, default_value = "config.yaml")] + pub config: Option, + + // TODO: This is temporary and will be removed soon + // See: https://github.com/kamu-data/kamu-cli/issues/342 + /// Indicates that target repo is multi-tenant (for file:// only) + #[arg(long)] + pub multi_tenant: bool, + + #[command(subcommand)] + pub command: Command, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, clap::Subcommand)] +pub enum Command { + Run(Run), + Gql(GqlGroup), + Metrics(Metrics), +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Run the server +#[derive(Debug, clap::Args)] +pub struct Run { + /// Expose HTTP server on specific network interface + #[arg(long)] + pub address: Option, + + /// Expose HTTP server on specific port + #[arg(long)] + pub http_port: Option, + + /// Expose Flight SQL server on specific port + #[arg(long)] + pub flightsql_port: Option, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// GraphQL related command group +#[derive(Debug, clap::Args)] +pub struct GqlGroup { + #[command(subcommand)] + pub subcommand: Gql, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, clap::Subcommand)] +pub enum Gql { + Schema(GqlSchema), + Query(GqlQuery), +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Prints out GraphQL schema +#[derive(Debug, clap::Args)] +pub struct GqlSchema {} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Executes the GraphQL query and prints out the result +#[derive(Debug, clap::Args)] +#[command(after_help = r#" +Example: + kamu-api-server gql query '{ apiVersion }' +"#)] +pub struct GqlQuery { + /// Display the full result including extensions + #[arg(long)] + pub full: bool, + + /// GQL query + #[arg(index = 1)] + pub query: String, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Prints out GraphQL schema +#[derive(Debug, clap::Args)] +pub struct Metrics {} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/app/api-server/src/cli_parser.rs b/src/app/api-server/src/cli_parser.rs deleted file mode 100644 index 6dc6bbb3..00000000 --- a/src/app/api-server/src/cli_parser.rs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use std::path::PathBuf; - -use clap::*; -use indoc::indoc; - -pub fn cli() -> Command { - Command::new(crate::BINARY_NAME) - .subcommand_required(true) - .arg_required_else_help(true) - .version(crate::VERSION) - .after_help(indoc!( - r" - To get help around individual commands use: - kamu-api-server -h - kamu-api-server -h - " - )) - .args([ - Arg::new("config") - .long("config") - .value_parser(value_parser!(PathBuf)) - .help("Path to the config file"), - // TODO: This is temporary and will be removed soon - // See: https://github.com/kamu-data/kamu-cli/issues/342 - Arg::new("multi-tenant") - .long("multi-tenant") - .action(ArgAction::SetTrue) - .help("Indicates that target repo is multi-tenant (for file:// only)"), - ]) - .subcommands([ - Command::new("run").about("Run the server").args([ - Arg::new("address") - .long("address") - .value_parser(value_parser!(std::net::IpAddr)) - .help("Expose HTTP server on specific network interface"), - Arg::new("http-port") - .long("http-port") - .value_parser(value_parser!(u16)) - .help("Expose HTTP server on specific port"), - Arg::new("flightsql-port") - .long("flightsql-port") - .value_parser(value_parser!(u16)) - .help("Expose Flight SQL server on specific port"), - ]), - Command::new("gql") - .about("GraphQL related command group") - .subcommand_required(true) - .arg_required_else_help(true) - .subcommands([ - Command::new("schema").about("Prints out GraphQL schema"), - Command::new("query") - .about("Executes the GraphQL query and prints out the result") - .args(&[ - Arg::new("full") - .long("full") - .action(ArgAction::SetTrue) - .help("Display the full result including extensions"), - Arg::new("query").index(1).required(true), - ]) - .after_help(indoc!( - r" - Example: - kamu-api-server gql query '{ apiVersion }' - " - )), - ]), - ]) -} diff --git a/src/app/api-server/src/flightsql_server.rs b/src/app/api-server/src/flightsql_server.rs index cb6b7bf1..f0d0ada6 100644 --- a/src/app/api-server/src/flightsql_server.rs +++ b/src/app/api-server/src/flightsql_server.rs @@ -56,6 +56,7 @@ impl FlightSqlServer { pub fn run(self) -> impl Future> { Server::builder() + .trace_fn(trace_grpc_request) .add_service(FlightServiceServer::new(self.service)) .serve_with_incoming(tokio_stream::wrappers::TcpListenerStream::new( self.listener, @@ -65,6 +66,27 @@ impl FlightSqlServer { ///////////////////////////////////////////////////////////////////////////////////////// +// TODO: Move to `kamu-adapter-flightsql` +fn trace_grpc_request(request: &http::Request<()>) -> tracing::Span { + let (service, method) = request + .uri() + .path() + .strip_prefix('/') + .and_then(|s| s.split_once('/')) + .unzip(); + + let otel_name = request.uri().path().strip_prefix('/'); + + observability::tracing::root_span!( + "flightsql_request", + service = service.unwrap_or_default(), + method = method.unwrap_or_default(), + "otel.name" = otel_name.unwrap_or_default(), + ) +} + +///////////////////////////////////////////////////////////////////////////////////////// + struct SessionFactoryImpl { base_catalog: dill::Catalog, } diff --git a/src/app/api-server/src/http_server.rs b/src/app/api-server/src/http_server.rs index b827f7e7..09b9193a 100644 --- a/src/app/api-server/src/http_server.rs +++ b/src/app/api-server/src/http_server.rs @@ -75,12 +75,16 @@ pub(crate) fn build_server( .allow_headers(tower_http::cors::Any), ) .layer(observability::axum::http_layer()) - // Note: Healthcheck route is placed before the tracing layer (as layers execute bottom-up) - // to avoid spam in logs + // Note: Healthcheck and metrics routes are placed before the tracing layer (layers + // execute bottom-up) to avoid spam in logs .route( "/system/health", axum::routing::get(observability::health::health_handler), ) + .route( + "/system/metrics", + axum::routing::get(observability::metrics::metrics_handler), + ) .layer(axum::extract::Extension(gql_schema)) .layer(axum::extract::Extension(catalog)); diff --git a/src/app/api-server/src/lib.rs b/src/app/api-server/src/lib.rs index 881c1657..118a3128 100644 --- a/src/app/api-server/src/lib.rs +++ b/src/app/api-server/src/lib.rs @@ -11,7 +11,7 @@ #![feature(let_chains)] pub mod app; -pub(crate) mod cli_parser; +pub mod cli; pub mod config; pub(crate) mod database; pub(crate) mod flightsql_server; @@ -19,5 +19,4 @@ pub(crate) mod gql_server; pub(crate) mod http_server; pub use app::*; -pub use cli_parser::*; pub(crate) use database::*; diff --git a/src/app/api-server/src/main.rs b/src/app/api-server/src/main.rs index f1793c75..30bf5b73 100644 --- a/src/app/api-server/src/main.rs +++ b/src/app/api-server/src/main.rs @@ -9,10 +9,13 @@ ///////////////////////////////////////////////////////////////////////////////////////// +use clap::Parser; +use kamu_api_server::cli::Cli; + fn main() { - let matches = kamu_api_server::cli().get_matches(); + let args = Cli::parse(); - let config = kamu_api_server::load_config(matches.get_one("config")).unwrap(); + let config = kamu_api_server::load_config(args.config.as_ref()).unwrap(); let mut builder = tokio::runtime::Builder::new_multi_thread(); @@ -28,15 +31,12 @@ fn main() { let rt = builder.enable_all().build().unwrap(); - let code = rt.block_on(main_async(matches, config)); + let code = rt.block_on(main_async(args, config)); std::process::exit(code) } -async fn main_async( - args: clap::ArgMatches, - config: kamu_api_server::config::ApiServerConfig, -) -> i32 { +async fn main_async(args: Cli, config: kamu_api_server::config::ApiServerConfig) -> i32 { use kamu_api_server::app; let _guard = app::init_observability(); diff --git a/src/app/oracle-provider/Cargo.toml b/src/app/oracle-provider/Cargo.toml index 39e41f57..f66986e1 100644 --- a/src/app/oracle-provider/Cargo.toml +++ b/src/app/oracle-provider/Cargo.toml @@ -13,6 +13,10 @@ edition = { workspace = true } publish = { workspace = true } +[lib] +doctest = false + + [dependencies] graceful-shutdown = { workspace = true, default-features = false } observability = { workspace = true, default-features = false, features = [ diff --git a/src/utils/observability/Cargo.toml b/src/utils/observability/Cargo.toml deleted file mode 100644 index abeebfc9..00000000 --- a/src/utils/observability/Cargo.toml +++ /dev/null @@ -1,70 +0,0 @@ -[package] -name = "observability" -description = "Utilities for tracing, structured logging, and metrics" -version = { workspace = true } -homepage = { workspace = true } -repository = { workspace = true } -authors = { workspace = true } -readme = { workspace = true } -license-file = { workspace = true } -keywords = { workspace = true } -include = { workspace = true } -edition = { workspace = true } -publish = { workspace = true } - - -[lints] -workspace = true - - -[lib] -doctest = false - - -[features] -default = [] - -opentelemetry = [ - "dep:opentelemetry", - "dep:opentelemetry_sdk", - "dep:opentelemetry-otlp", - "dep:opentelemetry-semantic-conventions", - "dep:tracing-opentelemetry", -] - -prometheus = ["dep:prometheus"] - - -[dependencies] -async-trait = { version = "0.1" } -axum = { version = "0.6", default-features = false, features = [ - "json", - "matched-path", - "query", -] } -dill = { version = "0.9", default-features = false } -http = { version = "0.2", default-features = false } -serde = { version = "1", default-features = false, features = ["derive"] } -serde_json = { version = "1", default-features = false } -thiserror = { version = "1", default-features = false } -tracing = { version = "0.1", default-features = false } -tracing-appender = { version = "0.2", default-features = false } -tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } -tower = { version = "0.4", default-features = false } -tower-http = { version = "0.4", default-features = false, features = ["trace"] } - -opentelemetry = { optional = true, version = "0.23", default-features = false } -opentelemetry_sdk = { optional = true, version = "0.23", default-features = false, features = [ - "rt-tokio", -] } -opentelemetry-otlp = { optional = true, version = "0.16", default-features = false, features = [ - "trace", - "grpc-tonic", -] } -opentelemetry-semantic-conventions = { optional = true, version = "0.16", default-features = false } -tracing-opentelemetry = { optional = true, version = "0.24", default-features = false } - -prometheus = { optional = true, version = "0.13", default-features = false } - - -[dev-dependencies] diff --git a/src/utils/observability/src/axum.rs b/src/utils/observability/src/axum.rs deleted file mode 100644 index 662bfa2d..00000000 --- a/src/utils/observability/src/axum.rs +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use http::Uri; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pub fn http_layer() -> tower_http::trace::TraceLayer< - tower_http::classify::SharedClassifier, - MakeSpan, - OnRequest, - OnResponse, -> { - tower_http::trace::TraceLayer::new_for_http() - .on_request(OnRequest) - .on_response(OnResponse) - .make_span_with(MakeSpan) -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Clone, Debug)] -pub struct OnRequest; - -impl tower_http::trace::OnRequest for OnRequest { - fn on_request(&mut self, request: &http::Request, _: &tracing::Span) { - tracing::info!( - uri = %request.uri(), - version = ?request.version(), - headers = ?request.headers(), - "HTTP request", - ); - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Clone, Debug)] -pub struct OnResponse; - -impl tower_http::trace::OnResponse for OnResponse { - fn on_response( - self, - response: &http::Response, - latency: std::time::Duration, - _span: &tracing::Span, - ) { - tracing::info!( - status = response.status().as_u16(), - headers = ?response.headers(), - latency = %Latency(latency), - "HTTP response" - ); - } -} - -struct Latency(std::time::Duration); - -impl std::fmt::Display for Latency { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{} ms", self.0.as_millis()) - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, Clone)] -pub struct MakeSpan; - -impl tower_http::trace::MakeSpan for MakeSpan { - fn make_span(&mut self, request: &http::Request) -> tracing::Span { - // TODO: Extract parent context from the request - - let method = request.method(); - let route = RouteOrUri::from(request); - - let span = crate::tracing::root_span!( - "http_request", - %method, - %route, - "otel.name" = tracing::field::Empty, - ); - - #[cfg(feature = "opentelemetry")] - { - crate::tracing::include_otel_trace_id(&span); - - span.record( - "otel.name", - tracing::field::display(SpanName::new(method, route)), - ); - } - - span - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -struct SpanName<'a> { - method: &'a http::Method, - route: RouteOrUri<'a>, -} - -impl<'a> SpanName<'a> { - fn new(method: &'a http::Method, route: RouteOrUri<'a>) -> Self { - Self { method, route } - } -} - -impl<'a> std::fmt::Display for SpanName<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{} {}", self.method, self.route) - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -enum RouteOrUri<'a> { - Route(&'a str), - Uri(&'a Uri), -} - -impl<'a, B> From<&'a http::Request> for RouteOrUri<'a> { - fn from(request: &'a http::Request) -> Self { - request - .extensions() - .get::() - .map_or_else( - || RouteOrUri::Uri(request.uri()), - |m| RouteOrUri::Route(m.as_str()), - ) - } -} - -impl<'a> std::fmt::Display for RouteOrUri<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - RouteOrUri::Route(s) => write!(f, "{s}"), - RouteOrUri::Uri(uri) => write!(f, "{uri}"), - } - } -} diff --git a/src/utils/observability/src/config.rs b/src/utils/observability/src/config.rs deleted file mode 100644 index 324c69a0..00000000 --- a/src/utils/observability/src/config.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug)] -pub struct Config { - /// Name of the service that will appear e.g. in OTEL traces - pub service_name: String, - /// Version of the service that will appear e.g. in OTEL traces - pub service_version: String, - /// Log levels that will be used if `RUST_LOG` was not specified explicitly - pub default_log_levels: String, - /// OpenTelemetry protocol endpoint to export traces to - pub otlp_endpoint: Option, -} - -impl Default for Config { - fn default() -> Self { - Self { - service_name: "unnamed-service".to_string(), - service_version: "0.0.0".to_string(), - default_log_levels: "info".to_string(), - otlp_endpoint: None, - } - } -} - -impl Config { - pub fn from_env() -> Self { - Self::from_env_with_prefix("") - } - - pub fn from_env_with_prefix(prefix: &str) -> Self { - let mut cfg = Self::default(); - if let Some(service_name) = std::env::var(format!("{prefix}SERVICE_NAME")) - .ok() - .filter(|v| !v.is_empty()) - { - cfg.service_name = service_name; - } - if let Some(service_version) = std::env::var(format!("{prefix}SERVICE_VERSION")) - .ok() - .filter(|v| !v.is_empty()) - { - cfg.service_version = service_version; - } - if let Some(otlp_endpoint) = std::env::var(format!("{prefix}OTLP_ENDPOINT")) - .ok() - .filter(|v| !v.is_empty()) - { - cfg.otlp_endpoint = Some(otlp_endpoint); - } - cfg - } - - pub fn with_service_name(mut self, service_name: impl Into) -> Self { - self.service_name = service_name.into(); - self - } - - pub fn with_service_version(mut self, service_version: impl Into) -> Self { - self.service_version = service_version.into(); - self - } - - pub fn with_default_log_levels(mut self, default_log_levels: impl Into) -> Self { - self.default_log_levels = default_log_levels.into(); - self - } - - pub fn with_otlp_endpoint(mut self, otlp_endpoint: impl Into) -> Self { - self.otlp_endpoint = Some(otlp_endpoint.into()); - self - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/observability/src/health.rs b/src/utils/observability/src/health.rs deleted file mode 100644 index f4929b18..00000000 --- a/src/utils/observability/src/health.rs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Type implementing this trait will be called during the health check -/// procedure. Multiple checks can be added, in which case they will be called -/// one by one with check considered failed upon the first error. -#[async_trait::async_trait] -pub trait HealthCheck: Send + Sync { - async fn check(&self, check_type: CheckType) -> Result<(), CheckError>; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Axum handler for serving the health checks. Depends on [`dill::Catalog`] to -/// instantiate the implementations of the [`HealthCheck`] trait. -pub async fn health_handler( - axum::Extension(catalog): axum::Extension, - axum::extract::Query(args): axum::extract::Query, -) -> Result, CheckError> { - for checker in catalog.get::>().unwrap() { - checker.check(args.r#type).await?; - } - - Ok(axum::Json(CheckSuccess { ok: true })) -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, serde::Serialize)] -#[serde(rename_all = "camelCase")] -pub struct CheckSuccess { - pub ok: bool, -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, thiserror::Error)] -#[error("{reason}")] -pub struct CheckError { - pub reason: String, -} - -impl axum::response::IntoResponse for CheckError { - fn into_response(self) -> axum::response::Response { - ( - http::status::StatusCode::SERVICE_UNAVAILABLE, - axum::Json(serde_json::json!({ - "ok": false, - "reason": self.reason, - })), - ) - .into_response() - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, serde::Deserialize, serde::Serialize)] -#[serde(rename_all = "lowercase")] -pub enum CheckType { - /// Determines when to restart the container - #[default] - Liveness, - /// Determines when to remove the instance from the loadbalancer - Readiness, - /// Is sent before liveness and readiness checks to determine when the - /// startup procedure is finished procedure is finished - Startup, -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, serde::Deserialize)] -pub struct CheckArgs { - #[serde(default)] - pub r#type: CheckType, -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/observability/src/init.rs b/src/utils/observability/src/init.rs deleted file mode 100644 index 2021c902..00000000 --- a/src/utils/observability/src/init.rs +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use std::io::IsTerminal; - -use tracing_subscriber::fmt::format::FmtSpan; -use tracing_subscriber::layer::SubscriberExt as _; -use tracing_subscriber::util::SubscriberInitExt as _; -use tracing_subscriber::EnvFilter; - -use super::config::Config; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/// If application is started under a terminal will use the [`dev`] mode, -/// otherwise will use [`service`] mode. -pub fn auto(cfg: Config) -> Guard { - if std::io::stderr().is_terminal() { - dev(cfg) - } else { - service(cfg) - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[allow(clippy::needless_pass_by_value)] -pub fn dev(cfg: Config) -> Guard { - let env_filter = - EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new(cfg.default_log_levels.clone())); - - let text_layer = tracing_subscriber::fmt::layer() - .pretty() - .with_writer(std::io::stderr) - .with_line_number(true) - .with_thread_names(true) - .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE); - - #[cfg(feature = "opentelemetry")] - let (otel_layer, otlp_guard) = if cfg.otlp_endpoint.is_none() { - (None, None) - } else { - ( - Some( - tracing_opentelemetry::layer() - .with_error_records_to_exceptions(true) - .with_tracer(init_otel_tracer(&cfg)), - ), - Some(OtlpGuard), - ) - }; - - #[cfg(feature = "opentelemetry")] - tracing_subscriber::registry() - .with(env_filter) - .with(otel_layer) - .with(text_layer) - .init(); - - #[cfg(not(feature = "opentelemetry"))] - tracing_subscriber::registry() - .with(env_filter) - .with(text_layer) - .init(); - - Guard { - non_blocking_appender: None, - - #[cfg(feature = "opentelemetry")] - otlp_guard, - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[allow(clippy::needless_pass_by_value)] -pub fn service(cfg: Config) -> Guard { - let env_filter = - EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new(cfg.default_log_levels.clone())); - - let (non_blocking, guard) = tracing_appender::non_blocking(std::io::stderr()); - - let text_layer = tracing_subscriber::fmt::layer() - .json() - .with_writer(non_blocking) - .with_line_number(true) - .with_thread_names(true) - .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE); - - #[cfg(feature = "opentelemetry")] - let (otel_layer, otlp_guard) = if cfg.otlp_endpoint.is_none() { - (None, None) - } else { - ( - Some( - tracing_opentelemetry::layer() - .with_error_records_to_exceptions(true) - .with_tracer(init_otel_tracer(&cfg)), - ), - Some(OtlpGuard), - ) - }; - - #[cfg(feature = "opentelemetry")] - tracing_subscriber::registry() - .with(env_filter) - .with(otel_layer) - .with(text_layer) - .init(); - - #[cfg(not(feature = "opentelemetry"))] - tracing_subscriber::registry() - .with(env_filter) - .with(text_layer) - .init(); - - Guard { - non_blocking_appender: Some(guard), - - #[cfg(feature = "opentelemetry")] - otlp_guard, - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[cfg(feature = "opentelemetry")] -fn init_otel_tracer(cfg: &Config) -> opentelemetry_sdk::trace::Tracer { - use std::time::Duration; - - use opentelemetry::KeyValue; - use opentelemetry_otlp::WithExportConfig as _; - use opentelemetry_semantic_conventions::resource as otel_resource; - - let otel_exporter = opentelemetry_otlp::new_exporter() - .tonic() - .with_endpoint(cfg.otlp_endpoint.as_ref().unwrap()) - .with_timeout(Duration::from_secs(5)); - - opentelemetry_otlp::new_pipeline() - .tracing() - .with_exporter(otel_exporter) - .with_trace_config( - opentelemetry_sdk::trace::config() - .with_max_events_per_span(64) - .with_max_attributes_per_span(16) - .with_resource(opentelemetry_sdk::Resource::new([ - KeyValue::new(otel_resource::SERVICE_NAME, cfg.service_name.clone()), - KeyValue::new(otel_resource::SERVICE_VERSION, cfg.service_version.clone()), - ])) - .with_sampler(opentelemetry_sdk::trace::Sampler::AlwaysOn), - ) - .install_batch(opentelemetry_sdk::runtime::Tokio) - .expect("Creating tracer") -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[must_use] -#[allow(dead_code)] -#[derive(Default)] -pub struct Guard { - pub non_blocking_appender: Option, - - #[cfg(feature = "opentelemetry")] - pub otlp_guard: Option, -} - -pub struct OtlpGuard; - -#[cfg(feature = "opentelemetry")] -impl Drop for OtlpGuard { - fn drop(&mut self) { - opentelemetry::global::shutdown_tracer_provider(); - } -} diff --git a/src/utils/observability/src/lib.rs b/src/utils/observability/src/lib.rs deleted file mode 100644 index 6106ac8d..00000000 --- a/src/utils/observability/src/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -pub mod axum; -pub mod config; -pub mod health; -pub mod init; -pub mod tracing; - -#[cfg(feature = "prometheus")] -pub mod metrics; diff --git a/src/utils/observability/src/metrics.rs b/src/utils/observability/src/metrics.rs deleted file mode 100644 index b2ef3f80..00000000 --- a/src/utils/observability/src/metrics.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use prometheus::Encoder as _; - -#[allow(clippy::unused_async)] -pub async fn metrics_handler(axum::Extension(catalog): axum::Extension) -> String { - let reg = catalog.get_one::().unwrap(); - - let mut buf = Vec::new(); - - prometheus::TextEncoder::new() - .encode(®.gather(), &mut buf) - .unwrap(); - - String::from_utf8(buf).unwrap() -} diff --git a/src/utils/observability/src/tracing.rs b/src/utils/observability/src/tracing.rs deleted file mode 100644 index 8483daa2..00000000 --- a/src/utils/observability/src/tracing.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -#[cfg(feature = "opentelemetry")] -#[macro_export] -macro_rules! root_span { - ($name:expr) => { - $crate::tracing::root_span!($name,) - }; - ($name:expr, $($field:tt)*) => { - { - let span = ::tracing::info_span!( - $name, - trace_id = tracing::field::Empty, - $($field)* - ); - - $crate::tracing::include_otel_trace_id(&span); - - span - } - }; -} - -#[cfg(not(feature = "opentelemetry"))] -#[macro_export] -macro_rules! root_span { - ($name:expr) => { - $crate::tracing::root_span!($name,) - }; - ($name:expr, $($field:tt)*) => { - tracing::info_span!( - $name, - trace_id = tracing::field::Empty, - $($field)* - ) - }; -} - -pub use root_span; - -/// Extracts trace ID from the OTEL layer and adds it to the tracing span to -/// allow cross-linking between logs and traces -#[cfg(feature = "opentelemetry")] -pub fn include_otel_trace_id(span: &tracing::Span) { - use opentelemetry::trace::TraceContextExt as _; - use tracing_opentelemetry::OpenTelemetrySpanExt as _; - - let context = span.context(); - let otel_span = context.span(); - let span_context = otel_span.span_context(); - let trace_id = span_context.trace_id(); - - if span_context.is_valid() { - span.record("trace_id", tracing::field::display(trace_id)); - } -}