diff --git a/src/prometheus/alert.rs b/src/prometheus/alert.rs index b673f51..c618020 100644 --- a/src/prometheus/alert.rs +++ b/src/prometheus/alert.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use crate::{ crd::{ReplicaAlert, ServiceAlertSpec}, - prometheus::replica_alerts::replica_count_rules, + prometheus::{http_alerts::http_rules, replica_alerts::replica_count_rules}, }; #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] @@ -88,6 +88,10 @@ impl TryFrom for PromAlerts { }); } + if spec.alerts.rest.is_some() { + alerts.groups.push(http_rules(&spec)) + } + Ok(alerts) } } diff --git a/src/prometheus/http_alerts.rs b/src/prometheus/http_alerts.rs new file mode 100644 index 0000000..406898e --- /dev/null +++ b/src/prometheus/http_alerts.rs @@ -0,0 +1,169 @@ +use crate::crd::{AlertConfig, NetworkAlert, Operation, ServiceAlertSpec}; + +use super::alert::{AlertGroup, AlertRules, Annotations, Labels, PrometheusSeverity}; + +pub fn http_rules(spec: &ServiceAlertSpec) -> AlertGroup { + let mut rules: Vec = vec![]; + + if let Some(rest_alerts) = &spec.alerts.rest { + rest_alerts.iter().for_each(|(key, val)| match key { + NetworkAlert::ErrorPercent => rules.append(&mut error_percent_alerts(spec, val)), + NetworkAlert::TrafficPerSecond => { + rules.append(&mut traffic_per_second_alerts(spec, val)) + } + NetworkAlert::LatencyMillisecondsP50 => { + rules.append(&mut latency_percentile_alerts(spec, 50, val)) + } + NetworkAlert::LatencyMillisecondsP90 => { + rules.append(&mut latency_percentile_alerts(spec, 90, val)) + } + NetworkAlert::LatencyMillisecondsP95 => { + rules.append(&mut latency_percentile_alerts(spec, 95, val)) + } + NetworkAlert::LatencyMillisecondsP99 => { + rules.append(&mut latency_percentile_alerts(spec, 99, val)) + } + }) + } + + AlertGroup { + name: String::from("HTTP Alerts"), + rules, + } +} + +fn error_percent_alerts(spec: &ServiceAlertSpec, alert_configs: &[AlertConfig]) -> Vec { + alert_configs + .iter() + .enumerate() + .map(|(i, conf)| AlertRules { + alert: format!("HTTPErrorPercentRule-{0}-{1}", spec.deployment_name, i), + expr: format!(r#"error percent {} {}"#, conf.operation, i), + for_: conf.for_.clone(), + labels: Labels { + severity: PrometheusSeverity::from(&conf.with_labels), + source: spec.common_labels.origin.clone(), + owner: spec.common_labels.owner.clone(), + }, + annotations: error_percent_annotations(conf), + }) + .collect() +} + +fn error_percent_annotations(alert_config: &AlertConfig) -> Annotations { + match alert_config.operation { + Operation::EqualTo => Annotations { + summary: String::from("Request errors percentage reached alert boundary"), + description: format!( + "Current error percentage is exactly {}%", + alert_config.value + ), + }, + Operation::LessThan => Annotations { + summary: String::from("Request errors percentage is less than alert boundary"), + description: format!( + "Current error percentage is {{{{ $value }}}}%, boundary is {}", + alert_config.value + ), + }, + Operation::MoreThan => Annotations { + summary: String::from("Request errors percentage is higher than alert boundary"), + description: format!( + "Current error percentage is {{{{ $value }}}}%, boundary is {}", + alert_config.value + ), + }, + } +} + +fn latency_percentile_alerts( + spec: &ServiceAlertSpec, + percentile: i8, + alert_configs: &[AlertConfig], +) -> Vec { + alert_configs + .iter() + .enumerate() + .map(|(i, conf)| AlertRules { + alert: format!("HTTPLatencyPercentileRule-{0}-{1}", spec.deployment_name, i), + expr: format!(r#"latency percentile {} {}"#, conf.operation, percentile), + for_: conf.for_.clone(), + labels: Labels { + severity: PrometheusSeverity::from(&conf.with_labels), + source: spec.common_labels.origin.clone(), + owner: spec.common_labels.owner.clone(), + }, + annotations: latency_percentile_annotations(conf), + }) + .collect() +} + +fn latency_percentile_annotations(alert_config: &AlertConfig) -> Annotations { + match alert_config.operation { + Operation::EqualTo => Annotations { + summary: String::from("Average request latency reached alert boundary"), + description: format!( + "Current request latency is exactly {}ms", + alert_config.value + ), + }, + Operation::LessThan => Annotations { + summary: String::from("Average request latency is less than alert boundary"), + description: format!( + "Current request latency is {{{{ $value }}}}ms, boundary is {}ms", + alert_config.value + ), + }, + Operation::MoreThan => Annotations { + summary: String::from("Average request latency is higher than alert boundary"), + description: format!( + "Current request latency is {{{{ $value }}}}ms, boundary is {}ms", + alert_config.value + ), + }, + } +} + +fn traffic_per_second_alerts( + spec: &ServiceAlertSpec, + alert_configs: &[AlertConfig], +) -> Vec { + alert_configs + .iter() + .enumerate() + .map(|(i, conf)| AlertRules { + alert: format!("HTTPTrafficPerSecondRule-{0}-{1}", spec.deployment_name, i), + expr: format!(r#"traffic per second {} {}"#, conf.operation, i), + for_: conf.for_.clone(), + labels: Labels { + severity: PrometheusSeverity::from(&conf.with_labels), + source: spec.common_labels.origin.clone(), + owner: spec.common_labels.owner.clone(), + }, + annotations: traffic_per_second_annotations(conf), + }) + .collect() +} + +fn traffic_per_second_annotations(alert_config: &AlertConfig) -> Annotations { + match alert_config.operation { + Operation::EqualTo => Annotations { + summary: String::from("HTTP requests per second reached alert boundary"), + description: format!("Requests per second is exactly {}/s", alert_config.value), + }, + Operation::LessThan => Annotations { + summary: String::from("HTTP requests per second is less than alert boundary"), + description: format!( + "Requests per second is {{{{ $value }}}}/s, boundary is {}/s", + alert_config.value + ), + }, + Operation::MoreThan => Annotations { + summary: String::from("HTTP requests per second is higher than alert boundary"), + description: format!( + "Requests per second is {{{{ $value }}}}/s, boundary is {}/s", + alert_config.value + ), + }, + } +} diff --git a/src/prometheus/mod.rs b/src/prometheus/mod.rs index e6290cf..9338dcd 100644 --- a/src/prometheus/mod.rs +++ b/src/prometheus/mod.rs @@ -6,6 +6,7 @@ //! Prometheus alert that Cactuar can produce as a Kubernetes `ConfigMap`. pub mod alert; +pub mod http_alerts; pub mod replica_alerts; #[cfg(test)]