feat: Basic prometheus exporter & documentation fix

keep/api/api.py

@@ -32,6 +32,7 @@
     healthcheck,
     incidents,
     mapping,
+    metrics,
     preset,
     providers,
     pusher,
@@ -196,6 +197,9 @@ def get_app(
     app.include_router(
         mapping.router, prefix="/mapping", tags=["enrichment", "mapping"]
     )
+    app.include_router(
+        metrics.router, prefix="/metrics", tags=["metrics"]
+    )
     app.include_router(
         extraction.router, prefix="/extraction", tags=["enrichment", "extraction"]
     )

keep/api/routes/metrics.py

@@ -0,0 +1,57 @@
+from fastapi import Response
+
+from fastapi import APIRouter, Depends
+from prometheus_client import CONTENT_TYPE_LATEST
+from keep.api.core.dependencies import AuthenticatedEntity, AuthVerifier
+
+from keep.api.core.db import (
+    get_last_incidents,
+)
+
+router = APIRouter()
+
+
+@router.get("")
+def get_metrics(
+    authenticated_entity: AuthenticatedEntity = Depends(AuthVerifier(["read:metrics"])),
+):
+    """
+    This endpoint is used by Prometheus to scrape such metrics from the application:
+    - alerts_total {incident_name, incident_id} - The total number of alerts per incident.
+    - open_incidents_total - The total number of open incidents
+
+    Please note that those metrics are per-tenant and are not designed to be used for the monitoring of the application itself.
+
+    Example prometheus configuration:
+    ```
+    scrape_configs:
+    - job_name: "scrape_keep"
+        static_configs:
+        - targets: ["https://api.keephq.dev/metrics"]  # Or your own domain.
+        authorization:
+            type: Bearer
+            credentials: "{Your API Key}"
+    ```
+    """
+    # We don't use im-memory metrics countrs here which is typical for prometheus exporters, 
+    # they would make us expose our app's pod id's. This is a customer-facing endpoing 
+    # we're deploying to SaaS, and we want to hide our internal infra.
+
+    tenant_id = authenticated_entity.tenant_id
+
+    export = str()
+
+    #Exporting alerts per incidents
+    export += "# HELP alerts_total The total number of alerts per incident.\n"
+    export += "# TYPE alerts_total counter\n"
+    incidents, incidents_total = get_last_incidents(tenant_id=tenant_id, limit=1000, is_confirmed=True, )
+    for incident in incidents:
+        export += f'alerts_total{{incident_name="{incident.name}" incident_id="{incident.id}"}} {incident.alerts_count}\n'
+
+    # Exporting stats about open incidents
+    export += "\n\n"
+    export += "# HELP open_incidents_total The total number of open incidents.\r\n"
+    export += "# TYPE open_incidents_total counter\n"
+    export += f'open_incidents_total {incidents_total}\n'
+
+    return Response(content=export, media_type=CONTENT_TYPE_LATEST)

pyproject.toml

@@ -91,6 +91,7 @@ pandas = "^2.2.2"
 quickchart-io = "^2.0.0"
 scipy = "^1.14.1"
 networkx = "^3.3"
+prometheus-client = "^0.20.0"
 [tool.poetry.group.dev.dependencies]
 pre-commit = "^3.0.4"
 pre-commit-hooks = "^4.4.0"

tests/test_metrics.py

@@ -0,0 +1,36 @@
+
+import pytest
+
+from keep.api.core.db import (
+    add_alerts_to_incident_by_incident_id,
+    create_incident_from_dict
+)
+
+from tests.fixtures.client import client, setup_api_key, test_app
+
+
+@pytest.mark.parametrize(
+    "test_app", ["NO_AUTH"], indirect=True
+)
+def test_add_remove_alert_to_incidents(client, db_session, test_app, setup_stress_alerts_no_elastic):
+    alerts = setup_stress_alerts_no_elastic(14)
+    incident = create_incident_from_dict("keep", {"name": "test", "description": "test"})
+    valid_api_key = "valid_api_key"
+    setup_api_key(db_session, valid_api_key)
+
+    add_alerts_to_incident_by_incident_id(
+        "keep",
+        incident.id,
+        [a.id for a in alerts]
+    )
+
+    response = client.get(
+        "/metrics",
+        headers={"X-API-KEY": "valid_api_key"}
+    )
+
+    # Checking for alert_total metric
+    assert f"alerts_total{{incident_name=\"test\" incident_id=\"{incident.id}\"}} 14" in response.text.split("\n")
+
+    # Checking for open_incidents_total metric
+    assert "open_incidents_total 1" in response.text.split("\n")