Support metrics & health endpoints PR#820 Issue#818

Add support for genearting metrics using prometheus. Support HTTP basic auth for scraping metrics from the endpoint.

* Add support for collecting prometheus metrics

* Add health endpoint

* Implement more test for the rest of the codebase reaching 100% code coverage

* Bump to release candidate version

* Run legacy api tests only on master since they take too long

.github/workflows/test_fastpath.yml

@@ -1,5 +1,8 @@
 name: test fastpath
-on: push
+on:
+  push:
+    branches:
+      - master
 
 jobs:
   test:

.github/workflows/test_legacy_ooniapi.yml

@@ -1,6 +1,8 @@
 name: test legacy/ooniapi
 on:
   push:
+    branches:
+      - master
   workflow_dispatch:
     inputs:
       debug_enabled:

ooniapi/common/src/common/config.py

@@ -17,3 +17,4 @@ class Settings(BaseSettings):
     statsd_port: int = 8125
     statsd_prefix: str = "ooniapi"
     jwt_encryption_key: str = "CHANGEME"
+    prometheus_metrics_password: str = "CHANGEME"

ooniapi/common/src/common/metrics.py

@@ -0,0 +1,42 @@
+import secrets
+
+from typing import Annotated
+
+from fastapi import FastAPI, Response, Depends, HTTPException, status
+from fastapi.security import HTTPBasic, HTTPBasicCredentials
+
+from .dependencies import get_settings
+from prometheus_client import (
+    CONTENT_TYPE_LATEST,
+    CollectorRegistry,
+    generate_latest,
+)
+
+security = HTTPBasic()
+
+
+def mount_metrics(app: FastAPI, registry: CollectorRegistry):
+    def metrics(
+        credentials: Annotated[HTTPBasicCredentials, Depends(security)],
+        settings=Depends(get_settings),
+    ):
+        is_correct_username = secrets.compare_digest(
+            credentials.username.encode("utf8"), b"prom"
+        )
+        is_correct_password = secrets.compare_digest(
+            credentials.password.encode("utf8"),
+            settings.prometheus_metrics_password.encode("utf-8"),
+        )
+        if not (is_correct_username and is_correct_password):
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail="Incorrect username or password",
+                headers={"WWW-Authenticate": "Basic"},
+            )
+
+        resp = Response(content=generate_latest(registry))
+        resp.headers["Content-Type"] = CONTENT_TYPE_LATEST
+        return resp
+
+    endpoint = "/metrics"
+    app.get(endpoint, include_in_schema=True, tags=None)(metrics)

ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py

@@ -83,5 +83,5 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
-    op.drop_table("oonirun")
     op.drop_table("oonirun_nettest")
+    op.drop_table("oonirun")

ooniapi/services/oonirun/pyproject.toml

@@ -21,6 +21,8 @@ dependencies = [
   "httpx ~= 0.26.0",
   "pyjwt ~= 2.8.0",
   "alembic ~= 1.13.1",
+  "prometheus-fastapi-instrumentator ~= 6.1.0",
+  "prometheus-client",
 ]
 
 readme = "README.md"
@@ -56,7 +58,7 @@ packages = ["src/oonirun"]
 artifacts = ["BUILD_LABEL"]
 
 [tool.hatch.envs.default]
-dependencies = ["pytest", "pytest-cov", "click", "black", "pytest-postgresql"]
+dependencies = ["pytest", "pytest-cov", "click", "black", "pytest-postgresql", "pytest-asyncio"]
 path = ".venv/"
 
 [tool.hatch.envs.default.scripts]

ooniapi/services/oonirun/src/oonirun/__about__.py

@@ -1 +1 @@
-VERSION = "0.5.0dev1"
+VERSION = "0.5.0rc0"

ooniapi/services/oonirun/src/oonirun/main.py

@@ -1,15 +1,20 @@
-from functools import lru_cache
-from fastapi import FastAPI
+import logging
+from contextlib import asynccontextmanager
+
+from fastapi import Depends, FastAPI
+from fastapi.middleware.cors import CORSMiddleware
 
+from pydantic import BaseModel
+
+from prometheus_fastapi_instrumentator import Instrumentator
+
+from . import models
 from .routers import oonirun
 
-from .dependencies import get_settings
+from .dependencies import get_postgresql_session, get_settings
 from .common.version import get_build_label, get_pkg_version
-from fastapi.middleware.cors import CORSMiddleware
+from .common.metrics import mount_metrics
 
-from contextlib import asynccontextmanager
-
-import logging
 
 pkg_name = "oonirun"
 
@@ -21,11 +26,16 @@
 async def lifespan(app: FastAPI):
     settings = get_settings()
     logging.basicConfig(level=getattr(logging, settings.log_level.upper()))
+    mount_metrics(app, instrumentor.registry)
     yield
 
 
 app = FastAPI(lifespan=lifespan)
 
+instrumentor = Instrumentator().instrument(
+    app, metric_namespace="ooniapi", metric_subsystem="oonirun"
+)
+
 # TODO: temporarily enable all
 origins = ["*"]
 app.add_middleware(
@@ -44,9 +54,41 @@ async def version():
     return {"version": pkg_version, "build_label": build_label}
 
 
+class HealthStatus(BaseModel):
+    status: str
+    errors: list[str] = []
+    version: str
+    build_label: str
+
+
 @app.get("/health")
-async def health():
-    return {"status": "ok", "version": pkg_version, "build_label": build_label}
+async def health(
+    settings=Depends(get_settings),
+    db=Depends(get_postgresql_session),
+):
+    errors = []
+    try:
+        db.query(models.OONIRunLink).limit(1).all()
+    except Exception as exc:
+        print(exc)
+        errors.append("db_error")
+
+    if settings.jwt_encryption_key == "CHANGEME":
+        errors.append("bad_jwt_secret")
+
+    if settings.prometheus_metrics_password == "CHANGEME":
+        errors.append("bad_prometheus_password")
+
+    status = "ok"
+    if len(errors) > 0:
+        status = "fail"
+
+    return {
+        "status": status,
+        "errors": errors,
+        "version": pkg_version,
+        "build_label": build_label,
+    }
 
 
 @app.get("/")

ooniapi/services/oonirun/src/oonirun/routers/oonirun.py

@@ -5,9 +5,7 @@
 """
 
 from datetime import datetime, timedelta, timezone, date
-from os import urandom
-from sys import byteorder
-from typing import Dict, Any, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
 import logging
 
 import sqlalchemy as sa
@@ -17,7 +15,6 @@
 from pydantic import BaseModel as PydandicBaseModel
 from typing_extensions import Annotated
 
-
 from .. import models
 
 from ..common.dependencies import get_settings, role_required
@@ -28,7 +25,6 @@
 )
 from ..dependencies import get_postgresql_session
 
-
 ISO_FORMAT_DATETIME = "%Y-%m-%dT%H:%M:%S.%fZ"
 ISO_FORMAT_DATE = "%Y-%m-%d"
 

ooniapi/services/oonirun/tests/conftest.py

@@ -35,20 +35,30 @@ def alembic_migration(postgresql):
     yield db_url
 
 
+@pytest.fixture
+def client_with_bad_settings():
+    app.dependency_overrides[get_settings] = make_override_get_settings(
+        postgresql_url="postgresql://bad:bad@localhost/bad"
+    )
+
+    client = TestClient(app)
+    yield client
+
+
 @pytest.fixture
 def client(alembic_migration):
     app.dependency_overrides[get_settings] = make_override_get_settings(
-        postgresql_url=alembic_migration
+        postgresql_url=alembic_migration,
+        jwt_encryption_key="super_secure",
+        prometheus_metrics_password="super_secure",
     )
 
     client = TestClient(app)
     yield client
 
 
 def create_jwt(payload: dict) -> str:
-    settings = Settings()
-    key = settings.jwt_encryption_key
-    return jwt.encode(payload, key, algorithm="HS256")
+    return jwt.encode(payload, "super_secure", algorithm="HS256")
 
 
 def create_session_token(account_id: str, role: str) -> str: