Merge ooni/data API and new measurements service

ooniapi/common/src/common/clickhouse.py

@@ -0,0 +1,4 @@
+from clickhouse_sqlalchemy import get_declarative_base
+
+
+Base = get_declarative_base()

ooniapi/common/src/common/routers.py

@@ -1,4 +1,5 @@
 from datetime import date, datetime
+from typing import Union
 from pydantic import BaseModel as PydandicBaseModel
 from pydantic import ConfigDict
 

ooniapi/common/src/common/utils.py

@@ -51,6 +51,8 @@ def commasplit(p: str) -> List[str]:
 def convert_to_csv(r) -> str:
     """Convert aggregation result dict/list to CSV"""
     csvf = StringIO()
+    if len(r) == 0:
+        return ""
     if isinstance(r, dict):
         # 0-dimensional data
         fieldnames = sorted(r.keys())

ooniapi/services/oonifindings/Makefile

@@ -61,4 +61,7 @@ clean:
 run:
 	hatch run uvicorn $(SERVICE_NAME).main:app
 
+apidocs:
+	hatch run python -m oonifindings.mkapidocs apidocs.json
+
 .PHONY: init test build clean docker print-labels

ooniapi/services/oonifindings/pyproject.toml

@@ -19,6 +19,7 @@ dependencies = [
   "psycopg2 ~= 2.9.9",
   "pyjwt ~= 2.8.0",
   "alembic ~= 1.13.1",
+  "python-dateutil ~= 2.9.0",
   "prometheus-fastapi-instrumentator ~= 6.1.0",
   "prometheus-client",
 ]
@@ -65,6 +66,8 @@ dependencies = [
   "black",
   "pytest-asyncio",
   "pytest-postgresql",
+  "pytest-docker",
+  "requests"
 ]
 path = ".venv/"
 

ooniapi/services/oonifindings/src/oonifindings/dependencies.py

@@ -1,5 +1,7 @@
 from typing import Annotated
 
+from clickhouse_driver import Client as Clickhouse
+
 from fastapi import Depends
 
 from sqlalchemy import create_engine
@@ -18,3 +20,11 @@ def get_postgresql_session(settings: Annotated[Settings, Depends(get_settings)])
         yield db
     finally:
         db.close()
+
+
+def get_clickhouse_session(settings: Annotated[Settings, Depends(get_settings)]):
+    db = Clickhouse.from_url(settings.clickhouse_url)
+    try:
+        yield db
+    finally:
+        db.disconnect()

ooniapi/services/oonifindings/src/oonifindings/main.py

@@ -11,6 +11,7 @@
 from . import models
 from .routers import v1
 
+
 from .dependencies import get_settings, get_postgresql_session
 from .common.version import get_build_label, get_pkg_version
 from .common.metrics import mount_metrics

ooniapi/services/oonifindings/src/oonifindings/mkapidocs.py

@@ -0,0 +1,13 @@
+import sys
+import json
+
+from fastapi.openapi.utils import get_openapi
+from .main import app
+from .__about__ import VERSION
+
+if __name__ == "__main__":
+    openapi = get_openapi(title="OONI Findings", version=VERSION, routes=app.routes)
+    assert len(sys.argv) == 2, "must specify outfile file"
+    with open(sys.argv[1], "w") as out_file:
+        out_file.write(json.dumps(openapi))
+        out_file.write("\n")

ooniapi/services/oonifindings/tests/conftest.py

@@ -1,7 +1,7 @@
 from pathlib import Path
-import pytest
-
 import time
+
+import pytest
 import jwt
 
 from fastapi.testclient import TestClient
@@ -56,7 +56,7 @@ def client(alembic_migration):
         postgresql_url=alembic_migration,
         jwt_encryption_key="super_secure",
         prometheus_metrics_password="super_secure",
-        account_id_hashing_key="super_secure"
+        account_id_hashing_key="super_secure",
     )
 
     client = TestClient(app)
@@ -100,7 +100,7 @@ def client_with_admin_role(client):
 
 @pytest.fixture
 def client_with_hashed_email(client):
-    
+
     def _hashed_email(email: str, role: str):
         client = TestClient(app)
         account_id = hash_email_address(email, "super_secure")

ooniapi/services/oonimeasurements/pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
   "fastapi ~= 0.108.0",
   "psycopg2 ~= 2.9.5",
   "clickhouse-driver ~= 0.2.6",
+  "clickhouse-sqlalchemy ~= 0.3.2",
   "sqlalchemy ~= 2.0.27",
   "ujson ~= 5.9.0",
   "urllib3 ~= 2.1.0",

ooniapi/services/oonimeasurements/src/oonimeasurements/dependencies.py

@@ -7,9 +7,10 @@
 from .common.config import Settings
 from .common.dependencies import get_settings
 
+
 def get_clickhouse_session(settings: Annotated[Settings, Depends(get_settings)]):
     db = Clickhouse.from_url(settings.clickhouse_url)
     try:
         yield db
-    finally: 
+    finally:
         db.disconnect()

ooniapi/services/oonimeasurements/src/oonimeasurements/main.py

@@ -8,7 +8,14 @@
 
 from prometheus_fastapi_instrumentator import Instrumentator
 
-from .routers import aggregation, measurements
+from .routers.v1 import aggregation
+from .routers.v1 import measurements
+from .routers.data import (
+    list_analysis,
+    list_observations,
+    aggregate_observations,
+    aggregate_analysis,
+)
 
 from .dependencies import get_clickhouse_session
 from .common.dependencies import get_settings
@@ -49,6 +56,10 @@ async def lifespan(app: FastAPI):
 
 app.include_router(measurements.router, prefix="/api")
 app.include_router(aggregation.router, prefix="/api")
+app.include_router(list_analysis.router, prefix="/api")
+app.include_router(list_observations.router, prefix="/api")
+app.include_router(aggregate_observations.router, prefix="/api")
+app.include_router(aggregate_analysis.router, prefix="/api")
 
 
 @app.get("/version")