Move oonidata endpoints into oonimeasurements service

ooniapi/services/oonifindings/src/oonifindings/main.py

@@ -10,12 +10,7 @@
 
 from . import models
 from .routers import v1
-from .routers.data import (
-    list_analysis,
-    list_observations,
-    aggregate_observations,
-    aggregate_analysis,
-)
+
 
 from .dependencies import get_settings, get_postgresql_session
 from .common.version import get_build_label, get_pkg_version
@@ -52,10 +47,6 @@ async def lifespan(app: FastAPI):
 )
 
 app.include_router(v1.router, prefix="/api")
-app.include_router(list_analysis.router, prefix="/api")
-app.include_router(list_observations.router, prefix="/api")
-app.include_router(aggregate_observations.router, prefix="/api")
-app.include_router(aggregate_analysis.router, prefix="/api")
 
 
 @app.get("/version")

ooniapi/services/oonifindings/tests/conftest.py

@@ -2,11 +2,8 @@
 import time
 
 import pytest
-import requests
 import jwt
 
-from clickhouse_driver import Client as ClickhouseClient
-
 from fastapi.testclient import TestClient
 
 from oonifindings.common.config import Settings
@@ -17,47 +14,6 @@
 THIS_DIR = Path(__file__).parent.resolve()
 
 
-def get_file_path(file_path: str):
-    return Path(__file__).parent / file_path
-
-
-@pytest.fixture(scope="session")
-def maybe_download_fixtures():
-    base_url = "https://ooni-data-eu-fra.s3.eu-central-1.amazonaws.com/samples/"
-    filenames = [
-        "analysis_web_measurement-sample.sql.gz",
-        "obs_web-sample.sql.gz",
-    ]
-    for fn in filenames:
-        dst_path = get_file_path(f"fixtures/{fn}")
-        if dst_path.exists():
-            continue
-        url = base_url + fn
-        print(f"Downloading {url} to {dst_path}")
-        r = requests.get(url)
-        dst_path.write_bytes(r.content)
-
-
-def is_clickhouse_running(url):
-    try:
-        with ClickhouseClient.from_url(url) as client:
-            client.execute("SELECT 1")
-        return True
-    except Exception:
-        return False
-
-
-@pytest.fixture(scope="session")
-def clickhouse_server(maybe_download_fixtures, docker_ip, docker_services):
-    """Ensure that HTTP service is up and responsive."""
-    port = docker_services.port_for("clickhouse", 9000)
-    url = "clickhouse://{}:{}/default".format(docker_ip, port)
-    docker_services.wait_until_responsive(
-        timeout=30.0, pause=0.1, check=lambda: is_clickhouse_running(url)
-    )
-    yield url
-
-
 def make_override_get_settings(**kw):
     def override_get_settings():
         return Settings(**kw)
@@ -95,10 +51,9 @@ def client_with_bad_settings():
 
 
 @pytest.fixture
-def client(alembic_migration, clickhouse_server):
+def client(alembic_migration):
     app.dependency_overrides[get_settings] = make_override_get_settings(
         postgresql_url=alembic_migration,
-        clickhouse_url=clickhouse_server,
         jwt_encryption_key="super_secure",
         prometheus_metrics_password="super_secure",
         account_id_hashing_key="super_secure",

ooniapi/services/oonimeasurements/src/oonimeasurements/main.py

@@ -10,6 +10,12 @@
 
 from .routers.v1 import aggregation
 from .routers.v1 import measurements
+from .routers.data import (
+    list_analysis,
+    list_observations,
+    aggregate_observations,
+    aggregate_analysis,
+)
 
 from .dependencies import get_clickhouse_session
 from .common.dependencies import get_settings
@@ -50,6 +56,10 @@ async def lifespan(app: FastAPI):
 
 app.include_router(measurements.router, prefix="/api")
 app.include_router(aggregation.router, prefix="/api")
+app.include_router(list_analysis.router, prefix="/api")
+app.include_router(list_observations.router, prefix="/api")
+app.include_router(aggregate_observations.router, prefix="/api")
+app.include_router(aggregate_analysis.router, prefix="/api")
 
 
 @app.get("/version")

ooniapi/services/oonimeasurements/tests/conftest.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 import pytest
 
+import requests
 import time
 import jwt
 
@@ -14,6 +15,27 @@
 THIS_DIR = Path(__file__).parent.resolve()
 
 
+def get_file_path(file_path: str):
+    return Path(__file__).parent / file_path
+
+
+@pytest.fixture(scope="session")
+def maybe_download_fixtures():
+    base_url = "https://ooni-data-eu-fra.s3.eu-central-1.amazonaws.com/samples/"
+    filenames = [
+        "analysis_web_measurement-sample.sql.gz",
+        "obs_web-sample.sql.gz",
+    ]
+    for fn in filenames:
+        dst_path = get_file_path(f"fixtures/{fn}")
+        if dst_path.exists():
+            continue
+        url = base_url + fn
+        print(f"Downloading {url} to {dst_path}")
+        r = requests.get(url)
+        dst_path.write_bytes(r.content)
+
+
 def is_clickhouse_running(url):
     try:
         with ClickhouseClient.from_url(url) as client:
@@ -24,7 +46,7 @@ def is_clickhouse_running(url):
 
 
 @pytest.fixture(scope="session")
-def clickhouse_server(docker_ip, docker_services):
+def clickhouse_server(maybe_download_fixtures, docker_ip, docker_services):
     port = docker_services.port_for("clickhouse", 9000)
     url = "clickhouse://{}:{}".format(docker_ip, port)
     docker_services.wait_until_responsive(

ooniapi/services/oonimeasurements/tests/docker-compose.yml

@@ -4,3 +4,6 @@ services:
     image: "clickhouse/clickhouse-server"
     ports:
       - "9000"
+    volumes:
+        - ./fixtures:/fixtures
+        - ./fixtures/initdb:/docker-entrypoint-initdb.d/