diff --git a/ooniapi/services/oonifindings/src/oonifindings/main.py b/ooniapi/services/oonifindings/src/oonifindings/main.py index 4511c4f5..7e269d71 100644 --- a/ooniapi/services/oonifindings/src/oonifindings/main.py +++ b/ooniapi/services/oonifindings/src/oonifindings/main.py @@ -10,12 +10,7 @@ from . import models from .routers import v1 -from .routers.data import ( - list_analysis, - list_observations, - aggregate_observations, - aggregate_analysis, -) + from .dependencies import get_settings, get_postgresql_session from .common.version import get_build_label, get_pkg_version @@ -52,10 +47,6 @@ async def lifespan(app: FastAPI): ) app.include_router(v1.router, prefix="/api") -app.include_router(list_analysis.router, prefix="/api") -app.include_router(list_observations.router, prefix="/api") -app.include_router(aggregate_observations.router, prefix="/api") -app.include_router(aggregate_analysis.router, prefix="/api") @app.get("/version") diff --git a/ooniapi/services/oonifindings/tests/conftest.py b/ooniapi/services/oonifindings/tests/conftest.py index 5afd31b8..42dac7e0 100644 --- a/ooniapi/services/oonifindings/tests/conftest.py +++ b/ooniapi/services/oonifindings/tests/conftest.py @@ -2,11 +2,8 @@ import time import pytest -import requests import jwt -from clickhouse_driver import Client as ClickhouseClient - from fastapi.testclient import TestClient from oonifindings.common.config import Settings @@ -17,47 +14,6 @@ THIS_DIR = Path(__file__).parent.resolve() -def get_file_path(file_path: str): - return Path(__file__).parent / file_path - - -@pytest.fixture(scope="session") -def maybe_download_fixtures(): - base_url = "https://ooni-data-eu-fra.s3.eu-central-1.amazonaws.com/samples/" - filenames = [ - "analysis_web_measurement-sample.sql.gz", - "obs_web-sample.sql.gz", - ] - for fn in filenames: - dst_path = get_file_path(f"fixtures/{fn}") - if dst_path.exists(): - continue - url = base_url + fn - print(f"Downloading {url} to {dst_path}") - r = requests.get(url) - dst_path.write_bytes(r.content) - - -def is_clickhouse_running(url): - try: - with ClickhouseClient.from_url(url) as client: - client.execute("SELECT 1") - return True - except Exception: - return False - - -@pytest.fixture(scope="session") -def clickhouse_server(maybe_download_fixtures, docker_ip, docker_services): - """Ensure that HTTP service is up and responsive.""" - port = docker_services.port_for("clickhouse", 9000) - url = "clickhouse://{}:{}/default".format(docker_ip, port) - docker_services.wait_until_responsive( - timeout=30.0, pause=0.1, check=lambda: is_clickhouse_running(url) - ) - yield url - - def make_override_get_settings(**kw): def override_get_settings(): return Settings(**kw) @@ -95,10 +51,9 @@ def client_with_bad_settings(): @pytest.fixture -def client(alembic_migration, clickhouse_server): +def client(alembic_migration): app.dependency_overrides[get_settings] = make_override_get_settings( postgresql_url=alembic_migration, - clickhouse_url=clickhouse_server, jwt_encryption_key="super_secure", prometheus_metrics_password="super_secure", account_id_hashing_key="super_secure", diff --git a/ooniapi/services/oonifindings/tests/docker-compose.yml b/ooniapi/services/oonifindings/tests/docker-compose.yml deleted file mode 100644 index bf383642..00000000 --- a/ooniapi/services/oonifindings/tests/docker-compose.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: '2' -services: - clickhouse: - image: "clickhouse/clickhouse-server" - ports: - - "9000" - volumes: - - ./fixtures:/fixtures - - ./fixtures/initdb:/docker-entrypoint-initdb.d/ diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py index c101ba15..d31b0dca 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py @@ -10,6 +10,12 @@ from .routers.v1 import aggregation from .routers.v1 import measurements +from .routers.data import ( + list_analysis, + list_observations, + aggregate_observations, + aggregate_analysis, +) from .dependencies import get_clickhouse_session from .common.dependencies import get_settings @@ -50,6 +56,10 @@ async def lifespan(app: FastAPI): app.include_router(measurements.router, prefix="/api") app.include_router(aggregation.router, prefix="/api") +app.include_router(list_analysis.router, prefix="/api") +app.include_router(list_observations.router, prefix="/api") +app.include_router(aggregate_observations.router, prefix="/api") +app.include_router(aggregate_analysis.router, prefix="/api") @app.get("/version") diff --git a/ooniapi/services/oonifindings/src/oonifindings/routers/data/__init__.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/__init__.py similarity index 100% rename from ooniapi/services/oonifindings/src/oonifindings/routers/data/__init__.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/__init__.py diff --git a/ooniapi/services/oonifindings/src/oonifindings/routers/data/aggregate_analysis.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_analysis.py similarity index 100% rename from ooniapi/services/oonifindings/src/oonifindings/routers/data/aggregate_analysis.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_analysis.py diff --git a/ooniapi/services/oonifindings/src/oonifindings/routers/data/aggregate_observations.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_observations.py similarity index 100% rename from ooniapi/services/oonifindings/src/oonifindings/routers/data/aggregate_observations.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_observations.py diff --git a/ooniapi/services/oonifindings/src/oonifindings/routers/data/list_analysis.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/list_analysis.py similarity index 100% rename from ooniapi/services/oonifindings/src/oonifindings/routers/data/list_analysis.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/list_analysis.py diff --git a/ooniapi/services/oonifindings/src/oonifindings/routers/data/list_observations.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/list_observations.py similarity index 100% rename from ooniapi/services/oonifindings/src/oonifindings/routers/data/list_observations.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/list_observations.py diff --git a/ooniapi/services/oonifindings/src/oonifindings/routers/data/utils.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/utils.py similarity index 100% rename from ooniapi/services/oonifindings/src/oonifindings/routers/data/utils.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/utils.py diff --git a/ooniapi/services/oonimeasurements/tests/conftest.py b/ooniapi/services/oonimeasurements/tests/conftest.py index 9bab9029..3cb1dcd5 100644 --- a/ooniapi/services/oonimeasurements/tests/conftest.py +++ b/ooniapi/services/oonimeasurements/tests/conftest.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest +import requests import time import jwt @@ -14,6 +15,27 @@ THIS_DIR = Path(__file__).parent.resolve() +def get_file_path(file_path: str): + return Path(__file__).parent / file_path + + +@pytest.fixture(scope="session") +def maybe_download_fixtures(): + base_url = "https://ooni-data-eu-fra.s3.eu-central-1.amazonaws.com/samples/" + filenames = [ + "analysis_web_measurement-sample.sql.gz", + "obs_web-sample.sql.gz", + ] + for fn in filenames: + dst_path = get_file_path(f"fixtures/{fn}") + if dst_path.exists(): + continue + url = base_url + fn + print(f"Downloading {url} to {dst_path}") + r = requests.get(url) + dst_path.write_bytes(r.content) + + def is_clickhouse_running(url): try: with ClickhouseClient.from_url(url) as client: @@ -24,7 +46,7 @@ def is_clickhouse_running(url): @pytest.fixture(scope="session") -def clickhouse_server(docker_ip, docker_services): +def clickhouse_server(maybe_download_fixtures, docker_ip, docker_services): port = docker_services.port_for("clickhouse", 9000) url = "clickhouse://{}:{}".format(docker_ip, port) docker_services.wait_until_responsive( diff --git a/ooniapi/services/oonimeasurements/tests/docker-compose.yml b/ooniapi/services/oonimeasurements/tests/docker-compose.yml index 7453035b..bf383642 100644 --- a/ooniapi/services/oonimeasurements/tests/docker-compose.yml +++ b/ooniapi/services/oonimeasurements/tests/docker-compose.yml @@ -4,3 +4,6 @@ services: image: "clickhouse/clickhouse-server" ports: - "9000" + volumes: + - ./fixtures:/fixtures + - ./fixtures/initdb:/docker-entrypoint-initdb.d/ diff --git a/ooniapi/services/oonifindings/tests/fixtures/.gitignore b/ooniapi/services/oonimeasurements/tests/fixtures/.gitignore similarity index 100% rename from ooniapi/services/oonifindings/tests/fixtures/.gitignore rename to ooniapi/services/oonimeasurements/tests/fixtures/.gitignore diff --git a/ooniapi/services/oonifindings/tests/fixtures/initdb/clickhouse.sql b/ooniapi/services/oonimeasurements/tests/fixtures/initdb/clickhouse.sql similarity index 100% rename from ooniapi/services/oonifindings/tests/fixtures/initdb/clickhouse.sql rename to ooniapi/services/oonimeasurements/tests/fixtures/initdb/clickhouse.sql diff --git a/ooniapi/services/oonifindings/tests/fixtures/initdb/init.sh b/ooniapi/services/oonimeasurements/tests/fixtures/initdb/init.sh similarity index 100% rename from ooniapi/services/oonifindings/tests/fixtures/initdb/init.sh rename to ooniapi/services/oonimeasurements/tests/fixtures/initdb/init.sh diff --git a/ooniapi/services/oonifindings/tests/test_oonidata.py b/ooniapi/services/oonimeasurements/tests/test_oonidata.py similarity index 100% rename from ooniapi/services/oonifindings/tests/test_oonidata.py rename to ooniapi/services/oonimeasurements/tests/test_oonidata.py