From 6791bc0292e4def111f32a61334f499fcb786f79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 10:21:27 +0100 Subject: [PATCH 01/12] Remove dbmigration codebuild task. We should run them inside of the container repl --- .../oonirun/buildspec-dbmigration.yml | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 ooniapi/services/oonirun/buildspec-dbmigration.yml diff --git a/ooniapi/services/oonirun/buildspec-dbmigration.yml b/ooniapi/services/oonirun/buildspec-dbmigration.yml deleted file mode 100644 index bb7dd126..00000000 --- a/ooniapi/services/oonirun/buildspec-dbmigration.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 0.2 -env: - variables: - OONI_CODE_PATH: api/fastapi/ - -phases: - install: - runtime-versions: - python: 3.11 - commands: - - echo "Installing Poetry" - - curl -fsS https://install.python-poetry.org | python - --preview -y - - export PATH="$HOME/.local/bin:$PATH" - - pre_build: - commands: - - aws --version - - build: - commands: - - PROJECT_ROOT=$(pwd) - - cd $OONI_CODE_PATH - - echo "Installing project dependencies with poetry..." - - poetry install --no-root - - export OONI_PG_URL=$(aws secretsmanager get-secret-value --secret-id OONI_PROD_POSTGRES_URL --query SecretString --output text) - - poetry run alembic upgrade head From 7925f493b19ced4b317b6d450d00e55c74b7aeaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 10:32:53 +0100 Subject: [PATCH 02/12] Move almebic script to top level directory --- ooniapi/services/oonirun/Dockerfile | 2 +- ooniapi/services/oonirun/Makefile | 1 + .../{src/oonirun => }/alembic/Readme.md | 0 .../{src/oonirun => }/alembic/__init__.py | 0 .../oonirun/{src/oonirun => }/alembic/env.py | 0 .../{src/oonirun => }/alembic/script.py.mako | 0 .../versions/981d92cf8790_init_tables.py | 0 ...841cb9549_make_oonirun_link_id_a_string.py | 37 --- ...add_expiration_date_color_columns_drop_.py | 32 --- .../9973a7e1f96c_oonirun_nettest_table.py | 256 ------------------ .../f96cf47f2791_create_oonirun_db.py | 44 --- 11 files changed, 2 insertions(+), 370 deletions(-) rename ooniapi/services/oonirun/{src/oonirun => }/alembic/Readme.md (100%) rename ooniapi/services/oonirun/{src/oonirun => }/alembic/__init__.py (100%) rename ooniapi/services/oonirun/{src/oonirun => }/alembic/env.py (100%) rename ooniapi/services/oonirun/{src/oonirun => }/alembic/script.py.mako (100%) rename ooniapi/services/oonirun/{src/oonirun => }/alembic/versions/981d92cf8790_init_tables.py (100%) delete mode 100644 ooniapi/services/oonirun/src/oonirun/alembic/old_versions/7d5841cb9549_make_oonirun_link_id_a_string.py delete mode 100644 ooniapi/services/oonirun/src/oonirun/alembic/old_versions/836b3451a168_add_expiration_date_color_columns_drop_.py delete mode 100644 ooniapi/services/oonirun/src/oonirun/alembic/old_versions/9973a7e1f96c_oonirun_nettest_table.py delete mode 100644 ooniapi/services/oonirun/src/oonirun/alembic/old_versions/f96cf47f2791_create_oonirun_db.py diff --git a/ooniapi/services/oonirun/Dockerfile b/ooniapi/services/oonirun/Dockerfile index 99b8ccb2..86cd3fbe 100644 --- a/ooniapi/services/oonirun/Dockerfile +++ b/ooniapi/services/oonirun/Dockerfile @@ -17,7 +17,7 @@ FROM python:3.11-bookworm as runner WORKDIR /app -COPY --from=builder /build /build/dist/*.whl /app/ +COPY --from=builder /build/alembic /build/alembic.ini /build/dist/*.whl /app/ RUN pip install /app/*whl && rm /app/*whl diff --git a/ooniapi/services/oonirun/Makefile b/ooniapi/services/oonirun/Makefile index 0bf8477e..140313f9 100644 --- a/ooniapi/services/oonirun/Makefile +++ b/ooniapi/services/oonirun/Makefile @@ -29,6 +29,7 @@ docker-build: -t ${IMAGE_NAME}:${VERSION_LABEL} \ -t ${IMAGE_NAME}:${ENV_LABEL} \ - + echo "built image: ${IMAGE_NAME}:${BUILD_LABEL} (${IMAGE_NAME}:${VERSION_LABEL} ${IMAGE_NAME}:${ENV_LABEL})" docker-push: # We need to use tar -czh to resolve the common dir symlink diff --git a/ooniapi/services/oonirun/src/oonirun/alembic/Readme.md b/ooniapi/services/oonirun/alembic/Readme.md similarity index 100% rename from ooniapi/services/oonirun/src/oonirun/alembic/Readme.md rename to ooniapi/services/oonirun/alembic/Readme.md diff --git a/ooniapi/services/oonirun/src/oonirun/alembic/__init__.py b/ooniapi/services/oonirun/alembic/__init__.py similarity index 100% rename from ooniapi/services/oonirun/src/oonirun/alembic/__init__.py rename to ooniapi/services/oonirun/alembic/__init__.py diff --git a/ooniapi/services/oonirun/src/oonirun/alembic/env.py b/ooniapi/services/oonirun/alembic/env.py similarity index 100% rename from ooniapi/services/oonirun/src/oonirun/alembic/env.py rename to ooniapi/services/oonirun/alembic/env.py diff --git a/ooniapi/services/oonirun/src/oonirun/alembic/script.py.mako b/ooniapi/services/oonirun/alembic/script.py.mako similarity index 100% rename from ooniapi/services/oonirun/src/oonirun/alembic/script.py.mako rename to ooniapi/services/oonirun/alembic/script.py.mako diff --git a/ooniapi/services/oonirun/src/oonirun/alembic/versions/981d92cf8790_init_tables.py b/ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py similarity index 100% rename from ooniapi/services/oonirun/src/oonirun/alembic/versions/981d92cf8790_init_tables.py rename to ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py diff --git a/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/7d5841cb9549_make_oonirun_link_id_a_string.py b/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/7d5841cb9549_make_oonirun_link_id_a_string.py deleted file mode 100644 index f7dccf63..00000000 --- a/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/7d5841cb9549_make_oonirun_link_id_a_string.py +++ /dev/null @@ -1,37 +0,0 @@ -"""make oonirun link id a string - -Revision ID: 7d5841cb9549 -Revises: 836b3451a168 -Create Date: 2024-02-28 15:41:53.811746 - -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision: str = "7d5841cb9549" -down_revision: Union[str, None] = "836b3451a168" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute( - """ - ALTER TABLE oonirun - ALTER COLUMN oonirun_link_id TYPE TEXT USING oonirun_link_id::TEXT - """ - ) - - -def downgrade() -> None: - op.execute( - """ - ALTER TABLE oonirun - ALTER COLUMN oonirun TYPE INTEGER USING oonirun::INTEGER - """ - ) diff --git a/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/836b3451a168_add_expiration_date_color_columns_drop_.py b/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/836b3451a168_add_expiration_date_color_columns_drop_.py deleted file mode 100644 index 2ac09b5c..00000000 --- a/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/836b3451a168_add_expiration_date_color_columns_drop_.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Add expiration_date, color columns. Drop is_archived column. - -Revision ID: 836b3451a168 -Revises: f96cf47f2791 -Create Date: 2024-02-27 09:44:26.833238 - -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision: str = "836b3451a168" -down_revision: Union[str, None] = "f96cf47f2791" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "oonirun", sa.Column("expiration_date", sa.DateTime(), nullable=False) - ) - op.add_column("oonirun", sa.Column("color", sa.String(), nullable=True)) - op.drop_column("oonirun", "is_archived") - - -def downgrade() -> None: - op.drop_column("oonirun", "expiration_date") - op.drop_column("oonirun", "color") diff --git a/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/9973a7e1f96c_oonirun_nettest_table.py b/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/9973a7e1f96c_oonirun_nettest_table.py deleted file mode 100644 index 660128a4..00000000 --- a/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/9973a7e1f96c_oonirun_nettest_table.py +++ /dev/null @@ -1,256 +0,0 @@ -"""oonirun_nettest table - -Revision ID: 9973a7e1f96c -Revises: 7d5841cb9549 -Create Date: 2024-03-08 14:38:53.154821 - -""" - -from datetime import datetime, timezone -from collections import defaultdict -from typing import Dict, List, Sequence, Union, Any -from copy import deepcopy - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.sql import table, column -from sqlalchemy.orm import Session, DeclarativeBase -from sqlalchemy import ForeignKey -from sqlalchemy.orm import Mapped -from sqlalchemy.orm import mapped_column -from sqlalchemy.orm import relationship - - -# revision identifiers, used by Alembic. -revision: str = "9973a7e1f96c" -down_revision: Union[str, None] = "7d5841cb9549" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -oonirun_nettest_table = table( - "oonirun_nettest", - sa.Column("oonirun_link_id", sa.String(), nullable=False), - sa.Column("revision", sa.Integer(), nullable=False, server_default=sa.text("1")), - sa.Column( - "nettest_index", sa.Integer(), nullable=False, server_default=sa.text("1") - ), - sa.Column("date_created", sa.DateTime(), nullable=True), - sa.Column("test_name", sa.String(), nullable=True), - sa.Column("inputs", sa.JSON(), nullable=True), - sa.Column("options", sa.JSON(), nullable=True), - sa.Column("backend_options", sa.JSON(), nullable=True), - sa.Column( - "is_background_run_enabled_default", - sa.Boolean(), - nullable=True, - server_default=sa.text("false"), - ), - sa.Column( - "is_manual_run_enabled_default", - sa.Boolean(), - nullable=True, - server_default=sa.text("false"), - ), -) - - -def upgrade() -> None: - op.create_table( - "oonirun_nettest", - sa.Column("oonirun_link_id", sa.String(), nullable=False), - sa.Column( - "revision", sa.Integer(), nullable=False, server_default=sa.text("1") - ), - sa.Column( - "nettest_index", sa.Integer(), nullable=False, server_default=sa.text("0") - ), - sa.Column("date_created", sa.DateTime(), nullable=True), - sa.Column("test_name", sa.String(), nullable=True), - sa.Column("inputs", sa.JSON(), nullable=True), - sa.Column("options", sa.JSON(), nullable=True), - sa.Column("backend_options", sa.JSON(), nullable=True), - sa.Column( - "is_background_run_enabled_default", - sa.Boolean(), - nullable=True, - server_default=sa.text("false"), - ), - sa.Column( - "is_manual_run_enabled_default", - sa.Boolean(), - nullable=True, - server_default=sa.text("false"), - ), - sa.PrimaryKeyConstraint("oonirun_link_id", "revision", "nettest_index"), - ) - - bind = op.get_bind() - session = Session(bind=bind) - oonirun_rows = session.execute( - sa.select( - column("oonirun_link_id", sa.String), - column("revision", sa.Integer), - column("nettests", sa.JSON), - column("date_created", sa.DateTime), - ).select_from(table("oonirun")) - ).fetchall() - - for record in oonirun_rows: - nettests_data = record.nettests - for index, nettest in enumerate(nettests_data): - nettest = dict( - oonirun_link_id=record.oonirun_link_id, - revision=record.revision, - nettest_index=index, - date_created=record.date_created, - test_name=nettest["test_name"], - inputs=nettest.get("inputs", []), - options=nettest.get("options", {}), - backend_options=nettest.get("backend_options", {}), - is_background_run_enabled_default=nettest.get( - "is_background_run_enabled", False - ), - is_manual_run_enabled_default=nettest.get( - "is_manual_run_enabled", False - ), - ) - print(nettest) - session.execute(oonirun_nettest_table.insert().values(**nettest)) - session.commit() - - tmp_table_name = "tmp__oonirun_new" - op.create_table( - tmp_table_name, - sa.Column("oonirun_link_id", sa.String(), nullable=False, primary_key=True), - sa.Column("date_created", sa.DateTime(), nullable=False), - sa.Column("date_updated", sa.DateTime(), nullable=False), - sa.Column("creator_account_id", sa.String(), nullable=False), - sa.Column("expiration_date", sa.DateTime(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("short_description", sa.String(), nullable=False), - sa.Column("description", sa.String(), nullable=False), - sa.Column("author", sa.String(), nullable=True), - sa.Column("icon", sa.String(), nullable=True), - sa.Column("color", sa.String(), nullable=True), - sa.Column("name_intl", sa.JSON(), nullable=True), - sa.Column("short_description_intl", sa.JSON(), nullable=True), - sa.Column("description_intl", sa.JSON(), nullable=True), - ) - conn = op.get_bind() - # This SQL query is to deduplicate the oonirun link rows such that we can - # have the highest revision kept so we have the latest metadata for the row - conn.execute( - sa.text( - f""" - INSERT INTO {tmp_table_name} (oonirun_link_id, date_created, - date_updated, creator_account_id, expiration_date, name, - short_description, description, author, icon, color, name_intl, - short_description_intl, description_intl) - SELECT oonirun_link_id, date_created, - date_updated, creator_account_id, expiration_date, name, - short_description, description, author, icon, color, name_intl, - short_description_intl, description_intl - FROM ( - SELECT oonirun_link_id, date_created, - date_updated, creator_account_id, expiration_date, name, - short_description, description, author, icon, color, name_intl, - short_description_intl, description_intl, ROW_NUMBER() OVER (PARTITION BY oonirun_link_id ORDER BY revision DESC) AS rn - FROM oonirun - ) sub - WHERE rn = 1 - """ - ) - ) - op.drop_table("oonirun") - # We then swap these deduplcated table with the new one - op.rename_table(tmp_table_name, "oonirun") - - op.create_foreign_key( - "fk_oonirun_nettest_oonirun_link_id", - "oonirun_nettest", - "oonirun", - ["oonirun_link_id"], - ["oonirun_link_id"], - ) - - -def downgrade() -> None: - """ - XXX This migration is not reversible. - - If you really must do it, check below for code that might work with some massaging. - - op.add_column( - "oonirun", - sa.Column( - "revision", - sa.INTEGER(), - autoincrement=False, - nullable=False, - server_default="1", - ), - ) - op.add_column( - "oonirun", - sa.Column("nettests", sa.JSON(), nullable=False, server_default="{}"), - ) - - bind = op.get_bind() - session = Session(bind=bind) - - # Select distinct `oonirun_link_id` to iterate through each `oonirun` record - distinct_ids = session.execute( - sa.select(column("oonirun_link_id").distinct()).select_from(table("oonirun")) - ).fetchall() - - for oonirun_link_id in distinct_ids: - # Fetch nettest records for each `oonirun_link_id` - nettest_records = session.execute( - sa.select(oonirun_nettest_table) - .where( - oonirun_nettest_table.c.oonirun_link_id - == oonirun_link_id.oonirun_link_id - ) - .order_by( - oonirun_nettest_table.c.revision, oonirun_nettest_table.c.nettest_index - ) - ).fetchall() - - # Construct the JSON structure from the nettest records - records_by_revision: Dict[int, List] = defaultdict(list) - for record in nettest_records: - inputs = [] - for d in record.test_inputs: - if d: - # Kind of ghetto, but anyways we were just handling "url" in the upgrade, so it's better than nothing - inputs.append(list(d.values())[0]) - nettests_json = { - "test_name": record.test_name, - "options": record.test_options, - "backend_options": record.backend_config, - "is_background_run_enabled": record.is_background_run_enabled_default, - "is_manual_run_enabled": record.is_manual_run_enabled_default, - "inputs": inputs, - } - records_by_revision[record.revision].append(nettests_json) - - existing_run_link = session.execute( - sa.select(OONIRunLink).where(OONIRunLink.oonirun_link_id == oonirun_link_id) - ).one() - print(existing_run_link) - - existing_run_link.nettests = records_by_revision.pop(existing_run_link.revision) - session.commit() - - for revision, nettests in records_by_revision.items(): - new_oonirun_link = deepcopy(existing_run_link) - session.expunge(new_oonirun_link) - new_oonirun_link.nettests = nettests - new_oonirun_link.revision = revision - session.add(new_oonirun_link) - session.commit() - - op.drop_table("oonirun_nettest") - """ - raise Exception("This migration is not reversible.") diff --git a/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/f96cf47f2791_create_oonirun_db.py b/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/f96cf47f2791_create_oonirun_db.py deleted file mode 100644 index 5dcded74..00000000 --- a/ooniapi/services/oonirun/src/oonirun/alembic/old_versions/f96cf47f2791_create_oonirun_db.py +++ /dev/null @@ -1,44 +0,0 @@ -"""create oonirun db - -Revision ID: f96cf47f2791 -Revises: -Create Date: 2024-02-15 14:39:47.867136 - -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision: str = "f96cf47f2791" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "oonirun", - sa.Column("oonirun_link_id", sa.Integer, primary_key=True), - sa.Column("revision", sa.Integer(), nullable=False, primary_key=True), - sa.Column("date_created", sa.DateTime(), nullable=False), - sa.Column("date_updated", sa.DateTime(), nullable=False), - sa.Column("creator_account_id", sa.String(), nullable=False), - sa.Column("name", sa.String()), - sa.Column("name_intl", sa.JSON()), - sa.Column("short_description", sa.String()), - sa.Column("short_description_intl", sa.JSON()), - sa.Column("description", sa.String()), - sa.Column("description_intl", sa.JSON()), - sa.Column("author", sa.String()), - sa.Column("icon", sa.String()), - sa.Column("nettests", sa.JSON(), nullable=False), - sa.Column("is_archived", sa.Boolean()), - ) - - -def downgrade() -> None: - op.drop_table("oonirun") From a7bfc16d05a0f82f89479b03b6b4deb41145ffe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 13:48:49 +0100 Subject: [PATCH 03/12] Fix docker build --- ooniapi/services/oonirun/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ooniapi/services/oonirun/Dockerfile b/ooniapi/services/oonirun/Dockerfile index 86cd3fbe..d3d4c413 100644 --- a/ooniapi/services/oonirun/Dockerfile +++ b/ooniapi/services/oonirun/Dockerfile @@ -17,7 +17,9 @@ FROM python:3.11-bookworm as runner WORKDIR /app -COPY --from=builder /build/alembic /build/alembic.ini /build/dist/*.whl /app/ +COPY --from=builder /build/alembic/ /app/alembic/ +COPY --from=builder /build/alembic.ini /app/ +COPY --from=builder /build/dist/*.whl /app/ RUN pip install /app/*whl && rm /app/*whl From 334f64c8eea528ba3164235c39899d63d6d75867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 13:53:48 +0100 Subject: [PATCH 04/12] Fix hatch version getting --- ooniapi/services/oonirun/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/oonirun/Makefile b/ooniapi/services/oonirun/Makefile index 140313f9..7a8c70ca 100644 --- a/ooniapi/services/oonirun/Makefile +++ b/ooniapi/services/oonirun/Makefile @@ -5,7 +5,7 @@ IMAGE_NAME ?= ooni/api-$(SERVICE_NAME) DATE := $(shell python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d'))") GIT_FULL_SHA ?= $(shell git rev-parse HEAD) SHORT_SHA := $(shell echo ${GIT_FULL_SHA} | cut -c1-8) -PKG_VERSION := $(shell cat pyproject.toml | grep -e 'version\s*=' | cut -d '"' -f2) +PKG_VERSION := $(shell hatch version) BUILD_LABEL := $(DATE)-$(SHORT_SHA) VERSION_LABEL = v$(PKG_VERSION) From 48529e5bb3a3009d4660ec4d25452993fac1f38b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 14:42:43 +0100 Subject: [PATCH 05/12] Fix db migration --- ooniapi/services/oonirun/.dockerignore | 10 +++++++ ooniapi/services/oonirun/Dockerfile | 12 ++++++--- ooniapi/services/oonirun/README.md | 15 +++++++++++ ooniapi/services/oonirun/alembic/Readme.md | 15 ----------- .../services/oonirun/alembic/script.py.mako | 26 ------------------- ooniapi/services/oonirun/pyproject.toml | 5 +--- 6 files changed, 35 insertions(+), 48 deletions(-) create mode 100644 ooniapi/services/oonirun/.dockerignore delete mode 100644 ooniapi/services/oonirun/alembic/Readme.md delete mode 100644 ooniapi/services/oonirun/alembic/script.py.mako diff --git a/ooniapi/services/oonirun/.dockerignore b/ooniapi/services/oonirun/.dockerignore new file mode 100644 index 00000000..4f7a82b5 --- /dev/null +++ b/ooniapi/services/oonirun/.dockerignore @@ -0,0 +1,10 @@ +.DS_Store +*.log +*.pyc +*.swp +*.env +.coverage +coverage.xml +dist/ +.venv/ +__pycache__/ diff --git a/ooniapi/services/oonirun/Dockerfile b/ooniapi/services/oonirun/Dockerfile index d3d4c413..cef25c48 100644 --- a/ooniapi/services/oonirun/Dockerfile +++ b/ooniapi/services/oonirun/Dockerfile @@ -8,6 +8,10 @@ RUN python -m pip install hatch COPY . /build +# When you build stuff on macOS you end up with ._ files +# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them +RUN find /build -type f -name '._*' -delete + RUN echo "$BUILD_LABEL" > /build/src/oonirun/BUILD_LABEL RUN hatch build @@ -17,11 +21,13 @@ FROM python:3.11-bookworm as runner WORKDIR /app -COPY --from=builder /build/alembic/ /app/alembic/ -COPY --from=builder /build/alembic.ini /app/ +COPY --from=builder /build/README.md /app/ COPY --from=builder /build/dist/*.whl /app/ - RUN pip install /app/*whl && rm /app/*whl +COPY --from=builder /build/alembic/ /app/alembic/ +COPY --from=builder /build/alembic.ini /app/ +RUN rm -rf /app/alembic/__pycache__ + CMD ["uvicorn", "oonirun.main:app", "--host", "0.0.0.0", "--port", "80"] EXPOSE 80 diff --git a/ooniapi/services/oonirun/README.md b/ooniapi/services/oonirun/README.md index e69de29b..6041f8df 100644 --- a/ooniapi/services/oonirun/README.md +++ b/ooniapi/services/oonirun/README.md @@ -0,0 +1,15 @@ +## Alembic database migrations + +When you make changes to the DB schema you will have to run the alembic scripts for generating an appropriate migration file. + +This is how you do it: + +1. Create the template migration script +``` +poetry run alembic revision -m "name of the revision" +``` +2. Edit the newly created python file and fill out the `upgrade()` and `downgrade()` function with the relevant code bits +3. You can now run the migration like so: +``` +OONI_PG_URL=postgresql://oonipg:oonipg@localhost/oonipg hatch run alembic upgrade head +``` diff --git a/ooniapi/services/oonirun/alembic/Readme.md b/ooniapi/services/oonirun/alembic/Readme.md deleted file mode 100644 index a8d77a90..00000000 --- a/ooniapi/services/oonirun/alembic/Readme.md +++ /dev/null @@ -1,15 +0,0 @@ -# Alembic database migrations - -When you make changes to the DB schema you will have to run the alembic scripts for generating an appropriate migration file. - -This is how you do it: - -1. Create the template migration script -``` -poetry run alembic revision -m "name of the revision" -``` -2. Edit the newly created python file and fill out the `upgrade()` and `downgrade()` function with the relevant code bits -3. You can now run the migration like so: -``` -OONI_PG_URL=postgresql://oonipg:oonipg@localhost/oonipg poetry run alembic upgrade head -``` diff --git a/ooniapi/services/oonirun/alembic/script.py.mako b/ooniapi/services/oonirun/alembic/script.py.mako deleted file mode 100644 index fbc4b07d..00000000 --- a/ooniapi/services/oonirun/alembic/script.py.mako +++ /dev/null @@ -1,26 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision: str = ${repr(up_revision)} -down_revision: Union[str, None] = ${repr(down_revision)} -branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} -depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} - - -def upgrade() -> None: - ${upgrades if upgrades else "pass"} - - -def downgrade() -> None: - ${downgrades if downgrades else "pass"} diff --git a/ooniapi/services/oonirun/pyproject.toml b/ooniapi/services/oonirun/pyproject.toml index dc0e1bb3..90f4399c 100644 --- a/ooniapi/services/oonirun/pyproject.toml +++ b/ooniapi/services/oonirun/pyproject.toml @@ -72,10 +72,7 @@ addopts = ["--import-mode=importlib"] branch = true parallel = true source_pkgs = ["oonirun", "tests"] -omit = [ - "src/oonirun/common/*", - "src/oonirun/__about__.py" -] +omit = ["src/oonirun/common/*", "src/oonirun/__about__.py"] [tool.coverage.paths] oonirun = ["src/oonirun"] From 8c74d280fe20fe66cffd40435800def3c48cc42c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 14:45:00 +0100 Subject: [PATCH 06/12] Fix path to alembic migrations --- ooniapi/services/oonirun/tests/conftest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ooniapi/services/oonirun/tests/conftest.py b/ooniapi/services/oonirun/tests/conftest.py index 0fa1f53e..896ce74d 100644 --- a/ooniapi/services/oonirun/tests/conftest.py +++ b/ooniapi/services/oonirun/tests/conftest.py @@ -25,9 +25,7 @@ def alembic_migration(postgresql): db_url = f"postgresql://{postgresql.info.user}:@{postgresql.info.host}:{postgresql.info.port}/{postgresql.info.dbname}" - migrations_path = ( - pathlib.Path(__file__).parent.parent / "src" / "oonirun" / "alembic" - ).resolve() + migrations_path = (pathlib.Path(__file__).parent.parent / "alembic").resolve() alembic_cfg = Config() alembic_cfg.set_main_option("script_location", str(migrations_path)) From 5791ecf9956ee4af315c8a32df9f4d580efccc3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 14:45:32 +0100 Subject: [PATCH 07/12] Add coverage to gitignore --- ooniapi/services/oonirun/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/ooniapi/services/oonirun/.gitignore b/ooniapi/services/oonirun/.gitignore index d7053580..9a1b4f54 100644 --- a/ooniapi/services/oonirun/.gitignore +++ b/ooniapi/services/oonirun/.gitignore @@ -1,2 +1,3 @@ /dist /coverage_html +*.coverage* From a16ad7f6a5909efea293ddc48f7fc75307c76504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 17:08:02 +0100 Subject: [PATCH 08/12] Add support for collecting prometheus metrics --- ooniapi/common/src/common/config.py | 1 + ooniapi/common/src/common/metrics.py | 42 +++++++++++++++++++ ooniapi/services/oonirun/pyproject.toml | 2 + ooniapi/services/oonirun/src/oonirun/main.py | 9 +++- .../oonirun/src/oonirun/routers/oonirun.py | 6 +-- 5 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 ooniapi/common/src/common/metrics.py diff --git a/ooniapi/common/src/common/config.py b/ooniapi/common/src/common/config.py index 245ab572..e275444c 100644 --- a/ooniapi/common/src/common/config.py +++ b/ooniapi/common/src/common/config.py @@ -17,3 +17,4 @@ class Settings(BaseSettings): statsd_port: int = 8125 statsd_prefix: str = "ooniapi" jwt_encryption_key: str = "CHANGEME" + prometheus_metrics_password: str = "CHANGEME" diff --git a/ooniapi/common/src/common/metrics.py b/ooniapi/common/src/common/metrics.py new file mode 100644 index 00000000..4ee2097f --- /dev/null +++ b/ooniapi/common/src/common/metrics.py @@ -0,0 +1,42 @@ +import secrets + +from typing import Annotated + +from fastapi import FastAPI, Response, Depends, HTTPException, status +from fastapi.security import HTTPBasic, HTTPBasicCredentials + +from .dependencies import get_settings +from prometheus_client import ( + CONTENT_TYPE_LATEST, + CollectorRegistry, + generate_latest, +) + +security = HTTPBasic() + + +def mount_metrics(app: FastAPI, registry: CollectorRegistry): + def metrics( + credentials: Annotated[HTTPBasicCredentials, Depends(security)], + settings=Depends(get_settings), + ): + is_correct_username = secrets.compare_digest( + credentials.username.encode("utf8"), b"prom" + ) + is_correct_password = secrets.compare_digest( + credentials.password.encode("utf8"), + settings.prometheus_metrics_password.encode("utf-8"), + ) + if not (is_correct_username and is_correct_password): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + headers={"WWW-Authenticate": "Basic"}, + ) + + resp = Response(content=generate_latest(registry)) + resp.headers["Content-Type"] = CONTENT_TYPE_LATEST + return resp + + endpoint = "/metrics" + app.get(endpoint, include_in_schema=True, tags=None)(metrics) diff --git a/ooniapi/services/oonirun/pyproject.toml b/ooniapi/services/oonirun/pyproject.toml index 90f4399c..206fba06 100644 --- a/ooniapi/services/oonirun/pyproject.toml +++ b/ooniapi/services/oonirun/pyproject.toml @@ -21,6 +21,8 @@ dependencies = [ "httpx ~= 0.26.0", "pyjwt ~= 2.8.0", "alembic ~= 1.13.1", + "prometheus-fastapi-instrumentator ~= 6.1.0", + "prometheus-client", ] readme = "README.md" diff --git a/ooniapi/services/oonirun/src/oonirun/main.py b/ooniapi/services/oonirun/src/oonirun/main.py index 485c6335..9922318b 100644 --- a/ooniapi/services/oonirun/src/oonirun/main.py +++ b/ooniapi/services/oonirun/src/oonirun/main.py @@ -1,12 +1,14 @@ -from functools import lru_cache from fastapi import FastAPI from .routers import oonirun from .dependencies import get_settings from .common.version import get_build_label, get_pkg_version +from .common.metrics import mount_metrics from fastapi.middleware.cors import CORSMiddleware +from prometheus_fastapi_instrumentator import Instrumentator + from contextlib import asynccontextmanager import logging @@ -21,11 +23,16 @@ async def lifespan(app: FastAPI): settings = get_settings() logging.basicConfig(level=getattr(logging, settings.log_level.upper())) + mount_metrics(app, instrumentor.registry) yield app = FastAPI(lifespan=lifespan) +instrumentor = Instrumentator().instrument( + app, metric_namespace="ooniapi", metric_subsystem="oonirun" +) + # TODO: temporarily enable all origins = ["*"] app.add_middleware( diff --git a/ooniapi/services/oonirun/src/oonirun/routers/oonirun.py b/ooniapi/services/oonirun/src/oonirun/routers/oonirun.py index 2b900733..d216dc88 100644 --- a/ooniapi/services/oonirun/src/oonirun/routers/oonirun.py +++ b/ooniapi/services/oonirun/src/oonirun/routers/oonirun.py @@ -5,9 +5,7 @@ """ from datetime import datetime, timedelta, timezone, date -from os import urandom -from sys import byteorder -from typing import Dict, Any, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple import logging import sqlalchemy as sa @@ -17,7 +15,6 @@ from pydantic import BaseModel as PydandicBaseModel from typing_extensions import Annotated - from .. import models from ..common.dependencies import get_settings, role_required @@ -28,7 +25,6 @@ ) from ..dependencies import get_postgresql_session - ISO_FORMAT_DATETIME = "%Y-%m-%dT%H:%M:%S.%fZ" ISO_FORMAT_DATE = "%Y-%m-%d" From 0c86c6e9c4db513a18e13cd2f3b84e54d24708f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 17:50:38 +0100 Subject: [PATCH 09/12] Add health endpoint Implement more test for the rest of the codebase Reach 100% code coverage --- .../versions/981d92cf8790_init_tables.py | 9 +- ooniapi/services/oonirun/pyproject.toml | 2 +- ooniapi/services/oonirun/src/oonirun/main.py | 55 +++++- ooniapi/services/oonirun/tests/conftest.py | 18 +- .../services/oonirun/tests/test_database.py | 176 ++++++++++++++++++ ooniapi/services/oonirun/tests/test_main.py | 35 ++++ .../services/oonirun/tests/test_oonirun.py | 97 ---------- 7 files changed, 274 insertions(+), 118 deletions(-) create mode 100644 ooniapi/services/oonirun/tests/test_database.py create mode 100644 ooniapi/services/oonirun/tests/test_main.py diff --git a/ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py b/ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py index fb98453d..c92919cd 100644 --- a/ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py +++ b/ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py @@ -75,13 +75,10 @@ def upgrade() -> None: server_default=sa.text("false"), ), sa.PrimaryKeyConstraint("oonirun_link_id", "revision", "nettest_index"), - sa.ForeignKeyConstraint( - ["oonirun_link_id"], - ["oonirun.oonirun_link_id"], - ), + sa.ForeignKeyConstraint(["oonirun_link_id"], ["oonirun.oonirun_link_id"]), ) -def downgrade() -> None: - op.drop_table("oonirun") +def downgrade() -> None: # no cov op.drop_table("oonirun_nettest") + op.drop_table("oonirun") diff --git a/ooniapi/services/oonirun/pyproject.toml b/ooniapi/services/oonirun/pyproject.toml index 206fba06..93f24f2c 100644 --- a/ooniapi/services/oonirun/pyproject.toml +++ b/ooniapi/services/oonirun/pyproject.toml @@ -58,7 +58,7 @@ packages = ["src/oonirun"] artifacts = ["BUILD_LABEL"] [tool.hatch.envs.default] -dependencies = ["pytest", "pytest-cov", "click", "black", "pytest-postgresql"] +dependencies = ["pytest", "pytest-cov", "click", "black", "pytest-postgresql", "pytest-asyncio"] path = ".venv/" [tool.hatch.envs.default.scripts] diff --git a/ooniapi/services/oonirun/src/oonirun/main.py b/ooniapi/services/oonirun/src/oonirun/main.py index 9922318b..ef4ef96e 100644 --- a/ooniapi/services/oonirun/src/oonirun/main.py +++ b/ooniapi/services/oonirun/src/oonirun/main.py @@ -1,17 +1,20 @@ -from fastapi import FastAPI - -from .routers import oonirun +import logging +from contextlib import asynccontextmanager -from .dependencies import get_settings -from .common.version import get_build_label, get_pkg_version -from .common.metrics import mount_metrics +from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + from prometheus_fastapi_instrumentator import Instrumentator -from contextlib import asynccontextmanager +from . import models +from .routers import oonirun + +from .dependencies import get_postgresql_session, get_settings +from .common.version import get_build_label, get_pkg_version +from .common.metrics import mount_metrics -import logging pkg_name = "oonirun" @@ -51,9 +54,41 @@ async def version(): return {"version": pkg_version, "build_label": build_label} +class HealthStatus(BaseModel): + status: str + errors: list[str] = [] + version: str + build_label: str + + @app.get("/health") -async def health(): - return {"status": "ok", "version": pkg_version, "build_label": build_label} +async def health( + settings=Depends(get_settings), + db=Depends(get_postgresql_session), +): + errors = [] + try: + db.query(models.OONIRunLink).limit(1).all() + except Exception as exc: + print(exc) + errors.append("db_error") + + if settings.jwt_encryption_key == "CHANGEME": + errors.append("bad_jwt_secret") + + if settings.prometheus_metrics_password == "CHANGEME": + errors.append("bad_prometheus_password") + + status = "ok" + if len(errors) > 0: + status = "fail" + + return { + "status": status, + "errors": errors, + "version": pkg_version, + "build_label": build_label, + } @app.get("/") diff --git a/ooniapi/services/oonirun/tests/conftest.py b/ooniapi/services/oonirun/tests/conftest.py index 896ce74d..3bf337e3 100644 --- a/ooniapi/services/oonirun/tests/conftest.py +++ b/ooniapi/services/oonirun/tests/conftest.py @@ -35,10 +35,22 @@ def alembic_migration(postgresql): yield db_url +@pytest.fixture +def client_with_bad_settings(): + app.dependency_overrides[get_settings] = make_override_get_settings( + postgresql_url="postgresql://bad:bad@localhost/bad" + ) + + client = TestClient(app) + yield client + + @pytest.fixture def client(alembic_migration): app.dependency_overrides[get_settings] = make_override_get_settings( - postgresql_url=alembic_migration + postgresql_url=alembic_migration, + jwt_encryption_key="super_secure", + prometheus_metrics_password="super_secure", ) client = TestClient(app) @@ -46,9 +58,7 @@ def client(alembic_migration): def create_jwt(payload: dict) -> str: - settings = Settings() - key = settings.jwt_encryption_key - return jwt.encode(payload, key, algorithm="HS256") + return jwt.encode(payload, "super_secure", algorithm="HS256") def create_session_token(account_id: str, role: str) -> str: diff --git a/ooniapi/services/oonirun/tests/test_database.py b/ooniapi/services/oonirun/tests/test_database.py new file mode 100644 index 00000000..0606a441 --- /dev/null +++ b/ooniapi/services/oonirun/tests/test_database.py @@ -0,0 +1,176 @@ +from copy import deepcopy +from datetime import datetime +import pathlib +from oonirun.routers.oonirun import utcnow_seconds +import pytest + +import sqlalchemy as sa +from sqlalchemy.orm import sessionmaker +from oonirun import models +from oonirun.dependencies import get_postgresql_session +from sqlalchemy import create_engine + +SAMPLE_OONIRUN = { + "name": "", + "name_intl": {}, + "description": "integ-test description in English", + "description_intl": { + "es": "integ-test descripción en español", + }, + "short_description": "integ-test short description in English", + "short_description_intl": { + "it": "integ-test descrizione breve in italiano", + }, + "icon": "myicon", + "author": "integ-test author", + "nettests": [ + { + "inputs": [ + "https://example.com/", + "https://ooni.org/", + ], + "options": { + "HTTP3Enabled": True, + }, + "backend_options": {}, + "is_background_run_enabled_default": False, + "is_manual_run_enabled_default": False, + "test_name": "web_connectivity", + }, + { + "inputs": [], + "options": {}, + "backend_options": {}, + "is_background_run_enabled_default": False, + "is_manual_run_enabled_default": False, + "test_name": "dnscheck", + }, + ], +} + + +def config_alembic(db_url): + from alembic.config import Config + + migrations_path = (pathlib.Path(__file__).parent.parent / "alembic").resolve() + + alembic_cfg = Config() + alembic_cfg.set_main_option("script_location", str(migrations_path)) + alembic_cfg.set_main_option("sqlalchemy.url", db_url) + return alembic_cfg + + +def upgrade_to_head(db_url): + from alembic import command + + command.upgrade(config_alembic(db_url), "head") + + +def get_db(pg_url): + engine = create_engine(pg_url) + SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + return SessionLocal() + + +def test_downgrade(postgresql): + from alembic import command + + db_url = f"postgresql://{postgresql.info.user}:@{postgresql.info.host}:{postgresql.info.port}/{postgresql.info.dbname}" + + command.upgrade(config_alembic(db_url), "head") + command.downgrade(config_alembic(db_url), "-1") + + +def test_upgrade_to_head(postgresql): + db_url = f"postgresql://{postgresql.info.user}:@{postgresql.info.host}:{postgresql.info.port}/{postgresql.info.dbname}" + upgrade_to_head(db_url) + db = get_db(db_url) + + run_link = deepcopy(SAMPLE_OONIRUN) + nettests = run_link.pop("nettests") + + new_row = db.query(models.OONIRunLink).first() + + db_runlink = models.OONIRunLink( + **run_link, + oonirun_link_id="000000000", + date_created=utcnow_seconds(), + date_updated=utcnow_seconds(), + expiration_date=utcnow_seconds(), + creator_account_id="000000000", + ) + db_runlink.nettests = [ + models.OONIRunLinkNettest( + **nettests[0], + revision=1, + nettest_index=0, + date_created=utcnow_seconds(), + ), + models.OONIRunLinkNettest( + **nettests[1], + revision=1, + nettest_index=1, + date_created=utcnow_seconds(), + ), + models.OONIRunLinkNettest( + **nettests[1], + revision=2, + nettest_index=0, + date_created=utcnow_seconds(), + ), + models.OONIRunLinkNettest( + **nettests[1], + revision=3, + nettest_index=0, + date_created=utcnow_seconds(), + ), + ] + db.add(db_runlink) + db.commit() + + new_row = db.query(models.OONIRunLink).first() + assert new_row + assert new_row.nettests[0].revision == 3 + + db.close() + + with pytest.raises(sa.exc.StatementError): + db_runlink = models.OONIRunLink( + **run_link, + oonirun_link_id="000000000", + date_created="NOT A DATE", + date_updated=utcnow_seconds(), + expiration_date=utcnow_seconds(), + creator_account_id="000000000", + ) + db.add(db_runlink) + db.commit() + db.rollback() + + with pytest.raises(sa.exc.StatementError): + naive_datetime = datetime.now() + db_runlink = models.OONIRunLink( + **run_link, + oonirun_link_id="000000000", + date_created=naive_datetime, + date_updated=utcnow_seconds(), + expiration_date=utcnow_seconds(), + creator_account_id="000000000", + ) + db.add(db_runlink) + db.commit() + db.rollback() + + with pytest.raises(sa.exc.StatementError): + db_runlink = models.OONIRunLink( + **run_link, + oonirun_link_id="000000000", + date_created=None, + date_updated=utcnow_seconds(), + expiration_date=utcnow_seconds(), + creator_account_id="000000000", + ) + db.add(db_runlink) + db.commit() + db.rollback() diff --git a/ooniapi/services/oonirun/tests/test_main.py b/ooniapi/services/oonirun/tests/test_main.py new file mode 100644 index 00000000..cb70a9c2 --- /dev/null +++ b/ooniapi/services/oonirun/tests/test_main.py @@ -0,0 +1,35 @@ +import pytest + +import httpx +from fastapi.testclient import TestClient +from oonirun.main import lifespan, app + + +def test_health_good(client): + r = client.get("health") + j = r.json() + assert j["status"] == "ok", j + assert len(j["errors"]) == 0, j + + +def test_health_bad(client_with_bad_settings): + r = client_with_bad_settings.get("health") + j = r.json() + assert j["status"] == "fail", j + assert len(j["errors"]) > 0, j + + +def test_metrics(client): + r = client.get("/metrics") + + +@pytest.mark.asyncio +async def test_lifecycle(): + async with lifespan(app) as ls: + client = TestClient(app) + r = client.get("/metrics") + assert r.status_code == 401 + + auth = httpx.BasicAuth(username="prom", password="super_secure") + r = client.get("/metrics", auth=auth) + assert r.status_code == 200, r.text diff --git a/ooniapi/services/oonirun/tests/test_oonirun.py b/ooniapi/services/oonirun/tests/test_oonirun.py index 871f01d5..6faaa2cd 100644 --- a/ooniapi/services/oonirun/tests/test_oonirun.py +++ b/ooniapi/services/oonirun/tests/test_oonirun.py @@ -73,103 +73,6 @@ ] -def test_oonirun_models(tmp_path_factory): - db_path = tmp_path_factory.mktemp("oonidb") / "db.sqlite3" - db_url = f"sqlite:///{db_path}" - - engine = create_engine(db_url, connect_args={"check_same_thread": False}) - metadata = models.OONIRunLink.metadata - metadata.create_all(engine) - SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) - db = SessionLocal() - - run_link = deepcopy(SAMPLE_OONIRUN) - nettests = run_link.pop("nettests") - - db_runlink = models.OONIRunLink( - **run_link, - oonirun_link_id="000000000", - date_created=utcnow_seconds(), - date_updated=utcnow_seconds(), - expiration_date=utcnow_seconds(), - creator_account_id="000000000", - ) - db_runlink.nettests = [ - models.OONIRunLinkNettest( - **nettests[0], - revision=1, - nettest_index=0, - date_created=utcnow_seconds(), - ), - models.OONIRunLinkNettest( - **nettests[1], - revision=1, - nettest_index=1, - date_created=utcnow_seconds(), - ), - models.OONIRunLinkNettest( - **nettests[1], - revision=2, - nettest_index=0, - date_created=utcnow_seconds(), - ), - models.OONIRunLinkNettest( - **nettests[1], - revision=3, - nettest_index=0, - date_created=utcnow_seconds(), - ), - ] - db.add(db_runlink) - db.commit() - - new_row = db.query(models.OONIRunLink).first() - assert new_row - assert new_row.nettests[0].revision == 3 - - db.close() - - with pytest.raises(sa.exc.StatementError): - db_runlink = models.OONIRunLink( - **run_link, - oonirun_link_id="000000000", - date_created="NOT A DATE", - date_updated=utcnow_seconds(), - expiration_date=utcnow_seconds(), - creator_account_id="000000000", - ) - db.add(db_runlink) - db.commit() - db.rollback() - - with pytest.raises(sa.exc.StatementError): - naive_datetime = datetime.now() - db_runlink = models.OONIRunLink( - **run_link, - oonirun_link_id="000000000", - date_created=naive_datetime, - date_updated=utcnow_seconds(), - expiration_date=utcnow_seconds(), - creator_account_id="000000000", - ) - db.add(db_runlink) - db.commit() - db.rollback() - - with pytest.raises(sa.exc.StatementError): - db_runlink = models.OONIRunLink( - **run_link, - oonirun_link_id="000000000", - date_created=None, - date_updated=utcnow_seconds(), - expiration_date=utcnow_seconds(), - creator_account_id="000000000", - ) - db.add(db_runlink) - db.commit() - db.rollback() - - def test_get_version(client): r = client.get("/version") j = r.json() From bd13f130378f3da998702121f406a5af43f593b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 17:51:14 +0100 Subject: [PATCH 10/12] Drop unneeded no cov directive since it's now covered --- .../oonirun/alembic/versions/981d92cf8790_init_tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py b/ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py index c92919cd..0b937bd2 100644 --- a/ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py +++ b/ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py @@ -79,6 +79,6 @@ def upgrade() -> None: ) -def downgrade() -> None: # no cov +def downgrade() -> None: op.drop_table("oonirun_nettest") op.drop_table("oonirun") From 5d110f28be494c5fea833a8f18267a2414a8531a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 17:54:46 +0100 Subject: [PATCH 11/12] Bump to release candidate version --- ooniapi/services/oonirun/src/oonirun/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/oonirun/src/oonirun/__about__.py b/ooniapi/services/oonirun/src/oonirun/__about__.py index 342f1d78..d04bdc97 100644 --- a/ooniapi/services/oonirun/src/oonirun/__about__.py +++ b/ooniapi/services/oonirun/src/oonirun/__about__.py @@ -1 +1 @@ -VERSION = "0.5.0dev1" +VERSION = "0.5.0rc0" From a70597cbf692d6fc1208f1dd930cdf2e6d5e9ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 13 Mar 2024 17:57:36 +0100 Subject: [PATCH 12/12] Run legacy api tests only on master since they take too long --- .github/workflows/test_fastpath.yml | 5 ++++- .github/workflows/test_legacy_ooniapi.yml | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_fastpath.yml b/.github/workflows/test_fastpath.yml index c03e5a3f..7e039e3d 100644 --- a/.github/workflows/test_fastpath.yml +++ b/.github/workflows/test_fastpath.yml @@ -1,5 +1,8 @@ name: test fastpath -on: push +on: + push: + branches: + - master jobs: test: diff --git a/.github/workflows/test_legacy_ooniapi.yml b/.github/workflows/test_legacy_ooniapi.yml index 2b551738..5b1c8db7 100644 --- a/.github/workflows/test_legacy_ooniapi.yml +++ b/.github/workflows/test_legacy_ooniapi.yml @@ -1,6 +1,8 @@ name: test legacy/ooniapi on: push: + branches: + - master workflow_dispatch: inputs: debug_enabled: