Skip to content

Commit

Permalink
feat: add docker dev and test workflows
Browse files Browse the repository at this point in the history
  • Loading branch information
jamesgorrie committed Dec 5, 2024
1 parent 8d55ea5 commit b73d15f
Show file tree
Hide file tree
Showing 9 changed files with 361 additions and 61 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -176,3 +176,6 @@ plugins
user_trunk.yaml
user.yaml
tmp

# rds dumps
dumps/
9 changes: 9 additions & 0 deletions Dockerfile.postgres
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
FROM postgres:14

WORKDIR /app

# It's unsure where this is generated or used, but if we don't create it we get a constant stream of logs saying
# `FATAL: role "navigator" does not exist`
RUN cat <<EOF > /docker-entrypoint-initdb.d/navigator-role.sql
CREATE ROLE navigator WITH LOGIN;
EOF
16 changes: 16 additions & 0 deletions Dockerfile.webapp
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
FROM python:3.10-slim

WORKDIR /usr/src
ENV PYTHONPATH=/usr/src

RUN apt-get update && \
apt-get install -y --no-install-recommends \
postgresql-client && \
rm -rf /var/lib/apt/lists/*

RUN pip install --no-cache-dir poetry==1.7.1
COPY poetry.lock pyproject.toml ./
RUN poetry config virtualenvs.create false && poetry install --no-directory --no-root

COPY ./app ./app
CMD python app/main.py
99 changes: 41 additions & 58 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,72 +1,55 @@
install_trunk:
$(eval trunk_installed=$(shell trunk --version > /dev/null 2>&1 ; echo $$? ))
ifneq (${trunk_installed},0)
$(eval OS_NAME=$(shell uname -s | tr A-Z a-z))
curl https://get.trunk.io -fsSL | bash
endif

uninstall_trunk:
sudo rm -if `which trunk`
rm -ifr ${HOME}/.cache/trunk
# <!-- please update this list in the Readme "Global dependencies" list if you add others
GLOBAL_DEPENDENCIES = trunk poetry aws docker

create_env:
# Copy .env
cp .env.example .env
# --- Setup --- #
bootstrap: bootstrap_check_dependencies bootstrap_configure_pyright

configure_pyright:
trunk actions run configure-pyright

setup_with_pyenv: install_trunk create_env ## Sets up a local dev environment using Pyenv
$(eval venv_name=$(shell grep 'venv =' pyproject.toml | cut -d '"' -f 2 ))
if [ -n "$(venv_name)" ] && ! pyenv versions --bare | grep -q "^$(venv_name)$$"; then \
$(eval python_version=$(shell grep 'python =' pyproject.toml | cut -d '"' -f 2 | sed 's/^\^//')) \
$(eval pyenv_version=$(shell pyenv versions --bare | grep$(python_version) )) \
pyenv virtualenv $(pyenv_version) $(venv_name); \
fi
@eval "$$(pyenv init -)" && \
pyenv activate $(venv_name) && \
poetry install

make configure_pyright

check:
trunk fmt
trunk check
bootstrap_check_dependencies:
@for dep in $(GLOBAL_DEPENDENCIES); do \
if ! command -v $${dep} &> /dev/null; then \
echo "$${dep} is not installed. See readme for links to instructions."; \
exit 1; \
fi \
done

build:
docker build -t navigator-admin-backend .
bootstrap_configure_pyright:
trunk actions run configure-pyright

build_dev:
docker compose build

unit_test: build
docker compose run --rm navigator-admin-backend pytest -vvv tests/unit_tests
# --- Dev ---#
dev: dev_rds_dump
docker-compose -f docker-compose-dev.yml up

integration_test: build_dev
docker compose run --rm navigator-admin-backend pytest -vvv tests/integration_tests
dev_rds_dump:
[ ! -f ./dumps/navigator.sql ] && aws --profile staging s3 cp s3://cpr-staging-rds/dumps/navigator.sql ./dumps/navigator.sql

test: build_dev
docker compose run --rm navigator-admin-backend pytest -vvv tests
# this should only need to be run if there are significant schema changes
# TODO: make this a little more inteligent
dev_rds_dump_update:
aws --profile staging s3 cp s3://cpr-staging-rds/dumps/navigator.sql ./dumps/navigator.sql

test_local:
poetry run pytest -vvv tests/integration_tests

run:
docker compose -f docker-compose.yml up -d --remove-orphans
# --- Test --- #
# If you'd like to run more specific, you can run e.g.
# - `make test TEST=tests/integration_tests`
# - `make test TEST=tests/unit_tests`
# - `make test TEST=tests/integration_tests/ingest/test_ingest.py::test_ingest_when_ok`
#
# We've left it specifically unprefixed so autocomplete works in the terminal
override TEST ?=
test:
TEST=$(TEST) docker-compose -f docker-compose-test.yml up --exit-code-from webapp

start: build_dev run
# --- CI --- #
build:
docker build --tag navigator-admin-backend .

start_local: build
# - docker stop navigator-admin-backend
docker run -p 8888:8888 \
--name navigator-admin-backend \
--network=navigator-backend_default \
-e ADMIN_POSTGRES_HOST=backend_db \
-e SECRET_KEY="secret_test_key" \
-d navigator-admin-backend
build_dev:
docker compose build

restart: build
docker stop navigator-admin-backend && docker rm navigator-admin-backend && make start_local && docker logs -f navigator-admin-backend

show_logs:
- docker compose logs -f
# --- Clean --- #
clean:
docker compose down
rm ./dumps/navigator.sql
40 changes: 39 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,44 @@
# navigator-admin-backend

Backend for the Admin Pages
Backend for the Admin webapp

## Getting started

### Global dependencies

These are global tools and libraries that are required to run the backend:

<!-- trunk-ignore(markdownlint/MD013) -->
<!-- please update this list in the Makefile GLOBAL_DEPENDENCIES if you add others -->

- [trunk](https://docs.trunk.io/cli/install#using-curl)
- [poetry](https://python-poetry.org/docs/#installation)
- [awscli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
- [docker](https://docs.docker.com/desktop/)

### Development

To get started run:

```shell
make bootstrap
```

To get a running version of the API run:

```shell
make dev
```

To run the test suite run:

```shell
make test
```

```shell
make test
```

## Background

Expand Down
37 changes: 37 additions & 0 deletions docker-compose-dev.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
services:
webapp:
command: poetry run fastapi dev --port 8888 --host 0.0.0.0
build:
context: ./
dockerfile: Dockerfile.webapp
tty: true
volumes:
- ./app:/usr/src/app
ports:
- 8888:8888
environment:
PYTHONPATH: .
ADMIN_POSTGRES_HOST: postgres
ADMIN_POSTGRES_USER: navigator_admin
ADMIN_POSTGRES_DATABASE: navigator
ADMIN_POSTGRES_PASSWORD: password
SECRET_KEY: ???
depends_on:
postgres:
condition: service_healthy

postgres:
environment:
POSTGRES_DB: navigator
POSTGRES_PASSWORD: password
POSTGRES_USER: navigator_admin
build:
context: .
dockerfile: ./Dockerfile.postgres
volumes:
- ./dumps/navigator.sql:/docker-entrypoint-initdb.d/navigator.sql
healthcheck:
test: [CMD-SHELL, "pg_isready -U ${POSTGRES_USER}"]
interval: 5s
timeout: 3s
retries: 30
41 changes: 41 additions & 0 deletions docker-compose-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
services:
webapp:
command: poetry run pytest --disable-warnings ${TEST}
build:
context: ./
dockerfile: Dockerfile.webapp
tty: true
volumes:
- ./tests:/usr/src/tests
environment:
PYTHONPATH: .
ADMIN_POSTGRES_HOST: postgres
ADMIN_POSTGRES_USER: navigator_admin
ADMIN_POSTGRES_PASSWORD: password
ADMIN_POSTGRES_DATABASE: navigator
PGUSER: navigator_admin
PGPASSWORD: password
PGHOST: postgres
PGDATABASE: navigator
SECRET_KEY: ???
# API_HOST: http://navigator-admin-backend:8888
# This is for when people explicitly set the tests they want to run
# It is used in the makefile
TEST: ${TEST}
depends_on:
postgres:
condition: service_healthy

postgres:
environment:
POSTGRES_USER: navigator_admin
POSTGRES_PASSWORD: password
POSTGRES_DB: navigator
build:
context: .
dockerfile: ./Dockerfile.postgres
healthcheck:
test: [CMD-SHELL, "pg_isready -U ${POSTGRES_USER}"]
interval: 5s
timeout: 3s
retries: 30
Loading

0 comments on commit b73d15f

Please sign in to comment.