bird-house · mishaschwartz · Jan 6, 2025 · Jan 7, 2025 · Jan 7, 2025 · Jan 7, 2025
@@ -18,4 +18,4 @@ jobs:
           pip install -r ./tests/requirements.txt
       - name: Test with pytest
         run: |
-          pytest ./tests/
+          pytest ./tests/unit
@@ -15,7 +15,32 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+
+- Add integration test framework
+
+  This update adds a framework for testing the deployed stack using pytest. This will allow developers to check
+  that their changes are consistent with the existing stack and to add additionally tests when new functionality
+  is introduced. 
+
+  Changes to implement this include:
+
+  - existing unit tests are moved to the `tests/unit/` directory
+  - new integration tests are written in the `tests/integration/` directory. More tests will be added in the
+    future!
+  - `conftest.py` scripts updated to bring the stack up/down in a consistent way for the integration tests.
+  - unit tests updated to accomodate new testing infrastructure as needed.
+  - unit tests updated to test logging outputs better
+  - `birdhouse` interface script updated to support testing infrastructure (this should not change anything for
+    other end-users).
+  - additional documentation added to `birdhouse` interface to improve user experience.
+  - docker healthchecks added to more components so that the readiness of the stack can be determined with or
+    without the use of the `canarie-api` component.
+
+  Next steps:
+
+  - add more integration tests as needed
+  - add a framework for testing migrating the stack from one version to another
 
 [2.7.1](https://github.com/bird-house/birdhouse-deploy/tree/2.7.1) (2024-12-20)
 ------------------------------------------------------------------------------------------------------------------

@@ -10,12 +10,11 @@ export __BIRDHOUSE_SUPPORTED_INTERFACE=True
 
 USAGE="USAGE: $THIS_BASENAME [-h|--help] 
                              [-b|--backwards-compatible]
-                             [-e|--env-file local-env-file]
-                             [-q|--quiet] [-q|--quiet {DEBUG|INFO|WARN|ERROR|CRITICAL}]
-                             [-s|--log-stdout] [-s|--log-stdout {DEBUG|INFO|WARN|ERROR|CRITICAL}]
-                             [-l|--log-file log-file-path]
-                             [-l|--log-file {DEBUG|INFO|WARN|ERROR|CRITICAL} log-file-path]
-                             [-L|--log-level {DEBUG|INFO|WARN|ERROR|CRITICAL}]
+                             [-e|--env-file path]
+                             [-q|--quiet [level]]
+                             [-s|--log-stdout [level]]
+                             [-l|--log-file [level] path]
+                             [-L|--log-level level]
                              {info|compose|configs}"
 USAGE=$(echo $USAGE | tr "\n" " ")
 
@@ -25,13 +24,13 @@ Manage the Birdhouse software stack.
 
 Commands:
   info      Print build information
-  compose   Run a \"docker compose\" command for the Birdhouse project
+  compose   Run a \"docker compose\" command for the Birdhouse project (run '${THIS_BASENAME} compose --help' for more options)
   configs   Load or execute commands in the Birdhouse configuration environment
 
 Options:
   -h, --help                                              Print this message and exit
   -b, --backwards-compatible                              Run in backwards compatible mode
-  -e, --env-file string                                   Override the local environment file, default is ${COMPOSE_DIR}/env.local
+  -e, --env-file path                                     Override the local environment file, default is ${COMPOSE_DIR}/env.local
   -s, --log-stdout                                        Write logs to stdout for all log levels, default is to write to stderr
   -s, --log-stdout {DEBUG|INFO|WARN|ERROR|CRITICAL}       Write logs to stdout for the given log level only (this option can be repeated)
   -l, --log-file path                                     Write logs to this file path for all log levels
@@ -41,12 +40,13 @@ Options:
   -L, --log-level {DEBUG|INFO|WARN|ERROR}                 Set log level, default is INFO
 "
 
-CONFIGS_USAGE="USAGE: $THIS_BASENAME configs [-h|--help] [-d|--default] {[-p|--print-config-command] | [-c|--command command]}"
+CONFIGS_USAGE="USAGE: $THIS_BASENAME [${THIS_BASENAME} options] configs [-h|--help] [-d|--default] {[-p|--print-config-command] | [-c|--command command]}"
 CONFIGS_HELP="$CONFIGS_USAGE
 
 Load or execute commands in the Birdhouse configuration environment.
 
 Options:
+  -h, --help                   Print this message and exit
   -d, --default                Only load/print a command for the default configuration settings, not those specified by the local environment file
   -p, --print-config-command   Print a command that can be used to load configuration settings as environment variables
   -c, --command string         Execute the given command after loading configuration settings
@@ -78,6 +78,7 @@ READ_CONFIGS_CMD=read_configs
 #   $ echo $BIRDHOUSE_BACKWARD_COMPATIBLE_ALLOWED
 #   True
 print_config_command() {
+  option="$1"
   configs_cmd_prefix="export __BIRDHOUSE_SUPPORTED_INTERFACE=True ;"
   configs_cmd_suffix="unset __BIRDHOUSE_SUPPORTED_INTERFACE ;"
   if [ "${BIRDHOUSE_BACKWARD_COMPATIBLE_ALLOWED+set}" = 'set' ]; then
@@ -98,6 +99,9 @@ print_config_command() {
     configs_cmd_suffix="${configs_cmd_suffix} unset BIRDHOUSE_LOCAL_ENV ;"
   fi
 
+  if [ "${option}" = "no-suffix" ]; then
+    configs_cmd_suffix=
+  fi
   echo "${configs_cmd_prefix} . ${COMPOSE_DIR}/read-configs.include.sh; ${READ_CONFIGS_CMD} ; ${configs_cmd_suffix}"
 }
 
@@ -167,9 +171,9 @@ parse_configs_args() {
         print_config_command
       elif [ "${CONFIGS_CMD+set}" = 'set' ]; then
         if [ "${CONFIGS_QUIET}" = "True" ]; then
-          eval "$(print_config_command)" 2> /dev/null
+          eval "$(print_config_command no-suffix)" 2> /dev/null
         else
-          eval "$(print_config_command)"
+          eval "$(print_config_command no-suffix)"
         fi
         exec /usr/bin/env sh -c "${CONFIGS_CMD}"
       else
@@ -285,7 +289,7 @@ parse_args() {
       parse_configs_args "$@"
       ;;
     -h|--help)
-      echo "$HELP"
+      echo "$HELP" | more
       ;;
     -??*)
       parse_multiple_short_flags parse_args "$@"

@@ -91,12 +91,14 @@ done
 export BIRDHOUSE_AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES
 
 # we apply all the templates
-find ${ALL_CONF_DIRS} -name '*.template' 2>/dev/null |
-  while read FILE
-  do
-    DEST=${FILE%.template}
-    cat "${FILE}" | envsubst "$VARS" | envsubst "$OPTIONAL_VARS" > "${DEST}"
-  done
+if [ x"$1" = x"up" ]; then
+  find ${ALL_CONF_DIRS} -name '*.template' 2>/dev/null |
+    while read FILE
+    do
+      DEST=${FILE%.template}
+      cat "${FILE}" | envsubst "$VARS" | envsubst "$OPTIONAL_VARS" > "${DEST}"
+    done
+fi
 
 SHELL_EXEC_FLAGS=
 if [ "${BIRDHOUSE_LOG_LEVEL}" = "DEBUG" ]; then

@@ -21,6 +21,13 @@ services:
     restart: always
     entrypoint: /entrypoint
     logging: *default-logging
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:2000"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
 
   canarie-api-cron:
     image: ${CANARIE_IMAGE}

@@ -45,6 +45,13 @@ services:
       - "${BIRDHOUSE_DATA_PERSIST_SHARED_ROOT}:${BIRDHOUSE_DATA_PERSIST_SHARED_ROOT}"
     restart: always
     logging: *default-logging
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://0.0.0.0:7000"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
 
   cowbird-worker:
     image: pavics/cowbird:${COWBIRD_VERSION}-worker
@@ -74,6 +81,7 @@ services:
       - "${BIRDHOUSE_DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:${BIRDHOUSE_DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}"
     restart: always
     logging: *default-logging
+    # TODO: create a stable healthcheck for this service (`inspect ping` or `status` is not reliable)
 
   # Dedicated database for Cowbird, since other 'mongodb' image does not employ the same version.
   cowbird-mongodb:
@@ -85,6 +93,13 @@ services:
       - ${COWBIRD_MONGODB_DATA_DIR}:/data/db
     restart: always
     logging: *default-logging
+    healthcheck:
+      test: ["CMD", "mongo", "--eval", "print('connected')"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
 
 networks:
   cowbird-mongodb: {}
@@ -31,10 +31,12 @@ services:
     restart: always
     logging: *default-logging
     healthcheck:
-      test: curl --fail --silent http://localhost:8080/ || exit 1
+      test: curl --fail --silent http://localhost:8080/geoserver || exit 1
       interval: 1m30s
       timeout: 10s
       retries: 3
+      start_period: 30s
+      start_interval: 5s
 
   postgis:
     image: pavics/postgis:2.2

@@ -46,6 +46,13 @@ services:
     command: bash -c "jupyterhub upgrade-db && jupyterhub"
     restart: always
     logging: *default-logging
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8000/jupyter"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
 
 # need external network so the folder name is not prefixed to network name
 networks:

@@ -35,6 +35,13 @@ services:
       - ./components/magpie/magpie.ini:/opt/local/src/magpie/config/magpie.ini
     restart: always
     logging: *default-logging
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://0.0.0.0:2001"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
 
   postgres-magpie:
     image: postgres:9.6
@@ -49,3 +56,10 @@ services:
       - ./components/magpie/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
     restart: always
     logging: *default-logging
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
@@ -19,6 +19,13 @@ services:
     command: bash -c 'chown -R mongodb:mongodb /data && chmod -R 755 /data && mongod'
     restart: always
     logging: *default-logging
-
+    healthcheck:
+      test: ["CMD", "mongo", "--eval", "print('connected')"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
+
 networks:
   mongodb: {}
@@ -14,6 +14,13 @@ services:
     devices:
       - /dev/kmsg
     restart: always
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:8080"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
 
   # https://github.com/prometheus/node_exporter
   # Collect system-wide metrics.
@@ -26,6 +33,13 @@ services:
     pid: "host"
     command: --path.rootfs=/host
     restart: always
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:9100"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
 
   # https://prometheus.io/docs/prometheus/latest/installation
   # Monitor and store collected metrics.
@@ -47,6 +61,13 @@ services:
       # wrong default was http://container-hash:9090/
       - --web.external-url=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/prometheus/
     restart: always
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:9090"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
 
   # https://grafana.com/docs/grafana/latest/installation/docker/
   # https://grafana.com/docs/grafana/latest/installation/configure-docker/
@@ -65,6 +86,13 @@ services:
       GF_SERVER_SERVE_FROM_SUB_PATH: 'true'
       GF_SERVER_DOMAIN: ${BIRDHOUSE_FQDN_PUBLIC}
     restart: always
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:3000"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
 
   # https://github.com/prometheus/alertmanager
   # https://prometheus.io/docs/alerting/latest/overview/
@@ -85,6 +113,13 @@ services:
       # wrong default was http://container-hash:9093/
       - --web.external-url=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/alertmanager
     restart: always
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:9093"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
 
 volumes:
   prometheus_persistence:

@@ -22,3 +22,10 @@ services:
       - ./components/postgres/credentials.env
     restart: always
     logging: *default-logging
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s
@@ -22,3 +22,10 @@ services:
       REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
     restart: always
     logging: *default-logging
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "${BIRDHOUSE_FQDN_PUBLIC}"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+      start_interval: 5s