add changes and formatting updates

CHANGES.md

@@ -15,7 +15,32 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+
+- Add integration test framework
+
+  This update adds a framework for testing the deployed stack using pytest. This will allow developers to check
+  that their changes are consistent with the existing stack and to add additionally tests when new functionality
+  is introduced. 
+
+  Changes to implement this include:
+
+  - existing unit tests are moved to the `tests/unit/` directory
+  - new integration tests are written in the `tests/integration/` directory. More tests will be added in the
+    future!
+  - `conftest.py` scripts updated to bring the stack up/down in a consistent way for the integration tests.
+  - unit tests updated to accomodate new testing infrastructure as needed.
+  - unit tests updated to test logging outputs better
+  - `birdhouse` interface script updated to support testing infrastructure (this should not change anything for
+    other end-users).
+  - additional documentation added to `birdhouse` interface to improve user experience.
+  - docker healthchecks added to more components so that the readiness of the stack can be determined with or
+    without the use of the `canarie-api` component.
+
+  Next steps:
+
+  - add more integration tests as needed
+  - add a framework for testing migrating the stack from one version to another
 
 [2.7.1](https://github.com/bird-house/birdhouse-deploy/tree/2.7.1) (2024-12-20)
 ------------------------------------------------------------------------------------------------------------------

tests/conftest.py

@@ -2,6 +2,7 @@
 import pathlib
 import pytest
 
+
 # These are used for integration tests only but are defined here so that tests run from the top level tests/
 # directory will execute without error.
 def pytest_addoption(parser):
@@ -27,6 +28,7 @@ def pytest_addoption(parser):
         help="Number of seconds to wait for the stack to be healthy after it starts up",
     )
 
+
 @pytest.fixture(scope="module")
 def root_dir(request):
     # implement this for every testing subfolder
@@ -35,6 +37,4 @@ def root_dir(request):
 
 @pytest.fixture(scope="module")
 def local_env_file(root_dir):
-    yield pathlib.Path(
-        os.getenv("TEST_BIRDHOUSE_LOCAL_ENV", root_dir / "tests" / "env.local.test")
-    )
+    yield pathlib.Path(os.getenv("TEST_BIRDHOUSE_LOCAL_ENV", root_dir / "tests" / "env.local.test"))

tests/integration/conftest.py

@@ -37,9 +37,7 @@ def check_stack_not_running(request):
     project_name = os.getenv("COMPOSE_PROJECT_NAME", "birdhouse")
     lines = proc.stdout.splitlines()
     if test_project_name in lines or project_name in lines:
-        pytest.fail(
-            "Birdhouse is currently running. Please stop the software before running tests."
-        )
+        pytest.fail("Birdhouse is currently running. Please stop the software before running tests.")
 
 
 @pytest.fixture(scope="session")
@@ -70,9 +68,7 @@ def load_stack_env(cli_path, local_env_file, stack_info):
     )
     if proc.returncode:
         pytest.fail(f"Unable to get environment variables. Error:\n{proc.stderr}")
-    env_vars = dict(
-        line.split("=", 1) for line in proc.stdout.split("\x00") if "=" in line
-    )
+    env_vars = dict(line.split("=", 1) for line in proc.stdout.split("\x00") if "=" in line)
     print(env_vars["MAGPIE_ADMIN_USERNAME"])
     stack_info["env_vars"] = env_vars
 
@@ -84,15 +80,11 @@ def stack_env(stack_info):
 
 @pytest.fixture(scope="module")
 def birdhouse_url(stack_env):
-    return (
-        f"{stack_env['BIRDHOUSE_PROXY_SCHEME']}://{stack_env['BIRDHOUSE_FQDN_PUBLIC']}"
-    )
+    return f"{stack_env['BIRDHOUSE_PROXY_SCHEME']}://{stack_env['BIRDHOUSE_FQDN_PUBLIC']}"
 
 
 @pytest.fixture(scope="module", autouse=True)
-def start_stack(
-    request, cli_path, local_env_file, tmp_data_persist_root, stack_info, pytestconfig
-):
+def start_stack(request, cli_path, local_env_file, tmp_data_persist_root, stack_info, pytestconfig):
     """
     Starts the birdhouse stack at the beginning of the test session.
 
@@ -139,8 +131,7 @@ def start_stack(
     while start + timeout > time.time():
         proc = subprocess.run(
             'docker inspect --format \'{{if .State.Health}}{"health": {{json .State.Health.Status}}, '
-            '"name": {{json .Name}}}{{end}}\' '
-            + containers_proc.stdout.replace("\n", " "),
+            '"name": {{json .Name}}}{{end}}\' ' + containers_proc.stdout.replace("\n", " "),
             shell=True,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
@@ -153,13 +144,9 @@ def start_stack(
             if status.strip():
                 status = json.loads(status)
                 if status["health"] != "healthy":
-                    health_stats.append(
-                        f"name: '{status['name'].strip().strip('/')}', status: '{status['health']}'"
-                    )
+                    health_stats.append(f"name: '{status['name'].strip().strip('/')}', status: '{status['health']}'")
         if any(health_stats):
-            msg = "Waiting on the following containers to be healthy:\n" + "\n".join(
-                health_stats
-            )
+            msg = "Waiting on the following containers to be healthy:\n" + "\n".join(health_stats)
             print(msg, file=sys.stderr)
         else:
             break
@@ -179,8 +166,7 @@ def stop_stack(request, stack_info, pytestconfig):
     flags = "-s" if pytestconfig.option.capture == "no" else ""
     if stack_info["started"] and not request.config.getoption("--no-stop-stack", None):
         proc = subprocess.run(
-            f"{stack_info['cli_path']} {flags} -e '{stack_info['local_env_file']}' "
-            "compose down -v --remove-orphans",
+            f"{stack_info['cli_path']} {flags} -e '{stack_info['local_env_file']}' " "compose down -v --remove-orphans",
             shell=True,
             stderr=subprocess.PIPE,
             universal_newlines=True,

tests/integration/test_magpie.py

@@ -22,7 +22,4 @@ def test_admin_can_log_in(magpie_url, stack_env):
         },
     )
     response.raise_for_status()
-    assert any(
-        cookie.domain == stack_env["BIRDHOUSE_FQDN_PUBLIC"]
-        for cookie in response.cookies
-    )
+    assert any(cookie.domain == stack_env["BIRDHOUSE_FQDN_PUBLIC"] for cookie in response.cookies)

tests/unit/test_cli.py

@@ -124,9 +124,7 @@ def test_compose_backwards_compatible(cli_path, run, printenv_script, flag):
 
 
 @pytest.mark.parametrize("flag", ["--env-file ", "-e ", "--env-file=", "-e="])
-def test_compose_set_env_file(
-    cli_path, run, printenv_script, local_env_file, tmp_path, flag
-):
+def test_compose_set_env_file(cli_path, run, printenv_script, local_env_file, tmp_path, flag):
     other_local_env_file = tmp_path / "env.local.other"
     with open(local_env_file) as f:
         other_local_env_file.write_text(f.read())
@@ -176,10 +174,7 @@ def test_configs_set_env_file(cli_path, run, local_env_file, tmp_path, flag):
         other_local_env_file.write_text(f.read())
     proc = run(f"{cli_path} {flag}{other_local_env_file} configs -p")
     assert f"BIRDHOUSE_LOCAL_ENV='{other_local_env_file}'" in proc.stdout
-    assert (
-        f"BIRDHOUSE_LOCAL_ENV='{local_env_file}'"
-        in proc.stdout.split(str(other_local_env_file))[-1]
-    )
+    assert f"BIRDHOUSE_LOCAL_ENV='{local_env_file}'" in proc.stdout.split(str(other_local_env_file))[-1]
 
 
 @pytest.mark.parametrize("flag", ["-s", "--log-stdout"])
@@ -245,18 +240,14 @@ def test_log_level(cli_path, run, logging_script, level):
 @pytest.mark.parametrize("level", LOG_LEVELS)
 def test_log_override_stdout(cli_path, run, logging_script, level):
     proc = run(f"{cli_path} -L DEBUG -s {level} compose", compose=logging_script)
-    check_log_output(
-        [level_ for level_ in LOG_CHECK_LEVELS if level_ != level], proc.stderr
-    )
+    check_log_output([level_ for level_ in LOG_CHECK_LEVELS if level_ != level], proc.stderr)
     check_log_output([level], proc.stdout)
 
 
 @pytest.mark.parametrize("level", LOG_LEVELS)
 def test_log_override_quiet(cli_path, run, logging_script, level):
     proc = run(f"{cli_path} -L DEBUG -q {level} compose", compose=logging_script)
-    check_log_output(
-        [level_ for level_ in LOG_CHECK_LEVELS if level_ != level], proc.stderr
-    )
+    check_log_output([level_ for level_ in LOG_CHECK_LEVELS if level_ != level], proc.stderr)
     check_log_output([], proc.stdout)
 
 
@@ -275,8 +266,7 @@ def test_log_override_file(cli_path, run, logging_script, tmp_path, level):
 def test_configs_log_override_multiple(cli_path, run, logging_script, tmp_path):
     log_file = tmp_path / "test.log"
     proc = run(
-        f"{cli_path} -L DEBUG -l DEBUG {log_file} -s INFO "
-        f"-q WARN -l ERROR {log_file} -q ERROR compose",
+        f"{cli_path} -L DEBUG -l DEBUG {log_file} -s INFO " f"-q WARN -l ERROR {log_file} -q ERROR compose",
         compose=logging_script,
     )
     check_log_output(["DEBUG"], proc.stderr)

tests/unit/test_deployment.py

@@ -12,21 +12,15 @@
 TEMPLATE_SUBSTITUTIONS = {
     "BIRDHOUSE_FQDN_PUBLIC": os.environ.get("BIRDHOUSE_FQDN_PUBLIC", "example.com"),
     "WEAVER_MANAGER_NAME": os.environ.get("WEAVER_MANAGER_NAME", "weaver"),
-    "TWITCHER_PROTECTED_PATH": os.environ.get(
-        "TWITCHER_PROTECTED_PATH", "/twitcher/ows/proxy"
-    ),
+    "TWITCHER_PROTECTED_PATH": os.environ.get("TWITCHER_PROTECTED_PATH", "/twitcher/ows/proxy"),
     "BIRDHOUSE_PROXY_SCHEME": os.environ.get("BIRDHOUSE_PROXY_SCHEME", "http"),
 }
 
 
 @pytest.fixture(scope="module")
 def component_paths(root_dir):
     # type: (str) -> List[str]
-    yield [
-        path
-        for loc in COMPONENT_LOCATIONS
-        for path in glob.glob(os.path.join(root_dir, "birdhouse", loc, "*"))
-    ]
+    yield [path for loc in COMPONENT_LOCATIONS for path in glob.glob(os.path.join(root_dir, "birdhouse", loc, "*"))]
 
 
 @pytest.fixture(scope="module")
@@ -51,19 +45,15 @@ def template_substitutions(component_paths):
     return templates
 
 
-@pytest.fixture(
-    scope="module", params=[pytest.lazy_fixture("component_service_configs")]
-)
+@pytest.fixture(scope="module", params=[pytest.lazy_fixture("component_service_configs")])
 def resolved_services_config_schema(request):
     """
     For each of the services provided by ``component_paths`` fixture, obtain the referenced ``$schema``.
 
     If variable ``DACCS_NODE_REGISTRY_BRANCH`` is defined, the referenced ``$schema`` is ignored in favor of it.
     """
     service_config_paths = request.param
-    assert (
-        service_config_paths
-    ), "Invalid service configuration. No service config found."
+    assert service_config_paths, "Invalid service configuration. No service config found."
 
     # test override
     branch = os.environ.get("DACCS_NODE_REGISTRY_BRANCH", None)
@@ -95,23 +85,17 @@ def load_templated_service_config(service_config_path, template_variables):
     Each service configuration file is expected to be an array of 'service' to allow multiple entries.
     """
     with open(service_config_path) as service_config_file:
-        service_config_json = Template(service_config_file.read()).safe_substitute(
-            template_variables
-        )
+        service_config_json = Template(service_config_file.read()).safe_substitute(template_variables)
         service_configs = json.loads(service_config_json)
     return service_configs
 
 
 class TestDockerCompose:
-    def test_service_config_name_same_as_dirname(
-        self, component_service_configs, template_substitutions
-    ):
+    def test_service_config_name_same_as_dirname(self, component_service_configs, template_substitutions):
         invalid_names = []
 
         for service_config_path in component_service_configs:
-            service_configs = load_templated_service_config(
-                service_config_path, template_substitutions
-            )
+            service_configs = load_templated_service_config(service_config_path, template_substitutions)
             invalid_config_names = []
             for service_config in service_configs:
                 config_name = service_config.get("name")
@@ -125,24 +109,16 @@ def test_service_config_name_same_as_dirname(
         assert not invalid_names, "service names in service-config.json.template should match the directory name"
 
     @pytest.mark.online
-    def test_service_config_valid(
-        self, resolved_services_config_schema, template_substitutions
-    ):
+    def test_service_config_valid(self, resolved_services_config_schema, template_substitutions):
         invalid_schemas = []
         for (
             service_config_schema,
             service_config_path,
         ) in resolved_services_config_schema:
-            service_configs = load_templated_service_config(
-                service_config_path, template_substitutions
-            )
+            service_configs = load_templated_service_config(service_config_path, template_substitutions)
             for service_config in service_configs:
                 try:
-                    jsonschema.validate(
-                        instance=service_config, schema=service_config_schema
-                    )
+                    jsonschema.validate(instance=service_config, schema=service_config_schema)
                 except jsonschema.exceptions.ValidationError as e:
-                    invalid_schemas.append(
-                        f"{service_config_path} contains invalid service configuration: {e}"
-                    )
+                    invalid_schemas.append(f"{service_config_path} contains invalid service configuration: {e}")
         assert not invalid_schemas, "\n".join(invalid_schemas)