Refactor testing (#3357)

* init more tests * Update snapshots * update fixtures * sort files * Remove tests before running hassfest * fix json * more test adjustments * lint * add psutil_home_assistant test requirement * sqlalchemy * fnv_hash_fast * more * PIL * Pillow * janus * hass_frontend * home-assistant-frontend * full entry for newer * lint * get entity for update test * renames * pin test packages * single t * updates * fix stuff * store entities and ws for setup test * remove base * Add basic fixtures for all categories * Use WS command to register custom repository * lint * cleanup * unused import * Test addons and core * start generate tests * Output all calls to proxy * change messaging * lint * remove old * plugin * lint * try with minor_version first * theme * lint * Setup all categories for tests * lint * Cleanup stale * Remove unnecessary test files * Remove aresponses * remove proxy_session usage * more! * remove backup arg * fix * lint * Remove autouse from setup_integration fixture * add entities to snapshots * Fix HACS test manifest * filter calls * replace diagnostics test * Fix sorting * better output * adjust resp * ignore minor_version * replace print in test * add appdaemon * cleanup * lint * Add test for removing repositories * lint * update output * extend generate * More usage of common test data * data client test * output * add python_script tests * typo * enable all * config_flow tests * remaining generate * use xfail * lint * lint * sort * adjust * cover bad cases * one more * update snapshots * init better action testing * track loader.async_suggest_report_issue * lint * Sort action results * lint... * Guard async_suggest_report_issue * lint * Run dev tests against 3.12 as well * change names * move report check to fixture * fix version selector * Add setuptools to base * fix check_report_issue check * Use set_state to bust cached propery * Use async_test_home_assistant test helpers from homeassistant core repo * Lint * Update snapshots * Align with HA Core 2024.4.0 * Fix test_update_repository_entity * Fix import * Fix test_hacs_action_integration * Disable testing dev core with Python 3.11 * Allow more issues * Use Python 3.12 run in codecov * Enable config flow tests * Adjust tests * Update generate_category_data tests * Clear stored responses after usage * Rename tests/z_action to tests/action * Update snapshot * Update action test * Remove cleanups * use with expectation * fix typing * Use python version from matrix * Test option flow * Lint * Update snapshots * Use token constant --------- Co-authored-by: Erik <[email protected]>
hacs · Apr 10, 2024 · 3f09f5f · 3f09f5f
1 parent ff00c18
commit 3f09f5f
Show file tree

Hide file tree

Showing 286 changed files with 16,449 additions and 2,273 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -2,22 +2,11 @@
 source = custom_components
 
 omit =
-    # omit pieces of code that rely on external API calls
-    # These should be mocked one day....
-    custom_components/hacs/helpers/get_defaults.py
-    custom_components/hacs/operational/setup_actions/load_hacs_repository.py
-    custom_components/hacs/operational/factory.py
-    custom_components/hacs/operational/relaod.py
-    custom_components/hacs/operational/remove.py
-    custom_components/hacs/operational/setup.py
-    custom_components/hacs/config_flow.py
-    custom_components/hacs/__init__.py
-
     # omit tests
     tests/*
 
     # omit scripts
-    scripts/*
+    scripts/update/*
 
 [report]
 exclude_lines =

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -13,45 +13,8 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run:
-    name: With pytest for Python ${{ matrix.python-version }}
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.11"]
-    steps:
-      - name: 📥 Checkout the repository
-        uses: actions/[email protected]
-
-      - name: 🛠️ Set up Python
-        uses: actions/[email protected]
-        id: python
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: |
-            requirements_base.txt
-            requirements_test.txt
-
-      - name: 📦 Install requirements
-        run: |
-          scripts/install/frontend
-          scripts/install/pip_packages --requirement requirements_test.txt
-
-      - name: 🏃 Run tests
-        env:
-          PYTEST: true
-        run: |
-          python3 -m pytest
-
-      - name: 📤 Upload coverage to Codecov
-        if: ${{ matrix.python-version == '3.11' }}
-        run: |
-          scripts/coverage
-          curl -sfSL https://codecov.io/bash | bash -
-
   legacy:
-    name: With pytest with Home Assistant 2023.6.0
+    name: With pytest with Home Assistant (min. supported version)
     runs-on: ubuntu-latest
     steps:
       - name: 📥 Checkout the repository
@@ -78,17 +41,23 @@ jobs:
         run: |
           python3 -m pytest
 
-  core_dev:
-    name: With pytest with Home Assistant dev
+  dev:
+    name: With pytest with Home Assistant (${{ matrix.homeassistant-version }}) & Python (${{ matrix.python-version }})
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        homeassistant-version:
+          - "dev"
+        python-version:
+          - "3.12"
     steps:
       - name: 📥 Checkout the repository
         uses: actions/[email protected]
 
-      - name: 🛠️ Set up Python 3.12
+      - name: 🛠️ Set up Python ${{ matrix.python-version }}
         uses: actions/[email protected]
         with:
-          python-version: "3.12"
+          python-version: ${{ matrix.python-version }}
           cache: 'pip'
           cache-dependency-path: |
             requirements_base.txt
@@ -105,3 +74,9 @@ jobs:
           PYTEST: true
         run: |
           python3 -m pytest
+
+      - name: 📤 Upload coverage to Codecov
+        if: ${{ matrix.python-version == '3.12' }}
+        run: |
+          scripts/coverage
+          curl -sfSL https://codecov.io/bash | bash -
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
@@ -44,6 +44,10 @@ jobs:
       - name: Checkout the repository
         uses: actions/[email protected]
 
+      # Test files conflics with running hassfest
+      - name: Remove tests
+        run: rm -rf tests
+
       - name: Hassfest validation
         uses: "home-assistant/actions/hassfest@master"
 

diff --git a/action/action.py b/action/action.py
@@ -168,5 +168,5 @@ async def validate_repository(hacs, repository, category, ref=None):
     except HacsException as exception:
         error(exception)
 
-
-asyncio.run(preflight())
+if __name__ == "__main__":
+    asyncio.run(preflight())
diff --git a/pyproject.toml b/pyproject.toml
@@ -19,7 +19,8 @@ combine_as_imports = true
 
 
 [tool.pytest.ini_options]
-addopts = "-rxf -x -v -l --cov=./ --cov-report=xml"
+asyncio_mode="auto"
+addopts = "-rf -v -l --cov=./ --cov-report=xml"
 filterwarnings = [
     "ignore::DeprecationWarning",
 ]

diff --git a/requirements_base.txt b/requirements_base.txt
@@ -4,4 +4,4 @@ aiohttp_cors==0.7.0
 async-timeout>=4.0.3
 asynctest==0.13.0
 colorlog==6.7.0
-
+setuptools==68.0.0
diff --git a/requirements_test.txt b/requirements_test.txt
@@ -1,8 +1,15 @@
 -r requirements_base.txt
-aresponses==2.1.6
 asynctest==0.13.0
+fnv-hash-fast==0.5.0
+home-assistant-frontend
 homeassistant
+janus==1.0.0
+Pillow==10.1.0
+psutil-home-assistant==0.0.1
 pytest==7.4.3
 pytest-asyncio==0.21.1
 pytest-cov==4.1.0
+pytest-snapshot==0.9.0
 pytest-socket==0.6.0
+RestrictedPython==7.0
+SQLAlchemy==2.0.23
diff --git a/scripts/snapshot-update b/scripts/snapshot-update
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+python3 -m \
+    pytest \
+    tests \
+    --snapshot-update
+
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -0,0 +1,28 @@
+import json
+
+from awesomeversion import AwesomeVersion
+from homeassistant import const, core, loader
+
+_async_suggest_report_issue_mock_call_tracker = []
+
+try:
+    _orig_async_suggest_report_issue = loader.async_suggest_report_issue
+except AttributeError:
+    if AwesomeVersion(const.__version__) >= "2023.11.0":
+        raise RuntimeError("loader.async_suggest_report_issue does not exist")
+
+    @core.callback
+    def _fallback_async_suggest_report_issue(*args, **kwargs):
+        return json.dumps(kwargs)
+
+    _orig_async_suggest_report_issue = _fallback_async_suggest_report_issue
+
+
+@core.callback
+def async_suggest_report_issue_mock(*args, **kwargs):
+    result = _orig_async_suggest_report_issue(*args, **kwargs)
+    _async_suggest_report_issue_mock_call_tracker.append(result)
+    return result
+
+
+loader.async_suggest_report_issue = async_suggest_report_issue_mock
diff --git a/tests/action/test_hacs_action_integration.py b/tests/action/test_hacs_action_integration.py
@@ -0,0 +1,73 @@
+"""Tests for the HACS action."""
+import base64
+import json
+import os
+from unittest import mock
+
+import pytest
+
+from tests.common import TOKEN, MockedResponse, ResponseMocker, current_function_name
+from tests.conftest import SnapshotFixture
+
+
+@pytest.mark.parametrize(
+    "key,manifest,succeed",
+    (
+        ("bad_documentation", {"documentation": None}, False),
+        ("bad_issue_tracker", {"issue_tracker": None}, False),
+        ("valid_manifest1", {}, True),
+    ),
+)
+async def test_hacs_action_integration(
+    key: str,
+    manifest: dict[str, str | bool | None],
+    succeed: bool,
+    caplog: pytest.LogCaptureFixture,
+    response_mocker: ResponseMocker,
+    snapshots: SnapshotFixture,
+):
+    """Test the action."""
+    basemanifest = {
+        "domain": "example",
+        "version": "1.0.0",
+        "documentation": "https://example.com",
+        "codeowners": ["hacs-test-org"],
+        "issue_tracker": "https://example.com",
+        "name": "Example",
+    }
+    envpatch = {
+        "INPUT_GITHUB_TOKEN": TOKEN,
+        "INPUT_REPOSITORY": "hacs-test-org/integration-basic",
+        "INPUT_CATEGORY": "integration",
+    }
+
+    response_mocker.add(
+        "https://brands.home-assistant.io/domains.json",
+        response=MockedResponse(status=200, content={"custom": ["example"]}),
+    )
+    response_mocker.add(
+        f"https://api.github.com/repos/hacs-test-org/integration-basic/contents/custom_components/example/manifest.json",
+        response=MockedResponse(
+            status=200,
+            content={
+                "content": base64.b64encode(
+                    json.dumps({**basemanifest, **manifest}).encode("ascii")
+                ).decode("ascii")
+            },
+            keep=True,
+        ),
+    )
+
+    with mock.patch.dict(os.environ, envpatch), mock.patch("builtins.exit", mock.MagicMock()):
+        from action.action import preflight
+
+        await preflight()
+
+    assert ("All (8) checks passed" if succeed else "1/8 checks failed") in caplog.text
+
+    splitlines = [f"<{l.rsplit(' <')[1]}" for l in caplog.text.split("\n") if " <" in l]
+
+    snapshots.assert_match(
+        "\n".join(splitlines[0:2] + sorted(splitlines[2:-2]) + splitlines[-2:]),
+        f"action/{current_function_name()}/{key}.log",
+    )
diff --git a/tests/async_mock.py b/tests/async_mock.py