WIP: Improved testing of new features

tests/site/site_distro_finder.json

@@ -15,8 +15,18 @@
                 }
             ]
         ],
-        "downloads": {
-            "cache_root": "hab testable/download/path"
+        "downloads":
+        {
+            "cache_root": "hab testable/download/path",
+            "distros":
+            [
+                [
+                    "hab.distro_finders.df_zip:DistroFinderZip",
+                    "network_server/distro/source"
+                ]
+            ],
+            "install_root": "{relative_root}/distros",
+            "relative_path": "{{distro_name}}_v{{version}}"
         }
     }
 }

tests/templates/site_download.json

@@ -0,0 +1,20 @@
+{
+    "set": {
+        "config_paths": [
+            "{relative_root}/configs"
+        ],
+        "distro_paths": [
+            "{relative_root}/distros/*"
+        ],
+        "downloads": {
+            "cache_root": "{relative_root}/downloads",
+            "distros": [
+                [
+                    "hab.distro_finders.df_zip:DistroFinderZip",
+                    "{{ zip_root }}"
+                ]
+            ],
+            "install_root": "{relative_root}/distros"
+        }
+    }
+}

tests/test_distro_finder.py

@@ -1,8 +1,9 @@
+import glob
 from pathlib import Path
 
 import pytest
 
-from hab import Resolver, Site
+from hab import Resolver, Site, utils
 from hab.distro_finders import df_zip, distro_finder, zip_sidecar
 from hab.parsers import DistroVersion
 
@@ -49,6 +50,28 @@ def test_eq():
     assert a == b
 
 
+@pytest.mark.parametrize(
+    "glob_str,count",
+    (
+        ("{root}/reference*/sh_*", 12),
+        ("{root}/reference/*", 0),
+        ("{root}/reference_scripts/*/*.sh", 20),
+    ),
+)
+def test_glob_path(config_root, glob_str, count):
+    """Ensure `hab.utils.glob_path` returns the expected results."""
+    glob_str = glob_str.format(root=config_root)
+    # Check against the `glob.glob` result.
+    check = sorted([Path(p) for p in glob.glob(glob_str)])
+
+    path_with_glob = Path(glob_str)
+    result = sorted(utils.glob_path(path_with_glob))
+
+    assert result == check
+    # Sanity check to ensure that the expected results were found by `glob.glob`
+    assert len(result) == count
+
+
 @pytest.mark.parametrize("distro_info", ("zip_distro", "zip_distro_sidecar"))
 def test_zip(request, distro_info, helpers, tmp_path):
     # Convert the distro_info parameter to testing values.

tests/test_parsing.py

@@ -21,7 +21,7 @@
 from hab.parsers import Config, DistroVersion, FlatConfig
 
 
-class TestLoadJsonFile:
+class TestLoadJson:
     """Tests various conditions when using `hab.utils.load_json_file` to ensure
     expected output.
     """
@@ -35,16 +35,28 @@ def test_missing(self, tmpdir):
             utils.load_json_file(path)
         assert Path(excinfo.value.filename) == path
 
+    @classmethod
+    def check_exception(cls, excinfo, native_json, path):
+        if native_json:
+            # If built-in json was used, check that filename was appended to the message
+            assert f'Source("{path}")' in str(excinfo.value)
+        else:
+            # If pyjson5 was used, check that the filename was added to str
+            assert f"'source': {str(path)!r}" in str(excinfo.value)
+            # Check that the filename was added to the result dict
+            assert excinfo.value.result["source"] == str(path)
+
     def test_binary(self, tmpdir):
         """If attempting to read a binary file, filename is included in exception.
 
         This is a problem we run into rarely where a text file gets
         replaced/generated with a binary file containing noting but a lot of null bytes.
         """
+        bin_data = b"\x00" * 32
         path = Path(tmpdir) / "binary.json"
         # Create a binary test file containing multiple binary null values.
         with path.open("wb") as fle:
-            fle.write(b"\x00" * 32)
+            fle.write(bin_data)
 
         # Detect if using pyjson5 or not
         native_json = False
@@ -57,15 +69,15 @@ def test_binary(self, tmpdir):
         else:
             exc_type = pyjson5.pyjson5.Json5IllegalCharacter
 
+        # Test load_json_file
         with pytest.raises(exc_type) as excinfo:
             utils.load_json_file(path)
+        self.check_exception(excinfo, native_json, path)
 
-        if native_json:
-            # If built-in json was used, check that filename was appended to the message
-            assert f'Filename("{path}")' in str(excinfo.value)
-        else:
-            # If pyjson5 was used, check that the filename was added to the result dict
-            assert f"{{'filename': {str(path)!r}}}" in str(excinfo.value)
+        # Test loads_json
+        with pytest.raises(exc_type) as excinfo:
+            utils.loads_json(bin_data.decode(), path)
+        self.check_exception(excinfo, native_json, path)
 
     def test_config_load(self, uncached_resolver):
         cfg = Config({}, uncached_resolver)
@@ -78,6 +90,18 @@ def test_config_load(self, uncached_resolver):
         with pytest.raises(FileNotFoundError):
             cfg.load("invalid_path.json")
 
+    def test_loads_json(self, config_root):
+        """Test that `loads_json` is able to parse a valid json string."""
+        filename = config_root / "site_main.json"
+        with filename.open() as fle:
+            text = fle.read()
+        # Test an existing file is able to be parsed successfully.
+        data = utils.loads_json(text, filename)
+        # Spot check that we were able to parse data from the file.
+        assert isinstance(data, dict)
+        assert "append" in data
+        assert "set" in data
+
 
 def test_distro_parse(config_root, resolver):
     """Check that a distro json can be parsed correctly"""
@@ -688,13 +712,7 @@ def test_invalid_config(config_root, resolver):
 
     with pytest.raises(_JsonException) as excinfo:
         Config({}, resolver, filename=path)
-
-    if native_json:
-        # If built-in json was used, check that filename was appended to the message
-        assert f'Filename("{path}")' in str(excinfo.value)
-    else:
-        # If pyjson5 was used, check that the filename was added to the result dict
-        assert excinfo.value.result["filename"] == str(path)
+    TestLoadJson.check_exception(excinfo, native_json, path)
 
 
 def test_misc_coverage(resolver):

tests/test_resolver.py

@@ -3,14 +3,16 @@
 import sys
 from collections import OrderedDict
 from pathlib import Path
+from zipfile import ZipFile
 
 import anytree
 import pytest
 from packaging.requirements import Requirement
 
-from hab import NotSet, Resolver, Site, utils
+from hab import DistroMode, NotSet, Resolver, Site, utils
 from hab.distro_finders.distro_finder import DistroFinder
 from hab.errors import InvalidRequirementError
+from hab.parsers import DistroVersion
 from hab.solvers import Solver
 
 
@@ -119,6 +121,52 @@ def test_closest_config(resolver, path, result, reason):
     assert resolver.closest_config(path).fullpath == result, reason
 
 
+def test_distro_mode(zip_distro, helpers, tmp_path):
+    """Test `Resolver.distro_mode` is respected when calling `distros`.
+
+    Also test that the `distro_mode_override` with context updates and restores
+    the distro_mode.
+    """
+    site_file = tmp_path / "site.json"
+    helpers.render_template(
+        "site_download.json", site_file, zip_root=zip_distro.root.as_posix()
+    )
+    resolver = Resolver(Site([site_file]))
+
+    # Install some distros so the resolver can find them
+    for distro, version in (("dist_a", "0.1"), ("dist_b", "0.5")):
+        with ZipFile(zip_distro.root / f"{distro}_v{version}.zip") as zip_info:
+            zip_info.extractall(tmp_path / "distros" / distro / version)
+
+    def get_dist_names():
+        return [
+            row.node.name
+            for row in resolver.dump_forest(resolver.distros, attr=None)
+            if isinstance(row.node, DistroVersion)
+        ]
+
+    # Get the installed distros and check that `.distros` is the correct return
+    installed = get_dist_names()
+    assert resolver.distros is resolver._installed_distros
+    # Get the download distros and check that `.distros` is the correct return
+    with resolver.distro_mode_override(DistroMode.Downloaded):
+        downloads = get_dist_names()
+        assert resolver.distros is resolver._downloadable_distros
+    # Check that the installed distros are accessible again
+    installed_after = get_dist_names()
+    assert resolver.distros is resolver._installed_distros
+
+    assert installed == ["dist_a==0.1", "dist_b==0.5"]
+    assert installed_after == installed
+    assert downloads == [
+        "dist_a==0.1",
+        "dist_a==0.2",
+        "dist_a==1.0",
+        "dist_b==0.5",
+        "dist_b==0.6",
+    ]
+
+
 class TestDumpForest:
     """Test the dump_forest method on resolver"""
 

tests/test_site.py

@@ -775,23 +775,92 @@ def test_habcache_cls(self, config_root, uncached_resolver):
         ):
             Site([config_root / "site" / "eps" / "site_habcache_cls.json"])
 
+    def test_entry_point_init(self, config_root):
+        site = Site([config_root / "site_main.json"])
+        instance = site.entry_point_init(
+            "group.name",
+            "hab.distro_finders.distro_finder:DistroFinder",
+            ["a/root/path", {"site": "a Site Instance"}],
+        )
+        # The entry_point class was imported and initialized
+        assert isinstance(instance, DistroFinder)
+        # The instance had the requested arguments passed to it
+        assert instance.root == Path("a/root/path")
+        # The last item was a dictionary, that was removed from args and passed
+        # as kwargs.
+        # NOTE: you should not pass site using this method. It's being used here
+        # to test the kwargs feature and ensure the default site setting doesn't
+        # overwrite site if it was passed as a kwarg.
+        assert instance.site == "a Site Instance"
+
+        # Don't pass a kwargs dict, it should get site from itself.
+        instance = site.entry_point_init(
+            "group.name",
+            "hab.distro_finders.distro_finder:DistroFinder",
+            ["b/root/path"],
+        )
+        assert instance.root == Path("b/root/path")
+        assert instance.site is site
 
-def test_download_cache(config_root, uncached_resolver):
-    """Test how `site.downloads["cache_root"]` is processed."""
+
+class TestDownloads:
     # Defaults to `$TEMP/hab_downloads` if not specified
-    default = Path(tempfile.gettempdir()) / "hab_downloads"
-    site = uncached_resolver.site
-    assert site.downloads["cache_root"] == default
-    # `Platform.default_download_cache()` returns the expected default value
-    assert utils.Platform.default_download_cache() == default
-
-    # If specified, only the first path is used. This is using a non-valid
-    # relative path for testing, in practice this should be a absolute path.
-    paths = [config_root / "site" / "site_distro_finder.json"]
-    site = Site(paths)
-    assert site.downloads["cache_root"] == Path("hab testable") / "download" / "path"
+    default_cache_root = Path(tempfile.gettempdir()) / "hab_downloads"
+
+    def test_download_cache(self, config_root, uncached_resolver):
+        """Test how `site.downloads["cache_root"]` is processed."""
+        site = uncached_resolver.site
+        assert site.downloads["cache_root"] == self.default_cache_root
+        # `Platform.default_download_cache()` returns the expected default value
+        assert utils.Platform.default_download_cache() == self.default_cache_root
+
+        # If specified, only the first path is used. This is using a non-valid
+        # relative path for testing, in practice this should be a absolute path.
+        paths = [config_root / "site" / "site_distro_finder.json"]
+        site = Site(paths)
+        assert (
+            site.downloads["cache_root"] == Path("hab testable") / "download" / "path"
+        )
 
-    # Use the default if site specifies cache_root but its an empty string.
-    paths = [config_root / "site" / "site_distro_finder_empty.json"]
-    site = Site(paths)
-    assert site.downloads["cache_root"] == default
+        # Use the default if site specifies cache_root but its an empty string.
+        paths = [config_root / "site" / "site_distro_finder_empty.json"]
+        site = Site(paths)
+        assert site.downloads["cache_root"] == self.default_cache_root
+
+    def test_lazy(self, config_root):
+        site = Site([config_root / "site" / "site_distro_finder.json"])
+        # Check that downloads is not parsed before the downloads property
+        # is first called.
+        assert site._downloads_parsed is False
+        downloads = site.downloads
+        assert site._downloads_parsed is True
+        assert site.downloads is downloads
+
+    def test_default_settings(self, config_root):
+        """Test the default downloads values if not defined by site files."""
+        site = Site([config_root / "site_main.json"])
+        downloads = site.downloads
+        assert len(downloads["distros"]) == 0
+
+        # cache_root is always defined
+        assert downloads["cache_root"] == self.default_cache_root
+        # These are only defined if the json file defines them.
+        assert "install_root" not in downloads
+        assert "relative_path" not in downloads
+
+    def test_all_settings_defined(self, config_root):
+        """Test the resolved downloads values defined by a site file."""
+        from hab.distro_finders.df_zip import DistroFinderZip
+
+        site = Site([config_root / "site" / "site_distro_finder.json"])
+        downloads = site.downloads
+
+        # Check that each part of downloads was processed correctly
+        assert len(downloads["distros"]) == 1
+        finder = downloads["distros"][0]
+        assert isinstance(finder, DistroFinderZip)
+        assert finder.root == Path("network_server/distro/source")
+
+        assert downloads["cache_root"] == Path("hab testable/download/path")
+        assert downloads["install_root"] == config_root / "site" / "distros"
+        assert downloads["relative_path"] == "{distro_name}_v{version}"