Create a single tmp dir per test session (#20)

* Add test for lsdb.to_hipscat

* Only create one temp dir per execution.

* Address lint introduced in merge

* Remove comment.

README.md

@@ -62,7 +62,7 @@ There are various steps to have tests run on another cloud bucket provider (like
 ...
 #...line 38...
 @pytest.fixture
-def example_cloud_path(cloud):
+def cloud_path(cloud):
     if cloud == "abfs":
         return "abfs://hipscat/pytests/hipscat"
 
@@ -73,7 +73,7 @@ def example_cloud_path(cloud):
     raise NotImplementedError("Cloud format not implemented for hipscat tests!")
 
 @pytest.fixture
-def example_cloud_storage_options(cloud):
+def storage_options(cloud):
     if cloud == "abfs":
         storage_options = {
             "account_key" : os.environ.get("ABFS_LINCCDATA_ACCOUNT_KEY"),

src/hipscat_cloudtests/__init__.py

@@ -3,4 +3,3 @@
 __all__ = ["greetings", "meaning"]
 
 from .file_checks import assert_parquet_file_ids, assert_text_file_matches
-from .temp_cloud_directory import TempCloudDirectory

src/hipscat_cloudtests/temp_cloud_directory.py

@@ -32,6 +32,15 @@ def __init__(self, prefix_path, method_name="", storage_options: dict = None):
     def __enter__(self):
         """Create a new temporary path
 
+        Returns:
+            string path that's been created. it will take the form of
+            <prefix_path>/<method_name><some random string>
+        """
+        return self.open()
+
+    def open(self):
+        """Create a new temporary path
+
         Returns:
             string path that's been created. it will take the form of
             <prefix_path>/<method_name><some random string>
@@ -45,15 +54,22 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 
         This will try to delete 3 times, with exponential backoff.
         We give up after the third attempt."""
+        self.close()
+
+    def close(self, num_retries=4):
+        """Recursively delete the created resources.
+
+        This will try to delete `num_retries` times, with exponential backoff.
+        We give up after the last attempt."""
         sleep_time = 2
         if self.temp_path:
-            for attempt_number in range(3):
+            for attempt_number in range(1, num_retries + 1):
                 ## Try
                 try:
                     file_io.remove_directory(self.temp_path, storage_options=self.storage_options)
                     return
                 except RuntimeError:
-                    if attempt_number == 2:
+                    if attempt_number == num_retries:
                         print(f"Failed to remove directory {self.temp_path}. Giving up.")
                         return
                 print(f"Failed to remove directory {self.temp_path}. Trying again.")

tests/conftest.py

@@ -1,6 +1,9 @@
 import os
 
 import pytest
+import shortuuid
+
+from hipscat_cloudtests.temp_cloud_directory import TempCloudDirectory
 
 ALMANAC_DIR_NAME = "almanac"
 SMALL_SKY_DIR_NAME = "small_sky"
@@ -15,24 +18,21 @@ def pytest_addoption(parser):
     parser.addoption("--cloud", action="store", default="abfs")
 
 
-def pytest_generate_tests(metafunc):
-    # This is called for every test. Only get/set command line arguments
-    # if the argument is specified in the list of test "fixturenames".
-    option_value = metafunc.config.option.cloud
-    if "cloud" in metafunc.fixturenames and option_value is not None:
-        metafunc.parametrize("cloud", [option_value])
+@pytest.fixture(scope="session", name="cloud")
+def cloud(request):
+    return request.config.getoption("--cloud")
 
 
-@pytest.fixture
-def example_cloud_path(cloud):
+@pytest.fixture(scope="session", name="cloud_path")
+def cloud_path(cloud):
     if cloud == "abfs":
         return "abfs://hipscat/pytests/"
 
     raise NotImplementedError("Cloud format not implemented for tests!")
 
 
-@pytest.fixture
-def example_cloud_storage_options(cloud):
+@pytest.fixture(scope="session", name="storage_options")
+def storage_options(cloud):
     if cloud == "abfs":
         storage_options = {
             "account_key": os.environ.get("ABFS_LINCCDATA_ACCOUNT_KEY"),
@@ -65,35 +65,51 @@ def small_sky_parts_dir_local(local_data_dir):
 
 
 @pytest.fixture
-def tmp_dir_cloud(example_cloud_path):
-    return os.path.join(example_cloud_path, "tmp")
+def test_data_dir_cloud(cloud_path):
+    return os.path.join(cloud_path, "data")
 
 
 @pytest.fixture
-def test_data_dir_cloud(example_cloud_path):
-    return os.path.join(example_cloud_path, "data")
+def almanac_dir_cloud(cloud_path):
+    return os.path.join(cloud_path, "data", ALMANAC_DIR_NAME)
 
 
 @pytest.fixture
-def almanac_dir_cloud(example_cloud_path):
-    return os.path.join(example_cloud_path, "data", ALMANAC_DIR_NAME)
+def small_sky_dir_cloud(cloud_path):
+    return os.path.join(cloud_path, "data", SMALL_SKY_DIR_NAME)
 
 
 @pytest.fixture
-def small_sky_dir_cloud(example_cloud_path):
-    return os.path.join(example_cloud_path, "data", SMALL_SKY_DIR_NAME)
+def small_sky_order1_dir_cloud(cloud_path):
+    return os.path.join(cloud_path, "data", SMALL_SKY_ORDER1_DIR_NAME)
 
 
 @pytest.fixture
-def small_sky_order1_dir_cloud(example_cloud_path):
-    return os.path.join(example_cloud_path, "data", SMALL_SKY_ORDER1_DIR_NAME)
+def small_sky_index_dir_cloud(cloud_path):
+    return os.path.join(cloud_path, "data", "small_sky_object_index")
 
 
 @pytest.fixture
-def small_sky_index_dir_cloud(example_cloud_path):
-    return os.path.join(example_cloud_path, "data", "small_sky_object_index")
+def small_sky_margin_dir_cloud(cloud_path):
+    return os.path.join(cloud_path, "data", "small_sky_order1_margin")
+
+
+@pytest.fixture(scope="session", name="tmp_dir_cloud")
+def tmp_dir_cloud(cloud_path, storage_options):
+    """Create a single client for use by all unit test cases."""
+    tmp = TempCloudDirectory(
+        os.path.join(cloud_path, "tmp"),
+        method_name="full_test",
+        storage_options=storage_options,
+    )
+    yield tmp.open()
+    tmp.close()
 
 
 @pytest.fixture
-def small_sky_margin_dir_cloud(example_cloud_path):
-    return os.path.join(example_cloud_path, "data", "small_sky_order1_margin")
+def tmp_cloud_path(request, tmp_dir_cloud):
+    name = request.node.name
+    my_uuid = shortuuid.uuid()
+    # Strip out the "test_" at the beginning of each method name, make it a little
+    # shorter, and add a disambuating UUID.
+    return f"{tmp_dir_cloud}/{name[5:25]}_{my_uuid}"

tests/hipscat/catalog/dataset/test_base_catalog_info_cloud.py

@@ -4,13 +4,11 @@
 from hipscat.io import file_io
 
 
-def test_read_from_file(base_catalog_info_file_cloud, example_cloud_storage_options):
+def test_read_from_file(base_catalog_info_file_cloud, storage_options):
     base_cat_info_fp = file_io.get_file_pointer_from_path(base_catalog_info_file_cloud)
-    catalog_info = BaseCatalogInfo.read_from_metadata_file(
-        base_cat_info_fp, storage_options=example_cloud_storage_options
-    )
+    catalog_info = BaseCatalogInfo.read_from_metadata_file(base_cat_info_fp, storage_options=storage_options)
     catalog_info_json = file_io.file_io.load_json_file(
-        base_catalog_info_file_cloud, storage_options=example_cloud_storage_options
+        base_catalog_info_file_cloud, storage_options=storage_options
     )
 
     catalog_info_dict = dataclasses.asdict(catalog_info)

tests/hipscat/catalog/test_catalog_cloud.py

@@ -9,57 +9,52 @@
 from hipscat.loaders import read_from_hipscat
 from hipscat.pixel_math import HealpixPixel
 
-from hipscat_cloudtests import TempCloudDirectory
 
-
-def test_load_catalog_small_sky(small_sky_dir_cloud, example_cloud_storage_options):
+def test_load_catalog_small_sky(small_sky_dir_cloud, storage_options):
     """Instantiate a catalog with 1 pixel"""
-    cat = Catalog.read_from_hipscat(small_sky_dir_cloud, storage_options=example_cloud_storage_options)
+    cat = Catalog.read_from_hipscat(small_sky_dir_cloud, storage_options=storage_options)
 
     assert cat.catalog_name == "small_sky"
     assert len(cat.get_healpix_pixels()) == 1
 
-    assert is_valid_catalog(small_sky_dir_cloud, storage_options=example_cloud_storage_options)
+    assert is_valid_catalog(small_sky_dir_cloud, storage_options=storage_options)
 
 
-def test_load_catalog_small_sky_with_loader(small_sky_dir_cloud, example_cloud_storage_options):
+def test_load_catalog_small_sky_with_loader(small_sky_dir_cloud, storage_options):
     """Instantiate a catalog with 1 pixel"""
-    cat = read_from_hipscat(small_sky_dir_cloud, storage_options=example_cloud_storage_options)
+    cat = read_from_hipscat(small_sky_dir_cloud, storage_options=storage_options)
 
     assert isinstance(cat, Catalog)
     assert cat.catalog_name == "small_sky"
     assert len(cat.get_healpix_pixels()) == 1
 
-    assert is_valid_catalog(small_sky_dir_cloud, storage_options=example_cloud_storage_options)
+    assert is_valid_catalog(small_sky_dir_cloud, storage_options=storage_options)
 
 
-def test_empty_directory(tmp_dir_cloud, example_cloud_storage_options):
+def test_empty_directory(tmp_cloud_path, storage_options):
     """Test loading empty or incomplete data"""
-    with TempCloudDirectory(tmp_dir_cloud, "empty", example_cloud_storage_options) as temp_path:
-        catalog_path = temp_path
-
-        ## Path exists but there's nothing there (which means it doesn't exist!)
-        with pytest.raises(FileNotFoundError, match="No directory"):
-            Catalog.read_from_hipscat(catalog_path, storage_options=example_cloud_storage_options)
-
-        ## catalog_info file exists - getting closer
-        file_name = os.path.join(catalog_path, "catalog_info.json")
-        file_io.write_string_to_file(
-            file_name,
-            string='{"catalog_name":"empty", "catalog_type":"source"}',
-            storage_options=example_cloud_storage_options,
-        )
-
-        with pytest.raises(FileNotFoundError, match="metadata"):
-            Catalog.read_from_hipscat(catalog_path, storage_options=example_cloud_storage_options)
-
-        ## partition_info file exists - enough to create a catalog
-        ## Now we create the needed _metadata and everything is right.
-        part_info = PartitionInfo.from_healpix([HealpixPixel(0, 11)])
-        part_info.write_to_metadata_files(
-            catalog_path=catalog_path, storage_options=example_cloud_storage_options
-        )
-
-        with pytest.warns(UserWarning, match="slow"):
-            catalog = Catalog.read_from_hipscat(catalog_path, storage_options=example_cloud_storage_options)
-        assert catalog.catalog_name == "empty"
+    catalog_path = tmp_cloud_path
+
+    ## Path exists but there's nothing there (which means it doesn't exist!)
+    with pytest.raises(FileNotFoundError, match="No directory"):
+        Catalog.read_from_hipscat(catalog_path, storage_options=storage_options)
+
+    ## catalog_info file exists - getting closer
+    file_name = os.path.join(catalog_path, "catalog_info.json")
+    file_io.write_string_to_file(
+        file_name,
+        string='{"catalog_name":"empty", "catalog_type":"source"}',
+        storage_options=storage_options,
+    )
+
+    with pytest.raises(FileNotFoundError, match="metadata"):
+        Catalog.read_from_hipscat(catalog_path, storage_options=storage_options)
+
+    ## partition_info file exists - enough to create a catalog
+    ## Now we create the needed _metadata and everything is right.
+    part_info = PartitionInfo.from_healpix([HealpixPixel(0, 11)])
+    part_info.write_to_metadata_files(catalog_path=catalog_path, storage_options=storage_options)
+
+    with pytest.warns(UserWarning, match="slow"):
+        catalog = Catalog.read_from_hipscat(catalog_path, storage_options=storage_options)
+    assert catalog.catalog_name == "empty"

tests/hipscat/catalog/test_index_catalog.py

@@ -4,8 +4,8 @@
 from hipscat.pixel_math import HealpixPixel
 
 
-def test_loc_partition(small_sky_index_dir_cloud, example_cloud_storage_options):
-    catalog = read_from_hipscat(small_sky_index_dir_cloud, storage_options=example_cloud_storage_options)
+def test_loc_partition(small_sky_index_dir_cloud, storage_options):
+    catalog = read_from_hipscat(small_sky_index_dir_cloud, storage_options=storage_options)
 
     assert isinstance(catalog, IndexCatalog)
     assert catalog.on_disk

tests/hipscat/catalog/test_margin_catalog.py

@@ -3,11 +3,8 @@
 from hipscat.pixel_math.healpix_pixel import HealpixPixel
 
 
-def test_read_margin_from_file(
-    small_sky_margin_dir_cloud,
-    example_cloud_storage_options,
-):
-    catalog = read_from_hipscat(small_sky_margin_dir_cloud, storage_options=example_cloud_storage_options)
+def test_read_margin_from_file(small_sky_margin_dir_cloud, storage_options):
+    catalog = read_from_hipscat(small_sky_margin_dir_cloud, storage_options=storage_options)
 
     assert isinstance(catalog, MarginCatalog)
     assert catalog.on_disk

tests/hipscat/inspection/test_almanac_cloud.py

@@ -3,19 +3,19 @@
 from hipscat.inspection.almanac import Almanac
 
 
-def test_default(almanac_dir_cloud, test_data_dir_cloud, example_cloud_storage_options):
+def test_default(almanac_dir_cloud, test_data_dir_cloud, storage_options):
     """Test loading from a default directory"""
 
     os.environ["HIPSCAT_ALMANAC_DIR"] = ""
     os.environ["HIPSCAT_DEFAULT_DIR"] = test_data_dir_cloud
 
-    alms = Almanac(include_default_dir=True, storage_options=example_cloud_storage_options)
+    alms = Almanac(include_default_dir=True, storage_options=storage_options)
     assert len(alms.catalogs()) == 0
 
     os.environ["HIPSCAT_ALMANAC_DIR"] = almanac_dir_cloud
-    alms = Almanac(include_default_dir=True, storage_options=example_cloud_storage_options)
+    alms = Almanac(include_default_dir=True, storage_options=storage_options)
     assert len(alms.catalogs()) == 2
 
     os.environ.pop("HIPSCAT_ALMANAC_DIR")
-    alms = Almanac(include_default_dir=True, storage_options=example_cloud_storage_options)
+    alms = Almanac(include_default_dir=True, storage_options=storage_options)
     assert len(alms.catalogs()) == 0

tests/hipscat/inspection/test_visualize_catalog_cloud.py

@@ -5,9 +5,9 @@
 # pylint: disable=no-member
 
 
-def test_generate_map_order1(small_sky_dir_cloud, example_cloud_storage_options, mocker):
+def test_generate_map_order1(small_sky_dir_cloud, storage_options, mocker):
     """Basic test that map data can be generated (does not test that a plot is rendered)"""
-    cat = Catalog.read_from_hipscat(small_sky_dir_cloud, storage_options=example_cloud_storage_options)
+    cat = Catalog.read_from_hipscat(small_sky_dir_cloud, storage_options=storage_options)
 
     mocker.patch("healpy.mollview")
     plot_pixels(cat)