diff --git a/forest/__init__.py b/forest/__init__.py index 72c35ab2b..f03cf9717 100644 --- a/forest/__init__.py +++ b/forest/__init__.py @@ -28,7 +28,7 @@ .. automodule:: forest.services """ -__version__ = '0.20.5' +__version__ = '0.20.6' from .config import * from . import ( diff --git a/forest/drivers/unified_model.py b/forest/drivers/unified_model.py index c3d815d43..eae293e1c 100644 --- a/forest/drivers/unified_model.py +++ b/forest/drivers/unified_model.py @@ -63,7 +63,13 @@ def __call__(self): with forest.db.Database.connect(self.database_path) as database: for path in extra_paths: print("inserting: '{}'".format(path)) - database.insert_netcdf(path) + try: + database.insert_netcdf(path) + except OSError as e: + # S3 Glacier objects inaccessible via goofys + print(e) + print(f"skip file: {path}") + continue print("finished") def full_path(self, name): diff --git a/test/test_drivers_unified_model.py b/test/test_drivers_unified_model.py index 405ed9a64..83d18c470 100644 --- a/test/test_drivers_unified_model.py +++ b/test/test_drivers_unified_model.py @@ -9,6 +9,25 @@ import iris +def test_sync_skips_oserror(tmpdir): + """Files stored in S3 Glacier are inaccessible via goofys""" + database_path = str(tmpdir / "file.db") + connection = sqlite3.connect(database_path) + connection.close() + file_name = str(tmpdir / "file.nc") + with open(file_name, "w"): + # Simulate NetCDF files stored in S3 Glacier + pass + settings = { + "locator": "database", + "pattern": file_name, + "directory": str(tmpdir), + "database_path": database_path + } + dataset = forest.drivers.get_dataset("unified_model", settings) + dataset.sync() + + def test_dataset_loader_pattern(): settings = { "pattern": "*.nc",