Release 1.7.11 Merge pull request #352 from sentinel-hub/develop

Release 1.7.11

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
         language_version: python3
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.4.4"
+    rev: "v0.4.9"
     hooks:
       - id: ruff
 

CHANGELOG.md

@@ -1,3 +1,8 @@
+## [Version 1.7.11] - 2024-06-19
+
+- `BatchDownloadPipeline` retries if the first connection to batch-id specific endpoints fails with a 404.
+
+
 ## [Version 1.7.10] - 2024-05-13
 
 - Minor changes to documentation.

eogrow/__init__.py

@@ -1,3 +1,3 @@
 """The main module of the eo-grow package."""
 
-__version__ = "1.7.10"
+__version__ = "1.7.11"

eogrow/pipelines/batch_to_eopatch.py

@@ -47,6 +47,14 @@ class FeatureMappingSchema(BaseSchema):
 
 
 class BatchToEOPatchPipeline(Pipeline):
+    """Transforms the `tiff` files into `EOPatch` objects.
+
+    Temporal tiffs are expected to be of shape `(h, w, t)` and one tiff per band.
+
+    If the pipeline knows that `timestamps=[]` (through `userdata.json`) and the tiffs contain only `0` elements, it
+    transforms them into "temporally empty" `EOPatch` objects of shape `(0, h, w, b)`.
+    """
+
     class Schema(Pipeline.Schema):
         input_folder_key: str = Field(description="Storage manager key pointing to the path with Batch results")
         _ensure_input_folder_key = ensure_storage_key_presence("input_folder_key")

eogrow/tasks/batch_to_eopatch.py

@@ -82,6 +82,7 @@ class FixImportedTimeDependentFeatureTask(EOTask):
 
     It performs the following:
 
+    -  If the timestamps are empty (but not `None`) and all the data is `0` then it creates an empty eopatch.
     -  rotates bands axis to time axis,
     -  reverses order of timestamps and feature
     -  potentially removes redundant timeframes according to timestamps. This is necessary because in case there were
@@ -101,6 +102,11 @@ def execute(self, eopatch: EOPatch) -> EOPatch:
         data = data[np.newaxis, ...]
         data = np.swapaxes(data, 0, -1)
 
+        if eopatch.timestamps == [] and list(np.unique(data)) == [0]:
+            # there is no actual data
+            shape = data.shape
+            data = np.empty((0, *shape[1:]), dtype=data.dtype)
+
         if eopatch.timestamps:
             timeframe_num = len(eopatch.timestamps)
             if data.shape[0] != timeframe_num:  # Handling a case where data would contain some empty timeframes

tests/pipelines/test_batch_to_eopatch.py

@@ -5,6 +5,7 @@
 import fs
 import numpy as np
 import pytest
+import rasterio
 from fs.base import FS
 
 from sentinelhub import BBox
@@ -78,3 +79,42 @@ def test_batch_to_eopatch_pipeline(config_and_stats_paths, experiment_name):
 
     output_path = run_config(config_path)
     compare_content(output_path, stats_path)
+
+
+@pytest.mark.parametrize("experiment_name", [pytest.param("batch_to_eopatch_empty", marks=pytest.mark.chain)])
+def test_batch_to_eopatch_pipeline_no_timestamps(config_and_stats_paths, experiment_name):
+    config_path, stats_path = config_and_stats_paths("download_and_batch", experiment_name)
+
+    pipeline = BatchToEOPatchPipeline.from_path(config_path)
+
+    filesystem = pipeline.storage.filesystem
+    input_folder = pipeline.storage.get_folder(pipeline.config.input_folder_key)
+    output_folder = pipeline.storage.get_folder(pipeline.config.output_folder_key)
+    filesystem.removetree(input_folder)
+    filesystem.removetree(output_folder)
+
+    # tiffs are made by hand so that they only contain zeros
+    height, width = 200, 300
+    for patch_name, bbox in pipeline.get_patch_list():
+        patch_path = fs.path.combine(input_folder, patch_name)
+        filesystem.makedirs(patch_path, recreate=True)
+
+        for filename in ["B01.tif", "B02.tif", "B03.tif"]:
+            with filesystem.openbin(fs.path.join(patch_path, filename), "w") as file_handle:
+                with rasterio.open(
+                    file_handle,
+                    "w",
+                    driver="GTiff",
+                    width=width,
+                    height=height,
+                    count=1,
+                    dtype=np.int16,
+                    transform=rasterio.transform.from_bounds(*bbox, width=width, height=height),
+                    crs=bbox.crs.ogc_string(),
+                ) as dst:
+                    dst.write(np.zeros((1, height, width)))
+
+        filesystem.writetext(fs.path.combine(patch_path, "userdata.json"), json.dumps({"timestamps": []}))
+
+    output_path = run_config(config_path)
+    compare_content(output_path, stats_path)

tests/test_config_files/download_and_batch/batch_to_eopatch_empty.json

@@ -0,0 +1,23 @@
+{
+  "pipeline": "eogrow.pipelines.batch_to_eopatch.BatchToEOPatchPipeline",
+  "**global_config": "${config_path}/../global_config.json",
+  "input_folder_key": "batch_data",
+  "output_folder_key": "temp",
+  "mapping": [
+    {
+      "batch_files": ["B01.tif", "B02.tif", "B03.tif"],
+      "feature": ["data", "bands"],
+      "multiply_factor": "0.12345",
+      "dtype": "float32"
+    },
+    {
+      "batch_files": ["B01.tif", "B02.tif", "B03.tif"],
+      "feature": ["data", "bands_repeated"],
+      "multiply_factor": "0.12345",
+      "dtype": "float32"
+    }
+  ],
+  "userdata_feature_name": "metainfo",
+  "userdata_timestamp_reader": "eogrow.utils.batch.read_timestamps",
+  "remove_batch_data": "false"
+}

tests/test_stats/download_and_batch/batch_to_eopatch_empty.json

@@ -0,0 +1,40 @@
+{
+  "eopatch-id-0-col-0-row-0": {
+    "bbox": "BBox(((729480.0, 4390045.0), (732120.0, 4391255.0)), crs=CRS('32638'))",
+    "data": {
+      "bands": {
+        "array_shape": [0, 200, 300, 3],
+        "dtype": "float32",
+        "values": []
+      },
+      "bands_repeated": {
+        "array_shape": [0, 200, 300, 3],
+        "dtype": "float32",
+        "values": []
+      }
+    },
+    "meta_info": {
+      "metainfo": "{'timestamps': []}"
+    },
+    "timestamps": []
+  },
+  "eopatch-id-1-col-0-row-1": {
+    "bbox": "BBox(((729480.0, 4391145.0), (732120.0, 4392355.0)), crs=CRS('32638'))",
+    "data": {
+      "bands": {
+        "array_shape": [0, 200, 300, 3],
+        "dtype": "float32",
+        "values": []
+      },
+      "bands_repeated": {
+        "array_shape": [0, 200, 300, 3],
+        "dtype": "float32",
+        "values": []
+      }
+    },
+    "meta_info": {
+      "metainfo": "{'timestamps': []}"
+    },
+    "timestamps": []
+  }
+}