Release version 1.6.3, Merge pull request #296 from sentinel-hub/develop

Release version 1.6.3

.github/workflows/ci_action.yml

@@ -113,19 +113,3 @@ jobs:
           files: coverage.xml
           fail_ci_if_error: true
           verbose: false
-
-  mirror-to-gitlab:
-    if: github.event_name == 'push'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - name: Mirror + trigger CI
-        uses: SvanBoxel/gitlab-mirror-and-ci-action@master
-        with:
-          args: "https://git.sinergise.com/eo/code/eo-grow"
-        env:
-          GITLAB_HOSTNAME: "git.sinergise.com"
-          GITLAB_USERNAME: "github-action"
-          GITLAB_PASSWORD: ${{ secrets.GITLAB_PASSWORD }}
-          GITLAB_PROJECT_ID: "878"
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci_trigger.yml

@@ -0,0 +1,29 @@
+name: mirror_and_trigger
+
+on:
+  pull_request:
+  push:
+    branches:
+      - "master"
+      - "develop"
+  workflow_call:
+  release:
+    types:
+      - published
+
+jobs:
+  mirror-to-gitlab:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Mirror + trigger CI
+        uses: SvanBoxel/gitlab-mirror-and-ci-action@master
+        with:
+          args: "https://git.sinergise.com/eo/code/eo-grow"
+        env:
+          FOLLOW_TAGS: "true"
+          GITLAB_HOSTNAME: "git.sinergise.com"
+          GITLAB_USERNAME: "github-action"
+          GITLAB_PASSWORD: ${{ secrets.GITLAB_PASSWORD }}
+          GITLAB_PROJECT_ID: "878"
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitlab-ci.yml

@@ -0,0 +1,18 @@
+image: python:3.9
+
+stages:
+  - build
+
+build_docker_image:
+  stage: build
+  needs: []
+  rules:
+    - if: $CI_COMMIT_TAG # run only on releases
+      when: always
+      variables:
+        CUSTOM_RUN_TAG: auto # this will create images with the latest tag and the version tag
+        LAYER_NAME: dotai-eo
+    - when: manual
+  trigger:
+    project: eo/infra/docker
+  allow_failure: true

.pre-commit-config.yaml

@@ -20,13 +20,13 @@ repos:
         types_or: [json]
 
   - repo: https://github.com/psf/black
-    rev: 23.9.1
+    rev: 23.10.1
     hooks:
       - id: black
         language_version: python3
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.0.292"
+    rev: "v0.1.4"
     hooks:
       - id: ruff
 

CHANGELOG.md

@@ -1,3 +1,10 @@
+## [Version 1.6.3] - 2023-11-07
+
+- Pipelines can request specific type of worker when run on a ray cluster with the `ray_worker_type` field.
+- Area managers now generate the patch lists more efficiently.
+- Pipelines have option to fail when temporally-ill defined EOPatches are encountered with the `raise_on_temporal_mismatch` flag.
+
+
 ## [Version 1.6.2] - 2023-10-17
 
 - Fixed a bug in `BatchDownloadPipeline` where the evalscript was not read correctly.

eogrow/__init__.py

@@ -1,3 +1,3 @@
 """The main module of the eo-grow package."""
 
-__version__ = "1.6.2"
+__version__ = "1.6.3"

eogrow/cli.py

@@ -1,4 +1,5 @@
 """Implements the command line interface for `eo-grow`."""
+
 import json
 import os
 import re

eogrow/core/area/base.py

@@ -1,8 +1,10 @@
 """Implementation of the base AreaManager."""
+
 from __future__ import annotations
 
 import logging
 from abc import ABCMeta, abstractmethod
+from functools import partial
 from typing import Literal, Optional
 
 import fiona
@@ -131,10 +133,10 @@ def get_grid_cache_filename(self) -> str:
     def get_patch_list(self) -> PatchList:
         """Returns a list of eopatch names and appropriate BBoxes."""
 
-        named_bboxes = []
+        named_bboxes: PatchList = []
         for crs, grid in self.get_grid(filtered=True).items():
-            for _, row in grid.iterrows():
-                named_bboxes.append((row.eopatch_name, BBox(row.geometry.bounds, crs=crs)))
+            bounds = grid.geometry.bounds.apply(tuple, axis=1)
+            named_bboxes.extend(zip(grid[self.NAME_COLUMN], bounds.map(partial(BBox, crs=crs))))
 
         return named_bboxes
 

eogrow/core/area/batch.py

@@ -1,4 +1,5 @@
 """Area manager implementation for Sentinel Hub batch grids."""
+
 from __future__ import annotations
 
 import logging

eogrow/core/area/custom_grid.py

@@ -1,4 +1,5 @@
 """Area manager implementation for custom grids."""
+
 from __future__ import annotations
 
 import logging

eogrow/core/base.py

@@ -1,4 +1,5 @@
 """Base object from which all configurable eo-grow objects inherit."""
+
 from __future__ import annotations
 
 from typing import Any, TypeVar

eogrow/core/config.py

@@ -1,4 +1,5 @@
 """Implements functions that transform raw dictionaries/JSON files according to the config language of eo-grow."""
+
 from __future__ import annotations
 
 import copy

eogrow/core/logging.py

@@ -1,4 +1,5 @@
 """Implementation of LoggingManager and different handlers used for logging."""
+
 from __future__ import annotations
 
 import contextlib

eogrow/core/pipeline.py

@@ -1,4 +1,5 @@
 """Implementation of the base Pipeline class."""
+
 from __future__ import annotations
 
 import logging
@@ -197,7 +198,8 @@ def run_execution(
             filesystem=self.storage.filesystem,
             logs_filter=EOExecutionFilter(ignore_packages=self.logging_manager.config.eoexecution_ignore_packages),
             logs_handler_factory=EOExecutionHandler,
-            **extra_kwargs,  # type: ignore[arg-type]
+            raise_on_temporal_mismatch=self.config.raise_on_temporal_mismatch,
+            **extra_kwargs,
         )
         execution_results = executor.run(**executor_run_params)
 

eogrow/core/schemas.py

@@ -4,6 +4,7 @@
 For each pipeline a separate schema has to be defined which inherits from PipelineSchema. Such schema should be placed
 as an internal class of the implemented pipeline class
 """
+
 from __future__ import annotations
 
 from inspect import isclass
@@ -75,6 +76,9 @@ class PipelineSchema(BaseSchema):
             "must implement the `filter_patch_list` method."
         ),
     )
+    raise_on_temporal_mismatch: bool = Field(
+        False, description="Whether to treat `TemporalDimensionWarning` as an exception during EOExecution."
+    )
 
 
 def build_schema_template(

eogrow/core/storage.py

@@ -1,4 +1,5 @@
 """Implementation of the StorageManager class for handling project storage."""
+
 from __future__ import annotations
 
 from typing import Any, ClassVar, Dict, Literal, Optional

eogrow/pipelines/batch_to_eopatch.py

@@ -1,4 +1,5 @@
 """Pipeline for conversion of batch results to EOPatches."""
+
 from __future__ import annotations
 
 from typing import Any, List, Optional
@@ -174,9 +175,7 @@ def _get_tiff_mapping_node(self, mapping: FeatureMappingSchema, previous_node: E
             ), feature[1]
 
             import_task = ImportFromTiffTask(
-                tmp_timeless_feature,
-                folder=self._input_folder,
-                filesystem=self.storage.filesystem,
+                tmp_timeless_feature, self._input_folder, filesystem=self.storage.filesystem
             )
             # Filename is written into the dependency name to be used later for execution arguments:
             import_node = EONode(

eogrow/pipelines/byoc.py

@@ -1,4 +1,5 @@
 """Defines pipelines for ingesting and modifying BYOC collections."""
+
 from __future__ import annotations
 
 import datetime

eogrow/pipelines/download.py

@@ -1,4 +1,5 @@
 """Implements different customizeable pipelines for downloading data."""
+
 from __future__ import annotations
 
 import abc

eogrow/pipelines/download_batch.py

@@ -1,4 +1,5 @@
 """Download pipeline that works with Sentinel Hub batch service."""
+
 from __future__ import annotations
 
 import logging

eogrow/pipelines/export_maps.py

@@ -1,4 +1,5 @@
 """Implements a pipeline for exporting data to TIFF files, can be used to prepare BYOC tiles."""
+
 from __future__ import annotations
 
 import datetime as dt
@@ -186,7 +187,7 @@ def build_workflow(self) -> EOWorkflow:
 
         export_to_tiff_task = ExportToTiffTask(
             self.config.feature,
-            folder=self.storage.get_folder(self.config.output_folder_key),
+            self.storage.get_folder(self.config.output_folder_key),
             filesystem=self.storage.filesystem,
             no_data_value=self.config.no_data_value,
             image_dtype=np.dtype(self.config.map_dtype),

eogrow/pipelines/features.py

@@ -1,4 +1,5 @@
 """Implements a pipeline to construct features for training/prediction."""
+
 from __future__ import annotations
 
 import logging

eogrow/pipelines/import_tiff.py

@@ -1,4 +1,5 @@
 """Implements a pipeline for importing reference data from a raster image."""
+
 from __future__ import annotations
 
 from typing import Optional
@@ -54,10 +55,6 @@ class Schema(Pipeline.Schema):
         )
         dtype: Optional[np.dtype] = Field(description="Custom dtype for the imported feature.")
         _parse_dtype = optional_field_validator("dtype", parse_dtype, pre=True)
-        use_vsi: bool = Field(
-            True,
-            description="Whether to use the VSI for reading. Enabled by default as a remote filesystem is assumed.",
-        )
         resize: Optional[ResizeSchema] = Field(
             description="Settings for SpatialResizeTask applied at the end. When omitted resizing is not performed."
         )
@@ -84,7 +81,6 @@ def build_workflow(self) -> EOWorkflow:
             no_data_value=self.config.no_data_value,
             filesystem=self.storage.filesystem,
             image_dtype=self.config.dtype,
-            use_vsi=self.config.use_vsi,
         )
         import_node = EONode(import_task, inputs=[create_eopatch_node])
 

eogrow/pipelines/import_vector.py

@@ -1,4 +1,5 @@
 """Implements a pipeline for importing vector data from a file."""
+
 from __future__ import annotations
 
 import fs

eogrow/pipelines/merge_samples.py

@@ -1,4 +1,5 @@
 """Implements a pipeline for merging sampled features into numpy arrays fit for training models."""
+
 from __future__ import annotations
 
 import logging

eogrow/pipelines/prediction.py

@@ -1,4 +1,5 @@
 """Implements a base prediction pipeline and a LGBM specialized classification and regression pipelines."""
+
 from __future__ import annotations
 
 import abc

eogrow/pipelines/rasterize.py

@@ -1,4 +1,5 @@
 """Implements a pipeline for rasterizing vector datasets."""
+
 from __future__ import annotations
 
 import logging

eogrow/pipelines/sampling.py

@@ -1,4 +1,5 @@
 """Implements different pipelines for sampling from data."""
+
 from __future__ import annotations
 
 import abc

eogrow/pipelines/split_grid.py

@@ -1,4 +1,5 @@
 """Implements a pipeline that creates a finer grid and splits EOPatches accordingly."""
+
 from __future__ import annotations
 
 import itertools as it

eogrow/pipelines/testing.py

@@ -1,4 +1,5 @@
 """Implements pipelines used for data preparation in testing."""
+
 from __future__ import annotations
 
 import logging

eogrow/tasks/batch_to_eopatch.py

@@ -1,4 +1,5 @@
 """Tasks used to transform Sentinel Hub Batch results into EOPatches."""
+
 from __future__ import annotations
 
 import concurrent.futures

eogrow/tasks/common.py

@@ -1,4 +1,5 @@
 """Common tasks shared between pipelines."""
+
 from __future__ import annotations
 
 from functools import partial

eogrow/tasks/features.py

@@ -1,4 +1,5 @@
 """Implements tasks needed for calculating features in FeaturesPipeline."""
+
 from __future__ import annotations
 
 import abc

eogrow/tasks/prediction.py

@@ -1,4 +1,5 @@
 """Defines task needed in prediction pipelines."""
+
 from __future__ import annotations
 
 import abc

eogrow/tasks/spatial.py

@@ -1,4 +1,5 @@
 """Tasks for spatial operations on EOPatches, used in grid-switching."""
+
 from __future__ import annotations
 
 import numpy as np

eogrow/tasks/testing.py

@@ -1,4 +1,5 @@
 """Tasks used to generate test data."""
+
 from __future__ import annotations
 
 import datetime as dt

eogrow/types.py

@@ -1,5 +1,6 @@
 """ Includes custom types used in schemas
 """
+
 import datetime
 import sys
 from enum import Enum

eogrow/utils/batch.py

@@ -1,6 +1,7 @@
 """
 A module with useful utilities related to batch processing
 """
+
 from __future__ import annotations
 
 

eogrow/utils/filter.py

@@ -1,6 +1,7 @@
 """
 Utilities for filtering eopatch lists
 """
+
 from __future__ import annotations
 
 from concurrent.futures import ThreadPoolExecutor

eogrow/utils/fs.py

@@ -1,6 +1,7 @@
 """
 Module containing utilities for working with filesystems
 """
+
 from __future__ import annotations
 
 import abc

eogrow/utils/general.py

@@ -1,6 +1,7 @@
 """
 A module containing general utilities that haven't been sorted in any other module
 """
+
 from __future__ import annotations
 
 import datetime as dt