Adv notebooks

AgriDataValue/Air pollution example/data/eu_capitals.geojson

@@ -0,0 +1,33 @@
+{
+"type": "FeatureCollection",
+"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
+"features": [
+{ "type": "Feature", "properties": { "name": "Vilnius" }, "geometry": { "type": "Point", "coordinates": [ 25.261688, 54.684946997861104 ] } },
+{ "type": "Feature", "properties": { "name": "Luxembourg" }, "geometry": { "type": "Point", "coordinates": [ 6.13, 49.611659996798664 ] } },
+{ "type": "Feature", "properties": { "name": "Amsterdam" }, "geometry": { "type": "Point", "coordinates": [ 4.890444, 52.370196997443337 ] } },
+{ "type": "Feature", "properties": { "name": "Madrid" }, "geometry": { "type": "Point", "coordinates": [ -3.71666, 40.383332993344226 ] } },
+{ "type": "Feature", "properties": { "name": "Sofia" }, "geometry": { "type": "Point", "coordinates": [ 23.33333, 42.699999994391369 ] } },
+{ "type": "Feature", "properties": { "name": "Athens" }, "geometry": { "type": "Point", "coordinates": [ 23.716666, 37.966665992158354 ] } },
+{ "type": "Feature", "properties": { "name": "Stockholm" }, "geometry": { "type": "Point", "coordinates": [ 18.068611, 59.329439998431397 ] } },
+{ "type": "Feature", "properties": { "name": "Berlin" }, "geometry": { "type": "Point", "coordinates": [ 13.383333, 52.516665997472963 ] } },
+{ "type": "Feature", "properties": { "name": "Warsaw" }, "geometry": { "type": "Point", "coordinates": [ 21.01666, 52.233329997415282 ] } },
+{ "type": "Feature", "properties": { "name": "Bratislava" }, "geometry": { "type": "Point", "coordinates": [ 17.109722, 48.14388799638381 ] } },
+{ "type": "Feature", "properties": { "name": "Nicosia" }, "geometry": { "type": "Point", "coordinates": [ 33.36666, 35.166659990718962 ] } },
+{ "type": "Feature", "properties": { "name": "Brussels" }, "geometry": { "type": "Point", "coordinates": [ 4.35, 50.849999997109201 ] } },
+{ "type": "Feature", "properties": { "name": "Tallinn" }, "geometry": { "type": "Point", "coordinates": [ 24.745278, 59.437221998441302 ] } },
+{ "type": "Feature", "properties": { "name": "Bucharest" }, "geometry": { "type": "Point", "coordinates": [ 26.103888, 44.432499995101445 ] } },
+{ "type": "Feature", "properties": { "name": "Paris" }, "geometry": { "type": "Point", "coordinates": [ 2.3508, 48.856699996591779 ] } },
+{ "type": "Feature", "properties": { "name": "Budapest" }, "geometry": { "type": "Point", "coordinates": [ 19.050293, 47.494008996183503 ] } },
+{ "type": "Feature", "properties": { "name": "Copenhagen" }, "geometry": { "type": "Point", "coordinates": [ 12.568333, 55.676110998009861 ] } },
+{ "type": "Feature", "properties": { "name": "Dublin" }, "geometry": { "type": "Point", "coordinates": [ -6.259722, 53.347769997632476 ] } },
+{ "type": "Feature", "properties": { "name": "Helsinki" }, "geometry": { "type": "Point", "coordinates": [ 24.9375, 60.170832998505439 ] } },
+{ "type": "Feature", "properties": { "name": "Prague" }, "geometry": { "type": "Point", "coordinates": [ 14.41666, 50.083329996921123 ] } },
+{ "type": "Feature", "properties": { "name": "Lisbon" }, "geometry": { "type": "Point", "coordinates": [ -9.139444, 38.713888992533079 ] } },
+{ "type": "Feature", "properties": { "name": "Ljubljana" }, "geometry": { "type": "Point", "coordinates": [ 14.508333, 46.055549995703409 ] } },
+{ "type": "Feature", "properties": { "name": "Riga" }, "geometry": { "type": "Point", "coordinates": [ 24.106389, 56.94888799817759 ] } },
+{ "type": "Feature", "properties": { "name": "Zagreb" }, "geometry": { "type": "Point", "coordinates": [ 15.98333, 45.816659995618771 ] } },
+{ "type": "Feature", "properties": { "name": "Rome" }, "geometry": { "type": "Point", "coordinates": [ 12.5, 41.899999994041522 ] } },
+{ "type": "Feature", "properties": { "name": "Valletta" }, "geometry": { "type": "Point", "coordinates": [ 14.5125, 35.89776999109754 ] } },
+{ "type": "Feature", "properties": { "name": "Vienna" }, "geometry": { "type": "Point", "coordinates": [ 16.36666, 48.199999996400642 ] } }
+]
+}

AgriDataValue/Air pollution example/data/ne_50m_admin_0_countries/ne_50m_admin_0_countries.VERSION.txt

@@ -0,0 +1 @@
+5.1.1

AgriDataValue/Air pollution example/data/ne_50m_admin_0_countries/ne_50m_admin_0_countries.cpg

@@ -0,0 +1 @@
+UTF-8

AgriDataValue/Air pollution example/data/ne_50m_admin_0_countries/ne_50m_admin_0_countries.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

AgriDataValue/Burned-up_area/eogrow_project/input-data/area.geojson

@@ -0,0 +1,8 @@
+{
+"type": "FeatureCollection",
+"name": "area",
+"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
+"features": [
+{ "type": "Feature", "properties": { }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 13.691619686000024, 45.888306475000036 ], [ 13.690687564000029, 45.797223798000061 ], [ 13.670730485000036, 45.797314483000036 ], [ 13.670806500000026, 45.797882 ], [ 13.670241, 45.798872500000073 ], [ 13.669363, 45.799702 ], [ 13.658106500000031, 45.800363 ], [ 13.657217, 45.800243 ], [ 13.650501, 45.799312500000042 ], [ 13.648942500000032, 45.799267 ], [ 13.642528, 45.798615 ], [ 13.642215500000077, 45.798378 ], [ 13.640498, 45.798483500000032 ], [ 13.637662, 45.798148 ], [ 13.637519, 45.798283500000025 ], [ 13.637475, 45.801366500000029 ], [ 13.632243500000072, 45.799458 ], [ 13.628746, 45.798182 ], [ 13.626913619000049, 45.797513587000026 ], [ 13.623570838000035, 45.797528777000025 ], [ 13.623453500000039, 45.798358500000063 ], [ 13.623754, 45.800162500000056 ], [ 13.623053, 45.80026950000007 ], [ 13.622945500000071, 45.800286 ], [ 13.622798, 45.800444500000026 ], [ 13.621656, 45.800403 ], [ 13.619471, 45.800725500000055 ], [ 13.61860050000007, 45.801099500000078 ], [ 13.617389500000058, 45.801381500000048 ], [ 13.615660500000047, 45.80229 ], [ 13.615237500000035, 45.802769500000068 ], [ 13.614414, 45.803056500000025 ], [ 13.613744, 45.803175 ], [ 13.61285950000007, 45.803079 ], [ 13.611990500000047, 45.802985 ], [ 13.611229500000036, 45.80263750000006 ], [ 13.609835, 45.803194 ], [ 13.607611, 45.803725500000041 ], [ 13.604122, 45.803997 ], [ 13.599285497000039, 45.806415865000076 ], [ 13.596243, 45.807937500000037 ], [ 13.595874, 45.808122 ], [ 13.59597750000006, 45.809332500000039 ], [ 13.59645850000004, 45.809316500000079 ], [ 13.596203500000058, 45.810586500000056 ], [ 13.597283, 45.810743 ], [ 13.597223, 45.811554 ], [ 13.596246500000063, 45.811539500000038 ], [ 13.596378500000071, 45.813576 ], [ 13.59644650000007, 45.814618 ], [ 13.596456, 45.814768 ], [ 13.596609, 45.815824500000076 ], [ 13.597126, 45.819393 ], [ 13.597145, 45.819522500000062 ], [ 13.593308500000035, 45.822656 ], [ 13.592402, 45.82337 ], [ 13.590965500000038, 45.82785350000006 ], [ 13.588562500000023, 45.835353500000053 ], [ 13.58258150000006, 45.838712500000042 ], [ 13.577138, 45.84177 ], [ 13.576974500000063, 45.841862 ], [ 13.576512, 45.841731500000037 ], [ 13.576162, 45.842318500000033 ], [ 13.574596500000041, 45.842342500000029 ], [ 13.574449, 45.843357 ], [ 13.575809, 45.843100500000048 ], [ 13.576632500000073, 45.843157500000075 ], [ 13.576665, 45.843341 ], [ 13.575652500000047, 45.844343500000036 ], [ 13.575467, 45.845001 ], [ 13.575076500000023, 45.845389500000067 ], [ 13.575802500000066, 45.847736500000053 ], [ 13.576555, 45.850358 ], [ 13.576733, 45.850979 ], [ 13.576224500000023, 45.851857500000051 ], [ 13.575498500000037, 45.852232 ], [ 13.575811500000043, 45.852988 ], [ 13.575231500000029, 45.853203 ], [ 13.575167, 45.853499500000055 ], [ 13.576939500000037, 45.855152500000031 ], [ 13.577259, 45.855013500000041 ], [ 13.578404, 45.855522500000063 ], [ 13.579042, 45.856195 ], [ 13.579406500000061, 45.855957500000045 ], [ 13.579858, 45.856429500000047 ], [ 13.579679, 45.856970500000045 ], [ 13.57958, 45.857270500000027 ], [ 13.579826, 45.85766650000005 ], [ 13.580355, 45.857886500000063 ], [ 13.580775500000072, 45.858924 ], [ 13.581699, 45.859878 ], [ 13.582107500000063, 45.860942500000078 ], [ 13.58283, 45.862824500000045 ], [ 13.584126500000025, 45.866223500000046 ], [ 13.584269500000062, 45.866598 ], [ 13.584290500000066, 45.866653 ], [ 13.58411250000006, 45.867427500000076 ], [ 13.584794500000044, 45.867532500000038 ], [ 13.587249, 45.871291500000041 ], [ 13.59325750000005, 45.874677500000075 ], [ 13.593295500000067, 45.874834500000077 ], [ 13.592392500000074, 45.875734500000078 ], [ 13.590741, 45.87738 ], [ 13.589563500000054, 45.878553500000066 ], [ 13.588173500000039, 45.87940350000008 ], [ 13.587599500000067, 45.879754500000047 ], [ 13.586661, 45.880712 ], [ 13.586496500000067, 45.881916500000045 ], [ 13.586767500000065, 45.88243650000004 ], [ 13.587572, 45.883741 ], [ 13.589887500000032, 45.886543 ], [ 13.590174, 45.88689050000005 ], [ 13.590588500000024, 45.88739350000003 ], [ 13.590856, 45.887718 ], [ 13.590874692000057, 45.887740680000036 ], [ 13.691619686000024, 45.888306475000036 ] ] ] ] } }
+]
+}

AgriDataValue/Burned-up_area/eogrow_project/input-data/evalscript.js

@@ -0,0 +1,30 @@
+//VERSION=3
+
+function setup() {
+  return {
+    input: [{
+      bands: ["B01", "B02", "B03", "B04", "B05", "B06", "B07", "B08", "B8A", "B09", "B11", "B12", "dataMask"],
+      units: "DN"
+    }],
+    output: [
+        {
+            id: "BANDS",
+            bands: 12,
+            sampleType: SampleType.UINT16
+        },
+        {
+            id: "dataMask",
+            bands: 1,
+            sampleType: SampleType.UINT8
+        }
+    ]
+  };
+}
+
+function evaluatePixel(sample) {
+  results = {
+      "BANDS": [sample.B01, sample.B02, sample.B03, sample.B04, sample.B05, sample.B06, sample.B07, sample.B08, sample.B8A, sample.B09, sample.B11, sample.B12],
+      "dataMask": [sample.dataMask]
+  }
+  return results;
+}

AgriDataValue/Burned-up_area/utils.py

@@ -0,0 +1,287 @@
+import datetime
+from typing import Iterable, List, Optional, Tuple
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+
+from eolearn.core import EOPatch, EOTask, FeatureType
+from sentinelhub import BBox
+
+Feature = Tuple[FeatureType, str]
+
+ID_X_COLUMN, ID_Y_COLUMN = "ID_X", "ID_Y"
+ID_COLUMNS = [ID_X_COLUMN, ID_Y_COLUMN]
+P_ID_COLUMN = "P_ID"
+COORD_X_COLUMN, COORD_Y_COLUMN = "COORD_X", "COORD_Y"
+TIMESTAMP_COLUMN = "TIMESTAMP"
+CRS_COLUMN = "CRS"
+
+
+class PatchToDataframeTask(EOTask):
+    """Task for converting an EOPatch to a pd.DataFrame
+
+    It transfers all the data from patch features to a dataframe, which is saved as a META_INFO feature.
+    """
+
+    def __init__(
+        self,
+        *,
+        append_coords: bool = False,
+        append_p_id: bool = False,
+        output_feature: Feature,
+        shape_info_feature: Feature,
+        mask_feature: Optional[Feature],
+    ) -> None:
+        self.append_coords = append_coords
+        self.append_p_id = append_p_id
+        self.output_feature = self.parse_feature(output_feature, allowed_feature_types=[FeatureType.META_INFO])
+        self.shape_info_feature = self.parse_feature(shape_info_feature, allowed_feature_types=[FeatureType.META_INFO])
+
+        self.mask_feature = None
+        if mask_feature is not None:
+            self.mask_feature = self.parse_feature(mask_feature, allowed_feature_types=[FeatureType.MASK_TIMELESS])
+
+        self.relevant_feature_types = [
+            FeatureType.DATA,
+            FeatureType.DATA_TIMELESS,
+            FeatureType.MASK,
+            FeatureType.MASK_TIMELESS,
+        ]
+
+    def get_basic_info(self, eopatch: EOPatch) -> Tuple[BBox, List[datetime.date], Tuple[int, int]]:
+        """Extracts the BBox, timestamps, and spatial size of EOPatch."""
+        bbox = eopatch.bbox
+        timestamps = eopatch.timestamps or []
+
+        some_features = self.parse_features(self.relevant_feature_types, eopatch=eopatch)
+
+        if not some_features:
+            ValueError("EOPatch has no relevant features.")
+        height, width = eopatch.get_spatial_dimension(*some_features[0])
+        return bbox, timestamps, (height, width)
+
+    def prepare_content(
+        self,
+        bbox: BBox,
+        timestamp: List[datetime.date],
+        spatial_shape: Tuple[int, int],
+        eopatch_id: int,
+        mask: np.ndarray,
+    ) -> pd.DataFrame:
+        """Prepares dataframe with columns containing basic information."""
+        height, width = spatial_shape
+        time_dim = len(timestamp)
+
+        id_x = np.full(spatial_shape, np.arange(width), dtype=np.uint16)[mask].ravel()
+        id_y = np.full(spatial_shape, np.expand_dims(np.arange(height), axis=-1), dtype=np.uint16)[mask].ravel()
+
+        if time_dim == 0:
+            dataframe = pd.DataFrame({ID_X_COLUMN: id_x, ID_Y_COLUMN: id_y})
+        else:
+            ts_array = np.full((time_dim, height, width), np.expand_dims(timestamp, axis=(-2, -1)))[..., mask].ravel()
+            dataframe = pd.DataFrame(
+                {
+                    TIMESTAMP_COLUMN: pd.to_datetime(ts_array),
+                    ID_X_COLUMN: np.full((time_dim, len(id_x)), id_x).ravel(),
+                    ID_Y_COLUMN: np.full((time_dim, len(id_y)), id_y).ravel(),
+                }
+            )
+
+        if self.append_p_id:
+            dataframe[P_ID_COLUMN] = (
+                eopatch_id * height * width + dataframe[ID_Y_COLUMN] * width + dataframe[ID_X_COLUMN]
+            ).astype(np.uint64)
+
+        if self.append_coords:
+            x1, y1, x2, y2 = bbox.geometry.bounds
+            res_x, res_y = (x2 - x1) / width, (y2 - y1) / height
+
+            dataframe[COORD_X_COLUMN] = (
+                np.linspace(x1, x2, width, endpoint=False, dtype=np.float32)[dataframe[ID_X_COLUMN]] + res_x / 2
+            )
+            dataframe[COORD_Y_COLUMN] = (
+                np.linspace(y1, y2, height, endpoint=False, dtype=np.float32)[dataframe[ID_Y_COLUMN]] + res_y / 2
+            )
+            dataframe[CRS_COLUMN] = bbox.crs.epsg
+            dataframe[CRS_COLUMN] = dataframe[CRS_COLUMN].astype("category")
+
+        dataframe.reset_index(drop=True, inplace=True)
+        return dataframe
+
+    def transfer_features(
+        self,
+        dataframe: pd.DataFrame,
+        features: Iterable[Feature],
+        eopatch: EOPatch,
+        time_dim: int,
+        spatial_shape: Tuple[int, int],
+        mask: np.ndarray,
+    ) -> None:
+        """Transfers features from eopatch to dataframe. Mutates existing structures."""
+        for f_type, f_name in features:
+            data_shape = eopatch.get_spatial_dimension(f_type, f_name)
+            if data_shape != spatial_shape:
+                raise ValueError(
+                    f"Features have different spatial shapes, {(f_type, f_name)} has {data_shape} but"
+                    f" {spatial_shape} was expected."
+                )
+
+            data = eopatch[f_type, f_name]
+            del eopatch[f_type, f_name]
+
+            if data.shape[-1] != 1:
+                raise ValueError(f"Features should have depth of 1, {(f_type, f_name)} has {data.shape[-1]}.")
+
+            data = data[..., mask, 0]
+
+            if time_dim > 0 and f_type.is_timeless():
+                data = np.full((time_dim, np.count_nonzero(mask)), data)
+
+            dataframe[f_name] = data.ravel()
+            del data
+
+    def execute(self, eopatch: EOPatch, eopatch_global_id: int = 0) -> EOPatch:
+        bbox, timestamp, spatial_shape = self.get_basic_info(eopatch)
+
+        mask = np.ones(spatial_shape, dtype=bool)
+        features = set(self.parse_features(self.relevant_feature_types, eopatch=eopatch))
+
+        if self.mask_feature is not None:
+            mask = eopatch[self.mask_feature].squeeze(axis=-1).astype(bool)
+            features -= {self.mask_feature}
+
+        dataframe = self.prepare_content(bbox, timestamp, spatial_shape, eopatch_global_id, mask)
+        self.transfer_features(dataframe, features, eopatch, len(timestamp), spatial_shape, mask)
+
+        new_eopatch = EOPatch(bbox=bbox)
+        new_eopatch[self.output_feature] = dataframe
+        new_eopatch[self.shape_info_feature] = spatial_shape
+        return new_eopatch
+
+
+def plot_data(
+    data: np.ndarray,
+    timestamp_index: int,
+    brightness_factor: float = 1,
+    clip=None,
+    title="",
+    ax=None,
+    **kwargs,
+):
+    image = data[timestamp_index] * brightness_factor
+
+    if clip is not None:
+        image = np.clip(image, *clip)
+
+    if ax is None:
+        _, ax = plt.subplots(figsize=(10, 10))
+
+    ax.imshow(image, **kwargs)
+
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.set_title(title)
+
+
+def plot_mask(mask: np.ndarray, title="", ax=None):
+    prepared_mask = mask.squeeze(-1)
+
+    if ax is None:
+        _, ax = plt.subplots(figsize=(10, 10))
+
+    ax.imshow(prepared_mask, vmin=0, vmax=1)
+
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.set_title(title)
+
+
+def plot_combo(
+    data: np.ndarray,
+    mask: np.ndarray,
+    brightness_factor: float = 1,
+    clip=None,
+    **kwargs,
+):
+    _, axs = plt.subplots(ncols=3, figsize=(15, 5))
+
+    plot_data(
+        data,
+        brightness_factor=brightness_factor,
+        timestamp_index=0,
+        clip=clip,
+        title="Before",
+        ax=axs[0],
+        **kwargs,
+    )
+    plot_data(
+        data,
+        brightness_factor=brightness_factor,
+        timestamp_index=1,
+        clip=clip,
+        title="After",
+        ax=axs[1],
+        **kwargs,
+    )
+    plot_mask(mask, title="Burned area mask", ax=axs[2])
+
+    # plot mask
+    axs[2].imshow(mask, vmin=0, vmax=1)
+
+    plt.tight_layout()
+
+
+def plot_band_hist(df, band, ax=None, **kwargs):
+    if ax is None:
+        _, ax = plt.subplots(figsize=(15, 5))
+
+    ax.hist(df.query("BURN_AREA == 0")[band], **kwargs, label="Not burned", alpha=0.5)
+    ax.hist(df.query("BURN_AREA == 1")[band], **kwargs, label="Burned", alpha=0.5)
+
+    ax.set_xlabel(f"Band: {band}")
+    ax.grid("on")
+    ax.legend()
+
+
+def convert_eop_to_df(eopatch, timestamp_index=None):
+    patch2df_task = PatchToDataframeTask(
+        output_feature=(FeatureType.META_INFO, "DF"),
+        shape_info_feature=(FeatureType.META_INFO, "SHAPE"),
+        mask_feature=None,
+    )
+    df = patch2df_task(eopatch.copy(deep=True)).meta_info["DF"]
+    df.loc[(df.BURN_AREA == 1) & (df.TIMESTAMP == df.TIMESTAMP.min()), "BURN_AREA"] = 0
+
+    if timestamp_index is None:
+        return df
+
+    ts_array = pd.to_datetime(eopatch.timestamps)
+    df.query(f'TIMESTAMP == "{ts_array[timestamp_index]}"', inplace=True)
+    return df
+
+
+def apply_function_to_eopatch(eopatch, timestamp_index, function, output_mask_name):
+    df = convert_eop_to_df(eopatch, timestamp_index)
+    mask_timeless_shape = eopatch.mask_timeless["BURN_AREA"].shape
+    values = df.apply(function, axis=1).values
+    eopatch.mask_timeless[output_mask_name] = values.reshape(mask_timeless_shape)
+    return eopatch
+
+
+def apply_decision_trees_to_eopatch(eopatch, timestamp_index, classifier, output_mask_name, features):
+    df = convert_eop_to_df(eopatch, timestamp_index)
+    mask_timeless_shape = eopatch.mask_timeless["BURN_AREA"].shape
+    values = classifier.predict(df[features].values)
+    eopatch.mask_timeless[output_mask_name] = values.reshape(mask_timeless_shape)
+    return eopatch
+
+
+def apply_lgbm_to_eopatch(eopatch, timestamp_index, classifier, output_mask_name):
+    df = convert_eop_to_df(eopatch, timestamp_index)
+    mask_timeless_shape = eopatch.mask_timeless["BURN_AREA"].shape
+    mask = classifier.predict(df[classifier.feature_name_].values)
+    proba = classifier.predict_proba(df[classifier.feature_name_].values)[:, -1]
+    eopatch.mask_timeless[output_mask_name] = mask.reshape(mask_timeless_shape)
+    eopatch.data_timeless[f"{output_mask_name}_PROBA"] = proba.reshape(mask_timeless_shape)
+    return eopatch