Merge branch 'main' into bugfix-fill-value-palette

.github/workflows/ci.yaml

@@ -30,9 +30,7 @@ jobs:
       - name: Setup Conda Environment
         uses: conda-incubator/setup-miniconda@v3
         with:
-          miniforge-variant: Mambaforge
           miniforge-version: latest
-          use-mamba: true
           python-version: ${{ matrix.python-version }}
           activate-environment: test-environment
           channels: conda-forge
@@ -65,6 +63,7 @@ jobs:
       - name: Install trollimage
         shell: bash -l {0}
         run: |
+          conda list
           pip install --no-deps -e .
           # pip forces non-wheel builds if we provide --cython-coverage as a --build-option
           # and that's way too slow

.github/workflows/deploy.yaml

@@ -59,7 +59,7 @@ jobs:
           platforms: all
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.19.2
+        uses: pypa/cibuildwheel@v2.21.1
         env:
           CIBW_SKIP: "cp36-* cp37-* cp38-* pp* *-manylinux_i686 *-musllinux_i686 *-musllinux_aarch64 *-win32"
           CIBW_ARCHS: "${{ matrix.cibw_archs }}"
@@ -101,14 +101,14 @@ jobs:
           path: dist
       - name: Publish package to Test PyPI
         if: github.event.action != 'published' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
-        uses: pypa/gh-action-pypi-publish@v1.9.0
+        uses: pypa/gh-action-pypi-publish@v1.10.2
         with:
           user: __token__
           password: ${{ secrets.test_pypi_password }}
           repository_url: https://test.pypi.org/legacy/
       - name: Publish package to PyPI
         if: github.event.action == 'published'
-        uses: pypa/gh-action-pypi-publish@v1.9.0
+        uses: pypa/gh-action-pypi-publish@v1.10.2
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

continuous_integration/environment.yaml

@@ -13,6 +13,7 @@ dependencies:
   - coverage
   - codecov
   - rasterio
+  - libgdal-jp2openjpeg
   - libtiff
   - pyproj
   - pyresample

trollimage/tests/test_image.py

@@ -1612,8 +1612,8 @@ def test_crude_stretch_integer_data(self, dtype, max_stretch):
         np.testing.assert_allclose(img.data.values, arr.astype(np.float32) / max_stretch, rtol=1e-6)
 
     @pytest.mark.parametrize("dtype", (np.float32, np.float64, float))
-    def test_invert(self, dtype):
-        """Check inversion of the image."""
+    def test_invert_single_parameter(self, dtype):
+        """Check inversion of the image for single inversion parameter."""
         arr = np.arange(75, dtype=dtype).reshape(5, 5, 3) / 75.
         data = xr.DataArray(arr.copy(), dims=['y', 'x', 'bands'],
                             coords={'bands': ['R', 'G', 'B']})
@@ -1625,7 +1625,11 @@ def test_invert(self, dtype):
         assert img.data.dtype == dtype
         assert np.allclose(img.data.values, 1 - arr)
 
-        data = xr.DataArray(arr.copy(), dims=['y', 'x', 'bands'],
+    @pytest.mark.parametrize("dtype", (np.float32, np.float64, float))
+    def test_invert_parameter_for_each_channel(self, dtype):
+        """Check inversion of the image for single inversion parameter."""
+        arr = np.arange(75, dtype=dtype).reshape(5, 5, 3) / 75.
+        data = xr.DataArray(arr, dims=['y', 'x', 'bands'],
                             coords={'bands': ['R', 'G', 'B']})
         img = xrimage.XRImage(data)
 
@@ -1635,6 +1639,7 @@ def test_invert(self, dtype):
         scale = xr.DataArray(np.array([-1, 1, -1]), dims=['bands'],
                              coords={'bands': ['R', 'G', 'B']})
         np.testing.assert_allclose(img.data.values, (data * scale + offset).values)
+        assert img.data.dtype == dtype
 
     @pytest.mark.parametrize("dtype", (np.float32, np.float64, float))
     def test_linear_stretch(self, dtype):
@@ -2626,6 +2631,16 @@ def test_palettize_fill_value(self):
         img.palettize(brbg)
         assert img.data[0, 2, 2] == img.data.attrs["_FillValue"]
 
+    def test_palettize_bad_fill_value(self):
+        """Test that palettize warns with a strange fill value."""
+        arr = np.arange(25, dtype="uint8").reshape(5, 5)
+        data = xr.DataArray(arr.copy(), dims=['y', 'x'], attrs={"_FillValue": 10})
+        img = xrimage.XRImage(data)
+        with pytest.warns(UserWarning,
+                          match="Palettizing uint8 data with the _FillValue attribute set to 10, "
+                                "but palettize is not generally fill value aware"):
+            img.palettize(brbg)
+
 
 class TestXRImageSaveScaleOffset:
     """Test case for saving an image with scale and offset tags."""

trollimage/xrimage.py

@@ -1133,7 +1133,6 @@ def _check_stretch_value(self, val, kind='min'):
         if val is None:
             non_band_dims = tuple(x for x in self.data.dims if x != 'bands')
             val = getattr(self.data, kind)(dim=non_band_dims)
-
         if isinstance(val, (list, tuple)):
             val = self.xrify_tuples(val)
 
@@ -1303,7 +1302,7 @@ def invert(self, invert=True):
         logger.debug("Applying invert with parameters %s", str(invert))
         if isinstance(invert, (tuple, list)):
             invert = self.xrify_tuples(invert)
-            offset = invert.astype(int)
+            offset = invert.astype(self.data.dtype)
             scale = (-1) ** offset
         elif invert:
             offset = 1
@@ -1485,9 +1484,10 @@ def palettize(self, colormap):
             mode = "PA"
             new_data = da.concatenate([new_data, self.data.sel(bands=['A'])], axis=0)
 
+        old_dtype = self.data.dtype
         self.data.data = new_data
         self.data.coords['bands'] = list(mode)
-        self.data.attrs["_FillValue"] = colormap.values.shape[0]-1
+        self._set_new_fill_value_after_palettize(colormap, old_dtype)
         # See docstring notes above for how scale/offset should be used
         scale_factor, offset = self._get_colormap_scale_offset(colormap)
         self.data.attrs.setdefault('enhancement_history', []).append({
@@ -1504,6 +1504,28 @@ def _get_colormap_scale_offset(colormap):
         offset = -cmap_min * scale_factor
         return scale_factor, offset
 
+    def _set_new_fill_value_after_palettize(self, colormap, old_dtype):
+        """Set new fill value after palettizing."""
+        # OK: float without fill value or fill value nan
+        # OK: int without fill value or fill value to max value
+        if ((np.issubdtype(old_dtype, np.inexact) and
+             np.isnan(self.data.attrs.get("_FillValue", np.nan))) or
+            (np.issubdtype(old_dtype, np.integer) and
+             self.data.attrs.get("_FillValue", np.iinfo(old_dtype).max) == np.iinfo(old_dtype).max)):
+            self.data.attrs["_FillValue"] = colormap.values.shape[0]-1
+        # not OK: float or int with different fill value
+        elif "_FillValue" in self.data.attrs:
+            warnings.warn(
+                f"Palettizing {old_dtype.name:s} data with the _FillValue attribute set to "
+                f"{self.data.attrs['_FillValue']!s}, "
+                "but palettize is not generally fill value aware (masked data "
+                "will be correctly palettized only for float with NaN or for "
+                "ints with fill value set to dtype max.",
+                UserWarning,
+                stacklevel=3)
+        # else: non-numeric data, probably doesn't work at all and will fail
+        # elsewhere anyway
+
     def blend(self, src):
         r"""Alpha blend *src* on top of the current image.