Merge pull request #61 from wpreimes/zlib

Zlib compression only for numeric data

.github/workflows/ubuntu.yml

@@ -24,7 +24,7 @@ jobs:
         run : |
           git submodule init
           git submodule update
-      - uses: conda-incubator/setup-miniconda@v2.0.1
+      - uses: conda-incubator/setup-miniconda@v2
         with:
           miniconda-version: "latest"
           auto-update-conda: true

.github/workflows/windows.yml

@@ -24,7 +24,7 @@ jobs:
         run : |
           git submodule init
           git submodule update
-      - uses: conda-incubator/setup-miniconda@v2.0.1
+      - uses: conda-incubator/setup-miniconda@v2
         with:
           miniconda-version: "latest"
           auto-update-conda: true

CHANGELOG.rst

@@ -6,6 +6,7 @@ Unreleased changes in master
 ============================
 
 - Add test for writing strings to netcdf
+- Zlib compress always off for non-numeric data
 
 Version 0.4.0
 =============

environment.yml

@@ -1,9 +1,11 @@
 name: pynetcf
+channels:
+  - conda-forge
 dependencies:
 - h5py
 - numpy
 - pandas
-- netCDF4!=1.6.2
+- netCDF4
 - pykdtree
 - pyproj
 - pip

setup.cfg

@@ -47,7 +47,7 @@ python_requires = >=3.8
 # Version specifiers like >=2.2,<3.0 avoid problems due to API changes in
 # new major versions. This works if the required packages follow Semantic Versioning.
 # For more information, check out https://semver.org/.
-install_requires = importlib-metadata; numpy; scipy; pandas; netCDF4!=1.6.2; pygeogrids; pygeobase
+install_requires = importlib-metadata; numpy; pandas; netCDF4; pygeogrids; pygeobase
 
 [options.packages.find]
 where = src

src/pynetcf/base.py

@@ -206,6 +206,9 @@ def write_var(self,
 
         if zlib is None:
             zlib = self.zlib
+        if not np.issubdtype(dtype, np.number):
+            # Only numeric data can be compressed
+            zlib = False
         if complevel is None:
             complevel = self.complevel
 

src/pynetcf/image.py

@@ -337,3 +337,4 @@ def __getitem__(self, key):
                 data[var] = self.dataset.variables[var][:, row, column]
 
         return pd.DataFrame(data, index=self.times)
+

src/pynetcf/point_data.py

@@ -355,10 +355,15 @@ def write(self,
                         if "dims" in metadata and var_data in metadata["dims"]:
                             dimensions = metadata["dims"][var_data]
 
+                    comp = self.compression_info.copy()
+
+                    if not np.issubdtype(dtype, np.number):
+                        comp['zlib'] = False
+
                     self.nc.createVariable(var_data,
                                            dtype,
                                            dimensions=dimensions,
-                                           **self.compression_info)
+                                           **comp)
 
                 self.nc.variables[var_data][idx] = data[var_data]
 

tests/test_point_data.py

@@ -55,6 +55,26 @@ def tearDown(self):
         except OSError:
             pass
 
+    def test_io_str(self):
+        """
+        Write/read test.
+        """
+        loc_ids = np.arange(0, 5)
+        data1 = np.array([str(n) for n in np.arange(5, 10)])
+        data2 = np.arange(10, 15)
+
+        with PointData(self.fn, mode="w", n_obs=5) as nc:
+            for loc_id, d1, d2 in zip(loc_ids, data1, data2):
+                nc.write(loc_id, {"var1": d1, "var2": d2})
+
+        with PointData(self.fn) as nc:
+            nptest.assert_array_equal(
+                nc["var1"], np.array([str(n) for n in range(5, 10)]))
+            assert nc.nc["var1"].filters()["zlib"] is False
+            assert nc.nc["var2"].filters()["zlib"] is True
+            nptest.assert_array_equal(nc["var1"][1], np.array(['6']))
+            nptest.assert_array_equal(nc["var2"][1], np.array([11]))
+
     def test_io(self):
         """
         Write/read test.
@@ -69,7 +89,7 @@ def test_io(self):
 
         with PointData(self.fn) as nc:
             nptest.assert_array_equal(nc["var1"], range(5, 10))
-            assert nc.nc["var1"].filters()["zlib"] == True
+            assert nc.nc["var1"].filters()["zlib"] is True
             nptest.assert_array_equal(nc["var2"][1], np.array([11]))
 
     def test_ioerror(self):