Merge branch 'master' into zarr-sink

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Improvements
 - Log and recover from occasional openslide failures ([#1461](../../pull/1461))
 - Add support for Imaging Data Commons ([#1450](../../pull/1450))
+- Speed up some retiling ([#1471](../../pull/1471))
 
 ### Changes
 - Make GDAL an optional dependency for the converter ([#1464](../../pull/1464))

large_image/cache_util/cache.py

@@ -65,7 +65,7 @@ def strhash(*args, **kwargs) -> str:
     """
     if kwargs:
         return '%r,%r' % (args, sorted(kwargs.items()))
-    return '%r' % (args, )
+    return repr(args)
 
 
 def methodcache(key: Optional[Callable] = None) -> Callable:  # noqa

large_image/config.py

@@ -1,3 +1,4 @@
+import functools
 import json
 import logging
 import os
@@ -60,6 +61,8 @@
 }
 
 
+# Fix when we drop Python 3.8 to just be @functools.cache
+@functools.lru_cache(maxsize=None)
 def getConfig(key: Optional[str] = None,
               default: Optional[Union[str, bool, int, logging.Logger]] = None) -> Any:
     """
@@ -110,6 +113,7 @@ def setConfig(key: str, value: Optional[Union[str, bool, int, logging.Logger]])
     curConfig = getConfig()
     if curConfig.get(key) is not value:
         curConfig[key] = value
+        getConfig.cache_clear()
 
 
 def _ignoreSourceNames(

large_image/tilesource/tiledict.py

@@ -120,35 +120,43 @@ def _retileTile(self) -> np.ndarray:
         Given the tile information, create a numpy array and merge multiple
         tiles together to form a tile of a different size.
         """
+        tileWidth = self.metadata['tileWidth']
+        tileHeight = self.metadata['tileHeight']
+        level = self.level
+        frame = self.frame
+        width = self.width
+        height = self.height
+        tx = self['x']
+        ty = self['y']
+
         retile = None
-        xmin = int(max(0, self['x'] // self.metadata['tileWidth']))
-        xmax = int((self['x'] + self.width - 1) // self.metadata['tileWidth'] + 1)
-        ymin = int(max(0, self['y'] // self.metadata['tileHeight']))
-        ymax = int((self['y'] + self.height - 1) // self.metadata['tileHeight'] + 1)
+        xmin = int(max(0, tx // tileWidth))
+        xmax = int((tx + width - 1) // tileWidth + 1)
+        ymin = int(max(0, ty // tileHeight))
+        ymax = int((ty + height - 1) // tileHeight + 1)
         for y in range(ymin, ymax):
             for x in range(xmin, xmax):
                 tileData = self.source.getTile(
-                    x, y, self.level,
-                    numpyAllowed='always', sparseFallback=True, frame=self.frame)
-                tileData, _ = _imageToNumpy(tileData)
-                if retile is None:
-                    retile = np.empty(
-                        (self.height, self.width, tileData.shape[2]),
-                        dtype=tileData.dtype)
-                x0 = int(x * self.metadata['tileWidth'] - self['x'])
-                y0 = int(y * self.metadata['tileHeight'] - self['y'])
+                    x, y, level,
+                    numpyAllowed='always', sparseFallback=True, frame=frame)
+                if not isinstance(tileData, np.ndarray) or len(tileData.shape) != 3:
+                    tileData, _ = _imageToNumpy(tileData)
+                x0 = int(x * tileWidth - tx)
+                y0 = int(y * tileHeight - ty)
                 if x0 < 0:
                     tileData = tileData[:, -x0:]
                     x0 = 0
                 if y0 < 0:
                     tileData = tileData[-y0:, :]
                     y0 = 0
-                tileData = tileData[:min(tileData.shape[0], self.height - y0),
-                                    :min(tileData.shape[1], self.width - x0)]
-                if tileData.shape[2] < retile.shape[2]:  # type: ignore[misc]
+                tw = min(tileData.shape[1], width - x0)
+                th = min(tileData.shape[0], height - y0)
+                if retile is None:
+                    retile = np.empty((height, width, tileData.shape[2]), dtype=tileData.dtype)
+                elif tileData.shape[2] < retile.shape[2]:
                     retile = retile[:, :, :tileData.shape[2]]
-                retile[y0:y0 + tileData.shape[0], x0:x0 + tileData.shape[1]] = tileData[
-                    :, :, :retile.shape[2]]  # type: ignore[misc]
+                retile[y0:y0 + th, x0:x0 + tw] = tileData[
+                    :th, :tw, :retile.shape[2]]  # type: ignore[misc]
         return cast(np.ndarray, retile)
 
     def __getitem__(self, key: str, *args, **kwargs) -> Any:

sources/tiff/large_image_source_tiff/tiff_reader.py

@@ -596,13 +596,27 @@ def _getUncompressedTile(self, tileNum):
                     self._tiffFile).value
             stripsCount = min(self._stripsPerTile, self._stripCount - tileNum)
             tileSize = stripSize * self._stripsPerTile
-        imageBuffer = ctypes.create_string_buffer(tileSize)
+        tw, th = self._tileWidth, self._tileHeight
+        if self._tiffInfo.get('orientation') in {
+                libtiff_ctypes.ORIENTATION_LEFTTOP,
+                libtiff_ctypes.ORIENTATION_RIGHTTOP,
+                libtiff_ctypes.ORIENTATION_RIGHTBOT,
+                libtiff_ctypes.ORIENTATION_LEFTBOT}:
+            tw, th = th, tw
+        format = (
+            self._tiffInfo.get('bitspersample'),
+            self._tiffInfo.get('sampleformat') if self._tiffInfo.get(
+                'sampleformat') is not None else libtiff_ctypes.SAMPLEFORMAT_UINT)
+        image = np.empty((th, tw, self._tiffInfo['samplesperpixel']),
+                         dtype=_ctypesFormattbl[format])
+        imageBuffer = image.ctypes.data_as(ctypes.POINTER(ctypes.c_char))
         if self._tiffInfo.get('istiled'):
             with self._tileLock:
                 readSize = libtiff_ctypes.libtiff.TIFFReadEncodedTile(
                     self._tiffFile, tileNum, imageBuffer, tileSize)
         else:
             readSize = 0
+            imageBuffer = ctypes.cast(imageBuffer, ctypes.POINTER(ctypes.c_char * 2)).contents
             for stripNum in range(stripsCount):
                 with self._tileLock:
                     chunkSize = libtiff_ctypes.libtiff.TIFFReadEncodedStrip(
@@ -621,21 +635,6 @@ def _getUncompressedTile(self, tileNum):
             raise IOTiffError(
                 'Read an unexpected number of bytes from an encoded tile' if readSize >= 0 else
                 'Failed to read from an encoded tile')
-        tw, th = self._tileWidth, self._tileHeight
-        if self._tiffInfo.get('orientation') in {
-                libtiff_ctypes.ORIENTATION_LEFTTOP,
-                libtiff_ctypes.ORIENTATION_RIGHTTOP,
-                libtiff_ctypes.ORIENTATION_RIGHTBOT,
-                libtiff_ctypes.ORIENTATION_LEFTBOT}:
-            tw, th = th, tw
-        format = (
-            self._tiffInfo.get('bitspersample'),
-            self._tiffInfo.get('sampleformat') if self._tiffInfo.get(
-                'sampleformat') is not None else libtiff_ctypes.SAMPLEFORMAT_UINT)
-        image = np.ctypeslib.as_array(ctypes.cast(
-            imageBuffer, ctypes.POINTER(ctypes.c_uint8)), (tileSize, )).view(
-                _ctypesFormattbl[format]).reshape(
-                    (th, tw, self._tiffInfo['samplesperpixel']))
         if (self._tiffInfo.get('samplesperpixel') == 3 and
                 self._tiffInfo.get('photometric') == libtiff_ctypes.PHOTOMETRIC_YCBCR):
             if self._tiffInfo.get('bitspersample') == 16: