Merge pull request #469 from girder/subifd-pyramids

Support reading OME Tiff files from sub-ifds.

sources/ometiff/large_image_source_ometiff/__init__.py

@@ -114,16 +114,14 @@ def __init__(self, path, **kwargs):
         self._omeLevels = [omebylevel.get(key) for key in range(max(omebylevel.keys()) + 1)]
         if base._tiffInfo.get('istiled'):
             self._tiffDirectories = [
-                TiledTiffDirectory(largeImagePath, int(entry['TiffData'][0]['IFD']))
+                TiledTiffDirectory(largeImagePath, int(entry['TiffData'][0].get('IFD', 0)))
                 if entry else None
                 for entry in self._omeLevels]
         else:
             self._tiffDirectories = [
                 TiledTiffDirectory(largeImagePath, 0, mustBeTiled=None)
                 if entry else None
                 for entry in self._omeLevels]
-        self._directoryCache = {}
-        self._directoryCacheMaxSize = max(20, len(self._omebase['TiffData']) * 3)
         self.tileWidth = base.tileWidth
         self.tileHeight = base.tileHeight
         self.levels = len(self._tiffDirectories)
@@ -184,37 +182,47 @@ def _checkForOMEZLoop(self, largeImagePath):
         info['Image']['Pixels']['PlanesFromZloop'] = 'true'
         info['Image']['Pixels']['SizeZ'] = str(zloop)
 
-    def _parseOMEInfo(self):
+    def _parseOMEInfo(self):  # noqa
         if isinstance(self._omeinfo['Image'], dict):
             self._omeinfo['Image'] = [self._omeinfo['Image']]
         for img in self._omeinfo['Image']:
             if isinstance(img['Pixels'].get('TiffData'), dict):
                 img['Pixels']['TiffData'] = [img['Pixels']['TiffData']]
             if isinstance(img['Pixels'].get('Plane'), dict):
                 img['Pixels']['Plane'] = [img['Pixels']['Plane']]
+            if isinstance(img['Pixels'].get('Channels'), dict):
+                img['Pixels']['Channels'] = [img['Pixels']['Channels']]
         try:
             self._omebase = self._omeinfo['Image'][0]['Pixels']
+            if isinstance(self._omebase.get('Plane'), dict):
+                self._omebase['Plane'] = [self._omebase['Plane']]
             if ((not len(self._omebase['TiffData']) or
                     len(self._omebase['TiffData']) == 1) and
-                    len(self._omebase['Plane'])):
+                    (len(self._omebase.get('Plane', [])) or
+                     len(self._omebase.get('Channel', [])))):
                 if not len(self._omebase['TiffData']) or self._omebase['TiffData'][0] == {}:
-                    self._omebase['TiffData'] = self._omebase['Plane']
+                    self._omebase['TiffData'] = self._omebase.get(
+                        'Plane', self._omebase.get('Channel'))
                 elif (int(self._omebase['TiffData'][0].get('PlaneCount', 0)) ==
-                        len(self._omebase['Plane'])):
-                    planes = copy.deepcopy(self._omebase['Plane'])
+                        len(self._omebase.get('Plane', self._omebase.get('Channel', [])))):
+                    planes = copy.deepcopy(self._omebase.get('Plane', self._omebase.get('Channel')))
                     for idx, plane in enumerate(planes):
                         plane['IFD'] = plane.get(
                             'IFD', int(self._omebase['TiffData'][0].get('IFD', 0)) + idx)
                     self._omebase['TiffData'] = planes
+            if isinstance(self._omebase['TiffData'], dict):
+                self._omebase['TiffData'] = [self._omebase['TiffData']]
             if len({entry.get('UUID', {}).get('FileName', '')
                     for entry in self._omebase['TiffData']}) > 1:
                 raise TileSourceException('OME Tiff references multiple files')
             if (len(self._omebase['TiffData']) != int(self._omebase['SizeC']) *
                     int(self._omebase['SizeT']) * int(self._omebase['SizeZ']) or
                     len(self._omebase['TiffData']) != len(
                         self._omebase.get('Plane', self._omebase['TiffData']))):
-                raise TileSourceException('OME Tiff contains frames that contain multiple planes')
+                raise TileSourceException(
+                    'OME Tiff contains frames that contain multiple planes')
         except (KeyError, ValueError, IndexError):
+            print('B')
             raise TileSourceException('OME Tiff does not contain an expected record')
 
     def getMetadata(self):
@@ -282,23 +290,32 @@ def getNativeMagnification(self):
     @methodcache()
     def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False,
                 sparseFallback=False, **kwargs):
-        if (z < 0 or z >= len(self._omeLevels) or self._omeLevels[z] is None or
-                kwargs.get('frame') in (None, 0, '0', '')):
+        if (z < 0 or z >= len(self._omeLevels) or (
+                self._omeLevels[z] is not None and kwargs.get('frame') in (None, 0, '0', ''))):
             return super(OMETiffFileTileSource, self).getTile(
                 x, y, z, pilImageAllowed=pilImageAllowed,
                 numpyAllowed=numpyAllowed, sparseFallback=sparseFallback,
                 **kwargs)
-        frame = int(kwargs['frame'])
+        frame = int(kwargs.get('frame') or 0)
         if frame < 0 or frame >= len(self._omebase['TiffData']):
             raise TileSourceException('Frame does not exist')
-        dirnum = int(self._omeLevels[z]['TiffData'][frame].get('IFD', frame))
-        if dirnum in self._directoryCache:
-            dir = self._directoryCache[dirnum]
+        subdir = None
+        if self._omeLevels[z] is not None:
+            dirnum = int(self._omeLevels[z]['TiffData'][frame].get('IFD', frame))
         else:
-            if len(self._directoryCache) >= self._directoryCacheMaxSize:
-                self._directoryCache = {}
-            dir = TiledTiffDirectory(self._getLargeImagePath(), dirnum, mustBeTiled=None)
-            self._directoryCache[dirnum] = dir
+            dirnum = int(self._omeLevels[-1]['TiffData'][frame].get('IFD', frame))
+            subdir = self.levels - 1 - z
+        dir = self._getDirFromCache(dirnum, subdir)
+        if subdir:
+            scale = int(2 ** subdir)
+            if (dir is None or
+                    dir.tileWidth != self.tileWidth or dir.tileHeight != self.tileHeight or
+                    abs(dir.imageWidth * scale - self.sizeX) > scale or
+                    abs(dir.imageHeight * scale - self.sizeY) > scale):
+                return super(OMETiffFileTileSource, self).getTile(
+                    x, y, z, pilImageAllowed=pilImageAllowed,
+                    numpyAllowed=numpyAllowed, sparseFallback=sparseFallback,
+                    **kwargs)
         try:
             tile = dir.getTile(x, y)
             format = 'JPEG'

sources/tiff/large_image_source_tiff/__init__.py

@@ -83,41 +83,12 @@ def __init__(self, path, **kwargs):
         super(TiffFileTileSource, self).__init__(path, **kwargs)
 
         largeImagePath = self._getLargeImagePath()
-        lastException = None
-        # Associated images are smallish TIFF images that have an image
-        # description and are not tiled.  They have their own TIFF directory.
-        # Individual TIFF images can also have images embedded into their
-        # directory as tags (this is a vendor-specific method of adding more
-        # images into a file) -- those are stored in the individual
-        # directories' _embeddedImages field.
-        self._associatedImages = {}
+        try:
+            alldir = self._scanDirectories()
+        except (ValidationTiffException, TiffException) as exc:
+            alldir = []
+            lastException = exc
 
-        # Query all know directories in the tif file.  Only keep track of
-        # directories that contain tiled images.
-        alldir = []
-        for directoryNum in itertools.count():  # pragma: no branch
-            try:
-                td = TiledTiffDirectory(largeImagePath, directoryNum)
-            except ValidationTiffException as exc:
-                lastException = exc
-                self._addAssociatedImage(largeImagePath, directoryNum)
-                continue
-            except TiffException as exc:
-                if not lastException:
-                    lastException = exc
-                break
-            if not td.tileWidth or not td.tileHeight:
-                continue
-            # Calculate the tile level, where 0 is a single tile, 1 is up to a
-            # set of 2x2 tiles, 2 is 4x4, etc.
-            level = int(math.ceil(math.log(max(
-                float(td.imageWidth) / td.tileWidth,
-                float(td.imageHeight) / td.tileHeight)) / math.log(2)))
-            if level < 0:
-                continue
-            # Store information for sorting with the directory.
-            alldir.append((level > 0, td.tileWidth * td.tileHeight, level,
-                           td.imageWidth * td.imageHeight, directoryNum, td))
         # If there are no tiled images, raise an exception.
         if not len(alldir):
             msg = "File %s didn't meet requirements for tile source: %s" % (
@@ -162,6 +133,57 @@ def __init__(self, path, **kwargs):
         self.sizeX = highest.imageWidth
         self.sizeY = highest.imageHeight
 
+    def _scanDirectories(self):
+        largeImagePath = self._getLargeImagePath()
+        lastException = None
+        # Associated images are smallish TIFF images that have an image
+        # description and are not tiled.  They have their own TIFF directory.
+        # Individual TIFF images can also have images embedded into their
+        # directory as tags (this is a vendor-specific method of adding more
+        # images into a file) -- those are stored in the individual
+        # directories' _embeddedImages field.
+        self._associatedImages = {}
+
+        dir = None
+        # Query all know directories in the tif file.  Only keep track of
+        # directories that contain tiled images.
+        alldir = []
+        associatedDirs = []
+        for directoryNum in itertools.count():  # pragma: no branch
+            try:
+                if dir is None:
+                    dir = TiledTiffDirectory(largeImagePath, directoryNum, validate=False)
+                else:
+                    dir._setDirectory(directoryNum)
+                    dir._loadMetadata()
+                dir._validate()
+            except ValidationTiffException as exc:
+                lastException = exc
+                associatedDirs.append(directoryNum)
+                continue
+            except TiffException as exc:
+                if not lastException:
+                    lastException = exc
+                break
+            if not dir.tileWidth or not dir.tileHeight:
+                continue
+            # Calculate the tile level, where 0 is a single tile, 1 is up to a
+            # set of 2x2 tiles, 2 is 4x4, etc.
+            level = int(math.ceil(math.log(max(
+                float(dir.imageWidth) / dir.tileWidth,
+                float(dir.imageHeight) / dir.tileHeight)) / math.log(2)))
+            if level < 0:
+                continue
+            td, dir = dir, None
+            # Store information for sorting with the directory.
+            alldir.append((level > 0, td.tileWidth * td.tileHeight, level,
+                           td.imageWidth * td.imageHeight, directoryNum, td))
+        if not alldir and lastException:
+            raise lastException
+        for directoryNum in associatedDirs:
+            self._addAssociatedImage(largeImagePath, directoryNum)
+        return alldir
+
     def _addAssociatedImage(self, largeImagePath, directoryNum, mustBeTiled=False, topImage=None):
         """
         Check if the specified TIFF directory contains an image with a sensible
@@ -349,6 +371,23 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False,
                 numpyAllowed=numpyAllowed, sparseFallback=sparseFallback,
                 exception=e, **kwargs)
 
+    def _getDirFromCache(self, dirnum, subdir=None):
+        if not hasattr(self, '_directoryCache'):
+            self._directoryCache = {}
+            self._directoryCacheMaxSize = max(20, self.levels * 3)
+        key = (dirnum, subdir)
+        result = self._directoryCache.get(key)
+        if result is None:
+            if len(self._directoryCache) >= self._directoryCacheMaxSize:
+                self._directoryCache = {}
+            try:
+                result = TiledTiffDirectory(
+                    self._getLargeImagePath(), dirnum, mustBeTiled=None, subDirectoryNum=subdir)
+            except IOTiffException:
+                result = None
+            self._directoryCache[key] = result
+        return result
+
     def getTileIOTiffException(self, x, y, z, pilImageAllowed=False,
                                numpyAllowed=False, sparseFallback=False,
                                exception=None, **kwargs):

sources/tiff/large_image_source_tiff/tiff_reader.py

@@ -99,55 +99,62 @@ class ValidationTiffException(TiffException):
 class TiledTiffDirectory(object):
 
     CoreFunctions = [
-        'SetDirectory', 'GetField', 'LastDirectory', 'GetMode', 'IsTiled',
-        'IsByteSwapped', 'IsUpSampled', 'IsMSB2LSB', 'NumberOfStrips'
+        'SetDirectory', 'SetSubDirectory', 'GetField',
+        'LastDirectory', 'GetMode', 'IsTiled', 'IsByteSwapped', 'IsUpSampled',
+        'IsMSB2LSB', 'NumberOfStrips',
     ]
 
-    def __init__(self, filePath, directoryNum, mustBeTiled=True):
+    def __init__(self, filePath, directoryNum, mustBeTiled=True, subDirectoryNum=0, validate=True):
         """
         Create a new reader for a tiled image file directory in a TIFF file.
 
         :param filePath: A path to a TIFF file on disk.
         :type filePath: str
         :param directoryNum: The number of the TIFF image file directory to
-        open.
+            open.
         :type directoryNum: int
         :param mustBeTiled: if True, only tiled images validate.  If False,
             only non-tiled images validate.  None validates both.
+        :type mustBeTiled: bool
+        :param subDirectoryNum: if set, the number of the TIFF subdirectory.
+        :type subDirectoryNum: int
+        :param validate: if False, don't validate that images can be read.
+        :type mustBeTiled: bool
         :raises: InvalidOperationTiffException or IOTiffException or
         ValidationTiffException
         """
-        # TODO how many to keep in the cache
-        # create local cache to store Jpeg tables and
-        # getTileByteCountsType
-
+        # create local cache to store Jpeg tables and getTileByteCountsType
         self.cache = LRUCache(10)
         self._mustBeTiled = mustBeTiled
 
         self._tiffFile = None
         self._tileLock = threading.RLock()
 
-        self._open(filePath, directoryNum)
+        self._open(filePath, directoryNum, subDirectoryNum)
         self._loadMetadata()
         config.getConfig('logger').debug(
-            'TiffDirectory %d Information %r', directoryNum, self._tiffInfo)
+            'TiffDirectory %d:%d Information %r',
+            directoryNum, subDirectoryNum or 0, self._tiffInfo)
         try:
-            self._validate()
+            if validate:
+                self._validate()
         except ValidationTiffException:
             self._close()
             raise
 
     def __del__(self):
         self._close()
 
-    def _open(self, filePath, directoryNum):
+    def _open(self, filePath, directoryNum, subDirectoryNum=0):
         """
         Open a TIFF file to a given file and IFD number.
 
         :param filePath: A path to a TIFF file on disk.
         :type filePath: str
         :param directoryNum: The number of the TIFF IFD to be used.
         :type directoryNum: int
+        :param subDirectoryNum: The number of the TIFF sub-IFD to be used.
+        :type subDirectoryNum: int
         :raises: InvalidOperationTiffException or IOTiffException
         """
         self._close()
@@ -170,12 +177,26 @@ def _open(self, filePath, directoryNum):
                     hasattr(self._tiffFile, func.lower())):
                 setattr(self._tiffFile, func, getattr(
                     self._tiffFile, func.lower()))
+        self._setDirectory(directoryNum, subDirectoryNum)
 
+    def _setDirectory(self, directoryNum, subDirectoryNum=0):
         self._directoryNum = directoryNum
         if self._tiffFile.SetDirectory(self._directoryNum) != 1:
             self._tiffFile.close()
             raise IOTiffException(
                 'Could not set TIFF directory to %d' % directoryNum)
+        self._subDirectoryNum = subDirectoryNum
+        if self._subDirectoryNum:
+            subifds = self._tiffFile.GetField('subifd')
+            if (subifds is None or self._subDirectoryNum < 1 or
+                    self._subDirectoryNum > len(subifds)):
+                raise IOTiffException(
+                    'Could not set TIFF subdirectory to %d' % subDirectoryNum)
+            subifd = subifds[self._subDirectoryNum - 1]
+            if self._tiffFile.SetSubDirectory(subifd) != 1:
+                self._tiffFile.close()
+                raise IOTiffException(
+                    'Could not set TIFF subdirectory to %d' % subDirectoryNum)
 
     def _close(self):
         if self._tiffFile:
@@ -277,7 +298,7 @@ def _loadMetadata(self):
                     'Loading field "%s" in directory number %d resulted in TypeError - "%s"',
                     field, self._directoryNum, err)
 
-        for func in self.CoreFunctions[2:]:
+        for func in self.CoreFunctions[3:]:
             if hasattr(self._tiffFile, func):
                 value = getattr(self._tiffFile, func)()
                 if value:

test/data/sample.subifd.ome.tif.sha512

@@ -0,0 +1 @@
+35ec252c94b1ad0b9d5bd42c89c1d15c83065d6734100d6f596237ff36e8d4495bcfed2c9ea24ab0b4a35aef59871da429dbd48faf0232219dc4391215ba59ce

test/test_cached_tiles.py

@@ -132,20 +132,20 @@ def countInit(*args, **kwargs):
         self.delCount = 0
         source = large_image.getTileSource(imagePath)
         assert source is not None
-        assert self.initCount == 14
-        assert self.delCount < 14
+        assert self.initCount == 12
+        assert self.delCount < 12
         # Create another source; we shouldn't init it again, as it should be
         # cached.
         source = large_image.getTileSource(imagePath)
         assert source is not None
-        assert self.initCount == 14
-        assert self.delCount < 14
+        assert self.initCount == 12
+        assert self.delCount < 12
         source = None
         # Clear the cache to free references and force garbage collection
         cachesClear()
         gc.collect(2)
         cachesClear()
-        assert self.delCount == 14
+        assert self.delCount == 12
 
 
 class TestMemcachedCache(LargeImageCachedTilesTest):