Better release file handles.

girder/girder_large_image/rest/large_image_resource.py

@@ -244,6 +244,8 @@ def __init__(self):
     )
     @access.admin(scope=TokenScope.DATA_WRITE)
     def cacheClear(self, params):
+        import gc
+
         before = cache_util.cachesInfo()
         cache_util.cachesClear()
         after = cache_util.cachesInfo()
@@ -252,6 +254,7 @@ def cacheClear(self, params):
         while time.time() < stoptime and any(after[key]['used'] for key in after):
             time.sleep(0.1)
             after = cache_util.cachesInfo()
+        gc.collect()
         return {'cacheCleared': datetime.datetime.utcnow(), 'before': before, 'after': after}
 
     @describeRoute(

large_image/cache_util/__init__.py

@@ -26,6 +26,8 @@
 
 from .cachefactory import CacheFactory, pickAvailableCache
 
+_cacheClearFuncs = []
+
 
 @atexit.register
 def cachesClear(*args, **kwargs):
@@ -44,6 +46,8 @@ def cachesClear(*args, **kwargs):
                 tileCache.clear()
         except Exception:
             pass
+    for func in _cacheClearFuncs:
+        func()
 
 
 def cachesInfo(*args, **kwargs):

sources/bioformats/large_image_source_bioformats/__init__.py

@@ -28,6 +28,7 @@
 import os
 import threading
 import types
+import weakref
 
 import numpy
 
@@ -71,6 +72,7 @@ def _monitor_thread():
             javabridge.attach()
             while len(_openImages):
                 source = _openImages.pop()
+                source = source()
                 try:
                     source._bioimage.close()
                 except Exception:
@@ -195,7 +197,7 @@ def __init__(self, path, **kwargs):  # noqa
                     raise TileSourceFileNotFoundError(largeImagePath) from None
                 self.logger.debug('File cannot be opened via Bioformats. (%r)' % exc)
                 raise TileSourceError('File cannot be opened via Bioformats (%r)' % exc)
-            _openImages.append(self)
+            _openImages.append(weakref.ref(self))
 
             rdr = self._bioimage.rdr
             # Bind additional functions not done by bioformats module.
@@ -295,7 +297,8 @@ def __del__(self):
             try:
                 javabridge.attach()
                 self._bioimage.close()
-                _openImages.remove(self)
+                del self._bioimage
+                _openImages.remove(weakref.ref(self))
             finally:
                 if javabridge.get_env():
                     javabridge.detach()

sources/nd2/large_image_source_nd2/__init__.py

@@ -140,7 +140,11 @@ def __init__(self, path, **kwargs):
                 raise TileSourceFileNotFoundError(self._largeImagePath) from None
             raise TileSourceError('File cannot be opened via the nd2 source.')
         # We use dask to allow lazy reading of large images
-        self._nd2array = self._nd2.to_dask(copy=False, wrapper=False)  # ##DWM::
+        try:
+            self._nd2array = self._nd2.to_dask(copy=False, wrapper=False)
+        except TypeError as exc:
+            self.logger.debug('Failed to read nd2 file: %s', exc)
+            raise TileSourceError('File cannot be opened via the nd2 source.')
         arrayOrder = list(self._nd2.sizes)
         # Reorder this so that it is XY (P), T, Z, C, Y, X, S (or at least end
         # in Y, X[, S]).
@@ -167,13 +171,24 @@ def __init__(self, path, **kwargs):
             self.tileHeight = self.sizeY
         self.levels = int(max(1, math.ceil(math.log(
             float(max(self.sizeX, self.sizeY)) / self.tileWidth) / math.log(2)) + 1))
-        self._framecount = (
-            self._nd2.metadata.contents.channelCount * self._nd2.metadata.contents.frameCount)
+        try:
+            self._framecount = (
+                self._nd2.metadata.contents.channelCount * self._nd2.metadata.contents.frameCount)
+        except Exception:
+            self._nd2.close()
+            del self._nd2
+            raise TileSourceError(
+                'File cannot be parsed with the nd2 source.  Is it a legacy nd2 file?')
         self._bandnames = {
             chan.channel.name.lower(): idx for idx, chan in enumerate(self._nd2.metadata.channels)}
         self._channels = [chan.channel.name for chan in self._nd2.metadata.channels]
         self._tileLock = threading.RLock()
 
+    def __del__(self):
+        if hasattr(self, '_nd2'):
+            self._nd2.close()
+            del self._nd2
+
     def getNativeMagnification(self):
         """
         Get the magnification at a particular level.

sources/vips/large_image_source_vips/__init__.py

@@ -11,7 +11,7 @@
 import pyvips
 
 from large_image import config
-from large_image.cache_util import LruCacheMetaclass, methodcache
+from large_image.cache_util import LruCacheMetaclass, _cacheClearFuncs, methodcache
 from large_image.constants import (NEW_IMAGE_PATH_FLAG, TILE_FORMAT_NUMPY,
                                    GValueToDtype, SourcePriority,
                                    dtypeToGValue)
@@ -24,6 +24,18 @@
     r'(^[^.]*|\.(yml|yaml|json|png|svs))$'
 
 
+def _clearVipsCache():
+    old = pyvips.voperation.cache_get_max_files()
+    pyvips.voperation.cache_set_max_files(0)
+    pyvips.voperation.cache_set_max_files(old)
+    old = pyvips.voperation.cache_get_max()
+    pyvips.voperation.cache_set_max(0)
+    pyvips.voperation.cache_set_max(old)
+
+
+_cacheClearFuncs.append(_clearVipsCache)
+
+
 class VipsFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
     """
     Provides tile access to any libvips compatible file.
@@ -79,6 +91,10 @@ def __init__(self, path, **kwargs):
             self._frames = [page]
             self.sizeX = subImage.width
             self.sizeY = subImage.height
+            try:
+                self._image.close()
+            except Exception:
+                pass
             self._image = subImage
         self.levels = int(max(1, math.ceil(math.log(
             float(max(self.sizeX, self.sizeY)) / self.tileWidth) / math.log(2)) + 1))