Merge pull request #820 from girder/feat/gdal-vfs

Handle file URLs with GDAL
girder · Apr 11, 2022 · ffc0882 · ffc0882
2 parents 88cb078 + aa87237
commit ffc0882
Show file tree

Hide file tree

Showing 2 changed files with 46 additions and 1 deletion.
diff --git a/sources/gdal/large_image_source_gdal/__init__.py b/sources/gdal/large_image_source_gdal/__init__.py
@@ -20,6 +20,7 @@
 import struct
 import tempfile
 import threading
+from urllib.parse import urlencode, urlparse
 
 import numpy
 import PIL.Image
@@ -68,6 +69,21 @@
 NeededInitPrefix = '' if int(pyproj.proj_version_str.split('.')[0]) >= 6 else InitPrefix
 
 
+def make_vsi(url: str, **options):
+    if str(url).startswith('s3://'):
+        s3_path = url.replace('s3://', '')
+        vsi = f'/vsis3/{s3_path}'
+    else:
+        gdal_options = {
+            'url': str(url),
+            'use_head': 'no',
+            'list_dir': 'no',
+        }
+        gdal_options.update(options)
+        vsi = f'/vsicurl?{urlencode(gdal_options)}'
+    return vsi
+
+
 class GDALFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
     """
     Provides tile access to geospatial files.
@@ -114,7 +130,7 @@ def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs):
         """
         super().__init__(path, **kwargs)
         self._bounds = {}
-        self._largeImagePath = str(self._getLargeImagePath())
+        self._largeImagePath = self._getLargeImagePath()
         try:
             self.dataset = gdal.Open(self._largeImagePath, gdalconst.GA_ReadOnly)
         except RuntimeError:
@@ -159,6 +175,17 @@ def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs):
         self._getTileLock = threading.Lock()
         self._setDefaultStyle()
 
+    def _getLargeImagePath(self):
+        """Get GDAL-compatible image path.
+
+        This will cast the output to a string and can also handle URLs
+        ('http', 'https', 'ftp', 's3') for use with GDAL
+        `Virtual Filesystems Interface <https://gdal.org/user/virtual_file_systems.html>`_.
+        """
+        if urlparse(str(self.largeImagePath)).scheme in {'http', 'https', 'ftp', 's3'}:
+            return make_vsi(self.largeImagePath)
+        return str(self.largeImagePath)
+
     def _checkNetCDF(self):
         return False
 

diff --git a/test/test_source_gdal.py b/test/test_source_gdal.py
@@ -532,3 +532,21 @@ def testMatplotlibPalette():
     image = PIL.Image.open(io.BytesIO(image))
     image = numpy.asarray(image)
     assert list(image[0, 0, :]) == [68, 1, 84, 0]
+
+
+def testHttpVfsPath():
+    imagePath = datastore.get_url('landcover_sample_1000.tif')
+    source = large_image_source_gdal.open(
+        imagePath, projection='EPSG:3857', encoding='PNG')
+    tileMetadata = source.getMetadata()
+    assert tileMetadata['tileWidth'] == 256
+    assert tileMetadata['tileHeight'] == 256
+    assert tileMetadata['sizeX'] == 65536
+    assert tileMetadata['sizeY'] == 65536
+    assert tileMetadata['levels'] == 9
+    assert tileMetadata['bounds']['xmax'] == pytest.approx(-7837888, 1)
+    assert tileMetadata['bounds']['xmin'] == pytest.approx(-8909162, 1)
+    assert tileMetadata['bounds']['ymax'] == pytest.approx(5755717, 1)
+    assert tileMetadata['bounds']['ymin'] == pytest.approx(4876273, 1)
+    assert tileMetadata['bounds']['srs'] == 'epsg:3857'
+    assert tileMetadata['geospatial']