From fdec6d76662783f376bec5b812121f57f907ffec Mon Sep 17 00:00:00 2001 From: banesullivan Date: Thu, 7 Apr 2022 23:09:50 -0600 Subject: [PATCH 1/3] Handle file URLs with GDAL --- .../gdal/large_image_source_gdal/__init__.py | 28 ++++++++++++++++++- test/test_source_gdal.py | 18 ++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/sources/gdal/large_image_source_gdal/__init__.py b/sources/gdal/large_image_source_gdal/__init__.py index 708adbdd2..0533b43b4 100644 --- a/sources/gdal/large_image_source_gdal/__init__.py +++ b/sources/gdal/large_image_source_gdal/__init__.py @@ -20,6 +20,7 @@ import struct import tempfile import threading +from urllib.parse import urlencode, urlparse import numpy import PIL.Image @@ -68,6 +69,21 @@ NeededInitPrefix = '' if int(pyproj.proj_version_str.split('.')[0]) >= 6 else InitPrefix +def make_vsi(url: str, **options): + if str(url).startswith("s3://"): + s3_path = url.replace("s3://", "") + vsi = f"/vsis3/{s3_path}" + else: + gdal_options = { + "url": str(url), + "use_head": "no", + "list_dir": "no", + } + gdal_options.update(options) + vsi = f"/vsicurl?{urlencode(gdal_options)}" + return vsi + + class GDALFileTileSource(FileTileSource, metaclass=LruCacheMetaclass): """ Provides tile access to geospatial files. @@ -114,7 +130,7 @@ def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs): """ super().__init__(path, **kwargs) self._bounds = {} - self._largeImagePath = str(self._getLargeImagePath()) + self._largeImagePath = self._getLargeImagePath() try: self.dataset = gdal.Open(self._largeImagePath, gdalconst.GA_ReadOnly) except RuntimeError: @@ -159,6 +175,16 @@ def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs): self._getTileLock = threading.Lock() self._setDefaultStyle() + def _getLargeImagePath(self): + """Get GDAL-compatible image path. + + This will cast the outout to a string and can also handle URLs for use + with GDAL VFS interface. + """ + if urlparse(str(self.largeImagePath)).scheme in {'http', 'https', 'ftp', 's3'}: + return make_vsi(self.largeImagePath) + return str(self.largeImagePath) + def _checkNetCDF(self): return False diff --git a/test/test_source_gdal.py b/test/test_source_gdal.py index 6c188e712..c9692d295 100644 --- a/test/test_source_gdal.py +++ b/test/test_source_gdal.py @@ -532,3 +532,21 @@ def testMatplotlibPalette(): image = PIL.Image.open(io.BytesIO(image)) image = numpy.asarray(image) assert list(image[0, 0, :]) == [68, 1, 84, 0] + + +def testHttpVfsPath(): + imagePath = datastore.get_url('landcover_sample_1000.tif') + source = large_image_source_gdal.open( + imagePath, projection='EPSG:3857', encoding='PNG') + tileMetadata = source.getMetadata() + assert tileMetadata['tileWidth'] == 256 + assert tileMetadata['tileHeight'] == 256 + assert tileMetadata['sizeX'] == 65536 + assert tileMetadata['sizeY'] == 65536 + assert tileMetadata['levels'] == 9 + assert tileMetadata['bounds']['xmax'] == pytest.approx(-7837888, 1) + assert tileMetadata['bounds']['xmin'] == pytest.approx(-8909162, 1) + assert tileMetadata['bounds']['ymax'] == pytest.approx(5755717, 1) + assert tileMetadata['bounds']['ymin'] == pytest.approx(4876273, 1) + assert tileMetadata['bounds']['srs'] == 'epsg:3857' + assert tileMetadata['geospatial'] From 469a5b2c6881ebc4554bf8732b97018a3899c25a Mon Sep 17 00:00:00 2001 From: banesullivan Date: Thu, 7 Apr 2022 23:11:36 -0600 Subject: [PATCH 2/3] Fix strings --- sources/gdal/large_image_source_gdal/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sources/gdal/large_image_source_gdal/__init__.py b/sources/gdal/large_image_source_gdal/__init__.py index 0533b43b4..98fae3dff 100644 --- a/sources/gdal/large_image_source_gdal/__init__.py +++ b/sources/gdal/large_image_source_gdal/__init__.py @@ -70,17 +70,17 @@ def make_vsi(url: str, **options): - if str(url).startswith("s3://"): - s3_path = url.replace("s3://", "") - vsi = f"/vsis3/{s3_path}" + if str(url).startswith('s3://'): + s3_path = url.replace('s3://', '') + vsi = f'/vsis3/{s3_path}' else: gdal_options = { - "url": str(url), - "use_head": "no", - "list_dir": "no", + 'url': str(url), + 'use_head': 'no', + 'list_dir': 'no', } gdal_options.update(options) - vsi = f"/vsicurl?{urlencode(gdal_options)}" + vsi = f'/vsicurl?{urlencode(gdal_options)}' return vsi From aa87237c350f6b7f06e1641cb020914a9e0038d1 Mon Sep 17 00:00:00 2001 From: banesullivan Date: Thu, 7 Apr 2022 23:13:06 -0600 Subject: [PATCH 3/3] Fix docstring --- sources/gdal/large_image_source_gdal/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sources/gdal/large_image_source_gdal/__init__.py b/sources/gdal/large_image_source_gdal/__init__.py index 98fae3dff..263b95e50 100644 --- a/sources/gdal/large_image_source_gdal/__init__.py +++ b/sources/gdal/large_image_source_gdal/__init__.py @@ -178,8 +178,9 @@ def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs): def _getLargeImagePath(self): """Get GDAL-compatible image path. - This will cast the outout to a string and can also handle URLs for use - with GDAL VFS interface. + This will cast the output to a string and can also handle URLs + ('http', 'https', 'ftp', 's3') for use with GDAL + `Virtual Filesystems Interface `_. """ if urlparse(str(self.largeImagePath)).scheme in {'http', 'https', 'ftp', 's3'}: return make_vsi(self.largeImagePath)