From 362a4af1ce27136adbe5ba8017e12fe035deccda Mon Sep 17 00:00:00 2001 From: Kanahiro Date: Sun, 27 Nov 2022 14:32:39 +0900 Subject: [PATCH 1/2] use generator efficiently --- tileget/__main__.py | 196 ++++++++++++++++++++++---------------------- 1 file changed, 97 insertions(+), 99 deletions(-) diff --git a/tileget/__main__.py b/tileget/__main__.py index d7f4407..e40fdb8 100644 --- a/tileget/__main__.py +++ b/tileget/__main__.py @@ -1,7 +1,6 @@ import os import argparse import math -import itertools import time import urllib.request import json @@ -12,26 +11,34 @@ def get_args(): - parser = argparse.ArgumentParser(description='xyz-tile download tool') - parser.add_argument('tileurl', - help=r'xyz-tile url in {z}/{x}/{y} template') - parser.add_argument('output_dir', help='output dir') - parser.add_argument('--extent', - help='min_lon min_lat max_lon max_lat, whitespace delimited', - nargs=4) - parser.add_argument('--geojson', - help="path to geojson which is Feature or FeatureCollection with geometry in EPSG:3857") - parser.add_argument('--minzoom', default="0", help="default to 0") - parser.add_argument('--maxzoom', default="16", help="default to 16") - parser.add_argument('--interval', default="500", - help="time taken after each-request, set as miliseconds in interger, default to 500") - parser.add_argument('--overwrite', - help='overwrite existing files', - action='store_true') - parser.add_argument('--timeout', default="5", - help="wait response until this value, set as seconds in integer, default to 5") - parser.add_argument('--parallel', default="1", - help='num of parallel requests') + parser = argparse.ArgumentParser(description="xyz-tile download tool") + parser.add_argument("tileurl", help=r"xyz-tile url in {z}/{x}/{y} template") + parser.add_argument("output_dir", help="output dir") + parser.add_argument( + "--extent", + help="min_lon min_lat max_lon max_lat, whitespace delimited", + nargs=4, + ) + parser.add_argument( + "--geojson", + help="path to geojson which is Feature or FeatureCollection with geometry in EPSG:3857", + ) + parser.add_argument("--minzoom", default="0", help="default to 0") + parser.add_argument("--maxzoom", default="16", help="default to 16") + parser.add_argument( + "--interval", + default="500", + help="time taken after each-request, set as miliseconds in interger, default to 500", + ) + parser.add_argument( + "--overwrite", help="overwrite existing files", action="store_true" + ) + parser.add_argument( + "--timeout", + default="5", + help="wait response until this value, set as seconds in integer, default to 5", + ) + parser.add_argument("--parallel", default="1", help="num of parallel requests") args = parser.parse_args() verified_args = { @@ -44,7 +51,7 @@ def get_args(): "interval": int(args.interval), "overwrite": args.overwrite, "timeout": int(args.timeout), - "parallel": int(args.parallel) + "parallel": int(args.parallel), } if args.extent is None and args.geojson is None: @@ -60,120 +67,111 @@ def get_args(): def lonlat_to_webmercator(lonlat: list): - return (lonlat[0] * 20037508.34 / 180, math.log(math.tan((90 + lonlat[1]) * math.pi / 360)) / (math.pi / 180) * 20037508.34 / 180) + return ( + lonlat[0] * 20037508.34 / 180, + math.log(math.tan((90 + lonlat[1]) * math.pi / 360)) + / (math.pi / 180) + * 20037508.34 + / 180, + ) + + +def get_geometry_as_3857(extent: list) -> dict: + """ + returns GeoJSON Polygon geometry dict + extent must be latitudes and longitudes and reprojected to WebMercator EPSG:3857 + + Args: + extent (list): [min_lon, min_lat, max_lon, max_lat] + + Returns: + dict: Polygon geometry + """ + return { + "type": "Polygon", + "coordinates": ( + tuple( + map( + lonlat_to_webmercator, + ( + (extent[0], extent[1]), + (extent[2], extent[1]), + (extent[2], extent[3]), + (extent[0], extent[3]), + (extent[0], extent[1]), + ), + ) + ), + ), + } def main(): args = get_args() if args["extent"] is not None: - geometries = (get_geometry_as_3857(args["extent"]),) + geometry = shapely.geometry.shape(get_geometry_as_3857(args["extent"])) elif args["geojson"] is not None: - with open(args["geojson"], mode='r') as f: + with open(args["geojson"], mode="r") as f: geojson = json.load(f) if geojson.get("features") is None: - geometries = (geojson["geometry"],) + geometry = shapely.geometry.shape(geojson["geometry"]) else: - geometries = tuple( - map(lambda feature: feature["geometry"], geojson["features"])) - - all_tiles = [] - for geom in geometries: - all_tiles += tuple(itertools.chain.from_iterable((get_tiles_generator( - geom, zoom) for zoom in range(args["minzoom"], args["maxzoom"] + 1)))) - all_tiles = tuple(set(all_tiles)) + geometries = [ + shapely.geometry.shape(g) + for g in list(map(lambda f: f["geometry"], geojson["features"])) + ] + geometry = shapely.ops.unary_union(geometries) - def download(tile, idx): + def download(tile): ext = args["tileurl"].split(".")[-1] - write_dir = os.path.join( - args["output_dir"], str(tile[2]), str(tile[0])) + write_dir = os.path.join(args["output_dir"], str(tile[2]), str(tile[0])) write_filepath = os.path.join(write_dir, str(tile[1]) + "." + ext) if os.path.exists(write_filepath) and args["overwrite"] == False: return - os.makedirs(write_dir, exist_ok=True) - - url = args["tileurl"].replace( - r"{x}", str(tile[0])).replace( - r"{y}", str(tile[1])).replace( - r"{z}", str(tile[2])) + url = ( + args["tileurl"] + .replace(r"{x}", str(tile[0])) + .replace(r"{y}", str(tile[1])) + .replace(r"{z}", str(tile[2])) + ) data = None - while(True): + while True: try: data = urllib.request.urlopen(url, timeout=args["timeout"]) break except urllib.error.HTTPError as e: raise Exception(str(e) + ":" + url) except Exception as e: - if str(e.args) == "(timeout('_ssl.c:1091: The handshake operation timed out'),)": + if ( + str(e.args) + == "(timeout('_ssl.c:1091: The handshake operation timed out'),)" + ): print("timeout, retrying... :" + url) else: raise Exception(str(e) + ":" + url) if data is not None: - with open(write_filepath, mode='wb') as f: + os.makedirs(write_dir, exist_ok=True) + with open(write_filepath, mode="wb") as f: f.write(data.read()) time.sleep(args["interval"] / 1000) - if idx % 10 == 0: - print(str(idx + 1) + "/" + str(len(all_tiles))) - with ThreadPoolExecutor(max_workers=args["parallel"]) as executor: - futures = [executor.submit(download, tile, idx) - for idx, tile in enumerate(all_tiles)] - - results = [future.exception() for future in futures] - errors = list(map(str, filter(lambda val: val is not None, results))) - if len(errors) == 0: - print("no error occurs.") - else: - print("\n".join(errors)) - print(str(len(errors)) + " errors occured.") + for zoom in range(args["minzoom"], args["maxzoom"] + 1): + generator = tiletanic.tilecover.cover_geometry( + tiletanic.tileschemes.WebMercator(), geometry, zoom + ) + for tile in generator: + future = executor.submit(download, tile) + if future.exception() is not None: + print(future.exception()) print("finished") -def get_tiles_generator(geometry: dict, zoomlevel: int): - """ - returns a generator to yield tile-indices covering the geometry at the zoomlevel - - Args: - geometry (dict): GeoJSON Polygon geometry - zoomlevel (int) - - Returns: - Generator: yields tile-index [x, y, z] - """ - tilesceme = tiletanic.tileschemes.WebMercator() - feature_shape = shapely.geometry.shape(geometry) - generator = tiletanic.tilecover.cover_geometry( - tilesceme, feature_shape, zoomlevel) - return generator - - -def get_geometry_as_3857(extent: list) -> dict: - """ - returns GeoJSON Polygon geometry dict - extent must be latitudes and longitudes and reprojected to WebMercator EPSG:3857 - - Args: - extent (list): [min_lon, min_lat, max_lon, max_lat] - - Returns: - dict: Polygon geometry - """ - return { - "type": "Polygon", - "coordinates": (tuple(map(lonlat_to_webmercator, ( - (extent[0], extent[1]), - (extent[2], extent[1]), - (extent[2], extent[3]), - (extent[0], extent[3]), - (extent[0], extent[1]),))),) - } - - if __name__ == "__main__": main() From 76c7a09cf352a554cb42830d70a555732f159f04 Mon Sep 17 00:00:00 2001 From: Kanahiro Date: Sun, 27 Nov 2022 14:33:07 +0900 Subject: [PATCH 2/2] 0.1.0 --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9f541e4..d0fd77d 100644 --- a/setup.py +++ b/setup.py @@ -7,16 +7,16 @@ def _requires_from_file(filename): setup( name="tileget", - version="0.0.2", + version="0.1.0", description="Tile download utility - easily download xyz-tile data", author="Kanahiro Iguchi", license="MIT", url="https://github.com/Kanahiro/tileget", packages=find_packages(), - install_requires=_requires_from_file('requirements.txt'), + install_requires=_requires_from_file("requirements.txt"), entry_points={ "console_scripts": [ "tileget=tileget.__main__:main", ] - } + }, )