From 729ae4f06abc6a00622d29007f77373db79be4e2 Mon Sep 17 00:00:00 2001 From: Peiti Li Date: Thu, 20 May 2021 13:13:00 -0700 Subject: [PATCH 1/5] python3.9 initial port --- batch-setup/batch.py | 10 +++++----- batch-setup/batch_setup.py | 6 +++--- batch-setup/cancel_all_jobs.py | 2 +- batch-setup/cron.py | 12 ++++++------ batch-setup/ecr.py | 2 +- batch-setup/make_meta_tiles.py | 22 +++++++++++----------- batch-setup/make_rawr_tiles.py | 18 +++++++++--------- batch-setup/teardown.py | 24 ++++++++++++------------ batch-setup/tileset.py | 8 ++++---- import/osm2pgsql.py | 14 +++++++------- utils/count_jobs.py | 4 ++-- utils/list-batch-logs.py | 4 ++-- utils/meta-high-job-times.py | 2 +- utils/test_tiles.py | 4 ++-- utils/tiles.py | 4 ++-- 15 files changed, 68 insertions(+), 68 deletions(-) diff --git a/batch-setup/batch.py b/batch-setup/batch.py index 5831322..67d4497 100644 --- a/batch-setup/batch.py +++ b/batch-setup/batch.py @@ -164,9 +164,9 @@ def s3_policy(bucket_or_buckets, date_prefix, allow_write=False): for bucket in buckets: object_resources.extend([ # allow access to objects under the date prefix - "arn:aws:s3:::%s/%s/*" % (bucket, date_prefix), + f"arn:aws:s3:::{bucket}/{date_prefix}/*", # and also objects under a hash + date prefix - "arn:aws:s3:::%s/*/%s/*" % (bucket, date_prefix), + f"arn:aws:s3:::{bucket}/*/{date_prefix}/*", ]) bucket_resources.append("arn:aws:s3:::%s" % (bucket)) @@ -234,7 +234,7 @@ def kebab_to_camel(name): def ensure_job_role_arn(iam, run_id, name, buckets, date_prefixes): role_name = kebab_to_camel( - "batch-%s-%s" % (name, run_id)) + f"batch-{name}-{run_id}") arn = None try: @@ -285,7 +285,7 @@ def create_role(iam, image_name, role_name, buckets, date_prefixes): Description='Role to perform %s batch jobs in %s/%s environment.' % (image_name, date_prefixes.rawr, date_prefixes.meta)) - class RolePolicies(object): + class RolePolicies: def __init__(self, iam, role_name): self.iam = iam self.role_name = role_name @@ -349,7 +349,7 @@ def make_job_definitions( retry_value = retry_attempts[name] \ if isinstance(retry_attempts, dict) else retry_attempts - job_name = "%s-%s" % (name, run_id) + job_name = f"{name}-{run_id}" definition = { 'name': job_name, 'job-role-arn': job_role_arn, diff --git a/batch-setup/batch_setup.py b/batch-setup/batch_setup.py index daa9556..9a55667 100644 --- a/batch-setup/batch_setup.py +++ b/batch-setup/batch_setup.py @@ -15,7 +15,7 @@ def get_or_create_role(boto_iam, role_name, role_document, role_path=None): RoleName=role_name, ) role_arn = response['Role']['Arn'] - print("Found role %s at %s" % (role_name, role_arn)) + print(f"Found role {role_name} at {role_arn}") except boto_iam.exceptions.NoSuchEntityException: response = boto_iam.create_role( Path=role_path, @@ -23,7 +23,7 @@ def get_or_create_role(boto_iam, role_name, role_document, role_path=None): AssumeRolePolicyDocument=json.dumps(role_document), ) role_arn = response['Role']['Arn'] - print("Created role %s at %s" % (role_name, role_arn)) + print(f"Created role {role_name} at {role_arn}") return role_arn @@ -93,7 +93,7 @@ def batch_setup(region_name, run_id, vpc_id, securityGroupIds, computeEnvironmen ] ) subnet_ids = [s['SubnetId'] for s in response['Subnets']] - print("Found subnets %s on VPC %s" % (subnet_ids, vpc_id)) + print(f"Found subnets {subnet_ids} on VPC {vpc_id}") # Create the batch service role # https://docs.aws.amazon.com/batch/latest/userguide/service_IAM_role.html diff --git a/batch-setup/cancel_all_jobs.py b/batch-setup/cancel_all_jobs.py index af2cc01..cf6bce3 100644 --- a/batch-setup/cancel_all_jobs.py +++ b/batch-setup/cancel_all_jobs.py @@ -9,5 +9,5 @@ "misconfiguration.") args = parser.parse_args() -job_queue = 'job-queue-%s' % (args.date,) +job_queue = f'job-queue-{args.date}' terminate_all_jobs(job_queue, args.reason) diff --git a/batch-setup/cron.py b/batch-setup/cron.py index ed1fa7a..8ff9570 100644 --- a/batch-setup/cron.py +++ b/batch-setup/cron.py @@ -561,7 +561,7 @@ def bucket_name(arg_name, bucket_function): # otherwise, default to a value constructed from the bucket prefix. if args.bucket_prefix: - return '%s-%s-%s' % (args.bucket_prefix, bucket_function, region) + return f'{args.bucket_prefix}-{bucket_function}-{region}' # finally, error out if we can't figure out a value. raise RuntimeError('Must provide either --bucket-prefix or %s.' @@ -597,7 +597,7 @@ def bucket_name(arg_name, bucket_function): smgr = boto3.client('secretsmanager') smgr_name = (args.db_password_secret_name or ('TilesDatabasePassword' + args.run_id)) - smgr_description = 'Tiles database password for %s import' % (args.run_id,) + smgr_description = f'Tiles database password for {args.run_id} import' db_password = generate_or_update_password( smgr, args.db_password, smgr_name, smgr_description) @@ -625,7 +625,7 @@ def bucket_name(arg_name, bucket_function): ) script_dir = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(script_dir, 'provision.sh'), 'r') as fh: + with open(os.path.join(script_dir, 'provision.sh')) as fh: provision = fh.read() % provision_params provision_base64 = b64encode(provision.encode('utf8')) @@ -669,8 +669,8 @@ def bucket_name(arg_name, bucket_function): instance = response['Instances'][0] instance_id = instance['InstanceId'] - print('reservation ID: %s' % (reservation_id,)) - print('instance ID: %s' % (instance_id,)) + print(f'reservation ID: {reservation_id}') + print(f'instance ID: {instance_id}') print('Waiting for instance to come up...') waiter = ec2.get_waiter('instance_status_ok') @@ -678,4 +678,4 @@ def bucket_name(arg_name, bucket_function): response = ec2.describe_instances(InstanceIds=[instance_id]) instance = response['Reservations'][0]['Instances'][0] - print('public IP: %r' % (instance.get('PublicIpAddress'),)) + print('public IP: {!r}'.format(instance.get('PublicIpAddress'))) diff --git a/batch-setup/ecr.py b/batch-setup/ecr.py index 546032e..9cd35e7 100644 --- a/batch-setup/ecr.py +++ b/batch-setup/ecr.py @@ -39,7 +39,7 @@ def ensure_ecr(run_id): repo_uris = {} for repo_name in repo_names: - full_name = 'tilezen/%s-%s' % (repo_name, run_id) + full_name = f'tilezen/{repo_name}-{run_id}' repo_uris[repo_name] = ensure_repo(ecr, full_name) return repo_uris diff --git a/batch-setup/make_meta_tiles.py b/batch-setup/make_meta_tiles.py index 22302f2..818ce97 100644 --- a/batch-setup/make_meta_tiles.py +++ b/batch-setup/make_meta_tiles.py @@ -27,7 +27,7 @@ MissingTiles = namedtuple('MissingTiles', 'low_zoom_file high_zoom_file') -class MissingTileFinder(object): +class MissingTileFinder: """ Finds tiles missing from an S3 bucket and provides convenience methods to navigate them. @@ -54,7 +54,7 @@ def __init__(self, missing_bucket, tile_bucket, src_date_prefix, assert self.region assert self.key_format_type is not None - with open(config, 'r') as fh: + with open(config) as fh: conf = yaml.load(fh.read()) self.job_queue_name = conf['batch']['job-queue'] self.job_definition = conf['batch']['job-definition'] @@ -132,8 +132,8 @@ def read_metas_to_file(self, filename, present=False, compress=False): '-bucket', self.missing_bucket, '-date-prefix', self.dst_date_prefix, '-region', self.region, - '-present=%r' % (bool(present),), - '-compress-output=%r' % (bool(compress),), + f'-present={bool(present)!r}', + f'-compress-output={bool(compress)!r}', '-max-zoom', str(self.max_zoom), stdout=filename) @@ -237,7 +237,7 @@ def present_tiles(self): shutil.rmtree(tmpdir) -class _JobSizer(object): +class _JobSizer: """ Individual instance of a callable which can evaluate the size of a job (i.e: grouped set of RAWR tiles). @@ -340,7 +340,7 @@ def update_memory_request(cfg, mem_multiplier, mem_max): # adaptor class for MissingTiles to see just the high zoom parts, this is used # along with the LowZoomLense to loop over missing tiles generically but # separately. -class HighZoomLense(object): +class HighZoomLense: def __init__(self, config): self.config = config self.description = "high zoom tiles" @@ -349,7 +349,7 @@ def missing_file(self, missing): return missing.high_zoom_file -class LowZoomLense(object): +class LowZoomLense: def __init__(self, config): self.config = config self.description = "low zoom tiles" @@ -361,7 +361,7 @@ def missing_file(self, missing): # abstracts away the logic for a re-rendering loop, splitting between high and # low zoom tiles and stopping if all the tiles aren't rendered within a # certain number of retries. -class TileRenderer(object): +class TileRenderer: def __init__(self, tile_finder, big_jobs, split_zoom, zoom_max, allowed_missing_tiles=0, tile_coords_generator=None): @@ -489,10 +489,10 @@ def render(self, num_retries, lense): generator = None if args.use_tile_coords_generator: bboxes = args.tile_coords_generator_bbox.split(',') - assert len(bboxes) == 4, 'Seed config: custom bbox {} does not have exactly four elements!'.format(bboxes) + assert len(bboxes) == 4, f'Seed config: custom bbox {bboxes} does not have exactly four elements!' min_x, min_y, max_x, max_y = list(map(float, bboxes)) - assert min_x < max_x, 'Invalid bbox. X: {} not less than {}'.format(min_x, max_x) - assert min_y < max_y, 'Invalid bbox. Y: {} not less than {}'.format(min_y, max_y) + assert min_x < max_x, f'Invalid bbox. X: {min_x} not less than {max_x}' + assert min_y < max_y, f'Invalid bbox. Y: {min_y} not less than {max_y}' generator = BoundingBoxTileCoordinatesGenerator(min_x=min_x, min_y=min_y, max_x=max_x, diff --git a/batch-setup/make_rawr_tiles.py b/batch-setup/make_rawr_tiles.py index 63adafd..3f86b6f 100644 --- a/batch-setup/make_rawr_tiles.py +++ b/batch-setup/make_rawr_tiles.py @@ -44,8 +44,8 @@ def missing_tiles(missing_bucket, rawr_bucket, date_prefix, region, key_format_type, config, zoom, tile_coords_generator): from make_meta_tiles import MissingTileFinder if bool(tile_coords_generator): - return set([c for c in tile_coords_generator. - generate_tiles_coordinates([zoom])]) + return {c for c in tile_coords_generator. + generate_tiles_coordinates([zoom])} else: present = set() finder = MissingTileFinder( @@ -79,7 +79,7 @@ def missing_jobs(missing_bucket, rawr_bucket, date_prefix, region, config, # the rawr tiles of `tile_zoom` is actually built by AWS batch jobs of # `job_zoom` so we need to do a zoomTo here to find the corresponding jobs - jobs = set(coord.zoomTo(job_zoom).container() for coord in tiles) + jobs = {coord.zoomTo(job_zoom).container() for coord in tiles} print("[make_rawr_tiles] Missing %d tiles (%d jobs)" % (len(tiles), len(jobs))) @@ -103,7 +103,7 @@ def wc_line(filename): line utility `wc -l`. """ - with open(filename, 'r') as fh: + with open(filename) as fh: count = sum(1 for _ in fh) return count @@ -116,7 +116,7 @@ def head_lines(filename, n_lines): sample = [] - with open(filename, 'r') as fh: + with open(filename) as fh: try: for _ in range(n_lines): sample.append(next(fh).strip()) @@ -176,7 +176,7 @@ def make_rawr_tiles(rawr_config_file, missing_config_file, missing_bucket, """ assert os.path.isfile(rawr_config_file), rawr_config_file - with open(rawr_config_file, 'r') as fh: + with open(rawr_config_file) as fh: config = yaml.load(fh.read()) job_zoom = config['batch']['queue-zoom'] logging_config = config['logging']['config'] @@ -259,10 +259,10 @@ def make_rawr_tiles(rawr_config_file, missing_config_file, missing_bucket, generator = None if args.use_tile_coords_generator: bboxes = args.tile_coords_generator_bbox.split(',') - assert len(bboxes) == 4, 'Seed config: custom bbox {} does not have exactly four elements!'.format(bboxes) + assert len(bboxes) == 4, f'Seed config: custom bbox {bboxes} does not have exactly four elements!' min_x, min_y, max_x, max_y = list(map(float, bboxes)) - assert min_x < max_x, 'Invalid bbox. X: {} not less than {}'.format(min_x, max_x) - assert min_y < max_y, 'Invalid bbox. Y: {} not less than {}'.format(min_y, max_y) + assert min_x < max_x, f'Invalid bbox. X: {min_x} not less than {max_x}' + assert min_y < max_y, f'Invalid bbox. Y: {min_y} not less than {max_y}' generator = BoundingBoxTileCoordinatesGenerator(min_x=min_x, min_y=min_y, max_x=max_x, diff --git a/batch-setup/teardown.py b/batch-setup/teardown.py index 60ac510..9e5f487 100644 --- a/batch-setup/teardown.py +++ b/batch-setup/teardown.py @@ -23,7 +23,7 @@ def delete_all_job_definitions(batch, run_id): response = batch.describe_job_definitions(nextToken=next_token) for def_name in job_definitions: - print("Deleting job definition %r" % (def_name,)) + print(f"Deleting job definition {def_name!r}") batch.deregister_job_definition(jobDefinition=def_name) @@ -41,7 +41,7 @@ def delete_job_queue(batch, job_queue, terminate): wait_for_jobs_to_finish(job_queue) # disable the job queue - print("Disabling job queue %r" % (job_queue,)) + print(f"Disabling job queue {job_queue!r}") batch.update_job_queue(jobQueue=job_queue, state='DISABLED') print("Waiting for job queue to disable.") @@ -52,7 +52,7 @@ def delete_job_queue(batch, job_queue, terminate): state = q['state'] status = q['status'] if status == 'UPDATING': - print("Queue %r is updating, waiting..." % (job_queue,)) + print(f"Queue {job_queue!r} is updating, waiting...") elif state == 'DISABLED': break @@ -65,7 +65,7 @@ def delete_job_queue(batch, job_queue, terminate): sleep(15) # delete the job queue - print("Deleting job queue %r" % (job_queue,)) + print(f"Deleting job queue {job_queue!r}") batch.delete_job_queue(jobQueue=job_queue) # wait for queue to be deleted, otherwise it causes issues when we try to @@ -98,7 +98,7 @@ def terminate_instances_by_tag(tags): for instance in reservation['Instances']: instance_id = instance['InstanceId'] if instance['State']['Name'] == 'running': - print("Terminating instance %r" % (instance_id,)) + print(f"Terminating instance {instance_id!r}") ec2.terminate_instances(InstanceIds=[instance_id]) @@ -121,14 +121,14 @@ def delete_role(name): InstanceProfileName=ipname, RoleName=role['RoleName'], ) - print("Deleting instance profile %r" % (ipname,)) + print(f"Deleting instance profile {ipname!r}") iam.delete_instance_profile(InstanceProfileName=ipname) # next, detach all policies paginator = iam.get_paginator('list_attached_role_policies') for page in paginator.paginate(RoleName=name): for policy in page['AttachedPolicies']: - print("Detaching role policy %r" % (policy['PolicyName'],)) + print("Detaching role policy {!r}".format(policy['PolicyName'])) iam.detach_role_policy( RoleName=name, PolicyArn=policy['PolicyArn'], @@ -146,7 +146,7 @@ def delete_role(name): ) # finally, delete the role - print("Deleting role %r" % (name,)) + print(f"Deleting role {name!r}") iam.delete_role(RoleName=name) @@ -157,13 +157,13 @@ def delete_policy(name): policy = find_policy(iam, name) if policy: - print("Deleting policy %r (arn=%r)" % (name, policy['Arn'])) + print("Deleting policy {!r} (arn={!r})".format(name, policy['Arn'])) iam.delete_policy(PolicyArn=policy['Arn']) def delete_compute_env(batch, compute_env): # disable the compute environment - print("Disabling compute environment %r" % (compute_env,)) + print(f"Disabling compute environment {compute_env!r}") batch.update_compute_environment( computeEnvironment=compute_env, state='DISABLED') @@ -176,7 +176,7 @@ def delete_compute_env(batch, compute_env): state = env['state'] status = env['status'] if status == 'UPDATING': - print("Environment %r is updating, waiting..." % (compute_env,)) + print(f"Environment {compute_env!r} is updating, waiting...") elif state == 'DISABLED': break @@ -189,7 +189,7 @@ def delete_compute_env(batch, compute_env): sleep(15) # delete the compute environment - print("Deleting compute environment %r" % (compute_env,)) + print(f"Deleting compute environment {compute_env!r}") batch.delete_compute_environment(computeEnvironment=compute_env) diff --git a/batch-setup/tileset.py b/batch-setup/tileset.py index 43678a6..905e68c 100644 --- a/batch-setup/tileset.py +++ b/batch-setup/tileset.py @@ -2,7 +2,7 @@ from itertools import chain -class BitSet(object): +class BitSet: """ A single-zoom set of Coordinates or, equivalently, a mapping from Coordinates to booleans. @@ -64,7 +64,7 @@ def __iter__(self): return BitSetIterator(self) -class BitSetIterator(object): +class BitSetIterator: """ Helper class to iterate over the True bits in a BitSet, yielding their Coordinates. @@ -95,7 +95,7 @@ def next(self): return coord -class CoordSet(object): +class CoordSet: """ A set of Coordinates, implemented as a mapping of Coordinate to True/False. @@ -160,7 +160,7 @@ def __iter__(self): assert list(c) == [] c[zoom0_coord] = True c[zoom10_coord] = True - assert set(c) == set([zoom0_coord, zoom10_coord]) + assert set(c) == {zoom0_coord, zoom10_coord} c = CoordSet(10, min_zoom=7) c[Coordinate(zoom=11, column=3, row=1)] = True diff --git a/import/osm2pgsql.py b/import/osm2pgsql.py index 492ff13..03e637c 100644 --- a/import/osm2pgsql.py +++ b/import/osm2pgsql.py @@ -7,7 +7,7 @@ def flat_nodes_key(run_id): - return 'flat-nodes-%s/flat.nodes' % (run_id,) + return f'flat-nodes-{run_id}/flat.nodes' def does_flat_nodes_file_exist(bucket, run_id): @@ -62,7 +62,7 @@ def reset_database(instance, db): def login_key(run_id): - filename = "import-private-key-%s.pem" % (run_id,) + filename = f"import-private-key-{run_id}.pem" if not os.path.exists(filename): return None @@ -82,7 +82,7 @@ def create_login_key(ec2, run_id, key_pair_name): pem = response['KeyMaterial'] key = RSAKey.from_private_key(BytesIO(pem)) - filename = "import-private-key-%s.pem" % (run_id,) + filename = f"import-private-key-{run_id}.pem" with open(filename, 'w') as fh: key.write_private_key(fh) @@ -268,7 +268,7 @@ def start_osm2pgsql_instance( return instance_id -class Instance(object): +class Instance: """ Represents an EC2 instance that we can run scripts on. """ @@ -310,7 +310,7 @@ def _ssh(self, script_file, parameters): yield ssh except Exception as e: - raise RuntimeError("ERROR: %s (on ubuntu@%s)" % (e, ip)) + raise RuntimeError(f"ERROR: {e} (on ubuntu@{ip})") finally: ssh.close() @@ -388,7 +388,7 @@ def repeatedly(self, script_file, **kwargs): stderr.read() status = stdout.read().rstrip().decode('utf8') - print("[%s] Import status: %r" % (time_now, status)) + print(f"[{time_now}] Import status: {status!r}") if status == "finished": break elif status == "failed": @@ -415,7 +415,7 @@ def shutdown_and_cleanup(ec2, import_instance_id, run_id, ip_addr): waiter.wait(InstanceIds=[import_instance_id]) print("Instance terminated.") - filename = "import-private-key-%s.pem" % (run_id,) + filename = f"import-private-key-{run_id}.pem" os.remove(filename) key_pair_name = 'osm2pgsql-import-' + run_id ec2.delete_key_pair(KeyName=key_pair_name) diff --git a/utils/count_jobs.py b/utils/count_jobs.py index ad5c0c8..9f321c9 100644 --- a/utils/count_jobs.py +++ b/utils/count_jobs.py @@ -35,7 +35,7 @@ def count_jobs(batch, job_queue): def log_msg(msg): now = datetime.datetime.now() - print("[%s] %s" % (now.strftime("%Y-%m-%d %H:%M:%S"), msg)) + print("[{}] {}".format(now.strftime("%Y-%m-%d %H:%M:%S"), msg)) parser = argparse.ArgumentParser(description=""" @@ -47,7 +47,7 @@ def log_msg(msg): "seconds between updates.") args = parser.parse_args() -job_queue = 'job-queue-%s' % (args.date,) +job_queue = f'job-queue-{args.date}' batch = boto3.client('batch') interval = args.interval diff --git a/utils/list-batch-logs.py b/utils/list-batch-logs.py index 49a0d1b..d6c7736 100644 --- a/utils/list-batch-logs.py +++ b/utils/list-batch-logs.py @@ -41,7 +41,7 @@ def print_failures(cwlogs, log_stream_name): coord = msg['coord'] coord_str = "%d/%d/%d" % (coord['z'], coord['x'], coord['y']) - print("%15s: %s" % (coord_str, msg['exception'])) + print("{:>15}: {}".format(coord_str, msg['exception'])) trace = msg.get('stacktrace') if trace: print(" " + trace.replace("|", "\n ")) @@ -81,5 +81,5 @@ def print_failures(cwlogs, log_stream_name): name = job['jobName'] last_attempt = job['attempts'][-1] log_stream_name = last_attempt['container']['logStreamName'] - print("=== %s ===\n" % (name,)) + print(f"=== {name} ===\n") print_failures(cwlogs, log_stream_name) diff --git a/utils/meta-high-job-times.py b/utils/meta-high-job-times.py index f8de36f..41f52bc 100644 --- a/utils/meta-high-job-times.py +++ b/utils/meta-high-job-times.py @@ -11,7 +11,7 @@ args = parser.parse_args() job_queue = 'job-queue-' + args.date -prefix = 'meta-batch-%s-' % (args.date,) +prefix = f'meta-batch-{args.date}-' batch = boto3.client('batch') next_token = None diff --git a/utils/test_tiles.py b/utils/test_tiles.py index 5e2c05f..8fc358f 100644 --- a/utils/test_tiles.py +++ b/utils/test_tiles.py @@ -40,8 +40,8 @@ def test_tiles_within_bbox(): 48.760348) res = generator4.generate_tiles_coordinates([10]) - zoomedCoords = set(coord.zoomTo(7).container() for coord in res) + zoomedCoords = {coord.zoomTo(7).container() for coord in res} - assert set([Coordinate(44, 20, 7), Coordinate(44, 21, 7), Coordinate(45, 20, 7), Coordinate(45, 21, 7)]) == \ + assert {Coordinate(44, 20, 7), Coordinate(44, 21, 7), Coordinate(45, 20, 7), Coordinate(45, 21, 7)} == \ zoomedCoords diff --git a/utils/tiles.py b/utils/tiles.py index 4847218..f8fc249 100644 --- a/utils/tiles.py +++ b/utils/tiles.py @@ -5,7 +5,7 @@ from utils.constants import MIN_TILE_ZOOM -class TileCoordinatesGenerator(object): +class TileCoordinatesGenerator: """ Generate tiles EPSG:3857 coordinates from a list of zooms within a range """ @@ -56,7 +56,7 @@ def __init__(self, min_x, min_y, max_x, max_y, min_zoom=MIN_TILE_ZOOM, :param min_zoom: the minimum zoom(inclusive) it can generate tiles for :param max_zoom: the maximum zoom(inclusive) it can generate tiles for """ - super(BoundingBoxTileCoordinatesGenerator, self).__init__(min_zoom=min_zoom, max_zoom=max_zoom) + super().__init__(min_zoom=min_zoom, max_zoom=max_zoom) self.min_x = min_x self.min_y = min_y self.max_x = max_x From 17312bb16d46660482889687bcb12e8091c0977f Mon Sep 17 00:00:00 2001 From: Peiti Li Date: Mon, 24 May 2021 15:39:57 -0700 Subject: [PATCH 2/5] upgrade dependencies --- batch-setup/provision.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/batch-setup/provision.sh b/batch-setup/provision.sh index 08cd021..27fe3e9 100644 --- a/batch-setup/provision.sh +++ b/batch-setup/provision.sh @@ -28,11 +28,11 @@ done # a per-branch basis without clobbering the (singleton) file on S3. # cat >/usr/local/etc/py-requirements.txt < Date: Mon, 24 May 2021 16:38:43 -0700 Subject: [PATCH 3/5] add util files for reference --- docker_meta_batch_requirements.txt | 29 ++++++++++++++++++++ docker_meta_low_zoom_batch_requirements.txt | 29 ++++++++++++++++++++ docker_rawr_batch_requirements.txt | 28 +++++++++++++++++++ import_requirements.txt | 3 +++ provision_requirements.txt | 30 +++++++++++++++++++++ 5 files changed, 119 insertions(+) create mode 100644 docker_meta_batch_requirements.txt create mode 100644 docker_meta_low_zoom_batch_requirements.txt create mode 100644 docker_rawr_batch_requirements.txt create mode 100644 import_requirements.txt create mode 100644 provision_requirements.txt diff --git a/docker_meta_batch_requirements.txt b/docker_meta_batch_requirements.txt new file mode 100644 index 0000000..f04c256 --- /dev/null +++ b/docker_meta_batch_requirements.txt @@ -0,0 +1,29 @@ +appdirs==1.4.3 +argparse==1.4.0 +boto==2.48.0 +enum34==1.1.6 +future==0.16.0 +hiredis==0.2.0 +Jinja2==3.0.1 +MarkupSafe==2.0.1 +ModestMaps==1.4.7 +protobuf==3.4.0 +psycopg2==2.8.6 +pyclipper==1.2.1 +pycountry==17.9.23 +pyproj==3.0.1 +python-dateutil==2.6.1 +PyYAML==4.2b4 +redis==2.10.6 +requests==2.20.1 +Shapely==1.7.1 +six==1.11.0 +StreetNames==0.1.5 +statsd==3.2.1 +ujson==1.35 +Werkzeug==0.12.2 +zope.dottedname==4.2 +edtf==2.6.0 +mapbox-vector-tile==1.2.1 +boto3==1.7.10 +#git+https://github.com/tilezen/coanacatl@v0.5.0#egg=coanacatl \ No newline at end of file diff --git a/docker_meta_low_zoom_batch_requirements.txt b/docker_meta_low_zoom_batch_requirements.txt new file mode 100644 index 0000000..f04c256 --- /dev/null +++ b/docker_meta_low_zoom_batch_requirements.txt @@ -0,0 +1,29 @@ +appdirs==1.4.3 +argparse==1.4.0 +boto==2.48.0 +enum34==1.1.6 +future==0.16.0 +hiredis==0.2.0 +Jinja2==3.0.1 +MarkupSafe==2.0.1 +ModestMaps==1.4.7 +protobuf==3.4.0 +psycopg2==2.8.6 +pyclipper==1.2.1 +pycountry==17.9.23 +pyproj==3.0.1 +python-dateutil==2.6.1 +PyYAML==4.2b4 +redis==2.10.6 +requests==2.20.1 +Shapely==1.7.1 +six==1.11.0 +StreetNames==0.1.5 +statsd==3.2.1 +ujson==1.35 +Werkzeug==0.12.2 +zope.dottedname==4.2 +edtf==2.6.0 +mapbox-vector-tile==1.2.1 +boto3==1.7.10 +#git+https://github.com/tilezen/coanacatl@v0.5.0#egg=coanacatl \ No newline at end of file diff --git a/docker_rawr_batch_requirements.txt b/docker_rawr_batch_requirements.txt new file mode 100644 index 0000000..c33b0c8 --- /dev/null +++ b/docker_rawr_batch_requirements.txt @@ -0,0 +1,28 @@ +appdirs==1.4.3 +argparse==1.4.0 +boto==2.48.0 +enum34==1.1.6 +future==0.16.0 +hiredis==0.2.0 +Jinja2==3.0.1 +MarkupSafe==2.0.1 +ModestMaps==1.4.7 +protobuf==3.4.0 +psycopg2==2.8.6 +pyclipper==1.2.1 +pycountry==17.9.23 +pyproj==3.0.1 +python-dateutil==2.6.1 +PyYAML==4.2b4 +redis==2.10.6 +requests==2.20.1 +Shapely==1.7.1 +six==1.11.0 +StreetNames==0.1.5 +statsd==3.2.1 +ujson==1.35 +Werkzeug==0.12.2 +zope.dottedname==4.2 +edtf==2.6.0 +mapbox-vector-tile==1.2.1 +boto3==1.7.10 diff --git a/import_requirements.txt b/import_requirements.txt new file mode 100644 index 0000000..35ce4b3 --- /dev/null +++ b/import_requirements.txt @@ -0,0 +1,3 @@ +boto3==1.7.10 +paramiko==2.4.2 +requests==2.20.1 \ No newline at end of file diff --git a/provision_requirements.txt b/provision_requirements.txt new file mode 100644 index 0000000..21b6b8a --- /dev/null +++ b/provision_requirements.txt @@ -0,0 +1,30 @@ +Jinja2==3.0.1 +MarkupSafe==2.0.1 +ModestMaps==1.4.7 +PyYAML==4.2b4 +Shapely==1.7.1 +StreetNames==0.1.5 +Werkzeug==0.12.2 +appdirs==1.4.3 +argparse==1.4.0 +boto3==1.9.32 +boto==2.48.0 +edtf==2.6.0 +enum34==1.1.6 +future==0.16.0 +hiredis==0.2.0 +mapbox-vector-tile==1.2.1 +paramiko==2.4.2 +protobuf==3.4.0 +psycopg2==2.8.6 +pyclipper==1.2.1 +pycountry==17.9.23 +pyproj==3.0.1 +python-dateutil==2.6.1 +redis==2.10.6 +requests==2.20.1 +six==1.11.0 +statsd==3.2.1 +ujson==1.35 +zope.dottedname==4.2 +mercantile==1.2.1 \ No newline at end of file From 1e8a53acc69d0ca751f97cba63c6b14832ce346f Mon Sep 17 00:00:00 2001 From: Peiti Li Date: Mon, 24 May 2021 17:27:03 -0700 Subject: [PATCH 4/5] more update --- docker/meta-batch/Dockerfile | 2 +- docker/meta-batch/requirements.txt | 15 +++++++-------- docker/meta-low-zoom-batch/Dockerfile | 2 +- docker/meta-low-zoom-batch/requirements.txt | 15 +++++++-------- docker/rawr-batch/Dockerfile | 2 +- docker/rawr-batch/requirements.txt | 15 +++++++-------- docker_meta_batch_requirements.txt | 3 ++- 7 files changed, 26 insertions(+), 28 deletions(-) diff --git a/docker/meta-batch/Dockerfile b/docker/meta-batch/Dockerfile index 1bc1715..02c6fa0 100644 --- a/docker/meta-batch/Dockerfile +++ b/docker/meta-batch/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2 +FROM python:3.9.5 RUN apt-get -y update \ && apt-get -y install libgeos-dev libboost-python-dev time \ diff --git a/docker/meta-batch/requirements.txt b/docker/meta-batch/requirements.txt index 4063135..2bad04d 100644 --- a/docker/meta-batch/requirements.txt +++ b/docker/meta-batch/requirements.txt @@ -4,27 +4,26 @@ boto==2.48.0 enum34==1.1.6 future==0.16.0 hiredis==0.2.0 -Jinja2==2.10.1 -MarkupSafe==1.0 +Jinja2==3.0.1 +MarkupSafe==2.0.1 ModestMaps==1.4.7 protobuf==3.4.0 -psycopg2==2.7.3.2 -pyclipper==1.0.6 +psycopg2==2.8.6 +pyclipper==1.2.1 pycountry==17.9.23 -pyproj==2.1.0 +pyproj==3.0.1 python-dateutil==2.6.1 PyYAML==4.2b4 redis==2.10.6 requests==2.20.1 -Shapely==1.6.2.post1 +Shapely==1.7.1 six==1.11.0 StreetNames==0.1.5 statsd==3.2.1 ujson==1.35 Werkzeug==0.12.2 -wsgiref==0.1.2 zope.dottedname==4.2 edtf==2.6.0 -mapbox-vector-tile==1.2.0 +mapbox-vector-tile==1.2.1 boto3==1.7.10 git+https://github.com/tilezen/coanacatl@v0.5.0#egg=coanacatl diff --git a/docker/meta-low-zoom-batch/Dockerfile b/docker/meta-low-zoom-batch/Dockerfile index 1c8544e..1bc02a9 100644 --- a/docker/meta-low-zoom-batch/Dockerfile +++ b/docker/meta-low-zoom-batch/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2 +FROM python:3.9.5 RUN apt-get -y update \ && apt-get -y install libgeos-dev libboost-python-dev time \ diff --git a/docker/meta-low-zoom-batch/requirements.txt b/docker/meta-low-zoom-batch/requirements.txt index 4063135..2bad04d 100644 --- a/docker/meta-low-zoom-batch/requirements.txt +++ b/docker/meta-low-zoom-batch/requirements.txt @@ -4,27 +4,26 @@ boto==2.48.0 enum34==1.1.6 future==0.16.0 hiredis==0.2.0 -Jinja2==2.10.1 -MarkupSafe==1.0 +Jinja2==3.0.1 +MarkupSafe==2.0.1 ModestMaps==1.4.7 protobuf==3.4.0 -psycopg2==2.7.3.2 -pyclipper==1.0.6 +psycopg2==2.8.6 +pyclipper==1.2.1 pycountry==17.9.23 -pyproj==2.1.0 +pyproj==3.0.1 python-dateutil==2.6.1 PyYAML==4.2b4 redis==2.10.6 requests==2.20.1 -Shapely==1.6.2.post1 +Shapely==1.7.1 six==1.11.0 StreetNames==0.1.5 statsd==3.2.1 ujson==1.35 Werkzeug==0.12.2 -wsgiref==0.1.2 zope.dottedname==4.2 edtf==2.6.0 -mapbox-vector-tile==1.2.0 +mapbox-vector-tile==1.2.1 boto3==1.7.10 git+https://github.com/tilezen/coanacatl@v0.5.0#egg=coanacatl diff --git a/docker/rawr-batch/Dockerfile b/docker/rawr-batch/Dockerfile index d234ead..b13f50f 100644 --- a/docker/rawr-batch/Dockerfile +++ b/docker/rawr-batch/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2 +FROM python:3.9.5 RUN apt-get -y update \ && apt-get -y install libgeos-dev time \ diff --git a/docker/rawr-batch/requirements.txt b/docker/rawr-batch/requirements.txt index 86e18f1..c33b0c8 100644 --- a/docker/rawr-batch/requirements.txt +++ b/docker/rawr-batch/requirements.txt @@ -4,26 +4,25 @@ boto==2.48.0 enum34==1.1.6 future==0.16.0 hiredis==0.2.0 -Jinja2==2.10.1 -MarkupSafe==1.0 +Jinja2==3.0.1 +MarkupSafe==2.0.1 ModestMaps==1.4.7 protobuf==3.4.0 -psycopg2==2.7.3.2 -pyclipper==1.0.6 +psycopg2==2.8.6 +pyclipper==1.2.1 pycountry==17.9.23 -pyproj==2.1.0 +pyproj==3.0.1 python-dateutil==2.6.1 PyYAML==4.2b4 redis==2.10.6 requests==2.20.1 -Shapely==1.6.2.post1 +Shapely==1.7.1 six==1.11.0 StreetNames==0.1.5 statsd==3.2.1 ujson==1.35 Werkzeug==0.12.2 -wsgiref==0.1.2 zope.dottedname==4.2 edtf==2.6.0 -mapbox-vector-tile==1.2.0 +mapbox-vector-tile==1.2.1 boto3==1.7.10 diff --git a/docker_meta_batch_requirements.txt b/docker_meta_batch_requirements.txt index f04c256..802afca 100644 --- a/docker_meta_batch_requirements.txt +++ b/docker_meta_batch_requirements.txt @@ -26,4 +26,5 @@ zope.dottedname==4.2 edtf==2.6.0 mapbox-vector-tile==1.2.1 boto3==1.7.10 -#git+https://github.com/tilezen/coanacatl@v0.5.0#egg=coanacatl \ No newline at end of file +#git+https://github.com/tilezen/coanacatl@v0.5.0#egg=coanacatl +peitilipeitili \ No newline at end of file From e842d02290c9cf1ad7b6aa786237c66ede79c6a2 Mon Sep 17 00:00:00 2001 From: Peiti Li Date: Mon, 24 May 2021 17:44:06 -0700 Subject: [PATCH 5/5] upgrade more --- batch-setup/provision.sh | 4 ++-- docker/meta-batch/requirements.txt | 2 +- docker/meta-low-zoom-batch/requirements.txt | 2 +- docker/rawr-batch/requirements.txt | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/batch-setup/provision.sh b/batch-setup/provision.sh index 27fe3e9..bf8bdd2 100644 --- a/batch-setup/provision.sh +++ b/batch-setup/provision.sh @@ -38,7 +38,7 @@ Werkzeug==0.12.2 appdirs==1.4.3 argparse==1.4.0 boto3==1.9.32 -boto==2.48.0 +boto==2.49.0 edtf==2.6.0 enum34==1.1.6 future==0.16.0 @@ -52,7 +52,7 @@ pycountry==17.9.23 pyproj==3.0.1 python-dateutil==2.6.1 redis==2.10.6 -requests==2.20.1 +requests==2.25.1 six==1.11.0 statsd==3.2.1 ujson==1.35 diff --git a/docker/meta-batch/requirements.txt b/docker/meta-batch/requirements.txt index 2bad04d..d96506f 100644 --- a/docker/meta-batch/requirements.txt +++ b/docker/meta-batch/requirements.txt @@ -1,6 +1,6 @@ appdirs==1.4.3 argparse==1.4.0 -boto==2.48.0 +boto==2.49.0 enum34==1.1.6 future==0.16.0 hiredis==0.2.0 diff --git a/docker/meta-low-zoom-batch/requirements.txt b/docker/meta-low-zoom-batch/requirements.txt index 2bad04d..d96506f 100644 --- a/docker/meta-low-zoom-batch/requirements.txt +++ b/docker/meta-low-zoom-batch/requirements.txt @@ -1,6 +1,6 @@ appdirs==1.4.3 argparse==1.4.0 -boto==2.48.0 +boto==2.49.0 enum34==1.1.6 future==0.16.0 hiredis==0.2.0 diff --git a/docker/rawr-batch/requirements.txt b/docker/rawr-batch/requirements.txt index c33b0c8..f30ed13 100644 --- a/docker/rawr-batch/requirements.txt +++ b/docker/rawr-batch/requirements.txt @@ -1,6 +1,6 @@ appdirs==1.4.3 argparse==1.4.0 -boto==2.48.0 +boto==2.49.0 enum34==1.1.6 future==0.16.0 hiredis==0.2.0