diff --git a/signing-manifests/bug1774221-3.yml b/signing-manifests/bug1774221-3.yml index 16e3e3d..d412f8f 100644 --- a/signing-manifests/bug1774221-3.yml +++ b/signing-manifests/bug1774221-3.yml @@ -1,4 +1,5 @@ --- +# comment to rerun dep signing bug: 1774221 sha256: b36e091ad9c4b3cd315b6bee0110c706b8f54a7b2bdf9ca292c6431e19fdb6c3 filesize: 395414 diff --git a/taskcluster/adhoc_taskgraph/routes.py b/taskcluster/adhoc_taskgraph/routes.py index a542712..30cef32 100644 --- a/taskcluster/adhoc_taskgraph/routes.py +++ b/taskcluster/adhoc_taskgraph/routes.py @@ -17,9 +17,6 @@ def add_signing_indexes(config, task, variant): routes = task.setdefault("routes", []) - if config.params["level"] != "3": - return task - subs = config.params.copy() subs["build_date"] = time.strftime( "%Y.%m.%d", time.gmtime(config.params["build_date"]) diff --git a/taskcluster/adhoc_taskgraph/transforms/cached.py b/taskcluster/adhoc_taskgraph/transforms/cached.py new file mode 100644 index 0000000..e9b092e --- /dev/null +++ b/taskcluster/adhoc_taskgraph/transforms/cached.py @@ -0,0 +1,81 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Build the cached_task digest to prevent rerunning tasks if the code hasn't changed. +""" + +import hashlib +import json +import os +import subprocess + +import taskgraph +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.hash import hash_path, hash_paths +from taskgraph.util.memoize import memoize + +from adhoc_taskgraph.signing_manifest import MANIFEST_DIR + +transforms = TransformSequence() + +BASE_DIR = os.getcwd() + + +@transforms.add +def add_resources(config, tasks): + for task in tasks: + resources = set(task.pop("resources", [])) + resources.add( + os.path.join(MANIFEST_DIR, f"{task['extra']['manifest-name']}.yml") + ) + resources = list(resources) + attributes = task.setdefault("attributes", {}) + if attributes.get("resources") is not None: + if resources and attributes["resources"] != resources: + raise Exception( + f"setting {config.kind} {task.get('name')} " + f"task.attributes.resources to {resources}: " + f"it's already set to {attributes['resources']}!" + ) + attributes["resources"] = resources + yield task + + +@transforms.add +def build_cache(config, tasks): + for task in tasks: + if task.get("cache", True) and not taskgraph.fast: + digest_data = [] + digest_data.append( + json.dumps( + task.get("attributes", {}).get("digest-extra", {}), + indent=2, + sort_keys=True, + ) + ) + resources = task["attributes"]["resources"] + for resource in resources: + path = os.path.join(BASE_DIR, resource) + if os.path.isdir(resource): + digest_data.append(hash_paths(path, [""])) + elif os.path.isfile(resource): + digest_data.append(hash_path(path)) + else: + raise Exception(f"Unknown resource {resource}") + cache_name = task["name"].replace(":", "-") + task["cache"] = { + "type": f"adhoc-signing.v1.{config.kind}", + "name": cache_name, + "digest-data": digest_data, + } + + yield task + + +@transforms.add +def set_label(config, jobs): + """Set the job label, which the `cached_tasks` transform needs""" + for job in jobs: + job.setdefault("label", f"{config.kind}-{job.pop('name')}") + yield job diff --git a/taskcluster/ci/dep-signing/kind.yml b/taskcluster/ci/dep-signing/kind.yml index 88d4817..e7318dd 100644 --- a/taskcluster/ci/dep-signing/kind.yml +++ b/taskcluster/ci/dep-signing/kind.yml @@ -10,6 +10,8 @@ kind-dependencies: transforms: - adhoc_taskgraph.transforms.signing:transforms + - adhoc_taskgraph.transforms.cached:transforms + - taskgraph.transforms.cached_tasks:transforms - taskgraph.transforms.task:transforms job-template: