Skip to content

Commit

Permalink
imgtestlib: read common CI runner from Schutzfile
Browse files Browse the repository at this point in the history
Use the common CI runner defined in the Schutzfile in all gitlab CI
generators.
  • Loading branch information
achilleas-k committed Nov 25, 2024
1 parent df429c3 commit fbc0771
Show file tree
Hide file tree
Showing 4 changed files with 50 additions and 12 deletions.
3 changes: 2 additions & 1 deletion test/scripts/generate-build-config
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ build/{distro}/{arch}/{image_type}/{config_name}:
- ./test/scripts/upload-results "{distro}" "{image_type}" "{config}"
extends: .terraform
variables:
RUNNER: aws/fedora-40-{arch}
RUNNER: {runner}-{arch}
INTERNAL_NETWORK: "{internal}"
"""

Expand Down Expand Up @@ -66,6 +66,7 @@ def generate_configs(build_requests, pipeline_file):

config_path = os.path.join(testlib.CONFIGS_PATH, config_name+".json")
pipeline_file.write(JOB_TEMPLATE.format(distro=distro, arch=arch, image_type=image_type,
runner=testlib.get_common_ci_runner(),
config_name=config_name, config=config_path,
internal="true" if "rhel" in distro else "false",
image_path=image_path))
Expand Down
40 changes: 30 additions & 10 deletions test/scripts/generate-gitlab-ci
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import imgtestlib as testlib

ARCHITECTURES = ["x86_64", "aarch64"]
MANIFEST_ONLY_ARCHES = ["ppc64le", "s390x"]
RUNNER = testlib.get_common_ci_runner()


BASE_CONFIG = """---
Expand Down Expand Up @@ -36,7 +37,7 @@ check-build-coverage:
stage: verify
extends: .terraform
variables:
RUNNER: aws/fedora-40-x86_64
RUNNER: {runner}-x86_64
INTERNAL_NETWORK: "true"
PYTHONUNBUFFERED: 1
script:
Expand Down Expand Up @@ -70,7 +71,7 @@ generate-build-config-{distro}-{arch}:
stage: gen
extends: .terraform
variables:
RUNNER: aws/fedora-40-{arch}
RUNNER: {runner}-{arch}
INTERNAL_NETWORK: "true"
script:
- sudo ./test/scripts/setup-osbuild-repo
Expand Down Expand Up @@ -102,7 +103,7 @@ generate-ostree-build-config-{distro}-{arch}:
stage: ostree-gen
extends: .terraform
variables:
RUNNER: aws/fedora-40-{arch}
RUNNER: {runner}-{arch}
INTERNAL_NETWORK: "true"
script:
- sudo ./test/scripts/setup-osbuild-repo
Expand Down Expand Up @@ -138,7 +139,7 @@ generate-manifests-{distro}-{arch}:
stage: gen
extends: .terraform
variables:
RUNNER: aws/fedora-40-x86_64
RUNNER: {runner}-x86_64
INTERNAL_NETWORK: "true"
script:
- sudo ./test/scripts/setup-osbuild-repo
Expand Down Expand Up @@ -175,10 +176,29 @@ def main():
continue

combos.add(combo)
gen_stage.append(GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache))
trigger_stage.append(TRIGGER_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache))
ostree_gen_stage.append(OSTREE_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache))
ostree_trigger_stage.append(OSTREE_TRIGGER_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache))
gen_stage.append(GEN_TEMPLATE.format(
distro=img["distro"],
arch=img["arch"],
runner=RUNNER,
cache=cache))

trigger_stage.append(TRIGGER_TEMPLATE.format(
distro=img["distro"],
arch=img["arch"],
runner=RUNNER,
cache=cache))

ostree_gen_stage.append(OSTREE_GEN_TEMPLATE.format(
distro=img["distro"],
arch=img["arch"],
runner=RUNNER,
cache=cache))

ostree_trigger_stage.append(OSTREE_TRIGGER_TEMPLATE.format(
distro=img["distro"],
arch=img["arch"],
runner=RUNNER,
cache=cache))

man_only_images = testlib.list_images(arches=MANIFEST_ONLY_ARCHES)
man_gen_stage = []
Expand All @@ -188,10 +208,10 @@ def main():
continue

combos.add(combo)
man_gen_stage.append(MANIFEST_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"]))
man_gen_stage.append(MANIFEST_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], runner=RUNNER))

with open(config_path, "w", encoding="utf-8") as config_file:
config_file.write(BASE_CONFIG)
config_file.write(BASE_CONFIG.format(runner=RUNNER))
config_file.write(testlib.BASE_CONFIG)
config_file.write("\n".join(gen_stage))
config_file.write("\n".join(trigger_stage))
Expand Down
3 changes: 2 additions & 1 deletion test/scripts/generate-ostree-build-config
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ build/{distro}/{arch}/{image_type}/{config_name}:
- ./test/scripts/upload-results "{distro}" "{image_type}" "{config}"
extends: .terraform
variables:
RUNNER: aws/fedora-40-{arch}
RUNNER: {runner}-{arch}
INTERNAL_NETWORK: "{internal}"
needs:
- pipeline: "$PARENT_PIPELINE_ID"
Expand Down Expand Up @@ -312,6 +312,7 @@ def generate_configs(build_requests, container_configs, pipeline_file, configs_d
run_container_cmd = f"sudo podman run -d --rm -p{container_port}:8080 oci-archive:container.tar"

pipeline_file.write(JOB_TEMPLATE.format(distro=distro, arch=arch, image_type=image_type,
runner=testlib.get_common_ci_runner(),
config_name=config_name, config=build_config_path,
dl_container=dl_container_cmd,
start_container=run_container_cmd,
Expand Down
16 changes: 16 additions & 0 deletions test/scripts/imgtestlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -497,3 +497,19 @@ def skopeo_inspect_id(image_name: str, arch: str) -> str:

# don't error out, just return an empty string and let the caller handle it
return ""


def get_common_ci_runner():
"""
CI runner for common tasks.
Currently this is used for all gitlab CI jobs. In the future, we might switch to running build jobs on the same host
distro as the target image, but this CI runner will still be used for generic tasks like check-build-coverage.
"""
with open(SCHUTZFILE, encoding="utf-8") as schutzfile:
data = json.load(schutzfile)

if (runner := data.get("common", {}).get("gitlab-ci-runner")) is None:
raise KeyError(f"gitlab-ci-runner not defined in {SCHUTZFILE}")

return runner

0 comments on commit fbc0771

Please sign in to comment.