build-si-containers

#!/usr/bin/env python3

# This script does the following.
# 1. Take one or more test names (e.g., mpich) from the command line
# 2. Take one or more testers found in testers/ (default to libabigail)
# 3. For each test, for each tester, build a container that spack installs the
#    packages specified in the test on top of the testers container.
# 4. Install the script/library here to run the tests
# 5. Output should be saved to a namespaced folder, <tester>/<package>/etc
# 6. We will need a way to compare output.


import argparse
import shutil
import logging
import tempfile
import jsonschema
import os
import re
import json
import calendar
import subprocess
import time
import shutil
import yaml
import sys
from abc import ABC

from jinja2 import Environment, FileSystemLoader, select_autoescape

logging.basicConfig(level=logging.INFO)

# We want the root
here = os.path.abspath(os.path.dirname(__file__))
templates = os.path.join(here, "templates")

env = Environment(
    autoescape=select_autoescape(["html"]), loader=FileSystemLoader(templates)
)

package_schema = {
    "$schema": "http://json-schema.org/schema#",
    "title": "build-abi-containers package schema",
    "type": "object",
    "additionalProperties": False,
    "required": ["package"],
    "properties": {
        "package": {
            "type": "object",
            "additionalProperties": False,
            "required": [
                "name",
                "versions",
                "headers",
            ],
            "properties": {
                "name": {"type": "string"},
                "versions": {"type": "array", "items": {"type": "string"}},
                "headers": {"type": "array", "items": {"type": "string"}},
                "libs": {"type": "array", "items": {"type": "string"}},
                "libregex": {"type": "array", "items": {"type": "string"}},
                "bins": {"type": "array", "items": {"type": "string"}},
                "run": {"type": "array", "items": {"type": "string"}},
            },
        },
        "test": {
            "type": "object",
            "additionalProperties": False,
            "properties": {
                "build_cache": {"type": "boolean"},
            },
        },
    },
}

test_schema = {
    "$schema": "http://json-schema.org/schema#",
    "title": "build-abi-containers test schema",
    "type": "object",
    "additionalProperties": False,
    "requiredProperties": ["packages", "experiment", "tester"],
    "properties": {
        "packages": {
            "type": "array",
            "items": {
                "type": "object",
                "additionalProperties": False,
                "properties": {
                    "name": {"type": "string"},
                    "versions": {"type": "array", "items": {"type": "string"}},
                },
            },
        },
        "experiment": {
            "type": "object",
            "additionalProperties": False,
            "required": ["name"],
            "properties": {
                "name": {
                    "type": "string",
                    "enum": ["single-test", "double-test", "manual-test"],
                },
            },
        },
        "tester": {
            "type": "object",
            "additionalProperties": False,
            "required": ["name"],
            "properties": {
                "name": {
                    "type": "string",
                    "enum": ["libabigail", "symbolator", "smeagle"],
                },
                "version": {"type": ["string"]},
            },
        },
        "test": {
            "type": "object",
            "additionalProperties": False,
            "properties": {
                "prebuilt": {"type": "boolean"},
                "cache_only": {"type": "boolean"},
                "dockerfile": {"type": "string"},
            },
        },
    },
}

tester_schema = {
    "$schema": "http://json-schema.org/schema#",
    "title": "build-abi-containers tester schema",
    "type": "object",
    "additionalProperties": False,
    "properties": {
        "tester": {
            "type": "object",
            "additionalProperties": False,
            "properties": {
                "name": {"type": "string"},
                "runscript": {"type": "string"},
                "container": {"type": "string"},
                "entrypoint": {"type": "string"},
                "version": {"type": "string"},
                "args": {"type": "array", "items": {"type": "string"}},
            },
        }
    },
}


class Config(ABC):
    """
    A general Config base to load a config and validate it
    """

    def __init__(self, config_file):
        self.config_file = os.path.abspath(config_file)
        self.config = read_yaml(self.config_file)
        self.config_basename = os.path.basename(self.config_file)
        jsonschema.validate(self.config, schema=self.schema)

    def __repr__(self):
        return self.__str__()

    def __getattr__(self, key):
        return self.config.get(key)


class Tester(Config):
    """
    A testing base like libabigail or smeagle
    """

    schema = tester_schema

    def __str__(self):
        return "<tester:%s>" % os.path.basename(self.config_file)

    def __getattr__(self, key):
        return self.config["tester"].get(key)


class TestPackage(Config):
    """
    A package to be tested with a tester.
    """

    schema = package_schema

    def __str__(self):
        return "<package:%s>" % self.config["package"]["name"]

    def __getattr__(self, key):
        return self.config["package"].get(key) or self.config.get(key)


class Test(Config):

    schema = test_schema

    def __init__(self, config_file, **kwargs):
        # Set and load config file
        super().__init__(config_file)
        if "test" not in self.config:
            self.config["test"] = {}

        # Command line args over-ride config
        for term in ["prebuilt", "use_cache"]:
            if term in kwargs:
                self.config["test"][term] = kwargs.get(term)

    @property
    def version(self):
        """
        If the test tester has a version, return it
        """
        return self.config["tester"].get("version")

    def __str__(self):
        return "<test:%s>" % os.path.basename(self.config_file)

    @property
    def name(self):
        """
        The name of the test is the yaml file without extension
        """
        return re.sub("[.](yaml|yml)", "", os.path.basename(self.config_file))


class TestSetup:
    def __init__(self, root):
        """A build-si-containers test setup will look for tests/testers"""
        self.testers = set()
        self.root = root
        self.check_root()
        self.docker_images()

    @property
    def test_dir(self):
        return os.path.join(self.root, "tests")

    @property
    def package_dir(self):
        return os.path.join(self.root, "packages")

    @property
    def templates_dir(self):
        return os.path.join(self.root, "templates")

    @property
    def testers_dir(self):
        return os.path.join(self.root, "testers")

    @property
    def spack_packages(self):
        """
        Custom spack test packages
        """
        return os.path.join(self.root, "spack")

    def check_root(self):
        """
        Ensure that the root is structured correctly.
        """
        for path in (
            self.root,
            self.test_dir,
            self.package_dir,
            self.testers_dir,
            self.templates_dir,
        ):
            if not os.path.exists(path):
                sys.exit("% does not exist in the root!" % path)

    def test(self, container, outdir):
        """
        Given a container, run it and bind to an output directory to test
        """
        res = run_command(
            ["docker", "run", "-t", "-v", "%s:/results" % outdir, container],
            to_stdout=True,
        )

    def deploy(self, container):
        """
        Given a container, deploy by pushing it.
        """
        res = run_command(["docker", "push", container], to_stdout=True)

    def get_container(self, test):
        """
        Given a tester name, generate the expected container name
        """
        # read in this test file
        test = Test(self.get_test_config(test))
        return "ghcr.io/buildsi/%s:latest" % test.name

    def docker_images(self):
        """
        Load docker images into the client to determine which already exist.
        """
        images = run_command(["docker", "images"])
        self.containers = []
        for image in images.split("\n"):
            if not image:
                continue
            image, rest = image.split(" ", 1)
            tag = rest.strip().split(" ")[0]
            self.containers.append("%s:%s" % (image, tag))

        self.containers = set(self.containers)

    def generate_single_tests(self, test):
        """
        Generate a list of tests for a library against itself.
        """
        # Assemble list of tests to run
        tests = []

        # If we have more than one package, no go
        package = test.config.get("packages", [])
        if len(package) > 1:
            sys.exit("A single-test experiment can only include one package")
        package = package[0]

        # Create the package to test
        package_file = self.get_package_config(package["name"])
        package = TestPackage(package_file)

        # Get versions
        if test.versions:
            if any(test.versions not in package.versions):
                sys.exit("Valid versions include %s" % package.versions)
        versions = test.versions or package.versions

        # We should still be able to retrieve the correct list from the package
        package.versions = versions

        # Test against all versions
        for version1 in versions:
            for version2 in versions:
                tests.append(
                    {
                        "package1": package,
                        "package2": package,
                        "version1": version1,
                        "version2": version2,
                    }
                )

        # Return a list of tests and unique packages
        return tests, [package]

    def build(
        self,
        test,
        use_cache=False,
        cache_only=False,
        fail_fast=True,
        skips=None,
        docker_no_cache=False,
        prebuilt=False,
    ):
        """
        Create a Dockerfile and build
        """
        # read in this test file
        test_file = self.get_test_config(test)
        test = Test(test_file, prebuilt=prebuilt, use_cache=use_cache)

        # Containers to skip building
        skips = skips or []

        # Get the experiment type to assemble list of tests
        experiment = test.config["experiment"]["name"]

        if experiment == "single-test":

            # Assemble list of tests to run
            tests, packages = self.generate_single_tests(test)

        else:
            sys.exit("Experiment type %s is not supported." % experiment)

        tester = test.tester["name"]

        # Get the tester build template
        template = self.get_tester_template(tester, test)
        tester = Tester(self.get_tester_config(tester))

        # Right now one container has all versions
        # Does the tester have extra scripts?
        bins = []
        tester_bin = os.path.join(self.testers_dir, tester.name, "bin")
        if os.path.exists(tester_bin):
            for filename in os.listdir(tester_bin):
                bins.append(filename)

        # Render the template and runtests.py file
        runscript = self.get_tester_runscript(tester).render(
            tests=tests, tester=tester, packages=packages, experiment=experiment
        )
        out = template.render(
            packages=packages,
            tester=tester,
            bins=bins,
            cache_only=cache_only,
            test=test,
        )
        container_name = self.get_container(test.name)

        # Don't build the container if requested to skip
        if container_name in skips:
            return container_name

        # Show dockerfile to the user
        print("Dockerfile:---------\n%s\n" % out)

        with tempfile.TemporaryDirectory() as tmp:

            # Copy spack packages
            shutil.copytree(self.spack_packages, os.path.join(tmp, "spack"))
            write_file(out, os.path.join(tmp, "Dockerfile"))
            write_file(runscript, os.path.join(tmp, tester.runscript))
            shutil.copyfile(test_file, os.path.join(tmp, os.path.basename(test_file)))

            # If we have extra files to add, copy them
            for binfile in bins:
                shutil.copyfile(
                    os.path.join(tester_bin, binfile), os.path.join(tmp, binfile)
                )
            cmd = ["docker", "build"]
            if docker_no_cache:
                cmd.append("--no-cache")
            cmd += ["-t", container_name, tmp]
            res = subprocess.call(cmd)
            if res == 0:
                return container_name
            elif res != 0 and fail_fast:
                sys.exit("Error building %s" % container_name)

            print("Issue building %s, but fail fast not set.")

    def get_tester_config(self, tester):
        """
        Given a package and tester, return the tester config.
        """
        config_file = os.path.join(self.testers_dir, tester, "tester.yaml")
        if not os.path.exists(config_file):
            sys.exit("%s does not exist!" % config_file)
        return config_file

    def get_test_config(self, path):
        """
        Given a test name, get the config for it.
        """
        return self._get_file(path, self.test_dir)

    def _get_file(self, name, dirname):
        """
        Get a known test or package file
        """
        # If we are given a file directly, honor it
        if os.path.exists(name):
            return name

        # Look for exact file in the required directory
        filename = os.path.join(dirname, name)
        if os.path.exists(filename):
            return filename

        # Last resort - add the extension
        filename = "%s.yaml" % filename
        if not os.path.exists(filename):
            sys.exit("%s does not exist!" % name)
        return filename

    def get_package_config(self, package):
        """
        Given a package and tester, return the tester config.
        """
        return self._get_file(package, self.package_dir)

    def get_tester_runscript(self, tester):
        """
        Get the runscript for the tester
        """
        template = os.path.join(self.templates_dir, tester.name, tester.runscript)
        if not os.path.exists(template):
            sys.exit("Tester runscript %s does not exist!" % template)
        template = template.replace(self.templates_dir, "").strip("/")
        return env.get_template(template)

    def get_tester_template(self, tester, test):
        """
        Given a package and tester, return the container template.
        """
        dockerfile = os.path.join(self.templates_dir, tester, "Dockerfile")
        default = os.path.join(self.templates_dir, "Dockerfile.default")

        # Does the test defines a specific dockerfile?
        test_dockerfile = test.test.get("dockerfile")

        if test_dockerfile and os.path.exists(test_dockerfile):
            dockerfile = os.path.abspath(test_dockerfile)
        elif not os.path.exists(dockerfile) and test.test.get("prebuilt") == True:
            dockerfile = os.path.join(self.templates_dir, "Dockerfile.prebuilt")
        elif not os.path.exists(dockerfile) and test.test.get("use_cache" == True):
            dockerfile = os.path.join(self.templates_dir, "Dockerfile.buildcache")
        elif not os.path.exists(dockerfile):
            dockerfile = default

        dockerfile = dockerfile.replace(self.templates_dir, "").strip("/")
        return env.get_template(dockerfile)


def read_yaml(filename):
    with open(filename, "r") as fd:
        content = yaml.load(fd, Loader=yaml.FullLoader)
    return content


def write_file(content, filename):
    with open(filename, "w") as fd:
        fd.write(content)


def run_command(cmd, to_stdout=False):
    stdout = subprocess.PIPE
    if to_stdout:
        stdout = None
    p = subprocess.Popen(cmd, stdout=stdout, stderr=subprocess.STDOUT)
    out = p.communicate()[0]
    if out:
        out = out.decode("utf-8")
    if p.returncode != 0:
        print(out)
        sys.exit("Error running %s." % " ".join(cmd))
    return out


def get_parser():
    parser = argparse.ArgumentParser(description="Build SI Container Tester")

    description = "actions for testing containers for the BUILD SI project"
    subparsers = parser.add_subparsers(
        help="run-tests actions",
        title="actions",
        description=description,
        dest="command",
    )

    # Deploy a container for a spec (push to docker hub) if found
    deploy = subparsers.add_parser("deploy", help="deploy test container.")

    # Run a complete test, which includes building the test container
    test = subparsers.add_parser("test", help="run tests.")
    test.add_argument(
        "--outdir",
        "-o",
        dest="outdir",
        help="Write test results to this directory (defaults to results in $PWD)",
        default=os.path.join(os.getcwd(), "results"),
    )
    test.add_argument(
        "--rebuild",
        dest="rebuild",
        help="Force rebuild of the container if it does not exist.",
        default=False,
        action="store_true",
    )

    # Build a testing container
    build = subparsers.add_parser("build", help="build a testing container.")

    for command in [test, build]:
        command.add_argument(
            "--use-cache",
            dest="use_cache",
            help="Install from build cache instead of autamus.",
            default=False,
            action="store_true",
        )
        command.add_argument(
            "--prebuilt",
            dest="prebuilt",
            help="Use a prebuilt container from ghcr.io/autamus/builds-<package>",
            default=False,
            action="store_true",
        )
        command.add_argument(
            "--cache-only",
            dest="cache_only",
            help="ONLY allow install from build cache.",
            default=False,
            action="store_true",
        )
        command.add_argument(
            "--docker-no-cache",
            dest="docker_no_cache",
            help="Do not use the docker cache.",
            default=False,
            action="store_true",
        )
        command.add_argument(
            "--fail-fast",
            dest="fail_fast",
            help="If a container build fails, exit.",
            default=True,
            action="store_false",
        )

    for command in [test, build, deploy]:
        command.add_argument("tests", help="tests to run", nargs="+")
        command.add_argument(
            "--root",
            "-r",
            dest="root",
            help="The root with the tests and testers directories.",
            default=os.getcwd(),
        )
    return parser


def main():
    """
    Entrypoint for running tests.
    """
    parser = get_parser()

    def help(return_code=0):
        parser.print_help()
        sys.exit(return_code)

    # If an error occurs while parsing the arguments, the interpreter will exit with value 2
    args, extra = parser.parse_known_args()
    if not args.command:
        help()

    setup = TestSetup(args.root)

    if args.command == "build":
        for test in args.tests:
            setup.build(
                test,
                use_cache=args.use_cache,
                fail_fast=args.fail_fast,
                docker_no_cache=args.docker_no_cache,
                cache_only=args.cache_only,
                prebuilt=args.prebuilt,
            )

    elif args.command == "deploy":
        for test in args.tests:
            container = setup.get_container(test)
            setup.deploy(container)

    elif args.command == "test":
        for test in args.tests:

            # By default, don't skip any builds, unless a rebuild is not wanted
            skips = []
            if not args.rebuild:

                # Skip containers that already exist
                container = setup.get_container(test)
                if container in setup.containers:
                    skips.append(container)

            container = setup.build(
                test,
                use_cache=args.use_cache,
                fail_fast=args.fail_fast,
                skips=skips,
                docker_no_cache=args.docker_no_cache,
                cache_only=args.cache_only,
                prebuilt=args.prebuilt,
            )
            if container:
                setup.test(container, args.outdir)
    else:
        help()


if __name__ == "__main__":
    main()