run-tests.py

"""
Copyright (c) 2019 Ash Wilding. All rights reserved.

SPDX-License-Identifier: MIT

This script attempts to compile every test source + header generated by the
run-build.py script.

Usage:

    python3.8 run-tests.py [--keep] COMPILER_PATH [COMPILER_FLAGS]

Where the optional `--keep` flag prevents the script from cleaning up the
generated .o object files.

Example:

    python3.8 run-tests.py aarch64-none-elf-gcc
    python3.8 run-tests.py --keep clang -std=c99 -O3

"""

import os
import shutil
import sys
import subprocess

def abort( msg:object, errno:int=1 ) -> None:
    print(msg)
    sys.exit(1)

if __name__ == "__main__":
    USAGE = (f"usage: python3.8 {sys.argv[0]} [--keep] COMPILER_PATH [COMPILER_FLAGS]\n\n"
             f"example: python3.8 {sys.argv[0]} aarch64-none-elf-gcc\n"
             f"example: python3.8 {sys.argv[0]} --keep clang -std=c99 -O3")

    if len(sys.argv) < 2:
        abort(USAGE)

    if len(sys.argv) == 2 and sys.argv[1] == "--keep":
        abort(USAGE)

    print("========== test setup ==========")
    keep_objs = False
    argv_compiler = 1
    argv_flags = 2
    if sys.argv[1] == "--keep":
        keep_objs = True
        argv_compiler = 2
        argv_flags = 3
    compiler = sys.argv[argv_compiler]

    try:
        os.chdir("test")
    except FileNotFoundError:
        abort("failed to cd into 'test/' directory; have you built the library?")

    def cleanup_test_dir_and_return_list_of_sources() -> str:
        files = []
        try:
            for (_, _, f) in os.walk("."):
                files.extend(f)
                break
            sources = sorted(list(filter(lambda s: s.endswith(".c"), files)))
            others = list(filter(lambda s: not s.endswith(".c"), files))
            if others:
                for o in others:
                    (shutil.rmtree if os.path.isdir(o) else os.remove)(o)
        except OSError as e:
            abort(e, e.errno if hasattr(e, "errno") else 1)
        return sources

    base_command = [
        compiler,
        "-g",
        "-c",
        "-I../include",
        "-Wall",
        "-Wextra",
        "-pedantic",
        "-Werror",
    ]

    if len(sys.argv) > argv_flags:
        base_command += sys.argv[argv_flags:]

    base_command_str = " ".join(base_command)

    if not "-O" in base_command_str:
        base_command.append("-O2")

    if not "-std" in base_command_str:
        base_command.append("-std=c11")

    if "clang" in compiler:
        if not "--target" in base_command_str:
            base_command.append("--target=aarch64-none-elf")

    ok = []
    failures = []
    print("========== test start ==========")
    with open(os.devnull, "wb") as devnull:
        for source in cleanup_test_dir_and_return_list_of_sources():
            command = f"{' '.join(base_command)} {source}"
            print(f"{command}")
            with subprocess.Popen(command.split()) as p:
                p.wait()
                if 0 == p.returncode:
                    ok.append(source)
                else:
                    failures.append(source)
                    print("\n\n")

    if failures:
        print("========== test failures ==========")
        print("\n".join(failures))

    print("========== test results ==========")
    print(f"using compiler: {subprocess.check_output([compiler, '--version']).splitlines()[0].decode('UTF-8')}")
    print(f"successfully built {len(ok)} sources out of {len(ok)+len(failures)} ({len(failures)} failures)")

    if not keep_objs:
        cleanup_test_dir_and_return_list_of_sources()

    sys.exit(not not len(failures))