From 0ed8863dad4aef8dba98e9f99ec2d0fd94324061 Mon Sep 17 00:00:00 2001 From: Tim Hutt Date: Wed, 2 Oct 2024 17:31:36 +0100 Subject: [PATCH] Implement tests --- .github/workflows/compile.yml | 12 +- .github/workflows/compile_new_test.yml | 61 +++ .github/workflows/test-results.yml | 4 +- .pre-commit-config.yaml | 2 +- HOWTO_add_new_tests.txt | 91 ++++ SAIL_RISCV_ROOTDIR | 5 + TESTING.md | 90 ++++ bin/run_tests.py | 621 +++++++++++++++++++++++++ 8 files changed, 881 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/compile_new_test.yml create mode 100644 HOWTO_add_new_tests.txt create mode 100644 SAIL_RISCV_ROOTDIR create mode 100644 TESTING.md create mode 100755 bin/run_tests.py diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 333d8ee79..2b90499cd 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -11,7 +11,7 @@ jobs: - name: Check out repository code uses: actions/checkout@HEAD with: - submodules: true + submodules: recursive - name: Ensure pre-commit checks pass run: python3 -m pip install pre-commit && pre-commit run --all-files --show-diff-on-failure --color=always - name: Install sail from binary @@ -22,13 +22,19 @@ jobs: run: test/run_tests.sh - name: Upload test results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: tests.xml path: test/tests.xml + - name: Upload test results (new) + if: always() + uses: actions/upload-artifact@v3 + with: + name: tests_new.xml + path: TEST_DIR_ROOT/riscv_tests.git.subtree/tests_new.xml - name: Upload event payload if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: event.json path: ${{ github.event_path }} diff --git a/.github/workflows/compile_new_test.yml b/.github/workflows/compile_new_test.yml new file mode 100644 index 000000000..a421cac47 --- /dev/null +++ b/.github/workflows/compile_new_test.yml @@ -0,0 +1,61 @@ +name: CI + +on: [push, pull_request, workflow_dispatch] + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - name: Install packages + run: sudo apt install -y opam zlib1g-dev pkg-config libgmp-dev z3 device-tree-compiler + - name: Check out repository code + uses: actions/checkout@HEAD + with: + submodules: recursive + - name: Ensure pre-commit checks pass + run: pip install pre-commit && pre-commit run --all-files --show-diff-on-failure --color=always + - name: Init opam + run: opam init --disable-sandboxing -y + - name: Install sail + run: opam install -y sail + - name: Download RISC-V toolchain (32-bit) + run: wget -c https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2024.04.12/riscv32-elf-ubuntu-22.04-gcc-nightly-2024.04.12-nightly.tar.gz + - name: Install RISC-V toolchain (32-bit) + run: sudo tar xvfz riscv32-elf-ubuntu-22.04-gcc-nightly-2024.04.12-nightly.tar.gz --directory /opt + - name: Download RISC-V toolchain (64-bit) + run: wget -c https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2024.04.12/riscv64-elf-ubuntu-22.04-gcc-nightly-2024.04.12-nightly.tar.gz + - name: Install RISC-V toolchain (64-bit) + run: sudo tar xvfz riscv64-elf-ubuntu-22.04-gcc-nightly-2024.04.12-nightly.tar.gz --directory /opt + - name: Build RISC-V testsuite + run: | + cd TEST_DIR_ROOT/riscv_tests.git.subtree ; \ + echo ${PWD} ; ls ; \ + autoconf ; \ + ./configure --prefix=${PWD}/target ; \ + export PATH=/opt/riscv/bin:${PATH} ; \ + make isa ; + - name: Build and test simulators + run: eval $(opam env) && test/run_tests.sh + - name: Build and test simulators (new method) + run: | + eval $(opam env) ; \ + cd TEST_DIR_ROOT/riscv_tests.git.subtree ; \ + ../../bin/run_tests.py --outfile=tests_new.xml --32bit=yes --64bit=yes --c_sim=yes --sailcov=no --clean_build=yes --test_switch_pyfile=test_command_line_switch.py --test_ignore_pyfile=test_ignore_list.py + - name: Upload test results + if: always() + uses: actions/upload-artifact@v3 + with: + name: tests.xml + path: test/tests.xml + - name: Upload test results (new) + if: always() + uses: actions/upload-artifact@v3 + with: + name: tests_new.xml + path: TEST_DIR_ROOT/riscv_tests.git.subtree/tests_new.xml + - name: Upload event payload + if: always() + uses: actions/upload-artifact@v3 + with: + name: event.json + path: ${{ github.event_path }} diff --git a/.github/workflows/test-results.yml b/.github/workflows/test-results.yml index 4201d46e5..8ebc18352 100644 --- a/.github/workflows/test-results.yml +++ b/.github/workflows/test-results.yml @@ -22,7 +22,7 @@ jobs: run_id: ${{github.event.workflow_run.id }}, }); var matchArtifacts = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == 'tests.xml' || artifact.name == 'event.json' + return artifact.name == 'tests.xml' || 'tests_new.xml' || artifact.name == 'event.json' }); var count = matchArtifacts.length; for (var i = 0; i < count; i++) { @@ -40,6 +40,8 @@ jobs: } - name: Extract test results run: unzip tests.xml.zip + - name: Extract test results (new) + run: unzip tests_new.xml.zip - name: Extract event payload run: unzip event.json.zip - name: Publish test results diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d8791c06f..c40b98c91 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks -exclude: '^(prover_snapshots)|(generated_definitions)|(c_emulator/SoftFloat-3e)' +exclude: '^(prover_snapshots)|(generated_definitions)|(c_emulator/SoftFloat-3e)|(TEST_DIR_ROOT/riscv_tests.git.subtree)' minimum_pre_commit_version: 2.10.0 repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/HOWTO_add_new_tests.txt b/HOWTO_add_new_tests.txt new file mode 100644 index 000000000..2a825ce4d --- /dev/null +++ b/HOWTO_add_new_tests.txt @@ -0,0 +1,91 @@ +2024-05-28 Bill McSpadden + +-.0 The following was done to create the subtree(s). This was then committed to + sail-riscv/ + + > git subtree add --prefix TEST_DIR_ROOT/riscv_tests.git.subtree git@github.com:riscv-software-src/riscv-tests.git riscv-tests-sail [--squash] + +# TODO: request sent to Andrew to be made contributor to the riscv-test-env repo +# DONE +# TODO: the TEST_DIR_ROOT/riscv_tests.git.subtree/env exists when riscv-tests subtree +# is added. And this leads to the error, "prefix 'TEST_DIR_ROOT/riscv_tests.git.subtree/env' already exists." +# So, I need figure out something. The path to "env" is needed by riscv-tests +# build process, so this might have an impact on the riscv-tests build method. +# THE FIX: removed env/ on the branch, 'riscv-tests-sail'. Then perform the +# following command: + > git subtree add --prefix TEST_DIR_ROOT/riscv_tests.git.subtree/env git@github.com:riscv/riscv-test-env.git riscv-test-env-sail [--squash] + +## +# ... In the meantime, use the master branch + > git subtree add --prefix TEST_DIR_ROOT/riscv_tests.git.subtree/env git@github.com:riscv/riscv-test-env.git master [--squash] + +0. Check out the sail-riscv repo + +0.1 Point to remote + This will persist within your working repository. But when you checkout + another repo, you'll need to execute this command from within the new repo. + TODO: is there a way to do this with git hooks? + + > git remote add riscv-tests-sail-remote git@github.com:riscv-software-src/riscv-tests.git + > git remote add riscv-tests-env-sail-remote https://github.com/riscv/riscv-test-env.git + + +1.0 Go to test directory. Add new tests/files + + > cd TEST_DIR_ROOT/riscv_tests.git.subtree + > autoconf + > ./configure --prefix $PWD/target + > make isa + +1.1 Make sure the new tests run against your model + +1.1.1 Individual test + + > cd TEST_DIR_ROOT/riscv_tests.git.subtree + > ../../c_emulator/riscv_sim_RV[32|64] [options] isa/ + + +1.1.2 Suite of tests + + > cd TEST_DIR_ROOT/riscv_tests.git.subtree + > ../../bin/run_tests.py --32bit=yes --64bit=yes --c_sim=yes --test_switch_pyfile=test_command_line_switch.py --test_ignore_pyfile=test_ignore_list.py + + +2.0 Push your new tests/files back to the riscv-tests repository (branch: riscv-tests-sail) + + > cd + > git subtree push --prefix TEST_DIR_ROOT/riscv_tests.git.subtree/ riscv-tests-sail-remote riscv-tests-sail + + This should put your changes into the riscv-tests repo under the riscv-tests-sail branch. + Steer your browser to ... + + https://github.com/riscv-software-src/riscv-tests/tree/riscv-tests-sail + + ... and make sure your files were properly committed to the branch. + + MAKE SURE YOUR CHANGES DID NOT GO ONTO 'master'. If they did, back out your last push. + +2.1 If you made any changes to TEST_DIR_ROOT/riscv_tests.git.subtree/env/, commit them and push + to the remote repo. + +3.0 Merging from remote riscv-tests/master into local repo + + Note : When I did the following command ... + + > git subtree pull -P TEST_DIR_ROOT/riscv_tests.git.subtree/ riscv-tests-sail-remote master + + ... I got the new code , but git assumed that I was going to commit + these changes and it opened a commit editor saying it was merging with + the following message: + + Merge commit '408e461da11e0b298c4b69e587729532787212f5' into new_test_3 + + I tried to 'q!' out of my vim editor (assuming that the commit would not + be made), but the commit was made with the default commit message. + + The commit was made onto the sail-riscv branch, NOT onto the riscv-tests + branch. + +3.1 Run tests to make sure new updates from riscv-tests/master work as expected. + Then commit the changes (if needed). + Then generate a PR for these new changes diff --git a/SAIL_RISCV_ROOTDIR b/SAIL_RISCV_ROOTDIR new file mode 100644 index 000000000..94dadf31f --- /dev/null +++ b/SAIL_RISCV_ROOTDIR @@ -0,0 +1,5 @@ +This file is created in the root directory for the RISC-V Sail model. +It is used as a reference point for various tools within the repo. + +Do not rename it. Do not move it. Do not duplicate it anywhere in +the repo, else tooling may get confused. diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 000000000..7ea29c079 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,90 @@ +# Testing the RISCV-Sail model + +This document contains information regarding the testing of the RISC-V +Sail model. + +There are several goals for the testing effort of the RISC-V Sail model. +First, we need a set of tests that get run during the CI (Continuous Improvement) +cycle when merging PRs to the main branch. We will refer to these as Build Verification Tests +(BVTs). These should cover basic instruction behavior. The set of tests should run +rather quickly (perhaps 30 minutes) so that the CI does not bog down. + +Second, we want to take Architectural Compatability Tests (ACTs) and run them +against the Sail model. However, many of the ACTs are not self-checking and therefore +cannot be used directly for validating the functionality of the model. But we can +run the Sail model and cross-check with the Spike model using signature checks. + +Third, for all of these test methods, we want to be able to make coverage measurements +of the Sail code. This will give us some objective insights into where we have coverage +holes in our testing. + +## Background + +For the longest time, the set of tests used to validate changes to the +model, were a set of precompiled .elf files (along with their .dump file counterparts) +that were stored in the repo under `test/riscv-tests`. The scripts used to +run theses tests (and to gather test results) were `test/run_tests.sh` and +`test/run_fp_tests.sh`. + +These tests are a compiled snapshot of tests that can be found at +https://github.com/riscv-software-src/riscv-tests +that date back to 2019. + +This methodolgy was defecient in several ways. +1. Original test source is difficult to track down. +1. Storing compiled code in a git repo is usually frowned upon. +1. There is no easy way to add new tests to the repo when you add a new feature. +1. `run_tests.sh` is difficult to enhance with new features. + +We anticipate that the `test/` directory will be removed once a more robust +testing methodolgy is put in place. (See next section.) + +## Adding new tests + + + +To fix the defeciencies of the old test methodology, we have done the +following: +1. Created a new test directory at the repo root, `TEST_DIR_ROOT/` under which +all new test collateral will be put. +1. Created a `bin/` directory under which various model scripts and executables +are added. +1. Installed https://github.com/riscv-software-src/riscv-tests as a subtree +at `TEST_DIR_ROOT/riscv-tests.git/`. We will be working on a special branch +in this repository: `riscv-tests-sail`. This allows us to add tests onto our +branch. And we can incorporate new tests into our testsuite as they appear +in the riscv-tests repo (by merging these new tests from the master branch +onto our branch). +1. Re-wrote `tests/run_tests.sh` in python and added run-time switches, the main +purpose of which was to be able to add command line switches to the execution of +particular tests. See the script, `bin/run_tests.py`, for execution parameters. +1. Updated `.github/workflows/compile.yml` to make use of the new run_tests python +script. + + + +## Future Plans + +### Sail and Spike Crosschecking with the Architecture Compatability Tests (ACTS) + +We intend to run the ACTs on both Sail and Spike. Test signatures will be compared +to check that the two simulators agree. + +### Fixing defecincies in the test environment + +We have the following defeciencies in the test environment that need to be fixed: + +1. A pass/fail can only be detected and reported from within the test itself. +If a test writer wanted to check the simluator log file to see +if certain strings existed (say, for example, you want to check the disassembly +of newly added instructions), there is no method to do so. The ability to +inspect the log file is a neccessary feature that needs to be added. + +1. Negative testing. We need to be able to check for proper detection of errors +which would then mean that the test "passed". For example, we might want to check +that if the vector extension is not enabled, that a test that uses a vector instruction +would "fail". + +1. Random test environment. We would like to add a directed random test environment +that allows us to hit unforeseen boundary conditions. We would like to randomize +both the instruction sequences as well as the model configuration. diff --git a/bin/run_tests.py b/bin/run_tests.py new file mode 100755 index 000000000..1e10aedc2 --- /dev/null +++ b/bin/run_tests.py @@ -0,0 +1,621 @@ +#! /usr/bin/env python3 +# vim: set tabstop=4 shiftwidth=4 expandtab set textwidth=79 +# ===========================================================================79 +# Filename: run_tests.py +# +# Description: Converting run_tests.sh to python +# +# Based on run_test.sh +# +# Same as run_test.sh: +# 1. output into .xml file is the same +# 2. colorization of output +# +# Differences with run_test.sh +# 1. Does not depend upon .elf file extension for filetype. +# Uses the unix 'file' command to get filetype +# 2. Added usage function +# 3. Added run switches (which didn't exist in run_test.sh) +# See the print_usage() function for a descriptionm of +# the supported switches. +# 4. Default search directories, rather than just one. +# 5. Removed use of $RISCV env var. +# 6. requires python3 +# 7. Added support to ignore tests via the --test_ignore_pyfile +# switch +# 8. Added support to supply switches to the riscv_sim command +# via the --test_switch_pyfile. +# +# Author(s): Bill McSpadden (bill@riscv.org) +# +# History: See revision control log +# ===========================================================================79 + +# ===========================================================================79 +# Necessary imports for this script: +import os # needed for os command interactions +import glob # needed for file list gathering useing wildcards +import re # regular expression +import sys # needed for command line arguments +import getopt # needed for command line arguments +import collections # needed for dequeues +import subprocess # needed for subprocesses where stdout is needed +from pathlib import Path +from inspect import currentframe, getframeinfo +from abc import ABC, abstractmethod +from copy import deepcopy +# Necessary imports for this script +# ===========================================================================79 + + +# ===========================================================================79 +# General data for use in this script. +# ===========================================================================79 +# Data structure for sim command line arguments +# TODO: make the key a regex +sim_test_command_line_switch_dict = { } +my_ignore_test_tuple = [] + +# Allowed command line options +opts, args = getopt.getopt ( + sys.argv[1:], + "dhuo:", + [ + "help", + "usage", + "outfile=", + "test_dir=", + "32bit=", + "64bit=", + "c_sim=", + "sailcov=", + "clean_build=", + "test_dir=", + "test_switch_pyfile=", + "test_ignore_pyfile=", + "debug" + ] + ) + +# Variables to be overridden with command line switches +xml_outfile = "./tests.xml" +run_32bit_tests = True +run_64bit_tests = True +run_csim = True +sailcov = False +clean_build = True +test_dir_list = [ "isa", "riscv-tests" ] +sail_riscv_rootdir = 'SAIL_RISCV_ROOTDIR' +test_switch_pyfile = '' +test_ignore_pyfile = '' +debug = False + +# Variables for tracking test status and test output +RED = '\033[0;91m' +GREEN = '\033[0;92m' +YELLOW = '\033[0;93m' +NC = '\033[0m' +test_pass = 0 +test_fail = 0 +all_pass = 0 +all_fail = 0 +SUITE_XML = "" +SUITES_XML = "" +# ===========================================================================79 + +# ===========================================================================79 +# Function prototypes: +# ==================================== +# Print Levels: +# print() Normal python print function. Goes to stdout +# debug_print() Print debug information. Goes to stdout. +# error_print() Print error message. Goes to stdout. Generally most errors should be fatal errors +# fatal_print() Print fatal error message. Exit with status 1. Generally most errors should be fatal errors. Goes to stdout. +# TRACE() For bringup debug only. TRACE() instances should be removed. + +def debug_print (text = "") : + if debug : + cf = currentframe() + of = cf.f_back + fi = getframeinfo(of) + filename = os.path.basename(fi.filename) + print("debug: file: " + filename + " line: " + str(of.f_lineno) + " : " + text) + return + +def error_print (text = "") : + cf = currentframe() + of = cf.f_back + fi = getframeinfo(of) + filename = os.path.basename(fi.filename) + print("error: file: " + filename + " line: " + str(of.f_lineno) + " : " + text) + return + +def fatal_print (text = "") : + cf = currentframe() + of = cf.f_back + fi = getframeinfo(of) + filename = os.path.basename(fi.filename) + print("fatal error: file: " + filename + " line: " + str(of.f_lineno) + " : " + text) + sys.exit(1) + return # never taken + +def TRACE(text = "") : + cf = currentframe() + of = cf.f_back + fi = getframeinfo(of) + filename = os.path.basename(fi.filename) + print("TRACE: file: " + filename + " line: " + str(of.f_lineno) + " : " + text) + return +# Print Levels: +# ==================================== + +# ==================================== +# Support for command line options +def print_usage(invocation) : + print(invocation + " usage: " + invocation + " []") + print(" Typically, invoke this script in the directory above where the elf-file tests live.") + print(" The script looks into test_dir, finds all of the elf files and then runs the simulator") + print(" with each elf file.") + print("") + print(" Output logs are put into [dir]/.cout (for C sim).") + print("") + print(" Some tests require specific command line switches to properly run. To add these") + print(" command line switches, you must use the '--test_switch_pyfile=' switch.") + print("") + print(" options:") + print(" -h --help -u -usage print out help/usage message") + print(" -o/--outfile= name of xml tests results file to be generated. default: ./tests.xml ") + print(" --32bit=[yes|y|no|n] run 32-bit tests. default: yes") + print(" --64bit=[yes|y|no|n] run 64-bit tests. default: yes") + print(" --c_sim=[yes|y|no|n] run the C simulator. default: yes") + print(" --sailcov=[yes|y|no|n] compile and run to get Sail model coverage. default: no. ") + print(" NOTE: sets 'clean_build' to yes. Coverage is gathered seperately for ") + print(" 32 and 64 bit models") + print(" --clean_build=[yes|y|no|n] do a 'make clean' before running 32/64/c_sim set of tests. default: yes") + print(" --test_dir= directory where test elf files live. default: ./isa ./riscv-tests") + print(" --test_switch_pyfile= a python fragment file that allows the user to pass in command line switches to the") + print(" riscv_sim command on a per-test basis. The format of the file should be:") + print(" sim_test_command_line_switch_dict = {") + print(" \"\" : \" [ ...] \",") + print(" \"\" : \" [ ...] \",") + print(" }") + print(" --test_ignore_pyfile= contains a tuple (immutable list) of tests to be ignored. The format of the file should be:") + print(" ignore_test_tuple = [") + print(" \"\",") + print(" \"\",") + print(" ]") + print(" -d,--debug turn on debug output") + +def process_command_line_args(opts) : + global xml_outfile + global run_32bit_tests + global run_32bit_tests + global run_64bit_tests + global run_csim + global sailcov + global clean_build + global test_dir_list + global test_switch_pyfile + global test_ignore_pyfile + global debug + + for opt, arg in opts : + if opt in ('-h', '--help', '-u', '--usage') : + print_usage(sys.argv[0]) + sys.exit(0) + elif opt in ('-o', "--outfile") : + xml_outfile = arg + elif opt in ('--32bit') : + if arg in ('yes', 'y') : + run_32bit_tests = True + elif arg in ('no', 'n') : + run_32bit_tests = False + else : + fatal_print("invalid argument to '--32bit' switch: " + arg) + elif opt in ('--64bit') : + if arg in ('yes', 'y') : + run_64bit_tests = True + elif arg in ('no', 'n') : + run_64bit_tests = False + else : + fatal_print("invalid argument to '--64bit' switch: " + arg) + elif opt in ('--c_sim') : + if arg in ('yes', 'y') : + run_csim = True + elif arg in ('no', 'n') : + run_csim = False + else : + fatal_print("invalid argument to '--run_csim' switch: " + arg) + elif opt in ('--sailcov') : + if arg in ('yes', 'y') : + sailcov = True + elif arg in ('no', 'n') : + sailcov = False + else : + fatal_print("invalid argument to '--sailcov' switch: " + arg) + sys.exit(1) + elif opt in ('--clean_build') : + if arg in ('yes', 'y') : + clean_build = True + elif arg in ('no', 'n') : + clean_build = False + else : + fatal_print("invalid argument to '--clean_build' switch: " + arg) + sys.exit(1) + elif opt in ('--test_dir') : + if not os.path.exists(arg) : + fatal_print("test_dir path, '" + arg + "', does not exist") + test_dir_list = [] + test_dir_list.append(arg) + elif opt in ('--test_switch_pyfile') : + if not os.path.isfile(arg) : + fatal_print("--test_switch_pyfile argument error. file, '" + arg + "', does not exist") + test_switch_pyfile = arg + elif opt in ('--test_ignore_pyfile') : + if not os.path.isfile(arg) : + fatal_print("--test_ignore_pyfile argument error. file, '" + arg + "', does not exist") + sys.exit(1) + test_ignore_pyfile = arg + elif opt in ('-d', '--debug') : + debug = True + else : + fatal_print("unexpected command line option: " + opt) + +# print_optional_settings AFTER the inmplicit overrides have happened +def print_optional_settings() : + global xml_outfile + global run_32bit_tests + global run_32bit_tests + global run_64bit_tests + global run_csim + global sailcov + global clean_build + global test_dir_list + global test_switch_pyfile + global test_ignore_pyfile + global debug + + print('================================================================') + print('Run time variable settings: ') + print(' {:32}'.format('debug: ') + str(debug)) + print(' {:32}'.format('outfile: ') + xml_outfile) + print(' {:32}'.format('run_32bit_tests: ') + str(run_32bit_tests)) + print(' {:32}'.format('run_64bit_tests: ') + str(run_64bit_tests)) + print(' {:32}'.format('run_csim: ') + str(run_csim)) + print(' {:32}'.format('sailcov: ') + str(sailcov)) + print(' {:32}'.format('clean_build: ') + str(clean_build)) + print(' {:32}'.format('test_dir_list: ') + str(test_dir_list)) + print(' {:32}'.format('test_ignore_pyfile: ') + test_ignore_pyfile) + print(' {:32}'.format('ignore_test: ') + str(my_ignore_test_tuple)) + print(' {:32}'.format('test_switch_pyfile: ') + test_switch_pyfile) + print(' {:32}'.format('sim_test_comand_line_switch: ') + str(sim_test_command_line_switch)) + print('================================================================') + +# Support for command line options +# ==================================== + +# ==================================== +# Functions from run_tests.sh +def green(test_str, ok_fail_str) : + global test_pass + global SUITE_XML + global GREEN + global NC + test_pass += 1 + print(test_str + ':' + GREEN + ok_fail_str + NC) + SUITE_XML += ' \n' + +def yellow(test_str, ok_fail_str) : + global test_fail + global SUITE_XML + global YELLOW + global NC + test_fail += 1 + print(test_str + ':' + YELLOW + ok_fail_str + NC) + SUITE_XML += ' \n ' + ok_fail_str + '\n \n' + +def red(test_str, ok_fail_str) : + global test_fail + global SUITE_XML + global RED + global NC + test_fail += 1 + print(test_str + ':' + RED + ok_fail_str + NC) + SUITE_XML += ' \n ' + ok_fail_str + '\n \n' + +def finish_suite(suite_name) : + global test_pass + global test_fail + global all_pass + global all_fail + global SUITE_XML + global SUITES_XML + + print(suite_name + ': Passed ' + str(test_pass) + ' out of ' + str(test_pass + test_fail) + '\n\n') + date_tmp = subprocess.check_output("date", shell=True, text=True) + date = date_tmp.rstrip() + SUITES_XML += ' \n' + SUITE_XML + ' \n' + SUITE_XML="" + all_pass += test_pass + all_fail += test_fail + test_pass = 0 + test_fail = 0 +# Functions from run_tests.sh +# ==================================== + + +# ==================================== +# Functions for determining file types +# +# TODO: there MUST be an equivalent to the 'file' command in python. +# Replace the 'file' command with a python equivalent. + +def is_elf(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $1 } ' | grep -q \"ELF\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_32bit(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $1 } ' | grep -q \"32-bit\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_64bit(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $1 } ' | grep -q \"64-bit\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_riscv(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $2 } ' | grep -q \"RISC-V\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_riscv_elf(filename) : + return is_elf(filename) and is_riscv(filename) + +def is_riscv_elf_32(filename) : + return is_riscv_elf(filename) and is_32bit(filename) + +def is_riscv_elf_64(filename) : + return is_riscv_elf(filename) and is_64bit(filename) + +def ignore_test(testname) : + for t in my_ignore_test_tuple : + debug_print("ignore testname: " + os.path.basename(testname) + " t: " + t) + if t == os.path.basename(testname) : + return True + else : + continue + return False +# Functions for determining file types +# ==================================== + +# Function prototypes +# ===========================================================================79 + +# ===========================================================================79 +# Start of execution.... + +debug_print("starting...") +debug_print("abspath to this script: " + os.path.abspath(sys.argv[0])) +debug_print("opts: " + str(opts)) + +process_command_line_args(opts) + +# ==================================== +# Implicit overrides of program varaibles +if sailcov : + clean_build = True + +if test_switch_pyfile : + exec(open(test_switch_pyfile).read()) + # check to see if variable set + if 'sim_test_command_line_switch_dict' in locals() : + sim_test_command_line_switch = sim_test_command_line_switch_dict; + else : + fatal_print("the python variable, sim_test_command_line_switch_dict, is not properly set in " + test_switch_pyfile) + sys.exit(1) + +if test_ignore_pyfile : + exec(open(test_ignore_pyfile).read()) + if 'ignore_test_tuple' in locals() : + my_ignore_test_tuple = ignore_test_tuple + else : + fatal_print("the python variable, ignore_test_tuple, is not properly set in " + test_ignore_pyfile) + +# Debug print out of important program variables +if debug : + print_optional_settings() + +# TODO: check that only 1 dir in test_dir_list exists +for d in test_dir_list : + if os.path.exists(d) : + TESTDIR = d + else : + pass + +debug_print('TESTDIR : ' + TESTDIR) + +# DIR points to the invocation directory. +DIR = os.getcwd() +SEARCH_DIR = DIR +while SEARCH_DIR != '/' : + if os.path.isfile(SEARCH_DIR + '/' + sail_riscv_rootdir) : + RISCVDIR = SEARCH_DIR + break + if SEARCH_DIR == '/' : + fatal_print("can't find root directory of repository") + SEARCH_DIR = os.path.dirname(SEARCH_DIR) +debug_print("RISCVDIR: " + RISCVDIR) + +if sailcov : + MAKE_SAILCOV = "SAILCOV=true" +else : + MAKE_SAILCOV = "" + +if os.path.isfile(DIR + xml_outfile) != False : + os.remove(DIR + xml_outfile) + +# TODO: Do you really want to run the tests from the RISCVDIR? +# TODO: check for success/failure of chdir +os.chdir(RISCVDIR) + +debug_print("DIR + '/' + TESTDIR + '/' + * :" + DIR + '/' + TESTDIR + '/' + "*") + +# Do 'make clean' to avoid cross-arch pollution. + +if clean_build : + cmd = "make ARCH=RV32 clean" + ret_val = os.system(cmd) + if ret_val != 0 : + fatal_print("non-zero exit value from command: '" + cmd + "'") + else : + pass +else : + pass + +if clean_build : + cmd = "make ARCH=RV32 clean" + ret_val = os.system(cmd) + if ret_val != 0 : + fatal_print("non-zero exit value from command: '" + cmd + "'") + sys.exit(1) + else : + pass +else : + pass + + +print("Building 32-bit RISCV specification...") +if run_csim : + if run_32bit_tests : + cmd = "make ARCH=RV32 " + MAKE_SAILCOV + " c_emulator/riscv_sim_RV32" + ret_val = os.system(cmd) + if ret_val == 0 : + green("Building 32-bit RISCV C emulator", "ok") + else : + red("Building 32-bit RISCV C emulator","fail") + error_print("non-zero exit value from command: '" + cmd + "'") + +if run_32bit_tests and run_csim : + for test in glob.glob(DIR + '/' + TESTDIR + '/' + "*") : + if not is_riscv_elf_32(test) : + continue + if ignore_test(test) : + debug_print("ignoring test: " + test) + continue + outfile = test + ".cout" + sim_switch = "" + for key in sim_test_command_line_switch : + pat = re.compile(key) + mo = pat.search(test) + if mo != None: + sim_switch = sim_test_command_line_switch[key] + break + + if sailcov : + run_sailcov = " --sailcov-file sailcov_RV32" + else : + run_sailcov = "" + + cmd = "timeout 5 " + RISCVDIR + "/c_emulator/riscv_sim_RV32" + run_sailcov + " " + sim_switch + " " + test + " > " + outfile + " 2>&1 && grep -q SUCCESS " + outfile + debug_print("cmd: '" + cmd + "'") + ret_val = os.system(cmd) + if ret_val == 0 : + green("C-32 " + os.path.basename(test), "ok") + else : + red("C-32 " + os.path.basename(test), "fail") +else : + pass + +finish_suite("32-bit RISCV C-simulator tests") + +if clean_build : + cmd = "make ARCH=RV64 clean" + ret_val = os.system(cmd) + if ret_val != 0 : + fatal_print("non-zero exit value from command: '" + cmd + "'") + else : + pass +else : + pass + +print("Building 64-bit RISCV specification...") + +if clean_build : + cmd = "make ARCH=RV64 clean" + ret_val = os.system(cmd) + if ret_val != 0 : + fatal_print("non-zero exit value from command: '" + cmd + "'") + else : + pass +else : + pass + +print("Building 64-bit RISCV specification...") +if run_csim : + if run_64bit_tests : + cmd = "make ARCH=RV64 " + MAKE_SAILCOV + " c_emulator/riscv_sim_RV64" + ret_val = os.system(cmd) + if ret_val == 0 : + green("Building 64-bit RISCV C emulator", "ok") + else : + red("Building 64-bit RISCV C emulator","fail") + error_print("non-zero exit value from command: '" + cmd + "'") + +if run_64bit_tests and run_csim : + for test in glob.glob(DIR + '/' + TESTDIR + '/' + "*") : + debug_print("test: " + test) + if not is_riscv_elf_64(test) : + continue + if ignore_test(test) : + debug_print("ignoring test: " + test) + continue + outfile = test + ".cout" + sim_switch = "" + for key in sim_test_command_line_switch : + pat = re.compile(key) + mo = pat.search(test) + if mo != None: + sim_switch = sim_test_command_line_switch[key] + break + + if sailcov : + run_sailcov = " --sailcov-file sailcov_RV64" + else : + run_sailcov = "" + + cmd = "timeout 5 " + RISCVDIR + "/c_emulator/riscv_sim_RV64" + run_sailcov + " " + sim_switch + " " + test + " > " + outfile + " 2>&1 && grep -q SUCCESS " + outfile + ret_val = os.system(cmd) + if ret_val == 0 : + green("C-64 " + os.path.basename(test), "ok") + else : + red("C-64 " + os.path.basename(test), "fail") +else : + pass + +finish_suite("64-bit RISCV C-simulator tests") + +print('Passed ' + str(all_pass) + ' out of ' + str(all_pass + all_fail) + '\n\n') +XML = '\n' + SUITES_XML + '\n' + +xml_outfile_fh = open(DIR + '/' + xml_outfile, 'w') +print(XML, file = xml_outfile_fh) +xml_outfile_fh.close() + +if all_fail > 0 : + sys.exit(1) +else : + sys.exit(0)