Implement tests

.github/workflows/compile.yml

@@ -11,7 +11,7 @@ jobs:
     - name: Check out repository code
       uses: actions/checkout@HEAD
       with:
-        submodules: true
+        submodules: recursive
     - name: Ensure pre-commit checks pass
       run: python3 -m pip install pre-commit && pre-commit run --all-files --show-diff-on-failure --color=always
     - name: Install sail from binary
@@ -22,13 +22,19 @@ jobs:
       run: test/run_tests.sh
     - name: Upload test results
       if: always()
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v3
       with:
         name: tests.xml
         path: test/tests.xml
+    - name: Upload test results (new)
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: tests_new.xml
+        path: TEST_DIR_ROOT/riscv_tests.git.subtree/tests_new.xml
     - name: Upload event payload
       if: always()
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v3
       with:
         name: event.json
         path: ${{ github.event_path }}

.github/workflows/compile_new_test.yml

@@ -0,0 +1,61 @@
+name: CI
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Install packages
+      run: sudo apt install -y opam zlib1g-dev pkg-config libgmp-dev z3 device-tree-compiler
+    - name: Check out repository code
+      uses: actions/checkout@HEAD
+      with:
+        submodules: recursive
+    - name: Ensure pre-commit checks pass
+      run: pip install pre-commit && pre-commit run --all-files --show-diff-on-failure --color=always
+    - name: Init opam
+      run: opam init --disable-sandboxing -y
+    - name: Install sail
+      run: opam install -y sail
+    - name: Download RISC-V toolchain (32-bit)
+      run: wget -c https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2024.04.12/riscv32-elf-ubuntu-22.04-gcc-nightly-2024.04.12-nightly.tar.gz
+    - name: Install RISC-V toolchain (32-bit)
+      run: sudo tar xvfz riscv32-elf-ubuntu-22.04-gcc-nightly-2024.04.12-nightly.tar.gz --directory /opt
+    - name: Download RISC-V toolchain (64-bit)
+      run: wget -c https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2024.04.12/riscv64-elf-ubuntu-22.04-gcc-nightly-2024.04.12-nightly.tar.gz
+    - name: Install RISC-V toolchain (64-bit)
+      run: sudo tar xvfz riscv64-elf-ubuntu-22.04-gcc-nightly-2024.04.12-nightly.tar.gz --directory /opt
+    - name: Build RISC-V testsuite
+      run: |
+        cd TEST_DIR_ROOT/riscv_tests.git.subtree ; \
+        echo ${PWD} ; ls ; \
+        autoconf ;  \
+        ./configure --prefix=${PWD}/target ; \
+        export PATH=/opt/riscv/bin:${PATH} ; \
+        make isa ;
+    - name: Build and test simulators
+      run: eval $(opam env) && test/run_tests.sh
+    - name: Build and test simulators (new method)
+      run: |
+        eval $(opam env) ; \
+        cd TEST_DIR_ROOT/riscv_tests.git.subtree ; \
+        ../../bin/run_tests.py --outfile=tests_new.xml --32bit=yes --64bit=yes --c_sim=yes --sailcov=no --clean_build=yes --test_switch_pyfile=test_command_line_switch.py --test_ignore_pyfile=test_ignore_list.py
+    - name: Upload test results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: tests.xml
+        path: test/tests.xml
+    - name: Upload test results (new)
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: tests_new.xml
+        path: TEST_DIR_ROOT/riscv_tests.git.subtree/tests_new.xml
+    - name: Upload event payload
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: event.json
+        path: ${{ github.event_path }}

.github/workflows/test-results.yml

@@ -22,7 +22,7 @@ jobs:
              run_id: ${{github.event.workflow_run.id }},
           });
           var matchArtifacts = artifacts.data.artifacts.filter((artifact) => {
-            return artifact.name == 'tests.xml' || artifact.name == 'event.json'
+            return artifact.name == 'tests.xml' || 'tests_new.xml' || artifact.name == 'event.json'
           });
           var count = matchArtifacts.length;
           for (var i = 0; i < count; i++) {
@@ -40,6 +40,8 @@ jobs:
           }
     - name: Extract test results
       run: unzip tests.xml.zip
+    - name: Extract test results (new)
+      run: unzip tests_new.xml.zip
     - name: Extract event payload
       run: unzip event.json.zip
     - name: Publish test results

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
-exclude: '^(prover_snapshots)|(generated_definitions)|(c_emulator/SoftFloat-3e)'
+exclude: '^(prover_snapshots)|(generated_definitions)|(c_emulator/SoftFloat-3e)|(TEST_DIR_ROOT/riscv_tests.git.subtree)'
 minimum_pre_commit_version: 2.10.0
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks

HOWTO_add_new_tests.txt

@@ -0,0 +1,91 @@
+2024-05-28  Bill McSpadden
+
+-.0 The following was done to create the subtree(s).  This was then committed to
+    sail-riscv/<branch>
+
+    > git subtree add --prefix TEST_DIR_ROOT/riscv_tests.git.subtree     [email protected]:riscv-software-src/riscv-tests.git riscv-tests-sail    [--squash]
+
+# TODO: request sent to Andrew to be made contributor to the riscv-test-env repo
+#       DONE
+# TODO: the TEST_DIR_ROOT/riscv_tests.git.subtree/env exists when riscv-tests subtree
+#       is added.  And this leads to the error, "prefix 'TEST_DIR_ROOT/riscv_tests.git.subtree/env' already exists."
+#       So, I need figure out something. The path to "env" is needed by riscv-tests
+#       build process, so this might have an impact on the riscv-tests build method.
+#       THE FIX: removed env/ on the branch, 'riscv-tests-sail'.  Then perform the
+#       following command:
+    > git subtree add --prefix TEST_DIR_ROOT/riscv_tests.git.subtree/env [email protected]:riscv/riscv-test-env.git           riscv-test-env-sail [--squash]
+
+##
+# ... In the meantime, use the master branch
+    > git subtree add --prefix TEST_DIR_ROOT/riscv_tests.git.subtree/env [email protected]:riscv/riscv-test-env.git            master              [--squash]
+
+0. Check out the sail-riscv repo
+
+0.1  Point to remote
+    This will persist within your working repository.  But when you checkout
+    another repo,  you'll need to execute this command from within the new repo.
+    TODO:  is there a way to do this with git hooks?
+
+    > git remote add riscv-tests-sail-remote      [email protected]:riscv-software-src/riscv-tests.git
+    > git remote add riscv-tests-env-sail-remote  https://github.com/riscv/riscv-test-env.git
+
+
+1.0 Go to test directory.  Add new tests/files
+
+    > cd TEST_DIR_ROOT/riscv_tests.git.subtree
+    > autoconf
+    > ./configure --prefix $PWD/target
+    > make isa
+
+1.1 Make sure the new tests run against your model
+
+1.1.1  Individual test
+
+    > cd TEST_DIR_ROOT/riscv_tests.git.subtree
+    > ../../c_emulator/riscv_sim_RV[32|64] [options] isa/<elf_file>
+
+
+1.1.2  Suite of tests
+
+    > cd TEST_DIR_ROOT/riscv_tests.git.subtree
+    > ../../bin/run_tests.py --32bit=yes --64bit=yes --c_sim=yes --test_switch_pyfile=test_command_line_switch.py --test_ignore_pyfile=test_ignore_list.py
+
+
+2.0 Push your new tests/files back to the riscv-tests repository (branch: riscv-tests-sail)
+
+    > cd <sail-riscv-root>
+    > git subtree push --prefix TEST_DIR_ROOT/riscv_tests.git.subtree/ riscv-tests-sail-remote riscv-tests-sail
+
+    This should put your changes into the riscv-tests repo under the riscv-tests-sail branch.
+    Steer your browser to ...
+
+        https://github.com/riscv-software-src/riscv-tests/tree/riscv-tests-sail
+
+    ... and make sure your files were properly committed to the branch.
+
+    MAKE SURE YOUR CHANGES DID NOT GO ONTO 'master'.  If they did,  back out your last push.
+
+2.1 If you made any changes to TEST_DIR_ROOT/riscv_tests.git.subtree/env/, commit them and push
+    to the remote repo.
+
+3.0  Merging from remote riscv-tests/master into local repo
+
+    Note :  When I did the following command ...
+
+        > git subtree pull -P TEST_DIR_ROOT/riscv_tests.git.subtree/ riscv-tests-sail-remote master
+
+    ... I got the new code , but git assumed that I was going to commit
+    these changes and it opened a commit editor saying it was merging with
+    the following message:
+
+        Merge commit '408e461da11e0b298c4b69e587729532787212f5' into new_test_3
+
+    I tried to 'q!' out of my vim editor (assuming that the commit would not
+    be made),  but the commit was made with the default commit message.
+
+    The commit was made onto the sail-riscv branch,  NOT onto the riscv-tests
+    branch.
+
+3.1 Run tests to make sure new updates from riscv-tests/master work as expected.
+    Then commit the changes (if needed).
+    Then generate a PR for these new changes

SAIL_RISCV_ROOTDIR

@@ -0,0 +1,5 @@
+This file is created in the root directory for the RISC-V Sail model.
+It is used as a reference point for various tools within the repo.
+
+Do not rename it.  Do not move it.  Do not duplicate it anywhere in
+the repo,  else tooling may get confused.

TESTING.md

@@ -0,0 +1,90 @@
+# Testing the RISCV-Sail model
+
+This document contains information regarding the testing of the RISC-V
+Sail model.
+
+There are several goals for the testing effort of the RISC-V Sail model.
+First, we need a set of tests that get run during the CI (Continuous Improvement)
+cycle when merging PRs to the main branch.  We will refer to these as Build Verification Tests
+(BVTs). These should cover basic instruction behavior.  The set of tests should run
+rather quickly (perhaps 30 minutes) so that the CI does not bog down.
+
+Second, we want to take Architectural Compatability Tests (ACTs) and run them
+against the Sail model.  However, many of the ACTs are not self-checking and therefore
+cannot be used directly for validating the functionality of the model.  But we can
+run the Sail model and cross-check with the Spike model using signature checks.
+
+Third,  for all of these test methods,  we want to be able to make coverage measurements
+of the Sail code.  This will give us some objective insights into where we have coverage
+holes in our testing.
+
+## Background
+
+For the longest time,  the set of tests used to validate changes to the
+model,  were a set of precompiled .elf files (along with their .dump file counterparts)
+that were stored in the repo under `test/riscv-tests`.  The scripts used to
+run theses tests (and to gather test results) were `test/run_tests.sh` and
+`test/run_fp_tests.sh`.
+
+These tests are a compiled snapshot of tests that can be found at
+https://github.com/riscv-software-src/riscv-tests
+that date back to 2019.
+
+This methodolgy was defecient in several ways.
+1. Original test source is difficult to track down.
+1. Storing compiled code in a git repo is usually frowned upon.
+1. There is no easy way to add new tests to the repo when you add a new feature.
+1. `run_tests.sh` is difficult to enhance with new features.
+
+We anticipate that the `test/` directory will be removed once a more robust
+testing methodolgy is put in place.  (See next section.)
+
+## Adding new tests
+
+
+
+To fix the defeciencies of the old test methodology,  we have done the
+following:
+1. Created a new test directory at the repo root, `TEST_DIR_ROOT/` under which
+all new test collateral will be put.
+1. Created a `bin/` directory under which various model scripts and executables
+are added.
+1. Installed https://github.com/riscv-software-src/riscv-tests as a subtree
+at `TEST_DIR_ROOT/riscv-tests.git/`.  We will be working on  a special branch
+in this repository: `riscv-tests-sail`.  This allows us to add tests onto our
+branch.  And we can incorporate new tests into our testsuite as they appear
+in the riscv-tests repo (by merging these new tests from the master branch
+onto our branch).
+1. Re-wrote `tests/run_tests.sh` in python and added run-time switches, the main
+purpose of which was to be able to add command line switches to the execution of
+particular tests. See the script, `bin/run_tests.py`, for execution parameters.
+1. Updated `.github/workflows/compile.yml` to make use of the new run_tests python
+script.
+
+
+
+## Future Plans
+
+### Sail and Spike Crosschecking with the Architecture Compatability Tests (ACTS)
+
+We intend to run the ACTs on both Sail and Spike. Test signatures will be compared
+to check that the two simulators agree.
+
+### Fixing defecincies in the test environment
+
+We have the following defeciencies in the test environment that need to be fixed:
+
+1. A pass/fail can only be detected and reported from within the test itself.
+If a test writer wanted to check the simluator log file to see
+if certain strings existed (say, for example, you want to check the disassembly
+of newly added instructions),  there is no method to do so.  The ability to
+inspect the log file is a neccessary feature that needs to be added.
+
+1. Negative testing. We need to be able to check for proper detection of errors
+which would then mean that the test "passed".  For example,  we might want to check
+that if the vector extension is not enabled, that a test that uses a vector instruction
+would "fail".
+
+1. Random test environment.  We would like to add a directed random test environment
+that allows us to hit unforeseen boundary conditions.  We would like to randomize
+both the instruction sequences as well as the model configuration.