Improve CI

Test with jit-no-reg-alloc

Signed-off-by: Laszlo Voros <[email protected]>

.github/workflows/actions.yml

@@ -89,8 +89,10 @@ jobs:
         rm $GITHUB_WORKSPACE/test/wasm-spec/core/call_indirect.wast
         $RUNNER --engine="$GITHUB_WORKSPACE/out/clang/x86/walrus"
         $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/clang/x64/walrus"
-        $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/pure/x86/walrus" basic-tests wasm-test-core jit
+        $RUNNER --no-reg-alloc --engine="$GITHUB_WORKSPACE/out/clang/x64/walrus"
         $RUNNER --engine="$GITHUB_WORKSPACE/out/pure/x64/walrus" basic-tests wasm-test-core jit
+        $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/pure/x86/walrus" basic-tests wasm-test-core jit
+        $RUNNER --no-reg-alloc --engine="$GITHUB_WORKSPACE/out/pure/x86/walrus" basic-tests wasm-test-core jit
 
   build-test-on-x86:
     runs-on: ubuntu-latest
@@ -112,6 +114,7 @@ jobs:
       run: |
           $RUNNER --engine="$GITHUB_WORKSPACE/out/linux/x86/walrus"
           $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/linux/x86/walrus"
+          $RUNNER --no-reg-alloc --engine="$GITHUB_WORKSPACE/out/linux/x86/walrus"
 
   build-test-on-x64:
     runs-on: ubuntu-latest
@@ -133,6 +136,7 @@ jobs:
       run: |
          $RUNNER --engine="$GITHUB_WORKSPACE/out/linux/x64/walrus"
          $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/linux/x64/walrus"
+         $RUNNER --no-reg-alloc --engine="$GITHUB_WORKSPACE/out/linux/x64/walrus"
 
   build-on-x64-with-perf:
           runs-on: ubuntu-latest
@@ -180,8 +184,10 @@ jobs:
           rm ./test/wasm-spec/core/call_indirect.wast
           python3 ./tools/run-tests.py --engine="./out/debug/walrus"
           python3 ./tools/run-tests.py --jit --engine="./out/debug/walrus"
+          python3 ./tools/run-tests.py --no-reg-alloc --engine="./out/debug/walrus"
           python3 ./tools/run-tests.py --engine="./out/pure/walrus" basic-tests wasm-test-core jit
           python3 ./tools/run-tests.py --jit --engine="./out/pure/walrus" basic-tests wasm-test-core jit
+          python3 ./tools/run-tests.py --no-reg-alloc --engine="./out/pure/walrus" basic-tests wasm-test-core jit
 
   build-test-on-aarch64:
     runs-on: ubuntu-latest
@@ -209,8 +215,10 @@ jobs:
           ninja -Cout/pure
           python3 ./tools/run-tests.py --engine="./out/release/walrus"
           python3 ./tools/run-tests.py --jit --engine="./out/release/walrus"
+          python3 ./tools/run-tests.py --no-reg-alloc --engine="./out/release/walrus"
           python3 ./tools/run-tests.py --engine="./out/pure/walrus" basic-tests wasm-test-core jit
           python3 ./tools/run-tests.py --jit --engine="./out/pure/walrus" basic-tests wasm-test-core jit
+          python3 ./tools/run-tests.py --no-reg-alloc --engine="./out/pure/walrus" basic-tests wasm-test-core jit
 
   test-on-windows-x86-x64:
     runs-on: windows-2022
@@ -272,7 +280,7 @@ jobs:
       run: |
         $RUNNER --engine="$GITHUB_WORKSPACE/out/extended/walrus" wasm-test-extended
 
-  build-test-performance:
+  build-test-performance-x64:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
@@ -295,6 +303,83 @@ jobs:
       run: |
         test/wasmBenchmarker/benchmark.py --engines $GITHUB_WORKSPACE/out/linux/x64/walrus --iterations 2 --verbose --summary --results i j2i n2i j n2j
 
+  build-test-performance-x86:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Install Packages
+      run: |
+        sudo apt update
+        sudo apt install -y ninja-build gcc-multilib g++-multilib
+        sudo pip install pandas
+        sudo pip install py-markdown-table
+        sudo pip install tqdm
+    - name: Build x86
+      env:
+        BUILD_OPTIONS: -DWALRUS_ARCH=x86 -DWALRUS_HOST=linux -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja
+      run: |
+        cmake -H. -Bout/linux/x86 $BUILD_OPTIONS
+        ninja -Cout/linux/x86
+    - name: Run Tests
+      run: |
+        test/wasmBenchmarker/benchmark.py --engines $GITHUB_WORKSPACE/out/linux/x86/walrus --iterations 2 --verbose --summary --results i j2i n2i j n2j
+
+  build-test-performance-armv7:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Build in arm32 container
+      uses: uraimo/[email protected]
+      with:
+        arch: armv7
+        distro: ubuntu_latest
+
+        # Install deps into the container. With the token, the container will be cached
+        # The image is cached publically like a package
+        githubToken: ${{ github.token }}
+
+        install: |
+          apt-get update
+          apt-get install -y cmake build-essential ninja-build pkg-config python3 clang git
+          pip install pandas
+          pip install py-markdown-table
+          pip install tqdm
+        run: |
+          CC=clang CXX=clang++ cmake -H. -Bout/release -DWALRUS_ARCH=arm -DWALRUS_HOST=linux -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja
+          ninja -Cout/release
+          test/wasmBenchmarker/benchmark.py --engines $GITHUB_WORKSPACE/out/release/walrus --iterations 2 --verbose --summary --results i j2i n2i j n2j
+
+  build-test-performance-aarch64:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Build in arm64 container
+      uses: uraimo/[email protected]
+      with:
+        arch: aarch64
+        distro: ubuntu22.04
+
+        # Install deps into the container. With the token, the container will be cached
+        # The image is cached publically like a package
+        githubToken: ${{ github.token }}
+
+        install: |
+          apt-get update
+          apt-get install -y cmake build-essential ninja-build pkg-config python3 clang git
+          pip install pandas
+          pip install py-markdown-table
+          pip install tqdm
+        run: |
+          CC=clang CXX=clang++ cmake -H. -Bout/release -DWALRUS_ARCH=aarch64 -DWALRUS_HOST=linux -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja
+          ninja -Cout/release
+          test/wasmBenchmarker/benchmark.py --engines $GITHUB_WORKSPACE/out/release/walrus --iterations 2 --verbose --summary --results i j2i n2i j n2j
+
   built-test-wasm-c-api:
     runs-on: ubuntu-latest
     steps:

tools/run-tests.py

@@ -47,6 +47,7 @@
 DEFAULT_RUNNERS = []
 JIT_EXCLUDE_FILES = []
 jit = False
+jit_no_reg_alloc = False
 
 
 class runner(object):
@@ -64,12 +65,18 @@ def __call__(self, fn):
 def _run_wast_tests(engine, files, is_fail):
     fails = 0
     for file in files:
-        if jit:
-            filename = os.path.basename(file) 
+        if jit or jit_no_reg_alloc:
+            filename = os.path.basename(file)
             if filename in JIT_EXCLUDE_FILES:
                 continue
 
-        proc = Popen([engine, "--mapdirs", "./test/wasi", "/var", file], stdout=PIPE) if not jit else Popen([engine, "--mapdirs", "./test/wasi", "/var", "--jit", file], stdout=PIPE)
+        openParams = [engine, "--mapdirs", "./test/wasi", "/var", file]
+        if jit:
+            openParams = [engine, "--mapdirs", "./test/wasi", "/var", "--jit", file]
+        elif jit_no_reg_alloc:
+            openParams = [engine, "--mapdirs", "./test/wasi", "/var", "--jit", "--jit-no-reg-alloc", file]
+
+        proc = Popen(openParams, stdout=PIPE)
         out, _ = proc.communicate()
 
         if is_fail and proc.returncode or not is_fail and not proc.returncode:
@@ -178,10 +185,18 @@ def main():
     parser.add_argument('suite', metavar='SUITE', nargs='*', default=sorted(DEFAULT_RUNNERS),
                         help='test suite to run (%s; default: %s)' % (', '.join(sorted(RUNNERS.keys())), ' '.join(sorted(DEFAULT_RUNNERS))))
     parser.add_argument('--jit', action='store_true', help='test with JIT')
+    parser.add_argument('--no-reg-alloc', action='store_true', help='test with JIT without register allocation')
     args = parser.parse_args()
     global jit
     jit = args.jit
-    if jit:
+
+    global jit_no_reg_alloc
+    jit_no_reg_alloc = args.no_reg_alloc
+
+    if jit and jit_no_reg_alloc:
+        parser.error('jit and no-reg-alloc cannot be used together')
+
+    if jit or jit_no_reg_alloc:
         exclude_list_file = join(PROJECT_SOURCE_DIR, 'tools', 'jit_exclude_list.txt')
         with open(exclude_list_file) as f:
             global JIT_EXCLUDE_FILES
@@ -195,7 +210,12 @@ def main():
     success, fail = [], []
 
     for suite in args.suite:
-        print(COLOR_PURPLE + f'running test suite{ " with jit" if jit else ""}: ' + suite + COLOR_RESET)
+        text = ""
+        if jit:
+            text = " with jit"
+        elif jit_no_reg_alloc:
+            text = " with jit without register allocation"
+        print(COLOR_PURPLE + f'running test suite{text}: ' + suite + COLOR_RESET)
         try:
             RUNNERS[suite](args.engine)
             success += [suite]