diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index a883ed6f5..981db51b6 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -89,8 +89,10 @@ jobs: rm $GITHUB_WORKSPACE/test/wasm-spec/core/call_indirect.wast $RUNNER --engine="$GITHUB_WORKSPACE/out/clang/x86/walrus" $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/clang/x64/walrus" - $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/pure/x86/walrus" basic-tests wasm-test-core jit + $RUNNER --no-reg-alloc --engine="$GITHUB_WORKSPACE/out/clang/x64/walrus" $RUNNER --engine="$GITHUB_WORKSPACE/out/pure/x64/walrus" basic-tests wasm-test-core jit + $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/pure/x86/walrus" basic-tests wasm-test-core jit + $RUNNER --no-reg-alloc --engine="$GITHUB_WORKSPACE/out/pure/x86/walrus" basic-tests wasm-test-core jit build-test-on-x86: runs-on: ubuntu-latest @@ -112,6 +114,7 @@ jobs: run: | $RUNNER --engine="$GITHUB_WORKSPACE/out/linux/x86/walrus" $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/linux/x86/walrus" + $RUNNER --no-reg-alloc --engine="$GITHUB_WORKSPACE/out/linux/x86/walrus" build-test-on-x64: runs-on: ubuntu-latest @@ -133,6 +136,7 @@ jobs: run: | $RUNNER --engine="$GITHUB_WORKSPACE/out/linux/x64/walrus" $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/linux/x64/walrus" + $RUNNER --no-reg-alloc --engine="$GITHUB_WORKSPACE/out/linux/x64/walrus" build-on-x64-with-perf: runs-on: ubuntu-latest @@ -180,8 +184,10 @@ jobs: rm ./test/wasm-spec/core/call_indirect.wast python3 ./tools/run-tests.py --engine="./out/debug/walrus" python3 ./tools/run-tests.py --jit --engine="./out/debug/walrus" + python3 ./tools/run-tests.py --no-reg-alloc --engine="./out/debug/walrus" python3 ./tools/run-tests.py --engine="./out/pure/walrus" basic-tests wasm-test-core jit python3 ./tools/run-tests.py --jit --engine="./out/pure/walrus" basic-tests wasm-test-core jit + python3 ./tools/run-tests.py --no-reg-alloc --engine="./out/pure/walrus" basic-tests wasm-test-core jit build-test-on-aarch64: runs-on: ubuntu-latest @@ -209,8 +215,10 @@ jobs: ninja -Cout/pure python3 ./tools/run-tests.py --engine="./out/release/walrus" python3 ./tools/run-tests.py --jit --engine="./out/release/walrus" + python3 ./tools/run-tests.py --no-reg-alloc --engine="./out/release/walrus" python3 ./tools/run-tests.py --engine="./out/pure/walrus" basic-tests wasm-test-core jit python3 ./tools/run-tests.py --jit --engine="./out/pure/walrus" basic-tests wasm-test-core jit + python3 ./tools/run-tests.py --no-reg-alloc --engine="./out/pure/walrus" basic-tests wasm-test-core jit test-on-windows-x86-x64: runs-on: windows-2022 @@ -272,7 +280,7 @@ jobs: run: | $RUNNER --engine="$GITHUB_WORKSPACE/out/extended/walrus" wasm-test-extended - build-test-performance: + build-test-performance-x64: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -295,6 +303,83 @@ jobs: run: | test/wasmBenchmarker/benchmark.py --engines $GITHUB_WORKSPACE/out/linux/x64/walrus --iterations 2 --verbose --summary --results i j2i n2i j n2j + build-test-performance-x86: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Packages + run: | + sudo apt update + sudo apt install -y ninja-build gcc-multilib g++-multilib + sudo pip install pandas + sudo pip install py-markdown-table + sudo pip install tqdm + - name: Build x86 + env: + BUILD_OPTIONS: -DWALRUS_ARCH=x86 -DWALRUS_HOST=linux -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja + run: | + cmake -H. -Bout/linux/x86 $BUILD_OPTIONS + ninja -Cout/linux/x86 + - name: Run Tests + run: | + test/wasmBenchmarker/benchmark.py --engines $GITHUB_WORKSPACE/out/linux/x86/walrus --iterations 2 --verbose --summary --results i j2i n2i j n2j + + build-test-performance-armv7: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Build in arm32 container + uses: uraimo/run-on-arch-action@v2.7.2 + with: + arch: armv7 + distro: ubuntu_latest + + # Install deps into the container. With the token, the container will be cached + # The image is cached publically like a package + githubToken: ${{ github.token }} + + install: | + apt-get update + apt-get install -y cmake build-essential ninja-build pkg-config python3 clang git + pip install pandas + pip install py-markdown-table + pip install tqdm + run: | + CC=clang CXX=clang++ cmake -H. -Bout/release -DWALRUS_ARCH=arm -DWALRUS_HOST=linux -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja + ninja -Cout/release + test/wasmBenchmarker/benchmark.py --engines $GITHUB_WORKSPACE/out/release/walrus --iterations 2 --verbose --summary --results i j2i n2i j n2j + + build-test-performance-aarch64: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Build in arm64 container + uses: uraimo/run-on-arch-action@v2.7.2 + with: + arch: aarch64 + distro: ubuntu22.04 + + # Install deps into the container. With the token, the container will be cached + # The image is cached publically like a package + githubToken: ${{ github.token }} + + install: | + apt-get update + apt-get install -y cmake build-essential ninja-build pkg-config python3 clang git + pip install pandas + pip install py-markdown-table + pip install tqdm + run: | + CC=clang CXX=clang++ cmake -H. -Bout/release -DWALRUS_ARCH=aarch64 -DWALRUS_HOST=linux -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja + ninja -Cout/release + test/wasmBenchmarker/benchmark.py --engines $GITHUB_WORKSPACE/out/release/walrus --iterations 2 --verbose --summary --results i j2i n2i j n2j + built-test-wasm-c-api: runs-on: ubuntu-latest steps: diff --git a/tools/run-tests.py b/tools/run-tests.py index 00767f94f..de5227a38 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -47,6 +47,7 @@ DEFAULT_RUNNERS = [] JIT_EXCLUDE_FILES = [] jit = False +jit_no_reg_alloc = False class runner(object): @@ -64,12 +65,18 @@ def __call__(self, fn): def _run_wast_tests(engine, files, is_fail): fails = 0 for file in files: - if jit: - filename = os.path.basename(file) + if jit or jit_no_reg_alloc: + filename = os.path.basename(file) if filename in JIT_EXCLUDE_FILES: continue - proc = Popen([engine, "--mapdirs", "./test/wasi", "/var", file], stdout=PIPE) if not jit else Popen([engine, "--mapdirs", "./test/wasi", "/var", "--jit", file], stdout=PIPE) + openParams = [engine, "--mapdirs", "./test/wasi", "/var", file] + if jit: + openParams = [engine, "--mapdirs", "./test/wasi", "/var", "--jit", file] + elif jit_no_reg_alloc: + openParams = [engine, "--mapdirs", "./test/wasi", "/var", "--jit", "--jit-no-reg-alloc", file] + + proc = Popen(openParams, stdout=PIPE) out, _ = proc.communicate() if is_fail and proc.returncode or not is_fail and not proc.returncode: @@ -178,10 +185,18 @@ def main(): parser.add_argument('suite', metavar='SUITE', nargs='*', default=sorted(DEFAULT_RUNNERS), help='test suite to run (%s; default: %s)' % (', '.join(sorted(RUNNERS.keys())), ' '.join(sorted(DEFAULT_RUNNERS)))) parser.add_argument('--jit', action='store_true', help='test with JIT') + parser.add_argument('--no-reg-alloc', action='store_true', help='test with JIT without register allocation') args = parser.parse_args() global jit jit = args.jit - if jit: + + global jit_no_reg_alloc + jit_no_reg_alloc = args.no_reg_alloc + + if jit and jit_no_reg_alloc: + parser.error('jit and no-reg-alloc cannot be used together') + + if jit or jit_no_reg_alloc: exclude_list_file = join(PROJECT_SOURCE_DIR, 'tools', 'jit_exclude_list.txt') with open(exclude_list_file) as f: global JIT_EXCLUDE_FILES @@ -195,7 +210,12 @@ def main(): success, fail = [], [] for suite in args.suite: - print(COLOR_PURPLE + f'running test suite{ " with jit" if jit else ""}: ' + suite + COLOR_RESET) + text = "" + if jit: + text = " with jit" + elif jit_no_reg_alloc: + text = " with jit without register allocation" + print(COLOR_PURPLE + f'running test suite{text}: ' + suite + COLOR_RESET) try: RUNNERS[suite](args.engine) success += [suite]