From b87a2540609786971650f854fffa54b981ea9294 Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Thu, 7 Mar 2024 17:37:31 +0100 Subject: [PATCH 1/7] bench --- .gitignore | 3 ++ bench/.gitignore | 2 ++ bench/bench.c | 72 +++++++++++++++++++++++++++++++++++++++++++++ bench/bench.sh | 30 +++++++++++++++++++ bench/factorial.con | 9 ++++++ bench/factorial.rs | 8 +++++ bench/fib.con | 9 ++++++ bench/fib.rs | 8 +++++ 8 files changed, 141 insertions(+) create mode 100644 bench/.gitignore create mode 100644 bench/bench.c create mode 100755 bench/bench.sh create mode 100644 bench/factorial.con create mode 100644 bench/factorial.rs create mode 100644 bench/fib.con create mode 100644 bench/fib.rs diff --git a/.gitignore b/.gitignore index 3e64010..6e91045 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ target/ lcov.info build_artifacts/ + +*.so +*.a diff --git a/bench/.gitignore b/bench/.gitignore new file mode 100644 index 0000000..7b6a50d --- /dev/null +++ b/bench/.gitignore @@ -0,0 +1,2 @@ +bench_* +*.so diff --git a/bench/bench.c b/bench/bench.c new file mode 100644 index 0000000..f3289c4 --- /dev/null +++ b/bench/bench.c @@ -0,0 +1,72 @@ +#include +#include +#include +#include +#include + +extern uint64_t concrete_function(uint64_t n); +extern uint64_t rust_function(uint64_t n); + +struct timespec timer_start() { + struct timespec start_time; + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start_time); + return start_time; +} + +long timer_end(struct timespec start_time) { + struct timespec end_time; + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end_time); + long diffInNanos = (end_time.tv_sec - start_time.tv_sec) * (long)1e9 + + (end_time.tv_nsec - start_time.tv_nsec); + return diffInNanos; +} + +int main(int argc, const char **argv) { + if (argc < 2) { + fprintf(stderr, "missing iteration arguments\n"); + return 1; + } + + if (argc < 3) { + fprintf(stderr, "missing input number argument\n"); + return 1; + } + + int num_iters = atoi(argv[1]); + int input = atoi(argv[2]); + + uint64_t result_concrete; + uint64_t result_rust; + + // warmup + sanity check + for (size_t i = 0; i < 3; ++i) { + assert(concrete_function(input) == rust_function(input)); + } + + printf("Running %d iterations\n", num_iters); + printf("Using input value:\t%d\n", input); + + { + struct timespec vartime = timer_start(); + for (size_t i = 0; i < num_iters; ++i) { + result_concrete = concrete_function(input); + } + long time_elapsed_nanos = timer_end(vartime); + printf("Concrete Result =\t%lu\t\tTime taken : %.2Lf ms\n", result_concrete, + (long double)time_elapsed_nanos / 1000000.0L); + } + + { + struct timespec vartime = timer_start(); + for (size_t i = 0; i < num_iters; ++i) { + result_rust = rust_function(input); + } + long time_elapsed_nanos = timer_end(vartime); + printf("Rust Result =\t\t%lu\t\tTime taken : %.2Lf ms\n", result_rust, + (long double)time_elapsed_nanos / 1000000.0L); + } + + assert(result_concrete == result_rust); + + return 0; +} diff --git a/bench/bench.sh b/bench/bench.sh new file mode 100755 index 0000000..4f926c1 --- /dev/null +++ b/bench/bench.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e + +cd "$(dirname "$0")" + +RED='\033[0;31m' +NC='\033[0m' # No Color + +# name without extension, num_iters, input number +function bench_program() { + local name=$1 + local num_iters=$2 + local input=$3 + + echo -e "### ${RED}Benchmarking $name ${NC}" + + rustc --crate-type=cdylib "$name.rs" -C opt-level=3 -o "${name}_rs.so" > /dev/null 2>&1 + cargo r -- "$name.con" --library --release > /dev/null 2>&1 + cp "./build_artifacts/$name.so" "${name}_con.so" + + cc bench.c -L . -l:./"${name}"_rs.so -l:./"${name}"_con.so -Wl,-rpath -o bench_"${name}" + + ./bench_"${name}" "$num_iters" "$input" +} + +bench_program "factorial" 5000000 20 +bench_program "fib" 5000 20 + +rm ./*.so diff --git a/bench/factorial.con b/bench/factorial.con new file mode 100644 index 0000000..1d8773d --- /dev/null +++ b/bench/factorial.con @@ -0,0 +1,9 @@ +mod Factorial { + fn concrete_function(n: u64) -> u64 { + if n == 0 { + return 1; + } else { + return n * concrete_function(n - 1); + } + } +} diff --git a/bench/factorial.rs b/bench/factorial.rs new file mode 100644 index 0000000..ff26f6d --- /dev/null +++ b/bench/factorial.rs @@ -0,0 +1,8 @@ +#[no_mangle] +pub extern "C" fn rust_function(n: u64) -> u64 { + if n == 0 { + return 1; + } else { + return n * rust_function(n - 1); + } +} diff --git a/bench/fib.con b/bench/fib.con new file mode 100644 index 0000000..a02aef8 --- /dev/null +++ b/bench/fib.con @@ -0,0 +1,9 @@ +mod Fibonacci { + pub fn concrete_function(n: u64) -> u64 { + if n < 2 { + return n; + } + + return concrete_function(n - 1) + concrete_function(n - 2); + } +} diff --git a/bench/fib.rs b/bench/fib.rs new file mode 100644 index 0000000..3f2a73d --- /dev/null +++ b/bench/fib.rs @@ -0,0 +1,8 @@ +#[no_mangle] +pub extern "C" fn rust_function(n: u64) -> u64 { + if n < 2 { + return n; + } + + return rust_function(n - 1) + rust_function(n - 2); +} From b206ca7bed5a0bc935b3973e8d4292019aa2baac Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Thu, 7 Mar 2024 17:42:25 +0100 Subject: [PATCH 2/7] docs --- bench/bench.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/bench/bench.sh b/bench/bench.sh index 4f926c1..3291941 100755 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -24,7 +24,25 @@ function bench_program() { ./bench_"${name}" "$num_iters" "$input" } +: ' +Bench program requirements: + +- Rust + +Function signature should match the following + +#[no_mangle] +pub extern "C" fn rust_function(n: u64) -> u64 + +- Concrete + +Function signature should match the following (in the future if manglign is added, make sure to add no_mangle) + +fn concrete_function(n: u64) -> u64 +' + bench_program "factorial" 5000000 20 bench_program "fib" 5000 20 +# Cleanup rm ./*.so From a51d5d55ea68bac05abf4b28287a159cef3d099f Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Thu, 7 Mar 2024 17:52:17 +0100 Subject: [PATCH 3/7] make bench more fair --- bench/bench.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/bench.sh b/bench/bench.sh index 3291941..a4edd4b 100755 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -15,11 +15,11 @@ function bench_program() { echo -e "### ${RED}Benchmarking $name ${NC}" - rustc --crate-type=cdylib "$name.rs" -C opt-level=3 -o "${name}_rs.so" > /dev/null 2>&1 + rustc --crate-type=cdylib "$name.rs" -C target-cpu=native -C opt-level=3 -o "${name}_rs.so" > /dev/null 2>&1 cargo r -- "$name.con" --library --release > /dev/null 2>&1 cp "./build_artifacts/$name.so" "${name}_con.so" - cc bench.c -L . -l:./"${name}"_rs.so -l:./"${name}"_con.so -Wl,-rpath -o bench_"${name}" + cc -march=native -mtune=native bench.c -L . -l:./"${name}"_rs.so -l:./"${name}"_con.so -Wl,-rpath -o bench_"${name}" ./bench_"${name}" "$num_iters" "$input" } @@ -45,4 +45,4 @@ bench_program "factorial" 5000000 20 bench_program "fib" 5000 20 # Cleanup -rm ./*.so +#rm ./*.so From 1c3b80fcc1671b37a83366e86e842ca8b66360fc Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Thu, 7 Mar 2024 18:01:01 +0100 Subject: [PATCH 4/7] fix --- Makefile | 3 +++ bench/bench.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 172d5be..ae136aa 100644 --- a/Makefile +++ b/Makefile @@ -30,3 +30,6 @@ test: check-deps coverage: check-deps cargo llvm-cov --verbose --all-features --workspace --lcov --output-path lcov.info + +bench: check-deps + ./bench/bench.sh diff --git a/bench/bench.sh b/bench/bench.sh index a4edd4b..c1677dd 100755 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -45,4 +45,4 @@ bench_program "factorial" 5000000 20 bench_program "fib" 5000 20 # Cleanup -#rm ./*.so +rm ./*.so From 990f0e0036bccaf9677450dc9fc12cd09bdebdb9 Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Thu, 7 Mar 2024 18:06:07 +0100 Subject: [PATCH 5/7] add ci --- .github/workflows/ci.yml | 59 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c48e794..d987f21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: TABLEGEN_170_PREFIX: /usr/lib/llvm-17/ steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@1.75.0 + - uses: dtolnay/rust-toolchain@1.76.0 with: components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@1.75.0 + - uses: dtolnay/rust-toolchain@1.76.0 with: components: rustfmt - run: cargo fmt --all -- --check @@ -58,7 +58,7 @@ jobs: sudo rm -rf /usr/share/dotnet/ sudo rm -rf /usr/local/lib/android - name: Setup rust env - uses: dtolnay/rust-toolchain@1.75.0 + uses: dtolnay/rust-toolchain@1.76.0 - name: Retreive cached dependecies uses: Swatinem/rust-cache@v2 - name: add llvm deb repository @@ -91,7 +91,7 @@ jobs: sudo rm -rf /usr/share/dotnet/ sudo rm -rf /usr/local/lib/android - name: Setup rust env - uses: dtolnay/rust-toolchain@1.75.0 + uses: dtolnay/rust-toolchain@1.76.0 - name: Retreive cached dependecies uses: Swatinem/rust-cache@v2 - name: add llvm deb repository @@ -114,3 +114,54 @@ jobs: #token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos files: lcov.info fail_ci_if_error: true + bench: + name: Bench + runs-on: ubuntu-latest + env: + MLIR_SYS_170_PREFIX: /usr/lib/llvm-17/ + LLVM_SYS_170_PREFIX: /usr/lib/llvm-17/ + TABLEGEN_170_PREFIX: /usr/lib/llvm-17/ + steps: + - uses: actions/checkout@v4 + - name: free HDD space + run: | + # deleting space + sudo rm -rf /usr/share/dotnet/ + sudo rm -rf /usr/local/lib/android + - uses: dtolnay/rust-toolchain@1.76.0 + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - name: Setup rust env + uses: dtolnay/rust-toolchain@1.76.0 + - name: Retreive cached dependecies + uses: Swatinem/rust-cache@v2 + - name: add llvm deb repository + uses: myci-actions/add-deb-repo@10 + with: + repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main + repo-name: llvm-repo + keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key + - name: Install LLVM + run: sudo apt-get install llvm-17 llvm-17-dev llvm-17-runtime clang-17 clang-tools-17 lld-17 libpolly-17-dev libmlir-17-dev mlir-17-tools + - name: Install Link deps + run: sudo apt-get install libc-dev build-essential + - name: Run bench + run: ./bench/bench.sh > bench.md + + - name: Find Bench Comment + continue-on-error: true + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Benchmarking + - name: Create or update bench comment + continue-on-error: true + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: bench.md + edit-mode: replace From 7ccb2cc5b65a9f9dab395b1617615e2edce928e8 Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Thu, 7 Mar 2024 18:50:24 +0100 Subject: [PATCH 6/7] filter out colors --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d987f21..70f6083 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,7 +147,7 @@ jobs: - name: Install Link deps run: sudo apt-get install libc-dev build-essential - name: Run bench - run: ./bench/bench.sh > bench.md + run: ./bench/bench.sh | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' > bench.md - name: Find Bench Comment continue-on-error: true From 02691b778917737a7453fa5e599b83f343485a2f Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Thu, 14 Mar 2024 15:12:14 +0100 Subject: [PATCH 7/7] bench readme --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 6f576b3..c5de262 100644 --- a/README.md +++ b/README.md @@ -262,3 +262,13 @@ Post-runtime features: - rustler like binding generator to call rust code :x: - embeddable concrete (no allocator, first-class support for no standard library, etc) :x: - capabilities 🤔 + +## Benchmarking + +There are some simple program benchmarks against Rust. + +You can run them using the following make target: + +``` +make bench +```