diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c48e794..70f6083 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: TABLEGEN_170_PREFIX: /usr/lib/llvm-17/ steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@1.75.0 + - uses: dtolnay/rust-toolchain@1.76.0 with: components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@1.75.0 + - uses: dtolnay/rust-toolchain@1.76.0 with: components: rustfmt - run: cargo fmt --all -- --check @@ -58,7 +58,7 @@ jobs: sudo rm -rf /usr/share/dotnet/ sudo rm -rf /usr/local/lib/android - name: Setup rust env - uses: dtolnay/rust-toolchain@1.75.0 + uses: dtolnay/rust-toolchain@1.76.0 - name: Retreive cached dependecies uses: Swatinem/rust-cache@v2 - name: add llvm deb repository @@ -91,7 +91,7 @@ jobs: sudo rm -rf /usr/share/dotnet/ sudo rm -rf /usr/local/lib/android - name: Setup rust env - uses: dtolnay/rust-toolchain@1.75.0 + uses: dtolnay/rust-toolchain@1.76.0 - name: Retreive cached dependecies uses: Swatinem/rust-cache@v2 - name: add llvm deb repository @@ -114,3 +114,54 @@ jobs: #token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos files: lcov.info fail_ci_if_error: true + bench: + name: Bench + runs-on: ubuntu-latest + env: + MLIR_SYS_170_PREFIX: /usr/lib/llvm-17/ + LLVM_SYS_170_PREFIX: /usr/lib/llvm-17/ + TABLEGEN_170_PREFIX: /usr/lib/llvm-17/ + steps: + - uses: actions/checkout@v4 + - name: free HDD space + run: | + # deleting space + sudo rm -rf /usr/share/dotnet/ + sudo rm -rf /usr/local/lib/android + - uses: dtolnay/rust-toolchain@1.76.0 + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - name: Setup rust env + uses: dtolnay/rust-toolchain@1.76.0 + - name: Retreive cached dependecies + uses: Swatinem/rust-cache@v2 + - name: add llvm deb repository + uses: myci-actions/add-deb-repo@10 + with: + repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main + repo-name: llvm-repo + keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key + - name: Install LLVM + run: sudo apt-get install llvm-17 llvm-17-dev llvm-17-runtime clang-17 clang-tools-17 lld-17 libpolly-17-dev libmlir-17-dev mlir-17-tools + - name: Install Link deps + run: sudo apt-get install libc-dev build-essential + - name: Run bench + run: ./bench/bench.sh | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' > bench.md + + - name: Find Bench Comment + continue-on-error: true + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Benchmarking + - name: Create or update bench comment + continue-on-error: true + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: bench.md + edit-mode: replace diff --git a/.gitignore b/.gitignore index 3e64010..6e91045 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ target/ lcov.info build_artifacts/ + +*.so +*.a diff --git a/Makefile b/Makefile index 133b2ac..b2ba699 100644 --- a/Makefile +++ b/Makefile @@ -30,3 +30,6 @@ test: check-deps coverage: check-deps cargo llvm-cov --verbose --all-features --all-targets --workspace --lcov --output-path lcov.info + +bench: check-deps + ./bench/bench.sh diff --git a/README.md b/README.md index 6f576b3..c5de262 100644 --- a/README.md +++ b/README.md @@ -262,3 +262,13 @@ Post-runtime features: - rustler like binding generator to call rust code :x: - embeddable concrete (no allocator, first-class support for no standard library, etc) :x: - capabilities 🤔 + +## Benchmarking + +There are some simple program benchmarks against Rust. + +You can run them using the following make target: + +``` +make bench +``` diff --git a/bench/.gitignore b/bench/.gitignore new file mode 100644 index 0000000..7b6a50d --- /dev/null +++ b/bench/.gitignore @@ -0,0 +1,2 @@ +bench_* +*.so diff --git a/bench/bench.c b/bench/bench.c new file mode 100644 index 0000000..f3289c4 --- /dev/null +++ b/bench/bench.c @@ -0,0 +1,72 @@ +#include +#include +#include +#include +#include + +extern uint64_t concrete_function(uint64_t n); +extern uint64_t rust_function(uint64_t n); + +struct timespec timer_start() { + struct timespec start_time; + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start_time); + return start_time; +} + +long timer_end(struct timespec start_time) { + struct timespec end_time; + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end_time); + long diffInNanos = (end_time.tv_sec - start_time.tv_sec) * (long)1e9 + + (end_time.tv_nsec - start_time.tv_nsec); + return diffInNanos; +} + +int main(int argc, const char **argv) { + if (argc < 2) { + fprintf(stderr, "missing iteration arguments\n"); + return 1; + } + + if (argc < 3) { + fprintf(stderr, "missing input number argument\n"); + return 1; + } + + int num_iters = atoi(argv[1]); + int input = atoi(argv[2]); + + uint64_t result_concrete; + uint64_t result_rust; + + // warmup + sanity check + for (size_t i = 0; i < 3; ++i) { + assert(concrete_function(input) == rust_function(input)); + } + + printf("Running %d iterations\n", num_iters); + printf("Using input value:\t%d\n", input); + + { + struct timespec vartime = timer_start(); + for (size_t i = 0; i < num_iters; ++i) { + result_concrete = concrete_function(input); + } + long time_elapsed_nanos = timer_end(vartime); + printf("Concrete Result =\t%lu\t\tTime taken : %.2Lf ms\n", result_concrete, + (long double)time_elapsed_nanos / 1000000.0L); + } + + { + struct timespec vartime = timer_start(); + for (size_t i = 0; i < num_iters; ++i) { + result_rust = rust_function(input); + } + long time_elapsed_nanos = timer_end(vartime); + printf("Rust Result =\t\t%lu\t\tTime taken : %.2Lf ms\n", result_rust, + (long double)time_elapsed_nanos / 1000000.0L); + } + + assert(result_concrete == result_rust); + + return 0; +} diff --git a/bench/bench.sh b/bench/bench.sh new file mode 100755 index 0000000..c1677dd --- /dev/null +++ b/bench/bench.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +set -e + +cd "$(dirname "$0")" + +RED='\033[0;31m' +NC='\033[0m' # No Color + +# name without extension, num_iters, input number +function bench_program() { + local name=$1 + local num_iters=$2 + local input=$3 + + echo -e "### ${RED}Benchmarking $name ${NC}" + + rustc --crate-type=cdylib "$name.rs" -C target-cpu=native -C opt-level=3 -o "${name}_rs.so" > /dev/null 2>&1 + cargo r -- "$name.con" --library --release > /dev/null 2>&1 + cp "./build_artifacts/$name.so" "${name}_con.so" + + cc -march=native -mtune=native bench.c -L . -l:./"${name}"_rs.so -l:./"${name}"_con.so -Wl,-rpath -o bench_"${name}" + + ./bench_"${name}" "$num_iters" "$input" +} + +: ' +Bench program requirements: + +- Rust + +Function signature should match the following + +#[no_mangle] +pub extern "C" fn rust_function(n: u64) -> u64 + +- Concrete + +Function signature should match the following (in the future if manglign is added, make sure to add no_mangle) + +fn concrete_function(n: u64) -> u64 +' + +bench_program "factorial" 5000000 20 +bench_program "fib" 5000 20 + +# Cleanup +rm ./*.so diff --git a/bench/factorial.con b/bench/factorial.con new file mode 100644 index 0000000..1d8773d --- /dev/null +++ b/bench/factorial.con @@ -0,0 +1,9 @@ +mod Factorial { + fn concrete_function(n: u64) -> u64 { + if n == 0 { + return 1; + } else { + return n * concrete_function(n - 1); + } + } +} diff --git a/bench/factorial.rs b/bench/factorial.rs new file mode 100644 index 0000000..ff26f6d --- /dev/null +++ b/bench/factorial.rs @@ -0,0 +1,8 @@ +#[no_mangle] +pub extern "C" fn rust_function(n: u64) -> u64 { + if n == 0 { + return 1; + } else { + return n * rust_function(n - 1); + } +} diff --git a/bench/fib.con b/bench/fib.con new file mode 100644 index 0000000..a02aef8 --- /dev/null +++ b/bench/fib.con @@ -0,0 +1,9 @@ +mod Fibonacci { + pub fn concrete_function(n: u64) -> u64 { + if n < 2 { + return n; + } + + return concrete_function(n - 1) + concrete_function(n - 2); + } +} diff --git a/bench/fib.rs b/bench/fib.rs new file mode 100644 index 0000000..3f2a73d --- /dev/null +++ b/bench/fib.rs @@ -0,0 +1,8 @@ +#[no_mangle] +pub extern "C" fn rust_function(n: u64) -> u64 { + if n < 2 { + return n; + } + + return rust_function(n - 1) + rust_function(n - 2); +}