Merge pull request #96 from lambdaclass/bench

Add bench vs rust

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
             TABLEGEN_170_PREFIX: /usr/lib/llvm-17/
         steps:
         - uses: actions/checkout@v3
-        - uses: dtolnay/rust-toolchain@1.75.0
+        - uses: dtolnay/rust-toolchain@1.76.0
           with:
             components: rustfmt, clippy
         - uses: Swatinem/rust-cache@v2
@@ -37,7 +37,7 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         - uses: actions/checkout@v3
-        - uses: dtolnay/rust-toolchain@1.75.0
+        - uses: dtolnay/rust-toolchain@1.76.0
           with:
             components: rustfmt
         - run: cargo fmt --all -- --check
@@ -58,7 +58,7 @@ jobs:
             sudo rm -rf /usr/share/dotnet/
             sudo rm -rf /usr/local/lib/android
         - name: Setup rust env
-          uses: dtolnay/rust-toolchain@1.75.0
+          uses: dtolnay/rust-toolchain@1.76.0
         - name: Retreive cached dependecies
           uses: Swatinem/rust-cache@v2
         - name: add llvm deb repository
@@ -91,7 +91,7 @@ jobs:
                     sudo rm -rf /usr/share/dotnet/
                     sudo rm -rf /usr/local/lib/android
             - name: Setup rust env
-              uses: dtolnay/rust-toolchain@1.75.0
+              uses: dtolnay/rust-toolchain@1.76.0
             - name: Retreive cached dependecies
               uses: Swatinem/rust-cache@v2
             - name: add llvm deb repository
@@ -114,3 +114,54 @@ jobs:
                 #token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
                 files: lcov.info
                 fail_ci_if_error: true
+    bench:
+      name: Bench
+      runs-on: ubuntu-latest
+      env:
+        MLIR_SYS_170_PREFIX: /usr/lib/llvm-17/
+        LLVM_SYS_170_PREFIX: /usr/lib/llvm-17/
+        TABLEGEN_170_PREFIX: /usr/lib/llvm-17/
+      steps:
+        - uses: actions/checkout@v4
+        - name: free HDD space
+          run: |
+                # deleting space
+                sudo rm -rf /usr/share/dotnet/
+                sudo rm -rf /usr/local/lib/android
+        - uses: dtolnay/[email protected]
+          with:
+            components: clippy
+        - uses: Swatinem/rust-cache@v2
+        - name: Setup rust env
+          uses: dtolnay/[email protected]
+        - name: Retreive cached dependecies
+          uses: Swatinem/rust-cache@v2
+        - name: add llvm deb repository
+          uses: myci-actions/add-deb-repo@10
+          with:
+                repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
+                repo-name: llvm-repo
+                keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key
+        - name: Install LLVM
+          run: sudo apt-get install llvm-17 llvm-17-dev llvm-17-runtime clang-17 clang-tools-17 lld-17 libpolly-17-dev libmlir-17-dev mlir-17-tools
+        - name: Install Link deps
+          run: sudo apt-get install libc-dev build-essential
+        - name: Run bench
+          run: ./bench/bench.sh | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' > bench.md
+
+        - name: Find Bench Comment
+          continue-on-error: true
+          uses: peter-evans/find-comment@v3
+          id: fc
+          with:
+            issue-number: ${{ github.event.pull_request.number }}
+            comment-author: 'github-actions[bot]'
+            body-includes: Benchmarking
+        - name: Create or update bench comment
+          continue-on-error: true
+          uses: peter-evans/create-or-update-comment@v4
+          with:
+            comment-id: ${{ steps.fc.outputs.comment-id }}
+            issue-number: ${{ github.event.pull_request.number }}
+            body-path: bench.md
+            edit-mode: replace

.gitignore

@@ -20,3 +20,6 @@ target/
 lcov.info
 
 build_artifacts/
+
+*.so
+*.a

Makefile

@@ -30,3 +30,6 @@ test: check-deps
 
 coverage: check-deps
 	cargo llvm-cov --verbose --all-features --all-targets --workspace --lcov --output-path lcov.info
+
+bench: check-deps
+	./bench/bench.sh

README.md

@@ -262,3 +262,13 @@ Post-runtime features:
 - rustler like binding generator to call rust code :x:
 - embeddable concrete (no allocator, first-class support for no standard library, etc) :x:
 - capabilities 🤔
+
+## Benchmarking
+
+There are some simple program benchmarks against Rust.
+
+You can run them using the following make target:
+
+```
+make bench
+```

bench/.gitignore

@@ -0,0 +1,2 @@
+bench_*
+*.so

bench/bench.c

@@ -0,0 +1,72 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+
+extern uint64_t concrete_function(uint64_t n);
+extern uint64_t rust_function(uint64_t n);
+
+struct timespec timer_start() {
+  struct timespec start_time;
+  clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start_time);
+  return start_time;
+}
+
+long timer_end(struct timespec start_time) {
+  struct timespec end_time;
+  clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end_time);
+  long diffInNanos = (end_time.tv_sec - start_time.tv_sec) * (long)1e9 +
+                     (end_time.tv_nsec - start_time.tv_nsec);
+  return diffInNanos;
+}
+
+int main(int argc, const char **argv) {
+  if (argc < 2) {
+    fprintf(stderr, "missing iteration arguments\n");
+    return 1;
+  }
+
+  if (argc < 3) {
+    fprintf(stderr, "missing input number argument\n");
+    return 1;
+  }
+
+  int num_iters = atoi(argv[1]);
+  int input = atoi(argv[2]);
+
+  uint64_t result_concrete;
+  uint64_t result_rust;
+
+  // warmup + sanity check
+  for (size_t i = 0; i < 3; ++i) {
+    assert(concrete_function(input) == rust_function(input));
+  }
+
+  printf("Running %d iterations\n", num_iters);
+  printf("Using input value:\t%d\n", input);
+
+  {
+    struct timespec vartime = timer_start();
+    for (size_t i = 0; i < num_iters; ++i) {
+      result_concrete = concrete_function(input);
+    }
+    long time_elapsed_nanos = timer_end(vartime);
+    printf("Concrete Result =\t%lu\t\tTime taken : %.2Lf ms\n", result_concrete,
+           (long double)time_elapsed_nanos / 1000000.0L);
+  }
+
+  {
+    struct timespec vartime = timer_start();
+    for (size_t i = 0; i < num_iters; ++i) {
+      result_rust = rust_function(input);
+    }
+    long time_elapsed_nanos = timer_end(vartime);
+    printf("Rust Result =\t\t%lu\t\tTime taken : %.2Lf ms\n", result_rust,
+           (long double)time_elapsed_nanos / 1000000.0L);
+  }
+
+  assert(result_concrete == result_rust);
+
+  return 0;
+}

bench/bench.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+
+cd "$(dirname "$0")"
+
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+# name without extension, num_iters, input number
+function bench_program() {
+    local name=$1
+    local num_iters=$2
+    local input=$3
+
+    echo -e "### ${RED}Benchmarking $name ${NC}"
+
+    rustc --crate-type=cdylib "$name.rs" -C target-cpu=native -C opt-level=3 -o "${name}_rs.so" > /dev/null 2>&1
+    cargo r -- "$name.con"  --library --release > /dev/null 2>&1
+    cp "./build_artifacts/$name.so" "${name}_con.so"
+
+    cc -march=native -mtune=native bench.c -L . -l:./"${name}"_rs.so -l:./"${name}"_con.so -Wl,-rpath -o bench_"${name}"
+
+    ./bench_"${name}" "$num_iters" "$input"
+}
+
+: '
+Bench program requirements:
+
+- Rust
+
+Function signature should match the following
+
+#[no_mangle]
+pub extern "C" fn rust_function(n: u64) -> u64
+
+- Concrete
+
+Function signature should match the following (in the future if manglign is added, make sure to add no_mangle)
+
+fn concrete_function(n: u64) -> u64
+'
+
+bench_program "factorial" 5000000 20
+bench_program "fib" 5000 20
+
+# Cleanup
+rm ./*.so

bench/factorial.con

@@ -0,0 +1,9 @@
+mod Factorial {
+    fn concrete_function(n: u64) -> u64 {
+        if n == 0 {
+            return 1;
+        } else {
+            return n * concrete_function(n - 1);
+        }
+    }
+}

bench/factorial.rs

@@ -0,0 +1,8 @@
+#[no_mangle]
+pub extern "C" fn rust_function(n: u64) -> u64 {
+    if n == 0 {
+        return 1;
+    } else {
+        return n * rust_function(n - 1);
+    }
+}

bench/fib.con

@@ -0,0 +1,9 @@
+mod Fibonacci {
+    pub fn concrete_function(n: u64) -> u64 {
+        if n < 2 {
+            return n;
+        }
+
+        return concrete_function(n - 1) + concrete_function(n - 2);
+    }
+}

bench/fib.rs

@@ -0,0 +1,8 @@
+#[no_mangle]
+pub extern "C" fn rust_function(n: u64) -> u64 {
+    if n < 2 {
+        return n;
+    }
+
+    return rust_function(n - 1) + rust_function(n - 2);
+}