Skip to content

Commit

Permalink
Add benchmarks comparing to Cairo VM (#692)
Browse files Browse the repository at this point in the history 
* Add benches

* Fix clippy

* Add more cases

* Use fibonacci instead of fib

* Differenciate expect messages

* Separate into groups per program

* Add result checks

* Fix clippy

* Add compile bench

* Add linear search

* Add linear search bench

* Lower time

* Fix program

* Add linear_search.c

* Add builtin

* Fix

* Fix symbol number

* Update benches

* Fix bench

* Remove comp benchmark

* Remove extra dependency

---------

Co-authored-by: Edgar <[email protected]>
Co-authored-by: Pedro Fontana <[email protected]>
  • Loading branch information
@JulianGCalderon @edg-l @pefontana
3 people authored Dec 9, 2024
1 parent 2bad480 commit 5378651
Show file tree
Hide file tree
Showing 3 changed files with 295 additions and 20 deletions.
241 changes: 221 additions & 20 deletions benches/benches.rs
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
use cairo_lang_compiler::{
compile_prepared_db, db::RootDatabase, project::setup_project, CompilerConfig,
};
use cairo_lang_runner::{RunResultValue, SierraCasmRunner, StarknetState};
use cairo_lang_sierra::program::Program;
use cairo_lang_sierra_generator::replace_ids::DebugReplacer;
use cairo_lang_starknet::contract::{find_contracts, get_contracts_info};
use cairo_lang_utils::Upcast;
use cairo_native::{
cache::{AotProgramCache, JitProgramCache},
context::NativeContext,
utils::find_function_id,
OptLevel,
OptLevel, Value,
};
use criterion::{criterion_group, criterion_main, Criterion};
use starknet_types_core::felt::Felt;
Expand All @@ -20,6 +24,7 @@ fn criterion_benchmark(c: &mut Criterion) {
let factorial = load_contract("programs/benches/factorial_2M.cairo");
let fibonacci = load_contract("programs/benches/fib_2M.cairo");
let logistic_map = load_contract("programs/benches/logistic_map.cairo");
let linear_search = load_contract("programs/benches/linear_search.cairo");

let aot_factorial = aot_cache
.compile_and_insert(Felt::ZERO, &factorial, OptLevel::Aggressive)
Expand All @@ -30,6 +35,9 @@ fn criterion_benchmark(c: &mut Criterion) {
let aot_logistic_map = aot_cache
.compile_and_insert(Felt::from(2), &logistic_map, OptLevel::Aggressive)
.unwrap();
let aot_linear_search = aot_cache
.compile_and_insert(Felt::from(2), &linear_search, OptLevel::Aggressive)
.unwrap();

let jit_factorial = jit_cache
.compile_and_insert(Felt::ZERO, &factorial, OptLevel::Aggressive)
Expand All @@ -40,32 +48,195 @@ fn criterion_benchmark(c: &mut Criterion) {
let jit_logistic_map = jit_cache
.compile_and_insert(Felt::from(2), &logistic_map, OptLevel::Aggressive)
.unwrap();
let jit_linear_search = jit_cache
.compile_and_insert(Felt::from(2), &linear_search, OptLevel::Aggressive)
.unwrap();

let factorial_function_id =
find_function_id(&factorial, "factorial_2M::factorial_2M::main").unwrap();
let fibonacci_function_id = find_function_id(&fibonacci, "fib_2M::fib_2M::main").unwrap();
let logistic_map_function_id =
find_function_id(&logistic_map, "logistic_map::logistic_map::main").unwrap();
let linear_search_function_id =
find_function_id(&linear_search, "linear_search::linear_search::main").unwrap();

let factorial_runner = load_contract_for_vm("programs/benches/factorial_2M.cairo");
let fibonacci_runner = load_contract_for_vm("programs/benches/fib_2M.cairo");
let logistic_map_runner = load_contract_for_vm("programs/benches/logistic_map.cairo");
let linear_search_runner = load_contract_for_vm("programs/benches/linear_search.cairo");

let factorial_function = factorial_runner
.find_function("main")
.expect("failed to find main factorial function");
let fibonacci_function = fibonacci_runner
.find_function("main")
.expect("failed to find main fibonacci function");
let logistic_map_function = logistic_map_runner
.find_function("main")
.expect("failed to find main logistic map function");
let linear_search_function = linear_search_runner
.find_function("main")
.expect("failed to find main logistic map function");

{
let mut linear_search_group = c.benchmark_group("linear_search");

linear_search_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_linear_search
.invoke_dynamic(linear_search_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});
linear_search_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_linear_search
.invoke_dynamic(linear_search_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

linear_search_group.bench_function("VM", |b| {
b.iter(|| {
let result = linear_search_runner
.run_function_with_starknet_context(
linear_search_function,
&[],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});

linear_search_group.finish();
}

{
let mut factorial_group = c.benchmark_group("factorial_2M");

factorial_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_factorial
.invoke_dynamic(factorial_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});
factorial_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_factorial
.invoke_dynamic(factorial_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

factorial_group.bench_function("VM", |b| {
b.iter(|| {
let result = factorial_runner
.run_function_with_starknet_context(
factorial_function,
&[],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});

factorial_group.finish();
}

{
let mut fibonacci_group = c.benchmark_group("fibonacci_2M");

fibonacci_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_fibonacci
.invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});
fibonacci_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_fibonacci
.invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
})
});
fibonacci_group.bench_function("VM", |b| {
b.iter(|| {
let result = fibonacci_runner
.run_function_with_starknet_context(
fibonacci_function,
&[],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});

fibonacci_group.finish();
}

c.bench_function("Cached JIT factorial_2M", |b| {
b.iter(|| jit_factorial.invoke_dynamic(factorial_function_id, &[], Some(u64::MAX)));
});
c.bench_function("Cached JIT fib_2M", |b| {
b.iter(|| jit_fibonacci.invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX)));
});
c.bench_function("Cached JIT logistic_map", |b| {
b.iter(|| jit_logistic_map.invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX)));
});

c.bench_function("Cached AOT factorial_2M", |b| {
b.iter(|| aot_factorial.invoke_dynamic(factorial_function_id, &[], Some(u64::MAX)));
});
c.bench_function("Cached AOT fib_2M", |b| {
b.iter(|| aot_fibonacci.invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX)));
});
c.bench_function("Cached AOT logistic_map", |b| {
b.iter(|| aot_logistic_map.invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX)));
});
{
let mut logistic_map_group = c.benchmark_group("logistic_map");

logistic_map_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_logistic_map
.invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

logistic_map_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_logistic_map
.invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

logistic_map_group.bench_function("VM", |b| {
b.iter(|| {
let result = logistic_map_runner
.run_function_with_starknet_context(
logistic_map_function,
&[],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});

logistic_map_group.finish();
}
}

fn load_contract(path: impl AsRef<Path>) -> Program {
Expand All @@ -84,5 +255,35 @@ fn load_contract(path: impl AsRef<Path>) -> Program {
sirrra_program.program
}

fn load_contract_for_vm(path: impl AsRef<Path>) -> SierraCasmRunner {
let mut db = RootDatabase::builder()
.detect_corelib()
.build()
.expect("failed to build database");
let main_crate_ids = setup_project(&mut db, path.as_ref()).expect("failed to setup project");
let program = compile_prepared_db(
&db,
main_crate_ids.clone(),
CompilerConfig {
replace_ids: true,
..Default::default()
},
)
.expect("failed to compile program");

let replacer = DebugReplacer { db: &db };
let contracts = find_contracts((db).upcast(), &main_crate_ids);
let contracts_info =
get_contracts_info(&db, contracts, &replacer).expect("failed to get contracts info");

SierraCasmRunner::new(
program.program.clone(),
Some(Default::default()),
contracts_info,
None,
)
.expect("failed to create runner")
}

criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
37 changes: 37 additions & 0 deletions programs/benches/linear_search.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#include <assert.h>
#include <stdint.h>


typedef struct linear_search_return_values
{
uint64_t range_check_counter;
uint64_t remaining_gas;
struct {
uint8_t discriminant;
struct {
void *ptr;
uint32_t start;
uint32_t end;
uint32_t cap;
} err;
} result;
} linear_search_return_values_t;

static void run_bench(linear_search_return_values_t *, uint64_t)
__attribute__((weakref("_mlir_ciface_linear_search::linear_search::main(f4)")));

extern uint64_t* cairo_native__set_costs_builtin(uint64_t*);

int main()
{
uint64_t BuiltinCosts[7] = {1, 4050, 583, 4085, 491, 230, 604};

cairo_native__set_costs_builtin(&BuiltinCosts[0]);

linear_search_return_values_t return_values;

run_bench(&return_values, 0xFFFFFFFFFFFFFFFF);
assert((return_values.result.discriminant & 0x1) == 0);

return 0;
}
37 changes: 37 additions & 0 deletions programs/benches/linear_search.cairo
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
fn search(array: @Array<felt252>, number: felt252) -> u32 {
let mut index = 0;

while index < array.len() {
if *array[index] == number {
break;
}

index += 1;
};

return index;
}

fn init_array(length: u32) -> Array<felt252> {
let mut array = ArrayTrait::new();
for i in 0..length {
array.append(i.into());
};

return array;
}

fn main() {
let array = init_array(4001);

let index = search(@array, 4000);
assert(
index == 4000,
'invalid result'
);
let index = search(@array, 2000);
assert(
index == 2000,
'invalid result'
);
}

0 comments on commit 5378651

Please sign in to comment.