From f53c9479d69c7dbd6dd568d9a06160c4f2e784dc Mon Sep 17 00:00:00 2001 From: Julian Gonzalez Calderon Date: Thu, 19 Dec 2024 12:59:15 -0300 Subject: [PATCH] Add more dict benches (#977) * Add more benches * Rename tests * Fix hyperfine benches * Add segment arena builtin * Add RangeCheck as first argument to `run_bench` * Increase bench size --- benches/benches.rs | 263 ++++++--------------------- programs/benches/dict_insert.c | 38 ++++ programs/benches/dict_insert.cairo | 19 ++ programs/benches/dict_snapshot.c | 38 ++++ programs/benches/dict_snapshot.cairo | 38 ++++ programs/benches/factorial_2M.c | 4 +- programs/benches/fib_2M.c | 4 +- programs/benches/linear_search.c | 4 +- programs/benches/logistic_map.c | 4 +- 9 files changed, 199 insertions(+), 213 deletions(-) create mode 100644 programs/benches/dict_insert.c create mode 100644 programs/benches/dict_insert.cairo create mode 100644 programs/benches/dict_snapshot.c create mode 100644 programs/benches/dict_snapshot.cairo diff --git a/benches/benches.rs b/benches/benches.rs index 9f034ab4e..066c58909 100644 --- a/benches/benches.rs +++ b/benches/benches.rs @@ -16,227 +16,80 @@ use criterion::{criterion_group, criterion_main, Criterion}; use starknet_types_core::felt::Felt; use std::path::Path; -fn criterion_benchmark(c: &mut Criterion) { +fn compare(c: &mut Criterion, path: impl AsRef) { let context = NativeContext::new(); let mut aot_cache = AotProgramCache::new(&context); let mut jit_cache = JitProgramCache::new(&context); - let factorial = load_contract("programs/benches/factorial_2M.cairo"); - let fibonacci = load_contract("programs/benches/fib_2M.cairo"); - let logistic_map = load_contract("programs/benches/logistic_map.cairo"); - let linear_search = load_contract("programs/benches/linear_search.cairo"); - - let aot_factorial = aot_cache - .compile_and_insert(Felt::ZERO, &factorial, OptLevel::Aggressive) - .unwrap(); - let aot_fibonacci = aot_cache - .compile_and_insert(Felt::ONE, &fibonacci, OptLevel::Aggressive) - .unwrap(); - let aot_logistic_map = aot_cache - .compile_and_insert(Felt::from(2), &logistic_map, OptLevel::Aggressive) - .unwrap(); - let aot_linear_search = aot_cache - .compile_and_insert(Felt::from(2), &linear_search, OptLevel::Aggressive) - .unwrap(); - - let jit_factorial = jit_cache - .compile_and_insert(Felt::ZERO, &factorial, OptLevel::Aggressive) - .unwrap(); - let jit_fibonacci = jit_cache - .compile_and_insert(Felt::ONE, &fibonacci, OptLevel::Aggressive) - .unwrap(); - let jit_logistic_map = jit_cache - .compile_and_insert(Felt::from(2), &logistic_map, OptLevel::Aggressive) + let stem = path + .as_ref() + .file_stem() + .unwrap() + .to_str() + .unwrap() + .to_string(); + + let program = load_contract(&path); + let aot_executor = aot_cache + .compile_and_insert(Felt::ZERO, &program, OptLevel::Aggressive) .unwrap(); - let jit_linear_search = jit_cache - .compile_and_insert(Felt::from(2), &linear_search, OptLevel::Aggressive) + let jit_executor = jit_cache + .compile_and_insert(Felt::ZERO, &program, OptLevel::Aggressive) .unwrap(); - let factorial_function_id = - find_function_id(&factorial, "factorial_2M::factorial_2M::main").unwrap(); - let fibonacci_function_id = find_function_id(&fibonacci, "fib_2M::fib_2M::main").unwrap(); - let logistic_map_function_id = - find_function_id(&logistic_map, "logistic_map::logistic_map::main").unwrap(); - let linear_search_function_id = - find_function_id(&linear_search, "linear_search::linear_search::main").unwrap(); + let main_name = format!("{stem}::{stem}::main"); + let main_id = find_function_id(&program, &main_name).unwrap(); - let factorial_runner = load_contract_for_vm("programs/benches/factorial_2M.cairo"); - let fibonacci_runner = load_contract_for_vm("programs/benches/fib_2M.cairo"); - let logistic_map_runner = load_contract_for_vm("programs/benches/logistic_map.cairo"); - let linear_search_runner = load_contract_for_vm("programs/benches/linear_search.cairo"); - - let factorial_function = factorial_runner - .find_function("main") - .expect("failed to find main factorial function"); - let fibonacci_function = fibonacci_runner - .find_function("main") - .expect("failed to find main fibonacci function"); - let logistic_map_function = logistic_map_runner + let vm_runner = load_contract_for_vm(&path); + let vm_main_id = vm_runner .find_function("main") - .expect("failed to find main logistic map function"); - let linear_search_function = linear_search_runner - .find_function("main") - .expect("failed to find main logistic map function"); - - { - let mut linear_search_group = c.benchmark_group("linear_search"); - - linear_search_group.bench_function("Cached JIT", |b| { - b.iter(|| { - let result = jit_linear_search - .invoke_dynamic(linear_search_function_id, &[], Some(u64::MAX)) - .unwrap(); - let value = result.return_value; - assert!(matches!(value, Value::Enum { tag: 0, .. })) - }); - }); - linear_search_group.bench_function("Cached AOT", |b| { - b.iter(|| { - let result = aot_linear_search - .invoke_dynamic(linear_search_function_id, &[], Some(u64::MAX)) - .unwrap(); - let value = result.return_value; - assert!(matches!(value, Value::Enum { tag: 0, .. })) - }); - }); - - linear_search_group.bench_function("VM", |b| { - b.iter(|| { - let result = linear_search_runner - .run_function_with_starknet_context( - linear_search_function, - vec![], - Some(usize::MAX), - StarknetState::default(), - ) - .unwrap(); - let value = result.value; - assert!(matches!(value, RunResultValue::Success(_))) - }); - }); - - linear_search_group.finish(); - } - - { - let mut factorial_group = c.benchmark_group("factorial_2M"); - - factorial_group.bench_function("Cached JIT", |b| { - b.iter(|| { - let result = jit_factorial - .invoke_dynamic(factorial_function_id, &[], Some(u64::MAX)) - .unwrap(); - let value = result.return_value; - assert!(matches!(value, Value::Enum { tag: 0, .. })) - }); - }); - factorial_group.bench_function("Cached AOT", |b| { - b.iter(|| { - let result = aot_factorial - .invoke_dynamic(factorial_function_id, &[], Some(u64::MAX)) - .unwrap(); - let value = result.return_value; - assert!(matches!(value, Value::Enum { tag: 0, .. })) - }); - }); - - factorial_group.bench_function("VM", |b| { - b.iter(|| { - let result = factorial_runner - .run_function_with_starknet_context( - factorial_function, - vec![], - Some(usize::MAX), - StarknetState::default(), - ) - .unwrap(); - let value = result.value; - assert!(matches!(value, RunResultValue::Success(_))) - }); - }); - - factorial_group.finish(); - } + .expect("failed to find main function"); - { - let mut fibonacci_group = c.benchmark_group("fibonacci_2M"); + let mut group = c.benchmark_group(stem); - fibonacci_group.bench_function("Cached JIT", |b| { - b.iter(|| { - let result = jit_fibonacci - .invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX)) - .unwrap(); - let value = result.return_value; - assert!(matches!(value, Value::Enum { tag: 0, .. })) - }); + group.bench_function("Cached JIT", |b| { + b.iter(|| { + let result = jit_executor + .invoke_dynamic(main_id, &[], Some(u64::MAX)) + .unwrap(); + let value = result.return_value; + assert!(matches!(value, Value::Enum { tag: 0, .. })) }); - fibonacci_group.bench_function("Cached AOT", |b| { - b.iter(|| { - let result = aot_fibonacci - .invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX)) - .unwrap(); - let value = result.return_value; - assert!(matches!(value, Value::Enum { tag: 0, .. })) - }) + }); + group.bench_function("Cached AOT", |b| { + b.iter(|| { + let result = aot_executor + .invoke_dynamic(main_id, &[], Some(u64::MAX)) + .unwrap(); + let value = result.return_value; + assert!(matches!(value, Value::Enum { tag: 0, .. })) }); - fibonacci_group.bench_function("VM", |b| { - b.iter(|| { - let result = fibonacci_runner - .run_function_with_starknet_context( - fibonacci_function, - vec![], - Some(usize::MAX), - StarknetState::default(), - ) - .unwrap(); - let value = result.value; - assert!(matches!(value, RunResultValue::Success(_))) - }); + }); + group.bench_function("VM", |b| { + b.iter(|| { + let result = vm_runner + .run_function_with_starknet_context( + vm_main_id, + vec![], + Some(usize::MAX), + StarknetState::default(), + ) + .unwrap(); + let value = result.value; + assert!(matches!(value, RunResultValue::Success(_))) }); + }); - fibonacci_group.finish(); - } - - { - let mut logistic_map_group = c.benchmark_group("logistic_map"); - - logistic_map_group.bench_function("Cached JIT", |b| { - b.iter(|| { - let result = jit_logistic_map - .invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX)) - .unwrap(); - let value = result.return_value; - assert!(matches!(value, Value::Enum { tag: 0, .. })) - }); - }); - - logistic_map_group.bench_function("Cached AOT", |b| { - b.iter(|| { - let result = aot_logistic_map - .invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX)) - .unwrap(); - let value = result.return_value; - assert!(matches!(value, Value::Enum { tag: 0, .. })) - }); - }); - - logistic_map_group.bench_function("VM", |b| { - b.iter(|| { - let result = logistic_map_runner - .run_function_with_starknet_context( - logistic_map_function, - vec![], - Some(usize::MAX), - StarknetState::default(), - ) - .unwrap(); - let value = result.value; - assert!(matches!(value, RunResultValue::Success(_))) - }); - }); + group.finish(); +} - logistic_map_group.finish(); - } +fn criterion_benchmark(c: &mut Criterion) { + compare(c, "programs/benches/dict_snapshot.cairo"); + compare(c, "programs/benches/dict_insert.cairo"); + compare(c, "programs/benches/factorial_2M.cairo"); + compare(c, "programs/benches/fib_2M.cairo"); + compare(c, "programs/benches/linear_search.cairo"); + compare(c, "programs/benches/logistic_map.cairo"); } fn load_contract(path: impl AsRef) -> Program { diff --git a/programs/benches/dict_insert.c b/programs/benches/dict_insert.c new file mode 100644 index 000000000..ed0b2ad6f --- /dev/null +++ b/programs/benches/dict_insert.c @@ -0,0 +1,38 @@ +#include +#include + + +typedef struct dict_insert_return_values +{ + uint64_t range_check_counter; + uint64_t segment_arena_counter; + uint64_t remaining_gas; + struct { + uint8_t discriminant; + struct { + void *ptr; + uint32_t start; + uint32_t end; + uint32_t cap; + } err; + } result; +} dict_insert_return_values_t; + +static void run_bench(dict_insert_return_values_t *, uint64_t, uint64_t, uint64_t) + __attribute__((weakref("_mlir_ciface_dict_insert::dict_insert::main(f3)"))); + +extern uint64_t* cairo_native__set_costs_builtin(uint64_t*); + +int main() +{ + uint64_t BuiltinCosts[7] = {1, 4050, 583, 4085, 491, 230, 604}; + + cairo_native__set_costs_builtin(&BuiltinCosts[0]); + + dict_insert_return_values_t return_values; + + run_bench(&return_values, 0, 0, 0xFFFFFFFFFFFFFFFF); + assert((return_values.result.discriminant & 0x1) == 0); + + return 0; +} diff --git a/programs/benches/dict_insert.cairo b/programs/benches/dict_insert.cairo new file mode 100644 index 000000000..742452a1c --- /dev/null +++ b/programs/benches/dict_insert.cairo @@ -0,0 +1,19 @@ +fn init_dict(length: u64) -> Felt252Dict { + let mut balances: Felt252Dict = Default::default(); + + for i in 0..length { + let x: felt252 = i.into(); + balances.insert(x, x); + }; + + return balances; +} + +fn main() { + let mut dict = init_dict(1000001); + let last = dict.get(1000000); + assert( + last == 1000000, + 'invalid result' + ); +} diff --git a/programs/benches/dict_snapshot.c b/programs/benches/dict_snapshot.c new file mode 100644 index 000000000..789bcb3bd --- /dev/null +++ b/programs/benches/dict_snapshot.c @@ -0,0 +1,38 @@ +#include +#include + + +typedef struct dict_snapshot_return_values +{ + uint64_t range_check_counter; + uint64_t segment_arena_counter; + uint64_t remaining_gas; + struct { + uint8_t discriminant; + struct { + void *ptr; + uint32_t start; + uint32_t end; + uint32_t cap; + } err; + } result; +} dict_snapshot_return_values_t; + +static void run_bench(dict_snapshot_return_values_t *, uint64_t, uint64_t, uint64_t) + __attribute__((weakref("_mlir_ciface_dict_snapshot::dict_snapshot::main(f4)"))); + +extern uint64_t* cairo_native__set_costs_builtin(uint64_t*); + +int main() +{ + uint64_t BuiltinCosts[7] = {1, 4050, 583, 4085, 491, 230, 604}; + + cairo_native__set_costs_builtin(&BuiltinCosts[0]); + + dict_snapshot_return_values_t return_values; + + run_bench(&return_values, 0, 0, 0xFFFFFFFFFFFFFFFF); + assert((return_values.result.discriminant & 0x1) == 0); + + return 0; +} diff --git a/programs/benches/dict_snapshot.cairo b/programs/benches/dict_snapshot.cairo new file mode 100644 index 000000000..5ac7ec0ae --- /dev/null +++ b/programs/benches/dict_snapshot.cairo @@ -0,0 +1,38 @@ +fn init_dict(length: u64) -> Felt252Dict { + let mut balances: Felt252Dict = Default::default(); + + for i in 0..length { + let x: felt252 = i.into(); + balances.insert(x, x); + }; + + return balances; +} + +#[derive(Destruct)] +struct VM { + marker: u64, + dict: Felt252Dict, +} + +fn main() { + let dict = init_dict(1000); + let vm = VM { + dict: dict, + marker: 1, + }; + + let dups: u64 = 10000; + let mut counter: u64 = 0; + + for _ in 0..dups { + let vmsnap = @vm; + let vmmarker = vmsnap.marker; + counter += *vmmarker; + }; + + assert( + counter == dups, + 'invalid result' + ); +} diff --git a/programs/benches/factorial_2M.c b/programs/benches/factorial_2M.c index 15b811783..0435f3740 100644 --- a/programs/benches/factorial_2M.c +++ b/programs/benches/factorial_2M.c @@ -17,7 +17,7 @@ typedef struct factorial_return_values } result; } factorial_return_values_t; -static void run_bench(factorial_return_values_t*, uint64_t) +static void run_bench(factorial_return_values_t*, uint64_t, uint64_t) __attribute__((weakref("_mlir_ciface_factorial_2M::factorial_2M::main(f1)"))); extern uint64_t* cairo_native__set_costs_builtin(uint64_t*); @@ -30,7 +30,7 @@ int main() cairo_native__set_costs_builtin(&BuiltinCosts[0]); - run_bench(&return_values, 0xFFFFFFFFFFFFFFFF); + run_bench(&return_values, 0, 0xFFFFFFFFFFFFFFFF); assert((return_values.result.discriminant & 0x1) == 0); return 0; diff --git a/programs/benches/fib_2M.c b/programs/benches/fib_2M.c index 696dc5f60..343807619 100644 --- a/programs/benches/fib_2M.c +++ b/programs/benches/fib_2M.c @@ -17,7 +17,7 @@ typedef struct fib_return_values } result; } fib_return_values_t; -static void run_bench(fib_return_values_t *, uint64_t) +static void run_bench(fib_return_values_t *, uint64_t, uint64_t) __attribute__((weakref("_mlir_ciface_fib_2M::fib_2M::main(f1)"))); extern uint64_t* cairo_native__set_costs_builtin(uint64_t*); @@ -30,7 +30,7 @@ int main() fib_return_values_t return_values; - run_bench(&return_values, 0xFFFFFFFFFFFFFFFF); + run_bench(&return_values, 0, 0xFFFFFFFFFFFFFFFF); assert((return_values.result.discriminant & 0x1) == 0); return 0; diff --git a/programs/benches/linear_search.c b/programs/benches/linear_search.c index 7a81f3b40..85c9b706a 100644 --- a/programs/benches/linear_search.c +++ b/programs/benches/linear_search.c @@ -17,7 +17,7 @@ typedef struct linear_search_return_values } result; } linear_search_return_values_t; -static void run_bench(linear_search_return_values_t *, uint64_t) +static void run_bench(linear_search_return_values_t *, uint64_t, uint64_t) __attribute__((weakref("_mlir_ciface_linear_search::linear_search::main(f4)"))); extern uint64_t* cairo_native__set_costs_builtin(uint64_t*); @@ -30,7 +30,7 @@ int main() linear_search_return_values_t return_values; - run_bench(&return_values, 0xFFFFFFFFFFFFFFFF); + run_bench(&return_values, 0, 0xFFFFFFFFFFFFFFFF); assert((return_values.result.discriminant & 0x1) == 0); return 0; diff --git a/programs/benches/logistic_map.c b/programs/benches/logistic_map.c index e8d118291..b88cd985a 100644 --- a/programs/benches/logistic_map.c +++ b/programs/benches/logistic_map.c @@ -17,7 +17,7 @@ typedef struct map_return_values } result; } map_return_values_t; -static void run_bench(map_return_values_t *, uint64_t) +static void run_bench(map_return_values_t *, uint64_t, uint64_t) __attribute__((weakref("_mlir_ciface_logistic_map::logistic_map::main(f2)"))); extern uint64_t* cairo_native__set_costs_builtin(uint64_t*); @@ -30,7 +30,7 @@ int main() map_return_values_t return_values; - run_bench(&return_values, 0xFFFFFFFFFFFFFFFF); + run_bench(&return_values, 0, 0xFFFFFFFFFFFFFFFF); assert((return_values.result.discriminant & 0x1) == 0); return 0;