Skip to content

Commit

Permalink
Add more dict benches (#977)
Browse files Browse the repository at this point in the history 
* Add more benches

* Rename tests

* Fix hyperfine benches

* Add segment arena builtin

* Add RangeCheck as first argument to `run_bench`

* Increase bench size
  • Loading branch information
@JulianGCalderon
JulianGCalderon authored Dec 19, 2024
1 parent e47a42e commit f53c947
Show file tree
Hide file tree
Showing 9 changed files with 199 additions and 213 deletions.
263 changes: 58 additions & 205 deletions benches/benches.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,227 +16,80 @@ use criterion::{criterion_group, criterion_main, Criterion};
use starknet_types_core::felt::Felt;
use std::path::Path;

fn criterion_benchmark(c: &mut Criterion) {
fn compare(c: &mut Criterion, path: impl AsRef<Path>) {
let context = NativeContext::new();
let mut aot_cache = AotProgramCache::new(&context);
let mut jit_cache = JitProgramCache::new(&context);

let factorial = load_contract("programs/benches/factorial_2M.cairo");
let fibonacci = load_contract("programs/benches/fib_2M.cairo");
let logistic_map = load_contract("programs/benches/logistic_map.cairo");
let linear_search = load_contract("programs/benches/linear_search.cairo");

let aot_factorial = aot_cache
.compile_and_insert(Felt::ZERO, &factorial, OptLevel::Aggressive)
.unwrap();
let aot_fibonacci = aot_cache
.compile_and_insert(Felt::ONE, &fibonacci, OptLevel::Aggressive)
.unwrap();
let aot_logistic_map = aot_cache
.compile_and_insert(Felt::from(2), &logistic_map, OptLevel::Aggressive)
.unwrap();
let aot_linear_search = aot_cache
.compile_and_insert(Felt::from(2), &linear_search, OptLevel::Aggressive)
.unwrap();

let jit_factorial = jit_cache
.compile_and_insert(Felt::ZERO, &factorial, OptLevel::Aggressive)
.unwrap();
let jit_fibonacci = jit_cache
.compile_and_insert(Felt::ONE, &fibonacci, OptLevel::Aggressive)
.unwrap();
let jit_logistic_map = jit_cache
.compile_and_insert(Felt::from(2), &logistic_map, OptLevel::Aggressive)
let stem = path
.as_ref()
.file_stem()
.unwrap()
.to_str()
.unwrap()
.to_string();

let program = load_contract(&path);
let aot_executor = aot_cache
.compile_and_insert(Felt::ZERO, &program, OptLevel::Aggressive)
.unwrap();
let jit_linear_search = jit_cache
.compile_and_insert(Felt::from(2), &linear_search, OptLevel::Aggressive)
let jit_executor = jit_cache
.compile_and_insert(Felt::ZERO, &program, OptLevel::Aggressive)
.unwrap();

let factorial_function_id =
find_function_id(&factorial, "factorial_2M::factorial_2M::main").unwrap();
let fibonacci_function_id = find_function_id(&fibonacci, "fib_2M::fib_2M::main").unwrap();
let logistic_map_function_id =
find_function_id(&logistic_map, "logistic_map::logistic_map::main").unwrap();
let linear_search_function_id =
find_function_id(&linear_search, "linear_search::linear_search::main").unwrap();
let main_name = format!("{stem}::{stem}::main");
let main_id = find_function_id(&program, &main_name).unwrap();

let factorial_runner = load_contract_for_vm("programs/benches/factorial_2M.cairo");
let fibonacci_runner = load_contract_for_vm("programs/benches/fib_2M.cairo");
let logistic_map_runner = load_contract_for_vm("programs/benches/logistic_map.cairo");
let linear_search_runner = load_contract_for_vm("programs/benches/linear_search.cairo");

let factorial_function = factorial_runner
.find_function("main")
.expect("failed to find main factorial function");
let fibonacci_function = fibonacci_runner
.find_function("main")
.expect("failed to find main fibonacci function");
let logistic_map_function = logistic_map_runner
let vm_runner = load_contract_for_vm(&path);
let vm_main_id = vm_runner
.find_function("main")
.expect("failed to find main logistic map function");
let linear_search_function = linear_search_runner
.find_function("main")
.expect("failed to find main logistic map function");

{
let mut linear_search_group = c.benchmark_group("linear_search");

linear_search_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_linear_search
.invoke_dynamic(linear_search_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});
linear_search_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_linear_search
.invoke_dynamic(linear_search_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

linear_search_group.bench_function("VM", |b| {
b.iter(|| {
let result = linear_search_runner
.run_function_with_starknet_context(
linear_search_function,
vec![],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});

linear_search_group.finish();
}

{
let mut factorial_group = c.benchmark_group("factorial_2M");

factorial_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_factorial
.invoke_dynamic(factorial_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});
factorial_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_factorial
.invoke_dynamic(factorial_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

factorial_group.bench_function("VM", |b| {
b.iter(|| {
let result = factorial_runner
.run_function_with_starknet_context(
factorial_function,
vec![],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});

factorial_group.finish();
}
.expect("failed to find main function");

{
let mut fibonacci_group = c.benchmark_group("fibonacci_2M");
let mut group = c.benchmark_group(stem);

fibonacci_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_fibonacci
.invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_executor
.invoke_dynamic(main_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
fibonacci_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_fibonacci
.invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
})
});
group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_executor
.invoke_dynamic(main_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
fibonacci_group.bench_function("VM", |b| {
b.iter(|| {
let result = fibonacci_runner
.run_function_with_starknet_context(
fibonacci_function,
vec![],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});
group.bench_function("VM", |b| {
b.iter(|| {
let result = vm_runner
.run_function_with_starknet_context(
vm_main_id,
vec![],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});

fibonacci_group.finish();
}

{
let mut logistic_map_group = c.benchmark_group("logistic_map");

logistic_map_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_logistic_map
.invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

logistic_map_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_logistic_map
.invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

logistic_map_group.bench_function("VM", |b| {
b.iter(|| {
let result = logistic_map_runner
.run_function_with_starknet_context(
logistic_map_function,
vec![],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});
group.finish();
}

logistic_map_group.finish();
}
fn criterion_benchmark(c: &mut Criterion) {
compare(c, "programs/benches/dict_snapshot.cairo");
compare(c, "programs/benches/dict_insert.cairo");
compare(c, "programs/benches/factorial_2M.cairo");
compare(c, "programs/benches/fib_2M.cairo");
compare(c, "programs/benches/linear_search.cairo");
compare(c, "programs/benches/logistic_map.cairo");
}

fn load_contract(path: impl AsRef<Path>) -> Program {
Expand Down
38 changes: 38 additions & 0 deletions programs/benches/dict_insert.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#include <assert.h>
#include <stdint.h>


typedef struct dict_insert_return_values
{
uint64_t range_check_counter;
uint64_t segment_arena_counter;
uint64_t remaining_gas;
struct {
uint8_t discriminant;
struct {
void *ptr;
uint32_t start;
uint32_t end;
uint32_t cap;
} err;
} result;
} dict_insert_return_values_t;

static void run_bench(dict_insert_return_values_t *, uint64_t, uint64_t, uint64_t)
__attribute__((weakref("_mlir_ciface_dict_insert::dict_insert::main(f3)")));

extern uint64_t* cairo_native__set_costs_builtin(uint64_t*);

int main()
{
uint64_t BuiltinCosts[7] = {1, 4050, 583, 4085, 491, 230, 604};

cairo_native__set_costs_builtin(&BuiltinCosts[0]);

dict_insert_return_values_t return_values;

run_bench(&return_values, 0, 0, 0xFFFFFFFFFFFFFFFF);
assert((return_values.result.discriminant & 0x1) == 0);

return 0;
}
19 changes: 19 additions & 0 deletions programs/benches/dict_insert.cairo
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
fn init_dict(length: u64) -> Felt252Dict<felt252> {
let mut balances: Felt252Dict<felt252> = Default::default();

for i in 0..length {
let x: felt252 = i.into();
balances.insert(x, x);
};

return balances;
}

fn main() {
let mut dict = init_dict(1000001);
let last = dict.get(1000000);
assert(
last == 1000000,
'invalid result'
);
}
38 changes: 38 additions & 0 deletions programs/benches/dict_snapshot.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#include <assert.h>
#include <stdint.h>


typedef struct dict_snapshot_return_values
{
uint64_t range_check_counter;
uint64_t segment_arena_counter;
uint64_t remaining_gas;
struct {
uint8_t discriminant;
struct {
void *ptr;
uint32_t start;
uint32_t end;
uint32_t cap;
} err;
} result;
} dict_snapshot_return_values_t;

static void run_bench(dict_snapshot_return_values_t *, uint64_t, uint64_t, uint64_t)
__attribute__((weakref("_mlir_ciface_dict_snapshot::dict_snapshot::main(f4)")));

extern uint64_t* cairo_native__set_costs_builtin(uint64_t*);

int main()
{
uint64_t BuiltinCosts[7] = {1, 4050, 583, 4085, 491, 230, 604};

cairo_native__set_costs_builtin(&BuiltinCosts[0]);

dict_snapshot_return_values_t return_values;

run_bench(&return_values, 0, 0, 0xFFFFFFFFFFFFFFFF);
assert((return_values.result.discriminant & 0x1) == 0);

return 0;
}
Loading

0 comments on commit f53c947

Please sign in to comment.