Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add more dict benches #977

Merged
merged 8 commits into from
Dec 19, 2024
Merged

Add more dict benches #977

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c879315
Add more benches
JulianGCalderon Dec 16, 2024
8c7c107
Rename tests
JulianGCalderon Dec 16, 2024
93917a4
Fix hyperfine benches
JulianGCalderon Dec 17, 2024
f1bf858
Add segment arena builtin
JulianGCalderon Dec 17, 2024
12d1e87
Add RangeCheck as first argument to `run_bench`
JulianGCalderon Dec 17, 2024
b1a080d
Increase bench size
JulianGCalderon Dec 18, 2024
e0847c6
Merge branch 'main' into dict-benches
JulianGCalderon Dec 18, 2024
fa43b91
Merge branch 'main' into dict-benches
JulianGCalderon Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
263 changes: 58 additions & 205 deletions benches/benches.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,227 +16,80 @@ use criterion::{criterion_group, criterion_main, Criterion};
use starknet_types_core::felt::Felt;
use std::path::Path;

fn criterion_benchmark(c: &mut Criterion) {
fn compare(c: &mut Criterion, path: impl AsRef<Path>) {
let context = NativeContext::new();
let mut aot_cache = AotProgramCache::new(&context);
let mut jit_cache = JitProgramCache::new(&context);

let factorial = load_contract("programs/benches/factorial_2M.cairo");
let fibonacci = load_contract("programs/benches/fib_2M.cairo");
let logistic_map = load_contract("programs/benches/logistic_map.cairo");
let linear_search = load_contract("programs/benches/linear_search.cairo");

let aot_factorial = aot_cache
.compile_and_insert(Felt::ZERO, &factorial, OptLevel::Aggressive)
.unwrap();
let aot_fibonacci = aot_cache
.compile_and_insert(Felt::ONE, &fibonacci, OptLevel::Aggressive)
.unwrap();
let aot_logistic_map = aot_cache
.compile_and_insert(Felt::from(2), &logistic_map, OptLevel::Aggressive)
.unwrap();
let aot_linear_search = aot_cache
.compile_and_insert(Felt::from(2), &linear_search, OptLevel::Aggressive)
.unwrap();

let jit_factorial = jit_cache
.compile_and_insert(Felt::ZERO, &factorial, OptLevel::Aggressive)
.unwrap();
let jit_fibonacci = jit_cache
.compile_and_insert(Felt::ONE, &fibonacci, OptLevel::Aggressive)
.unwrap();
let jit_logistic_map = jit_cache
.compile_and_insert(Felt::from(2), &logistic_map, OptLevel::Aggressive)
let stem = path
.as_ref()
.file_stem()
.unwrap()
.to_str()
.unwrap()
.to_string();

let program = load_contract(&path);
let aot_executor = aot_cache
.compile_and_insert(Felt::ZERO, &program, OptLevel::Aggressive)
.unwrap();
let jit_linear_search = jit_cache
.compile_and_insert(Felt::from(2), &linear_search, OptLevel::Aggressive)
let jit_executor = jit_cache
.compile_and_insert(Felt::ZERO, &program, OptLevel::Aggressive)
.unwrap();

let factorial_function_id =
find_function_id(&factorial, "factorial_2M::factorial_2M::main").unwrap();
let fibonacci_function_id = find_function_id(&fibonacci, "fib_2M::fib_2M::main").unwrap();
let logistic_map_function_id =
find_function_id(&logistic_map, "logistic_map::logistic_map::main").unwrap();
let linear_search_function_id =
find_function_id(&linear_search, "linear_search::linear_search::main").unwrap();
let main_name = format!("{stem}::{stem}::main");
let main_id = find_function_id(&program, &main_name).unwrap();

let factorial_runner = load_contract_for_vm("programs/benches/factorial_2M.cairo");
let fibonacci_runner = load_contract_for_vm("programs/benches/fib_2M.cairo");
let logistic_map_runner = load_contract_for_vm("programs/benches/logistic_map.cairo");
let linear_search_runner = load_contract_for_vm("programs/benches/linear_search.cairo");

let factorial_function = factorial_runner
.find_function("main")
.expect("failed to find main factorial function");
let fibonacci_function = fibonacci_runner
.find_function("main")
.expect("failed to find main fibonacci function");
let logistic_map_function = logistic_map_runner
let vm_runner = load_contract_for_vm(&path);
let vm_main_id = vm_runner
.find_function("main")
.expect("failed to find main logistic map function");
let linear_search_function = linear_search_runner
.find_function("main")
.expect("failed to find main logistic map function");

{
let mut linear_search_group = c.benchmark_group("linear_search");

linear_search_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_linear_search
.invoke_dynamic(linear_search_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});
linear_search_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_linear_search
.invoke_dynamic(linear_search_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

linear_search_group.bench_function("VM", |b| {
b.iter(|| {
let result = linear_search_runner
.run_function_with_starknet_context(
linear_search_function,
vec![],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});

linear_search_group.finish();
}

{
let mut factorial_group = c.benchmark_group("factorial_2M");

factorial_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_factorial
.invoke_dynamic(factorial_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});
factorial_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_factorial
.invoke_dynamic(factorial_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

factorial_group.bench_function("VM", |b| {
b.iter(|| {
let result = factorial_runner
.run_function_with_starknet_context(
factorial_function,
vec![],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});

factorial_group.finish();
}
.expect("failed to find main function");

{
let mut fibonacci_group = c.benchmark_group("fibonacci_2M");
let mut group = c.benchmark_group(stem);

fibonacci_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_fibonacci
.invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_executor
.invoke_dynamic(main_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
fibonacci_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_fibonacci
.invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
})
});
group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_executor
.invoke_dynamic(main_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
fibonacci_group.bench_function("VM", |b| {
b.iter(|| {
let result = fibonacci_runner
.run_function_with_starknet_context(
fibonacci_function,
vec![],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});
group.bench_function("VM", |b| {
b.iter(|| {
let result = vm_runner
.run_function_with_starknet_context(
vm_main_id,
vec![],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});

fibonacci_group.finish();
}

{
let mut logistic_map_group = c.benchmark_group("logistic_map");

logistic_map_group.bench_function("Cached JIT", |b| {
b.iter(|| {
let result = jit_logistic_map
.invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

logistic_map_group.bench_function("Cached AOT", |b| {
b.iter(|| {
let result = aot_logistic_map
.invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX))
.unwrap();
let value = result.return_value;
assert!(matches!(value, Value::Enum { tag: 0, .. }))
});
});

logistic_map_group.bench_function("VM", |b| {
b.iter(|| {
let result = logistic_map_runner
.run_function_with_starknet_context(
logistic_map_function,
vec![],
Some(usize::MAX),
StarknetState::default(),
)
.unwrap();
let value = result.value;
assert!(matches!(value, RunResultValue::Success(_)))
});
});
group.finish();
}

logistic_map_group.finish();
}
fn criterion_benchmark(c: &mut Criterion) {
compare(c, "programs/benches/dict_snapshot.cairo");
compare(c, "programs/benches/dict_insert.cairo");
compare(c, "programs/benches/factorial_2M.cairo");
compare(c, "programs/benches/fib_2M.cairo");
compare(c, "programs/benches/linear_search.cairo");
compare(c, "programs/benches/logistic_map.cairo");
}

fn load_contract(path: impl AsRef<Path>) -> Program {
Expand Down
38 changes: 38 additions & 0 deletions programs/benches/dict_insert.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#include <assert.h>
#include <stdint.h>


typedef struct dict_insert_return_values
{
uint64_t range_check_counter;
uint64_t segment_arena_counter;
uint64_t remaining_gas;
struct {
uint8_t discriminant;
struct {
void *ptr;
uint32_t start;
uint32_t end;
uint32_t cap;
} err;
} result;
} dict_insert_return_values_t;

static void run_bench(dict_insert_return_values_t *, uint64_t, uint64_t, uint64_t)
__attribute__((weakref("_mlir_ciface_dict_insert::dict_insert::main(f3)")));

extern uint64_t* cairo_native__set_costs_builtin(uint64_t*);

int main()
{
uint64_t BuiltinCosts[7] = {1, 4050, 583, 4085, 491, 230, 604};

cairo_native__set_costs_builtin(&BuiltinCosts[0]);

dict_insert_return_values_t return_values;

run_bench(&return_values, 0, 0, 0xFFFFFFFFFFFFFFFF);
assert((return_values.result.discriminant & 0x1) == 0);

return 0;
}
19 changes: 19 additions & 0 deletions programs/benches/dict_insert.cairo
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
fn init_dict(length: u64) -> Felt252Dict<felt252> {
let mut balances: Felt252Dict<felt252> = Default::default();

for i in 0..length {
let x: felt252 = i.into();
balances.insert(x, x);
};

return balances;
}

fn main() {
let mut dict = init_dict(1000001);
let last = dict.get(1000000);
assert(
last == 1000000,
'invalid result'
);
}
38 changes: 38 additions & 0 deletions programs/benches/dict_snapshot.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#include <assert.h>
#include <stdint.h>


typedef struct dict_snapshot_return_values
{
uint64_t range_check_counter;
uint64_t segment_arena_counter;
uint64_t remaining_gas;
struct {
uint8_t discriminant;
struct {
void *ptr;
uint32_t start;
uint32_t end;
uint32_t cap;
} err;
} result;
} dict_snapshot_return_values_t;

static void run_bench(dict_snapshot_return_values_t *, uint64_t, uint64_t, uint64_t)
__attribute__((weakref("_mlir_ciface_dict_snapshot::dict_snapshot::main(f4)")));
JulianGCalderon marked this conversation as resolved.
Show resolved Hide resolved

extern uint64_t* cairo_native__set_costs_builtin(uint64_t*);

int main()
{
uint64_t BuiltinCosts[7] = {1, 4050, 583, 4085, 491, 230, 604};

cairo_native__set_costs_builtin(&BuiltinCosts[0]);

dict_snapshot_return_values_t return_values;

run_bench(&return_values, 0, 0, 0xFFFFFFFFFFFFFFFF);
assert((return_values.result.discriminant & 0x1) == 0);

return 0;
}
Loading
Loading