Compilation time benches (#381)

* compilation benches

* clean

* better expect msg

* typo

* typo

* format

Cargo.toml

@@ -62,6 +62,7 @@ cairo-lang-starknet = "2.3.1"
 
 [dev-dependencies]
 cairo-lang-runner = "2.3.1"
+criterion = { version = "0.5.1", features = ["html_reports"] }
 lambdaworks-math = "0.1"
 num-traits = "0.2"
 pretty_assertions_sorted = "1.2.3"
@@ -88,5 +89,9 @@ opt-level = 3
 [profile.dev.package."*"]
 opt-level = 1
 
+[[bench]]
+name = "compile_time"
+harness = false
+
 [workspace]
 members = ["runtime"]

benches/compile_time.rs

@@ -0,0 +1,80 @@
+use cairo_native::{context::NativeContext, module_to_object};
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use util::prepare_programs;
+
+mod util;
+
+pub fn bench_compile_time(c: &mut Criterion) {
+    {
+        let mut c = c.benchmark_group("Compilation With Independent Context");
+
+        let programs = prepare_programs();
+
+        for (program, filename) in programs {
+            c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
+                b.iter(|| {
+                    let native_context = NativeContext::new();
+                    native_context.compile(program).unwrap();
+                    // pass manager internally verifies the MLIR output is correct.
+                })
+            });
+        }
+    }
+
+    {
+        let mut c = c.benchmark_group("Compilation With Shared Context");
+
+        let programs = prepare_programs();
+
+        let native_context = NativeContext::new();
+
+        for (program, filename) in programs {
+            c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
+                b.iter(|| {
+                    native_context.compile(program).unwrap();
+                    // pass manager internally verifies the MLIR output is correct.
+                })
+            });
+        }
+    }
+
+    {
+        let mut c = c.benchmark_group("Compilation With Independent Context To Object Code");
+
+        let programs = prepare_programs();
+
+        for (program, filename) in programs {
+            c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
+                b.iter(|| {
+                    let native_context = NativeContext::new();
+                    let module = native_context.compile(black_box(program)).unwrap();
+                    let object = module_to_object(module.module())
+                        .expect("to compile correctly to a object file");
+                    black_box(object)
+                })
+            });
+        }
+    }
+
+    {
+        let mut c = c.benchmark_group("Compilation With Shared Context To Object Code");
+
+        let programs = prepare_programs();
+
+        let native_context = NativeContext::new();
+
+        for (program, filename) in programs {
+            c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
+                b.iter(|| {
+                    let module = native_context.compile(black_box(program)).unwrap();
+                    let object = module_to_object(module.module())
+                        .expect("to compile correctly to a object file");
+                    black_box(object)
+                })
+            });
+        }
+    }
+}
+
+criterion_group!(benches, bench_compile_time);
+criterion_main!(benches);

benches/util.rs

@@ -0,0 +1,22 @@
+use std::{path::Path, sync::Arc};
+
+use cairo_lang_sierra::program::Program;
+
+pub fn prepare_programs() -> impl Iterator<Item = (Arc<Program>, String)> {
+    let programs = Path::new("programs/compile_benches")
+        .read_dir()
+        .unwrap()
+        .filter_map(|entry| {
+            let path = entry.unwrap().path();
+            match path.extension().map(|x| x.to_str().unwrap()) {
+                Some("cairo") => Some((
+                    cairo_native::utils::cairo_to_sierra(&path),
+                    path.file_name().unwrap().to_str().unwrap().to_string(),
+                )),
+                _ => None,
+            }
+        })
+        .collect::<Vec<_>>(); // collect so iter is not lazy evaluated on bench
+
+    programs.into_iter()
+}