Add Llvm debug ir info with sierra source code (#736)

* Simplify debug utils. * Add auto breakpoint. * Fix stuff. * Add fallback libfunc location name if missing * llvm debug * works * prog * progress * progress * move to own module * progress * add arrays * wip prog * fix * reset2 * only relate to sierra * fix * make prettier * cleanup * remove unused code * remove debug trap * fixes * docs * fix * fix doct test * fixes * add docu * docs --------- Co-authored-by: Esteve Soler Arderiu <[email protected]>
lambdaclass · Aug 15, 2024 · 51d2a42 · 51d2a42
1 parent 1114cf2
commit 51d2a42
Show file tree

Hide file tree

Showing 32 changed files with 1,207 additions and 1,104 deletions.
diff --git a/README.md b/README.md
@@ -893,20 +893,30 @@ the `target/` folder besides the generated JSON sierra files.
 
 ### Useful environment variables
 
-These 2 env vars will dump the generated MLIR code from any compilation on the current working directory as:
+This env will dump the generated MLIR code from any compilation on the current working directory as:
 
 - `dump.mlir`: The MLIR code after passes without locations.
 - `dump-debug.mlir`: The MLIR code after passes with locations.
 - `dump-prepass.mlir`: The MLIR code before without locations.
 - `dump-prepass-debug.mlir`: The MLIR code before passes with locations.
+- `program.sierra`: The compiled sierra code, if using a debugger such as lldb, this file path is the default used in debug info to show sources.
 
 Do note that the MLIR with locations is in pretty form and thus not suitable to pass to `mlir-opt`.
 
 ```bash
-export NATIVE_DEBUG_DUMP_PREPASS=1
 export NATIVE_DEBUG_DUMP=1
 ```
 
+Add a debugger breakpoint trap at the given sierra statement:
+
+The trap instruction may not end up exactly where the statement is.
+
+```bash
+export NATIVE_DEBUG_TRAP_AT_STMT=10
+```
+
+Note: The debugger will only show source locations when using AOT. Also you need enable the cairo-native feature `with-debug-utils` and have the env var `NATIVE_DEBUG_DUMP` set.
+
 Enable logging to see the compilation process:
 
 ```bash

diff --git a/benches/compile_time.rs b/benches/compile_time.rs
@@ -14,7 +14,7 @@ pub fn bench_compile_time(c: &mut Criterion) {
             c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
                 b.iter(|| {
                     let native_context = NativeContext::new();
-                    native_context.compile(program, None).unwrap();
+                    native_context.compile(program).unwrap();
                     // pass manager internally verifies the MLIR output is correct.
                 })
             });
@@ -29,7 +29,7 @@ pub fn bench_compile_time(c: &mut Criterion) {
         for (program, filename) in &programs {
             c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
                 b.iter(|| {
-                    native_context.compile(program, None).unwrap();
+                    native_context.compile(program).unwrap();
                     // pass manager internally verifies the MLIR output is correct.
                 })
             });
@@ -43,7 +43,7 @@ pub fn bench_compile_time(c: &mut Criterion) {
             c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
                 b.iter(|| {
                     let native_context = NativeContext::new();
-                    let module = native_context.compile(black_box(program), None).unwrap();
+                    let module = native_context.compile(black_box(program)).unwrap();
                     let object = module_to_object(module.module(), OptLevel::None)
                         .expect("to compile correctly to a object file");
                     black_box(object)
@@ -60,7 +60,7 @@ pub fn bench_compile_time(c: &mut Criterion) {
         for (program, filename) in &programs {
             c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
                 b.iter(|| {
-                    let module = native_context.compile(black_box(program), None).unwrap();
+                    let module = native_context.compile(black_box(program)).unwrap();
                     let object = module_to_object(module.module(), OptLevel::None)
                         .expect("to compile correctly to a object file");
                     black_box(object)

diff --git a/benches/libfuncs.rs b/benches/libfuncs.rs
@@ -50,7 +50,7 @@ pub fn bench_libfuncs(c: &mut Criterion) {
                 |b, program| {
                     let native_context = NativeContext::new();
                     b.iter(|| {
-                        let module = native_context.compile(program, None).unwrap();
+                        let module = native_context.compile(program).unwrap();
                         // pass manager internally verifies the MLIR output is correct.
                         let native_executor =
                             JitNativeExecutor::from_native_module(module, Default::default());
@@ -69,7 +69,7 @@ pub fn bench_libfuncs(c: &mut Criterion) {
                 program,
                 |b, program| {
                     let native_context = NativeContext::new();
-                    let module = native_context.compile(program, None).unwrap();
+                    let module = native_context.compile(program).unwrap();
                     // pass manager internally verifies the MLIR output is correct.
                     let native_executor =
                         JitNativeExecutor::from_native_module(module, Default::default());

diff --git a/examples/easy_api.rs b/examples/easy_api.rs
@@ -1,4 +1,6 @@
-use cairo_native::{context::NativeContext, executor::JitNativeExecutor, values::JitValue};
+use cairo_native::{
+    context::NativeContext, executor::JitNativeExecutor, utils::cairo_to_sierra, values::JitValue,
+};
 use starknet_types_core::felt::Felt;
 use std::path::Path;
 
@@ -10,16 +12,10 @@ fn main() {
     let native_context = NativeContext::new();
 
     // Compile the cairo program to sierra.
-    let (sierra_program, debug_locations) = cairo_native::utils::cairo_to_sierra_with_debug_info(
-        native_context.context(),
-        program_path,
-    )
-    .unwrap();
+    let sierra_program = cairo_to_sierra(program_path);
 
     // Compile the sierra program into a MLIR module.
-    let native_program = native_context
-        .compile(&sierra_program, Some(debug_locations))
-        .unwrap();
+    let native_program = native_context.compile(&sierra_program).unwrap();
 
     // The parameters of the entry point.
     let params = &[JitValue::Felt252(Felt::from_bytes_be_slice(b"user"))];

diff --git a/examples/erc20.rs b/examples/erc20.rs
@@ -296,7 +296,7 @@ fn main() {
 
     let native_context = NativeContext::new();
 
-    let native_program = native_context.compile(&sierra_program, None).unwrap();
+    let native_program = native_context.compile(&sierra_program).unwrap();
 
     let entry_point_fn =
         find_entry_point_by_idx(&sierra_program, entry_point.function_idx).unwrap();

diff --git a/examples/invoke.rs b/examples/invoke.rs
@@ -20,7 +20,7 @@ fn main() {
 
     let native_context = NativeContext::new();
 
-    let native_program = native_context.compile(&sierra_program, None).unwrap();
+    let native_program = native_context.compile(&sierra_program).unwrap();
 
     // Call the echo function from the contract using the generated wrapper.
 

diff --git a/examples/starknet.rs b/examples/starknet.rs
@@ -424,7 +424,7 @@ fn main() {
 
     let native_context = NativeContext::new();
 
-    let native_program = native_context.compile(&sierra_program, None).unwrap();
+    let native_program = native_context.compile(&sierra_program).unwrap();
 
     // Call the echo function from the contract using the generated wrapper.
 

diff --git a/src/bin/cairo-native-compile.rs b/src/bin/cairo-native-compile.rs
@@ -1,8 +1,7 @@
 use anyhow::Context;
 use cairo_lang_compiler::project::check_compiler_path;
 use cairo_native::{
-    context::NativeContext, module_to_object, object_to_shared_lib,
-    utils::cairo_to_sierra_with_debug_info,
+    context::NativeContext, module_to_object, object_to_shared_lib, utils::cairo_to_sierra,
 };
 use clap::{Parser, ValueEnum};
 use std::path::PathBuf;
@@ -53,13 +52,10 @@ fn main() -> anyhow::Result<()> {
     check_compiler_path(args.single_file, &args.path)?;
 
     let native_context = NativeContext::new();
-    let (sierra_program, debug_locations) =
-        cairo_to_sierra_with_debug_info(native_context.context(), &args.path)?;
+    let sierra_program = cairo_to_sierra(&args.path);
 
     // Compile the sierra program into a MLIR module.
-    let native_module = native_context
-        .compile(&sierra_program, Some(debug_locations))
-        .unwrap();
+    let native_module = native_context.compile(&sierra_program).unwrap();
 
     let output_mlir = args
         .output_mlir
@@ -71,12 +67,19 @@ fn main() -> anyhow::Result<()> {
     )
     .context("Failed to write output.")?;
 
-    if let Some(output_library) = &args.output_library {
-        let object_data = module_to_object(native_module.module(), args.opt_level.into())
-            .context("Failed to convert module to object.")?;
-        object_to_shared_lib(&object_data, output_library)
-            .context("Failed to write shared library.")?;
-    }
+    let output_lib = args.output_library.unwrap_or_else(|| {
+        PathBuf::from({
+            if cfg!(target_os = "macos") {
+                "out.dylib"
+            } else {
+                "out.so"
+            }
+        })
+    });
+
+    let object_data = module_to_object(native_module.module(), args.opt_level.into())
+        .context("Failed to convert module to object.")?;
+    object_to_shared_lib(&object_data, &output_lib).context("Failed to write shared library.")?;
 
     Ok(())
 }
diff --git a/src/bin/cairo-native-dump.rs b/src/bin/cairo-native-dump.rs
@@ -1,6 +1,5 @@
 use cairo_lang_compiler::{
-    compile_prepared_db, db::RootDatabase, diagnostics::DiagnosticsReporter,
-    project::setup_project, CompilerConfig,
+    compile_prepared_db, db::RootDatabase, project::setup_project, CompilerConfig,
 };
 use cairo_lang_defs::plugin::NamedPlugin;
 use cairo_lang_semantic::plugin::PluginSuite;
@@ -9,12 +8,9 @@ use cairo_lang_starknet::{
     compile::compile_contract_in_prepared_db, inline_macros::selector::SelectorMacro,
     plugin::StarkNetPlugin,
 };
-use cairo_native::{
-    context::NativeContext,
-    debug_info::{DebugInfo, DebugLocations},
-};
+use cairo_native::context::NativeContext;
 use clap::Parser;
-use melior::{ir::operation::OperationPrintingFlags, Context};
+use melior::ir::operation::OperationPrintingFlags;
 use std::{
     ffi::OsStr,
     fs,
@@ -36,15 +32,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // Load the program.
     let context = NativeContext::new();
-    // TODO: Reconnect debug information.
-    let (program, debug_info) = load_program(
-        Path::new(&args.input),
-        Some(context.context()),
-        args.starknet,
-    )?;
+    let program = load_program(Path::new(&args.input), args.starknet)?;
 
     // Compile the program.
-    let module = context.compile(&program, debug_info)?;
+    let module = context.compile(&program)?;
 
     // Write the output.
     let output_str = module
@@ -59,37 +50,20 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
-fn load_program<'c>(
-    path: &Path,
-    context: Option<&'c Context>,
-    is_contract: bool,
-) -> Result<(Program, Option<DebugLocations<'c>>), Box<dyn std::error::Error>> {
+fn load_program(path: &Path, is_contract: bool) -> Result<Program, Box<dyn std::error::Error>> {
     Ok(match path.extension().and_then(OsStr::to_str) {
         Some("cairo") if !is_contract => {
             let mut db = RootDatabase::builder().detect_corelib().build()?;
             let main_crate_ids = setup_project(&mut db, path)?;
-            let program = compile_prepared_db(
+
+            compile_prepared_db(
                 &mut db,
                 main_crate_ids,
                 CompilerConfig {
                     replace_ids: true,
                     ..Default::default()
                 },
-            )?;
-
-            let debug_locations = if let Some(context) = context {
-                let debug_info = DebugInfo::extract(&db, &program).map_err(|_| {
-                    let mut buffer = String::new();
-                    assert!(DiagnosticsReporter::write_to_string(&mut buffer).check(&db));
-                    buffer
-                })?;
-
-                Some(DebugLocations::extract(context, &db, &debug_info))
-            } else {
-                None
-            };
-
-            (program, debug_locations)
+            )?
         }
         Some("cairo") if is_contract => {
             // mimics cairo_lang_starknet::contract_class::compile_path
@@ -114,31 +88,16 @@ fn load_program<'c>(
                 },
             )?;
 
-            let program = contract.extract_sierra_program()?;
-
-            let debug_locations = if let Some(context) = context {
-                let debug_info = DebugInfo::extract(&db, &program).map_err(|_| {
-                    let mut buffer = String::new();
-                    assert!(DiagnosticsReporter::write_to_string(&mut buffer).check(&db));
-                    buffer
-                })?;
-
-                Some(DebugLocations::extract(context, &db, &debug_info))
-            } else {
-                None
-            };
-
-            (program, debug_locations)
+            contract.extract_sierra_program()?
         }
         Some("sierra") => {
             let program_src = fs::read_to_string(path)?;
 
             let program_parser = ProgramParser::new();
-            let program = program_parser
-                .parse(&program_src)
-                .map_err(|e| e.map_token(|t| t.to_string()))?;
 
-            (program, None)
+            program_parser
+                .parse(&program_src)
+                .map_err(|e| e.map_token(|t| t.to_string()))?
         }
         _ => unreachable!(),
     })

diff --git a/src/bin/cairo-native-run.rs b/src/bin/cairo-native-run.rs
@@ -15,7 +15,6 @@ use cairo_lang_sierra_generator::{
 use cairo_lang_starknet::contract::get_contracts_info;
 use cairo_native::{
     context::NativeContext,
-    debug_info::{DebugInfo, DebugLocations},
     executor::{AotNativeExecutor, JitNativeExecutor, NativeExecutor},
     metadata::gas::{GasMetadata, MetadataComputationConfig},
 };
@@ -95,22 +94,8 @@ fn main() -> anyhow::Result<()> {
 
     let native_context = NativeContext::new();
 
-    let debug_locations = {
-        let debug_info = DebugInfo::extract(db, &sierra_program)
-            .map_err(|_| {
-                let mut buffer = String::new();
-                assert!(DiagnosticsReporter::write_to_string(&mut buffer).check(db));
-                buffer
-            })
-            .unwrap();
-
-        DebugLocations::extract(native_context.context(), db, &debug_info)
-    };
-
     // Compile the sierra program into a MLIR module.
-    let native_module = native_context
-        .compile(&sierra_program, Some(debug_locations))
-        .unwrap();
+    let native_module = native_context.compile(&sierra_program).unwrap();
 
     let native_executor: NativeExecutor = match args.run_mode {
         RunMode::Aot => {

diff --git a/src/bin/cairo-native-stress/main.rs b/src/bin/cairo-native-stress/main.rs
@@ -275,7 +275,7 @@ where
     ) -> Arc<AotNativeExecutor> {
         let native_module = self
             .context
-            .compile(program, None)
+            .compile(program)
             .expect("failed to compile program");
 
         let registry = ProgramRegistry::new(program).expect("failed to get program registry");

diff --git a/src/bin/scarb-native-dump.rs b/src/bin/scarb-native-dump.rs
@@ -39,7 +39,7 @@ fn main() -> anyhow::Result<()> {
 
                 // Compile the sierra program into a MLIR module.
                 let native_module = native_context
-                    .compile(&compiled.into_v1().unwrap().program, None)
+                    .compile(&compiled.into_v1().unwrap().program)
                     .unwrap();
 
                 // Write the output.

diff --git a/src/bin/utils/test.rs b/src/bin/utils/test.rs
@@ -134,16 +134,7 @@ pub fn run_tests(
     let native_context = NativeContext::new();
 
     // Compile the sierra program into a MLIR module.
-    let native_module = native_context
-        .compile_with_metadata(
-            &sierra_program,
-            MetadataComputationConfig {
-                function_set_costs: function_set_costs.clone(),
-                linear_ap_change_solver: true,
-                linear_gas_solver: true,
-            },
-        )
-        .unwrap();
+    let native_module = native_context.compile(&sierra_program).unwrap();
 
     let native_executor: NativeExecutor = match args.run_mode {
         RunMode::Aot => {

diff --git a/src/cache/aot.rs b/src/cache/aot.rs
@@ -44,7 +44,7 @@ where
             module,
             registry,
             metadata,
-        } = self.context.compile(program, None).expect("should compile");
+        } = self.context.compile(program).expect("should compile");
 
         // Compile module into an object.
         let object_data = crate::ffi::module_to_object(&module, opt_level).unwrap();

diff --git a/src/cache/jit.rs b/src/cache/jit.rs
@@ -45,7 +45,7 @@ where
         program: &Program,
         opt_level: OptLevel,
     ) -> Arc<JitNativeExecutor<'a>> {
-        let module = self.context.compile(program, None).expect("should compile");
+        let module = self.context.compile(program).expect("should compile");
         let executor = JitNativeExecutor::from_native_module(module, opt_level);
 
         let executor = Arc::new(executor);