From 9741a8fdc3b644c63a3b050cdc4b1b3d4ff4dda7 Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Tue, 12 Mar 2024 15:14:43 +0100 Subject: [PATCH 1/2] implement the concrete cli, project manager --- Cargo.lock | 222 ++++++++ crates/concrete/src/main.rs | 2 +- crates/concrete_ast/src/lib.rs | 3 + crates/concrete_check/src/lib.rs | 244 +++++---- crates/concrete_codegen_mlir/src/codegen.rs | 8 +- crates/concrete_codegen_mlir/src/context.rs | 7 +- crates/concrete_codegen_mlir/src/lib.rs | 33 +- crates/concrete_driver/Cargo.toml | 8 +- crates/concrete_driver/src/config.rs | 23 + crates/concrete_driver/src/lib.rs | 494 ++++++++++++++---- .../src/linker.rs | 87 +-- crates/concrete_driver/tests/checks.rs | 11 +- crates/concrete_driver/tests/common.rs | 32 +- crates/concrete_ir/src/lib.rs | 3 + crates/concrete_ir/src/lowering.rs | 62 ++- crates/concrete_ir/src/lowering/errors.rs | 59 ++- crates/concrete_ir/src/lowering/prepass.rs | 13 + crates/concrete_parser/src/grammar.lalrpop | 1 + crates/concrete_session/src/lib.rs | 6 +- 19 files changed, 1013 insertions(+), 305 deletions(-) create mode 100644 crates/concrete_driver/src/config.rs rename crates/{concrete_codegen_mlir => concrete_driver}/src/linker.rs (59%) diff --git a/Cargo.lock b/Cargo.lock index b05b8d1..098a33d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -226,6 +226,10 @@ name = "cc" version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +dependencies = [ + "jobserver", + "libc", +] [[package]] name = "cexpr" @@ -386,6 +390,7 @@ dependencies = [ name = "concrete_driver" version = "0.1.0" dependencies = [ + "anyhow", "ariadne", "clap", "concrete_ast", @@ -394,11 +399,16 @@ dependencies = [ "concrete_ir", "concrete_parser", "concrete_session", + "git2", + "owo-colors", "salsa-2022", + "serde", "tempfile", "test-case", + "toml", "tracing", "tracing-subscriber", + "walkdir", ] [[package]] @@ -745,6 +755,15 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + [[package]] name = "getrandom" version = "0.2.12" @@ -756,6 +775,21 @@ dependencies = [ "wasi", ] +[[package]] +name = "git2" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b3ba52851e73b46a4c3df1d89343741112003f0f6f13beb0dfac9e457c3fdcd" +dependencies = [ + "bitflags 2.4.2", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "glob" version = "0.3.1" @@ -808,6 +842,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indenter" version = "0.3.3" @@ -859,6 +903,15 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "jobserver" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +dependencies = [ + "libc", +] + [[package]] name = "lalrpop" version = "0.20.2" @@ -908,6 +961,20 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libgit2-sys" +version = "0.16.2+1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + [[package]] name = "libloading" version = "0.8.3" @@ -929,6 +996,32 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libssh2-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "line-wrap" version = "0.1.1" @@ -1141,12 +1234,36 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "owo-colors" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caff54706df99d2a78a5a4e3455ff45448d81ef1bb63c22cd14052ca0e993a3f" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1182,6 +1299,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + [[package]] name = "petgraph" version = "0.6.4" @@ -1471,6 +1594,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +dependencies = [ + "serde", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1732,6 +1864,55 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "toml" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af06656561d28735e9c1cd63dfd57132c8155426aa6af24f36a00a351f88c48e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18769cd1cec395d70860ceb4d932812a0b4d06b1a4bb336745a4d21b9496e992" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tracing" version = "0.1.40" @@ -1808,12 +1989,27 @@ dependencies = [ "thiserror", ] +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.11.0" @@ -1844,6 +2040,17 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.1" @@ -1856,6 +2063,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -2173,6 +2386,15 @@ version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +[[package]] +name = "winnow" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" +dependencies = [ + "memchr", +] + [[package]] name = "xdg" version = "2.5.2" diff --git a/crates/concrete/src/main.rs b/crates/concrete/src/main.rs index 740ed80..5b66d47 100644 --- a/crates/concrete/src/main.rs +++ b/crates/concrete/src/main.rs @@ -1,5 +1,5 @@ use std::error::Error; fn main() -> Result<(), Box> { - concrete_driver::main() + Ok(concrete_driver::main()?) } diff --git a/crates/concrete_ast/src/lib.rs b/crates/concrete_ast/src/lib.rs index 6b7c36c..5b0224a 100644 --- a/crates/concrete_ast/src/lib.rs +++ b/crates/concrete_ast/src/lib.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use modules::Module; pub mod common; @@ -12,5 +14,6 @@ pub mod types; #[derive(Clone, Debug, Eq, PartialEq)] pub struct Program { + pub file_path: Option, pub modules: Vec, } diff --git a/crates/concrete_check/src/lib.rs b/crates/concrete_check/src/lib.rs index 2590944..608d53b 100644 --- a/crates/concrete_check/src/lib.rs +++ b/crates/concrete_check/src/lib.rs @@ -1,20 +1,23 @@ -use std::ops::Range; +use std::{ops::Range, path::PathBuf}; use ariadne::{ColorGenerator, Label, Report, ReportKind}; use concrete_ir::lowering::errors::LoweringError; -use concrete_session::Session; /// Creates a report from a lowering error. pub fn lowering_error_to_report( error: LoweringError, - session: &Session, -) -> Report<(String, Range)> { - let path = session.file_path.display().to_string(); + file_paths: &[PathBuf], +) -> Report<'static, (String, Range)> { let mut colors = ColorGenerator::new(); colors.next(); match error { - LoweringError::ModuleNotFound { span, module } => { + LoweringError::ModuleNotFound { + span, + module, + program_id, + } => { let offset = span.from; + let path = file_paths[program_id].to_str().unwrap().to_string(); Report::build(ReportKind::Error, path.clone(), offset) .with_code("ModuleNotFound") .with_label( @@ -25,65 +28,94 @@ pub fn lowering_error_to_report( .with_message("Unresolved import.") .finish() } - LoweringError::FunctionNotFound { span, function } => { + LoweringError::FunctionNotFound { + span, + function, + program_id, + } => { + let path = file_paths[program_id].to_str().unwrap().to_string(); Report::build(ReportKind::Error, path.clone(), span.from) - .with_code("FunctionNotFound") - .with_label( - Label::new((path, span.into())) - .with_message(format!("Function {function:?} not found.")) - .with_color(colors.next()), - ) - .finish() - }, - LoweringError::StructFieldNotFound { span, name } => { + .with_code("FunctionNotFound") + .with_label( + Label::new((path, span.into())) + .with_message(format!("Function {function:?} not found.")) + .with_color(colors.next()), + ) + .finish() + } + LoweringError::StructFieldNotFound { + span, + name, + program_id, + } => { + let path = file_paths[program_id].to_str().unwrap().to_string(); Report::build(ReportKind::Error, path.clone(), span.from) - .with_code("StructFieldNotFound") - .with_label( - Label::new((path, span.into())) - .with_message(format!("Struct field {name:?} not found.")) - .with_color(colors.next()), - ) - .finish() - }, - LoweringError::ImportNotFound { import_span, module_span, symbol } => { - let offset = symbol.span.from; - Report::build(ReportKind::Error, path.clone(), offset) - .with_code("ImportNotFound") - .with_label( - Label::new((path.clone(), module_span.into())) - .with_message("In module this module."), - ) - .with_label( - Label::new((path.clone(), import_span.into())) - .with_message("In this import statement"), - ) - .with_label( - Label::new((path, symbol.span.into())) - .with_message(format!("Failed to find symbol {:?}", symbol.name)) - .with_color(colors.next()), - ) - .with_message("Unresolved import.") - .finish() - }, - LoweringError::BorrowNotMutable { span, name, type_span } => { - let mut labels = vec![ - Label::new((path.clone(), span.into())) - .with_message(format!("Can't mutate {name:?} because it's behind a immutable borrow")) - .with_color(colors.next()) - ]; + .with_code("StructFieldNotFound") + .with_label( + Label::new((path, span.into())) + .with_message(format!("Struct field {name:?} not found.")) + .with_color(colors.next()), + ) + .finish() + } + LoweringError::ImportNotFound { + import_span, + module_span, + symbol, + program_id, + } => { + let path = file_paths[program_id].to_str().unwrap().to_string(); + let offset = symbol.span.from; + Report::build(ReportKind::Error, path.clone(), offset) + .with_code("ImportNotFound") + .with_label( + Label::new((path.clone(), module_span.into())) + .with_message("In module this module."), + ) + .with_label( + Label::new((path.clone(), import_span.into())) + .with_message("In this import statement"), + ) + .with_label( + Label::new((path, symbol.span.into())) + .with_message(format!("Failed to find symbol {:?}", symbol.name)) + .with_color(colors.next()), + ) + .with_message("Unresolved import.") + .finish() + } + LoweringError::BorrowNotMutable { + span, + name, + type_span, + program_id, + } => { + let path = file_paths[program_id].to_str().unwrap().to_string(); + let mut labels = vec![Label::new((path.clone(), span.into())) + .with_message(format!( + "Can't mutate {name:?} because it's behind a immutable borrow" + )) + .with_color(colors.next())]; if let Some(type_span) = type_span { - labels.push(Label::new((path.clone(), type_span.into())) - .with_message(format!("Variable {name:?} has this type")) - .with_color(colors.next())); + labels.push( + Label::new((path.clone(), type_span.into())) + .with_message(format!("Variable {name:?} has this type")) + .with_color(colors.next()), + ); } Report::build(ReportKind::Error, path.clone(), span.from) - .with_code("BorrowNotMutable") - .with_labels(labels) - .finish() - }, - LoweringError::UnrecognizedType { span, name } => { + .with_code("BorrowNotMutable") + .with_labels(labels) + .finish() + } + LoweringError::UnrecognizedType { + span, + name, + program_id, + } => { + let path = file_paths[program_id].to_str().unwrap().to_string(); Report::build(ReportKind::Error, path.clone(), span.from) .with_code("UnrecognizedType") .with_label( @@ -93,8 +125,13 @@ pub fn lowering_error_to_report( ) .with_message(format!("Unresolved type {:?}.", name)) .finish() - }, - LoweringError::IdNotFound { span, id } => { + } + LoweringError::IdNotFound { + span, + id, + program_id, + } => { + let path = file_paths[program_id].to_str().unwrap().to_string(); Report::build(ReportKind::Error, path.clone(), span.from) .with_code("E_ID") .with_label( @@ -104,8 +141,13 @@ pub fn lowering_error_to_report( ) .with_message(format!("Failed to find definition id {id:?}, this is most likely a compiler bug or a unimplemented lowering")) .finish() - }, - LoweringError::NotYetImplemented { span, message } => { + } + LoweringError::NotYetImplemented { + span, + message, + program_id, + } => { + let path = file_paths[program_id].to_str().unwrap().to_string(); Report::build(ReportKind::Error, path.clone(), span.from) .with_code("NotYetImplemented") .with_label( @@ -114,19 +156,26 @@ pub fn lowering_error_to_report( .with_color(colors.next()), ) .finish() - }, - LoweringError::UnexpectedType { span, found, expected } => { - let mut labels = vec![ - Label::new((path.clone(), span.into())) - .with_message(format!("Unexpected type '{}', expected '{}'", found.kind, expected.kind)) - .with_color(colors.next()) - ]; + } + LoweringError::UnexpectedType { + span, + found, + expected, + program_id, + } => { + let path = file_paths[program_id].to_str().unwrap().to_string(); + let mut labels = vec![Label::new((path.clone(), span.into())) + .with_message(format!( + "Unexpected type '{}', expected '{}'", + found.kind, expected.kind + )) + .with_color(colors.next())]; if let Some(span) = expected.span { labels.push( Label::new((path.clone(), span.into())) .with_message(format!("expected '{}' due to this type", expected.kind)) - .with_color(colors.next()) + .with_color(colors.next()), ); } @@ -135,26 +184,43 @@ pub fn lowering_error_to_report( .with_labels(labels) .with_message(format!("expected type {}.", expected.kind)) .finish() - }, - LoweringError::UseOfUndeclaredVariable { span, name } => { + } + LoweringError::UseOfUndeclaredVariable { + span, + name, + program_id, + } => { + let path = file_paths[program_id].to_str().unwrap().to_string(); Report::build(ReportKind::Error, path.clone(), span.from) - .with_code("UseOfUndeclaredVariable") - .with_label( - Label::new((path, span.into())) - .with_message(format!("Use of undeclared variable {:?}", name)) - .with_color(colors.next()), - ) - .finish() - }, - LoweringError::ExternFnWithBody { span, name } => { + .with_code("UseOfUndeclaredVariable") + .with_label( + Label::new((path, span.into())) + .with_message(format!("Use of undeclared variable {:?}", name)) + .with_color(colors.next()), + ) + .finish() + } + LoweringError::ExternFnWithBody { + span, + name, + program_id, + } => { + let path = file_paths[program_id].to_str().unwrap().to_string(); Report::build(ReportKind::Error, path.clone(), span.from) - .with_code("ExternFnWithBody") - .with_label( - Label::new((path, span.into())) - .with_message(format!("extern function {:?} declared with body", name)) - .with_color(colors.next()), - ) - .finish() - }, + .with_code("ExternFnWithBody") + .with_label( + Label::new((path, span.into())) + .with_message(format!("extern function {:?} declared with body", name)) + .with_color(colors.next()), + ) + .finish() + } + LoweringError::InternalError(msg, program_id) => { + let path = file_paths[program_id].to_str().unwrap().to_string(); + Report::build(ReportKind::Error, path.clone(), 0) + .with_code("InternalError") + .with_message(msg) + .finish() + } } } diff --git a/crates/concrete_codegen_mlir/src/codegen.rs b/crates/concrete_codegen_mlir/src/codegen.rs index 9e2a2c7..c509996 100644 --- a/crates/concrete_codegen_mlir/src/codegen.rs +++ b/crates/concrete_codegen_mlir/src/codegen.rs @@ -59,12 +59,12 @@ impl<'a> ModuleCodegenCtx<'a> { /// Gets a MLIR location from the given span, or unknown if the span is `None`. pub fn get_location(&self, span: Option) -> Location { if let Some(span) = span { - let (_, line, col) = self.ctx.session.source.get_offset_line(span.from).unwrap(); + let (_, line, col) = self.ctx.session.sources[self.module_id.program_id] + .get_offset_line(span.from) + .unwrap(); Location::new( self.ctx.mlir_context, - self.ctx - .session - .file_path + self.ctx.program.file_paths[&self.module_id.program_id] .file_name() .unwrap() .to_str() diff --git a/crates/concrete_codegen_mlir/src/context.rs b/crates/concrete_codegen_mlir/src/context.rs index 4b831d8..5b9d665 100644 --- a/crates/concrete_codegen_mlir/src/context.rs +++ b/crates/concrete_codegen_mlir/src/context.rs @@ -39,8 +39,7 @@ impl Context { session: &Session, program: &ProgramBody, ) -> Result { - let file_path = session.file_path.display().to_string(); - let location = Location::new(&self.melior_context, &file_path, 0, 0); + let location = Location::unknown(&self.melior_context); let target_triple = get_target_triple(session); let module_region = Region::new(); @@ -74,7 +73,7 @@ impl Context { super::codegen::compile_program(codegen_ctx)?; - if session.output_mlir || session.output_all { + if session.output_mlir { std::fs::write( session.output_file.with_extension("before-pass.mlir"), melior_module.as_operation().to_string(), @@ -96,7 +95,7 @@ impl Context { ); } - if session.output_mlir || session.output_all { + if session.output_mlir { std::fs::write( session.output_file.with_extension("after-pass.mlir"), melior_module.as_operation().to_string(), diff --git a/crates/concrete_codegen_mlir/src/lib.rs b/crates/concrete_codegen_mlir/src/lib.rs index 1c7c717..a8e30aa 100644 --- a/crates/concrete_codegen_mlir/src/lib.rs +++ b/crates/concrete_codegen_mlir/src/lib.rs @@ -39,7 +39,6 @@ use module::MLIRModule; mod codegen; mod context; pub mod errors; -pub mod linker; mod module; mod pass_manager; @@ -93,16 +92,10 @@ pub fn get_data_layout_rep(session: &Session) -> Result { let error_buffer = addr_of_mut!(null); let target_triple = LLVMGetDefaultTargetTriple(); - tracing::debug!("Target triple: {:?}", CStr::from_ptr(target_triple)); let target_cpu = LLVMGetHostCPUName(); - tracing::debug!("Target CPU: {:?}", CStr::from_ptr(target_cpu)); let target_cpu_features = LLVMGetHostCPUFeatures(); - tracing::debug!( - "Target CPU Features: {:?}", - CStr::from_ptr(target_cpu_features) - ); let mut target: MaybeUninit = MaybeUninit::uninit(); @@ -155,22 +148,10 @@ pub fn compile_to_object( module: &MLIRModule<'_>, ) -> Result { tracing::debug!("Compiling to object file"); - if !session.target_dir.exists() { - std::fs::create_dir_all(&session.target_dir)?; - } - let target_file = session - .file_path - .clone() - .file_stem() - .unwrap() - .to_string_lossy() - .to_string(); - let target_file = PathBuf::from(target_file).with_extension("o"); + let target_file = session.output_file.with_extension("o"); tracing::debug!("Target file: {:?}", target_file); - let target_path = session.target_dir.join(target_file); - // TODO: Rework so you can specify target and host features, etc. // Right now it compiles for the native cpu feature set and arch @@ -249,9 +230,9 @@ pub fn compile_to_object( LLVMDisposePassBuilderOptions(opts); - if session.output_ll || session.output_all { + if session.output_ll { let filename = CString::new( - target_path + target_file .with_extension("ll") .as_os_str() .to_string_lossy() @@ -270,7 +251,7 @@ pub fn compile_to_object( } } - let filename = CString::new(target_path.as_os_str().to_string_lossy().as_bytes()).unwrap(); + let filename = CString::new(target_file.as_os_str().to_string_lossy().as_bytes()).unwrap(); tracing::debug!("filename to llvm: {:?}", filename); let ok = LLVMTargetMachineEmitToFile( machine, @@ -290,9 +271,9 @@ pub fn compile_to_object( LLVMDisposeMessage(*error_buffer); } - if session.output_asm || session.output_all { + if session.output_asm { let filename = CString::new( - target_path + target_file .with_extension("asm") .as_os_str() .to_string_lossy() @@ -322,6 +303,6 @@ pub fn compile_to_object( LLVMDisposeModule(llvm_module); LLVMContextDispose(llvm_context); - Ok(target_path) + Ok(target_file) } } diff --git a/crates/concrete_driver/Cargo.toml b/crates/concrete_driver/Cargo.toml index 39d5194..4873cb4 100644 --- a/crates/concrete_driver/Cargo.toml +++ b/crates/concrete_driver/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -clap = { version = "4.4.13", features = ["derive"] } +clap = { version = "4.4.16", features = ["derive"] } tracing = { workspace = true } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } concrete_ast = { path = "../concrete_ast"} @@ -17,6 +17,12 @@ salsa = { git = "https://github.com/salsa-rs/salsa.git", package = "salsa-2022" ariadne = { version = "0.4.0", features = ["auto-color"] } concrete_ir = { version = "0.1.0", path = "../concrete_ir" } concrete_check = { version = "0.1.0", path = "../concrete_check" } +walkdir = "2.5.0" +anyhow = "1" +toml = "0.8.10" +git2 = "0.18.2" +owo-colors = "4.0.0" +serde = { version = "1.0.197", features = ["derive"] } [dev-dependencies] tempfile = "3.9.0" diff --git a/crates/concrete_driver/src/config.rs b/crates/concrete_driver/src/config.rs new file mode 100644 index 0000000..071e25b --- /dev/null +++ b/crates/concrete_driver/src/config.rs @@ -0,0 +1,23 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Config { + pub package: Package, + pub profile: HashMap, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Package { + pub name: String, + pub version: String, + pub license: String, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Profile { + pub release: bool, + pub opt_level: u8, + pub debug_info: bool, +} diff --git a/crates/concrete_driver/src/lib.rs b/crates/concrete_driver/src/lib.rs index 0ecce64..7eaae20 100644 --- a/crates/concrete_driver/src/lib.rs +++ b/crates/concrete_driver/src/lib.rs @@ -1,22 +1,75 @@ -use ariadne::Source; -use clap::Parser; -use concrete_codegen_mlir::linker::{link_binary, link_shared_lib}; -use concrete_ir::lowering::lower_program; +use anyhow::bail; +use anyhow::Context; +use anyhow::Result; +use clap::{Parser, Subcommand}; +use concrete_ir::lowering::lower_programs; use concrete_parser::{error::Diagnostics, ProgramSource}; use concrete_session::{ config::{DebugInfo, OptLevel}, Session, }; -use std::{error::Error, path::PathBuf, time::Instant}; +use config::{Package, Profile}; +use git2::{IndexAddOption, Repository}; +use owo_colors::OwoColorize; +use std::io::Read; +use std::{collections::HashMap, fs::File, path::PathBuf, time::Instant}; +use walkdir::WalkDir; +use crate::{ + config::Config, + linker::{link_binary, link_shared_lib}, +}; + +pub mod config; pub mod db; +pub mod linker; + +#[derive(Parser, Debug)] +#[command(author, version, about = "concrete", long_about = None, bin_name = "concrete")] +pub struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand, Debug)] +enum Commands { + /// Initialize a project + New { + path: PathBuf, + + /// The name of the project, defaults to the directory name + #[arg(long)] + name: Option, + + /// Use a binary (application) template [default] + #[arg(long, group = "binary", default_value_t = true)] + bin: bool, + + /// Use a library template + #[arg(long, group = "binary")] + lib: bool, + }, + /// Build a project + Build { + /// Build for release with all optimizations. + #[arg(short, long, default_value_t = false)] + release: bool, + + /// Override the profile to use. + #[arg(short, long)] + profile: Option, + }, +} #[derive(Parser, Debug)] -#[command(author, version, about, long_about = None)] +#[command(author, version, about = "concrete compiler", long_about = None)] pub struct CompilerArgs { /// The input file. input: PathBuf, + /// The output file. + pub output: PathBuf, + /// Build for release with all optimizations. #[arg(short, long, default_value_t = false)] release: bool, @@ -25,87 +78,339 @@ pub struct CompilerArgs { #[arg(short = 'O', long)] optlevel: Option, + /// Always add debug info + #[arg(long)] + pub debug_info: Option, + /// Build as a library. #[arg(short, long, default_value_t = false)] library: bool, - /// Prints the ast. + /// Also output the ast. #[arg(long, default_value_t = false)] - print_ast: bool, + ast: bool, - /// Prints the middle ir. + /// Also output the ir. #[arg(long, default_value_t = false)] - print_ir: bool, + ir: bool, /// Also output the llvm ir file. #[arg(long, default_value_t = false)] - output_ll: bool, + llvm: bool, /// Also output the mlir file #[arg(long, default_value_t = false)] - output_mlir: bool, + mlir: bool, /// Also output the asm file. #[arg(long, default_value_t = false)] - output_asm: bool, + asm: bool, - /// Print all file formats. + /// Also output the object file. #[arg(long, default_value_t = false)] - output_all: bool, + object: bool, } -pub fn main() -> Result<(), Box> { - let start_time = Instant::now(); - +pub fn main() -> Result<()> { tracing_subscriber::fmt::init(); - let args = CompilerArgs::parse(); + let cli = Cli::parse(); - let db = crate::db::Database::default(); - let source = ProgramSource::new( - &db, - std::fs::read_to_string(&args.input)?, - args.input.display().to_string(), - ); - tracing::debug!("source code:\n{}", source.input(&db)); - let parse_ast_time = Instant::now(); - let program = match concrete_parser::parse_ast(&db, source) { - Some(x) => x, - None => { - Diagnostics::dump( - &db, - source, - &concrete_parser::parse_ast::accumulated::( - &db, source, - ), + match cli.command { + Commands::New { + path, + name, + bin, + lib, + } => { + let name = name.unwrap_or_else(|| { + path.file_name() + .context("Failed to get project name") + .unwrap() + .to_string_lossy() + .to_string() + }); + + if !path.exists() { + std::fs::create_dir_all(&path).context("failed to create the project directory")?; + std::fs::create_dir_all(path.join("src")).context("failed to create src/")?; + } + + let config_path = path.join("Concrete.toml"); + + let mut profiles = HashMap::new(); + + profiles.insert( + "release".to_string(), + Profile { + release: true, + opt_level: 3, + debug_info: false, + }, ); - std::process::exit(1); + + profiles.insert( + "dev".to_string(), + Profile { + release: false, + opt_level: 0, + debug_info: true, + }, + ); + + let config = Config { + package: Package { + name: name.clone(), + version: "0.1.0".to_string(), + license: "MIT".to_string(), + }, + profile: profiles, + }; + + std::fs::write(config_path, toml::to_string_pretty(&config)?) + .context("failed to write Concrete.toml")?; + std::fs::write(path.join(".gitignore"), "/build\n") + .context("failed to write .gitignore")?; + std::fs::write( + path.join(".gitattributes"), + "*.con linguist-language=Rust\n", + ) + .context("failed to write .gitattributes")?; + + if bin { + std::fs::write( + path.join("src").join("main.con"), + format!( + r#" +mod {} {{ + pub fn main() -> i32 {{ + return 0; + }} +}}"#, + name + ), + )?; + } + + if lib { + std::fs::write( + path.join("src").join("lib.con"), + format!( + r#" +mod {} {{ + pub fn hello_world() -> i32 {{ + return 0; + }} +}}"#, + name + ), + )?; + } + + { + let repo = Repository::init(&path).context("failed to create repository")?; + let sig = repo.signature()?; + let tree_id = { + let mut index = repo.index()?; + + index.add_all(["."].iter(), IndexAddOption::DEFAULT, None)?; + index.write()?; + index.write_tree()? + }; + + let tree = repo.find_tree(tree_id).context("failed to find git tree")?; + repo.commit(Some("HEAD"), &sig, &sig, "Initial commit", &tree, &[]) + .context("failed to create initial commit")?; + } + + if bin { + println!( + " {} binary (application) `{}` package", + "Created".green().bold(), + name + ); + } else { + println!(" {} library `{}` package", "Created".green(), name); + } } - }; - let parse_ast_time = parse_ast_time.elapsed(); + Commands::Build { release, profile } => { + let mut current_dir = std::env::current_dir()?; + let mut config_path = None; + + // Go up to 3 parent folders to find Concrete.toml + for _ in 0..3 { + if !current_dir.join("Concrete.toml").exists() { + current_dir = if let Some(parent) = current_dir.parent() { + parent.to_path_buf() + } else { + bail!("Couldn't find Concrete.toml"); + }; + } else { + config_path = Some(current_dir.join("Concrete.toml")); + break; + } + } + + let config_path = match config_path { + Some(x) => x, + None => bail!("Couldn't find Concrete.toml"), + }; + + let base_dir = config_path + .parent() + .context("couldn't get config parent dir")?; + let mut config = File::open(&config_path).context("Failed to open Concrete.toml")?; + let mut buf = String::new(); + config.read_to_string(&mut buf)?; - if args.print_ast { - println!("{:#?}", program); + let config: Config = toml::from_str(&buf).context("failed to parse Concrete.toml")?; + + println!( + " {} {} v{} ({})", + "Compiling".green().bold(), + config.package.name, + config.package.version, + base_dir.display() + ); + + let src_dir = base_dir.join("src"); + let target_dir = base_dir.join("build"); + + if !target_dir.exists() { + std::fs::create_dir_all(&target_dir)?; + } + + let has_main = src_dir.join("main.con").exists(); + let output = target_dir.join(config.package.name); + + let (profile, profile_name) = if let Some(profile) = profile { + ( + config + .profile + .get(&profile) + .context("Couldn't get requested profile")?, + profile, + ) + } else if release { + ( + config + .profile + .get("release") + .context("Couldn't get profile: release")?, + "release".to_string(), + ) + } else { + ( + config + .profile + .get("dev") + .context("Couldn't get profile: dev")?, + "dev".to_string(), + ) + }; + + let compile_args = CompilerArgs { + input: src_dir, + output: output.clone(), + release, + optlevel: Some(profile.opt_level), + debug_info: Some(profile.debug_info), + library: !has_main, + ast: false, + ir: false, + llvm: true, + asm: false, + object: true, + mlir: true, + }; + + let start = Instant::now(); + let object = compile(&compile_args)?; + + if !has_main { + link_shared_lib(&[object], &output)?; + } else { + link_binary(&[object], &output)?; + } + + let elapsed = start.elapsed(); + + println!( + " {} {} [{}{}] in {elapsed:?}", + "Finished".green().bold(), + profile_name, + if profile.opt_level > 0 { + "optimized" + } else { + "unoptimized" + }, + if profile.debug_info { + " + debuginfo" + } else { + "" + } + ); + } + } + + Ok(()) +} + +pub fn compile(args: &CompilerArgs) -> Result { + let mut files = Vec::new(); + for entry in WalkDir::new(&args.input) { + let entry = entry?; + if let Some(ext) = entry.path().extension() { + if ext.eq_ignore_ascii_case("con") { + files.push(entry.path().to_path_buf()); + } + } } - let cwd = std::env::current_dir()?; - // todo: find a better name, "target" would clash with rust if running in the source tree. - let target_dir = cwd.join("build_artifacts/"); - if !target_dir.exists() { - std::fs::create_dir_all(&target_dir)?; + if files.is_empty() { + panic!("files is empty"); + } + + let start_time = Instant::now(); + + let mut programs = Vec::new(); + + let db = crate::db::Database::default(); + + let mut sources = Vec::new(); + let mut sources_str = Vec::new(); + + for path in files { + let source = std::fs::read_to_string(&path)?; + let source = ProgramSource::new(&db, source, args.input.display().to_string()); + + let mut program = match concrete_parser::parse_ast(&db, source) { + Some(x) => x, + None => { + Diagnostics::dump( + &db, + source, + &concrete_parser::parse_ast::accumulated::( + &db, source, + ), + ); + std::process::exit(1); + } + }; + program.file_path = Some(path); + sources.push(ariadne::Source::from(source.input(&db).clone())); + sources_str.push(source.input(&db).clone()); + programs.push(program); } - let output_file = target_dir.join(PathBuf::from(args.input.file_name().unwrap())); - let output_file = if args.library { - output_file.with_extension(Session::get_platform_library_ext()) - } else if cfg!(target_os = "windows") { - output_file.with_extension("exe") - } else { - output_file.with_extension("") - }; let session = Session { - file_path: args.input, - debug_info: if args.release { + debug_info: if let Some(debug_info) = args.debug_info { + if debug_info { + DebugInfo::Full + } else { + DebugInfo::None + } + } else if args.release { DebugInfo::None } else { DebugInfo::Full @@ -122,52 +427,65 @@ pub fn main() -> Result<(), Box> { } else { OptLevel::None }, - source: Source::from(source.input(&db).to_string()), + sources, library: args.library, - target_dir, - output_file, - output_mlir: args.output_mlir, - output_ll: args.output_ll, - output_asm: args.output_asm, - output_all: args.output_all, + output_file: args.output.with_extension("o"), + output_asm: args.asm, + output_ll: args.llvm, + output_mlir: args.mlir, }; + tracing::debug!("Output file: {:#?}", session.output_file); + tracing::debug!("Is library: {:#?}", session.library); + tracing::debug!("Optlevel: {:#?}", session.optlevel); + tracing::debug!("Debug Info: {:#?}", session.debug_info); - tracing::debug!("Compiling with session: {:#?}", session); + let path_cache: Vec<_> = programs + .iter() + .zip(&sources_str) + .map(|(program, source)| { + ( + program + .file_path + .clone() + .expect("path should always exist here") + .to_string_lossy() + .to_string(), + source.clone(), + ) + }) + .collect(); - let lower_time = Instant::now(); - let program_ir = match lower_program(&program) { + if args.ast { + std::fs::write( + session.output_file.with_extension("ast"), + format!("{:#?}", programs), + )?; + } + + let program_ir = match lower_programs(&programs) { Ok(ir) => ir, Err(error) => { - let report = concrete_check::lowering_error_to_report(error, &session); - let path = session.file_path.display().to_string(); - report.eprint((path, session.source.clone()))?; + let file_paths: Vec<_> = programs + .iter() + .map(|x| x.file_path.clone().unwrap()) + .collect(); + let report = concrete_check::lowering_error_to_report(error, &file_paths); + report.eprint(ariadne::sources(path_cache))?; std::process::exit(1); } }; - let lower_time = lower_time.elapsed(); - if args.print_ir { - println!("{:#?}", program_ir); + if args.ir { + std::fs::write( + session.output_file.with_extension("ir"), + format!("{:#?}", program_ir), + )?; } - let compile_time = Instant::now(); - let object_path = concrete_codegen_mlir::compile(&session, &program_ir)?; - let compile_time = compile_time.elapsed(); - - let link_time = Instant::now(); - if session.library { - link_shared_lib(&object_path, &session.output_file)?; - } else { - link_binary(&object_path, &session.output_file)?; - } - let link_time = link_time.elapsed(); + let object_path = concrete_codegen_mlir::compile(&session, &program_ir).unwrap(); let elapsed = start_time.elapsed(); - tracing::debug!("Parse time {:?}", parse_ast_time); - tracing::debug!("Lower time {:?}", lower_time); - tracing::debug!("Combined compile time {:?}", compile_time); - tracing::debug!("Link time {:?}", link_time); tracing::debug!("Done in {:?}", elapsed); - Ok(()) + Ok(object_path) } diff --git a/crates/concrete_codegen_mlir/src/linker.rs b/crates/concrete_driver/src/linker.rs similarity index 59% rename from crates/concrete_codegen_mlir/src/linker.rs rename to crates/concrete_driver/src/linker.rs index c3a564d..b8eeb47 100644 --- a/crates/concrete_codegen_mlir/src/linker.rs +++ b/crates/concrete_driver/src/linker.rs @@ -1,43 +1,41 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use tracing::instrument; -use crate::errors::CodegenError; - -// TODO: Implement a proper linker driver, passing only the arguments needed dynamically based on the requirements. - #[instrument(level = "debug")] -pub fn link_shared_lib(input_path: &Path, output_filename: &Path) -> Result<(), CodegenError> { - let args: &[&str] = { +pub fn link_shared_lib(objects: &[PathBuf], output_filename: &Path) -> std::io::Result<()> { + let objects: Vec<_> = objects.iter().map(|x| x.display().to_string()).collect(); + let output_filename = output_filename.to_string_lossy().to_string(); + + let args: Vec<_> = { #[cfg(target_os = "macos")] { - &[ + let mut args = vec![ "-demangle", "-no_deduplicate", "-dynamic", "-dylib", "-L/usr/local/lib", "-L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib", - &input_path.display().to_string(), - "-o", - &output_filename.display().to_string(), - "-lSystem", - ] + ]; + + args.extend(objects.iter().map(|x| x.as_str())); + + args.extend(&["-o", &output_filename, "-lSystem"]); + + args } #[cfg(target_os = "linux")] { - &[ - "--hash-style=gnu", - "--eh-frame-hdr", - "-shared", - "-o", - &output_filename.display().to_string(), - "-L/lib/../lib64", - "-L/usr/lib/../lib64", - "-lc", - "-O1", - &input_path.display().to_string(), - ] + let mut args = vec!["--hash-style=gnu", "--eh-frame-hdr", "-shared"]; + + args.extend(&["-o", &output_filename]); + + args.extend(&["-L/lib/../lib64", "-L/usr/lib/../lib64", "-lc", "-O1"]); + + args.extend(objects.iter().map(|x| x.as_str())); + + args } #[cfg(target_os = "windows")] { @@ -52,18 +50,24 @@ pub fn link_shared_lib(input_path: &Path, output_filename: &Path) -> Result<(), } #[instrument(level = "debug")] -pub fn link_binary(input_path: &Path, output_filename: &Path) -> Result<(), CodegenError> { - let args: &[&str] = { +pub fn link_binary(objects: &[PathBuf], output_filename: &Path) -> std::io::Result<()> { + let objects: Vec<_> = objects.iter().map(|x| x.display().to_string()).collect(); + let output_filename = output_filename.to_string_lossy().to_string(); + + let args: Vec<_> = { #[cfg(target_os = "macos")] { - &[ + let mut args = vec![ "-L/usr/local/lib", "-L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib", &input_path.display().to_string(), - "-o", - &output_filename.display().to_string(), - "-lSystem", - ] + ]; + + args.extend(objects.iter().map(|x| x.as_str())); + + args.extend(&["-o", &output_filename, "-lSystem"]); + + args } #[cfg(target_os = "linux")] { @@ -83,7 +87,7 @@ pub fn link_binary(input_path: &Path, output_filename: &Path) -> Result<(), Code } }; - &[ + let mut args = vec![ "-pie", "--hash-style=gnu", "--eh-frame-hdr", @@ -93,8 +97,11 @@ pub fn link_binary(input_path: &Path, output_filename: &Path) -> Result<(), Code "elf_x86_64", scrt1, crti, - "-o", - &output_filename.display().to_string(), + ]; + + args.extend(&["-o", &output_filename]); + + args.extend(&[ "-L/lib64", "-L/usr/lib64", "-L/lib/x86_64-linux-gnu", @@ -103,8 +110,11 @@ pub fn link_binary(input_path: &Path, output_filename: &Path) -> Result<(), Code "-lc", "-O1", crtn, - &input_path.display().to_string(), - ] + ]); + + args.extend(objects.iter().map(|x| x.as_str())); + + args } #[cfg(target_os = "windows")] { @@ -114,7 +124,8 @@ pub fn link_binary(input_path: &Path, output_filename: &Path) -> Result<(), Code let mut linker = std::process::Command::new("ld"); let proc = linker.args(args.iter()).spawn()?; - proc.wait_with_output()?; + let output = proc.wait_with_output()?; + tracing::debug!("Linker result: {:#?}", output); Ok(()) } diff --git a/crates/concrete_driver/tests/checks.rs b/crates/concrete_driver/tests/checks.rs index 42f586b..2a5e670 100644 --- a/crates/concrete_driver/tests/checks.rs +++ b/crates/concrete_driver/tests/checks.rs @@ -1,4 +1,6 @@ -use concrete_ir::lowering::{errors::LoweringError, lower_program}; +use std::path::PathBuf; + +use concrete_ir::lowering::{errors::LoweringError, lower_programs}; use concrete_parser::{error::Diagnostics, ProgramSource}; #[test] @@ -9,7 +11,7 @@ fn module_not_found() { assert!( matches!( &error, - LoweringError::ModuleNotFound { span: _, module } if module == "Other" + LoweringError::ModuleNotFound { span: _, module, .. } if module == "Other" ), "{:#?}", error @@ -65,7 +67,7 @@ pub fn check_invalid_program(source: &str, name: &str) -> LoweringError { let db = concrete_driver::db::Database::default(); let source = ProgramSource::new(&db, source.to_string(), name.to_string()); - let program = match concrete_parser::parse_ast(&db, source) { + let mut program = match concrete_parser::parse_ast(&db, source) { Some(x) => x, None => { Diagnostics::dump( @@ -78,8 +80,9 @@ pub fn check_invalid_program(source: &str, name: &str) -> LoweringError { panic!("error parsing ast"); } }; + program.file_path = Some(PathBuf::from(name).with_extension("con")); - lower_program(&program).expect_err("expected error") + lower_programs(&[program]).expect_err("expected error") } #[test] diff --git a/crates/concrete_driver/tests/common.rs b/crates/concrete_driver/tests/common.rs index 79ded4d..ba1e261 100644 --- a/crates/concrete_driver/tests/common.rs +++ b/crates/concrete_driver/tests/common.rs @@ -6,8 +6,8 @@ use std::{ }; use ariadne::Source; -use concrete_codegen_mlir::linker::{link_binary, link_shared_lib}; -use concrete_ir::lowering::lower_program; +use concrete_driver::linker::{link_binary, link_shared_lib}; +use concrete_ir::lowering::lower_programs; use concrete_parser::{error::Diagnostics, ProgramSource}; use concrete_session::{ config::{DebugInfo, OptLevel}, @@ -42,7 +42,7 @@ pub fn compile_program( let db = concrete_driver::db::Database::default(); let source = ProgramSource::new(&db, source.to_string(), name.to_string()); tracing::debug!("source code:\n{}", source.input(&db)); - let program = match concrete_parser::parse_ast(&db, source) { + let mut program = match concrete_parser::parse_ast(&db, source) { Some(x) => x, None => { Diagnostics::dump( @@ -58,12 +58,12 @@ pub fn compile_program( let test_dir = tempfile::tempdir()?; let test_dir_path = test_dir.path(); - // todo: find a better name, "target" would clash with rust if running in the source tree. - let target_dir = test_dir_path.join("build_artifacts/"); - if !target_dir.exists() { - std::fs::create_dir_all(&target_dir)?; - } - let output_file = target_dir.join(PathBuf::from(name)); + + let input_file = test_dir_path.join(name).with_extension(".con"); + std::fs::write(&input_file, source.input(&db))?; + program.file_path = Some(input_file); + + let output_file = test_dir_path.join(name); let output_file = if library { output_file.with_extension(Session::get_platform_library_ext()) } else if cfg!(target_os = "windows") { @@ -73,32 +73,32 @@ pub fn compile_program( }; let session = Session { - file_path: PathBuf::from(name), debug_info: DebugInfo::Full, optlevel, - source: Source::from(source.input(&db).to_string()), + sources: vec![Source::from(source.input(&db).to_string())], library, - target_dir, output_file, output_mlir: false, output_ll: false, output_asm: false, - output_all: false, }; - let program_ir = lower_program(&program)?; + let program_ir = lower_programs(&[program])?; let object_path = concrete_codegen_mlir::compile(&session, &program_ir)?; if library { link_shared_lib( - &object_path, + &[object_path.clone()], &session .output_file .with_extension(Session::get_platform_library_ext()), )?; } else { - link_binary(&object_path, &session.output_file.with_extension(""))?; + link_binary( + &[object_path.clone()], + &session.output_file.with_extension(""), + )?; } Ok(CompileResult { diff --git a/crates/concrete_ir/src/lib.rs b/crates/concrete_ir/src/lib.rs index d8e555a..4200995 100644 --- a/crates/concrete_ir/src/lib.rs +++ b/crates/concrete_ir/src/lib.rs @@ -1,6 +1,7 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt, + path::PathBuf, }; use concrete_ast::common::Ident; @@ -38,6 +39,8 @@ pub struct ProgramBody { pub structs: BTreeMap, /// The function signatures. pub function_signatures: HashMap, Ty)>, + /// The file paths (program_id from the DefId) -> path. + pub file_paths: HashMap, } #[derive(Debug, Clone)] diff --git a/crates/concrete_ir/src/lowering.rs b/crates/concrete_ir/src/lowering.rs index 87fe3bc..7663f7f 100644 --- a/crates/concrete_ir/src/lowering.rs +++ b/crates/concrete_ir/src/lowering.rs @@ -28,34 +28,44 @@ pub mod common; pub mod errors; pub mod prepass; -pub fn lower_program(program: &Program) -> Result { +pub fn lower_programs(program: &[Program]) -> Result { let mut ctx = BuildCtx { body: ProgramBody::default(), gen: IdGenerator::default(), unresolved_function_signatures: Default::default(), }; - // resolve symbols - for module in &program.modules { - ctx = prepass::prepass_module(ctx, module)?; - } + for (program_id, program) in program.iter().enumerate() { + ctx.gen.program_id = program_id; + ctx.gen.current_id = 0; + let file_path = program.file_path.as_ref().ok_or_else(|| { + LoweringError::InternalError("Missing program file path".to_string(), program_id) + })?; + ctx.body.file_paths.insert(program_id, file_path.clone()); + + // resolve symbols + for module in &program.modules { + ctx = prepass::prepass_module(ctx, module)?; + } - // resolve imports - for module in &program.modules { - ctx = prepass::prepass_imports(ctx, module)?; - } + // resolve imports + for module in &program.modules { + ctx = prepass::prepass_imports(ctx, module)?; + } - for mod_def in &program.modules { - let id = *ctx - .body - .top_level_module_names - .get(&mod_def.name.name) - .ok_or_else(|| LoweringError::ModuleNotFound { - span: mod_def.span, - module: mod_def.name.name.clone(), - })?; - - ctx = lower_module(ctx, mod_def, id)?; + for mod_def in &program.modules { + let id = *ctx + .body + .top_level_module_names + .get(&mod_def.name.name) + .ok_or_else(|| LoweringError::ModuleNotFound { + span: mod_def.span, + module: mod_def.name.name.clone(), + program_id, + })?; + + ctx = lower_module(ctx, mod_def, id)?; + } } Ok(ctx.body) @@ -87,6 +97,7 @@ fn lower_module(mut ctx: BuildCtx, module: &Module, id: DefId) -> Result Result Result<(), Lowering span: info.span, found: rvalue_ty, expected: ty.clone(), + program_id: builder.local_module.program_id, }); } @@ -628,6 +642,7 @@ fn lower_assign(builder: &mut FnBodyBuilder, info: &AssignStmt) -> Result<(), Lo span: info.target.first.span, name: info.target.first.name.clone(), type_span: ty.span, + program_id: builder.local_module.program_id, })?; } ty = *inner.clone(); @@ -661,6 +676,7 @@ fn lower_return( span: info.span, found: value_ty, expected: ret_type.clone(), + program_id: builder.local_module.program_id, }); } @@ -954,6 +970,7 @@ fn lower_fn_call( .ok_or(LoweringError::FunctionNotFound { span: info.target.span, function: info.target.name.clone(), + program_id: builder.local_module.program_id, })? } }; @@ -1056,6 +1073,7 @@ fn lower_binary_op( span: rhs_span, found: rhs_ty, expected: lhs_ty, + program_id: builder.local_module.program_id, }); } @@ -1294,6 +1312,7 @@ pub fn lower_path( LoweringError::UseOfUndeclaredVariable { span: info.span, name: info.first.name.clone(), + program_id: builder.local_module.program_id, }, )?; @@ -1316,6 +1335,7 @@ pub fn lower_path( LoweringError::StructFieldNotFound { span: *field_span, name: name.name.clone(), + program_id: builder.local_module.program_id, } })?; projection.push(PlaceElem::Field(idx)); @@ -1355,6 +1375,7 @@ pub fn lower_type(ctx: &BuildCtx, spec: &TypeSpec, module_id: DefId) -> Result Err(LoweringError::NotYetImplemented { span: *span, message: "Generics not yet implemented", + program_id: module_id.program_id, })?, TypeSpec::Array { of_type, @@ -1424,6 +1445,7 @@ pub fn name_to_tykind( Err(LoweringError::UnrecognizedType { span, name: other.to_string(), + program_id: module_id.program_id, })? } } diff --git a/crates/concrete_ir/src/lowering/errors.rs b/crates/concrete_ir/src/lowering/errors.rs index 334a945..48d9a58 100644 --- a/crates/concrete_ir/src/lowering/errors.rs +++ b/crates/concrete_ir/src/lowering/errors.rs @@ -6,33 +6,74 @@ use crate::{DefId, Ty}; #[derive(Debug, Error, Clone)] pub enum LoweringError { #[error("module {module:?} not found")] - ModuleNotFound { span: Span, module: String }, + ModuleNotFound { + span: Span, + module: String, + program_id: usize, + }, #[error("function {function:?} not found")] - FunctionNotFound { span: Span, function: String }, + FunctionNotFound { + span: Span, + function: String, + program_id: usize, + }, #[error("struct field {name:?} not found")] - StructFieldNotFound { span: Span, name: String }, + StructFieldNotFound { + span: Span, + name: String, + program_id: usize, + }, #[error("symbol {:?} not found", symbol.name)] ImportNotFound { module_span: Span, import_span: Span, symbol: Ident, + program_id: usize, }, #[error("use of underclared variable {name:?}")] - UseOfUndeclaredVariable { span: Span, name: String }, + UseOfUndeclaredVariable { + span: Span, + name: String, + program_id: usize, + }, #[error("trying to mutate a non-mutable reference")] BorrowNotMutable { span: Span, type_span: Option, name: String, + program_id: usize, }, #[error("unrecognized type {name}")] - UnrecognizedType { span: Span, name: String }, + UnrecognizedType { + span: Span, + name: String, + program_id: usize, + }, #[error("id not found")] - IdNotFound { span: Span, id: DefId }, + IdNotFound { + span: Span, + id: DefId, + program_id: usize, + }, #[error("feature not yet implemented: {message}")] - NotYetImplemented { span: Span, message: &'static str }, + NotYetImplemented { + span: Span, + message: &'static str, + program_id: usize, + }, #[error("unexpected type")] - UnexpectedType { span: Span, found: Ty, expected: Ty }, + UnexpectedType { + span: Span, + found: Ty, + expected: Ty, + program_id: usize, + }, #[error("extern function {name:?} has a body")] - ExternFnWithBody { span: Span, name: String }, + ExternFnWithBody { + span: Span, + name: String, + program_id: usize, + }, + #[error("internal error: {0}")] + InternalError(String, usize), } diff --git a/crates/concrete_ir/src/lowering/prepass.rs b/crates/concrete_ir/src/lowering/prepass.rs index 8da779c..b03e2b9 100644 --- a/crates/concrete_ir/src/lowering/prepass.rs +++ b/crates/concrete_ir/src/lowering/prepass.rs @@ -261,6 +261,7 @@ pub fn prepass_imports( .ok_or(LoweringError::ModuleNotFound { span: import.module[0].span, module: import.module[0].name.clone(), + program_id: mod_id.program_id, })?; let mut imported_module = ctx.body @@ -269,6 +270,7 @@ pub fn prepass_imports( .ok_or(LoweringError::IdNotFound { span: mod_def.span, id: *imported_module_id, + program_id: mod_id.program_id, })?; for x in import.module.iter().skip(1) { @@ -280,11 +282,13 @@ pub fn prepass_imports( .ok_or_else(|| LoweringError::ModuleNotFound { span: x.span, module: x.name.clone(), + program_id: mod_id.program_id, })?; imported_module = ctx.body.modules.get(imported_module_id).ok_or({ LoweringError::IdNotFound { span: x.span, id: *imported_module_id, + program_id: mod_id.program_id, } })?; } @@ -305,6 +309,7 @@ pub fn prepass_imports( module_span: mod_def.span, import_span: import.span, symbol: sym.clone(), + program_id: mod_id.program_id, })?; } } @@ -338,6 +343,7 @@ pub fn prepass_imports_submodule( .ok_or(LoweringError::IdNotFound { span: mod_def.span, id: parent_id, + program_id: parent_id.program_id, })? .symbols .modules @@ -345,6 +351,7 @@ pub fn prepass_imports_submodule( .ok_or_else(|| LoweringError::ModuleNotFound { span: mod_def.span, module: mod_def.name.name.clone(), + program_id: parent_id.program_id, })?; for import in &mod_def.imports { @@ -355,6 +362,7 @@ pub fn prepass_imports_submodule( .ok_or_else(|| LoweringError::ModuleNotFound { span: import.module[0].span, module: import.module[0].name.clone(), + program_id: parent_id.program_id, })?; let mut imported_module = ctx.body @@ -363,6 +371,7 @@ pub fn prepass_imports_submodule( .ok_or(LoweringError::IdNotFound { span: import.span, id: *imported_module_id, + program_id: parent_id.program_id, })?; for module_path in import.module.iter().skip(1) { @@ -373,11 +382,13 @@ pub fn prepass_imports_submodule( .ok_or_else(|| LoweringError::ModuleNotFound { span: module_path.span, module: module_path.name.clone(), + program_id: parent_id.program_id, })?; imported_module = ctx.body.modules.get(imported_module_id).ok_or({ LoweringError::IdNotFound { span: import.span, id: *imported_module_id, + program_id: parent_id.program_id, } })?; } @@ -398,6 +409,7 @@ pub fn prepass_imports_submodule( module_span: mod_def.span, import_span: import.span, symbol: sym.clone(), + program_id: parent_id.program_id, })?; } } @@ -408,6 +420,7 @@ pub fn prepass_imports_submodule( .ok_or(LoweringError::IdNotFound { span: mod_def.span, id: mod_id, + program_id: mod_id.program_id, })? .imports .extend(imports); diff --git a/crates/concrete_parser/src/grammar.lalrpop b/crates/concrete_parser/src/grammar.lalrpop index 4845537..e3fdd9f 100644 --- a/crates/concrete_parser/src/grammar.lalrpop +++ b/crates/concrete_parser/src/grammar.lalrpop @@ -184,6 +184,7 @@ pub(crate) GenericParams: Vec = { pub Program: ast::Program = { => { ast::Program { + file_path: None, modules: vec![<>], } }, diff --git a/crates/concrete_session/src/lib.rs b/crates/concrete_session/src/lib.rs index 1356383..c316f4a 100644 --- a/crates/concrete_session/src/lib.rs +++ b/crates/concrete_session/src/lib.rs @@ -7,19 +7,15 @@ pub mod config; #[derive(Debug, Clone)] pub struct Session { - pub file_path: PathBuf, pub debug_info: DebugInfo, pub optlevel: OptLevel, - pub source: Source, // for debugging locations + pub sources: Vec, // for debugging locations /// True if it should be compiled as a library false for binary. pub library: bool, - /// The directory where to store artifacts and intermediate files such as object files. - pub target_dir: PathBuf, pub output_file: PathBuf, pub output_mlir: bool, pub output_ll: bool, pub output_asm: bool, - pub output_all: bool, // todo: include target, host, etc } From 84f3fdff26b411b02d1694c6317c2c0754211f4b Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Tue, 12 Mar 2024 16:34:02 +0100 Subject: [PATCH 2/2] name mangling --- README.md | 1 + crates/concrete_codegen_mlir/src/codegen.rs | 8 +++++--- crates/concrete_driver/src/lib.rs | 2 +- crates/concrete_ir/src/lib.rs | 12 ++++++++++++ 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d5f66c9..bb543fa 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Make sure you have installed the dependencies: - git - Rust - LLVM 17 with MLIR enabled +- libgit2 If building LLVM from source, you'll need additional tools: - g++, clang++, or MSVC with versions listed on [LLVM's documentation](https://llvm.org/docs/GettingStarted.html#host-c-toolchain-both-compiler-and-standard-library) diff --git a/crates/concrete_codegen_mlir/src/codegen.rs b/crates/concrete_codegen_mlir/src/codegen.rs index c509996..faa1c33 100644 --- a/crates/concrete_codegen_mlir/src/codegen.rs +++ b/crates/concrete_codegen_mlir/src/codegen.rs @@ -343,8 +343,10 @@ fn compile_function(ctx: FunctionCodegenCtx) -> Result<(), CodegenError> { .iter() .map(|x| compile_rvalue(&ctx, mlir_block, x, &locals).map(|x| x.0)) .collect::>()?; - let fn_symbol = - FlatSymbolRefAttribute::new(ctx.context(), &target_fn_body.name); // todo: good name resolution + let fn_symbol = FlatSymbolRefAttribute::new( + ctx.context(), + &target_fn_body.get_mangled_name(), + ); let ret_type = match &target_fn_body_sig.1.kind { TyKind::Unit => None, _ => Some(compile_type(ctx.module_ctx, &target_fn_body_sig.1)), @@ -443,7 +445,7 @@ fn compile_function(ctx: FunctionCodegenCtx) -> Result<(), CodegenError> { let func_op = func::func( ctx.context(), - StringAttribute::new(ctx.context(), &body.name), + StringAttribute::new(ctx.context(), &body.get_mangled_name()), TypeAttribute::new(func_type.into()), region, &fn_attributes, diff --git a/crates/concrete_driver/src/lib.rs b/crates/concrete_driver/src/lib.rs index 7eaae20..95ba0f4 100644 --- a/crates/concrete_driver/src/lib.rs +++ b/crates/concrete_driver/src/lib.rs @@ -25,7 +25,7 @@ pub mod db; pub mod linker; #[derive(Parser, Debug)] -#[command(author, version, about = "concrete", long_about = None, bin_name = "concrete")] +#[command(author, version, about = "The Concrete Programming Language", long_about = None, bin_name = "concrete")] pub struct Cli { #[command(subcommand)] command: Commands, diff --git a/crates/concrete_ir/src/lib.rs b/crates/concrete_ir/src/lib.rs index 4200995..c015940 100644 --- a/crates/concrete_ir/src/lib.rs +++ b/crates/concrete_ir/src/lib.rs @@ -79,6 +79,18 @@ impl FnBody { .filter(|x| matches!(x.kind, LocalKind::Arg)) .collect() } + + pub fn get_mangled_name(&self) -> String { + if self.is_extern { + return self.name.clone(); + } + + if self.name == "main" { + "main".to_string() + } else { + format!("{}@{}@{}", self.name, self.id.program_id, self.id.id) + } + } } #[derive(Debug, Clone)]