diff --git a/.gitignore b/.gitignore index 1cc4cd6..19851c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target /outputs +/database \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 5288dc1..e85623b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "bytes", "futures-core", "futures-sink", @@ -52,7 +52,7 @@ dependencies = [ "actix-utils", "ahash 0.8.11", "base64 0.22.1", - "bitflags 2.5.0", + "bitflags 2.6.0", "brotli", "bytes", "bytestring", @@ -469,9 +469,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bitvec" @@ -720,6 +720,7 @@ dependencies = [ "anyhow", "clap", "dt_core", + "indicatif", "serde_json", ] @@ -994,6 +995,7 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" name = "dt_core" version = "0.1.0" dependencies = [ + "dt_database", "dt_graph", "dt_i18n", "dt_parser", @@ -1004,6 +1006,14 @@ dependencies = [ "dt_tracker", ] +[[package]] +name = "dt_database" +version = "0.1.0" +dependencies = [ + "anyhow", + "rusqlite", +] + [[package]] name = "dt_graph" version = "0.1.0" @@ -1089,6 +1099,7 @@ name = "dt_tracker" version = "0.1.0" dependencies = [ "anyhow", + "dt_database", "dt_graph", "dt_parser", "serde", @@ -1149,6 +1160,18 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fancy-regex" version = "0.13.0" @@ -1348,6 +1371,15 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -1613,6 +1645,17 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.13" @@ -2141,7 +2184,7 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -2238,6 +2281,20 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags 2.6.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -2274,7 +2331,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -2667,7 +2724,7 @@ version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69e9696b3d02197c16ba7548c95b31f7ca79532200d269ce3ad03a5b2174cf28" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "bytecheck", "is-macro", "num-bigint", @@ -2753,7 +2810,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b1e55ce789bd4411b1e0a8b83149c70dd1186e38471fd65860dcece8a522f2f" dependencies = [ "better_scoped_tls", - "bitflags 2.5.0", + "bitflags 2.6.0", "indexmap", "once_cell", "phf", @@ -3332,6 +3389,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vergen" version = "9.0.1" diff --git a/Cargo.toml b/Cargo.toml index 1e34545..220bf13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/cli", "crates/demo", "crates/dt_core", + "crates/dt_database", "crates/dt_graph", "crates/dt_i18n", "crates/dt_parser", @@ -23,3 +24,5 @@ serde_json = "1.0" swc_core = { version = "0.104.2", features = ["common", "ecma_ast", "ecma_visit", "ecma_plugin_transform"] } swc_ecma_parser = { version = "0.150.0", features = ["typescript"] } clap = { version = "4.5", features = ["derive"] } +rusqlite = { version = "0.32.1", features = ["bundled"] } +indicatif = "0.17.8" \ No newline at end of file diff --git a/README.md b/README.md index 7a1f8c7..a1d04ca 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ flowchart TD reexport all the library crates: +- database - graph - i18n - parser @@ -88,6 +89,12 @@ reexport all the library crates: - scheduler - tracker +### Database + +`Database` defines the models using in the `cli` and `api_server` crate. + +![ERD](./assets/erd.jpeg) + ### Graph `DependOnGraph` takes the `SymbolDependency` one by one to construct a DAG. You have to add the `SymbolDependency` by topological order so that `DependOnGraph` can handle the wildcard import and export for you. @@ -203,11 +210,58 @@ let paths = dt.trace("", TraceTarget::LocalVar("variable_name")).un ### Demo -See the `demo` crate. You can run `cargo run --bin demo -- -s ./test-project/everybodyyyy -d ~/tmp`. +See the `demo` crate. -### Portable +``` +Track fine-grained symbol dependency graph + +Usage: demo -s -d + +Options: + -s Path of project to trace + -d Path of the output folder + -h, --help Print help + -V, --version Print version +``` + +### CLI + +See the `cli` crate. + +``` +Parse a project and serialize its output -See the `cli` crate. You can run `cargo run --bin cli -- -i -o `. +Usage: cli + +Commands: + portable Parse and export the project in portable format + database Parse and export the project in database format + help Print this message or the help of the given subcommand(s) + +Options: + -h, --help Print help + -V, --version Print version +``` + +Usage: + +- `cli portable -i -t -o ` +- `cli database -i -t -o ` + +### API Server + +see the `api_server` crate. The database is the one generated by CLI with `database` command. + +``` +Start the server to provide search API + +Usage: api_server --db + +Options: + --db The path of your database + -h, --help Print help + -V, --version Print version +``` ## Client diff --git a/assets/erd.jpeg b/assets/erd.jpeg new file mode 100644 index 0000000..52b16fe Binary files /dev/null and b/assets/erd.jpeg differ diff --git a/crates/api_server/src/main.rs b/crates/api_server/src/main.rs index 492c287..5a886fa 100644 --- a/crates/api_server/src/main.rs +++ b/crates/api_server/src/main.rs @@ -2,24 +2,11 @@ use actix_cors::Cors; use actix_web::{error, get, web, App, HttpServer, Result}; use clap::Parser; use dt_core::{ - graph::used_by_graph::UsedByGraph, - portable::Portable, - tracker::{DependencyTracker, TraceTarget}, + database::{models, Database, SqliteDb}, + tracker::{db_version::DependencyTracker, TraceTarget}, }; use serde::{Deserialize, Serialize}; -use std::{ - collections::{HashMap, HashSet}, - fs::File, - io::Read, -}; - -struct AppState { - project_root: String, - translation_json: HashMap, - i18n_to_symbol: HashMap>>, - symbol_to_route: HashMap>>, - used_by_graph: UsedByGraph, -} +use std::collections::HashMap; #[derive(Serialize, Clone)] struct Step { @@ -39,133 +26,118 @@ struct Info { exact_match: bool, } +// Current implementation is mimick version of the trace with in-memory graph. +// We can refactor it after the database feature gets validated. #[get("/search")] async fn search( data: web::Data, info: web::Query, ) -> Result> { + let db = &data.db; let search = &info.q; let exact_match = info.exact_match; - - let mut matched_i18n_keys: Vec = Vec::new(); - match exact_match { - true => { - for (i18n_key, translation) in data.translation_json.iter() { - if translation == search { - matched_i18n_keys.push(i18n_key.to_owned()); - } - } - } - false => { - for (i18n_key, translation) in data.translation_json.iter() { - if translation.contains(search) { - matched_i18n_keys.push(i18n_key.to_owned()); - } - } - } - } - + // project name "default_project" can be different in feature "cross-project tracing" + let project = models::Project::retrieve_by_name(&db.conn, "default_project").unwrap(); + let matched_i18n_keys = project + .search_translation(&db.conn, search, exact_match) + .unwrap(); if matched_i18n_keys.len() == 0 { return Err(error::ErrorNotFound(format!("No result for {}", search))); } - - let mut dependency_tracker = DependencyTracker::new(&data.used_by_graph, true); + let mut dependency_tracker = DependencyTracker::new(&db, project.clone(), true); let mut trace_result = HashMap::new(); - for i18n_key in matched_i18n_keys.iter() { + for translation in matched_i18n_keys.iter() { let mut route_to_paths = HashMap::new(); - if let Some(i18n_key_usage) = data.i18n_to_symbol.get(i18n_key) { - for (module_path, symbols) in i18n_key_usage { - for symbol in symbols { - let full_paths = dependency_tracker - .trace((module_path.clone(), TraceTarget::LocalVar(symbol.clone()))) - .unwrap(); - // traverse each path and check if any symbol is used in some routes - for mut full_path in full_paths { - full_path.reverse(); - for (i, (step_module_path, step_trace_target)) in - full_path.iter().enumerate() - { - match step_trace_target { - TraceTarget::LocalVar(step_symbol_name) => { - if let Some(symbol_to_routes) = - data.symbol_to_route.get(step_module_path) - { - if let Some(routes) = symbol_to_routes.get(step_symbol_name) - { - let dependency_from_target_to_route: Vec = - full_path[0..i] - .iter() - .map(|(path, target)| Step { - module_path: path.clone(), - symbol_name: target.to_string(), - }) - .collect(); - for route in routes.iter() { - if !route_to_paths.contains_key(route) { - route_to_paths - .insert(route.clone(), HashMap::new()); - } - if !route_to_paths - .get(route) - .unwrap() - .contains_key(symbol) - { - route_to_paths - .get_mut(route) - .unwrap() - .insert(symbol.to_string(), vec![]); - } - route_to_paths - .get_mut(route) - .unwrap() - .get_mut(symbol) - .unwrap() - .push(dependency_from_target_to_route.clone()); - } - } + let translation_used_by = translation.get_used_by(&db.conn).unwrap(); + for symbol in translation_used_by.iter() { + let module = models::Module::retrieve_by_id(&db.conn, symbol.module_id).unwrap(); + let full_paths = dependency_tracker + .trace(( + module.path.to_string(), + TraceTarget::LocalVar(symbol.name.to_string()), + )) + .unwrap(); + // traverse each path and check if any symbol is used in some routes + for mut full_path in full_paths { + full_path.reverse(); + for (i, (step_module_path, step_trace_target)) in full_path.iter().enumerate() { + match step_trace_target { + TraceTarget::LocalVar(step_symbol_name) => { + let step_module = + project.get_module(&db.conn, &step_module_path).unwrap(); + let step_symbol = step_module + .get_symbol( + &db.conn, + models::SymbolVariant::LocalVariable, + &step_symbol_name, + ) + .unwrap(); + let routes = step_symbol.get_used_by_routes(&db.conn).unwrap(); + if routes.len() > 0 { + let dependency_from_target_to_route: Vec = full_path[0..i] + .iter() + .map(|(path, target)| Step { + module_path: path.clone(), + symbol_name: target.to_string(), + }) + .collect(); + for route in routes.iter() { + let route = &route.path; + let symbol = &symbol.name; + if !route_to_paths.contains_key(route) { + route_to_paths.insert(route.clone(), HashMap::new()); } + if !route_to_paths.get(route).unwrap().contains_key(symbol) { + route_to_paths + .get_mut(route) + .unwrap() + .insert(symbol.to_string(), vec![]); + } + route_to_paths + .get_mut(route) + .unwrap() + .get_mut(symbol) + .unwrap() + .push(dependency_from_target_to_route.clone()); } - _ => (), } } + _ => (), } } } } - trace_result.insert(i18n_key.to_string(), route_to_paths); + + trace_result.insert(translation.key.to_string(), route_to_paths); } Ok(web::Json(SearchResponse { - project_root: data.project_root.to_owned(), + project_root: "".to_string(), trace_result, })) } +struct AppState { + db: SqliteDb, +} + #[derive(Parser)] #[command(version, about = "Start the server to provide search API", long_about = None)] struct Cli { - /// Portable path - #[arg(short)] - portable: String, + /// The path of your database + #[arg(long)] + db: String, } #[actix_web::main] async fn main() -> std::io::Result<()> { let cli = Cli::parse(); - let mut file = File::open(cli.portable)?; - let mut exported = String::new(); - file.read_to_string(&mut exported)?; - let portable = Portable::import(&exported).unwrap(); HttpServer::new(move || { App::new() .wrap(Cors::default().allow_any_method().allow_any_origin()) .app_data(web::Data::new(AppState { - project_root: portable.project_root.clone(), - translation_json: portable.translation_json.clone(), - i18n_to_symbol: portable.i18n_to_symbol.clone(), - symbol_to_route: portable.symbol_to_route.clone(), - used_by_graph: portable.used_by_graph.clone(), + db: SqliteDb::open(&cli.db).expect(&format!("open database from {}", cli.db)), })) .service(search) }) diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index be5ddc4..28c009a 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -10,5 +10,6 @@ version = "0.1.0" anyhow = { workspace = true } clap = { workspace = true } serde_json = { workspace = true } +indicatif = { workspace = true } dt_core = { version = "0.1.0", path = "../dt_core" } \ No newline at end of file diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index e28fd76..b124bb2 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,40 +1,97 @@ use anyhow::Context; -use clap::Parser; +use clap::{Parser, Subcommand}; use dt_core::{ + database::{models, Database, SqliteDb}, graph::{depend_on_graph::DependOnGraph, used_by_graph::UsedByGraph}, - i18n::I18nToSymbol, - parser::{collect_symbol_dependency, Input}, - path_resolver::ToCanonicalString, + i18n::{collect_translation, I18nToSymbol}, + parser::{ + anonymous_default_export::SYMBOL_NAME_FOR_ANONYMOUS_DEFAULT_EXPORT, + collect_symbol_dependency, + types::{FromOtherModule, FromType, ModuleExport, ModuleScopedVariable, SymbolDependency}, + Input, + }, + path_resolver::{PathResolver, ToCanonicalString}, portable::Portable, - route::SymbolToRoutes, + route::{collect_route_dependency, Route, SymbolToRoutes}, scheduler::ParserCandidateScheduler, }; +use indicatif::{ProgressBar, ProgressStyle}; use std::{ + collections::{HashMap, HashSet}, fs::File, - io::{prelude::*, BufReader}, + io::{BufReader, Write}, path::PathBuf, }; #[derive(Parser)] #[command(version, about = "Parse a project and serialize its output", long_about = None)] struct Cli { - /// Input path - #[arg(short)] - input: String, + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Parse and export the project in portable format + Portable { + /// Input path + #[arg(short)] + input: String, + + /// translation.json path + #[arg(short)] + translation_path: String, + + /// Output path + #[arg(short)] + output: String, + }, + + /// Parse and export the project in database format + Database { + /// Input path + #[arg(short)] + input: String, - /// translation.json path - #[arg(short)] - translation_path: String, + /// translation.json path + #[arg(short)] + translation_path: String, - /// Output path - #[arg(short)] - output: String, + /// Output path + #[arg(short)] + output: String, + }, } fn main() -> anyhow::Result<()> { - let cli = Cli::parse(); - let project_root = PathBuf::from(&cli.input).to_canonical_string()?; - let translation_json = File::open(&cli.translation_path)?; + match Cli::parse().command { + Command::Portable { + input, + translation_path, + output, + } => { + parse_and_export_project_to_portable(&input, &output, &translation_path) + .context("parse and export project to portable")?; + } + Command::Database { + input, + translation_path, + output, + } => { + parse_and_export_project_to_database(&input, &output, &translation_path) + .context("parse and export project to database")?; + } + } + Ok(()) +} + +fn parse_and_export_project_to_portable( + project_root: &str, + output_portable_path: &str, + translation_file_path: &str, +) -> anyhow::Result<()> { + let project_root = PathBuf::from(project_root).to_canonical_string()?; + let translation_json = File::open(&translation_file_path)?; let translation_json_reader = BufReader::new(translation_json); let mut scheduler = ParserCandidateScheduler::new(&project_root); @@ -66,8 +123,487 @@ fn main() -> anyhow::Result<()> { ); let serialized = portable.export()?; - let mut file = File::create(&cli.output)?; + let mut file = File::create(&output_portable_path)?; file.write_all(serialized.as_bytes())?; Ok(()) } + +fn parse_and_export_project_to_database( + project_root: &str, + output_database_path: &str, + translation_file_path: &str, +) -> anyhow::Result<()> { + let project_root = PathBuf::from(project_root).to_canonical_string()?; + // project name "default_project" can be different in feature "cross-project tracing" + let project = + Project::new("default_project", &project_root, output_database_path).context(format!( + "ready to a emit the project to database, project: {}, database: {}", + project_root, output_database_path + ))?; + + let translation_file = File::open(translation_file_path).context(format!( + "open translation file, path: {}", + translation_file_path + ))?; + let translation_json_reader = BufReader::new(translation_file); + let translation_json: HashMap = + serde_json::from_reader(translation_json_reader).context(format!( + "deserialize translation file, path: {}", + translation_file_path + ))?; + project + .add_translation(&translation_json) + .context("add translation to project")?; + + let mut scheduler = ParserCandidateScheduler::new(&project_root); + let bar = ProgressBar::new(scheduler.get_total_remaining_candidate_count() as u64); + bar.set_style( + ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", + )? + .progress_chars("##-"), + ); + loop { + match scheduler.get_one_candidate() { + Some(c) => { + let module_src = c + .to_str() + .context(format!("get module_src, path_buf: {:?}", c))?; + let module_ast = Input::Path(module_src) + .get_module_ast() + .context(format!("get module ast, module_src: {}", module_src))?; + + let symbol_dependency = collect_symbol_dependency(&module_ast, module_src) + .context(format!( + "collect symbol dependency for module: {}", + &module_src + ))?; + let module = project + .add_module(&symbol_dependency) + .context(format!( + "add symbol dependency of module {} to project", + symbol_dependency.canonical_path + )) + .context(format!("add module {} to project", module_src))?; + + let i18n_usage = collect_translation(&module_ast) + .context(format!("collect i18n usage for module: {}", &module_src))?; + project + .add_i18n_usage(&module, &i18n_usage) + .context(format!( + "add i18n usage of module {} to project", + module_src + ))?; + + let route_usage = collect_route_dependency(&module_ast, &symbol_dependency) + .context(format!("collect route usage for module: {}", &module_src))?; + project + .add_route_usage(&module, &route_usage) + .context(format!( + "add route usage of module {} to project", + module_src + ))?; + + scheduler.mark_candidate_as_parsed(c); + bar.inc(1); + } + None => break, + } + } + bar.finish_with_message("all modules parsed 🌲"); + Ok(()) +} + +struct Project { + db: SqliteDb, + project_root: String, + project: models::Project, + path_resolver: PathResolver, +} + +impl Project { + pub fn new(project_name: &str, project_root: &str, db_path: &str) -> anyhow::Result { + let db = SqliteDb::open(db_path)?; + db.create_tables()?; + let project = models::Project::create(&db.conn, project_root, project_name)?; + Ok(Self { + db, + project_root: project_root.to_owned(), + project, + path_resolver: PathResolver::new(project_root), + }) + } + + fn remove_prefix(&self, canonical_path: &str) -> String { + match canonical_path.starts_with(&self.project_root) { + true => canonical_path[self.project_root.len()..].to_string(), + false => canonical_path.to_string(), + } + } + + fn resolve_path(&self, current_path: &str, import_src: &str) -> anyhow::Result { + Ok(self.remove_prefix(&self.path_resolver.resolve_path(current_path, import_src)?)) + } + + fn handle_local_variable_table( + &self, + module: &models::Module, + symbol_dependency: &SymbolDependency, + ) -> anyhow::Result<()> { + for ( + symbol_name, + ModuleScopedVariable { + depend_on, + import_from, + }, + ) in symbol_dependency.local_variable_table.iter() + { + let current_symbol = module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::LocalVariable, + symbol_name, + )?; + if let Some(depend_on) = depend_on { + // Items in depend_on vector is guranteed to be local variables of the same module. + // So we can create those symbols as local variable. + for depend_on_symbol_name in depend_on.iter() { + let depend_on_symbol = module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::LocalVariable, + depend_on_symbol_name, + )?; + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + } + if let Some(FromOtherModule { from, from_type }) = import_from { + if let Ok(from) = self.resolve_path(&symbol_dependency.canonical_path, &from) { + let import_from_module = + self.project.get_or_create_module(&self.db.conn, &from)?; + // It's ok to create a named export or default export symbol for other module + // even that module hasn't been parsed yet. + match from_type { + dt_core::parser::types::FromType::Named(depend_on_symbol_name) => { + let depend_on_symbol = import_from_module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::NamedExport, + &depend_on_symbol_name, + )?; + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + dt_core::parser::types::FromType::Default => { + let depend_on_symbol = import_from_module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::DefaultExport, + "", // default export doesn't have name + )?; + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + dt_core::parser::types::FromType::Namespace => { + // When A module import namespace from B module, B module is guranteed to be + // parsed before A module. So we can query all named exports from B module. + let named_export_symbols = + import_from_module.get_named_export_symbols(&self.db.conn)?; + for depend_on_symbol in named_export_symbols.iter() { + if depend_on_symbol.name != SYMBOL_NAME_FOR_ANONYMOUS_DEFAULT_EXPORT + { + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + } + } + }; + } + } + } + Ok(()) + } + + fn handle_named_export_table( + &self, + module: &models::Module, + symbol_dependency: &SymbolDependency, + ) -> anyhow::Result<()> { + for (exported_symbol_name, exported_from) in symbol_dependency.named_export_table.iter() { + let current_symbol = module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::NamedExport, + &exported_symbol_name, + )?; + match exported_from { + ModuleExport::Local(depend_on_symbol_name) => { + let depend_on_symbol = module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::LocalVariable, + &depend_on_symbol_name, + )?; + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + ModuleExport::ReExportFrom(FromOtherModule { from, from_type }) => { + if let Ok(from) = self.resolve_path(&symbol_dependency.canonical_path, &from) { + let import_from_module = + self.project.get_or_create_module(&self.db.conn, &from)?; + // It's ok to create a named export or default export symbol for other module + // even that module hasn't been parsed yet. + match from_type { + dt_core::parser::types::FromType::Named(depend_on_symbol_name) => { + let depend_on_symbol = import_from_module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::NamedExport, + &depend_on_symbol_name, + )?; + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + dt_core::parser::types::FromType::Default => { + let depend_on_symbol = import_from_module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::DefaultExport, + "", // default export doesn't have name + )?; + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + dt_core::parser::types::FromType::Namespace => { + // When A module import namespace from B module, B module is guranteed to be + // parsed before A module. So we can query all named exports from B module. + let named_export_symbols = + import_from_module.get_named_export_symbols(&self.db.conn)?; + for depend_on_symbol in named_export_symbols.iter() { + if depend_on_symbol.name + != SYMBOL_NAME_FOR_ANONYMOUS_DEFAULT_EXPORT + { + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + } + } + }; + } + } + } + } + Ok(()) + } + + fn handle_default_export( + &self, + module: &models::Module, + symbol_dependency: &SymbolDependency, + ) -> anyhow::Result<()> { + if let Some(default_export) = symbol_dependency.default_export.as_ref() { + let current_symbol = module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::DefaultExport, + "", // default export doesn't have name + )?; + match default_export { + ModuleExport::Local(depend_on_symbol_name) => { + let depend_on_symbol = module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::LocalVariable, + &depend_on_symbol_name, + )?; + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + ModuleExport::ReExportFrom(FromOtherModule { from, from_type }) => { + if let Ok(from) = self.resolve_path(&symbol_dependency.canonical_path, &from) { + let import_from_module = + self.project.get_or_create_module(&self.db.conn, &from)?; + // It's ok to create a named export or default export symbol for other module + // even that module hasn't been parsed yet. + match from_type { + dt_core::parser::types::FromType::Named(depend_on_symbol_name) => { + let depend_on_symbol = import_from_module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::NamedExport, + &depend_on_symbol_name, + )?; + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + dt_core::parser::types::FromType::Default => { + let depend_on_symbol = import_from_module.get_or_create_symbol( + &self.db.conn, + models::SymbolVariant::DefaultExport, + "", // default export doesn't have name + )?; + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + FromType::Namespace => { + unreachable!( + "can't not export namespace from other module as default export" + ) + } + } + } + } + } + } + + Ok(()) + } + + fn handle_re_export_star_from( + &self, + module: &models::Module, + symbol_dependency: &SymbolDependency, + ) -> anyhow::Result<()> { + if let Some(re_export_start_from) = symbol_dependency.re_export_star_from.as_ref() { + for from in re_export_start_from.iter() { + if let Ok(from) = self.resolve_path(&symbol_dependency.canonical_path, &from) { + // When A module do wildcard export from B module, B module is guranteed to be + // parsed before A module. So we can query all named exports from B module. + let import_from_module = self.project.get_module(&self.db.conn, &from)?; + let named_export_symbols = + import_from_module.get_named_export_symbols(&self.db.conn)?; + for depend_on_symbol in named_export_symbols.iter() { + // Create a named export symbol for this module, and set the dependency to + // the named export symbol of imported module. + if depend_on_symbol.name != SYMBOL_NAME_FOR_ANONYMOUS_DEFAULT_EXPORT { + let current_symbol = module.add_symbol( + &self.db.conn, + models::SymbolVariant::NamedExport, + &depend_on_symbol.name, + )?; + models::SymbolDependency::create( + &self.db.conn, + ¤t_symbol, + &depend_on_symbol, + )?; + } + } + } + } + } + Ok(()) + } + + pub fn add_module( + &self, + symbol_dependency: &SymbolDependency, + ) -> anyhow::Result { + let module = self.project.get_or_create_module( + &self.db.conn, + &self.remove_prefix(&symbol_dependency.canonical_path), + )?; + + self.handle_local_variable_table(&module, symbol_dependency)?; + self.handle_named_export_table(&module, symbol_dependency)?; + self.handle_default_export(&module, symbol_dependency)?; + self.handle_re_export_star_from(&module, symbol_dependency)?; + + Ok(module) + } + + pub fn add_translation( + &self, + translation_json: &HashMap, + ) -> anyhow::Result<()> { + for (key, value) in translation_json.iter() { + self.project.add_translation(&self.db.conn, key, value)?; + } + Ok(()) + } + + pub fn add_i18n_usage( + &self, + module: &models::Module, + i18n_usage: &HashMap>, + ) -> anyhow::Result<()> { + for (symbol_name, i18n_keys) in i18n_usage.iter() { + let symbol = module + .get_symbol( + &self.db.conn, + models::SymbolVariant::LocalVariable, + &symbol_name, + ) + .context(format!( + "try to add i18n keys for symbol {}, but symbol doesn't exist", + symbol_name, + ))?; + for key in i18n_keys.iter() { + match self.project.get_translation(&self.db.conn, key) { + Ok(translation) => { + models::TranslationUsage::create(&self.db.conn, &translation, &symbol) + .context(format!( + "relate symbol {} to translation {}", + symbol_name, key + ))?; + } + Err(_) => { + // you can uncomment this to debug + // println!("try to add translation for symbol {}, but translation {} doesn't exist", symbol_name, key); + } + } + } + } + Ok(()) + } + + pub fn add_route_usage( + &self, + module: &models::Module, + route_usage: &Vec, + ) -> anyhow::Result<()> { + for Route { path, depend_on } in route_usage.iter() { + let route = self + .project + .add_route(&self.db.conn, path) + .context(format!("create route {} for project", path))?; + for symbol_name in depend_on.iter() { + let symbol = module + .get_symbol( + &self.db.conn, + models::SymbolVariant::LocalVariable, + &symbol_name, + ) + .context(format!( + "try to add route for symbol {}, but symbol doesn't exist", + symbol_name, + ))?; + models::RouteUsage::create(&self.db.conn, &route, &symbol)?; + } + } + Ok(()) + } +} diff --git a/crates/demo/Cargo.toml b/crates/demo/Cargo.toml index 288be90..67c7710 100644 --- a/crates/demo/Cargo.toml +++ b/crates/demo/Cargo.toml @@ -7,12 +7,12 @@ version = "0.1.0" [dependencies] -anyhow = { workspace = true } -clap = { workspace = true } +anyhow = { workspace = true } +clap = { workspace = true } +indicatif = { workspace = true } umya-spreadsheet = "1.2.3" dialoguer = { version = "0.11.0", features = ["history"] } -indicatif = "0.17.8" console = "0.15.8" rand = "0.8.5" diff --git a/crates/demo/src/main.rs b/crates/demo/src/main.rs index 6c7ef6c..971f4d8 100644 --- a/crates/demo/src/main.rs +++ b/crates/demo/src/main.rs @@ -1,13 +1,8 @@ use anyhow::Context; use clap::Parser; use console::style; -use dialoguer::{theme::ColorfulTheme, BasicHistory, Confirm, Input, Select}; -use indicatif::{ProgressBar, ProgressStyle}; -use rand::distributions::Alphanumeric; -use rand::{thread_rng, Rng}; -use std::path::PathBuf; - use demo::spreadsheet::write_to_spreadsheet; +use dialoguer::{theme::ColorfulTheme, BasicHistory, Confirm, Input, Select}; use dt_core::{ graph::{depend_on_graph::DependOnGraph, used_by_graph::UsedByGraph}, parser::{collect_symbol_dependency, Input as ModuleInput}, @@ -15,6 +10,9 @@ use dt_core::{ scheduler::ParserCandidateScheduler, tracker::{DependencyTracker, TraceTarget}, }; +use indicatif::{ProgressBar, ProgressStyle}; +use rand::{distributions::Alphanumeric, thread_rng, Rng}; +use std::path::PathBuf; const SYMBOL_TYPE_SELECTIONS: [&str; 3] = ["Default Export", "Named Export", "Local Variable"]; diff --git a/crates/dt_core/Cargo.toml b/crates/dt_core/Cargo.toml index 7019982..dbf5c97 100644 --- a/crates/dt_core/Cargo.toml +++ b/crates/dt_core/Cargo.toml @@ -7,6 +7,7 @@ version = "0.1.0" [dependencies] +dt_database = { version = "0.1.0", path = "../dt_database" } dt_graph = { version = "0.1.0", path = "../dt_graph" } dt_i18n = { version = "0.1.0", path = "../dt_i18n" } dt_parser = { version = "0.1.0", path = "../dt_parser" } diff --git a/crates/dt_core/src/lib.rs b/crates/dt_core/src/lib.rs index 96d243b..ea55815 100644 --- a/crates/dt_core/src/lib.rs +++ b/crates/dt_core/src/lib.rs @@ -1,3 +1,7 @@ +pub mod database { + pub use dt_database::*; +} + pub mod graph { pub use dt_graph::*; } diff --git a/crates/dt_database/Cargo.toml b/crates/dt_database/Cargo.toml new file mode 100644 index 0000000..476e996 --- /dev/null +++ b/crates/dt_database/Cargo.toml @@ -0,0 +1,11 @@ +[package] +authors = ["Leo Lin "] +description = "relational database for dt" +edition = "2021" +name = "dt_database" +version = "0.1.0" + + +[dependencies] +anyhow = { workspace = true } +rusqlite = { workspace = true } diff --git a/crates/dt_database/src/lib.rs b/crates/dt_database/src/lib.rs new file mode 100644 index 0000000..4cbf12e --- /dev/null +++ b/crates/dt_database/src/lib.rs @@ -0,0 +1,50 @@ +pub mod models; + +use models::{ + Model, Module, Project, Route, RouteUsage, Symbol, SymbolDependency, Translation, + TranslationUsage, +}; +use rusqlite::Connection; +use std::path::Path; + +pub trait Database { + fn open(path: impl AsRef) -> anyhow::Result + where + Self: Sized; + fn create_tables(&self) -> anyhow::Result<()>; +} + +#[derive(Debug)] +pub struct SqliteDb { + pub conn: Connection, +} + +impl SqliteDb { + fn create_table_if_not_exists(&self, table: &str) -> anyhow::Result<()> { + let sql = format!("CREATE TABLE if not exists {}", table); + self.conn.execute(&sql, ())?; + + Ok(()) + } +} + +impl Database for SqliteDb { + fn open(path: impl AsRef) -> anyhow::Result { + Ok(Self { + conn: Connection::open(path)?, + }) + } + + fn create_tables(&self) -> anyhow::Result<()> { + self.create_table_if_not_exists(&Project::table())?; + self.create_table_if_not_exists(&Module::table())?; + self.create_table_if_not_exists(&Symbol::table())?; + self.create_table_if_not_exists(&SymbolDependency::table())?; + self.create_table_if_not_exists(&Translation::table())?; + self.create_table_if_not_exists(&TranslationUsage::table())?; + self.create_table_if_not_exists(&Route::table())?; + self.create_table_if_not_exists(&RouteUsage::table())?; + + Ok(()) + } +} diff --git a/crates/dt_database/src/models.rs b/crates/dt_database/src/models.rs new file mode 100644 index 0000000..aba62ce --- /dev/null +++ b/crates/dt_database/src/models.rs @@ -0,0 +1,631 @@ +use rusqlite::{params, Connection, Row, ToSql}; + +pub trait Model { + fn table() -> String; +} + +#[derive(Debug, Clone)] +pub struct Project { + pub id: usize, + pub path: String, + pub name: String, +} + +impl Model for Project { + fn table() -> String { + " + project ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL, + name TEXT UNIQUE NOT NULL + ) + " + .to_string() + } +} + +impl Project { + pub fn from_row(row: &Row) -> rusqlite::Result { + Ok(Self { + id: row.get(0)?, + path: row.get(1)?, + name: row.get(2)?, + }) + } + + /// single thread only: last_insert_rowid() + pub fn create(conn: &Connection, path: &str, name: &str) -> anyhow::Result { + conn.execute( + "INSERT INTO project (path, name) VALUES (?1, ?2)", + params![path, name], + )?; + let project = conn.query_row( + "SELECT * FROM project WHERE id=last_insert_rowid()", + (), + Self::from_row, + )?; + Ok(project) + } + + pub fn retrieve_by_name(conn: &Connection, name: &str) -> anyhow::Result { + let project = conn.query_row( + "SELECT * FROM project WHERE (name) = (?1)", + params![name], + Self::from_row, + )?; + Ok(project) + } + + pub fn add_module(&self, conn: &Connection, path: &str) -> anyhow::Result { + Module::create(conn, self, path) + } + + pub fn get_module(&self, conn: &Connection, path: &str) -> anyhow::Result { + Module::retrieve(conn, self, path) + } + + /// single thread only: retrieve then create + pub fn get_or_create_module(&self, conn: &Connection, path: &str) -> anyhow::Result { + match Module::retrieve(conn, self, path) { + Ok(module) => Ok(module), + Err(_) => self.add_module(conn, path), + } + } + + pub fn add_translation( + &self, + conn: &Connection, + key: &str, + value: &str, + ) -> anyhow::Result { + Translation::create(conn, self, key, value) + } + + pub fn get_translation(&self, conn: &Connection, key: &str) -> anyhow::Result { + Translation::retrieve(conn, self, key) + } + + pub fn search_translation( + &self, + conn: &Connection, + search: &str, + exact_match: bool, + ) -> anyhow::Result> { + match exact_match { + true => Translation::search_value_exact_match(conn, self, search), + false => Translation::search_value_contain(conn, self, search), + } + } + + pub fn add_route(&self, conn: &Connection, path: &str) -> anyhow::Result { + Route::create(conn, self, path) + } +} + +#[derive(Debug)] +pub struct Module { + pub id: usize, + pub project_id: usize, + pub path: String, +} + +impl Model for Module { + fn table() -> String { + " + module ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_id INTEGER REFERENCES project(id) ON DELETE CASCADE, + path TEXT NOT NULL + ) + " + .to_string() + } +} + +impl Module { + pub fn from_row(row: &Row) -> rusqlite::Result { + Ok(Self { + id: row.get(0)?, + project_id: row.get(1)?, + path: row.get(2)?, + }) + } + + /// single thread only: last_insert_rowid() + pub fn create(conn: &Connection, project: &Project, path: &str) -> anyhow::Result { + conn.execute( + "INSERT INTO module (project_id, path) VALUES (?1, ?2)", + params![project.id, path], + )?; + let module = conn.query_row( + "SELECT * FROM module WHERE id=last_insert_rowid()", + (), + Self::from_row, + )?; + Ok(module) + } + + pub fn retrieve_by_id(conn: &Connection, module_id: usize) -> anyhow::Result { + let module = conn.query_row( + "SELECT * FROM module WHERE id = ?1", + params![module_id], + Self::from_row, + )?; + Ok(module) + } + + pub fn retrieve(conn: &Connection, project: &Project, path: &str) -> anyhow::Result { + let module = conn.query_row( + "SELECT * FROM module WHERE (project_id, path) = (?1, ?2)", + params![project.id, path], + Self::from_row, + )?; + Ok(module) + } + + pub fn add_symbol( + &self, + conn: &Connection, + variant: SymbolVariant, + name: &str, + ) -> anyhow::Result { + Symbol::create(conn, self, variant, name) + } + + pub fn get_symbol( + &self, + conn: &Connection, + variant: SymbolVariant, + name: &str, + ) -> anyhow::Result { + Symbol::retrieve(conn, self, variant, name) + } + + /// single thread only: retrieve then create + pub fn get_or_create_symbol( + &self, + conn: &Connection, + variant: SymbolVariant, + name: &str, + ) -> anyhow::Result { + match Symbol::retrieve(conn, self, variant, name) { + Ok(symbol) => Ok(symbol), + Err(_) => self.add_symbol(conn, variant, name), + } + } + + pub fn get_named_export_symbols(&self, conn: &Connection) -> anyhow::Result> { + let named_export_symbols: Vec = conn + .prepare("SELECT * FROM symbol WHERE (module_id, variant) = (?1, ?2)")? + .query_map( + params![self.id, SymbolVariant::NamedExport], + Symbol::from_row, + )? + .map(|s| s.unwrap()) + .collect(); + Ok(named_export_symbols) + } +} + +#[derive(Debug, Clone, Copy)] +pub enum SymbolVariant { + LocalVariable, + NamedExport, + DefaultExport, +} + +impl SymbolVariant { + pub fn from(n: usize) -> Self { + match n { + 0 => Self::LocalVariable, + 1 => Self::NamedExport, + 2 => Self::DefaultExport, + _ => unreachable!(), + } + } +} + +impl ToSql for SymbolVariant { + fn to_sql(&self) -> rusqlite::Result> { + match self { + SymbolVariant::LocalVariable => 0.to_sql(), + SymbolVariant::NamedExport => 1.to_sql(), + SymbolVariant::DefaultExport => 2.to_sql(), + } + } +} + +#[derive(Debug)] +pub struct Symbol { + pub id: usize, + pub module_id: usize, + pub variant: SymbolVariant, + pub name: String, +} + +impl Model for Symbol { + fn table() -> String { + " + symbol ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + module_id INTEGER REFERENCES module(id) ON DELETE CASCADE, + variant INTEGER NOT NULL, + name TEXT NOT NULL + ) + " + .to_string() + } +} + +impl Symbol { + pub fn from_row(row: &Row) -> rusqlite::Result { + Ok(Self { + id: row.get(0)?, + module_id: row.get(1)?, + variant: SymbolVariant::from(row.get::<_, usize>(2)?), + name: row.get(3)?, + }) + } + + /// single thread only: last_insert_rowid() + pub fn create( + conn: &Connection, + module: &Module, + variant: SymbolVariant, + name: &str, + ) -> anyhow::Result { + conn.execute( + "INSERT INTO symbol (module_id, variant, name) VALUES (?1, ?2, ?3)", + params![module.id, variant, name], + )?; + let symbol = conn.query_row( + "SELECT * FROM symbol WHERE id=last_insert_rowid()", + (), + Self::from_row, + )?; + Ok(symbol) + } + + pub fn retrieve( + conn: &Connection, + module: &Module, + variant: SymbolVariant, + name: &str, + ) -> anyhow::Result { + let symbol = conn.query_row( + "SELECT * FROM symbol WHERE (module_id, variant, name) = (?1, ?2, ?3)", + params![module.id, variant, name], + Self::from_row, + )?; + Ok(symbol) + } + + pub fn get_used_by(&self, conn: &Connection) -> anyhow::Result> { + let used_by: Vec = conn + .prepare( + " + SELECT s.* + FROM symbol s + JOIN symbol_dependency sd ON s.id = sd.symbol_id + WHERE sd.depend_on_symbol_id = ?1; + ", + )? + .query_map(params![self.id], Symbol::from_row)? + .map(|s| s.unwrap()) + .collect(); + Ok(used_by) + } + + pub fn get_used_by_routes(&self, conn: &Connection) -> anyhow::Result> { + let used_by_routes: Vec = conn + .prepare( + " + SELECT r.* + FROM route r + JOIN route_usage ru ON r.id = ru.route_id + WHERE ru.symbol_id = ?1; + ", + )? + .query_map(params![self.id], Route::from_row)? + .map(|s| s.unwrap()) + .collect(); + Ok(used_by_routes) + } +} + +// Join Table +#[derive(Debug)] +pub struct SymbolDependency { + pub id: usize, + pub symbol_id: usize, + pub depend_on_symbol_id: usize, +} + +impl Model for SymbolDependency { + fn table() -> String { + " + symbol_dependency ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + symbol_id INTEGER REFERENCES symbol(id) ON DELETE CASCADE, + depend_on_symbol_id INTEGER REFERENCES symbol(id) ON DELETE CASCADE + ) + " + .to_string() + } +} + +impl SymbolDependency { + pub fn from_row(row: &Row) -> rusqlite::Result { + Ok(Self { + id: row.get(0)?, + symbol_id: row.get(1)?, + depend_on_symbol_id: row.get(2)?, + }) + } + + /// single thread only: last_insert_rowid() + pub fn create( + conn: &Connection, + current_symbol: &Symbol, + depend_on_symbol: &Symbol, + ) -> anyhow::Result { + conn.execute( + "INSERT INTO symbol_dependency (symbol_id, depend_on_symbol_id) VALUES (?1, ?2)", + params![current_symbol.id, depend_on_symbol.id], + )?; + let symbol_dependency = conn.query_row( + "SELECT * FROM symbol_dependency WHERE id=last_insert_rowid()", + (), + Self::from_row, + )?; + Ok(symbol_dependency) + } +} + +#[derive(Debug)] +pub struct Translation { + pub id: usize, + pub project_id: usize, + pub key: String, + pub value: String, +} + +impl Model for Translation { + fn table() -> String { + " + translation ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_id INTEGER REFERENCES project(id) ON DELETE CASCADE, + key TEXT NOT NULL, + value TEXT NOT NULL + ) + " + .to_string() + } +} + +impl Translation { + pub fn from_row(row: &Row) -> rusqlite::Result { + Ok(Self { + id: row.get(0)?, + project_id: row.get(1)?, + key: row.get(2)?, + value: row.get(3)?, + }) + } + + /// single thread only: last_insert_rowid() + pub fn create( + conn: &Connection, + project: &Project, + key: &str, + value: &str, + ) -> anyhow::Result { + conn.execute( + "INSERT INTO translation (project_id, key, value) VALUES (?1, ?2, ?3)", + params![project.id, key, value], + )?; + let translation = conn.query_row( + "SELECT * FROM translation WHERE id=last_insert_rowid()", + (), + Self::from_row, + )?; + Ok(translation) + } + + pub fn retrieve(conn: &Connection, project: &Project, key: &str) -> anyhow::Result { + let translation = conn.query_row( + "SELECT * FROM translation WHERE (project_id, key) = (?1, ?2)", + params![project.id, key], + Self::from_row, + )?; + Ok(translation) + } + + pub fn search_value_exact_match( + conn: &Connection, + project: &Project, + search: &str, + ) -> anyhow::Result> { + let translations: Vec = conn + .prepare("SELECT * FROM translation WHERE (project_id, value) = (?1, ?2)")? + .query_map(params![project.id, search], Self::from_row)? + .map(|s| s.unwrap()) + .collect(); + Ok(translations) + } + + pub fn search_value_contain( + conn: &Connection, + project: &Project, + search: &str, + ) -> anyhow::Result> { + let translations: Vec = conn + .prepare( + " + SELECT * + FROM translation + WHERE project_id = ?1 + AND value LIKE '%' || ?2 || '%' + ", + )? + .query_map(params![project.id, search], Self::from_row)? + .map(|s| s.unwrap()) + .collect(); + Ok(translations) + } + + pub fn get_used_by(&self, conn: &Connection) -> anyhow::Result> { + let used_by: Vec = conn + .prepare( + " + SELECT s.* + FROM symbol s + JOIN translation_usage tu ON s.id = tu.symbol_id + WHERE tu.translation_id = ?1; + ", + )? + .query_map(params![self.id], Symbol::from_row)? + .map(|s| s.unwrap()) + .collect(); + Ok(used_by) + } +} + +// Join Table +#[derive(Debug)] +pub struct TranslationUsage { + pub id: usize, + pub translation_id: usize, + pub symbol_id: usize, +} + +impl Model for TranslationUsage { + fn table() -> String { + " + translation_usage ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + translation_id INTEGER REFERENCES translation(id) ON DELETE CASCADE, + symbol_id INTEGER REFERENCES symbol(id) ON DELETE CASCADE + ) + " + .to_string() + } +} + +impl TranslationUsage { + pub fn from_row(row: &Row) -> rusqlite::Result { + Ok(Self { + id: row.get(0)?, + translation_id: row.get(1)?, + symbol_id: row.get(2)?, + }) + } + + /// single thread only: last_insert_rowid() + pub fn create( + conn: &Connection, + translation: &Translation, + symbol: &Symbol, + ) -> anyhow::Result { + conn.execute( + "INSERT INTO translation_usage (translation_id, symbol_id) VALUES (?1, ?2)", + params![translation.id, symbol.id], + )?; + let translation = conn.query_row( + "SELECT * FROM translation_usage WHERE id=last_insert_rowid()", + (), + Self::from_row, + )?; + Ok(translation) + } +} + +#[derive(Debug)] +pub struct Route { + pub id: usize, + pub project_id: usize, + pub path: String, +} + +impl Model for Route { + fn table() -> String { + " + route ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_id INTEGER REFERENCES project(id) ON DELETE CASCADE, + path TEXT NOT NULL + ) + " + .to_string() + } +} + +impl Route { + pub fn from_row(row: &Row) -> rusqlite::Result { + Ok(Self { + id: row.get(0)?, + project_id: row.get(1)?, + path: row.get(2)?, + }) + } + + /// single thread only: last_insert_rowid() + pub fn create(conn: &Connection, project: &Project, path: &str) -> anyhow::Result { + conn.execute( + "INSERT INTO route (project_id, path) VALUES (?1, ?2)", + params![project.id, path], + )?; + let route = conn.query_row( + "SELECT * FROM route WHERE id=last_insert_rowid()", + (), + Self::from_row, + )?; + Ok(route) + } +} + +// Join Table +#[derive(Debug)] +pub struct RouteUsage { + pub id: usize, + pub route_id: usize, + pub symbol_id: usize, +} + +impl Model for RouteUsage { + fn table() -> String { + " + route_usage ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + route_id INTEGER REFERENCES route(id) ON DELETE CASCADE, + symbol_id INTEGER REFERENCES symbol(id) ON DELETE CASCADE + ) + " + .to_string() + } +} + +impl RouteUsage { + pub fn from_row(row: &Row) -> rusqlite::Result { + Ok(Self { + id: row.get(0)?, + route_id: row.get(1)?, + symbol_id: row.get(2)?, + }) + } + + /// single thread only: last_insert_rowid() + pub fn create(conn: &Connection, route: &Route, symbol: &Symbol) -> anyhow::Result { + conn.execute( + "INSERT INTO route_usage (route_id, symbol_id) VALUES (?1, ?2)", + params![route.id, symbol.id], + )?; + let route = conn.query_row( + "SELECT * FROM route_usage WHERE id=last_insert_rowid()", + (), + Self::from_row, + )?; + Ok(route) + } +} diff --git a/crates/dt_i18n/src/collect.rs b/crates/dt_i18n/src/collect.rs index 2381b7f..2459983 100644 --- a/crates/dt_i18n/src/collect.rs +++ b/crates/dt_i18n/src/collect.rs @@ -19,7 +19,7 @@ impl I18nToSymbol { module_ast: &Module, ) -> anyhow::Result<()> { let i18n_usage = core::collect_translation(module_ast)?; - for (symbol, i18n_keys) in i18n_usage { + for (symbol, i18n_keys) in i18n_usage.iter() { for i18n_key in i18n_keys.iter() { if !self.table.contains_key(i18n_key) { self.table.insert(i18n_key.to_owned(), HashMap::new()); diff --git a/crates/dt_i18n/src/lib.rs b/crates/dt_i18n/src/lib.rs index 5437391..1a5a639 100644 --- a/crates/dt_i18n/src/lib.rs +++ b/crates/dt_i18n/src/lib.rs @@ -3,3 +3,4 @@ mod collect; mod core; pub use collect::I18nToSymbol; +pub use core::collect_translation; diff --git a/crates/dt_route/src/lib.rs b/crates/dt_route/src/lib.rs index e038818..5b0db14 100644 --- a/crates/dt_route/src/lib.rs +++ b/crates/dt_route/src/lib.rs @@ -8,6 +8,56 @@ use swc_core::ecma::{ visit::{Visit, VisitWith}, }; +fn should_collect(symbol_dependency: &SymbolDependency) -> bool { + // filename should be "routes.js" + if !symbol_dependency.canonical_path.ends_with("/routes.js") { + return false; + } + // routes.js should have default export + if symbol_dependency.default_export.is_none() { + return false; + } + // routes.js should have anonumous default export + match symbol_dependency.default_export.as_ref().unwrap() { + dt_parser::types::ModuleExport::Local(exported_symbol) => { + if exported_symbol != SYMBOL_NAME_FOR_ANONYMOUS_DEFAULT_EXPORT { + return false; + } + } + dt_parser::types::ModuleExport::ReExportFrom(_) => return false, + } + // default export should depend on some symbols + match symbol_dependency + .local_variable_table + .get(SYMBOL_NAME_FOR_ANONYMOUS_DEFAULT_EXPORT) + .unwrap() + .depend_on + { + Some(_) => return true, + None => return false, + } +} + +fn collect( + module_ast: &Module, + symbol_dependency: &SymbolDependency, +) -> anyhow::Result> { + let mut route_visitor = RouteVisitor::new(symbol_dependency); + module_ast.visit_with(&mut route_visitor); + Ok(route_visitor.routes) +} + +pub fn collect_route_dependency( + module_ast: &Module, + symbol_dependency: &SymbolDependency, +) -> anyhow::Result> { + if should_collect(symbol_dependency) { + let routes = collect(module_ast, symbol_dependency)?; + return Ok(routes); + } + Ok(vec![]) +} + #[derive(Debug)] pub struct SymbolToRoutes { // one symbol can be used in multiple routes @@ -26,60 +76,21 @@ impl SymbolToRoutes { module_ast: &Module, symbol_dependency: &SymbolDependency, ) -> anyhow::Result<()> { - if Self::should_collect(symbol_dependency) { - let routes = Self::collect(module_ast, symbol_dependency)?; - self.aggregate(symbol_dependency.canonical_path.as_str(), routes); + if should_collect(symbol_dependency) { + let routes = collect(module_ast, symbol_dependency)?; + self.aggregate(symbol_dependency.canonical_path.as_str(), &routes); } Ok(()) } - fn should_collect(symbol_dependency: &SymbolDependency) -> bool { - // filename should be "routes.js" - if !symbol_dependency.canonical_path.ends_with("/routes.js") { - return false; - } - // routes.js should have default export - if symbol_dependency.default_export.is_none() { - return false; - } - // routes.js should have anonumous default export - match symbol_dependency.default_export.as_ref().unwrap() { - dt_parser::types::ModuleExport::Local(exported_symbol) => { - if exported_symbol != SYMBOL_NAME_FOR_ANONYMOUS_DEFAULT_EXPORT { - return false; - } - } - dt_parser::types::ModuleExport::ReExportFrom(_) => return false, - } - // default export should depend on some symbols - match symbol_dependency - .local_variable_table - .get(SYMBOL_NAME_FOR_ANONYMOUS_DEFAULT_EXPORT) - .unwrap() - .depend_on - { - Some(_) => return true, - None => return false, - } - } - - fn collect( - module_ast: &Module, - symbol_dependency: &SymbolDependency, - ) -> anyhow::Result> { - let mut route_visitor = RouteVisitor::new(symbol_dependency); - module_ast.visit_with(&mut route_visitor); - Ok(route_visitor.routes) - } - - fn aggregate(&mut self, module_path: &str, routes: Vec) { + fn aggregate(&mut self, module_path: &str, routes: &Vec) { let mut map = HashMap::new(); for route in routes { - for symbol in route.depend_on { - if !map.contains_key(&symbol) { - map.insert(symbol, vec![route.path.to_owned()]); + for symbol in route.depend_on.iter() { + if !map.contains_key(symbol) { + map.insert(symbol.to_string(), vec![route.path.to_owned()]); } else { - map.get_mut(&symbol).unwrap().push(route.path.to_owned()); + map.get_mut(symbol).unwrap().push(route.path.to_owned()); } } } @@ -88,9 +99,9 @@ impl SymbolToRoutes { } #[derive(Debug)] -struct Route { - path: String, - depend_on: HashSet, +pub struct Route { + pub path: String, + pub depend_on: HashSet, } #[derive(Debug)] diff --git a/crates/dt_tracker/Cargo.toml b/crates/dt_tracker/Cargo.toml index 4ef0f34..ef1730e 100644 --- a/crates/dt_tracker/Cargo.toml +++ b/crates/dt_tracker/Cargo.toml @@ -12,8 +12,9 @@ swc_core = { workspace = true } swc_ecma_parser = { workspace = true } serde = { workspace = true } -dt_graph = { version = "0.1.0", path = "../dt_graph" } -dt_parser = { version = "0.1.0", path = "../dt_parser" } +dt_database = { version = "0.1.0", path = "../dt_database" } +dt_graph = { version = "0.1.0", path = "../dt_graph" } +dt_parser = { version = "0.1.0", path = "../dt_parser" } [dev-dependencies] \ No newline at end of file diff --git a/crates/dt_tracker/src/db_version.rs b/crates/dt_tracker/src/db_version.rs new file mode 100644 index 0000000..d3c03cb --- /dev/null +++ b/crates/dt_tracker/src/db_version.rs @@ -0,0 +1,132 @@ +use crate::TraceTarget; + +use super::ModuleSymbol; +use anyhow::Context; +use dt_database::{models, SqliteDb}; +use std::collections::HashMap; + +pub struct DependencyTracker<'db> { + cache: HashMap>>, + db: &'db SqliteDb, + project: models::Project, + trace_full_path_only: bool, +} + +impl<'db> DependencyTracker<'db> { + pub fn new(db: &'db SqliteDb, project: models::Project, trace_full_path_only: bool) -> Self { + Self { + cache: HashMap::new(), + db, + project, + trace_full_path_only, + } + } + + // Current implementation is mimick version of the trace with in-memory graph. + // We can refactor it after the database feature gets validated. + pub fn trace(&mut self, module_symbol: ModuleSymbol) -> anyhow::Result>> { + // Treat routeNmaes specially since they cause a lot of circular dependencies in + // some of our codebases. One assumption of this tool is "no circular dependency" + // , so let's workaround here for now. + if module_symbol.1.to_string() == "routeNames" { + return Ok(vec![]); + } + + // early return if cached + if let Some(cached) = self.cache.get(&module_symbol) { + return Ok(cached.clone()); + } + + let module = self + .project + .get_module(&self.db.conn, &module_symbol.0) + .context(format!("module {} not found", module_symbol.0))?; + + let symbol = match &module_symbol.1 { + crate::TraceTarget::NamedExport(name) => module + .get_symbol(&self.db.conn, models::SymbolVariant::NamedExport, name) + .context(format!( + "module {} doesn't have named export symbol {}", + module.path, name + ))?, + crate::TraceTarget::DefaultExport => module + .get_symbol(&self.db.conn, models::SymbolVariant::DefaultExport, "") + .context(format!( + "module {} doesn't have default export symbol", + module.path + ))?, + crate::TraceTarget::LocalVar(name) => module + .get_symbol(&self.db.conn, models::SymbolVariant::LocalVariable, name) + .context(format!( + "module {} doesn't have local variable symbol {}", + module.path, name + ))?, + }; + + let used_by = symbol + .get_used_by(&self.db.conn) + .context(format!("get used-by vector for symbol {}", symbol.name))?; + + let mut res: Vec> = vec![]; + for next_target in used_by.iter() { + let mut paths = match next_target.module_id == symbol.module_id { + true => { + // used by symbol from the same module + match next_target.variant { + models::SymbolVariant::NamedExport => self.trace(( + module_symbol.0.clone(), + TraceTarget::NamedExport(next_target.name.to_string()), + ))?, + models::SymbolVariant::DefaultExport => { + self.trace((module_symbol.0.clone(), TraceTarget::DefaultExport))? + } + models::SymbolVariant::LocalVariable => self.trace(( + module_symbol.0.clone(), + TraceTarget::LocalVar(next_target.name.to_string()), + ))?, + } + } + false => { + // used by symbol from other module + let other_module = + models::Module::retrieve_by_id(&self.db.conn, next_target.module_id)?; + match next_target.variant { + models::SymbolVariant::NamedExport => self.trace(( + other_module.path.clone(), + TraceTarget::NamedExport(next_target.name.to_string()), + ))?, + models::SymbolVariant::DefaultExport => { + self.trace((other_module.path.clone(), TraceTarget::DefaultExport))? + } + models::SymbolVariant::LocalVariable => self.trace(( + other_module.path.clone(), + TraceTarget::LocalVar(next_target.name.to_string()), + ))?, + } + } + }; + res.append(&mut paths); + } + + // append current ModuleSymbol to each path + for path in res.iter_mut() { + path.push(module_symbol.clone()); + } + if self.trace_full_path_only { + // because we only want to trace the full path, we only need to add a new path + // when this ModuleSymbol is not using by anyone. + if res.len() == 0 { + res.push(vec![module_symbol.clone()]); + } + } else { + // always append the current ModuleSymbol since we want to list every single path + // that is reachable from the target. + res.push(vec![module_symbol.clone()]); + } + + // update cache + self.cache.insert(module_symbol.clone(), res.clone()); + + Ok(res) + } +} diff --git a/crates/dt_tracker/src/lib.rs b/crates/dt_tracker/src/lib.rs index b53bc15..f9b27cf 100644 --- a/crates/dt_tracker/src/lib.rs +++ b/crates/dt_tracker/src/lib.rs @@ -1,3 +1,5 @@ +pub mod db_version; + use anyhow::{bail, Context}; use dt_graph::used_by_graph::{UsedBy, UsedByGraph, UsedByOther, UsedByType}; use serde::{Deserialize, Serialize}; diff --git a/web/src/search/components/TreeView.tsx b/web/src/search/components/TreeView.tsx index b45a19d..e680c20 100644 --- a/web/src/search/components/TreeView.tsx +++ b/web/src/search/components/TreeView.tsx @@ -23,13 +23,15 @@ const StyledTreeView = styled(SimpleTreeView)({ }); const mapAllModuleSymbolToString = ( - project_root: string, tracePaths: ModuleSymbol[][] ): string[][] => { return tracePaths.map((tracePath) => tracePath.map(({ module_path, symbol_name }) => { - let shorterPath = module_path.slice(project_root.length); - return `${symbol_name}@${shorterPath}`; + if (module_path.startsWith("/")) { + return `${symbol_name}@${module_path.slice(1)}`; + } else { + return `${symbol_name}@${module_path}`; + } }) ); }; @@ -86,7 +88,7 @@ export const TreeView = React.memo(function TreeView({ return ( @@ -103,10 +105,7 @@ export const TreeView = React.memo(function TreeView({ {Object.entries(traceTargets).map(([traceTarget, paths]) => ( ))}