Skip to content

Commit

Permalink
feat: replace in memory searching with database searching
Browse files Browse the repository at this point in the history
  • Loading branch information
@wtlin1228
wtlin1228 committed Oct 11, 2024
1 parent 4994621 commit fb2fb47
Show file tree
Hide file tree
Showing 5 changed files with 164 additions and 257 deletions.
154 changes: 17 additions & 137 deletions crates/api_server/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,10 @@ use actix_web::{error, get, web, App, HttpServer, Result};
use clap::Parser;
use dt_core::{
database::{models, Database, SqliteDb},
graph::used_by_graph::UsedByGraph,
portable::Portable,
tracker::{
db_version::DependencyTracker as DependencyTrackerV2, DependencyTracker, TraceTarget,
},
tracker::{db_version::DependencyTracker, TraceTarget},
};
use serde::{Deserialize, Serialize};
use std::{
collections::{HashMap, HashSet},
fs::File,
io::Read,
};

struct AppState {
project_root: String,
translation_json: HashMap<String, String>,
i18n_to_symbol: HashMap<String, HashMap<String, HashSet<String>>>,
symbol_to_route: HashMap<String, HashMap<String, Vec<String>>>,
used_by_graph: UsedByGraph,
}
use std::collections::HashMap;

#[derive(Serialize, Clone)]
struct Step {
Expand All @@ -42,122 +26,23 @@ struct Info {
exact_match: bool,
}

#[get("/search/in-memory")]
async fn search_in_memory(
#[get("/search")]
async fn search(
data: web::Data<AppState>,
info: web::Query<Info>,
) -> Result<web::Json<SearchResponse>> {
let db = &data.db;
let search = &info.q;
let exact_match = info.exact_match;

let mut matched_i18n_keys: Vec<String> = Vec::new();
match exact_match {
true => {
for (i18n_key, translation) in data.translation_json.iter() {
if translation == search {
matched_i18n_keys.push(i18n_key.to_owned());
}
}
}
false => {
for (i18n_key, translation) in data.translation_json.iter() {
if translation.contains(search) {
matched_i18n_keys.push(i18n_key.to_owned());
}
}
}
}

if matched_i18n_keys.len() == 0 {
return Err(error::ErrorNotFound(format!("No result for {}", search)));
}

let mut dependency_tracker = DependencyTracker::new(&data.used_by_graph, true);
let mut trace_result = HashMap::new();
for i18n_key in matched_i18n_keys.iter() {
let mut route_to_paths = HashMap::new();
if let Some(i18n_key_usage) = data.i18n_to_symbol.get(i18n_key) {
for (module_path, symbols) in i18n_key_usage {
for symbol in symbols {
let full_paths = dependency_tracker
.trace((module_path.clone(), TraceTarget::LocalVar(symbol.clone())))
.unwrap();
// traverse each path and check if any symbol is used in some routes
for mut full_path in full_paths {
full_path.reverse();
for (i, (step_module_path, step_trace_target)) in
full_path.iter().enumerate()
{
match step_trace_target {
TraceTarget::LocalVar(step_symbol_name) => {
if let Some(symbol_to_routes) =
data.symbol_to_route.get(step_module_path)
{
if let Some(routes) = symbol_to_routes.get(step_symbol_name)
{
let dependency_from_target_to_route: Vec<Step> =
full_path[0..i]
.iter()
.map(|(path, target)| Step {
module_path: path.clone(),
symbol_name: target.to_string(),
})
.collect();
for route in routes.iter() {
if !route_to_paths.contains_key(route) {
route_to_paths
.insert(route.clone(), HashMap::new());
}
if !route_to_paths
.get(route)
.unwrap()
.contains_key(symbol)
{
route_to_paths
.get_mut(route)
.unwrap()
.insert(symbol.to_string(), vec![]);
}
route_to_paths
.get_mut(route)
.unwrap()
.get_mut(symbol)
.unwrap()
.push(dependency_from_target_to_route.clone());
}
}
}
}
_ => (),
}
}
}
}
}
}
trace_result.insert(i18n_key.to_string(), route_to_paths);
}

Ok(web::Json(SearchResponse {
project_root: data.project_root.to_owned(),
trace_result,
}))
}

#[get("/search/db")]
async fn search_db(info: web::Query<Info>) -> Result<web::Json<SearchResponse>> {
let search = &info.q;
let exact_match = info.exact_match;
// TODO: share the db connection
let db = SqliteDb::open("./database/1010.db3").unwrap();
let project = models::Project::retrieve_by_name(&db.conn, "kirby").unwrap();
// project name "default_project" can be different in feature "cross-project tracing"
let project = models::Project::retrieve_by_name(&db.conn, "default_project").unwrap();
let matched_i18n_keys = project
.search_translation(&db.conn, search, exact_match)
.unwrap();
if matched_i18n_keys.len() == 0 {
return Err(error::ErrorNotFound(format!("No result for {}", search)));
}
let mut dependency_tracker = DependencyTrackerV2::new(&db, project.clone(), true);
let mut dependency_tracker = DependencyTracker::new(&db, project.clone(), true);
let mut trace_result = HashMap::new();
for translation in matched_i18n_keys.iter() {
let mut route_to_paths = HashMap::new();
Expand Down Expand Up @@ -230,34 +115,29 @@ async fn search_db(info: web::Query<Info>) -> Result<web::Json<SearchResponse>>
}))
}

struct AppState {
db: SqliteDb,
}

#[derive(Parser)]
#[command(version, about = "Start the server to provide search API", long_about = None)]
struct Cli {
/// Portable path
#[arg(short)]
portable: String,
/// The path of your database
#[arg(long)]
db: String,
}

#[actix_web::main]
async fn main() -> std::io::Result<()> {
let cli = Cli::parse();
let mut file = File::open(cli.portable)?;
let mut exported = String::new();
file.read_to_string(&mut exported)?;
let portable = Portable::import(&exported).unwrap();

HttpServer::new(move || {
App::new()
.wrap(Cors::default().allow_any_method().allow_any_origin())
.app_data(web::Data::new(AppState {
project_root: portable.project_root.clone(),
translation_json: portable.translation_json.clone(),
i18n_to_symbol: portable.i18n_to_symbol.clone(),
symbol_to_route: portable.symbol_to_route.clone(),
used_by_graph: portable.used_by_graph.clone(),
db: SqliteDb::open(&cli.db).expect(&format!("open database from {}", cli.db)),
}))
.service(search_in_memory)
.service(search_db)
.service(search)
})
.bind(("127.0.0.1", 8080))?
.run()
Expand Down
164 changes: 90 additions & 74 deletions crates/cli/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,21 @@ use anyhow::Context;
use clap::Parser;
use dt_core::{
database::{models, Database, SqliteDb},
graph::{depend_on_graph::DependOnGraph, used_by_graph::UsedByGraph},
i18n::I18nToSymbol,
i18n::collect_translation,
parser::{
anonymous_default_export::SYMBOL_NAME_FOR_ANONYMOUS_DEFAULT_EXPORT,
collect_symbol_dependency,
types::{FromOtherModule, FromType, ModuleExport, ModuleScopedVariable, SymbolDependency},
Input,
},
path_resolver::{PathResolver, ToCanonicalString},
portable::Portable,
route::{Route, SymbolToRoutes},
route::{collect_route_dependency, Route},
scheduler::ParserCandidateScheduler,
};
use std::{
collections::{HashMap, HashSet},
fs::File,
io::{prelude::*, BufReader},
io::BufReader,
path::PathBuf,
};

Expand All @@ -38,6 +36,93 @@ struct Cli {
output: String,
}

fn main() -> anyhow::Result<()> {
let cli = Cli::parse();

parse_export_project_to_database(&cli.input, &cli.output, &cli.translation_path)
.context("parse and export project to database")?;

Ok(())
}

fn parse_export_project_to_database(
project_root: &str,
output_database_path: &str,
translation_file_path: &str,
) -> anyhow::Result<()> {
let project_root = PathBuf::from(project_root).to_canonical_string()?;
// project name "default_project" can be different in feature "cross-project tracing"
let project =
Project::new("default_project", &project_root, output_database_path).context(format!(
"ready to a emit the project to database, project: {}, database: {}",
project_root, output_database_path
))?;

let translation_file = File::open(translation_file_path).context(format!(
"open translation file, path: {}",
translation_file_path
))?;
let translation_json_reader = BufReader::new(translation_file);
let translation_json: HashMap<String, String> =
serde_json::from_reader(translation_json_reader).context(format!(
"deserialize translation file, path: {}",
translation_file_path
))?;
project
.add_translation(&translation_json)
.context("add translation to project")?;

let mut scheduler = ParserCandidateScheduler::new(&project_root);
loop {
match scheduler.get_one_candidate() {
Some(c) => {
let module_src = c
.to_str()
.context(format!("get module_src, path_buf: {:?}", c))?;
let module_ast = Input::Path(module_src)
.get_module_ast()
.context(format!("get module ast, module_src: {}", module_src))?;

let symbol_dependency = collect_symbol_dependency(&module_ast, module_src)
.context(format!(
"collect symbol dependency for module: {}",
&module_src
))?;
let module = project
.add_module(&symbol_dependency)
.context(format!(
"add symbol dependency of module {} to project",
symbol_dependency.canonical_path
))
.context(format!("add module {} to project", module_src))?;

let i18n_usage = collect_translation(&module_ast)
.context(format!("collect i18n usage for module: {}", &module_src))?;
project
.add_i18n_usage(&module, &i18n_usage)
.context(format!(
"add i18n usage of module {} to project",
module_src
))?;

let route_usage = collect_route_dependency(&module_ast, &symbol_dependency)
.context(format!("collect route usage for module: {}", &module_src))?;
project
.add_route_usage(&module, &route_usage)
.context(format!(
"add route usage of module {} to project",
module_src
))?;

scheduler.mark_candidate_as_parsed(c);
}
None => break,
}
}

Ok(())
}

struct Project {
db: SqliteDb,
project_root: String,
Expand Down Expand Up @@ -425,72 +510,3 @@ impl Project {
Ok(())
}
}

fn main() -> anyhow::Result<()> {
let cli = Cli::parse();
let project_root = PathBuf::from(&cli.input).to_canonical_string()?;
let project = Project::new("kirby", &project_root, "./database/1010.db3")?;
let translation_file = File::open(&cli.translation_path)?;
let translation_json_reader = BufReader::new(translation_file);
let mut scheduler = ParserCandidateScheduler::new(&project_root);
let mut depend_on_graph = DependOnGraph::new(&project_root);
let mut symbol_to_route = SymbolToRoutes::new();
let mut i18n_to_symbol = I18nToSymbol::new();

let translation_json: HashMap<String, String> =
serde_json::from_reader(translation_json_reader)?;
project
.add_translation(&translation_json)
.context("add translation to project")?;

loop {
match scheduler.get_one_candidate() {
Some(c) => {
let module_src = c.to_str().context(format!("to_str() failed: {:?}", c))?;
let module_ast = Input::Path(module_src).get_module_ast()?;
let symbol_dependency = collect_symbol_dependency(&module_ast, module_src)?;
let i18n_usage = i18n_to_symbol.collect_i18n_usage(module_src, &module_ast)?;
let route_usage =
symbol_to_route.collect_route_dependency(&module_ast, &symbol_dependency)?;

let module = project
.add_module(&symbol_dependency)
.context(format!(
"add module {} to project",
symbol_dependency.canonical_path
))
.context(format!("add module {} to project", module_src))?;
project
.add_i18n_usage(&module, &i18n_usage)
.context(format!(
"add i18n usage of module {} to project",
module_src
))?;
project
.add_route_usage(&module, &route_usage)
.context(format!(
"add route usage of module {} to project",
module_src
))?;

depend_on_graph.add_symbol_dependency(symbol_dependency)?;
scheduler.mark_candidate_as_parsed(c);
}
None => break,
}
}

let portable = Portable::new(
project_root.to_owned(),
translation_json,
i18n_to_symbol.table,
symbol_to_route.table,
UsedByGraph::from(&depend_on_graph),
);

let serialized = portable.export()?;
let mut file = File::create(&cli.output)?;
file.write_all(serialized.as_bytes())?;

Ok(())
}
Loading

0 comments on commit fb2fb47

Please sign in to comment.