diff --git a/Cargo.lock b/Cargo.lock index 701a0fba..fe1623ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4196,6 +4196,7 @@ dependencies = [ "pollster", "reqwest", "rustc-hash", + "same-file", "serde", "serde-wasm-bindgen", "serde_json", diff --git a/cli/src/compile.rs b/cli/src/compile.rs index 051f4c95..ec3c3a53 100644 --- a/cli/src/compile.rs +++ b/cli/src/compile.rs @@ -2,7 +2,7 @@ use std::path::Path; use typst::doc::Document; use typst_ts_compiler::{ - service::{CompileDriver, CompileExporter, Compiler, DynamicLayoutCompiler, WatchDriver}, + service::{CompileActor, CompileDriver, CompileExporter, Compiler, DynamicLayoutCompiler}, TypstSystemWorld, }; use typst_ts_core::{config::CompileOpts, exporter_builtins::GroupExporter, path::PathClean}; @@ -104,9 +104,9 @@ pub fn compile_export(args: CompileArgs, exporter: GroupExporter) -> ! // CompileExporter + DynamicLayoutCompiler + WatchDriver let driver = CompileExporter::new(driver).with_exporter(exporter); let driver = DynamicLayoutCompiler::new(driver, output_dir).with_enable(args.dynamic_layout); - let mut driver = WatchDriver::new(driver, watch_root).with_enable(args.watch); + let actor = CompileActor::new(driver, watch_root).with_watch(args.watch); utils::async_continue(async move { - utils::logical_exit(driver.compile().await); + utils::logical_exit(actor.run()); }) } diff --git a/cli/src/query_repl.rs b/cli/src/query_repl.rs index 4433330f..20021280 100644 --- a/cli/src/query_repl.rs +++ b/cli/src/query_repl.rs @@ -10,6 +10,7 @@ use rustyline::hint::{Hinter, HistoryHinter}; use rustyline::validate::MatchingBracketValidator; use rustyline::{Cmd, CompletionType, Config, EditMode, Editor, KeyEvent}; use rustyline::{Helper, Validator}; +use typst_ts_core::TakeAs; use crate::query::serialize; use crate::CompileOnceArgs; @@ -123,7 +124,7 @@ impl Completer for ReplContext { driver.world.reset(); let typst_completions = driver .with_shadow_file_by_id(main_id, dyn_content.as_bytes().into(), |driver| { - let frames = driver.compile().map(|d| d.pages); + let frames = driver.compile().map(|d| d.take().pages); let frames = frames.as_ref().map(|v| v.as_slice()).unwrap_or_default(); let source = driver.world.main(); Ok(autocomplete(&driver.world, frames, &source, cursor, true)) @@ -226,13 +227,13 @@ pub fn start_repl_test(args: CompileOnceArgs) -> rustyline::Result<()> { impl ReplContext { fn repl_process_line(&mut self, line: String) { - let compiled = - self.driver - .borrow_mut() - .with_compile_diag::(|driver: &mut CompileDriver| { - let doc = driver.compile()?; - driver.query(line, &doc) - }); + let compiled = self.driver.borrow_mut().with_stage_diag::( + "compiling", + |driver: &mut CompileDriver| { + let doc = driver.compile()?; + driver.query(line, &doc) + }, + ); if let Some(compiled) = compiled { let serialized = serialize(&compiled, "json").unwrap(); diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index 97bd1f79..ec5f5de7 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -27,6 +27,8 @@ serde.workspace = true serde_json.workspace = true serde-wasm-bindgen = { workspace = true, optional = true } +same-file = { version = "1", optional = true } + memmap2 = { workspace = true, optional = true } dirs = { workspace = true, optional = true } walkdir = { workspace = true, optional = true } @@ -43,7 +45,6 @@ nohash-hasher.workspace = true pathdiff.workspace = true dissimilar.workspace = true tar.workspace = true - wasm-bindgen = { workspace = true, optional = true } wasm-bindgen-futures = { workspace = true, optional = true } js-sys = { workspace = true, optional = true } @@ -88,7 +89,7 @@ system-compile = [ "dep:notify", "dep:log", ] -system-watch = ["dep:notify", "dep:tokio"] +system-watch = ["dep:notify", "dep:tokio", "dep:same-file"] system = ["system-compile", "system-watch"] dynamic-layout = ["dep:typst-ts-svg-exporter"] __web = [ diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index b2d779d9..c2c1c659 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -49,7 +49,10 @@ pub mod service; /// Run the compiler in the system environment. #[cfg(feature = "system-compile")] pub(crate) mod system; -use std::path::{Path, PathBuf}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; #[cfg(feature = "system-compile")] pub use system::TypstSystemWorld; @@ -63,7 +66,8 @@ use typst::{ diag::{At, FileResult, SourceResult}, syntax::Span, }; -use typst_ts_core::{Bytes, TypstFileId}; +use typst_ts_core::{Bytes, ImmutPath, TypstFileId}; +use vfs::notify::FilesystemEvent; /// Latest version of the shadow api, which is in beta. pub trait ShadowApi { @@ -71,8 +75,15 @@ pub trait ShadowApi { unimplemented!() } + /// Get the shadow files. + fn shadow_paths(&self) -> Vec>; + /// Reset the shadow files. - fn reset_shadow(&mut self); + fn reset_shadow(&mut self) { + for path in self.shadow_paths() { + self.unmap_shadow(&path).unwrap(); + } + } /// Add a shadow file to the driver. fn map_shadow(&self, path: &Path, content: Bytes) -> FileResult<()>; @@ -123,3 +134,10 @@ pub trait ShadowApi { self.with_shadow_file(&file_path, content, f) } } + +/// Latest version of the notify api, which is in beta. +pub trait NotifyApi { + fn iter_dependencies<'a>(&'a self, f: &mut dyn FnMut(&'a ImmutPath, instant::SystemTime)); + + fn notify_fs_event(&mut self, event: FilesystemEvent); +} diff --git a/compiler/src/service/compile.rs b/compiler/src/service/compile.rs new file mode 100644 index 00000000..a246b372 --- /dev/null +++ b/compiler/src/service/compile.rs @@ -0,0 +1,564 @@ +use std::{ + collections::HashSet, + num::NonZeroUsize, + ops::Deref, + path::{Path, PathBuf}, + sync::Arc, + thread::JoinHandle, +}; + +use serde::Serialize; +use tokio::sync::{mpsc, oneshot}; +use typst::{ + doc::{Frame, FrameItem, Position}, + geom::Point, + syntax::{LinkedNode, Source, Span, SyntaxKind, VirtualPath}, + World, +}; + +use crate::{ + vfs::notify::{FilesystemEvent, MemoryEvent, NotifyMessage}, + world::{CompilerFeat, CompilerWorld}, + ShadowApi, +}; +use typst_ts_core::{ + error::prelude::ZResult, vector::span_id_from_u64, TypstDocument, TypstFileId, +}; + +use super::{Compiler, DiagObserver, WorkspaceProvider, WorldExporter}; + +/// A task that can be sent to the context (compiler thread) +/// +/// The internal function will be dereferenced and called on the context. +type BorrowTask = Box; + +/// Interrupts for the compiler thread. +enum CompilerInterrupt { + /// Interrupted by task. + /// + /// See [`CompileClient::steal`] for more information. + Task(BorrowTask), + /// Interrupted by memory file changes. + Memory(MemoryEvent), + /// Interrupted by file system event. + /// + /// If the event is `None`, it means the initial file system scan is done. + /// Otherwise, it means a file system event is received. + Fs(Option), +} + +/// Responses from the compiler thread. +enum CompilerResponse { + /// Response to the file watcher + Notify(NotifyMessage), +} + +/// A tagged memory event with logical tick. +struct TaggedMemoryEvent { + /// The logical tick when the event is received. + logical_tick: usize, + /// The memory event happened. + event: MemoryEvent, +} + +/// The compiler thread. +pub struct CompileActor { + /// The underlying compiler. + pub compiler: C, + /// The root path of the workspace. + pub root: PathBuf, + /// Whether to enable file system watching. + pub enable_watch: bool, + + /// The current logical tick. + logical_tick: usize, + /// Last logical tick when invalidation is caused by shadow update. + dirty_shadow_logical_tick: usize, + + /// Estimated latest set of shadow files. + estimated_shadow_files: HashSet>, + /// The latest compiled document. + latest_doc: Option>, + + /// Internal channel for stealing the compiler thread. + steal_send: mpsc::UnboundedSender>, + steal_recv: mpsc::UnboundedReceiver>, + + /// Internal channel for memory events. + memory_send: mpsc::UnboundedSender, + memory_recv: mpsc::UnboundedReceiver, +} + +impl CompileActor +where + C::World: for<'files> codespan_reporting::files::Files<'files, FileId = TypstFileId>, +{ + /// Create a new compiler thread. + pub fn new(compiler: C, root: PathBuf) -> Self { + let (steal_send, steal_recv) = mpsc::unbounded_channel(); + let (memory_send, memory_recv) = mpsc::unbounded_channel(); + + Self { + compiler, + root, + + logical_tick: 1, + enable_watch: false, + dirty_shadow_logical_tick: 0, + + estimated_shadow_files: Default::default(), + latest_doc: None, + + steal_send, + steal_recv, + + memory_send, + memory_recv, + } + } + + /// Run the compiler thread synchronously. + pub fn run(self) -> bool { + use tokio::runtime::Handle; + + if Handle::try_current().is_err() && self.enable_watch { + log::error!("Typst compiler thread with watch enabled must be run in a tokio runtime"); + return false; + } + + tokio::task::block_in_place(move || Handle::current().block_on(self.block_run_inner())) + } + + /// Inner function for `run`, it launches the compiler thread and blocks + /// until it exits. + async fn block_run_inner(mut self) -> bool { + if !self.enable_watch { + let compiled = self + .compiler + .with_stage_diag::("compiling", |driver| driver.compile()); + return compiled.is_some(); + } + + if let Some(h) = self.spawn().await { + // Note: this is blocking the current thread. + // Note: the block safety is ensured by `run` function. + h.join().unwrap(); + } + + true + } + + /// Spawn the compiler thread. + pub async fn spawn(mut self) -> Option> { + if !self.enable_watch { + self.compiler + .with_stage_diag::("compiling", |driver| driver.compile()); + return None; + } + + // Setup internal channels. + let (dep_tx, dep_rx) = tokio::sync::mpsc::unbounded_channel(); + let (fs_tx, mut fs_rx) = tokio::sync::mpsc::unbounded_channel(); + + // Wrap sender to send compiler response. + let compiler_ack = move |res: CompilerResponse| match res { + CompilerResponse::Notify(msg) => dep_tx.send(msg).unwrap(), + }; + + // Spawn file system watcher. + tokio::spawn(super::watch_deps(dep_rx, move |event| { + fs_tx.send(event).unwrap(); + })); + + // Spawn compiler thread. + let compile_thread = ensure_single_thread("typst-compiler", async move { + log::debug!("CompileActor: initialized"); + + // Wait for first events. + while let Some(event) = tokio::select! { + Some(it) = fs_rx.recv() => Some(CompilerInterrupt::Fs(it)), + Some(it) = self.memory_recv.recv() => Some(CompilerInterrupt::Memory(it)), + Some(it) = self.steal_recv.recv() => Some(CompilerInterrupt::Task(it)), + } { + // Small step to warp the logical clock. + self.logical_tick += 1; + + // Accumulate events. + let mut need_recompile = false; + need_recompile = self.process(event, &compiler_ack) || need_recompile; + while let Some(event) = fs_rx + .try_recv() + .ok() + .map(CompilerInterrupt::Fs) + .or_else(|| { + self.memory_recv + .try_recv() + .ok() + .map(CompilerInterrupt::Memory) + }) + .or_else(|| self.steal_recv.try_recv().ok().map(CompilerInterrupt::Task)) + { + need_recompile = self.process(event, &compiler_ack) || need_recompile; + } + + // Compile if needed. + if need_recompile { + self.compile(&compiler_ack); + } + } + + log::debug!("CompileActor: exited"); + }) + .unwrap(); + + // Return the thread handle. + Some(compile_thread) + } + + /// Compile the document. + fn compile(&mut self, send: impl Fn(CompilerResponse)) { + use CompilerResponse::*; + + // Compile the document. + self.latest_doc = self + .compiler + .with_stage_diag::("compiling", |driver| driver.compile()); + + // Evict compilation cache. + comemo::evict(30); + + // Notify the new file dependencies. + let mut deps = vec![]; + self.compiler + .iter_dependencies(&mut |dep, _| deps.push(dep.clone())); + send(Notify(NotifyMessage::SyncDependency(deps))); + } + + /// Process some interrupt. + fn process(&mut self, event: CompilerInterrupt, send: impl Fn(CompilerResponse)) -> bool { + use CompilerResponse::*; + // warp the logical clock by one. + self.logical_tick += 1; + + match event { + // Borrow the compiler thread and run the task. + // + // See [`CompileClient::steal`] for more information. + CompilerInterrupt::Task(task) => { + log::debug!("CompileActor: execute task"); + + task(self); + + // Will never trigger compilation + false + } + // Handle memory events. + CompilerInterrupt::Memory(event) => { + log::debug!("CompileActor: memory event incoming"); + + // Emulate memory changes. + let mut files = HashSet::new(); + if matches!(event, MemoryEvent::Sync(..)) { + files = self.estimated_shadow_files.clone(); + self.estimated_shadow_files.clear(); + } + match &event { + MemoryEvent::Sync(event) | MemoryEvent::Update(event) => { + for path in event.removes.iter().map(Deref::deref) { + self.estimated_shadow_files.remove(path); + files.insert(path.into()); + } + for path in event.inserts.iter().map(|e| e.0.deref()) { + self.estimated_shadow_files.insert(path.into()); + files.remove(path); + } + } + } + + // If there is no invalidation happening, apply memory changes directly. + if files.is_empty() && self.dirty_shadow_logical_tick == 0 { + self.apply_memory_changes(event); + + // Will trigger compilation + return true; + } + + // Otherwise, send upstream update event. + // Also, record the logical tick when shadow is dirty. + self.dirty_shadow_logical_tick = self.logical_tick; + send(Notify(NotifyMessage::UpstreamUpdate( + crate::vfs::notify::UpstreamUpdateEvent { + invalidates: files.into_iter().collect(), + opaque: Box::new(TaggedMemoryEvent { + logical_tick: self.logical_tick, + event, + }), + }, + ))); + + // Delayed trigger compilation + false + } + // Handle file system events. + CompilerInterrupt::Fs(event) => { + log::debug!("CompileActor: fs event incoming {:?}", event); + + // Handle file system event if any. + if let Some(mut event) = event { + // Handle delayed upstream update event before applying file system changes + if let FilesystemEvent::UpstreamUpdate { upstream_event, .. } = &mut event { + let event = upstream_event.take().unwrap().opaque; + let TaggedMemoryEvent { + logical_tick, + event, + } = *event.downcast().unwrap(); + + // Recovery from dirty shadow state. + if logical_tick == self.dirty_shadow_logical_tick { + self.dirty_shadow_logical_tick = 0; + } + + self.apply_memory_changes(event); + } + + // Apply file system changes. + self.compiler.notify_fs_event(event); + } + + // Will trigger compilation + true + } + } + } + + /// Apply memory changes to underlying compiler. + fn apply_memory_changes(&mut self, event: MemoryEvent) { + if matches!(event, MemoryEvent::Sync(..)) { + self.compiler.reset_shadow(); + } + match event { + MemoryEvent::Update(event) | MemoryEvent::Sync(event) => { + for removes in event.removes { + let _ = self.compiler.unmap_shadow(&removes); + } + for (p, t) in event.inserts { + let _ = self.compiler.map_shadow(&p, t.content().cloned().unwrap()); + } + } + } + } +} + +impl CompileActor { + pub fn with_watch(mut self, enable_watch: bool) -> Self { + self.enable_watch = enable_watch; + self + } + + pub fn split(self) -> (Self, CompileClient) { + let steal_send = self.steal_send.clone(); + let memory_send = self.memory_send.clone(); + ( + self, + CompileClient { + steal_send, + memory_send, + _ctx: std::marker::PhantomData, + }, + ) + } + + pub fn document(&self) -> Option> { + self.latest_doc.clone() + } +} +pub struct CompileClient { + steal_send: mpsc::UnboundedSender>, + memory_send: mpsc::UnboundedSender, + + _ctx: std::marker::PhantomData, +} + +impl CompileClient { + fn steal_inner( + &mut self, + f: impl FnOnce(&mut Ctx) -> Ret + Send + 'static, + ) -> oneshot::Receiver { + let (tx, rx) = oneshot::channel(); + + self.steal_send + .send(Box::new(move |this: &mut Ctx| { + if tx.send(f(this)).is_err() { + // Receiver was dropped. The main thread may have exited, or the request may + // have been cancelled. + log::warn!("could not send back return value from Typst thread"); + } + })) + .unwrap(); + rx + } + + pub fn steal( + &mut self, + f: impl FnOnce(&mut Ctx) -> Ret + Send + 'static, + ) -> ZResult { + Ok(self.steal_inner(f).blocking_recv().unwrap()) + } + + /// Steal the compiler thread and run the given function. + pub async fn steal_async( + &mut self, + f: impl FnOnce(&mut Ctx, tokio::runtime::Handle) -> Ret + Send + 'static, + ) -> ZResult { + // get current async handle + let handle = tokio::runtime::Handle::current(); + Ok(self + .steal_inner(move |this: &mut Ctx| f(this, handle.clone())) + .await + .unwrap()) + } + + pub fn add_memory_changes(&self, event: MemoryEvent) { + self.memory_send.send(event).unwrap(); + } +} + +#[derive(Debug, Serialize)] +pub struct DocToSrcJumpInfo { + filepath: String, + start: Option<(usize, usize)>, // row, column + end: Option<(usize, usize)>, +} + +// todo: remove constraint to CompilerWorld +impl>> CompileClient> +where + Ctx::World: WorkspaceProvider, +{ + /// fixme: character is 0-based, UTF-16 code unit. + /// We treat it as UTF-8 now. + pub async fn resolve_src_to_doc_jump( + &mut self, + filepath: PathBuf, + line: usize, + character: usize, + ) -> ZResult> { + self.steal_async(move |this, _| { + let doc = this.document()?; + + let world = this.compiler.world(); + + let relative_path = filepath + .strip_prefix(&this.compiler.world().workspace_root()) + .ok()?; + + let source_id = TypstFileId::new(None, VirtualPath::new(relative_path)); + let source = world.source(source_id).ok()?; + let cursor = source.line_column_to_byte(line, character)?; + + jump_from_cursor(&doc.pages, &source, cursor) + }) + .await + } + + pub async fn resolve_doc_to_src_jump(&mut self, id: u64) -> ZResult> { + let resolve_off = + |src: &Source, off: usize| src.byte_to_line(off).zip(src.byte_to_column(off)); + + self.steal_async(move |this, _| { + let world = this.compiler.world(); + let span = span_id_from_u64(id)?; + let src_id = span.id()?; + let source = world.source(src_id).ok()?; + let range = source.find(span)?.range(); + let filepath = world.path_for_id(src_id).ok()?; + Some(DocToSrcJumpInfo { + filepath: filepath.to_string_lossy().to_string(), + start: resolve_off(&source, range.start), + end: resolve_off(&source, range.end), + }) + }) + .await + } +} + +/// Spawn a thread and run the given future on it. +/// +/// Note: the future is run on a single-threaded tokio runtime. +fn ensure_single_thread + Send + 'static>( + name: &str, + f: F, +) -> std::io::Result> { + std::thread::Builder::new().name(name.to_owned()).spawn(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(f); + }) +} + +/// Find the output location in the document for a cursor position. +pub fn jump_from_cursor(frames: &[Frame], source: &Source, cursor: usize) -> Option { + let node = LinkedNode::new(source.root()).leaf_at(cursor)?; + if node.kind() != SyntaxKind::Text { + return None; + } + + let mut min_dis = u64::MAX; + let mut p = Point::default(); + let mut ppage = 0usize; + + let span = node.span(); + for (i, frame) in frames.iter().enumerate() { + let t_dis = min_dis; + if let Some(pos) = find_in_frame(frame, span, &mut min_dis, &mut p) { + return Some(Position { + page: NonZeroUsize::new(i + 1).unwrap(), + point: pos, + }); + } + if t_dis != min_dis { + ppage = i; + } + } + + if min_dis == u64::MAX { + return None; + } + + Some(Position { + page: NonZeroUsize::new(ppage + 1).unwrap(), + point: p, + }) +} + +/// Find the position of a span in a frame. +fn find_in_frame(frame: &Frame, span: Span, min_dis: &mut u64, p: &mut Point) -> Option { + for (mut pos, item) in frame.items() { + if let FrameItem::Group(group) = item { + // TODO: Handle transformation. + if let Some(point) = find_in_frame(&group.frame, span, min_dis, p) { + return Some(point + pos); + } + } + + if let FrameItem::Text(text) = item { + for glyph in &text.glyphs { + if glyph.span.0 == span { + return Some(pos); + } + if glyph.span.0.id() == span.id() { + let dis = glyph.span.0.number().abs_diff(span.number()); + if dis < *min_dis { + *min_dis = dis; + *p = pos; + } + } + pos.x += glyph.x_advance.at(text.size); + } + } + } + + None +} diff --git a/compiler/src/service/diag.rs b/compiler/src/service/diag.rs index f36d724e..af037e2b 100644 --- a/compiler/src/service/diag.rs +++ b/compiler/src/service/diag.rs @@ -82,7 +82,7 @@ fn label<'files, W: World + Files<'files, FileId = TypstFileId>>( pub fn status(entry_file: TypstFileId, status: DiagStatus) -> io::Result<()> { let input = entry_file; match status { - DiagStatus::Compiling => log::info!("{:?}: compiling ...", input), + DiagStatus::Stage(stage) => log::info!("{:?}: {} ...", input, stage), DiagStatus::Success(duration) => { log::info!("{:?}: Compilation succeeded in {:?}", input, duration) } diff --git a/compiler/src/service/driver.rs b/compiler/src/service/driver.rs index e1426399..8ca3e467 100644 --- a/compiler/src/service/driver.rs +++ b/compiler/src/service/driver.rs @@ -3,13 +3,11 @@ use std::{ sync::Arc, }; -use crate::ShadowApi; +use crate::{NotifyApi, ShadowApi}; use typst::{diag::SourceResult, syntax::VirtualPath, World}; -use typst_ts_core::{ - exporter_builtins::GroupExporter, path::PathClean, Bytes, Exporter, TakeAs, TypstFileId, -}; +use typst_ts_core::{path::PathClean, Bytes, ImmutPath, TypstFileId}; -use super::{Compiler, WorkspaceProvider, WrappedCompiler}; +use super::{Compiler, WorkspaceProvider}; /// CompileDriverImpl is a driver for typst compiler. /// It is responsible for operating the compiler without leaking implementation @@ -58,7 +56,7 @@ impl CompileDriverImpl { } } -impl Compiler for CompileDriverImpl { +impl Compiler for CompileDriverImpl { type World = W; fn world(&self) -> &Self::World { @@ -98,277 +96,43 @@ impl Compiler for CompileDriverImpl { self._relevant(event).unwrap_or(true) } + + fn iter_dependencies<'a>(&'a self, f: &mut dyn FnMut(&'a ImmutPath, instant::SystemTime)) { + self.world.iter_dependencies(f) + } + + fn notify_fs_event(&mut self, event: crate::vfs::notify::FilesystemEvent) { + self.world.notify_fs_event(event) + } } impl ShadowApi for CompileDriverImpl { + #[inline] fn _shadow_map_id(&self, file_id: TypstFileId) -> typst::diag::FileResult { self.world._shadow_map_id(file_id) } + #[inline] + fn shadow_paths(&self) -> Vec> { + self.world.shadow_paths() + } + + #[inline] fn reset_shadow(&mut self) { self.world.reset_shadow() } + #[inline] fn map_shadow(&self, path: &Path, content: Bytes) -> typst::diag::FileResult<()> { self.world.map_shadow(path, content) } + #[inline] fn unmap_shadow(&self, path: &Path) -> typst::diag::FileResult<()> { self.world.unmap_shadow(path) } } -pub struct CompileExporter { - pub compiler: C, - pub exporter: GroupExporter, -} - -impl CompileExporter { - pub fn new(compiler: C) -> Self { - Self { - compiler, - exporter: GroupExporter::new(vec![]), - } - } - - /// Wrap driver with a given exporter. - pub fn with_exporter(mut self, exporter: GroupExporter) -> Self { - self.exporter = exporter; - self - } - - /// set an exporter. - pub fn set_exporter(&mut self, exporter: GroupExporter) { - self.exporter = exporter; - } - - /// Export a typst document using `typst_ts_core::DocumentExporter`. - pub fn export(&self, output: Arc) -> SourceResult<()> { - self.exporter.export(self.compiler.world(), output) - } -} - -impl WrappedCompiler for CompileExporter { - type Compiler = C; - - fn inner(&self) -> &Self::Compiler { - &self.compiler - } - - fn inner_mut(&mut self) -> &mut Self::Compiler { - &mut self.compiler - } - - fn wrap_compile(&mut self) -> SourceResult { - let doc = Arc::new(self.inner_mut().compile()?); - self.export(doc.clone())?; - - // Note: when doc is not retained by the exporters, no clone happens, - // because of the `Arc` type detecting a single owner at runtime. - Ok(doc.take()) - } -} - -pub type LayoutWidths = Vec; - -pub struct DynamicLayoutCompiler { - pub compiler: C, - - pub enable_dynamic_layout: bool, - - // todo: abstract this - output: PathBuf, - pub extension: String, - - pub layout_widths: LayoutWidths, - - /// Specify the target. It's default value is `web`. - /// You can specify a sub target like `web-dark` to refine the target. - /// Though we even don't encourage you to do so. - /// - /// Before typst allowing passing arguments to the compiler, this is - /// (probably) the only way to control the typst code's behavior. - pub target: String, -} - -impl DynamicLayoutCompiler { - pub fn new(compiler: C, output: PathBuf) -> Self { - Self { - compiler, - output, - enable_dynamic_layout: false, - extension: "multi.sir.in".to_owned(), - layout_widths: LayoutWidths::from_iter( - (0..40) - .map(|i| typst::geom::Abs::pt(750.0) - typst::geom::Abs::pt(i as f64 * 10.0)), - ), - target: "web".to_owned(), - } - } - - pub fn set_output(&mut self, output: PathBuf) { - self.output = output; - } - - pub fn set_extension(&mut self, extension: String) { - self.extension = extension; - } - - pub fn set_layout_widths(&mut self, layout_widths: LayoutWidths) { - self.layout_widths = layout_widths; - } - - pub fn set_target(&mut self, target: String) { - self.target = target; - } - - pub fn with_enable(mut self, enable_dynamic_layout: bool) -> Self { - self.enable_dynamic_layout = enable_dynamic_layout; - self - } -} - -#[cfg(feature = "dynamic-layout")] -impl WrappedCompiler for DynamicLayoutCompiler { - type Compiler = C; - - fn inner(&self) -> &Self::Compiler { - &self.compiler - } - - fn inner_mut(&mut self) -> &mut Self::Compiler { - &mut self.compiler - } - - fn wrap_compile(&mut self) -> SourceResult { - use std::str::FromStr; - use typst::{ - diag::At, - syntax::{PackageSpec, Span}, - }; - use typst_ts_svg_exporter::{flat_ir::serialize_doc, DynamicLayoutSvgExporter}; - - if !self.enable_dynamic_layout { - return self.inner_mut().compile(); - } - - let variable_file = TypstFileId::new( - Some(PackageSpec::from_str("@preview/typst-ts-variables:0.1.0").at(Span::detached())?), - VirtualPath::new("lib.typ"), - ); - - let pure_doc = Arc::new(self.inner_mut().compile()?); - - // self.export(doc.clone())?; - // checkout the entry file - - let mut svg_exporter = DynamicLayoutSvgExporter::default(); - - // for each 10pt we rerender once - let instant_begin = instant::Instant::now(); - for (i, current_width) in self.layout_widths.clone().into_iter().enumerate() { - let instant = instant::Instant::now(); - // replace layout - - let variables: String = format!( - r##" -#let page-width = {:2}pt -#let target = "{}""##, - current_width.to_pt(), - self.target, - ); - - log::trace!( - "rerendering {} at {:?}, width={current_width:?} target={}", - i, - instant - instant_begin, - self.target, - ); - - self.with_shadow_file_by_id(variable_file, variables.as_bytes().into(), |this| { - // compile and export document - let output = Arc::new(this.inner_mut().compile()?); - svg_exporter.render(current_width, output); - log::trace!( - "rerendered {} at {:?}, {}", - i, - instant - instant_begin, - svg_exporter.debug_stat() - ); - Ok(()) - })?; - } - - let module_output = self.output.with_extension(&self.extension); - - let doc = svg_exporter.finalize(); - - std::fs::write(module_output, serialize_doc(doc)).unwrap(); - - let instant = instant::Instant::now(); - log::trace!("multiple layouts finished at {:?}", instant - instant_begin); - - Ok(pure_doc.take()) - } -} - -pub struct WatchDriver { - pub compiler: C, - pub root: PathBuf, - pub enable_watch: bool, -} - -// todo: remove cfg feature here -#[cfg(feature = "system-watch")] -use super::DiagObserver; -#[cfg(feature = "system-watch")] -impl WatchDriver -where - C::World: for<'files> codespan_reporting::files::Files<'files, FileId = TypstFileId>, -{ - pub fn new(compiler: C, root: PathBuf) -> Self { - Self { - compiler, - root, - enable_watch: false, - } - } - - pub fn with_enable(mut self, enable_watch: bool) -> Self { - self.enable_watch = enable_watch; - self - } - - pub async fn compile(&mut self) -> bool { - if !self.enable_watch { - let compiled = self - .compiler - .with_compile_diag::(|driver| driver.compile()); - return compiled.is_some(); - } - - super::watch_dir(&self.root.clone(), move |events| { - // relevance checking - if events.is_some() - && !events - .unwrap() - .iter() - // todo: inner - .any(|event| self.compiler.relevant(event)) - { - return; - } - - // compile - self.compiler - .with_compile_diag::(|driver| driver.compile()); - comemo::evict(30); - }) - .await; - true - } -} - // todo: Print that a package downloading is happening. // fn print_downloading(_spec: &PackageSpec) -> std::io::Result<()> { // let mut w = color_stream(); diff --git a/compiler/src/service/export.rs b/compiler/src/service/export.rs new file mode 100644 index 00000000..02d61270 --- /dev/null +++ b/compiler/src/service/export.rs @@ -0,0 +1,217 @@ +use std::{path::PathBuf, sync::Arc}; + +use crate::ShadowApi; +use typst::diag::SourceResult; +use typst_ts_core::{exporter_builtins::GroupExporter, DynExporter, TypstDocument}; + +use super::{Compiler, WrappedCompiler}; + +pub trait WorldExporter { + fn export(&mut self, output: Arc) -> SourceResult<()>; +} + +pub struct CompileExporter { + pub compiler: C, + pub exporter: DynExporter, +} + +impl CompileExporter { + pub fn new(compiler: C) -> Self { + Self { + compiler, + exporter: GroupExporter::new(vec![]).into(), + } + } + + /// Wrap driver with a given exporter. + pub fn with_exporter(mut self, exporter: impl Into>) -> Self { + self.set_exporter(exporter); + self + } + + /// set an exporter. + pub fn set_exporter(&mut self, exporter: impl Into>) { + self.exporter = exporter.into(); + } +} + +impl WorldExporter for CompileExporter { + /// Export a typst document using `typst_ts_core::DocumentExporter`. + fn export(&mut self, output: Arc) -> SourceResult<()> { + self.exporter.export(self.compiler.world(), output) + } +} + +impl WrappedCompiler for CompileExporter { + type Compiler = C; + + fn inner(&self) -> &Self::Compiler { + &self.compiler + } + + fn inner_mut(&mut self) -> &mut Self::Compiler { + &mut self.compiler + } + + fn wrap_compile(&mut self) -> SourceResult> { + let doc = self.inner_mut().compile()?; + self.export(doc.clone())?; + + Ok(doc) + } +} + +pub type LayoutWidths = Vec; + +pub struct DynamicLayoutCompiler { + pub compiler: C, + + pub enable_dynamic_layout: bool, + + // todo: abstract this + output: PathBuf, + pub extension: String, + + pub layout_widths: LayoutWidths, + + /// Specify the target. It's default value is `web`. + /// You can specify a sub target like `web-dark` to refine the target. + /// Though we even don't encourage you to do so. + /// + /// Before typst allowing passing arguments to the compiler, this is + /// (probably) the only way to control the typst code's behavior. + pub target: String, +} + +impl DynamicLayoutCompiler { + pub fn new(compiler: C, output: PathBuf) -> Self { + Self { + compiler, + output, + enable_dynamic_layout: false, + extension: "multi.sir.in".to_owned(), + layout_widths: LayoutWidths::from_iter( + (0..40) + .map(|i| typst::geom::Abs::pt(750.0) - typst::geom::Abs::pt(i as f64 * 10.0)), + ), + target: "web".to_owned(), + } + } + + pub fn set_output(&mut self, output: PathBuf) { + self.output = output; + } + + pub fn set_extension(&mut self, extension: String) { + self.extension = extension; + } + + pub fn set_layout_widths(&mut self, layout_widths: LayoutWidths) { + self.layout_widths = layout_widths; + } + + pub fn set_target(&mut self, target: String) { + self.target = target; + } + + pub fn with_enable(mut self, enable_dynamic_layout: bool) -> Self { + self.enable_dynamic_layout = enable_dynamic_layout; + self + } +} + +#[cfg(feature = "dynamic-layout")] +impl WorldExporter for DynamicLayoutCompiler { + /// Export a typst document using `typst_ts_core::DocumentExporter`. + fn export(&mut self, _output: Arc) -> SourceResult<()> { + use std::str::FromStr; + + use typst::{ + diag::At, + syntax::{PackageSpec, Span, VirtualPath}, + }; + + use typst_ts_core::TypstFileId; + use typst_ts_svg_exporter::{flat_ir::serialize_doc, DynamicLayoutSvgExporter}; + + let variable_file = TypstFileId::new( + Some(PackageSpec::from_str("@preview/typst-ts-variables:0.1.0").at(Span::detached())?), + VirtualPath::new("lib.typ"), + ); + + // self.export(doc.clone())?; + // checkout the entry file + + let mut svg_exporter = DynamicLayoutSvgExporter::default(); + + // for each 10pt we rerender once + let instant_begin = instant::Instant::now(); + for (i, current_width) in self.layout_widths.clone().into_iter().enumerate() { + let instant = instant::Instant::now(); + // replace layout + + let variables: String = format!( + r##" +#let page-width = {:2}pt +#let target = "{}""##, + current_width.to_pt(), + self.target, + ); + + log::trace!( + "rerendering {} at {:?}, width={current_width:?} target={}", + i, + instant - instant_begin, + self.target, + ); + + self.with_shadow_file_by_id(variable_file, variables.as_bytes().into(), |this| { + // compile and export document + let output = this.inner_mut().compile()?; + svg_exporter.render(current_width, output); + log::trace!( + "rerendered {} at {:?}, {}", + i, + instant - instant_begin, + svg_exporter.debug_stat() + ); + Ok(()) + })?; + } + + let module_output = self.output.with_extension(&self.extension); + + let doc = svg_exporter.finalize(); + + std::fs::write(module_output, serialize_doc(doc)).unwrap(); + + let instant = instant::Instant::now(); + log::trace!("multiple layouts finished at {:?}", instant - instant_begin); + + Ok(()) + } +} + +#[cfg(feature = "dynamic-layout")] +impl WrappedCompiler for DynamicLayoutCompiler { + type Compiler = C; + + fn inner(&self) -> &Self::Compiler { + &self.compiler + } + + fn inner_mut(&mut self) -> &mut Self::Compiler { + &mut self.compiler + } + + fn wrap_compile(&mut self) -> SourceResult> { + if !self.enable_dynamic_layout { + return self.inner_mut().compile(); + } + + let pure_doc = self.inner_mut().compile()?; + self.export(pure_doc.clone())?; + + Ok(pure_doc) + } +} diff --git a/compiler/src/service/mod.rs b/compiler/src/service/mod.rs index 8aa547b4..fb050a11 100644 --- a/compiler/src/service/mod.rs +++ b/compiler/src/service/mod.rs @@ -3,7 +3,7 @@ use std::{ sync::Arc, }; -use crate::ShadowApi; +use crate::{vfs::notify::FilesystemEvent, ShadowApi}; use typst::{ diag::{At, FileResult, SourceDiagnostic, SourceResult}, doc::Document, @@ -12,14 +12,27 @@ use typst::{ syntax::Span, World, }; -use typst_ts_core::{Bytes, TypstFileId}; +use typst_ts_core::{Bytes, ImmutPath, TypstFileId}; +// todo: remove cfg feature here #[cfg(feature = "system-compile")] pub(crate) mod diag; +#[cfg(feature = "system-watch")] +pub(crate) mod watch; +#[cfg(feature = "system-watch")] +pub use watch::*; + pub(crate) mod driver; pub use driver::*; +#[cfg(feature = "system-watch")] +pub(crate) mod compile; +#[cfg(feature = "system-watch")] +pub use compile::*; + +pub(crate) mod export; +pub use export::*; pub mod query; #[cfg(feature = "system-compile")] @@ -27,11 +40,6 @@ pub(crate) mod session; #[cfg(feature = "system-compile")] pub use session::*; -#[cfg(feature = "system-watch")] -pub(crate) mod watch; -#[cfg(feature = "system-watch")] -pub use watch::*; - #[cfg(feature = "system-compile")] pub type CompileDriver = CompileDriverImpl; @@ -58,12 +66,12 @@ pub trait Compiler { fn reset(&mut self) -> SourceResult<()>; /// Compile once from scratch. - fn pure_compile(&mut self) -> SourceResult { + fn pure_compile(&mut self) -> SourceResult> { self.reset()?; let mut tracer = Tracer::default(); // compile and export document - typst::compile(self.world(), &mut tracer) + typst::compile(self.world(), &mut tracer).map(Arc::new) } /// With **the compilation state**, query the matches for the selector. @@ -72,7 +80,7 @@ pub trait Compiler { } /// Compile once from scratch. - fn compile(&mut self) -> SourceResult { + fn compile(&mut self) -> SourceResult> { self.pure_compile() } @@ -81,6 +89,12 @@ pub trait Compiler { self.pure_query(selector, document) } + /// Iterate over the dependencies of found by the compiler. + /// Note: reset the compiler will clear the dependencies cache. + fn iter_dependencies<'a>(&'a self, _f: &mut dyn FnMut(&'a ImmutPath, instant::SystemTime)) {} + + fn notify_fs_event(&mut self, _event: FilesystemEvent) {} + /// Determine whether the event is relevant to the compiler. /// The default implementation is conservative, which means that /// `MaybeRelevant` implies `MustRelevant`. @@ -167,7 +181,7 @@ pub trait WrappedCompiler { } /// Hooked compile once from scratch. - fn wrap_compile(&mut self) -> SourceResult { + fn wrap_compile(&mut self) -> SourceResult> { self.inner_mut().compile() } @@ -205,7 +219,7 @@ impl Compiler for T { } #[inline] - fn pure_compile(&mut self) -> SourceResult { + fn pure_compile(&mut self) -> SourceResult> { self.inner_mut().pure_compile() } @@ -215,7 +229,7 @@ impl Compiler for T { } #[inline] - fn compile(&mut self) -> SourceResult { + fn compile(&mut self) -> SourceResult> { self.wrap_compile() } @@ -223,6 +237,16 @@ impl Compiler for T { fn query(&mut self, selector: String, document: &Document) -> SourceResult> { self.wrap_query(selector, document) } + + #[inline] + fn iter_dependencies<'a>(&'a self, f: &mut dyn FnMut(&'a ImmutPath, instant::SystemTime)) { + self.inner().iter_dependencies(f) + } + + #[inline] + fn notify_fs_event(&mut self, event: crate::vfs::notify::FilesystemEvent) { + self.inner_mut().notify_fs_event(event) + } } impl ShadowApi for T @@ -234,6 +258,11 @@ where self.inner()._shadow_map_id(_file_id) } + #[inline] + fn shadow_paths(&self) -> Vec> { + self.inner().shadow_paths() + } + #[inline] fn reset_shadow(&mut self) { self.inner_mut().reset_shadow() @@ -252,7 +281,7 @@ where /// The status in which the watcher can be. pub enum DiagStatus { - Compiling, + Stage(&'static str), Success(std::time::Duration), Error(std::time::Duration), } @@ -268,11 +297,20 @@ pub trait DiagObserver { fn print_status(&self, status: DiagStatus); /// Run inner function with print (optional) status and diagnostics to the - /// terminal. + /// terminal (for compilation). + #[deprecated(note = "use `with_stage_diag` instead")] fn with_compile_diag( &mut self, f: impl FnOnce(&mut Self) -> SourceResult, ) -> Option; + + /// Run inner function with print (optional) status and diagnostics to the + /// terminal. + fn with_stage_diag( + &mut self, + stage: &'static str, + f: impl FnOnce(&mut Self) -> SourceResult, + ) -> Option; } #[cfg(feature = "system-compile")] @@ -297,12 +335,22 @@ where } /// Run inner function with print (optional) status and diagnostics to the - /// terminal. + /// terminal (for compilation). fn with_compile_diag( &mut self, f: impl FnOnce(&mut Self) -> SourceResult, ) -> Option { - self.print_status::(DiagStatus::Compiling); + self.with_stage_diag::("compiling", f) + } + + /// Run inner function with print (optional) status and diagnostics to the + /// terminal (for stages). + fn with_stage_diag( + &mut self, + stage: &'static str, + f: impl FnOnce(&mut Self) -> SourceResult, + ) -> Option { + self.print_status::(DiagStatus::Stage(stage)); let start = instant::Instant::now(); match f(self) { Ok(val) => { diff --git a/compiler/src/service/watch.rs b/compiler/src/service/watch.rs index 75eb2ffc..654a0299 100644 --- a/compiler/src/service/watch.rs +++ b/compiler/src/service/watch.rs @@ -1,47 +1,549 @@ -use std::path::Path; +//! upstream +//! +//! An implementation of `watch_deps` using `notify` crate. +//! +//! The file watching bits here are untested and quite probably buggy. For this +//! reason, by default we don't watch files and rely on editor's file watching +//! capabilities. +//! +//! Hopefully, one day a reliable file watching/walking crate appears on +//! crates.io, and we can reduce this to trivial glue code. -use log::{error, info}; -use notify::{Event, RecommendedWatcher, RecursiveMode, Watcher}; -use typst::eval::eco_format; +use std::{collections::HashMap, fs}; -pub async fn watch_dir( - workspace_dir: &Path, - mut interrupted_by_events: impl FnMut(Option>), -) -> ! { - // Setup file watching. - let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - let mut watcher = RecommendedWatcher::new( - move |res: Result| match res { - Ok(e) => { - tx.send(e).unwrap(); +use notify::{Config, RecommendedWatcher, RecursiveMode, Watcher}; +use tokio::sync::mpsc; +use typst::diag::{FileError, FileResult}; + +use typst_ts_core::{Bytes, ImmutPath}; + +use crate::vfs::{ + notify::{FileChangeSet, FilesystemEvent, NotifyFile, NotifyMessage, UpstreamUpdateEvent}, + system::SystemAccessModel, + AccessModel, +}; + +type WatcherPair = (RecommendedWatcher, mpsc::UnboundedReceiver); +type NotifyEvent = notify::Result; +type FileEntry = (/* key */ ImmutPath, /* value */ NotifyFile); +type NotifyFilePair = FileResult<( + /* mtime */ instant::SystemTime, + /* content */ Bytes, +)>; + +/// The state of a watched file. +/// +/// It is used to determine some dirty editiors' implementation. +#[derive(Debug)] +enum WatchState { + /// The file is stable, which means we believe that it keeps syncthronized + /// as expected. + Stable, + /// The file is empty or removed, but there is a chance that the file is not + /// stable. So we need to recheck the file after a while. + EmptyOrRemoval { + recheck_at: usize, + payload: NotifyFilePair, + }, +} + +/// By default, the state is stable. +impl Default for WatchState { + fn default() -> Self { + Self::Stable + } +} + +/// The data entry of a watched file. +struct WatchedEntry { + /// The lifetime of the entry. + /// + /// The entry will be removed if the entry is too old. + // todo: generalize lifetime + lifetime: usize, + + /// The state of the entry. + state: WatchState, + + /// Previous content of the file. + prev: Option, + + prev_meta: Option, +} + +/// Self produced event that check whether the file is stable after a while. +#[derive(Debug)] +struct UndeterminedNotifyEvent { + /// The time when the event is produced. + at_realtime: instant::Instant, + /// The logical tick when the event is produced. + at_logical_tick: usize, + /// The path of the file. + path: ImmutPath, +} + +// Drop order is significant. +/// The actor that watches files. +/// It is used to watch files and send events to the consumers +pub struct NotifyActor { + /// The access model of the actor. + /// We concrete the access model to `SystemAccessModel` for now. + inner: SystemAccessModel, + + /// The lifetime tick of the actor. + lifetime: usize, + + /// The logical tick of the actor. + logical_tick: usize, + + /// Whether the actor is using builtin watcher. + /// + /// If it is [`None`], the actor need a upstream event to trigger the + /// updates + watch: Option<()>, + + /// Output of the actor. + /// See [`FilesystemEvent`] for more information. + sender: mpsc::UnboundedSender, + + /// Internal channel for recheck events. + undetermined_send: mpsc::UnboundedSender, + undetermined_recv: mpsc::UnboundedReceiver, + + /// The holded entries for watching, one entry for per file. + watched_entries: HashMap, + + /// The builtin watcher object. + watcher: Option, +} + +impl NotifyActor { + /// Create a new actor. + fn new(sender: mpsc::UnboundedSender) -> NotifyActor { + let (undetermined_send, undetermined_recv) = mpsc::unbounded_channel(); + + NotifyActor { + inner: SystemAccessModel, + // we start from 1 to distinguish from 0 (default value) + lifetime: 1, + logical_tick: 1, + + watch: Some(()), + sender, + + undetermined_send, + undetermined_recv, + + watched_entries: HashMap::new(), + watcher: None, + } + } + + /// Send a filesystem event to remove. + fn send(&mut self, msg: FilesystemEvent) { + self.sender.send(msg).unwrap(); + } + + /// Get the notify event from the watcher. + async fn get_notify_event(watcher: &mut Option) -> Option { + match watcher { + Some((_, watcher_receiver)) => watcher_receiver.recv().await, + None => None, + } + } + + /// Main loop of the actor. + async fn run(mut self, mut inbox: mpsc::UnboundedReceiver) { + /// The event of the actor. + #[derive(Debug)] + enum ActorEvent { + /// Recheck the notify event. + ReCheck(UndeterminedNotifyEvent), + /// external message to change notifer's state + Message(NotifyMessage), + /// notify event from builtin watcher + NotifyEvent(NotifyEvent), + } + + loop { + // Get the event from the inbox or the watcher. + let event = tokio::select! { + Some(it) = inbox.recv() => Some(ActorEvent::Message(it)), + Some(it) = Self::get_notify_event(&mut self.watcher) => Some(ActorEvent::NotifyEvent(it)), + Some(it) = self.undetermined_recv.recv() => Some(ActorEvent::ReCheck(it)), + }; + + // Failed to get the event. + let Some(event) = event else { + log::info!("failed to get event, exiting..."); + return; + }; + + // Increase the logical tick per event. + self.logical_tick += 1; + + // log::info!("vfs-notify event {event:?}"); + // function entries to handle some event + match event { + ActorEvent::Message(NotifyMessage::UpstreamUpdate(event)) => { + self.invalidate_upstream(event); + } + ActorEvent::Message(NotifyMessage::SyncDependency(paths)) => { + if let Some(changeset) = self.update_watches(&paths) { + self.send(FilesystemEvent::Update(changeset)); + } + } + ActorEvent::NotifyEvent(event) => { + // log::info!("notify event {event:?}"); + if let Some(event) = log_notify_error(event, "failed to notify") { + self.notify_event(event); + } + } + ActorEvent::ReCheck(event) => { + self.recheck_notify_event(event).await; + } } - Err(e) => error!("watch error: {:#}", e), - }, - notify::Config::default(), - ) - .map_err(|err| eco_format!("failed to watch directory ({err})")) - .unwrap(); - - // Add a path to be watched. All files and directories at that path and - // below will be monitored for changes. - watcher - .watch(workspace_dir, RecursiveMode::Recursive) - .unwrap(); + } + } - // Handle events. - info!("start watching files..."); - interrupted_by_events(None); - loop { - let mut events = vec![]; - while let Ok(e) = - tokio::time::timeout(tokio::time::Duration::from_millis(100), rx.recv()).await - { - if e.is_none() { + /// Update the watches of corresponding invalidation + fn invalidate_upstream(&mut self, event: UpstreamUpdateEvent) { + // Update watches of invalidated files. + let changeset = self.update_watches(&event.invalidates).unwrap_or_default(); + + // Send the event to the consumer. + self.send(FilesystemEvent::UpstreamUpdate { + changeset, + upstream_event: Some(event), + }); + } + + /// Update the watches of corresponding files. + fn update_watches(&mut self, paths: &[ImmutPath]) -> Option { + // Increase the lifetime per external message. + self.lifetime += 1; + + let mut changeset = FileChangeSet::default(); + + // Remove old watches, if any. + self.watcher = None; + if self.watch.is_some() { + match &mut self.watcher { + // Clear the old watches. + Some((old_watcher, _)) => { + for path in self.watched_entries.keys() { + // Remove the watch if it still exists. + if let Err(err) = old_watcher.unwatch(path) { + if !matches!(err.kind, notify::ErrorKind::WatchNotFound) { + log::warn!("failed to unwatch: {err}"); + } + } + } + } + // Create a new builtin watcher. + None => { + let (watcher_sender, watcher_receiver) = mpsc::unbounded_channel(); + let watcher = log_notify_error( + RecommendedWatcher::new( + move |event| { + let res = watcher_sender.send(event); + if let Err(err) = res { + log::warn!("error to send event: {err}"); + } + }, + Config::default(), + ), + "failed to create watcher", + ); + self.watcher = watcher.map(|it| (it, watcher_receiver)); + } + } + } + + // Update watched entries. + // + // Also check whether the file is updated since there is a window + // between unwatch the file and watch the file again. + for path in paths.iter() { + // Update or insert the entry with the new lifetime. + let entry = self + .watched_entries + .entry(path.clone()) + .and_modify(|watch_entry| { + watch_entry.lifetime = self.lifetime; + }) + .or_insert_with(|| WatchedEntry { + lifetime: self.lifetime, + state: WatchState::Stable, + prev: None, + prev_meta: None, + }); + + // Update in-memory metadata for now. + let Some(meta) = path.metadata().ok().or(entry.prev_meta.clone()) else { + // We cannot get the metadata even at the first time, so we are + // okay to ignore this file for watching. continue; + }; + entry.prev_meta = Some(meta.clone()); + + let watch = self.watch.is_some(); + if watch { + // Watch the file again if it's not a directory. + if !meta.is_dir() { + if let Some((watcher, _)) = &mut self.watcher { + log_notify_error( + watcher.watch(path.as_ref(), RecursiveMode::NonRecursive), + "failed to watch", + ); + + changeset.may_insert(self.notify_entry_update(path.clone(), Some(meta))); + } else { + unreachable!() + } + } + } else { + let watched = self + .inner + .content(path) + .map(|e| (meta.modified().unwrap(), e)); + changeset.inserts.push((path.clone(), watched.into())); + } + } + + // Remove old entries. + // Note: since we have increased the lifetime, it is safe to remove the + // old entries after updating the watched entries. + self.watched_entries.retain(|path, entry| { + if self.lifetime - entry.lifetime < 30 { + true + } else { + changeset.removes.push(path.clone()); + false + } + }); + + (!changeset.is_empty()).then_some(changeset) + } + + /// Notify the event from the builtin watcher. + fn notify_event(&mut self, event: notify::Event) { + // Account file updates. + let mut changeset = FileChangeSet::default(); + for path in event.paths.into_iter() { + // todo: remove this clone: path.into() + changeset.may_insert(self.notify_entry_update(path.into(), None)); + } + + // Send file updates. + if !changeset.is_empty() { + self.send(FilesystemEvent::Update(changeset)); + } + } + + /// Notify any update of the file entry + fn notify_entry_update( + &mut self, + path: ImmutPath, + meta: Option, + ) -> Option { + let meta = meta.or_else(|| fs::metadata(&path).ok())?; + + // The following code in rust-analyzer is commented out + // todo: check whether we need this + // if meta.file_type().is_dir() && self + // .watched_entriesiter().any(|entry| entry.contains_dir(&path)) + // { + // self.watch(path); + // return None; + // } + + if !meta.file_type().is_file() { + return None; + } + + // Check meta, path, and content + + // Get meta, real path and ignore errors + let mtime = meta.modified().ok()?; + + // Find entry and continue + let entry = self.watched_entries.get_mut(&path)?; + + let mut file = self.inner.content(&path).map(|it| (mtime, it)); + + // Check state in fast path: compare state, return None on not sending + // the file change + match (&entry.prev, &mut file) { + // update the content of the entry in the following cases: + // + Case 1: previous content is clear + // + Case 2: previous content is not clear but some error, and the + // current content is ok + (None, ..) | (Some(Err(..)), Ok(..)) => {} + // Meet some error currently + (Some(..), Err(err)) => match &mut entry.state { + // If the file is stable, check whether the editor is removing + // or truncating the file. They are possibly flushing the file + // but not finished yet. + WatchState::Stable => { + if matches!(err, FileError::NotFound(..) | FileError::Other(..)) { + entry.state = WatchState::EmptyOrRemoval { + recheck_at: self.logical_tick, + payload: file.clone(), + }; + entry.prev = Some(file); + self.undetermined_send + .send(UndeterminedNotifyEvent { + at_realtime: instant::Instant::now(), + at_logical_tick: self.logical_tick, + path: path.clone(), + }) + .unwrap(); + return None; + } + // Otherwise, we push the error to the consumer. + } + + // Very complicated case of check error sequence, so we simplify + // a bit, we regard any subsequent error as the same error. + WatchState::EmptyOrRemoval { payload, .. } => { + // update payload + *payload = file; + return None; + } + }, + // Compare content for transitinal the state + (Some(Ok((prev_tick, prev_content))), Ok((next_tick, next_content))) => { + // So far it is acurately no change for the file, skip it + if prev_content == next_content { + return None; + } + + match entry.state { + // If the file is stable, check whether the editor is + // removing or truncating the file. They are possibly + // flushing the file but not finished yet. + WatchState::Stable => { + if next_content.is_empty() { + entry.state = WatchState::EmptyOrRemoval { + recheck_at: self.logical_tick, + payload: file.clone(), + }; + entry.prev = Some(file); + self.undetermined_send + .send(UndeterminedNotifyEvent { + at_realtime: instant::Instant::now(), + at_logical_tick: self.logical_tick, + path, + }) + .unwrap(); + return None; + } + } + + // Still empty + WatchState::EmptyOrRemoval { .. } if next_content.is_empty() => return None, + // Otherwise, we push the diff to the consumer. + WatchState::EmptyOrRemoval { .. } => {} + } + + // We have found a change, however, we need to check whether the + // mtime is changed. Generally, the mtime should be changed. + // However, It is common that editor (VSCode) to change the + // mtime after writing + // + // this condition should be never happen, but we still check it + // + // There will be cases that user change content of a file and + // then also modify the mtime of the file, so we need to check + // `next_tick == prev_tick`: Whether mtime is changed. + // `matches!(entry.state, WatchState::Fresh)`: Whether the file + // is fresh. We have not submit the file to the compiler, so + // that is ok. + if next_tick == prev_tick && matches!(entry.state, WatchState::Stable) { + // this is necessary to invalidate our mtime-based cache + *next_tick = prev_tick + .checked_add(std::time::Duration::from_micros(1)) + .unwrap(); + log::warn!("same content but mtime is different...: {:?} content: prev:{:?} v.s. curr:{:?}", path, prev_content, next_content); + }; } - events.push(e.unwrap()); + }; + + // Send the update to the consumer + // Update the entry according to the state + entry.state = WatchState::Stable; + entry.prev = Some(file.clone()); + + // Slow path: trigger the file change for consumer + Some((path, file.into())) + } + + /// Recheck the notify event after a while. + async fn recheck_notify_event(&mut self, event: UndeterminedNotifyEvent) -> Option<()> { + let now = instant::Instant::now(); + log::debug!("recheck event {event:?} at {now:?}"); + + // The aysnc scheduler is not accurate, so we need to ensure a window here + let reserved = now - event.at_realtime; + if reserved < std::time::Duration::from_millis(50) { + let send = self.undetermined_send.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(50) - reserved).await; + send.send(event).unwrap(); + }); + return None; } - interrupted_by_events(Some(events)); + // Check whether the entry is still valid + let entry = self.watched_entries.get_mut(&event.path)?; + + // Check the state of the entry + match std::mem::take(&mut entry.state) { + // If the entry is stable, we do nothing + WatchState::Stable => {} + // If the entry is not stable, and no other event is produced after + // this event, we send the event to the consumer. + WatchState::EmptyOrRemoval { + recheck_at, + payload, + } => { + if recheck_at == event.at_logical_tick { + log::debug!("notify event real happened {event:?}, state: {:?}", payload); + + // Send the underlying change to the consumer + let mut changeset = FileChangeSet::default(); + changeset.inserts.push((event.path, payload.into())); + self.send(FilesystemEvent::Update(changeset)); + } + } + }; + + Some(()) + } +} + +#[inline] +fn log_notify_error(res: notify::Result, reason: &'static str) -> Option { + res.map_err(|err| log::warn!("{reason}: notify error: {}", err)) + .ok() +} + +pub async fn watch_deps( + inbox: mpsc::UnboundedReceiver, + mut interrupted_by_events: impl FnMut(Option), +) { + // Setup file watching. + let (tx, mut rx) = mpsc::unbounded_channel(); + let actor = NotifyActor::new(tx); + + // Watch messages to notify + tokio::spawn(actor.run(inbox)); + + // Handle events. + log::debug!("start watching files..."); + interrupted_by_events(None); + while let Some(event) = rx.recv().await { + interrupted_by_events(Some(event)); } } diff --git a/compiler/src/vfs/cached.rs b/compiler/src/vfs/cached.rs index bd0baad1..44869df3 100644 --- a/compiler/src/vfs/cached.rs +++ b/compiler/src/vfs/cached.rs @@ -38,6 +38,10 @@ impl CachedAccessModel { pub fn inner(&self) -> &Inner { &self.inner } + + pub fn inner_mut(&mut self) -> &mut Inner { + &mut self.inner + } } impl CachedAccessModel { @@ -146,8 +150,8 @@ impl AccessModel for CachedAccessModel { fn content(&self, src: &Path) -> FileResult { self.cache_entry(src, |entry| { - let data = entry.read_all.compute(|| self.inner.content(src))?; - Ok(data.clone()) + let data = entry.read_all.compute(|| self.inner.content(src)); + Ok(data?.clone()) }) } } diff --git a/compiler/src/vfs/mod.rs b/compiler/src/vfs/mod.rs index 0b600645..a52c0233 100644 --- a/compiler/src/vfs/mod.rs +++ b/compiler/src/vfs/mod.rs @@ -12,6 +12,7 @@ pub mod system; pub mod cached; pub mod dummy; +pub mod notify; pub mod overlay; pub mod trace; @@ -31,13 +32,7 @@ pub(crate) mod writable; pub use writable::Vfs as MemVfs; pub use writable::{ChangeKind, ChangedFile}; -use std::{ - collections::HashMap, - ffi::OsStr, - hash::Hash, - path::{Path, PathBuf}, - sync::Arc, -}; +use std::{collections::HashMap, ffi::OsStr, hash::Hash, path::Path, sync::Arc}; use append_only_vec::AppendOnlyVec; use parking_lot::{Mutex, RwLock, RwLockUpgradableReadGuard}; @@ -46,11 +41,15 @@ use typst::{ syntax::Source, }; -use typst_ts_core::{path::PathClean, Bytes, QueryRef, TypstFileId}; +use typst_ts_core::{path::PathClean, Bytes, ImmutPath, QueryRef, TypstFileId}; use crate::{parser::reparse, time::SystemTime}; -use self::{cached::CachedAccessModel, overlay::OverlayAccessModel}; +use self::{ + cached::CachedAccessModel, + notify::{FilesystemEvent, NotifyAccessModel}, + overlay::OverlayAccessModel, +}; /// Handle to a file in [`Vfs`] /// @@ -80,7 +79,7 @@ type FileQuery = QueryRef; /// Holds canonical data for all paths pointing to the same entity. pub struct PathSlot { idx: FileId, - sampled_path: once_cell::sync::OnceCell, + sampled_path: once_cell::sync::OnceCell, mtime: FileQuery, source: FileQuery, buffer: FileQuery, @@ -98,10 +97,13 @@ impl PathSlot { } } +type VfsAccessModel = CachedAccessModel>, Source>; + pub struct Vfs { lifetime_cnt: u64, - // access_model: TraceAccessModel>, - access_model: CachedAccessModel, Source>, + // access_model: TraceAccessModel>, + // we add notify access model here since notify access model doesn't introduce overheads + access_model: VfsAccessModel, path_interner: Mutex::RealPath, u64>>, path2slot: RwLock, FileId>>, @@ -112,6 +114,7 @@ pub struct Vfs { impl Vfs { pub fn new(access_model: M) -> Self { + let access_model = NotifyAccessModel::new(access_model); let access_model = OverlayAccessModel::new(access_model); let access_model = CachedAccessModel::new(access_model); // let access_model = TraceAccessModel::new(access_model); @@ -144,6 +147,10 @@ impl Vfs { self.access_model.inner().clear_shadow(); } + pub fn shadow_paths(&self) -> Vec> { + self.access_model.inner().file_paths() + } + /// Set the `do_reparse` flag. pub fn set_do_reparse(&mut self, do_reparse: bool) { self.do_reparse = do_reparse; @@ -169,7 +176,7 @@ impl Vfs { } /// Get all the files in the VFS. - pub fn iter_dependencies(&self) -> impl Iterator { + pub fn iter_dependencies(&self) -> impl Iterator { self.slots.iter().map(|slot| { let dep_path = slot.sampled_path.get().unwrap(); let dep_mtime = slot @@ -177,10 +184,26 @@ impl Vfs { .compute(|| Err(other_reason("vfs: uninitialized"))) .unwrap(); - (dep_path.as_path(), *dep_mtime) + (dep_path, *dep_mtime) }) } + /// Get all the files in the VFS. + pub fn iter_dependencies_dyn<'a>( + &'a self, + f: &mut dyn FnMut(&'a ImmutPath, instant::SystemTime), + ) { + for slot in self.slots.iter() { + let dep_path = slot.sampled_path.get().unwrap(); + let dep_mtime = slot + .mtime + .compute(|| Err(other_reason("vfs: uninitialized"))) + .unwrap(); + + f(dep_path, *dep_mtime) + } + } + /// File path corresponding to the given `file_id`. /// /// # Panics @@ -219,7 +242,7 @@ impl Vfs { } let slot = &self.slots[idx]; - slot.sampled_path.get_or_init(|| origin_path.to_path_buf()); + slot.sampled_path.get_or_init(|| origin_path.into()); Ok(&self.slots[idx]) } @@ -325,6 +348,10 @@ impl Vfs { self.access_model.inner().remove_file(path); } + pub fn notify_fs_event(&mut self, event: FilesystemEvent) { + self.access_model.inner_mut().inner_mut().notify(event); + } + pub fn file(&self, path: &Path) -> FileResult { let slot = self.slot(path)?; diff --git a/compiler/src/vfs/notify.rs b/compiler/src/vfs/notify.rs new file mode 100644 index 00000000..7a493f70 --- /dev/null +++ b/compiler/src/vfs/notify.rs @@ -0,0 +1,234 @@ +use std::{collections::HashMap, path::Path}; + +use typst::diag::FileResult; +use typst_ts_core::{Bytes, ImmutPath}; + +use crate::vfs::AccessModel; + +/// internal representation of [`NotifyFile`] +#[derive(Debug, Clone)] +struct NotifyFileRepr { + mtime: instant::SystemTime, + content: Bytes, +} + +/// A file snapshot that is notified by some external source +#[derive(Debug, Clone)] +pub struct NotifyFile(FileResult); + +impl NotifyFile { + /// mtime of the file + pub fn mtime(&self) -> FileResult<&instant::SystemTime> { + self.0.as_ref().map(|e| &e.mtime).map_err(|e| e.clone()) + } + + /// content of the file + pub fn content(&self) -> FileResult<&Bytes> { + self.0.as_ref().map(|e| &e.content).map_err(|e| e.clone()) + } + + /// Whether the related file is a file + pub fn is_file(&self) -> FileResult { + self.0.as_ref().map(|_| true).map_err(|e| e.clone()) + } +} + +/// Convenent function to create a [`NotifyFile`] from tuple +impl From> for NotifyFile { + fn from(result: FileResult<(instant::SystemTime, Bytes)>) -> Self { + Self(result.map(|(mtime, content)| NotifyFileRepr { mtime, content })) + } +} + +/// A set of changes to the filesystem. +/// +/// The correct order of applying changes is: +/// 1. Remove files +/// 2. Upsert (Insert or Update) files +#[derive(Debug, Clone, Default)] +pub struct FileChangeSet { + /// Files to remove + pub removes: Vec, + /// Files to insert or update + pub inserts: Vec<(ImmutPath, NotifyFile)>, +} + +impl FileChangeSet { + /// Create a new empty changeset + pub fn is_empty(&self) -> bool { + self.inserts.is_empty() && self.removes.is_empty() + } + + /// Create a new changeset with removing files + pub fn new_removes(removes: Vec) -> Self { + Self { + removes, + inserts: vec![], + } + } + + /// Create a new changeset with inserting files + pub fn new_inserts(inserts: Vec<(ImmutPath, NotifyFile)>) -> Self { + Self { + removes: vec![], + inserts, + } + } + + /// Utility function to insert a possible file to insert or update + pub fn may_insert(&mut self, v: Option<(ImmutPath, NotifyFile)>) { + if let Some(v) = v { + self.inserts.push(v); + } + } + + /// Utility function to insert multiple possible files to insert or update + pub fn may_extend(&mut self, v: Option>) { + if let Some(v) = v { + self.inserts.extend(v); + } + } +} + +/// A memory event that is notified by some external source +#[derive(Debug)] +pub enum MemoryEvent { + /// Reset all dependencies and update according to the given changeset + /// + /// We have not provided a way to reset all dependencies without updating + /// yet, but you can create a memory event with empty changeset to achieve + /// this: + /// + /// ``` + /// use typst_ts_compiler::vfs::notify::{MemoryEvent, FileChangeSet}; + /// let event = MemoryEvent::Sync(FileChangeSet::default()); + /// ``` + Sync(FileChangeSet), + /// Update according to the given changeset + Update(FileChangeSet), +} + +/// A upstream update event that is notified by some external source. +/// +/// This event is used to notify some file watcher to invalidate some files +/// before applying upstream changes. This is very important to make some atomic +/// changes. +#[derive(Debug)] +pub struct UpstreamUpdateEvent { + /// Associated files that the event causes to invalidate + pub invalidates: Vec, + /// Opaque data that is passed to the file watcher + pub opaque: Box, +} + +/// Aggregated filesystem events from some file watcher +#[derive(Debug)] +pub enum FilesystemEvent { + /// Update file system files according to the given changeset + Update(FileChangeSet), + /// See [`UpstreamUpdateEvent`] + UpstreamUpdate { + /// New changeset produced by invalidation + changeset: FileChangeSet, + upstream_event: Option, + }, +} + +/// A message that is sent to some file watcher +#[derive(Debug)] +pub enum NotifyMessage { + /// override all dependencies + SyncDependency(Vec), + /// upstream invalidation This is very important to make some atomic changes + /// + /// Example: + /// ```plain + /// /// Receive memory event + /// let event: MemoryEvent = retrieve(); + /// let invalidates = event.invalidates(); + /// + /// /// Send memory change event to [`NotifyActor`] + /// let event = Box::new(event); + /// self.send(NotifyMessage::UpstreamUpdate{ invalidates, opaque: event }); + /// + /// /// Wait for [`NotifyActor`] to finish + /// let fs_event = self.fs_notify.block_receive(); + /// let event: MemoryEvent = fs_event.opaque.downcast().unwrap(); + /// + /// /// Apply changes + /// self.lock(); + /// update_memory(event); + /// apply_fs_changes(fs_event.changeset); + /// self.unlock(); + /// ``` + UpstreamUpdate(UpstreamUpdateEvent), +} + +/// Notify shadowing access model, which the typical underlying access model is +/// [`crate::vfs::system::SystemAccessModel`] +pub struct NotifyAccessModel { + files: HashMap, + pub inner: M, +} + +impl NotifyAccessModel { + /// Create a new notify access model + pub fn new(inner: M) -> Self { + Self { + files: HashMap::new(), + inner, + } + } + + /// Notify the access model with a filesystem event + pub fn notify(&mut self, event: FilesystemEvent) { + match event { + FilesystemEvent::UpstreamUpdate { changeset, .. } + | FilesystemEvent::Update(changeset) => { + for path in changeset.removes { + self.files.remove(&path); + } + + for (path, contents) in changeset.inserts { + self.files.insert(path, contents); + } + } + } + } +} + +impl AccessModel for NotifyAccessModel { + type RealPath = M::RealPath; + + fn mtime(&self, src: &Path) -> FileResult { + if let Some(entry) = self.files.get(src) { + return entry.mtime().cloned(); + } + + self.inner.mtime(src) + } + + fn is_file(&self, src: &Path) -> FileResult { + if let Some(entry) = self.files.get(src) { + return entry.is_file(); + } + + self.inner.is_file(src) + } + + fn real_path(&self, src: &Path) -> FileResult { + if self.files.get(src).is_some() { + return Ok(src.into()); + } + + self.inner.real_path(src) + } + + fn content(&self, src: &Path) -> FileResult { + if let Some(entry) = self.files.get(src) { + return entry.content().cloned(); + } + + self.inner.content(src) + } +} diff --git a/compiler/src/vfs/overlay.rs b/compiler/src/vfs/overlay.rs index e4fc753e..032a598f 100644 --- a/compiler/src/vfs/overlay.rs +++ b/compiler/src/vfs/overlay.rs @@ -30,10 +30,22 @@ impl OverlayAccessModel { } } + pub fn inner(&self) -> &M { + &self.model + } + + pub fn inner_mut(&mut self) -> &mut M { + &mut self.model + } + pub fn clear_shadow(&self) { self.files.write().clear(); } + pub fn file_paths(&self) -> Vec> { + self.files.read().keys().cloned().collect() + } + pub fn add_file(&self, path: Arc, content: Bytes) { // we change mt every time, since content almost changes every time // Note: we can still benefit from cache, since we incrementally parse source diff --git a/compiler/src/vfs/trace.rs b/compiler/src/vfs/trace.rs index e37fe232..b0d39bd0 100644 --- a/compiler/src/vfs/trace.rs +++ b/compiler/src/vfs/trace.rs @@ -21,6 +21,14 @@ impl TraceAccessModel> } } + pub fn inner(&self) -> &M { + self.inner.inner() + } + + pub fn inner_mut(&mut self) -> &mut M { + self.inner.inner_mut() + } + pub fn read_all_diff( &self, src: &Path, diff --git a/compiler/src/workspace/dependency.rs b/compiler/src/workspace/dependency.rs index 0d0b176c..ee46cc87 100644 --- a/compiler/src/workspace/dependency.rs +++ b/compiler/src/workspace/dependency.rs @@ -6,6 +6,7 @@ use std::{ collections::{hash_map::DefaultHasher, HashMap}, ffi::OsStr, hash::Hash, + iter::Iterator, iter::Peekable, path::{Component, Path, PathBuf}, sync::Arc, diff --git a/compiler/src/world.rs b/compiler/src/world.rs index 83c88ed3..d54ce14d 100644 --- a/compiler/src/world.rs +++ b/compiler/src/world.rs @@ -17,7 +17,7 @@ use typst::{ use typst_ts_core::{ font::{FontProfile, FontResolverImpl}, - Bytes, FontResolver, TypstFileId as FileId, + Bytes, FontResolver, ImmutPath, TypstFileId as FileId, }; use crate::{ @@ -26,7 +26,7 @@ use crate::{ time::SystemTime, vfs::{AccessModel as VfsAccessModel, Vfs}, workspace::dependency::{DependencyTree, DependentFileInfo}, - ShadowApi, + NotifyApi, ShadowApi, }; type CodespanResult = Result; @@ -188,16 +188,14 @@ impl CompilerWorld { /// Get found dependencies in current state of vfs. pub fn get_dependencies(&self) -> DependencyTree { - let vfs_dependencies = - self.vfs - .iter_dependencies() - .map(|(path, mtime)| DependentFileInfo { - path: path.to_owned(), - mtime: mtime - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_micros() as u64, - }); + let t = self.vfs.iter_dependencies(); + let vfs_dependencies = t.map(|(path, mtime)| DependentFileInfo { + path: path.as_ref().to_owned(), + mtime: mtime + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_micros() as u64, + }); DependencyTree::from_iter(&self.root, vfs_dependencies) } @@ -223,18 +221,27 @@ impl CompilerWorld { } impl ShadowApi for CompilerWorld { + #[inline] fn _shadow_map_id(&self, file_id: FileId) -> FileResult { self.path_for_id(file_id) } + #[inline] + fn shadow_paths(&self) -> Vec> { + self.vfs.shadow_paths() + } + + #[inline] fn reset_shadow(&mut self) { self.vfs.reset_shadow() } + #[inline] fn map_shadow(&self, path: &Path, content: Bytes) -> FileResult<()> { self.vfs.map_shadow(path, content) } + #[inline] fn unmap_shadow(&self, path: &Path) -> FileResult<()> { self.vfs.remove_shadow(path); @@ -242,6 +249,18 @@ impl ShadowApi for CompilerWorld { } } +impl NotifyApi for CompilerWorld { + #[inline] + fn iter_dependencies<'a>(&'a self, f: &mut dyn FnMut(&'a ImmutPath, instant::SystemTime)) { + self.vfs.iter_dependencies_dyn(f) + } + + #[inline] + fn notify_fs_event(&mut self, event: crate::vfs::notify::FilesystemEvent) { + self.vfs.notify_fs_event(event) + } +} + impl WorkspaceProvider for CompilerWorld { fn reset(&mut self) -> SourceResult<()> { self.reset(); diff --git a/core/src/concepts/mod.rs b/core/src/concepts/mod.rs index 6c790b4f..5a314cfe 100644 --- a/core/src/concepts/mod.rs +++ b/core/src/concepts/mod.rs @@ -1,4 +1,6 @@ mod takable; +use std::{path::Path, sync::Arc}; + pub use takable::*; mod hash; @@ -16,3 +18,6 @@ pub use marker::*; /// Re-export of the typst crate. mod typst; pub use self::typst::*; + +pub type ImmutStr = Arc; +pub type ImmutPath = Arc; diff --git a/core/src/concepts/query.rs b/core/src/concepts/query.rs index 442befef..0eafb2b4 100644 --- a/core/src/concepts/query.rs +++ b/core/src/concepts/query.rs @@ -1,3 +1,4 @@ +use core::fmt; use std::{ cell::{RefCell, RefMut}, sync::Mutex, @@ -17,6 +18,12 @@ impl<'a, T> std::ops::Deref for QueryResult<'a, T> { } } +impl<'a, T: fmt::Debug> fmt::Debug for QueryResult<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("QueryResult").field(&self.0).finish() + } +} + /// Represent the result of an immutable query reference. /// The compute function should be pure enough. /// @@ -42,7 +49,8 @@ impl QueryRef { } impl QueryRef { - /// Clone the error so that it can escape the borrowed reference to the ref cell. + /// Clone the error so that it can escape the borrowed reference to the ref + /// cell. #[inline] fn clone_err(r: RefMut<'_, QueryCell>) -> E { let initialized_res = r.1.as_ref().unwrap(); @@ -50,7 +58,8 @@ impl QueryRef { checked_res.unwrap_err().clone() } - /// Get the reference to the query result, which asserts that the query result is initialized. + /// Get the reference to the query result, which asserts that the query + /// result is initialized. #[inline] fn get_ref(&self) -> Result<&T, E> { let holding = unsafe { (*self.cell.as_ptr()).1.as_ref().unwrap_unchecked() }; diff --git a/core/src/exporter.rs b/core/src/exporter.rs index fbabe8a6..11251583 100644 --- a/core/src/exporter.rs +++ b/core/src/exporter.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use typst::{diag::SourceResult, World}; -pub(crate) type DynExporter = Box + Send>; +pub type DynExporter = Box + Send>; pub trait Transformer { /// Export the given input with given world. @@ -36,6 +36,15 @@ where } } +impl From for DynExporter +where + F: (for<'a> Fn(&'a (dyn World + 'a), Arc) -> SourceResult) + Sized + Send + 'static, +{ + fn from(f: F) -> Self { + Box::new(f) + } +} + pub mod builtins { use std::{fs::File, sync::Arc}; @@ -78,6 +87,12 @@ pub mod builtins { } } + impl From> for DynExporter { + fn from(exporter: GroupExporter) -> Self { + Box::new(exporter) + } + } + /// The Exporter> must be explicitly constructed. pub struct FromExporter { exporter: GroupExporter, @@ -104,6 +119,15 @@ pub mod builtins { } } + impl From> for DynExporter + where + A: for<'a> From<&'a I>, + { + fn from(exporter: FromExporter) -> Self { + Box::new(exporter) + } + } + pub struct FsPathExporter { path: std::path::PathBuf, exporter: E, @@ -145,6 +169,25 @@ pub mod builtins { } } + impl From> for DynExporter + where + E: Exporter, + Bytes: AsRef<[u8]>, + { + fn from(exporter: FsPathExporter) -> Self { + Box::new(exporter) + } + } + + impl From> for DynExporter + where + E: Transformer<(Arc, File)>, + { + fn from(exporter: FsPathExporter) -> Self { + Box::new(exporter) + } + } + pub struct VecExporter { exporter: E, diff --git a/core/src/lib.rs b/core/src/lib.rs index 1d3d62b3..d5768342 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -19,7 +19,7 @@ pub mod config; // Core mechanism of typst-ts. pub(crate) mod exporter; pub use exporter::{builtins as exporter_builtins, utils as exporter_utils}; -pub use exporter::{Exporter, Transformer}; +pub use exporter::{DynExporter, Exporter, Transformer}; pub mod font; pub use font::{FontLoader, FontResolver, FontSlot}; pub mod package; diff --git a/core/src/vector/ir.rs b/core/src/vector/ir.rs index ff44df7c..396b7771 100644 --- a/core/src/vector/ir.rs +++ b/core/src/vector/ir.rs @@ -18,7 +18,7 @@ use crate::{ StaticHash128, }; -pub type ImmutStr = Arc; +pub use crate::ImmutStr; pub use super::geom::*; @@ -108,7 +108,8 @@ pub struct FontRef { /// Reference a glyph item in a more friendly format to compress and store /// information. The glyphs are locally stored in the svg module. /// With a glyph reference, we can get both the font metric and the glyph data. -/// The `font_hash` is to let it safe to be cached, please see [`FontItem`] for more details. +/// The `font_hash` is to let it safe to be cached, please see [`FontItem`] for +/// more details. #[derive(Debug, Clone, Hash, PartialEq, Eq)] #[cfg_attr(feature = "rkyv", derive(Archive, rDeser, rSer))] #[cfg_attr(feature = "rkyv-validation", archive(check_bytes))] diff --git a/github-pages/docs/data-flow-standalone.typ b/github-pages/docs/data-flow-standalone.typ index 25f1bdec..2f640d02 100644 --- a/github-pages/docs/data-flow-standalone.typ +++ b/github-pages/docs/data-flow-standalone.typ @@ -9,3 +9,5 @@ caption: [Browser-side module needed: $dagger$: compiler; $dagger.double$: renderer. ], numbering: none, ) + + diff --git a/github-pages/docs/test.typ b/github-pages/docs/test.typ new file mode 100644 index 00000000..e69de29b diff --git a/packages/compiler/src/lib.rs b/packages/compiler/src/lib.rs index 466f6289..00bee64f 100644 --- a/packages/compiler/src/lib.rs +++ b/packages/compiler/src/lib.rs @@ -1,4 +1,4 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; use base64::Engine; use js_sys::{JsString, Uint8Array}; @@ -181,7 +181,7 @@ impl TypstCompiler { // compile and export document let doc = self.compiler.compile().map_err(|e| format!("{e:?}"))?; let data = ast_exporter - .export(self.compiler.world(), Arc::new(doc)) + .export(self.compiler.world(), doc) .map_err(|e| format!("{e:?}"))?; let converted = ansi_to_html::convert_escaped( @@ -204,7 +204,7 @@ impl TypstCompiler { let doc = self.compiler.compile().map_err(|e| format!("{e:?}"))?; let artifact_bytes = ir_exporter - .export(self.compiler.world(), Arc::new(doc)) + .export(self.compiler.world(), doc) .map_err(|e| format!("{e:?}"))?; Ok(artifact_bytes) } diff --git a/server/dev/src/main.rs b/server/dev/src/main.rs index 68cc624b..14f28cf7 100644 --- a/server/dev/src/main.rs +++ b/server/dev/src/main.rs @@ -66,7 +66,7 @@ fn compile_corpus(args: CompileCorpusArgs) { driver.set_exporter(exporter); driver.inner_mut().set_entry_file(entry); - driver.with_compile_diag::(|driver| driver.compile()); + driver.with_stage_diag::("compiling", |driver| driver.compile()); // if status.code().unwrap() != 0 { // eprintln!("compile corpus failed."); diff --git a/tests/incremental/src/lib.rs b/tests/incremental/src/lib.rs index 9c0048c6..fe2bf3e4 100644 --- a/tests/incremental/src/lib.rs +++ b/tests/incremental/src/lib.rs @@ -1,4 +1,4 @@ -//! based on project https://github.com/frozolotl/typst-mutilate +//! based on project //! LICENSE: European Union Public License 1.2 use rand::{seq::IteratorRandom, Rng, SeedableRng}; use rand_xoshiro::Xoshiro256PlusPlus; diff --git a/tests/incremental/src/main.rs b/tests/incremental/src/main.rs index 9885596d..85536f7d 100644 --- a/tests/incremental/src/main.rs +++ b/tests/incremental/src/main.rs @@ -1,4 +1,4 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; use typst::doc::Document; use typst_ts_compiler::{ @@ -98,7 +98,6 @@ pub fn test_compiler( }) .unwrap(); - let doc = Arc::new(doc); let delta = incr_server.pack_delta(doc); let delta = BytesModuleStream::from_slice(&delta).checkout_owned(); incr_client.merge_delta(delta);