From 676be4597799ffaf5ba1e720c516eed91b5effe5 Mon Sep 17 00:00:00 2001 From: Myriad-Dreamin Date: Thu, 21 Sep 2023 22:28:51 +0800 Subject: [PATCH] test: add incremental compilation fuzzer --- Cargo.lock | 29 ++++ Cargo.toml | 1 + exporter/svg/src/frontend/incremental.rs | 23 ++- fuzzers/corpora/viewers/preview-incr_01.typ | 34 ++-- tests/incremental/Cargo.toml | 30 ++++ tests/incremental/src/lib.rs | 163 ++++++++++++++++++++ tests/incremental/src/main.rs | 121 +++++++++++++++ 7 files changed, 372 insertions(+), 29 deletions(-) create mode 100644 tests/incremental/Cargo.toml create mode 100644 tests/incremental/src/lib.rs create mode 100644 tests/incremental/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 1852cb38..701a0fba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2602,6 +2602,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core", +] + [[package]] name = "rayon" version = "1.7.0" @@ -4283,6 +4292,26 @@ dependencies = [ "typst-ts-test-common", ] +[[package]] +name = "typst-ts-incremental-test" +version = "0.4.0-rc6" +dependencies = [ + "anyhow", + "comemo", + "hex", + "rand", + "rand_xoshiro", + "sha2", + "tokio", + "typst", + "typst-syntax", + "typst-ts-compiler", + "typst-ts-core", + "typst-ts-dev-server", + "typst-ts-svg-exporter", + "typst-ts-test-common", +] + [[package]] name = "typst-ts-integration-test" version = "0.4.0-rc6" diff --git a/Cargo.toml b/Cargo.toml index 971402cc..929b1b95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ members = [ "tests/common", "tests/heap-profile", + "tests/incremental", "tests/integration", "tests/std", ] diff --git a/exporter/svg/src/frontend/incremental.rs b/exporter/svg/src/frontend/incremental.rs index 21c84cdb..5fa9afa6 100644 --- a/exporter/svg/src/frontend/incremental.rs +++ b/exporter/svg/src/frontend/incremental.rs @@ -64,20 +64,17 @@ impl SvgTask { } } - println!("reusable: {:?}", reusable); - println!("unused_prev: {:?}", unused_prev); - - for ( - idx, - Page { - content: entry, - size, - }, - ) in ctx.next.iter().enumerate() + // println!("reusable: {:?}", reusable); + // println!("unused_prev: {:?}", unused_prev); + + for Page { + content: entry, + size, + } in ctx.next.iter() { let size = Self::page_size(*size); if reusable.contains(entry) { - println!("reuse page: {} {:?}", idx, entry); + // println!("reuse page: {} {:?}", idx, entry); svg_body.push(SvgText::Content(Arc::new(SvgTextNode { attributes: vec![ ("class", "typst-page".into()), @@ -104,13 +101,13 @@ impl SvgTask { // todo: evaluate simlarity let item = if let Some(prev_entry) = unused_prev.pop_first().map(|(_, v)| v) { - println!("diff page: {} {:?} {:?}", idx, entry, prev_entry); + // println!("diff page: {} {:?} {:?}", idx, entry, prev_entry); attributes.push(("data-reuse-from", prev_entry.as_svg_id("p"))); render_task.render_diff_item(entry, &prev_entry) } else { // todo: find a box - println!("rebuild page: {} {:?}", idx, entry); + // println!("rebuild page: {} {:?}", idx, entry); render_task.render_flat_item(entry) }; diff --git a/fuzzers/corpora/viewers/preview-incr_01.typ b/fuzzers/corpora/viewers/preview-incr_01.typ index 1aed74e1..1e903341 100644 --- a/fuzzers/corpora/viewers/preview-incr_01.typ +++ b/fuzzers/corpora/viewers/preview-incr_01.typ @@ -4,7 +4,7 @@ #set page( paper: "a4", header: align(right)[ - Multi-purpose Combat Chassis [FALKEN] + Multi-purpose Combat Chassis ], numbering: "1", margin: (x:20mm, y:12.7mm) @@ -46,30 +46,32 @@ #let img = "/assets/files/tiger.jpg" -= #lorem(4) += Seed #outline(title:none, indent:auto, ) #booktab() -#lorem(5000) +Seed2 Seed4 + +Seed3 Seed4 #pagebreak() == #lorem(3) #emp_block()[ -#lorem(100) + Seed4 Seed4 Seed4 Seed4 ] #booktab() -#lorem(5000) + Seed4 Seed4 Seed4 Seed4 Seed4 Seed4 Seed4 Seed4 Seed4 Seed4 Seed4 Seed4 Seed4 Seed4 #pagebreak() == 隼的轻武器 #booktab() -#lorem(5000) +Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed #let images_rkg3=("tiger.jpg","tiger.jpg") @@ -80,7 +82,7 @@ ) ] -#lorem(300) +Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed #let images_pistol=("tiger.jpg","tiger.jpg") #let cell = rect.with( @@ -96,37 +98,37 @@ ..images_pistol.map(n=>align(center)[#image(img, width:20%)]) ) -#lorem(450) +Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed #figure( image(img, width: 50%) ) -#lorem(450) +Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed #figure( image(img, width: 50%) ) -#lorem(300) +Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed === 电磁枪 -#lorem(450) +Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed #pagebreak() == 武器舱和背部重武器 #booktab() -#lorem(300) +Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed #figure( image(img, width: 60%), caption: [30mm Rapid Railgun with Extended Barrel, also retrofitted as 2nd stage rail on Arclight] ) -#lorem(5000) +Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed #figure( image(img, width: 60%), @@ -140,18 +142,18 @@ caption: [炮管展开60°的状态] ) -#lorem(300) +Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed #figure( image(img, width: 80%), caption: [4Sure Ballistics Man-Portable ASAT Missile] ) -#lorem(500) +Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed #pagebreak() == 辅助机 —— "FRAMER" #booktab() -#lorem(500) +Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed Seed diff --git a/tests/incremental/Cargo.toml b/tests/incremental/Cargo.toml new file mode 100644 index 00000000..8c113e28 --- /dev/null +++ b/tests/incremental/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "typst-ts-incremental-test" +authors.workspace = true +version.workspace = true +license.workspace = true +edition.workspace = true +homepage.workspace = true +repository.workspace = true + +[dependencies] +typst.workspace = true +typst-syntax.workspace = true +comemo.workspace = true + +sha2.workspace = true +anyhow.workspace = true +tokio.workspace = true + +rand = "0.8.5" +rand_xoshiro = "0.6.0" + +typst-ts-dev-server.workspace = true +typst-ts-test-common.workspace = true +typst-ts-core.workspace = true +hex.workspace = true +typst-ts-compiler.workspace = true +typst-ts-svg-exporter.workspace = true + +[features] +generate = [] diff --git a/tests/incremental/src/lib.rs b/tests/incremental/src/lib.rs new file mode 100644 index 00000000..9c0048c6 --- /dev/null +++ b/tests/incremental/src/lib.rs @@ -0,0 +1,163 @@ +//! based on project https://github.com/frozolotl/typst-mutilate +//! LICENSE: European Union Public License 1.2 +use rand::{seq::IteratorRandom, Rng, SeedableRng}; +use rand_xoshiro::Xoshiro256PlusPlus; +use std::{ + borrow::Cow, + io::{self, Write}, + ops::Range, +}; + +use typst_syntax::{ast, SyntaxKind, SyntaxNode}; + +pub fn mutate(code: String) -> io::Result { + let mut tr = Mutator::build_context()?; + + let syntax = typst_syntax::parse(&code); + let errors = syntax.errors(); + if !errors.is_empty() { + panic!("Syntax errors: {:?}", errors); + } + + tr.mutate_syntax(&syntax)?; + Ok(String::from_utf8(tr.output).unwrap()) +} + +// unicode range +const JPWORD_RANGE: Range = 0x3040..0x30ff; +const HANZI_RANGE: Range = 0x4e00..0x9fa5; +const ENGLISH_RANGE: Range = 0x61..0x7b; + +struct Mutator { + rng: Xoshiro256PlusPlus, + aggressive: bool, + // language: Lang, + output: Vec, +} + +impl Mutator { + fn build_context() -> io::Result { + let rng = Xoshiro256PlusPlus::from_rng(rand::thread_rng()).unwrap(); + Ok(Self { + rng, + aggressive: false, + output: Vec::new(), + // language, + }) + } + + fn mutate_syntax(&mut self, syntax: &SyntaxNode) -> io::Result<()> { + match syntax.kind() { + SyntaxKind::Text => self.mutate_text(syntax.text()), + SyntaxKind::LineComment => { + write!(self.output, "//")?; + let content = &syntax.text()[2..]; + write!(self.output, "{content}")?; + // self.translate_text(content)?; + Ok(()) + } + SyntaxKind::BlockComment => { + write!(self.output, "/*")?; + let content = &syntax.text()[2..syntax.text().len() - 2]; + write!(self.output, "{content}")?; + // self.translate_text(content)?; + write!(self.output, "*/")?; + Ok(()) + } + SyntaxKind::Str if self.aggressive => { + write!(self.output, "\"")?; + let content = &syntax.text()[1..syntax.text().len() - 1]; + self.mutate_text(content)?; + write!(self.output, "\"")?; + Ok(()) + } + SyntaxKind::Raw => { + let raw: ast::Raw = syntax.cast().unwrap(); + let backticks = syntax.text().split(|c| c != '`').next().unwrap(); + write!(self.output, "{backticks}")?; + + let mut text = syntax + .text() + .trim_start_matches('`') + .strip_suffix(backticks) + .unwrap(); + if let Some(lang) = raw.lang() { + text = text.strip_prefix(lang).unwrap(); + write!(self.output, "{lang}")?; + } + + // will not translate text inside raw blocks + write!(self.output, "{text}")?; + // self.translate_text(text, self.output)?; + write!(self.output, "{backticks}")?; + Ok(()) + } + SyntaxKind::Link => { + let (scheme, rest) = syntax.text().split_once(':').unwrap(); + write!(self.output, "{scheme}:")?; + self.mutate_text(rest) + } + SyntaxKind::Equation + | SyntaxKind::Math + | SyntaxKind::MathIdent + | SyntaxKind::MathAlignPoint + | SyntaxKind::MathDelimited + | SyntaxKind::MathFrac + | SyntaxKind::MathPrimes + | SyntaxKind::MathRoot + | SyntaxKind::ModuleInclude + | SyntaxKind::ModuleImport => self.write_node(syntax), + _ if syntax.children().next().is_some() => { + for child in syntax.children() { + self.mutate_syntax(child)?; + } + Ok(()) + } + _ => self.write_node(syntax), + } + } + + fn mutate_text(&mut self, text: &str) -> io::Result<()> { + if self.rng.gen_bool(0.96) { + write!(self.output, "{}", text)?; + return Ok(()); + } + + let mutated = if self.rng.gen_bool(0.33) { + Cow::Owned(text.repeat(50.min(1000 / text.len()).max(1))) + } else if self.rng.gen_bool(0.33) { + Cow::Owned(text.chars().take(50).collect::()) + } else { + Cow::Borrowed(text) + } + .chars() + .flat_map(|c| { + if c.is_ascii_punctuation() || c.is_whitespace() { + return Some(c); + } + + let res = (0..3).choose(&mut self.rng).unwrap(); + let w = match res { + 0 => ENGLISH_RANGE.choose(&mut self.rng).unwrap(), + 1 => HANZI_RANGE.choose(&mut self.rng).unwrap(), + 2 => JPWORD_RANGE.choose(&mut self.rng).unwrap(), + _ => unreachable!(), + }; + char::from_u32(w) + }) + .collect::(); + write!(self.output, "{}", mutated)?; + Ok(()) + } + + fn write_node(&mut self, syntax: &SyntaxNode) -> io::Result<()> { + if syntax.children().next().is_some() { + for child in syntax.children() { + self.write_node(child)?; + } + } else { + write!(self.output, "{}", syntax.text())?; + } + Ok(()) + } +} diff --git a/tests/incremental/src/main.rs b/tests/incremental/src/main.rs new file mode 100644 index 00000000..9885596d --- /dev/null +++ b/tests/incremental/src/main.rs @@ -0,0 +1,121 @@ +use std::{path::Path, sync::Arc}; + +use typst::doc::Document; +use typst_ts_compiler::{ + service::{CompileDriver, CompileExporter, Compiler}, + ShadowApi, TypstSystemWorld, +}; +use typst_ts_core::{ + config::CompileOpts, + exporter_builtins::GroupExporter, + vector::{ + incr::{IncrDocClient, IncrDocServer}, + ir::{Abs, Point, Rect}, + stream::BytesModuleStream, + }, +}; +use typst_ts_incremental_test::mutate; +use typst_ts_svg_exporter::IncrSvgDocClient; + +fn get_driver( + workspace_dir: &Path, + entry_file_path: &Path, + exporter: GroupExporter, +) -> CompileExporter { + let project_base = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let font_path = project_base.join("assets/fonts"); + let world = TypstSystemWorld::new(CompileOpts { + root_dir: workspace_dir.to_owned(), + no_system_fonts: true, + font_paths: vec![font_path], + ..CompileOpts::default() + }) + .unwrap(); + + let driver = CompileDriver { + world, + entry_file: entry_file_path.to_owned(), + }; + + CompileExporter::new(driver).with_exporter(exporter) +} + +pub fn test_compiler( + workspace_dir: &Path, + entry_file_path: &Path, + exporter: GroupExporter, +) { + let mut driver = get_driver(workspace_dir, entry_file_path, exporter); + let mut content = { std::fs::read_to_string(entry_file_path).expect("Could not read file") }; + + #[cfg(feature = "generate")] + let mut incr_server = IncrDocServer::default(); + #[cfg(feature = "generate")] + let mut incr_client = IncrDocClient::default(); + #[cfg(feature = "generate")] + let mut incr_svg_client = IncrSvgDocClient::default(); + + let window = Rect { + lo: Point::new(Abs::from(0.), Abs::from(0.)), + hi: Point::new(Abs::from(1e33), Abs::from(1e33)), + }; + + #[cfg(feature = "generate")] + std::fs::write("mutate_sequence.log", "").unwrap(); + + for i in 0..200 { + println!("Iteration {}", i); + + if cfg!(feature = "generate") { + content.push_str(" #lorem(50)"); + content = mutate(content).unwrap(); + std::fs::write("test.typ", &content).unwrap(); + { + use std::io::Write; + let mut f = std::fs::OpenOptions::new() + .append(true) + .create(true) + .open("mutate_sequence.log") + .unwrap(); + f.write_all(hex::encode(&content).as_bytes()).unwrap(); + f.write_all(b"\n").unwrap(); + } + } + + #[cfg(not(feature = "generate"))] + let mut incr_server = IncrDocServer::default(); + #[cfg(not(feature = "generate"))] + let mut incr_client = IncrDocClient::default(); + #[cfg(not(feature = "generate"))] + let mut incr_svg_client = IncrSvgDocClient::default(); + + // checkout the entry file + let main_id = driver.main_id(); + + let doc = driver + .with_shadow_file_by_id(main_id, content.as_bytes().into(), |driver| { + driver.compile() + }) + .unwrap(); + + let doc = Arc::new(doc); + let delta = incr_server.pack_delta(doc); + let delta = BytesModuleStream::from_slice(&delta).checkout_owned(); + incr_client.merge_delta(delta); + incr_client.set_layout(incr_client.doc.layouts[0].unwrap_single()); + let _ = incr_svg_client.render_in_window(&mut incr_client, window); + + comemo::evict(10); + } +} + +pub fn main() { + let workspace_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + #[cfg(feature = "generate")] + let entry_file_path = workspace_dir.join("fuzzers/corpora/viewers/preview-incr_01.typ"); + #[cfg(not(feature = "generate"))] + let entry_file_path = workspace_dir.join("test.typ"); + + let noop_exporter = GroupExporter::new(vec![]); + test_compiler(&workspace_dir, &entry_file_path, noop_exporter); +}