From 13d5750a61ef90ca5e9d1af7720c053c8af2acde Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 7 Jan 2025 09:41:29 -0800 Subject: [PATCH] Relocate `SingleModuleGraph` to `turbopack-core` (#74571) While many use case-specific graphs belong in Next.js, this primitive should be kept in Turbopack as it will be used for other core usecases like chunking. Test Plan: CI --- Cargo.lock | 1 + crates/next-api/src/client_references.rs | 4 +- crates/next-api/src/dynamic_imports.rs | 3 +- crates/next-api/src/module_graph.rs | 533 +---------------- crates/next-api/src/server_actions.rs | 3 +- turbopack/crates/turbopack-core/Cargo.toml | 1 + turbopack/crates/turbopack-core/src/lib.rs | 2 + .../turbopack-core/src/module_graph/mod.rs | 535 ++++++++++++++++++ 8 files changed, 548 insertions(+), 534 deletions(-) create mode 100644 turbopack/crates/turbopack-core/src/module_graph/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 6ac85cac0a7ef..27c7e2581df6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8834,6 +8834,7 @@ dependencies = [ "lazy_static", "once_cell", "patricia_tree", + "petgraph", "ref-cast", "regex", "rstest", diff --git a/crates/next-api/src/client_references.rs b/crates/next-api/src/client_references.rs index 7bb5fa0c24572..e1270d9057be2 100644 --- a/crates/next-api/src/client_references.rs +++ b/crates/next-api/src/client_references.rs @@ -10,9 +10,7 @@ use turbo_tasks::{ debug::ValueDebugFormat, trace::TraceRawVcs, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, Vc, }; use turbopack::css::CssModuleAsset; -use turbopack_core::module::Module; - -use crate::module_graph::SingleModuleGraph; +use turbopack_core::{module::Module, module_graph::SingleModuleGraph}; #[derive( Clone, Serialize, Deserialize, Eq, PartialEq, TraceRawVcs, ValueDebugFormat, NonLocalValue, diff --git a/crates/next-api/src/dynamic_imports.rs b/crates/next-api/src/dynamic_imports.rs index 19ce191dca61c..96cb3618a05b9 100644 --- a/crates/next-api/src/dynamic_imports.rs +++ b/crates/next-api/src/dynamic_imports.rs @@ -34,10 +34,11 @@ use turbopack_core::{ ChunkingContext, ModuleId, }, module::Module, + module_graph::SingleModuleGraph, output::OutputAssets, }; -use crate::module_graph::{DynamicImportEntriesWithImporter, SingleModuleGraph}; +use crate::module_graph::DynamicImportEntriesWithImporter; pub(crate) enum NextDynamicChunkAvailability<'a> { /// In App Router, the client references diff --git a/crates/next-api/src/module_graph.rs b/crates/next-api/src/module_graph.rs index 0c7928140071e..b446f74516f7b 100644 --- a/crates/next-api/src/module_graph.rs +++ b/crates/next-api/src/module_graph.rs @@ -1,12 +1,9 @@ use std::{ borrow::Cow, collections::{HashMap, HashSet}, - future::Future, - hash::Hash, - ops::Deref, }; -use anyhow::{Context, Result}; +use anyhow::Result; use next_core::{ mode::NextMode, next_client_reference::{ @@ -16,26 +13,15 @@ use next_core::{ next_dynamic::NextDynamicEntryModule, next_manifests::ActionLayer, }; -use petgraph::{ - graph::{DiGraph, NodeIndex}, - visit::{Dfs, VisitMap, Visitable}, -}; -use serde::{Deserialize, Serialize}; use tracing::Instrument; -use turbo_rcstr::RcStr; use turbo_tasks::{ - debug::ValueDebugFormat, - graph::{AdjacencyMap, GraphTraversal, Visit, VisitControlFlow, VisitedNodes}, - trace::{TraceRawVcs, TraceRawVcsContext}, - CollectiblesSource, FxIndexMap, FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, - TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc, + CollectiblesSource, FxIndexMap, FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc, }; use turbopack_core::{ - chunk::ChunkingType, context::AssetContext, - issue::{Issue, IssueExt}, - module::{Module, Modules}, - reference::primary_chunkable_referenced_modules, + issue::Issue, + module::Module, + module_graph::{GraphTraversalAction, SingleModuleGraph}, }; use crate::{ @@ -49,515 +35,6 @@ use crate::{ #[derive(Clone, Debug)] struct SingleModuleGraphs(pub Vec>); -#[derive(PartialEq, Eq, Debug)] -pub enum GraphTraversalAction { - /// Continue visiting children - Continue, - /// Skip the immediate children - Skip, -} - -#[derive(Clone, Debug, Serialize, Deserialize, TraceRawVcs, NonLocalValue)] -pub struct SingleModuleGraphNode { - pub module: ResolvedVc>, - issues: Vec>>, - pub layer: Option>, -} - -impl SingleModuleGraphNode { - fn emit_issues(&self) { - for issue in &self.issues { - issue.emit(); - } - } -} - -#[derive(Clone, Debug, ValueDebugFormat, Serialize, Deserialize)] -struct TracedDiGraph(DiGraph); -impl Default for TracedDiGraph { - fn default() -> Self { - Self(Default::default()) - } -} -impl TraceRawVcs for TracedDiGraph { - fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) { - for node in self.0.node_weights() { - node.trace_raw_vcs(trace_context); - } - for edge in self.0.edge_weights() { - edge.trace_raw_vcs(trace_context); - } - } -} -impl Deref for TracedDiGraph { - type Target = DiGraph; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -#[turbo_tasks::value(cell = "new", eq = "manual", into = "new", local)] -#[derive(Clone, Default)] -pub struct SingleModuleGraph { - graph: TracedDiGraph, - // NodeIndex isn't necessarily stable, but these are first nodes in the graph, so shouldn't - // ever be involved in a swap_remove operation - // - // HashMaps have nondeterministic order, but this map is only used for lookups (in `get_entry`) - // and not iteration. - // - // This contains Vcs, but they are already contained in the graph, so no need to trace this. - #[turbo_tasks(trace_ignore)] - entries: HashMap>, NodeIndex>, -} - -#[turbo_tasks::value(transparent)] -#[derive(Clone, Debug)] -struct ModuleSet(pub HashSet>>); - -// These nodes are created while walking the Turbopack modules references, and are used to then -// afterwards build the SingleModuleGraph. -#[derive(Clone, Hash, PartialEq, Eq)] -enum SingleModuleGraphBuilderNode { - /// This edge is represented as a node: source Module -> ChunkableReference -> target Module - ChunkableReference { - chunking_type: ChunkingType, - source: ResolvedVc>, - source_ident: ReadRef, - target: ResolvedVc>, - target_ident: ReadRef, - }, - Module { - module: ResolvedVc>, - layer: Option>, - ident: ReadRef, - }, - /// Issues to be added to the parent Module node - #[allow(dead_code)] - Issues(Vec>>), -} - -impl SingleModuleGraphBuilderNode { - async fn new_module(module: ResolvedVc>) -> Result { - let ident = module.ident(); - Ok(Self::Module { - module, - layer: match ident.await?.layer { - Some(layer) => Some(layer.await?), - None => None, - }, - ident: ident.to_string().await?, - }) - } - async fn new_chunkable_ref( - source: ResolvedVc>, - target: ResolvedVc>, - chunking_type: ChunkingType, - ) -> Result { - Ok(Self::ChunkableReference { - chunking_type, - source, - source_ident: source.ident().to_string().await?, - target, - target_ident: target.ident().to_string().await?, - }) - } -} -struct SingleModuleGraphBuilderEdge { - // ty: Option, - to: SingleModuleGraphBuilderNode, -} - -/// The chunking type that occurs most often, is handled more efficiently by not creating -/// intermediate SingleModuleGraphBuilderNode::ChunkableReference nodes. -const COMMON_CHUNKING_TYPE: ChunkingType = ChunkingType::ParallelInheritAsync; - -struct SingleModuleGraphBuilder {} -impl Visit for SingleModuleGraphBuilder { - type Edge = SingleModuleGraphBuilderEdge; - type EdgesIntoIter = Vec; - type EdgesFuture = impl Future>; - - fn visit(&mut self, edge: Self::Edge) -> VisitControlFlow { - match edge.to { - SingleModuleGraphBuilderNode::Module { .. } - | SingleModuleGraphBuilderNode::ChunkableReference { .. } => { - VisitControlFlow::Continue(edge.to) - } - SingleModuleGraphBuilderNode::Issues(_) => VisitControlFlow::Skip(edge.to), - } - } - - fn edges(&mut self, node: &SingleModuleGraphBuilderNode) -> Self::EdgesFuture { - // Destructure beforehand to not have to clone the whole node when entering the async block - let (module, chunkable_ref_target) = match node { - SingleModuleGraphBuilderNode::Module { module, .. } => (Some(*module), None), - SingleModuleGraphBuilderNode::ChunkableReference { target, .. } => { - (None, Some(*target)) - } - SingleModuleGraphBuilderNode::Issues(_) => unreachable!(), - }; - async move { - Ok(match (module, chunkable_ref_target) { - (Some(module), None) => { - let refs_cell = primary_chunkable_referenced_modules(*module); - let refs = refs_cell.await?; - // TODO This is currently too slow - // let refs_issues = refs_cell - // .take_collectibles::>() - // .iter() - // .map(|issue| issue.to_resolved()) - // .try_join() - // .await?; - - refs.iter() - .flat_map(|(ty, modules)| { - if matches!(ty, ChunkingType::Traced) { - None - } else { - Some(modules.iter().map(|m| (ty.clone(), *m))) - } - }) - .flatten() - .map(|(ty, target)| async move { - Ok(SingleModuleGraphBuilderEdge { - to: if ty == COMMON_CHUNKING_TYPE { - SingleModuleGraphBuilderNode::new_module(target).await? - } else { - SingleModuleGraphBuilderNode::new_chunkable_ref( - module, target, ty, - ) - .await? - }, - }) - }) - .try_join() - .await? - } - (None, Some(chunkable_ref_target)) => { - vec![SingleModuleGraphBuilderEdge { - to: SingleModuleGraphBuilderNode::new_module(chunkable_ref_target).await?, - }] - } - _ => unreachable!(), - }) - } - } - - fn span(&mut self, node: &SingleModuleGraphBuilderNode) -> tracing::Span { - match node { - SingleModuleGraphBuilderNode::Module { ident, .. } => { - tracing::info_span!("module", name = display(ident)) - } - SingleModuleGraphBuilderNode::Issues(_) => { - tracing::info_span!("issues") - } - SingleModuleGraphBuilderNode::ChunkableReference { - chunking_type, - source_ident, - target_ident, - .. - } => { - tracing::info_span!( - "chunkable reference", - ty = debug(chunking_type), - source = display(source_ident), - target = display(target_ident) - ) - } - } - } -} - -impl SingleModuleGraph { - /// Walks the graph starting from the given entries and collects all reachable nodes, skipping - /// nodes listed in `visited_modules` - /// The resulting graph's outgoing edges are in reverse order. - /// If passed, `root` is connected to the entries and include in `self.entries`. - async fn new_inner( - root: Option>>, - entries: &Vec>>, - visited_modules: &HashSet>>, - ) -> Result> { - let mut graph = DiGraph::new(); - - let root_edges = entries - .iter() - .map(|e| async move { - Ok(SingleModuleGraphBuilderEdge { - to: SingleModuleGraphBuilderNode::new_module(*e).await?, - }) - }) - .try_join() - .await?; - - let children_nodes_iter = AdjacencyMap::new() - .skip_duplicates_with_visited_nodes(VisitedNodes( - visited_modules - .iter() - .map(|&module| SingleModuleGraphBuilderNode::new_module(module)) - .try_join() - .await? - .into_iter() - .collect(), - )) - .visit(root_edges, SingleModuleGraphBuilder {}) - .await - .completed()? - .into_inner(); - - let mut modules: HashMap>, NodeIndex> = HashMap::new(); - { - let _span = tracing::info_span!("build module graph").entered(); - for (parent, current) in children_nodes_iter.into_breadth_first_edges() { - let parent_edge = parent.map(|parent| match parent { - SingleModuleGraphBuilderNode::Module { module, .. } => { - (*modules.get(&module).unwrap(), COMMON_CHUNKING_TYPE) - } - SingleModuleGraphBuilderNode::ChunkableReference { - source, - chunking_type, - .. - } => (*modules.get(&source).unwrap(), chunking_type), - SingleModuleGraphBuilderNode::Issues { .. } => unreachable!(), - }); - - match current { - SingleModuleGraphBuilderNode::Module { - module, - layer, - ident: _, - } => { - // Find the current node, if it was already added - let current_idx = if let Some(current_idx) = modules.get(&module) { - *current_idx - } else { - let idx = graph.add_node(SingleModuleGraphNode { - module, - issues: Default::default(), - layer, - }); - modules.insert(module, idx); - idx - }; - // Add the edge - if let Some((parent_idx, chunking_type)) = parent_edge { - graph.add_edge(parent_idx, current_idx, chunking_type); - } - } - SingleModuleGraphBuilderNode::ChunkableReference { .. } => { - // Ignore. They are handled when visiting the next edge - // (ChunkableReference -> Module) - } - SingleModuleGraphBuilderNode::Issues(new_issues) => { - let (parent_idx, _) = parent_edge.unwrap(); - graph - .node_weight_mut(parent_idx) - .unwrap() - .issues - .extend(new_issues); - } - } - } - } - - let root_idx = if let Some(root) = root { - if !modules.contains_key(&root) { - let root_idx = graph.add_node(SingleModuleGraphNode { - module: root, - issues: Default::default(), - layer: None, - // ident: root.ident().to_string().await?, - }); - for entry in entries { - graph.add_edge( - root_idx, - *modules.get(entry).unwrap(), - ChunkingType::Parallel, - ); - } - Some((root, root_idx)) - } else { - None - } - } else { - None - }; - - Ok(SingleModuleGraph { - graph: TracedDiGraph(graph), - entries: entries - .iter() - .map(|e| (*e, *modules.get(e).unwrap())) - .chain(root_idx.into_iter()) - .collect(), - } - .cell()) - } - - fn get_entry(&self, module: ResolvedVc>) -> Result { - self.entries - .get(&module) - .copied() - .context("Couldn't find entry module in graph") - } - - /// Iterate over all nodes in the graph (potentially in the whole app!). - pub fn iter_nodes(&self) -> impl Iterator + '_ { - self.graph.node_weights() - } - - /// Enumerate over all nodes in the graph (potentially in the whole app!). - pub fn enumerate_nodes( - &self, - ) -> impl Iterator + '_ { - self.graph - .node_indices() - .map(move |idx| (idx, self.graph.node_weight(idx).unwrap())) - } - - /// Traverses all reachable nodes (once) - pub fn traverse_from_entry<'a>( - &'a self, - entry: ResolvedVc>, - mut visitor: impl FnMut(&'a SingleModuleGraphNode), - ) -> Result<()> { - let entry_node = self.get_entry(entry)?; - - let mut dfs = Dfs::new(&*self.graph, entry_node); - while let Some(nx) = dfs.next(&*self.graph) { - let weight = self.graph.node_weight(nx).unwrap(); - weight.emit_issues(); - visitor(weight); - } - Ok(()) - } - - /// Traverses all reachable edges exactly once and calls the visitor with the edge source and - /// target. - /// - /// This means that target nodes can be revisited (once per incoming edge). - pub fn traverse_edges_from_entry<'a>( - &'a self, - entry: ResolvedVc>, - mut visitor: impl FnMut( - (Option<&'a SingleModuleGraphNode>, &'a SingleModuleGraphNode), - ) -> GraphTraversalAction, - ) -> Result<()> { - let graph = &self.graph; - let entry_node = self.get_entry(entry)?; - - let mut stack = vec![entry_node]; - let mut discovered = graph.visit_map(); - let entry_weight = graph.node_weight(entry_node).unwrap(); - entry_weight.emit_issues(); - visitor((None, entry_weight)); - - while let Some(node) = stack.pop() { - let node_weight = graph.node_weight(node).unwrap(); - if discovered.visit(node) { - for succ in graph.neighbors(node).collect::>() { - let succ_weight = graph.node_weight(succ).unwrap(); - let action = visitor((Some(node_weight), succ_weight)); - if !discovered.is_visited(&succ) && action == GraphTraversalAction::Continue { - stack.push(succ); - } - } - } - } - - Ok(()) - } - - /// Traverses all reachable edges in topological order. The preorder visitor can be used to - /// forward state down the graph, and to skip subgraphs - /// - /// Use this to collect modules in evaluation order. - /// - /// Target nodes can be revisited (once per incoming edge). - /// Edges are traversed in normal order, so should correspond to reference order. - pub fn traverse_edges_from_entry_topological<'a, S>( - &'a self, - entry: ResolvedVc>, - state: &mut S, - mut visit_preorder: impl FnMut( - (Option<&'a SingleModuleGraphNode>, &'a SingleModuleGraphNode), - &mut S, - ) -> GraphTraversalAction, - mut visit_postorder: impl FnMut( - (Option<&'a SingleModuleGraphNode>, &'a SingleModuleGraphNode), - &mut S, - ), - ) -> Result<()> { - let graph = &self.graph; - let entry_node = self.get_entry(entry)?; - - enum ReverseTopologicalPass { - Visit, - ExpandAndVisit, - } - - let mut stack: Vec<(ReverseTopologicalPass, Option, NodeIndex)> = - vec![(ReverseTopologicalPass::ExpandAndVisit, None, entry_node)]; - let mut expanded = HashSet::new(); - while let Some((pass, parent, current)) = stack.pop() { - match pass { - ReverseTopologicalPass::Visit => { - visit_postorder( - ( - parent.map(|parent| graph.node_weight(parent).unwrap()), - graph.node_weight(current).unwrap(), - ), - state, - ); - } - ReverseTopologicalPass::ExpandAndVisit => { - let action = visit_preorder( - ( - parent.map(|parent| graph.node_weight(parent).unwrap()), - graph.node_weight(current).unwrap(), - ), - state, - ); - stack.push((ReverseTopologicalPass::Visit, parent, current)); - if expanded.insert(current) && action == GraphTraversalAction::Continue { - stack.extend( - graph - .neighbors(current) - // .collect::>() - // .rev() - .map(|child| { - (ReverseTopologicalPass::ExpandAndVisit, Some(current), child) - }), - ); - } - } - } - } - - Ok(()) - } -} - -#[turbo_tasks::value_impl] -impl SingleModuleGraph { - #[turbo_tasks::function] - async fn new_with_entries(entries: Vc) -> Result> { - SingleModuleGraph::new_inner(None, &*entries.await?, &Default::default()).await - } - - /// `root` is connected to the entries and include in `self.entries`. - #[turbo_tasks::function] - async fn new_with_entries_visited( - root: ResolvedVc>, - // This must not be a Vc> to ensure layout segment optimization hits the cache - entries: Vec>>, - visited_modules: Vc, - ) -> Result> { - SingleModuleGraph::new_inner(Some(root), &entries, &*visited_modules.await?).await - } -} - /// Implements layout segment optimization to compute a graph "chain" for each layout segment #[turbo_tasks::function] async fn get_module_graph_for_endpoint( diff --git a/crates/next-api/src/server_actions.rs b/crates/next-api/src/server_actions.rs index 7b63b183f1305..d928b58d8d47b 100644 --- a/crates/next-api/src/server_actions.rs +++ b/crates/next-api/src/server_actions.rs @@ -30,6 +30,7 @@ use turbopack_core::{ context::AssetContext, file_source::FileSource, module::Module, + module_graph::SingleModuleGraph, output::OutputAsset, reference_type::{EcmaScriptModulesReferenceSubType, ReferenceType}, resolve::ModulePart, @@ -41,8 +42,6 @@ use turbopack_ecmascript::{ tree_shake::asset::EcmascriptModulePartAsset, EcmascriptParsable, }; -use crate::module_graph::SingleModuleGraph; - #[turbo_tasks::value] pub(crate) struct ServerActionsManifest { pub loader: ResolvedVc>, diff --git a/turbopack/crates/turbopack-core/Cargo.toml b/turbopack/crates/turbopack-core/Cargo.toml index 612d202f96caa..be29ad91192f3 100644 --- a/turbopack/crates/turbopack-core/Cargo.toml +++ b/turbopack/crates/turbopack-core/Cargo.toml @@ -22,6 +22,7 @@ indexmap = { workspace = true } lazy_static = { workspace = true } once_cell = { workspace = true } patricia_tree = "0.5.5" +petgraph = { workspace = true } ref-cast = "1.0.20" regex = { workspace = true } serde = { workspace = true, features = ["rc"] } diff --git a/turbopack/crates/turbopack-core/src/lib.rs b/turbopack/crates/turbopack-core/src/lib.rs index 169a27bc48a58..46e0897aaa986 100644 --- a/turbopack/crates/turbopack-core/src/lib.rs +++ b/turbopack/crates/turbopack-core/src/lib.rs @@ -4,6 +4,7 @@ #![feature(assert_matches)] #![feature(arbitrary_self_types)] #![feature(arbitrary_self_types_pointers)] +#![feature(impl_trait_in_assoc_type)] #![feature(iter_intersperse)] pub mod asset; @@ -21,6 +22,7 @@ pub mod ident; pub mod introspect; pub mod issue; pub mod module; +pub mod module_graph; pub mod output; pub mod package_json; pub mod proxied_asset; diff --git a/turbopack/crates/turbopack-core/src/module_graph/mod.rs b/turbopack/crates/turbopack-core/src/module_graph/mod.rs new file mode 100644 index 0000000000000..7f49434075e2c --- /dev/null +++ b/turbopack/crates/turbopack-core/src/module_graph/mod.rs @@ -0,0 +1,535 @@ +use std::{ + collections::{HashMap, HashSet}, + future::Future, + ops::Deref, +}; + +use anyhow::{Context, Result}; +use petgraph::{ + graph::{DiGraph, NodeIndex}, + visit::{Dfs, VisitMap, Visitable}, +}; +use serde::{Deserialize, Serialize}; +use turbo_rcstr::RcStr; +use turbo_tasks::{ + debug::ValueDebugFormat, + graph::{AdjacencyMap, GraphTraversal, Visit, VisitControlFlow, VisitedNodes}, + trace::{TraceRawVcs, TraceRawVcsContext}, + NonLocalValue, ReadRef, ResolvedVc, TryJoinIterExt, ValueToString, Vc, +}; + +use crate::{ + chunk::ChunkingType, + issue::{Issue, IssueExt}, + module::{Module, Modules}, + reference::primary_chunkable_referenced_modules, +}; + +#[turbo_tasks::value(transparent)] +#[derive(Clone, Debug)] +pub struct ModuleSet(pub HashSet>>); + +#[turbo_tasks::value(cell = "new", eq = "manual", into = "new", local)] +#[derive(Clone, Default)] +pub struct SingleModuleGraph { + graph: TracedDiGraph, + // NodeIndex isn't necessarily stable, but these are first nodes in the graph, so shouldn't + // ever be involved in a swap_remove operation + // + // HashMaps have nondeterministic order, but this map is only used for lookups (in `get_entry`) + // and not iteration. + // + // This contains Vcs, but they are already contained in the graph, so no need to trace this. + #[turbo_tasks(trace_ignore)] + entries: HashMap>, NodeIndex>, +} + +impl SingleModuleGraph { + /// Walks the graph starting from the given entries and collects all reachable nodes, skipping + /// nodes listed in `visited_modules` + /// The resulting graph's outgoing edges are in reverse order. + /// If passed, `root` is connected to the entries and include in `self.entries`. + async fn new_inner( + root: Option>>, + entries: &Vec>>, + visited_modules: &HashSet>>, + ) -> Result> { + let mut graph = DiGraph::new(); + + let root_edges = entries + .iter() + .map(|e| async move { + Ok(SingleModuleGraphBuilderEdge { + to: SingleModuleGraphBuilderNode::new_module(*e).await?, + }) + }) + .try_join() + .await?; + + let children_nodes_iter = AdjacencyMap::new() + .skip_duplicates_with_visited_nodes(VisitedNodes( + visited_modules + .iter() + .map(|&module| SingleModuleGraphBuilderNode::new_module(module)) + .try_join() + .await? + .into_iter() + .collect(), + )) + .visit(root_edges, SingleModuleGraphBuilder {}) + .await + .completed()? + .into_inner(); + + let mut modules: HashMap>, NodeIndex> = HashMap::new(); + { + let _span = tracing::info_span!("build module graph").entered(); + for (parent, current) in children_nodes_iter.into_breadth_first_edges() { + let parent_edge = parent.map(|parent| match parent { + SingleModuleGraphBuilderNode::Module { module, .. } => { + (*modules.get(&module).unwrap(), COMMON_CHUNKING_TYPE) + } + SingleModuleGraphBuilderNode::ChunkableReference { + source, + chunking_type, + .. + } => (*modules.get(&source).unwrap(), chunking_type), + SingleModuleGraphBuilderNode::Issues { .. } => unreachable!(), + }); + + match current { + SingleModuleGraphBuilderNode::Module { + module, + layer, + ident: _, + } => { + // Find the current node, if it was already added + let current_idx = if let Some(current_idx) = modules.get(&module) { + *current_idx + } else { + let idx = graph.add_node(SingleModuleGraphNode { + module, + issues: Default::default(), + layer, + }); + modules.insert(module, idx); + idx + }; + // Add the edge + if let Some((parent_idx, chunking_type)) = parent_edge { + graph.add_edge(parent_idx, current_idx, chunking_type); + } + } + SingleModuleGraphBuilderNode::ChunkableReference { .. } => { + // Ignore. They are handled when visiting the next edge + // (ChunkableReference -> Module) + } + SingleModuleGraphBuilderNode::Issues(new_issues) => { + let (parent_idx, _) = parent_edge.unwrap(); + graph + .node_weight_mut(parent_idx) + .unwrap() + .issues + .extend(new_issues); + } + } + } + } + + let root_idx = if let Some(root) = root { + if !modules.contains_key(&root) { + let root_idx = graph.add_node(SingleModuleGraphNode { + module: root, + issues: Default::default(), + layer: None, + // ident: root.ident().to_string().await?, + }); + for entry in entries { + graph.add_edge( + root_idx, + *modules.get(entry).unwrap(), + ChunkingType::Parallel, + ); + } + Some((root, root_idx)) + } else { + None + } + } else { + None + }; + + Ok(SingleModuleGraph { + graph: TracedDiGraph(graph), + entries: entries + .iter() + .map(|e| (*e, *modules.get(e).unwrap())) + .chain(root_idx.into_iter()) + .collect(), + } + .cell()) + } + + fn get_entry(&self, module: ResolvedVc>) -> Result { + self.entries + .get(&module) + .copied() + .context("Couldn't find entry module in graph") + } + + /// Iterate over all nodes in the graph (potentially in the whole app!). + pub fn iter_nodes(&self) -> impl Iterator + '_ { + self.graph.node_weights() + } + + /// Enumerate over all nodes in the graph (potentially in the whole app!). + pub fn enumerate_nodes( + &self, + ) -> impl Iterator + '_ { + self.graph + .node_indices() + .map(move |idx| (idx, self.graph.node_weight(idx).unwrap())) + } + + /// Traverses all reachable nodes (once) + pub fn traverse_from_entry<'a>( + &'a self, + entry: ResolvedVc>, + mut visitor: impl FnMut(&'a SingleModuleGraphNode), + ) -> Result<()> { + let entry_node = self.get_entry(entry)?; + + let mut dfs = Dfs::new(&*self.graph, entry_node); + while let Some(nx) = dfs.next(&*self.graph) { + let weight = self.graph.node_weight(nx).unwrap(); + weight.emit_issues(); + visitor(weight); + } + Ok(()) + } + + /// Traverses all reachable edges exactly once and calls the visitor with the edge source and + /// target. + /// + /// This means that target nodes can be revisited (once per incoming edge). + pub fn traverse_edges_from_entry<'a>( + &'a self, + entry: ResolvedVc>, + mut visitor: impl FnMut( + (Option<&'a SingleModuleGraphNode>, &'a SingleModuleGraphNode), + ) -> GraphTraversalAction, + ) -> Result<()> { + let graph = &self.graph; + let entry_node = self.get_entry(entry)?; + + let mut stack = vec![entry_node]; + let mut discovered = graph.visit_map(); + let entry_weight = graph.node_weight(entry_node).unwrap(); + entry_weight.emit_issues(); + visitor((None, entry_weight)); + + while let Some(node) = stack.pop() { + let node_weight = graph.node_weight(node).unwrap(); + if discovered.visit(node) { + for succ in graph.neighbors(node).collect::>() { + let succ_weight = graph.node_weight(succ).unwrap(); + let action = visitor((Some(node_weight), succ_weight)); + if !discovered.is_visited(&succ) && action == GraphTraversalAction::Continue { + stack.push(succ); + } + } + } + } + + Ok(()) + } + + /// Traverses all reachable edges in topological order. The preorder visitor can be used to + /// forward state down the graph, and to skip subgraphs + /// + /// Use this to collect modules in evaluation order. + /// + /// Target nodes can be revisited (once per incoming edge). + /// Edges are traversed in normal order, so should correspond to reference order. + pub fn traverse_edges_from_entry_topological<'a, S>( + &'a self, + entry: ResolvedVc>, + state: &mut S, + mut visit_preorder: impl FnMut( + (Option<&'a SingleModuleGraphNode>, &'a SingleModuleGraphNode), + &mut S, + ) -> GraphTraversalAction, + mut visit_postorder: impl FnMut( + (Option<&'a SingleModuleGraphNode>, &'a SingleModuleGraphNode), + &mut S, + ), + ) -> Result<()> { + let graph = &self.graph; + let entry_node = self.get_entry(entry)?; + + enum ReverseTopologicalPass { + Visit, + ExpandAndVisit, + } + + let mut stack: Vec<(ReverseTopologicalPass, Option, NodeIndex)> = + vec![(ReverseTopologicalPass::ExpandAndVisit, None, entry_node)]; + let mut expanded = HashSet::new(); + while let Some((pass, parent, current)) = stack.pop() { + match pass { + ReverseTopologicalPass::Visit => { + visit_postorder( + ( + parent.map(|parent| graph.node_weight(parent).unwrap()), + graph.node_weight(current).unwrap(), + ), + state, + ); + } + ReverseTopologicalPass::ExpandAndVisit => { + let action = visit_preorder( + ( + parent.map(|parent| graph.node_weight(parent).unwrap()), + graph.node_weight(current).unwrap(), + ), + state, + ); + stack.push((ReverseTopologicalPass::Visit, parent, current)); + if expanded.insert(current) && action == GraphTraversalAction::Continue { + stack.extend( + graph + .neighbors(current) + // .collect::>() + // .rev() + .map(|child| { + (ReverseTopologicalPass::ExpandAndVisit, Some(current), child) + }), + ); + } + } + } + } + + Ok(()) + } +} + +#[turbo_tasks::value_impl] +impl SingleModuleGraph { + #[turbo_tasks::function] + pub async fn new_with_entries(entries: Vc) -> Result> { + SingleModuleGraph::new_inner(None, &*entries.await?, &Default::default()).await + } + + /// `root` is connected to the entries and include in `self.entries`. + #[turbo_tasks::function] + pub async fn new_with_entries_visited( + root: ResolvedVc>, + // This must not be a Vc> to ensure layout segment optimization hits the cache + entries: Vec>>, + visited_modules: Vc, + ) -> Result> { + SingleModuleGraph::new_inner(Some(root), &entries, &*visited_modules.await?).await + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, TraceRawVcs, NonLocalValue)] +pub struct SingleModuleGraphNode { + pub module: ResolvedVc>, + issues: Vec>>, + pub layer: Option>, +} + +impl SingleModuleGraphNode { + fn emit_issues(&self) { + for issue in &self.issues { + issue.emit(); + } + } +} + +#[derive(Clone, Debug, ValueDebugFormat, Serialize, Deserialize)] +struct TracedDiGraph(DiGraph); +impl Default for TracedDiGraph { + fn default() -> Self { + Self(Default::default()) + } +} +impl TraceRawVcs for TracedDiGraph { + fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) { + for node in self.0.node_weights() { + node.trace_raw_vcs(trace_context); + } + for edge in self.0.edge_weights() { + edge.trace_raw_vcs(trace_context); + } + } +} +impl Deref for TracedDiGraph { + type Target = DiGraph; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(PartialEq, Eq, Debug)] +pub enum GraphTraversalAction { + /// Continue visiting children + Continue, + /// Skip the immediate children + Skip, +} + +// These nodes are created while walking the Turbopack modules references, and are used to then +// afterwards build the SingleModuleGraph. +#[derive(Clone, Hash, PartialEq, Eq)] +enum SingleModuleGraphBuilderNode { + /// This edge is represented as a node: source Module -> ChunkableReference -> target Module + ChunkableReference { + chunking_type: ChunkingType, + source: ResolvedVc>, + source_ident: ReadRef, + target: ResolvedVc>, + target_ident: ReadRef, + }, + Module { + module: ResolvedVc>, + layer: Option>, + ident: ReadRef, + }, + /// Issues to be added to the parent Module node + #[allow(dead_code)] + Issues(Vec>>), +} + +impl SingleModuleGraphBuilderNode { + async fn new_module(module: ResolvedVc>) -> Result { + let ident = module.ident(); + Ok(Self::Module { + module, + layer: match ident.await?.layer { + Some(layer) => Some(layer.await?), + None => None, + }, + ident: ident.to_string().await?, + }) + } + async fn new_chunkable_ref( + source: ResolvedVc>, + target: ResolvedVc>, + chunking_type: ChunkingType, + ) -> Result { + Ok(Self::ChunkableReference { + chunking_type, + source, + source_ident: source.ident().to_string().await?, + target, + target_ident: target.ident().to_string().await?, + }) + } +} +struct SingleModuleGraphBuilderEdge { + // ty: Option, + to: SingleModuleGraphBuilderNode, +} + +/// The chunking type that occurs most often, is handled more efficiently by not creating +/// intermediate SingleModuleGraphBuilderNode::ChunkableReference nodes. +const COMMON_CHUNKING_TYPE: ChunkingType = ChunkingType::ParallelInheritAsync; + +struct SingleModuleGraphBuilder {} +impl Visit for SingleModuleGraphBuilder { + type Edge = SingleModuleGraphBuilderEdge; + type EdgesIntoIter = Vec; + type EdgesFuture = impl Future>; + + fn visit(&mut self, edge: Self::Edge) -> VisitControlFlow { + match edge.to { + SingleModuleGraphBuilderNode::Module { .. } + | SingleModuleGraphBuilderNode::ChunkableReference { .. } => { + VisitControlFlow::Continue(edge.to) + } + SingleModuleGraphBuilderNode::Issues(_) => VisitControlFlow::Skip(edge.to), + } + } + + fn edges(&mut self, node: &SingleModuleGraphBuilderNode) -> Self::EdgesFuture { + // Destructure beforehand to not have to clone the whole node when entering the async block + let (module, chunkable_ref_target) = match node { + SingleModuleGraphBuilderNode::Module { module, .. } => (Some(*module), None), + SingleModuleGraphBuilderNode::ChunkableReference { target, .. } => { + (None, Some(*target)) + } + SingleModuleGraphBuilderNode::Issues(_) => unreachable!(), + }; + async move { + Ok(match (module, chunkable_ref_target) { + (Some(module), None) => { + let refs_cell = primary_chunkable_referenced_modules(*module); + let refs = refs_cell.await?; + // TODO This is currently too slow + // let refs_issues = refs_cell + // .take_collectibles::>() + // .iter() + // .map(|issue| issue.to_resolved()) + // .try_join() + // .await?; + + refs.iter() + .flat_map(|(ty, modules)| { + if matches!(ty, ChunkingType::Traced) { + None + } else { + Some(modules.iter().map(|m| (ty.clone(), *m))) + } + }) + .flatten() + .map(|(ty, target)| async move { + Ok(SingleModuleGraphBuilderEdge { + to: if ty == COMMON_CHUNKING_TYPE { + SingleModuleGraphBuilderNode::new_module(target).await? + } else { + SingleModuleGraphBuilderNode::new_chunkable_ref( + module, target, ty, + ) + .await? + }, + }) + }) + .try_join() + .await? + } + (None, Some(chunkable_ref_target)) => { + vec![SingleModuleGraphBuilderEdge { + to: SingleModuleGraphBuilderNode::new_module(chunkable_ref_target).await?, + }] + } + _ => unreachable!(), + }) + } + } + + fn span(&mut self, node: &SingleModuleGraphBuilderNode) -> tracing::Span { + match node { + SingleModuleGraphBuilderNode::Module { ident, .. } => { + tracing::info_span!("module", name = display(ident)) + } + SingleModuleGraphBuilderNode::Issues(_) => { + tracing::info_span!("issues") + } + SingleModuleGraphBuilderNode::ChunkableReference { + chunking_type, + source_ident, + target_ident, + .. + } => { + tracing::info_span!( + "chunkable reference", + ty = debug(chunking_type), + source = display(source_ident), + target = display(target_ident) + ) + } + } + } +}