From 9eb8143772d510030de65bb0e2d5009a77dd5e2d Mon Sep 17 00:00:00 2001 From: Alexander Lyon Date: Thu, 3 Oct 2024 11:43:17 +0100 Subject: [PATCH] move core node-file-trace logic into turbopack --- Cargo.lock | 13 + turbopack/crates/node-file-trace/Cargo.toml | 3 + turbopack/crates/node-file-trace/readme.md | 18 + turbopack/crates/node-file-trace/src/lib.rs | 386 +++--------------- turbopack/crates/node-file-trace/src/main.rs | 4 +- turbopack/crates/turbopack/src/lib.rs | 2 + .../src/nft_json.rs | 0 turbopack/crates/turbopack/src/trace.rs | 346 ++++++++++++++++ 8 files changed, 435 insertions(+), 337 deletions(-) create mode 100644 turbopack/crates/node-file-trace/readme.md rename turbopack/crates/{node-file-trace => turbopack}/src/nft_json.rs (100%) create mode 100644 turbopack/crates/turbopack/src/trace.rs diff --git a/Cargo.lock b/Cargo.lock index f52f065c698ce..ed885aea5310d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3043,6 +3043,18 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "insta" +version = "1.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6593a41c7a73841868772495db7dc1e8ecab43bb5c0b6da2059246c4b506ab60" +dependencies = [ + "console", + "lazy_static", + "linked-hash-map", + "similar", +] + [[package]] name = "instant" version = "0.1.13" @@ -4190,6 +4202,7 @@ dependencies = [ "anyhow", "clap 4.5.2", "console-subscriber", + "insta", "serde", "serde_json", "tokio", diff --git a/turbopack/crates/node-file-trace/Cargo.toml b/turbopack/crates/node-file-trace/Cargo.toml index febd05f0d04b0..e4b7ddb1d4cb1 100644 --- a/turbopack/crates/node-file-trace/Cargo.toml +++ b/turbopack/crates/node-file-trace/Cargo.toml @@ -45,3 +45,6 @@ turbopack-resolve = { workspace = true } [build-dependencies] turbo-tasks-build = { workspace = true } + +[dev-dependencies] +insta = "1.40.0" diff --git a/turbopack/crates/node-file-trace/readme.md b/turbopack/crates/node-file-trace/readme.md new file mode 100644 index 0000000000000..e12408095b0ab --- /dev/null +++ b/turbopack/crates/node-file-trace/readme.md @@ -0,0 +1,18 @@ +# node-file-trace + +Node file trace (nft) is a binary / library that can be used to precisely determine +which files (including those in `node_modules`) are needed for a given application at +run time. + +## Usage + +There are a few commands available (accessed via the `Args` struct if consuming as a +library). You can see these by running the binary. The binary should be self-documenting. + +```bash +cargo run +``` + +## Externals + +Tracing externals is not currently supported, though support will be added soon. diff --git a/turbopack/crates/node-file-trace/src/lib.rs b/turbopack/crates/node-file-trace/src/lib.rs index 197248c429954..bc85497765d5a 100644 --- a/turbopack/crates/node-file-trace/src/lib.rs +++ b/turbopack/crates/node-file-trace/src/lib.rs @@ -2,53 +2,30 @@ #![feature(arbitrary_self_types)] #![feature(arbitrary_self_types_pointers)] -mod nft_json; - use std::{ - collections::BTreeSet, env::current_dir, future::Future, - path::{Path, PathBuf}, - pin::Pin, sync::Arc, time::{Duration, Instant}, }; -use anyhow::{anyhow, Context, Result}; +use anyhow::Result; #[cfg(feature = "cli")] use clap::Parser; #[cfg(feature = "node-api")] use serde::Deserialize; #[cfg(feature = "node-api")] use serde::Serialize; -use tokio::sync::mpsc::channel; +use tokio::sync::mpsc::{channel, Receiver, Sender}; use turbo_tasks::{ backend::Backend, util::FormatDuration, RcStr, ReadConsistency, TaskId, TransientInstance, - TransientValue, TurboTasks, UpdateInfo, Value, Vc, -}; -use turbo_tasks_fs::{ - glob::Glob, DirectoryEntry, DiskFileSystem, FileSystem, FileSystemPath, ReadGlobResult, -}; -use turbopack::{ - emit_asset, emit_with_completion, module_options::ModuleOptionsContext, rebase::RebasedAsset, - ModuleAssetContext, + TransientValue, TurboTasks, UpdateInfo, Vc, }; +use turbopack::module_options::ModuleOptionsContext; use turbopack_cli_utils::issue::{ConsoleUi, IssueSeverityCliOption, LogOptions}; -use turbopack_core::{ - compile_time_info::CompileTimeInfo, - context::AssetContext, - environment::{Environment, ExecutionEnvironment, NodeJsEnvironment}, - file_source::FileSource, - issue::{IssueDescriptionExt, IssueReporter, IssueSeverity}, - module::{Module, Modules}, - output::OutputAsset, - reference::all_modules_and_affecting_sources, - resolve::options::{ImportMapping, ResolvedMap}, -}; +use turbopack_core::issue::{IssueDescriptionExt, IssueReporter, IssueSeverity}; use turbopack_resolve::resolve_options_context::ResolveOptionsContext; -use crate::nft_json::NftJsonAsset; - #[cfg(feature = "persistent_cache")] #[cfg_attr(feature = "cli", derive(clap::Args))] #[cfg_attr( @@ -81,10 +58,13 @@ pub struct CacheArgs {} derive(Serialize, Deserialize), serde(rename_all = "camelCase") )] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct CommonArgs { + /// A list of input files to perform a trace on input: Vec, + /// The folder to consider as the root when performing the trace. All traced files must reside + /// in this directory #[cfg_attr(feature = "cli", clap(short, long))] #[cfg_attr(feature = "node-api", serde(default))] context_directory: Option, @@ -126,7 +106,13 @@ pub struct CommonArgs { /// MB. #[cfg_attr(feature = "cli", clap(long))] #[cfg_attr(feature = "serializable", serde(default))] - pub memory_limit: Option, + memory_limit: Option, +} + +impl CommonArgs { + pub fn memory_limit(&self) -> usize { + self.memory_limit.unwrap_or(usize::MAX) + } } #[cfg_attr(feature = "cli", derive(Parser))] @@ -138,21 +124,21 @@ pub struct CommonArgs { )] #[derive(Debug)] pub enum Args { - // Print all files that the input files reference + /// Print all files that the input files reference Print { #[cfg_attr(feature = "cli", clap(flatten))] #[cfg_attr(feature = "node-api", serde(flatten))] common: CommonArgs, }, - // Adds a *.nft.json file next to each input file which lists the referenced files + /// Adds a *.nft.json file next to each input file which lists the referenced files Annotate { #[cfg_attr(feature = "cli", clap(flatten))] #[cfg_attr(feature = "node-api", serde(flatten))] common: CommonArgs, }, - // Copy input files and all referenced files to the output directory + /// Copy input files and all referenced files to the output directory Build { #[cfg_attr(feature = "cli", clap(flatten))] #[cfg_attr(feature = "node-api", serde(flatten))] @@ -162,13 +148,6 @@ pub enum Args { #[cfg_attr(feature = "node-api", serde(default = "default_output_directory"))] output_directory: String, }, - - // Print total size of input and referenced files - Size { - #[cfg_attr(feature = "cli", clap(flatten))] - #[cfg_attr(feature = "node-api", serde(flatten))] - common: CommonArgs, - }, } #[cfg(feature = "node-api")] @@ -176,138 +155,23 @@ fn default_output_directory() -> String { "dist".to_string() } +type OutputPair = Option<(Sender, Receiver)>; + impl Args { pub fn common(&self) -> &CommonArgs { match self { Args::Print { common, .. } | Args::Annotate { common, .. } - | Args::Build { common, .. } - | Args::Size { common, .. } => common, + | Args::Build { common, .. } => common, } } -} - -async fn create_fs(name: &str, root: &str, watch: bool) -> Result>> { - let fs = DiskFileSystem::new(name.into(), root.into(), vec![]); - if watch { - fs.await?.start_watching(None).await?; - } else { - fs.await?.invalidate_with_reason(); - } - Ok(Vc::upcast(fs)) -} -async fn add_glob_results( - asset_context: Vc>, - result: Vc, - list: &mut Vec>>, -) -> Result<()> { - let result = result.await?; - for entry in result.results.values() { - if let DirectoryEntry::File(path) = entry { - let source = Vc::upcast(FileSource::new(**path)); - let module = asset_context - .process( - source, - Value::new(turbopack_core::reference_type::ReferenceType::Undefined), - ) - .module(); - list.push(module); - } - } - for result in result.inner.values() { - fn recurse<'a>( - asset_context: Vc>, - result: Vc, - list: &'a mut Vec>>, - ) -> Pin> + Send + 'a>> { - Box::pin(add_glob_results(asset_context, result, list)) + fn output_pair(&self) -> OutputPair> { + match self { + Args::Print { .. } | Args::Annotate { .. } => Some(channel(1)), + _ => None, } - // Boxing for async recursion - recurse(asset_context, **result, list).await?; - } - Ok(()) -} - -#[turbo_tasks::function] -async fn input_to_modules( - fs: Vc>, - input: Vec, - exact: bool, - process_cwd: Option, - context_directory: RcStr, - module_options: TransientInstance, - resolve_options: TransientInstance, -) -> Result> { - let root = fs.root(); - let process_cwd = process_cwd - .clone() - .map(|p| format!("/ROOT{}", p.trim_start_matches(&*context_directory)).into()); - - let asset_context: Vc> = Vc::upcast(create_module_asset( - root, - process_cwd, - module_options, - resolve_options, - )); - - let mut list = Vec::new(); - for input in input { - if exact { - let source = Vc::upcast(FileSource::new(root.join(input))); - let module = asset_context - .process( - source, - Value::new(turbopack_core::reference_type::ReferenceType::Undefined), - ) - .module(); - list.push(module); - } else { - let glob = Glob::new(input); - add_glob_results(asset_context, root.read_glob(glob, false), &mut list).await?; - }; - } - Ok(Vc::cell(list)) -} - -fn process_context(dir: &Path, context_directory: Option<&String>) -> Result { - let mut context_directory = PathBuf::from(context_directory.map_or(".", |s| s)); - if !context_directory.is_absolute() { - context_directory = dir.join(context_directory); - } - // context = context.canonicalize().unwrap(); - Ok(context_directory - .to_str() - .ok_or_else(|| anyhow!("context directory contains invalid characters")) - .unwrap() - .to_string()) -} - -fn make_relative_path(dir: &Path, context_directory: &str, input: &str) -> Result { - let mut input = PathBuf::from(input); - if !input.is_absolute() { - input = dir.join(input); } - // input = input.canonicalize()?; - let input = input.strip_prefix(context_directory).with_context(|| { - anyhow!( - "{} is not part of the context directory {}", - input.display(), - context_directory - ) - })?; - Ok(input - .to_str() - .ok_or_else(|| anyhow!("input contains invalid characters"))? - .replace('\\', "/") - .into()) -} - -fn process_input(dir: &Path, context_directory: &str, input: &[String]) -> Result> { - input - .iter() - .map(|input| make_relative_path(dir, context_directory, input)) - .collect() } pub async fn start( @@ -393,9 +257,7 @@ async fn run>( result } }; - let has_return_value = - matches!(&*args, Args::Annotate { .. }) || matches!(&*args, Args::Print { .. }); - let (sender, mut receiver) = channel(1); + let dir = current_dir().unwrap(); let module_options = TransientInstance::new(module_options.unwrap_or_default()); let resolve_options = TransientInstance::new(resolve_options.unwrap_or_default()); @@ -406,6 +268,9 @@ async fn run>( log_detail, log_level: log_level.map_or_else(|| IssueSeverity::Error, |l| l.0), }); + + let (sender, receiver) = args.output_pair().unzip(); + let task = tt.spawn_root_task(move || { let dir = dir.clone(); let args = args.clone(); @@ -414,9 +279,23 @@ async fn run>( let resolve_options = resolve_options.clone(); let log_options = log_options.clone(); Box::pin(async move { - let output = main_operation( + let common = args.common(); + let output = turbopack::trace::run_node_file_trace( TransientValue::new(dir.clone()), - TransientInstance::new(args.clone()), + TransientValue::new(match args.as_ref() { + Args::Print { .. } => turbopack::trace::Operation::Print, + Args::Annotate { .. } => turbopack::trace::Operation::Annotate, + Args::Build { + output_directory, .. + } => turbopack::trace::Operation::Build(output_directory.clone()), + }), + TransientInstance::new(Arc::new(turbopack::trace::Args { + context_directory: common.context_directory.clone(), + input: common.input.clone(), + exact: common.exact, + process_cwd: common.process_cwd.clone(), + watch: common.watch, + })), module_options, resolve_options, ); @@ -434,182 +313,21 @@ async fn run>( ) .await?; - if has_return_value { + if let Some(sender) = sender { let output_read_ref = output.await?; let output_iter = output_read_ref.iter().cloned(); sender.send(output_iter.collect::>()).await?; drop(sender); } + Ok::, _>(Default::default()) }) }); - finish(tt, task).await?; - let output = if has_return_value { - receiver.try_recv()? - } else { - Vec::new() - }; - Ok(output) -} - -#[turbo_tasks::function] -async fn main_operation( - current_dir: TransientValue, - args: TransientInstance>, - module_options: TransientInstance, - resolve_options: TransientInstance, -) -> Result>> { - let dir = current_dir.into_value(); - let args = &*args; - let &CommonArgs { - ref input, - watch, - exact, - ref context_directory, - ref process_cwd, - .. - } = args.common(); - let context_directory: RcStr = process_context(&dir, context_directory.as_ref()) - .unwrap() - .into(); - let fs = create_fs("context directory", &context_directory, watch).await?; - let process_cwd = process_cwd.clone().map(RcStr::from); - - match **args { - Args::Print { common: _ } => { - let input = process_input(&dir, &context_directory, input).unwrap(); - let mut result = BTreeSet::new(); - let modules = input_to_modules( - fs, - input, - exact, - process_cwd.clone(), - context_directory, - module_options, - resolve_options, - ) - .await?; - for module in modules.iter() { - let set = all_modules_and_affecting_sources(*module) - .issue_file_path(module.ident().path(), "gathering list of assets") - .await?; - for asset in set.await?.iter() { - let path = asset.ident().path().await?; - result.insert(RcStr::from(&*path.path)); - } - } - - return Ok(Vc::cell(result.into_iter().collect::>())); - } - Args::Annotate { common: _ } => { - let input = process_input(&dir, &context_directory, input).unwrap(); - let mut output_nft_assets = Vec::new(); - let mut emits = Vec::new(); - for module in input_to_modules( - fs, - input, - exact, - process_cwd.clone(), - context_directory, - module_options, - resolve_options, - ) - .await? - .iter() - { - let nft_asset = NftJsonAsset::new(*module); - let path = nft_asset.ident().path().await?.path.clone(); - output_nft_assets.push(path); - emits.push(emit_asset(Vc::upcast(nft_asset))); - } - // Wait for all files to be emitted - for emit in emits { - emit.await?; - } - return Ok(Vc::cell(output_nft_assets)); - } - Args::Build { - ref output_directory, - common: _, - } => { - let output = process_context(&dir, Some(output_directory)).unwrap(); - let input = process_input(&dir, &context_directory, input).unwrap(); - let out_fs = create_fs("output directory", &output, watch).await?; - let input_dir = fs.root(); - let output_dir = out_fs.root(); - let mut emits = Vec::new(); - for module in input_to_modules( - fs, - input, - exact, - process_cwd.clone(), - context_directory, - module_options, - resolve_options, - ) - .await? - .iter() - { - let rebased = Vc::upcast(RebasedAsset::new(*module, input_dir, output_dir)); - emits.push(emit_with_completion(rebased, output_dir)); - } - // Wait for all files to be emitted - for emit in emits { - emit.await?; - } - } - Args::Size { common: _ } => todo!(), - } - Ok(Vc::cell(Vec::new())) -} - -#[turbo_tasks::function] -async fn create_module_asset( - root: Vc, - process_cwd: Option, - module_options: TransientInstance, - resolve_options: TransientInstance, -) -> Vc { - let env = Environment::new(Value::new(ExecutionEnvironment::NodeJsLambda( - NodeJsEnvironment { - cwd: Vc::cell(process_cwd), - ..Default::default() - } - .into(), - ))); - let compile_time_info = CompileTimeInfo::builder(env).cell(); - let glob_mappings = vec![ - ( - root, - Glob::new("**/*/next/dist/server/next.js".into()), - ImportMapping::Ignore.into(), - ), - ( - root, - Glob::new("**/*/next/dist/bin/next".into()), - ImportMapping::Ignore.into(), - ), - ]; - let mut resolve_options = ResolveOptionsContext::clone(&*resolve_options); - if resolve_options.emulate_environment.is_none() { - resolve_options.emulate_environment = Some(env); - } - if resolve_options.resolved_map.is_none() { - resolve_options.resolved_map = Some( - ResolvedMap { - by_glob: glob_mappings, - } - .cell(), - ); - } - ModuleAssetContext::new( - Default::default(), - compile_time_info, - ModuleOptionsContext::clone(&*module_options).cell(), - resolve_options.cell(), - Vc::cell("node_file_trace".into()), - ) + finish(tt, task).await?; + receiver + .map(|mut r| Ok(r.try_recv()?)) + .unwrap_or(Ok(Vec::new())) } fn register() { diff --git a/turbopack/crates/node-file-trace/src/main.rs b/turbopack/crates/node-file-trace/src/main.rs index 12f98da9c6774..544d4bc797f8f 100644 --- a/turbopack/crates/node-file-trace/src/main.rs +++ b/turbopack/crates/node-file-trace/src/main.rs @@ -18,9 +18,7 @@ async fn main() -> Result<()> { console_subscriber::init(); let args = Arc::new(Args::parse()); let should_print = matches!(&*args, Args::Print { .. }); - let turbo_tasks = TurboTasks::new(MemoryBackend::new( - args.common().memory_limit.unwrap_or(usize::MAX), - )); + let turbo_tasks = TurboTasks::new(MemoryBackend::new(args.common().memory_limit())); let result = start(args, turbo_tasks, None, None).await?; if should_print { for file in result.iter() { diff --git a/turbopack/crates/turbopack/src/lib.rs b/turbopack/crates/turbopack/src/lib.rs index 06191ff7d7f32..bd1d044bdfa8a 100644 --- a/turbopack/crates/turbopack/src/lib.rs +++ b/turbopack/crates/turbopack/src/lib.rs @@ -10,7 +10,9 @@ pub mod evaluate_context; mod graph; pub mod module_options; +mod nft_json; pub mod rebase; +pub mod trace; pub mod transition; pub(crate) mod unsupported_sass; diff --git a/turbopack/crates/node-file-trace/src/nft_json.rs b/turbopack/crates/turbopack/src/nft_json.rs similarity index 100% rename from turbopack/crates/node-file-trace/src/nft_json.rs rename to turbopack/crates/turbopack/src/nft_json.rs diff --git a/turbopack/crates/turbopack/src/trace.rs b/turbopack/crates/turbopack/src/trace.rs new file mode 100644 index 0000000000000..906beeef7ec55 --- /dev/null +++ b/turbopack/crates/turbopack/src/trace.rs @@ -0,0 +1,346 @@ +use std::{ + collections::BTreeSet, + future::Future, + path::{Path, PathBuf}, + pin::Pin, + sync::Arc, +}; + +use anyhow::{anyhow, Context, Result}; +use turbo_tasks::{RcStr, TransientInstance, TransientValue, Value, Vc}; +use turbo_tasks_fs::{ + glob::Glob, DirectoryEntry, DiskFileSystem, FileSystem, FileSystemPath, ReadGlobResult, +}; +use turbopack_core::{ + compile_time_info::CompileTimeInfo, + context::AssetContext, + environment::{Environment, ExecutionEnvironment, NodeJsEnvironment}, + file_source::FileSource, + issue::IssueDescriptionExt, + module::{Module, Modules}, + output::OutputAsset, + reference::all_modules_and_affecting_sources, + resolve::options::{ImportMapping, ResolvedMap}, +}; +use turbopack_resolve::resolve_options_context::ResolveOptionsContext; + +use crate::{ + emit_asset, emit_with_completion, module_options::ModuleOptionsContext, rebase::RebasedAsset, + ModuleAssetContext, +}; + +#[derive(Debug, Clone, Hash, Eq, PartialEq)] +pub enum Operation { + Print, + Annotate, + Build(String), +} + +pub struct Args { + /// A list of input files to perform a trace on + pub input: Vec, + + /// The folder to consider as the root when performing the trace. All traced files must reside + /// in this directory + pub context_directory: Option, + + pub process_cwd: Option, + + /// Whether to watch the context directory for changes + pub watch: bool, + + /// Whether to skip the glob logic + /// assume the provided input is not glob even if it contains `*` and `[]` + pub exact: bool, +} + +#[turbo_tasks::function] +pub async fn run_node_file_trace( + current_dir: TransientValue, + operation: TransientValue, + args: TransientInstance>, + module_options: TransientInstance, + resolve_options: TransientInstance, +) -> Result>> { + let dir = current_dir.into_value(); + let &Args { + ref input, + watch, + exact, + ref context_directory, + ref process_cwd, + .. + } = &**args; + let context_directory: RcStr = process_context(&dir, context_directory.as_ref()) + .unwrap() + .into(); + let fs = create_fs("context directory", &context_directory, watch).await?; + let process_cwd = process_cwd.clone().map(RcStr::from); + + match &*operation { + Operation::Print => { + let input = get_inputs(&dir, &context_directory, input).unwrap(); + + let mut result = BTreeSet::new(); + let modules = input_to_modules( + fs, + input, + exact, + process_cwd.clone(), + context_directory, + module_options, + resolve_options, + ) + .await?; + for module in modules.iter() { + let set = all_modules_and_affecting_sources(*module) + .issue_file_path(module.ident().path(), "gathering list of assets") + .await?; + for asset in set.await?.iter() { + let path = asset.ident().path().await?; + result.insert(RcStr::from(&*path.path)); + } + } + + return Ok(Vc::cell(result.into_iter().collect::>())); + } + Operation::Annotate => { + let input = get_inputs(&dir, &context_directory, input).unwrap(); + + let mut output_nft_assets = Vec::new(); + let mut emits = Vec::new(); + for module in input_to_modules( + fs, + input, + exact, + process_cwd.clone(), + context_directory, + module_options, + resolve_options, + ) + .await? + .iter() + { + let nft_asset = crate::nft_json::NftJsonAsset::new(*module); + let path = nft_asset.ident().path().await?.path.clone(); + output_nft_assets.push(path); + emits.push(emit_asset(Vc::upcast(nft_asset))); + } + // Wait for all files to be emitted + for emit in emits { + emit.await?; + } + return Ok(Vc::cell(output_nft_assets)); + } + Operation::Build(output_directory) => { + let output = process_context(&dir, Some(output_directory)).unwrap(); + let input = get_inputs(&dir, &context_directory, input).unwrap(); + let out_fs = create_fs("output directory", &output, watch).await?; + let input_dir = fs.root(); + let output_dir = out_fs.root(); + let mut emits = Vec::new(); + for module in input_to_modules( + fs, + input, + exact, + process_cwd.clone(), + context_directory, + module_options, + resolve_options, + ) + .await? + .iter() + { + let rebased = Vc::upcast(RebasedAsset::new(*module, input_dir, output_dir)); + emits.push(emit_with_completion(rebased, output_dir)); + } + // Wait for all files to be emitted + for emit in emits { + emit.await?; + } + } + } + Ok(Vc::cell(Vec::new())) +} + +async fn create_fs(name: &str, root: &str, watch: bool) -> Result>> { + let fs = DiskFileSystem::new(name.into(), root.into(), vec![]); + if watch { + fs.await?.start_watching(None).await?; + } else { + fs.await?.invalidate_with_reason(); + } + Ok(Vc::upcast(fs)) +} + +fn process_context(dir: &Path, context_directory: Option<&String>) -> Result { + let mut context_directory = PathBuf::from(context_directory.map(|s| s.as_str()).unwrap_or(".")); + if !context_directory.is_absolute() { + context_directory = dir.join(context_directory); + } + // context = context.canonicalize().unwrap(); + Ok(context_directory + .to_str() + .ok_or_else(|| anyhow!("context directory contains invalid characters")) + .unwrap() + .to_string()) +} + +fn get_inputs(dir: &Path, context_directory: &str, input: &[String]) -> Result> { + let input = process_input(dir, context_directory, input); + let (ok, errs): (Vec<_>, Vec<_>) = input.into_iter().partition(Result::is_ok); + if !errs.is_empty() { + return Err(anyhow!("Input files could not be found {:?}", errs)); + } + let input = ok.into_iter().map(|i| i.unwrap()).collect(); + Ok(input) +} + +fn process_input(dir: &Path, context_directory: &str, input: &[String]) -> Vec> { + input + .iter() + .map(|input| make_relative_path(dir, context_directory, input)) + .collect() +} + +fn make_relative_path(dir: &Path, context_directory: &str, input: &str) -> Result { + let mut input = PathBuf::from(input); + if !input.is_absolute() { + input = dir.join(input); + } + // input = input.canonicalize()?; + let input = input.strip_prefix(context_directory).with_context(|| { + anyhow!( + "{} is not part of the context directory {}", + input.display(), + context_directory + ) + })?; + Ok(input + .to_str() + .ok_or_else(|| anyhow!("input contains invalid characters"))? + .replace('\\', "/") + .into()) +} + +#[turbo_tasks::function] +async fn input_to_modules( + fs: Vc>, + input: Vec, + exact: bool, + process_cwd: Option, + context_directory: RcStr, + module_options: TransientInstance, + resolve_options: TransientInstance, +) -> Result> { + let root = fs.root(); + let process_cwd = process_cwd + .clone() + .map(|p| format!("/ROOT{}", p.trim_start_matches(&*context_directory)).into()); + + let asset_context: Vc> = Vc::upcast(create_module_asset( + root, + process_cwd, + module_options, + resolve_options, + )); + + let mut list = Vec::new(); + for input in input { + if exact { + let source = Vc::upcast(FileSource::new(root.join(input))); + let module = asset_context + .process( + source, + Value::new(turbopack_core::reference_type::ReferenceType::Undefined), + ) + .module(); + list.push(module); + } else { + let glob = Glob::new(input); + add_glob_results(asset_context, root.read_glob(glob, false), &mut list).await?; + }; + } + Ok(Vc::cell(list)) +} + +async fn add_glob_results( + asset_context: Vc>, + result: Vc, + list: &mut Vec>>, +) -> Result<()> { + let result = result.await?; + for entry in result.results.values() { + if let DirectoryEntry::File(path) = entry { + let source = Vc::upcast(FileSource::new(**path)); + let module = asset_context + .process( + source, + Value::new(turbopack_core::reference_type::ReferenceType::Undefined), + ) + .module(); + list.push(module); + } + } + for result in result.inner.values() { + fn recurse<'a>( + asset_context: Vc>, + result: Vc, + list: &'a mut Vec>>, + ) -> Pin> + Send + 'a>> { + Box::pin(add_glob_results(asset_context, result, list)) + } + // Boxing for async recursion + recurse(asset_context, **result, list).await?; + } + Ok(()) +} + +#[turbo_tasks::function] +async fn create_module_asset( + root: Vc, + process_cwd: Option, + module_options: TransientInstance, + resolve_options: TransientInstance, +) -> Vc { + let env = Environment::new(Value::new(ExecutionEnvironment::NodeJsLambda( + NodeJsEnvironment { + cwd: Vc::cell(process_cwd), + ..Default::default() + } + .into(), + ))); + let compile_time_info = CompileTimeInfo::builder(env).cell(); + let glob_mappings = vec![ + ( + root, + Glob::new("**/*/next/dist/server/next.js".into()), + ImportMapping::Ignore.into(), + ), + ( + root, + Glob::new("**/*/next/dist/bin/next".into()), + ImportMapping::Ignore.into(), + ), + ]; + let mut resolve_options = ResolveOptionsContext::clone(&*resolve_options); + if resolve_options.emulate_environment.is_none() { + resolve_options.emulate_environment = Some(env); + } + if resolve_options.resolved_map.is_none() { + resolve_options.resolved_map = Some( + ResolvedMap { + by_glob: glob_mappings, + } + .cell(), + ); + } + + ModuleAssetContext::new( + Default::default(), + compile_time_info, + ModuleOptionsContext::clone(&*module_options).cell(), + resolve_options.cell(), + Vc::cell("node_file_trace".into()), + ) +}