From 614b361db15d4290cbf145f26950afcfa219d450 Mon Sep 17 00:00:00 2001 From: xusd320 Date: Fri, 14 Nov 2025 10:12:27 +0800 Subject: [PATCH 01/17] chore(turbopack-node): remove some outdated codes --- turbopack/crates/turbopack-node/src/lib.rs | 126 +----------------- turbopack/crates/turbopack-node/src/pool.rs | 52 -------- .../turbopack-node/src/source_map/mod.rs | 39 +----- 3 files changed, 5 insertions(+), 212 deletions(-) diff --git a/turbopack/crates/turbopack-node/src/lib.rs b/turbopack/crates/turbopack-node/src/lib.rs index ba0fb54f6b68e0..a86c9b5d1baa3e 100644 --- a/turbopack/crates/turbopack-node/src/lib.rs +++ b/turbopack/crates/turbopack-node/src/lib.rs @@ -2,30 +2,22 @@ #![feature(arbitrary_self_types)] #![feature(arbitrary_self_types_pointers)] -use std::{iter::once, thread::available_parallelism}; +use std::iter::once; -use anyhow::{Result, bail}; +use anyhow::Result; use rustc_hash::FxHashMap; -use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ FxIndexSet, ResolvedVc, TryJoinIterExt, Vc, graph::{AdjacencyMap, GraphTraversal}, }; -use turbo_tasks_env::ProcessEnv; -use turbo_tasks_fs::{File, FileSystemPath, to_sys_path}; +use turbo_tasks_fs::{File, FileSystemPath}; use turbopack_core::{ asset::{Asset, AssetContent}, - changed::content_changed, - chunk::{ChunkingContext, ChunkingContextExt, EvaluatableAsset, EvaluatableAssets}, - module::Module, - module_graph::{ModuleGraph, chunk_group_info::ChunkGroupEntry}, - output::{OutputAsset, OutputAssets, OutputAssetsSet}, + output::{OutputAsset, OutputAssetsSet}, source_map::GenerateSourceMap, virtual_output::VirtualOutputAsset, }; -use self::pool::NodeJsPool; - pub mod debug; pub mod embed_js; pub mod evaluate; @@ -100,26 +92,6 @@ async fn internal_assets_for_source_mapping( Ok(Vc::cell(internal_assets_for_source_mapping)) } -/// Returns a set of "external" assets on the boundary of the "internal" -/// subgraph -#[turbo_tasks::function] -pub async fn external_asset_entrypoints( - module: Vc>, - runtime_entries: Vc, - chunking_context: Vc>, - intermediate_output_path: FileSystemPath, -) -> Result> { - Ok(*separate_assets_operation( - get_intermediate_asset(chunking_context, module, runtime_entries) - .to_resolved() - .await?, - intermediate_output_path, - ) - .read_strongly_consistent() - .await? - .external_asset_entrypoints) -} - /// Splits the asset graph into "internal" assets and boundaries to "external" /// assets. #[turbo_tasks::function(operation)] @@ -199,93 +171,3 @@ fn emit_package_json(dir: FileSystemPath) -> Result> { dir, )) } - -/// Creates a node.js renderer pool for an entrypoint. -#[turbo_tasks::function(operation)] -pub async fn get_renderer_pool_operation( - cwd: FileSystemPath, - env: ResolvedVc>, - intermediate_asset: ResolvedVc>, - intermediate_output_path: FileSystemPath, - output_root: FileSystemPath, - project_dir: FileSystemPath, - debug: bool, -) -> Result> { - emit_package_json(intermediate_output_path.clone())?.await?; - - emit(*intermediate_asset, output_root.clone()) - .as_side_effect() - .await?; - let assets_for_source_mapping = - internal_assets_for_source_mapping(*intermediate_asset, output_root.clone()); - - let entrypoint = intermediate_asset.path().owned().await?; - - let Some(cwd) = to_sys_path(cwd.clone()).await? else { - bail!( - "can only render from a disk filesystem, but `cwd = {}`", - cwd.value_to_string().await? - ); - }; - let Some(entrypoint) = to_sys_path(entrypoint.clone()).await? else { - bail!( - "can only render from a disk filesystem, but `entrypoint = {}`", - entrypoint.value_to_string().await? - ); - }; - // Invalidate pool when code content changes - content_changed(*ResolvedVc::upcast(intermediate_asset)).await?; - - Ok(NodeJsPool::new( - cwd, - entrypoint, - env.read_all() - .await? - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), - assets_for_source_mapping.to_resolved().await?, - output_root, - project_dir, - available_parallelism().map_or(1, |v| v.get()), - debug, - ) - .cell()) -} - -/// Converts a module graph into node.js executable assets -#[turbo_tasks::function] -pub async fn get_intermediate_asset( - chunking_context: Vc>, - main_entry: ResolvedVc>, - other_entries: Vc, -) -> Result>> { - Ok(chunking_context.root_entry_chunk_group_asset( - chunking_context - .chunk_path(None, main_entry.ident(), None, rcstr!(".js")) - .owned() - .await?, - other_entries.with_entry(*main_entry), - ModuleGraph::from_modules( - Vc::cell(vec![ChunkGroupEntry::Entry( - other_entries - .await? - .into_iter() - .copied() - .chain(std::iter::once(main_entry)) - .map(ResolvedVc::upcast) - .collect(), - )]), - false, - ), - OutputAssets::empty(), - OutputAssets::empty(), - )) -} - -#[derive(Clone, Debug)] -#[turbo_tasks::value(shared)] -pub struct ResponseHeaders { - pub status: u16, - pub headers: Vec<(RcStr, RcStr)>, -} diff --git a/turbopack/crates/turbopack-node/src/pool.rs b/turbopack/crates/turbopack-node/src/pool.rs index 78fce6f7b58b9c..1d4532868d66e4 100644 --- a/turbopack/crates/turbopack-node/src/pool.rs +++ b/turbopack/crates/turbopack-node/src/pool.rs @@ -1,5 +1,4 @@ use std::{ - borrow::Cow, cmp::max, fmt::{Debug, Display}, future::Future, @@ -71,9 +70,6 @@ impl FormattingMode { struct NodeJsPoolProcess { child: Option, connection: TcpStream, - assets_for_source_mapping: ResolvedVc, - assets_root: FileSystemPath, - project_dir: FileSystemPath, stdout_handler: OutputStreamHandler, stderr_handler: OutputStreamHandler, debug: bool, @@ -107,39 +103,6 @@ impl PartialEq for NodeJsPoolProcess { } } -impl NodeJsPoolProcess { - pub async fn apply_source_mapping<'a>( - &self, - text: &'a str, - formatting_mode: FormattingMode, - ) -> Result> { - let text = unmangle_identifiers(text, |content| formatting_mode.magic_identifier(content)); - match text { - Cow::Borrowed(text) => { - apply_source_mapping( - text, - *self.assets_for_source_mapping, - self.assets_root.clone(), - self.project_dir.clone(), - formatting_mode, - ) - .await - } - Cow::Owned(ref text) => { - let cow = apply_source_mapping( - text, - *self.assets_for_source_mapping, - self.assets_root.clone(), - self.project_dir.clone(), - formatting_mode, - ) - .await?; - Ok(Cow::Owned(cow.into_owned())) - } - } - } -} - const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Clone, PartialEq, Eq, Hash)] @@ -456,9 +419,6 @@ impl NodeJsPoolProcess { let mut process = Self { child: Some(child), connection, - assets_for_source_mapping, - assets_root: assets_root.clone(), - project_dir: project_dir.clone(), stdout_handler, stderr_handler, debug, @@ -958,18 +918,6 @@ impl NodeJsOperation { self.allow_process_reuse = false; } } - - pub async fn apply_source_mapping<'a>( - &self, - text: &'a str, - formatting_mode: FormattingMode, - ) -> Result> { - if let Some(process) = self.process.as_ref() { - process.apply_source_mapping(text, formatting_mode).await - } else { - Ok(Cow::Borrowed(text)) - } - } } impl Drop for NodeJsOperation { diff --git a/turbopack/crates/turbopack-node/src/source_map/mod.rs b/turbopack/crates/turbopack-node/src/source_map/mod.rs index 58b2584b095a8a..eeaf2e6cab1f68 100644 --- a/turbopack/crates/turbopack-node/src/source_map/mod.rs +++ b/turbopack/crates/turbopack-node/src/source_map/mod.rs @@ -9,7 +9,6 @@ use const_format::concatcp; use once_cell::sync::Lazy; use regex::Regex; pub use trace::{StackFrame, TraceResult, trace_source_map}; -use tracing::{Level, instrument}; use turbo_tasks::{ReadRef, Vc}; use turbo_tasks_fs::{ FileLinesContent, FileSystemPath, source_context::get_source_context, to_sys_path, @@ -17,12 +16,11 @@ use turbo_tasks_fs::{ use turbopack_cli_utils::source_context::format_source_context_lines; use turbopack_core::{ PROJECT_FILESYSTEM_NAME, SOURCE_URL_PROTOCOL, - output::OutputAsset, source_map::{GenerateSourceMap, SourceMap}, }; use turbopack_ecmascript::magic_identifier::unmangle_identifiers; -use crate::{AssetsForSourceMapping, internal_assets_for_source_mapping, pool::FormattingMode}; +use crate::{AssetsForSourceMapping, pool::FormattingMode}; pub mod trace; @@ -332,38 +330,3 @@ impl StructuredError { Ok(message) } } - -pub async fn trace_stack( - error: StructuredError, - root_asset: Vc>, - output_path: FileSystemPath, - project_dir: FileSystemPath, -) -> Result { - let assets_for_source_mapping = - internal_assets_for_source_mapping(root_asset, output_path.clone()); - - trace_stack_with_source_mapping_assets( - error, - assets_for_source_mapping, - output_path, - project_dir, - ) - .await -} - -#[instrument(level = Level::TRACE, skip_all)] -pub async fn trace_stack_with_source_mapping_assets( - error: StructuredError, - assets_for_source_mapping: Vc, - output_path: FileSystemPath, - project_dir: FileSystemPath, -) -> Result { - error - .print( - assets_for_source_mapping, - output_path, - project_dir, - FormattingMode::Plain, - ) - .await -} From cd3c10418ab7e45b29f4edcd52ba53a264591c45 Mon Sep 17 00:00:00 2001 From: xusd320 Date: Tue, 18 Nov 2025 18:26:38 +0800 Subject: [PATCH 02/17] refactor(turbopack-node): support execution by napi and worker_threads --- Cargo.lock | 28 +++ Cargo.toml | 2 + crates/napi/Cargo.toml | 19 +- crates/next-api/src/project.rs | 11 +- .../next/src/build/swc/generated-native.d.ts | 12 ++ turbopack/crates/turbopack-node/Cargo.toml | 12 +- turbopack/crates/turbopack-node/build.rs | 6 + .../crates/turbopack-node/src/evaluate.rs | 146 ++++++++++----- turbopack/crates/turbopack-node/src/format.rs | 36 ++++ turbopack/crates/turbopack-node/src/lib.rs | 7 +- .../src/{pool.rs => process_pool.rs} | 174 ++++++++---------- .../turbopack-node/src/source_map/mod.rs | 2 +- .../turbopack-node/src/transforms/webpack.rs | 16 +- .../turbopack-node/src/worker_pool/mod.rs | 79 ++++++++ .../src/worker_pool/operation.rs | 141 ++++++++++++++ .../src/worker_pool/worker_thread.rs | 100 ++++++++++ 16 files changed, 620 insertions(+), 171 deletions(-) create mode 100644 turbopack/crates/turbopack-node/build.rs create mode 100644 turbopack/crates/turbopack-node/src/format.rs rename turbopack/crates/turbopack-node/src/{pool.rs => process_pool.rs} (91%) create mode 100644 turbopack/crates/turbopack-node/src/worker_pool/mod.rs create mode 100644 turbopack/crates/turbopack-node/src/worker_pool/operation.rs create mode 100644 turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs diff --git a/Cargo.lock b/Cargo.lock index 44fc569dff5359..b04885d3b6a9f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -328,6 +328,18 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" version = "0.3.15" @@ -2214,6 +2226,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -9880,14 +9902,19 @@ name = "turbopack-node" version = "0.1.0" dependencies = [ "anyhow", + "async-channel", "async-stream", "async-trait", "base64 0.21.4", "const_format", + "dashmap 6.1.0", "either", "futures", "futures-retry", "indoc", + "napi", + "napi-build", + "napi-derive", "once_cell", "owo-colors", "parking_lot", @@ -9907,6 +9934,7 @@ dependencies = [ "turbopack-core", "turbopack-ecmascript", "turbopack-resolve", + "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e685d439e41fec..ce489135bf852f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -423,6 +423,8 @@ napi = { version = "2", default-features = false, features = [ "napi5", "compat-mode", ] } +napi-derive = "2" +napi-build = "2" notify = "8.1.0" once_cell = "1.17.1" owo-colors = "4.2.2" diff --git a/crates/napi/Cargo.toml b/crates/napi/Cargo.toml index a0a3ddec1f0397..76f049f083ac69 100644 --- a/crates/napi/Cargo.toml +++ b/crates/napi/Cargo.toml @@ -46,12 +46,12 @@ workspace = true [package.metadata.cargo-shear] ignored = [ - # we need to set features on these packages when building for WASM, but we don't directly use them - "getrandom", - "iana-time-zone", - # the plugins feature needs to set a feature on this transitively depended-on package, we never - # directly import it - "turbopack-ecmascript-plugins", + # we need to set features on these packages when building for WASM, but we don't directly use them + "getrandom", + "iana-time-zone", + # the plugins feature needs to set a feature on this transitively depended-on package, we never + # directly import it + "turbopack-ecmascript-plugins", ] [dependencies] @@ -63,7 +63,7 @@ flate2 = { workspace = true } futures-util = { workspace = true } owo-colors = { workspace = true } napi = { workspace = true } -napi-derive = "2" +napi-derive = { workspace = true } next-custom-transforms = { workspace = true } next-taskless = { workspace = true } rand = { workspace = true } @@ -115,7 +115,7 @@ next-core = { workspace = true } mdxjs = { workspace = true, features = ["serializable"] } turbo-tasks-malloc = { workspace = true, default-features = false, features = [ - "custom_allocator" + "custom_allocator", ] } turbopack-core = { workspace = true } @@ -143,8 +143,7 @@ tokio = { workspace = true, features = ["full"] } [build-dependencies] anyhow = { workspace = true } -napi-build = "2" +napi-build = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } vergen-gitcl = { workspace = true } - diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs index e3b04f77b53532..6fe18d9d712be5 100644 --- a/crates/next-api/src/project.rs +++ b/crates/next-api/src/project.rs @@ -1057,11 +1057,12 @@ impl Project { // At this point all modules have been computed and we can get rid of the node.js // process pools - if *self.is_watch_enabled().await? { - turbopack_node::evaluate::scale_down(); - } else { - turbopack_node::evaluate::scale_zero(); - } + // TODO: + // if *self.is_watch_enabled().await? { + // turbopack_node::evaluate::scale_down(); + // } else { + // turbopack_node::evaluate::scale_zero(); + // } Ok(module_graphs_vc) } diff --git a/packages/next/src/build/swc/generated-native.d.ts b/packages/next/src/build/swc/generated-native.d.ts index 08fed01ec5009a..4d8b188f010366 100644 --- a/packages/next/src/build/swc/generated-native.d.ts +++ b/packages/next/src/build/swc/generated-native.d.ts @@ -34,6 +34,18 @@ export declare class ExternalObject { [K: symbol]: T } } +export declare function recvPoolRequest(): Promise +export declare function notifyPoolCreated(filename: string): Promise +export declare function recvWorkerRequest(poolId: string): Promise +export declare function notifyWorkerAck(poolId: string): Promise +export declare function recvEvaluation(poolId: string): Promise> +export declare function recvMessageInWorker( + workerId: number +): Promise> +export declare function sendTaskResponse( + taskId: string, + data: Array +): Promise export declare function lockfileTryAcquireSync( path: string ): { __napiType: 'Lockfile' } | null diff --git a/turbopack/crates/turbopack-node/Cargo.toml b/turbopack/crates/turbopack-node/Cargo.toml index 646d64b2597939..e22ec55d63fba2 100644 --- a/turbopack/crates/turbopack-node/Cargo.toml +++ b/turbopack/crates/turbopack-node/Cargo.toml @@ -10,8 +10,11 @@ autobenches = false bench = false [features] +default = ["worker_thread"] # enable "HMR" for embedded assets dynamic_embed_contents = ["turbo-tasks-fs/dynamic_embed_contents"] +child_process = ["tokio"] +worker_thread = ["napi", "napi-derive"] [lints] workspace = true @@ -20,6 +23,8 @@ workspace = true anyhow = { workspace = true } async-stream = "0.3.4" async-trait = { workspace = true } +async-channel = "2.5.0" +dashmap = { workspace = true } base64 = "0.21.0" const_format = { workspace = true } either = { workspace = true, features = ["serde"] } @@ -34,7 +39,10 @@ rustc-hash = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true, features = ["base64"] } -tokio = { workspace = true, features = ["full"] } +tokio = { workspace = true, optional = true, features = ["full"] } +uuid = { workspace = true, features = ["v4"] } +napi = { workspace = true, optional = true, features = ["anyhow"] } +napi-derive = { workspace = true, optional = true } tracing = { workspace = true } turbo-rcstr = { workspace = true } turbo-tasks = { workspace = true } @@ -46,3 +54,5 @@ turbopack-core = { workspace = true } turbopack-ecmascript = { workspace = true } turbopack-resolve = { workspace = true } +[build-dependencies] +napi-build = { workspace = true } diff --git a/turbopack/crates/turbopack-node/build.rs b/turbopack/crates/turbopack-node/build.rs new file mode 100644 index 00000000000000..8efb97e10c4e7f --- /dev/null +++ b/turbopack/crates/turbopack-node/build.rs @@ -0,0 +1,6 @@ +fn main() { + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + if std::env::var("CARGO_FEATURE_WORKER_THREAD").is_ok() { + napi_build::setup(); + } +} diff --git a/turbopack/crates/turbopack-node/src/evaluate.rs b/turbopack/crates/turbopack-node/src/evaluate.rs index 7d8aed4651ec19..9df17e85ceac18 100644 --- a/turbopack/crates/turbopack-node/src/evaluate.rs +++ b/turbopack/crates/turbopack-node/src/evaluate.rs @@ -10,7 +10,7 @@ use turbo_tasks::{ TryJoinIterExt, Vc, duration_span, fxindexmap, get_effects, trace::TraceRawVcs, }; use turbo_tasks_env::{EnvMap, ProcessEnv}; -use turbo_tasks_fs::{File, FileSystemPath, to_sys_path}; +use turbo_tasks_fs::{File, FileSystemPath, json::parse_json_with_source_context, to_sys_path}; use turbopack_core::{ asset::AssetContent, changed::content_changed, @@ -31,17 +31,18 @@ use turbopack_core::{ virtual_source::VirtualSource, }; +#[cfg(feature = "child_process")] +use crate::process_pool::ChildProcessPool as Pool; +#[cfg(feature = "worker_thread")] +use crate::worker_pool::WorkerThreadPool as Pool; use crate::{ - AssetsForSourceMapping, - embed_js::embed_file_path, - emit, emit_package_json, internal_assets_for_source_mapping, - pool::{FormattingMode, NodeJsOperation, NodeJsPool}, - source_map::StructuredError, + AssetsForSourceMapping, embed_js::embed_file_path, emit, emit_package_json, + format::FormattingMode, internal_assets_for_source_mapping, source_map::StructuredError, }; #[derive(Serialize)] #[serde(tag = "type", rename_all = "camelCase")] -enum EvalJavaScriptOutgoingMessage<'a> { +pub enum EvalJavaScriptOutgoingMessage<'a> { #[serde(rename_all = "camelCase")] Evaluate { args: Vec<&'a JsonValue> }, Result { @@ -53,13 +54,59 @@ enum EvalJavaScriptOutgoingMessage<'a> { #[derive(Deserialize, Debug)] #[serde(tag = "type", rename_all = "camelCase")] -enum EvalJavaScriptIncomingMessage { +pub enum EvalJavaScriptIncomingMessage { Info { data: JsonValue }, Request { id: u64, data: JsonValue }, End { data: Option }, Error(StructuredError), } +#[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)] +pub struct EvaluatePool { + pub id: RcStr, + #[turbo_tasks(trace_ignore, debug_ignore)] + pool: Box, + pub assets_for_source_mapping: ResolvedVc, + pub assets_root: FileSystemPath, + pub project_dir: FileSystemPath, +} + +impl EvaluatePool { + pub async fn operation(&self) -> Result> { + self.pool.operation().await + } +} + +impl EvaluatePool { + pub(crate) fn new( + id: RcStr, + pool: Box, + assets_for_source_mapping: ResolvedVc, + assets_root: FileSystemPath, + project_dir: FileSystemPath, + ) -> Self { + Self { + id, + pool, + assets_for_source_mapping, + assets_root, + project_dir, + } + } +} + +#[async_trait::async_trait] +pub trait EvaluateOperation: Send + Sync { + async fn operation(&self) -> Result>; +} + +#[async_trait::async_trait] +pub trait Operation: Send { + async fn recv(&mut self) -> Result>; + + async fn send(&mut self, data: Vec) -> Result<()>; +} + #[turbo_tasks::value] struct EmittedEvaluatePoolAssets { bootstrap: ResolvedVc>, @@ -161,7 +208,7 @@ pub async fn get_evaluate_pool( additional_invalidation: ResolvedVc, debug: bool, env_var_tracking: EnvVarTracking, -) -> Result> { +) -> Result> { let operation = emit_evaluate_pool_assets_with_effects_operation(entries, chunking_context, module_graph); let EmittedEvaluatePoolAssetsWithEffects { assets, effects } = @@ -199,7 +246,7 @@ pub async fn get_evaluate_pool( env.read_all().untracked().await? } }; - let pool = NodeJsPool::new( + let pool = Pool::create( cwd, entrypoint, env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), @@ -256,7 +303,7 @@ pub trait EvaluateContext { type ResponseMessage: Serialize; type State: Default; - fn pool(&self) -> OperationVc; + fn pool(&self) -> OperationVc; fn keep_alive(&self) -> bool { false } @@ -265,24 +312,24 @@ pub trait EvaluateContext { fn emit_error( &self, error: StructuredError, - pool: &NodeJsPool, + pool: &EvaluatePool, ) -> impl Future> + Send; fn info( &self, state: &mut Self::State, data: Self::InfoMessage, - pool: &NodeJsPool, + pool: &EvaluatePool, ) -> impl Future> + Send; fn request( &self, state: &mut Self::State, data: Self::RequestMessage, - pool: &NodeJsPool, + pool: &EvaluatePool, ) -> impl Future> + Send; fn finish( &self, state: Self::State, - pool: &NodeJsPool, + pool: &EvaluatePool, ) -> impl Future> + Send; } @@ -297,7 +344,7 @@ pub async fn custom_evaluate(evaluate_context: impl EvaluateContext) -> Result Result Result( - operation: &mut NodeJsOperation, - pool: &NodeJsPool, + operation: &mut Box, + pool: &EvaluatePool, evaluate_context: &T, state: &mut T::State, ) -> Result> { let _guard = duration_span!("Node.js evaluation"); loop { - match operation.recv().await? { + let buf = operation.recv().await?; + let message = parse_json_with_source_context(std::str::from_utf8(&buf)?)?; + + match message { EvalJavaScriptIncomingMessage::Error(error) => { evaluate_context.emit_error(error, pool).await?; // Do not reuse the process in case of error - operation.disallow_reuse(); + // operation.disallow_reuse(); // Issue emitted, we want to break but don't want to return an error return Ok(None); } @@ -484,20 +536,24 @@ async fn pull_operation( { Ok(response) => { operation - .send(EvalJavaScriptOutgoingMessage::Result { - id, - error: None, - data: Some(serde_json::to_value(response)?), - }) + .send(serde_json::to_vec( + &EvalJavaScriptOutgoingMessage::Result { + id, + error: None, + data: Some(serde_json::to_value(response)?), + }, + )?) .await?; } Err(e) => { operation - .send(EvalJavaScriptOutgoingMessage::Result { - id, - error: Some(PrettyPrintError(&e).to_string()), - data: None, - }) + .send(serde_json::to_vec( + &EvalJavaScriptOutgoingMessage::Result { + id, + error: Some(PrettyPrintError(&e).to_string()), + data: None, + }, + )?) .await?; } } @@ -525,7 +581,7 @@ impl EvaluateContext for BasicEvaluateContext { type ResponseMessage = (); type State = (); - fn pool(&self) -> OperationVc { + fn pool(&self) -> OperationVc { get_evaluate_pool( self.entries, self.cwd.clone(), @@ -550,7 +606,7 @@ impl EvaluateContext for BasicEvaluateContext { !self.args.is_empty() } - async fn emit_error(&self, error: StructuredError, pool: &NodeJsPool) -> Result<()> { + async fn emit_error(&self, error: StructuredError, pool: &EvaluatePool) -> Result<()> { EvaluationIssue { error, source: IssueSource::from_source_only(self.context_source_for_issue), @@ -567,7 +623,7 @@ impl EvaluateContext for BasicEvaluateContext { &self, _state: &mut Self::State, _data: Self::InfoMessage, - _pool: &NodeJsPool, + _pool: &EvaluatePool, ) -> Result<()> { bail!("BasicEvaluateContext does not support info messages") } @@ -576,24 +632,16 @@ impl EvaluateContext for BasicEvaluateContext { &self, _state: &mut Self::State, _data: Self::RequestMessage, - _pool: &NodeJsPool, + _pool: &EvaluatePool, ) -> Result { bail!("BasicEvaluateContext does not support request messages") } - async fn finish(&self, _state: Self::State, _pool: &NodeJsPool) -> Result<()> { + async fn finish(&self, _state: Self::State, _pool: &EvaluatePool) -> Result<()> { Ok(()) } } -pub fn scale_zero() { - NodeJsPool::scale_zero(); -} - -pub fn scale_down() { - NodeJsPool::scale_down(); -} - /// An issue that occurred while evaluating node code. #[turbo_tasks::value(shared)] pub struct EvaluationIssue { diff --git a/turbopack/crates/turbopack-node/src/format.rs b/turbopack/crates/turbopack-node/src/format.rs new file mode 100644 index 00000000000000..27dd550c28119a --- /dev/null +++ b/turbopack/crates/turbopack-node/src/format.rs @@ -0,0 +1,36 @@ +use std::fmt::Display; + +use owo_colors::{OwoColorize, Style}; + +#[derive(Clone, Copy)] +pub enum FormattingMode { + /// No formatting, just print the output + Plain, + /// Use ansi colors to format the output + AnsiColors, +} + +impl FormattingMode { + pub fn magic_identifier<'a>(&self, content: impl Display + 'a) -> impl Display + 'a { + match self { + FormattingMode::Plain => format!("{{{content}}}"), + FormattingMode::AnsiColors => format!("{{{content}}}").italic().to_string(), + } + } + + pub fn lowlight<'a>(&self, content: impl Display + 'a) -> impl Display + 'a { + match self { + FormattingMode::Plain => Style::new(), + FormattingMode::AnsiColors => Style::new().dimmed(), + } + .style(content) + } + + pub fn highlight<'a>(&self, content: impl Display + 'a) -> impl Display + 'a { + match self { + FormattingMode::Plain => Style::new(), + FormattingMode::AnsiColors => Style::new().bold().underline(), + } + .style(content) + } +} diff --git a/turbopack/crates/turbopack-node/src/lib.rs b/turbopack/crates/turbopack-node/src/lib.rs index a86c9b5d1baa3e..91b001da34b5ee 100644 --- a/turbopack/crates/turbopack-node/src/lib.rs +++ b/turbopack/crates/turbopack-node/src/lib.rs @@ -22,10 +22,15 @@ pub mod debug; pub mod embed_js; pub mod evaluate; pub mod execution_context; +mod format; +#[cfg(feature = "child_process")] mod heap_queue; -mod pool; +#[cfg(feature = "child_process")] +mod process_pool; pub mod source_map; pub mod transforms; +#[cfg(feature = "worker_thread")] +mod worker_pool; #[turbo_tasks::function] async fn emit( diff --git a/turbopack/crates/turbopack-node/src/pool.rs b/turbopack/crates/turbopack-node/src/process_pool.rs similarity index 91% rename from turbopack/crates/turbopack-node/src/pool.rs rename to turbopack/crates/turbopack-node/src/process_pool.rs index 1d4532868d66e4..0e8039976f2480 100644 --- a/turbopack/crates/turbopack-node/src/pool.rs +++ b/turbopack/crates/turbopack-node/src/process_pool.rs @@ -1,6 +1,6 @@ use std::{ cmp::max, - fmt::{Debug, Display}, + fmt::Debug, future::Future, mem::take, path::{Path, PathBuf}, @@ -12,10 +12,9 @@ use std::{ use anyhow::{Context, Result, bail}; use futures::join; use once_cell::sync::Lazy; -use owo_colors::{OwoColorize, Style}; +use owo_colors::OwoColorize; use parking_lot::Mutex; use rustc_hash::FxHashMap; -use serde::{Serialize, de::DeserializeOwned}; use tokio::{ io::{ AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, Stderr, @@ -29,43 +28,16 @@ use tokio::{ }; use turbo_rcstr::RcStr; use turbo_tasks::{FxIndexSet, ResolvedVc, Vc, duration_span}; -use turbo_tasks_fs::{FileSystemPath, json::parse_json_with_source_context}; +use turbo_tasks_fs::FileSystemPath; use turbopack_ecmascript::magic_identifier::unmangle_identifiers; -use crate::{AssetsForSourceMapping, heap_queue::HeapQueue, source_map::apply_source_mapping}; - -#[derive(Clone, Copy)] -pub enum FormattingMode { - /// No formatting, just print the output - Plain, - /// Use ansi colors to format the output - AnsiColors, -} - -impl FormattingMode { - pub fn magic_identifier<'a>(&self, content: impl Display + 'a) -> impl Display + 'a { - match self { - FormattingMode::Plain => format!("{{{content}}}"), - FormattingMode::AnsiColors => format!("{{{content}}}").italic().to_string(), - } - } - - pub fn lowlight<'a>(&self, content: impl Display + 'a) -> impl Display + 'a { - match self { - FormattingMode::Plain => Style::new(), - FormattingMode::AnsiColors => Style::new().dimmed(), - } - .style(content) - } - - pub fn highlight<'a>(&self, content: impl Display + 'a) -> impl Display + 'a { - match self { - FormattingMode::Plain => Style::new(), - FormattingMode::AnsiColors => Style::new().bold().underline(), - } - .style(content) - } -} +use crate::{ + AssetsForSourceMapping, + evaluate::{EvaluateOperation, EvaluatePool, Operation}, + format::FormattingMode, + heap_queue::HeapQueue, + source_map::apply_source_mapping, +}; struct NodeJsPoolProcess { child: Option, @@ -686,7 +658,7 @@ static ACTIVE_POOLS: Lazy = Lazy::new(Default::default); /// The worker will *not* use the env of the parent process by default. All env /// vars need to be provided to make the execution as pure as possible. #[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)] -pub struct NodeJsPool { +pub struct ChildProcessPool { cwd: PathBuf, entrypoint: PathBuf, env: FxHashMap, @@ -711,10 +683,10 @@ pub struct NodeJsPool { stats: Arc>, } -impl NodeJsPool { +impl ChildProcessPool { /// * debug: Whether to automatically enable Node's `--inspect-brk` when spawning it. Note: /// automatically overrides concurrency to 1. - pub(super) fn new( + pub fn create( cwd: PathBuf, entrypoint: PathBuf, env: FxHashMap, @@ -723,24 +695,53 @@ impl NodeJsPool { project_dir: FileSystemPath, concurrency: usize, debug: bool, - ) -> Self { - Self { - cwd, - entrypoint, - env, + ) -> EvaluatePool { + EvaluatePool::new( + entrypoint.to_string_lossy().to_string().into(), + Box::new(Self { + cwd, + entrypoint, + env, + assets_for_source_mapping, + assets_root: assets_root.clone(), + project_dir: project_dir.clone(), + concurrency_semaphore: Arc::new(Semaphore::new(if debug { + 1 + } else { + concurrency + })), + bootup_semaphore: Arc::new(Semaphore::new(1)), + idle_processes: Arc::new(HeapQueue::new()), + shared_stdout: Arc::new(Mutex::new(FxIndexSet::default())), + shared_stderr: Arc::new(Mutex::new(FxIndexSet::default())), + debug, + stats: Default::default(), + }), assets_for_source_mapping, assets_root, project_dir, - concurrency_semaphore: Arc::new(Semaphore::new(if debug { 1 } else { concurrency })), - bootup_semaphore: Arc::new(Semaphore::new(1)), - idle_processes: Arc::new(HeapQueue::new()), - shared_stdout: Arc::new(Mutex::new(FxIndexSet::default())), - shared_stderr: Arc::new(Mutex::new(FxIndexSet::default())), - debug, - stats: Default::default(), - } + ) } +} +#[async_trait::async_trait] +impl EvaluateOperation for ChildProcessPool { + async fn operation(&self) -> Result> { + // Acquire a running process (handles concurrency limits, boots up the process) + let (process, permits) = self.acquire_process().await?; + + Ok(Box::new(NodeJsOperation { + process: Some(process), + permits, + idle_processes: self.idle_processes.clone(), + start: Instant::now(), + stats: self.stats.clone(), + allow_process_reuse: true, + })) + } +} + +impl ChildProcessPool { async fn acquire_process(&self) -> Result<(NodeJsPoolProcess, AcquiredPermits)> { { self.stats.lock().add_queued_task(); @@ -797,20 +798,6 @@ impl NodeJsPool { Ok((process, start.elapsed())) } - pub async fn operation(&self) -> Result { - // Acquire a running process (handles concurrency limits, boots up the process) - let (process, permits) = self.acquire_process().await?; - - Ok(NodeJsOperation { - process: Some(process), - permits, - idle_processes: self.idle_processes.clone(), - start: Instant::now(), - stats: self.stats.clone(), - allow_process_reuse: true, - }) - } - pub fn scale_down() { let pools = ACTIVE_POOLS.lock().clone(); for pool in pools { @@ -837,6 +824,27 @@ pub struct NodeJsOperation { allow_process_reuse: bool, } +#[async_trait::async_trait] +impl Operation for NodeJsOperation { + async fn recv(&mut self) -> Result> { + self.with_process(|process| async move { + process.recv().await.context("failed to receive message") + }) + .await + } + + async fn send(&mut self, data: Vec) -> Result<()> { + self.with_process(|process| async move { + timeout(Duration::from_secs(30), process.send(data)) + .await + .context("timeout while sending message")? + .context("failed to send message")?; + Ok(()) + }) + .await + } +} + impl NodeJsOperation { async fn with_process<'a, F: Future> + Send + 'a, T>( &'a mut self, @@ -859,34 +867,7 @@ impl NodeJsOperation { result } - pub async fn recv(&mut self) -> Result - where - M: DeserializeOwned, - { - let message = self - .with_process(|process| async move { - process.recv().await.context("failed to receive message") - }) - .await?; - let message = std::str::from_utf8(&message).context("message is not valid UTF-8")?; - parse_json_with_source_context(message).context("failed to deserialize message") - } - - pub async fn send(&mut self, message: M) -> Result<()> - where - M: Serialize, - { - let message = serde_json::to_vec(&message).context("failed to serialize message")?; - self.with_process(|process| async move { - timeout(Duration::from_secs(30), process.send(message)) - .await - .context("timeout while sending message")? - .context("failed to send message")?; - Ok(()) - }) - .await - } - + #[allow(dead_code)] pub async fn wait_or_kill(mut self) -> Result { let mut process = self .process @@ -912,6 +893,7 @@ impl NodeJsOperation { Ok(status) } + #[allow(dead_code)] pub fn disallow_reuse(&mut self) { if self.allow_process_reuse { self.stats.lock().remove_worker(); diff --git a/turbopack/crates/turbopack-node/src/source_map/mod.rs b/turbopack/crates/turbopack-node/src/source_map/mod.rs index eeaf2e6cab1f68..13d486da523ef2 100644 --- a/turbopack/crates/turbopack-node/src/source_map/mod.rs +++ b/turbopack/crates/turbopack-node/src/source_map/mod.rs @@ -20,7 +20,7 @@ use turbopack_core::{ }; use turbopack_ecmascript::magic_identifier::unmangle_identifiers; -use crate::{AssetsForSourceMapping, pool::FormattingMode}; +use crate::{AssetsForSourceMapping, format::FormattingMode}; pub mod trace; diff --git a/turbopack/crates/turbopack-node/src/transforms/webpack.rs b/turbopack/crates/turbopack-node/src/transforms/webpack.rs index 899849fb155200..e013669230729e 100644 --- a/turbopack/crates/turbopack-node/src/transforms/webpack.rs +++ b/turbopack/crates/turbopack-node/src/transforms/webpack.rs @@ -55,11 +55,11 @@ use crate::{ debug::should_debug, embed_js::embed_file_path, evaluate::{ - EnvVarTracking, EvaluateContext, EvaluateEntries, EvaluationIssue, custom_evaluate, - get_evaluate_entries, get_evaluate_pool, + EnvVarTracking, EvaluateContext, EvaluateEntries, EvaluatePool, EvaluationIssue, + custom_evaluate, get_evaluate_entries, get_evaluate_pool, }, execution_context::ExecutionContext, - pool::{FormattingMode, NodeJsPool}, + format::FormattingMode, source_map::{StackFrame, StructuredError}, }; @@ -433,7 +433,7 @@ impl EvaluateContext for WebpackLoaderContext { type ResponseMessage = ResponseMessage; type State = Vec; - fn pool(&self) -> OperationVc { + fn pool(&self) -> OperationVc { get_evaluate_pool( self.entries, self.cwd.clone(), @@ -461,7 +461,7 @@ impl EvaluateContext for WebpackLoaderContext { true } - async fn emit_error(&self, error: StructuredError, pool: &NodeJsPool) -> Result<()> { + async fn emit_error(&self, error: StructuredError, pool: &EvaluatePool) -> Result<()> { EvaluationIssue { error, source: IssueSource::from_source_only(self.context_source_for_issue), @@ -478,7 +478,7 @@ impl EvaluateContext for WebpackLoaderContext { &self, state: &mut Self::State, data: Self::InfoMessage, - pool: &NodeJsPool, + pool: &EvaluatePool, ) -> Result<()> { match data { InfoMessage::Dependencies { @@ -554,7 +554,7 @@ impl EvaluateContext for WebpackLoaderContext { &self, _state: &mut Self::State, data: Self::RequestMessage, - _pool: &NodeJsPool, + _pool: &EvaluatePool, ) -> Result { match data { RequestMessage::Resolve { @@ -608,7 +608,7 @@ impl EvaluateContext for WebpackLoaderContext { } } - async fn finish(&self, state: Self::State, pool: &NodeJsPool) -> Result<()> { + async fn finish(&self, state: Self::State, pool: &EvaluatePool) -> Result<()> { let has_errors = state.iter().any(|log| log.log_type == LogType::Error); let has_warnings = state.iter().any(|log| log.log_type == LogType::Warn); if has_errors || has_warnings { diff --git a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs new file mode 100644 index 00000000000000..ad69187022a0d4 --- /dev/null +++ b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs @@ -0,0 +1,79 @@ +use std::path::PathBuf; + +use anyhow::Result; +use rustc_hash::FxHashMap; +use turbo_rcstr::RcStr; +use turbo_tasks::ResolvedVc; +use turbo_tasks_fs::FileSystemPath; + +use crate::{ + AssetsForSourceMapping, + evaluate::{EvaluateOperation, EvaluatePool, Operation}, + worker_pool::operation::{WorkerOperation, connect_to_worker, create_pool}, +}; + +mod operation; +mod worker_thread; + +#[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)] +pub struct WorkerThreadPool { + cwd: PathBuf, + entrypoint: PathBuf, + env: FxHashMap, + concurrency: usize, + pub assets_for_source_mapping: ResolvedVc, + pub assets_root: FileSystemPath, + pub project_dir: FileSystemPath, +} + +impl WorkerThreadPool { + pub fn create( + cwd: PathBuf, + entrypoint: PathBuf, + env: FxHashMap, + assets_for_source_mapping: ResolvedVc, + assets_root: FileSystemPath, + project_dir: FileSystemPath, + concurrency: usize, + _debug: bool, + ) -> EvaluatePool { + EvaluatePool::new( + entrypoint.to_string_lossy().to_string().into(), + Box::new(Self { + cwd, + entrypoint, + env, + concurrency, + assets_for_source_mapping, + assets_root: assets_root.clone(), + project_dir: project_dir.clone(), + }), + assets_for_source_mapping, + assets_root, + project_dir, + ) + } +} + +#[async_trait::async_trait] +impl EvaluateOperation for WorkerThreadPool { + async fn operation(&self) -> Result> { + create_pool( + self.entrypoint.to_string_lossy().to_string(), + self.concurrency, + ) + .await?; + + let task_id = uuid::Uuid::new_v4().to_string(); + + let worker_id = connect_to_worker( + self.entrypoint.to_string_lossy().to_string(), + task_id.clone(), + ) + .await?; + + let worker_operation = WorkerOperation { task_id, worker_id }; + + Ok(Box::new(worker_operation)) + } +} diff --git a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs new file mode 100644 index 00000000000000..c6f1bbc4d6a106 --- /dev/null +++ b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs @@ -0,0 +1,141 @@ +use std::sync::LazyLock; + +use anyhow::{Context, Result}; +use async_channel::{Receiver, Sender, bounded, unbounded}; +use dashmap::DashMap; + +use crate::evaluate::Operation; + +pub(crate) struct MessageChannel { + sender: Sender, + receiver: Receiver, +} + +impl MessageChannel { + pub(super) fn unbounded() -> Self { + let (sender, receiver) = unbounded::(); + Self { sender, receiver } + } + + pub(super) fn bounded(cap: usize) -> Self { + let (sender, receiver) = bounded::(cap); + Self { sender, receiver } + } +} + +impl MessageChannel { + pub(crate) async fn send(&self, data: T) -> Result<()> { + Ok(self.sender.send(data).await?) + } + + pub(crate) async fn recv(&self) -> Result { + Ok(self.receiver.recv().await?) + } +} + +pub(crate) static POOL_REQUEST_CHANNEL: LazyLock> = + LazyLock::new(MessageChannel::unbounded); + +pub(crate) static POOL_CREATION_CHANNEL: LazyLock>> = + LazyLock::new(DashMap::new); + +pub(crate) async fn create_pool(filename: String, concurrency: usize) -> anyhow::Result<()> { + POOL_REQUEST_CHANNEL + .send(filename.clone()) + .await + .context("failed to send pool request")?; + + let mut created_worker_count = 0; + + { + let channel = POOL_CREATION_CHANNEL + .entry(filename.clone()) + .or_insert_with(|| MessageChannel::bounded(concurrency)); + + while created_worker_count < concurrency { + channel + .recv() + .await + .context("failed to recv worker creation")?; + created_worker_count += 1; + } + }; + + POOL_CREATION_CHANNEL.remove(&filename); + + Ok(()) +} + +pub(crate) static EVALUATION_REQUEST_CHANNAL: LazyLock>>> = + LazyLock::new(DashMap::new); + +pub(crate) static WORKER_REQUEST_CHANNAL: LazyLock>> = + LazyLock::new(DashMap::new); + +pub(crate) static WORKER_ACK_CHANNAL: LazyLock>> = + LazyLock::new(DashMap::new); + +pub(crate) async fn connect_to_worker(pool_id: String, task_id: String) -> Result { + let channel = WORKER_REQUEST_CHANNAL + .entry(pool_id.clone()) + .or_insert_with(MessageChannel::unbounded); + channel + .send(()) + .await + .context("failed to send evaluation request")?; + let worker_id = async move { + let channel = WORKER_ACK_CHANNAL + .entry(task_id.clone()) + .or_insert_with(MessageChannel::unbounded); + channel + .recv() + .await + .context("failed to recv evaluation ack") + } + .await?; + Ok(worker_id) +} + +pub(crate) static WORKER_ROUTED_CHANNEL: LazyLock>>> = + LazyLock::new(DashMap::new); + +pub(crate) async fn send_message_to_worker(worker_id: u32, data: Vec) -> Result<()> { + let entry = WORKER_ROUTED_CHANNEL + .entry(worker_id) + .or_insert_with(MessageChannel::unbounded); + entry + .send(data) + .await + .with_context(|| format!("failed to send message to worker {worker_id}"))?; + Ok(()) +} + +pub(crate) static TASK_ROUTERD_CHANNEL: LazyLock>>> = + LazyLock::new(DashMap::new); + +pub async fn recv_task_response(task_id: String) -> Result> { + let channel = TASK_ROUTERD_CHANNEL + .entry(task_id.clone()) + .or_insert_with(MessageChannel::unbounded); + let data = channel + .recv() + .await + .with_context(|| format!("failed to send message to worker {task_id}"))?; + Ok(data) +} + +pub(crate) struct WorkerOperation { + pub(crate) task_id: String, + pub(crate) worker_id: u32, +} + +#[async_trait::async_trait] +impl Operation for WorkerOperation { + async fn recv(&mut self) -> Result> { + recv_task_response(self.task_id.clone()).await + } + + async fn send(&mut self, data: Vec) -> Result<()> { + send_message_to_worker(self.worker_id, data).await + } +} diff --git a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs new file mode 100644 index 00000000000000..22be69b3a13f9b --- /dev/null +++ b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs @@ -0,0 +1,100 @@ +use anyhow::Context; +use napi_derive::napi; + +use crate::worker_pool::operation::{ + EVALUATION_REQUEST_CHANNAL, MessageChannel, POOL_CREATION_CHANNEL, POOL_REQUEST_CHANNEL, + TASK_ROUTERD_CHANNEL, WORKER_REQUEST_CHANNAL, WORKER_ROUTED_CHANNEL, +}; + +#[napi] +pub async fn recv_pool_request() -> napi::Result { + Ok(POOL_REQUEST_CHANNEL + .recv() + .await + .context("failed to recv pool request")?) +} + +#[napi] +pub async fn notify_pool_created(filename: String) -> napi::Result<()> { + let channel = if let Some(channel) = POOL_CREATION_CHANNEL.get(&filename) { + channel + } else { + return Err(napi::Error::from_reason(format!( + "pool creation channel for {filename} not found" + ))); + }; + Ok(channel + .send(()) + .await + .context("failed to notify pool created")?) +} + +#[napi] +pub async fn recv_worker_request(pool_id: String) -> napi::Result<()> { + let channel = if let Some(channel) = WORKER_REQUEST_CHANNAL.get(&pool_id) { + channel + } else { + return Err(napi::Error::from_reason(format!( + "worker request channel for {pool_id} not found" + ))); + }; + Ok(channel + .send(()) + .await + .context("failed to recv worker request")?) +} + +#[napi] +pub async fn notify_worker_ack(pool_id: String) -> napi::Result<()> { + let channel = if let Some(channel) = POOL_CREATION_CHANNEL.get(&pool_id) { + channel + } else { + return Err(napi::Error::from_reason(format!( + "evaluation ack channel for {pool_id} not found" + ))); + }; + Ok(channel + .send(()) + .await + .context("failed to notify evaluation ack")?) +} + +#[napi] +pub async fn recv_evaluation(pool_id: String) -> napi::Result> { + let channel = if let Some(channel) = EVALUATION_REQUEST_CHANNAL.get(&pool_id) { + channel + } else { + return Err(napi::Error::from_reason(format!( + "evaluation request channel for {pool_id} not found" + ))); + }; + + Ok(channel + .recv() + .await + .context("failed to recv evaluate request")?) +} + +#[napi] +pub async fn recv_message_in_worker(worker_id: u32) -> napi::Result> { + let channel = WORKER_ROUTED_CHANNEL + .entry(worker_id) + .or_insert_with(MessageChannel::unbounded); + let data = channel + .recv() + .await + .with_context(|| format!("failed to recv message in worker {worker_id}"))?; + Ok(data) +} + +#[napi] +pub async fn send_task_response(task_id: String, data: Vec) -> napi::Result<()> { + let channel = TASK_ROUTERD_CHANNEL + .entry(task_id.clone()) + .or_insert_with(MessageChannel::unbounded); + channel + .send(data) + .await + .with_context(|| format!("failed to recv message in worker {task_id}"))?; + Ok(()) +} From da23168ad99be94cd3350074b69097f94b6cfdf5 Mon Sep 17 00:00:00 2001 From: xusd320 Date: Tue, 25 Nov 2025 12:44:38 +0800 Subject: [PATCH 03/17] feat(turbopack-node): run webpack loaders via napi, keep features for child_process --- Cargo.lock | 1 + Cargo.toml | 2 +- crates/napi/Cargo.toml | 1 + crates/next-api/Cargo.toml | 2 +- crates/next-api/src/project.rs | 2 +- crates/next-core/Cargo.toml | 2 +- .../next/src/build/swc/generated-native.d.ts | 22 +- packages/next/src/build/swc/index.ts | 28 ++ turbopack/crates/turbopack-cli/Cargo.toml | 2 +- turbopack/crates/turbopack-node/Cargo.toml | 17 +- .../js/src/{ipc => child_process}/evaluate.ts | 7 +- .../js/src/{ => child_process}/globals.ts | 0 .../js/src/{ipc => child_process}/index.ts | 22 +- .../turbopack-node/js/src/{ipc => }/error.ts | 21 ++ .../js/src/transforms/transforms.ts | 4 +- .../js/src/transforms/webpack-loaders.ts | 6 +- .../crates/turbopack-node/js/src/types.ts | 5 + .../js/src/worker_threads/evaluate.ts | 165 ++++++++++++ .../crates/turbopack-node/src/evaluate.rs | 84 ++++-- turbopack/crates/turbopack-node/src/lib.rs | 4 +- .../turbopack-node/src/process_pool/mod.rs | 39 +-- .../turbopack-node/src/worker_pool/mod.rs | 47 +++- .../src/worker_pool/operation.rs | 250 ++++++++++++------ .../src/worker_pool/web_worker.rs | 1 + .../src/worker_pool/worker_thread.rs | 107 +++----- turbopack/crates/turbopack-tests/Cargo.toml | 2 +- turbopack/crates/turbopack/Cargo.toml | 2 +- 27 files changed, 582 insertions(+), 263 deletions(-) rename turbopack/crates/turbopack-node/js/src/{ipc => child_process}/evaluate.ts (95%) rename turbopack/crates/turbopack-node/js/src/{ => child_process}/globals.ts (100%) rename turbopack/crates/turbopack-node/js/src/{ipc => child_process}/index.ts (91%) rename turbopack/crates/turbopack-node/js/src/{ipc => }/error.ts (74%) create mode 100644 turbopack/crates/turbopack-node/js/src/types.ts create mode 100644 turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts create mode 100644 turbopack/crates/turbopack-node/src/worker_pool/web_worker.rs diff --git a/Cargo.lock b/Cargo.lock index 3a69c1335a67ad..af2a3ef7c4741b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4469,6 +4469,7 @@ dependencies = [ "turbopack-core", "turbopack-ecmascript-hmr-protocol", "turbopack-ecmascript-plugins", + "turbopack-node", "turbopack-trace-server", "turbopack-trace-utils", "url", diff --git a/Cargo.toml b/Cargo.toml index ce489135bf852f..b4a835e050c713 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -327,7 +327,7 @@ turbopack-env = { path = "turbopack/crates/turbopack-env" } turbopack-image = { path = "turbopack/crates/turbopack-image" } turbopack-json = { path = "turbopack/crates/turbopack-json" } turbopack-mdx = { path = "turbopack/crates/turbopack-mdx" } -turbopack-node = { path = "turbopack/crates/turbopack-node" } +turbopack-node = { path = "turbopack/crates/turbopack-node", default-features = false } turbopack-resolve = { path = "turbopack/crates/turbopack-resolve" } turbopack-static = { path = "turbopack/crates/turbopack-static" } turbopack-swc-utils = { path = "turbopack/crates/turbopack-swc-utils" } diff --git a/crates/napi/Cargo.toml b/crates/napi/Cargo.toml index 76f049f083ac69..0da55ffc338151 100644 --- a/crates/napi/Cargo.toml +++ b/crates/napi/Cargo.toml @@ -123,6 +123,7 @@ turbopack-ecmascript-hmr-protocol = { workspace = true } turbopack-trace-utils = { workspace = true } turbopack-trace-server = { workspace = true } turbopack-ecmascript-plugins = { workspace = true, optional = true } +turbopack-node = { workspace = true, default-features = false, features = ["worker_pool"] } [target.'cfg(windows)'.dependencies] windows-sys = "0.60" diff --git a/crates/next-api/Cargo.toml b/crates/next-api/Cargo.toml index b5a1fd03a7df4a..41885c97b0cfc4 100644 --- a/crates/next-api/Cargo.toml +++ b/crates/next-api/Cargo.toml @@ -36,7 +36,7 @@ turbopack-analyze = { workspace = true } turbopack-browser = { workspace = true } turbopack-core = { workspace = true } turbopack-ecmascript = { workspace = true } -turbopack-node = { workspace = true } +turbopack-node = { workspace = true, default-features = false } turbopack-nodejs = { workspace = true } turbopack-wasm = { workspace = true } diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs index 9c4a2985cbdc9c..da684f10c5de16 100644 --- a/crates/next-api/src/project.rs +++ b/crates/next-api/src/project.rs @@ -1056,7 +1056,7 @@ impl Project { // At this point all modules have been computed and we can get rid of the node.js // process pools - // TODO: + // FIXME : // if *self.is_watch_enabled().await? { // turbopack_node::evaluate::scale_down(); // } else { diff --git a/crates/next-core/Cargo.toml b/crates/next-core/Cargo.toml index 90f77cbcd556fb..eaebd97c72f0a6 100644 --- a/crates/next-core/Cargo.toml +++ b/crates/next-core/Cargo.toml @@ -76,7 +76,7 @@ turbopack-ecmascript-plugins = { workspace = true, features = [ ] } turbopack-ecmascript-runtime = { workspace = true } turbopack-image = { workspace = true } -turbopack-node = { workspace = true } +turbopack-node = { workspace = true, default-features = false } turbopack-nodejs = { workspace = true } turbopack-static = { workspace = true } turbopack-trace-utils = { workspace = true } diff --git a/packages/next/src/build/swc/generated-native.d.ts b/packages/next/src/build/swc/generated-native.d.ts index 4d8b188f010366..12520ec9702ce6 100644 --- a/packages/next/src/build/swc/generated-native.d.ts +++ b/packages/next/src/build/swc/generated-native.d.ts @@ -34,17 +34,21 @@ export declare class ExternalObject { [K: symbol]: T } } -export declare function recvPoolRequest(): Promise -export declare function notifyPoolCreated(filename: string): Promise -export declare function recvWorkerRequest(poolId: string): Promise -export declare function notifyWorkerAck(poolId: string): Promise -export declare function recvEvaluation(poolId: string): Promise> -export declare function recvMessageInWorker( +export interface PoolOptions { + filename: string + concurrency: number +} +export declare function recvPoolCreation(): PoolOptions | null +export declare function recvWorkerRequest(poolId: string): Promise +export declare function recvMessageInWorker(workerId: number): Promise +export declare function notifyOneWorkerCreated(filename: string): Promise +export declare function notifyWorkerAck( + taskId: string, workerId: number -): Promise> -export declare function sendTaskResponse( +): Promise +export declare function sendTaskMessage( taskId: string, - data: Array + message: string ): Promise export declare function lockfileTryAcquireSync( path: string diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index 3b4071d4083e39..daab54c18155e7 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -1,6 +1,7 @@ import path from 'path' import { pathToFileURL } from 'url' import { arch, platform } from 'os' +import { Worker } from 'worker_threads' import { platformArchTriples } from 'next/dist/compiled/@napi-rs/triples' import * as Log from '../output/log' import { getParserOptions } from './options' @@ -668,8 +669,34 @@ function bindingToApi( class ProjectImpl implements Project { private readonly _nativeProject: { __napiType: 'Project' } + #poolCreated: Record> = {} + + #poolScheduler?: ReturnType + constructor(nativeProject: { __napiType: 'Project' }) { this._nativeProject = nativeProject + + if (binding.recvPoolCreation) { + this.#poolScheduler = setInterval(() => { + let poolOptions = binding.recvPoolCreation() + if (poolOptions) { + const { filename, concurrency } = poolOptions + if (!this.#poolCreated[filename]) { + const workers = [] + for (let i = 0; i < concurrency; i++) { + const worker = new Worker(filename, { + workerData: { + poolId: filename, + }, + }) + worker.unref() + workers.push(worker) + } + this.#poolCreated[filename] = workers + } + } + }, 0) + } } async update(options: Partial) { @@ -796,6 +823,7 @@ function bindingToApi( } shutdown(): Promise { + this.#poolScheduler && clearInterval(this.#poolScheduler) return binding.projectShutdown(this._nativeProject) } diff --git a/turbopack/crates/turbopack-cli/Cargo.toml b/turbopack/crates/turbopack-cli/Cargo.toml index d885720bec7420..0ed62123619d7d 100644 --- a/turbopack/crates/turbopack-cli/Cargo.toml +++ b/turbopack/crates/turbopack-cli/Cargo.toml @@ -66,7 +66,7 @@ turbopack-ecmascript-plugins = { workspace = true, features = [ ] } turbopack-ecmascript-runtime = { workspace = true } turbopack-env = { workspace = true } -turbopack-node = { workspace = true } +turbopack-node = { workspace = true, default-features = false, features = ["process_pool"] } turbopack-nodejs = { workspace = true } turbopack-resolve = { workspace = true } turbopack-trace-utils = { workspace = true } diff --git a/turbopack/crates/turbopack-node/Cargo.toml b/turbopack/crates/turbopack-node/Cargo.toml index e22ec55d63fba2..405131c08a3557 100644 --- a/turbopack/crates/turbopack-node/Cargo.toml +++ b/turbopack/crates/turbopack-node/Cargo.toml @@ -10,11 +10,11 @@ autobenches = false bench = false [features] -default = ["worker_thread"] +default = ["worker_pool"] # enable "HMR" for embedded assets dynamic_embed_contents = ["turbo-tasks-fs/dynamic_embed_contents"] -child_process = ["tokio"] -worker_thread = ["napi", "napi-derive"] +process_pool = ["tokio/full"] +worker_pool = ["async-channel", "tokio/sync", "napi", "napi-derive"] [lints] workspace = true @@ -23,7 +23,6 @@ workspace = true anyhow = { workspace = true } async-stream = "0.3.4" async-trait = { workspace = true } -async-channel = "2.5.0" dashmap = { workspace = true } base64 = "0.21.0" const_format = { workspace = true } @@ -39,10 +38,8 @@ rustc-hash = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true, features = ["base64"] } -tokio = { workspace = true, optional = true, features = ["full"] } +tokio = { workspace = true, optional = true } uuid = { workspace = true, features = ["v4"] } -napi = { workspace = true, optional = true, features = ["anyhow"] } -napi-derive = { workspace = true, optional = true } tracing = { workspace = true } turbo-rcstr = { workspace = true } turbo-tasks = { workspace = true } @@ -53,6 +50,12 @@ turbopack-cli-utils = { workspace = true } turbopack-core = { workspace = true } turbopack-ecmascript = { workspace = true } turbopack-resolve = { workspace = true } +napi = { workspace = true, features = ["anyhow"], optional = true } +napi-derive = { workspace = true, optional = true } +async-channel = { version = "2.5.0", optional = true } + +[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] +uuid = { workspace = true, features = ["js"] } [build-dependencies] napi-build = { workspace = true } diff --git a/turbopack/crates/turbopack-node/js/src/ipc/evaluate.ts b/turbopack/crates/turbopack-node/js/src/child_process/evaluate.ts similarity index 95% rename from turbopack/crates/turbopack-node/js/src/ipc/evaluate.ts rename to turbopack/crates/turbopack-node/js/src/child_process/evaluate.ts index 018a3505ddf18c..e7ed556f6c5382 100644 --- a/turbopack/crates/turbopack-node/js/src/ipc/evaluate.ts +++ b/turbopack/crates/turbopack-node/js/src/child_process/evaluate.ts @@ -1,6 +1,8 @@ import { IPC } from './index' import type { Ipc as GenericIpc } from './index' +import type { Channel as Ipc } from '../types' + type IpcIncomingMessage = | { type: 'evaluate' @@ -29,11 +31,6 @@ type IpcOutgoingMessage = data: any } -export type Ipc = { - sendInfo(message: IM): Promise - sendRequest(message: RM): Promise - sendError(error: Error): Promise -} const ipc = IPC as GenericIpc const queue: string[][] = [] diff --git a/turbopack/crates/turbopack-node/js/src/globals.ts b/turbopack/crates/turbopack-node/js/src/child_process/globals.ts similarity index 100% rename from turbopack/crates/turbopack-node/js/src/globals.ts rename to turbopack/crates/turbopack-node/js/src/child_process/globals.ts diff --git a/turbopack/crates/turbopack-node/js/src/ipc/index.ts b/turbopack/crates/turbopack-node/js/src/child_process/index.ts similarity index 91% rename from turbopack/crates/turbopack-node/js/src/ipc/index.ts rename to turbopack/crates/turbopack-node/js/src/child_process/index.ts index d01d80cd708f39..d1b4773f78230c 100644 --- a/turbopack/crates/turbopack-node/js/src/ipc/index.ts +++ b/turbopack/crates/turbopack-node/js/src/child_process/index.ts @@ -1,26 +1,6 @@ import { createConnection } from 'node:net' import { Writable } from 'node:stream' -import type { StackFrame } from '../compiled/stacktrace-parser' -import { parse as parseStackTrace } from '../compiled/stacktrace-parser' -import { getProperError } from './error' - -export type StructuredError = { - name: string - message: string - stack: StackFrame[] - cause: StructuredError | undefined -} - -export function structuredError(e: unknown): StructuredError { - e = getProperError(e) - - return { - name: e.name, - message: e.message, - stack: typeof e.stack === 'string' ? parseStackTrace(e.stack) : [], - cause: e.cause ? structuredError(getProperError(e.cause)) : undefined, - } -} +import { structuredError } from '../error' type State = | { diff --git a/turbopack/crates/turbopack-node/js/src/ipc/error.ts b/turbopack/crates/turbopack-node/js/src/error.ts similarity index 74% rename from turbopack/crates/turbopack-node/js/src/ipc/error.ts rename to turbopack/crates/turbopack-node/js/src/error.ts index 62f02e86a50525..f02ff477700a0e 100644 --- a/turbopack/crates/turbopack-node/js/src/ipc/error.ts +++ b/turbopack/crates/turbopack-node/js/src/error.ts @@ -1,3 +1,6 @@ +import type { StackFrame } from './compiled/stacktrace-parser' +import { parse as parseStackTrace } from './compiled/stacktrace-parser' + // merged from next.js // https://github.com/vercel/next.js/blob/e657741b9908cf0044aaef959c0c4defb19ed6d8/packages/next/src/lib/is-error.ts // https://github.com/vercel/next.js/blob/e657741b9908cf0044aaef959c0c4defb19ed6d8/packages/next/src/shared/lib/is-plain-object.ts @@ -50,3 +53,21 @@ function isPlainObject(value: any): boolean { */ return prototype === null || prototype.hasOwnProperty('isPrototypeOf') } + +export type StructuredError = { + name: string + message: string + stack: StackFrame[] + cause: StructuredError | undefined +} + +export function structuredError(e: Error | string): StructuredError { + e = getProperError(e) + + return { + name: e.name, + message: e.message, + stack: typeof e.stack === 'string' ? parseStackTrace(e.stack) : [], + cause: e.cause ? structuredError(getProperError(e.cause)) : undefined, + } +} diff --git a/turbopack/crates/turbopack-node/js/src/transforms/transforms.ts b/turbopack/crates/turbopack-node/js/src/transforms/transforms.ts index bfd925a7f234f9..84cc090afca61d 100644 --- a/turbopack/crates/turbopack-node/js/src/transforms/transforms.ts +++ b/turbopack/crates/turbopack-node/js/src/transforms/transforms.ts @@ -2,9 +2,9 @@ * Shared utilities for our 2 transform implementations. */ -import type { Ipc } from '../ipc/evaluate' +import type { Channel as Ipc } from '../types' import { relative, isAbsolute, join, sep } from 'path' -import { type StructuredError } from '../ipc' +import { type StructuredError } from '../error' import { type StackFrame } from '../compiled/stacktrace-parser' export type IpcInfoMessage = diff --git a/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts b/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts index e72e3065d2be03..248ff5382e6182 100644 --- a/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts +++ b/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts @@ -1,14 +1,14 @@ declare const __turbopack_external_require__: { - resolve: (name: string, opt: { paths: string[] }) => string + resolve: (name: string, opt?: { paths: string[] }) => string } & ((id: string, thunk: () => any, esm?: boolean) => any) -import type { Ipc } from '../ipc/evaluate' +import type { Channel as Ipc } from '../types' import { dirname, resolve as pathResolve, relative } from 'path' import { StackFrame, parse as parseStackTrace, } from '../compiled/stacktrace-parser' -import { structuredError, type StructuredError } from '../ipc' +import { structuredError, type StructuredError } from '../error' import { fromPath, getReadEnvVariables, diff --git a/turbopack/crates/turbopack-node/js/src/types.ts b/turbopack/crates/turbopack-node/js/src/types.ts new file mode 100644 index 00000000000000..f37d49d86cec27 --- /dev/null +++ b/turbopack/crates/turbopack-node/js/src/types.ts @@ -0,0 +1,5 @@ +export type Channel = { + sendInfo(message: IM): Promise + sendRequest(message: RM): Promise + sendError(error: Error): Promise +} diff --git a/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts new file mode 100644 index 00000000000000..ffedf350d55c66 --- /dev/null +++ b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts @@ -0,0 +1,165 @@ +import { threadId as workerId, workerData } from 'worker_threads' +import { structuredError } from '../error' +import type { Channel } from '../types' + +interface Binding { + recvWorkerRequest(poolId: string): Promise + recvMessageInWorker(workerId: number): Promise + notifyOneWorkerCreated(filename: string): Promise + notifyWorkerAck(taskId: string, workerId: number): Promise + sendTaskMessage(taskId: string, message: string): Promise +} + +// FIXME: require correct path on diffrent platform +const binding: Binding = require(/* turbopackIgnore: true */ '@next/swc/native/next-swc.darwin-arm64.node') + +binding.notifyOneWorkerCreated(workerData.poolId) + +const queue: string[][] = [] + +export const run = async ( + moduleFactory: () => Promise<{ + init?: () => Promise + default: (channel: Channel, ...deserializedArgs: any[]) => any + }> +) => { + const taskId = await binding.recvWorkerRequest(workerData.poolId) + + await binding.notifyWorkerAck(taskId, workerId) + + let nextId = 1 + const requests = new Map() + const internalIpc = { + sendInfo: (message: any) => + binding.sendTaskMessage( + taskId, + JSON.stringify({ + type: 'info', + data: message, + }) + ), + sendRequest: async (message: any) => { + const id = nextId++ + let resolve, reject + const promise = new Promise((res, rej) => { + resolve = res + reject = rej + }) + requests.set(id, { resolve, reject }) + return binding + .sendTaskMessage( + taskId, + JSON.stringify({ type: 'request', id, data: message }) + ) + .then(() => promise) + }, + sendError: async (error: Error) => { + try { + await binding.sendTaskMessage( + taskId, + JSON.stringify({ + type: 'error', + ...structuredError(error), + }) + ) + } catch (err) { + // There's nothing we can do about errors that happen after this point, we can't tell anyone + // about them. + console.error('failed to send error back to rust:', err) + } + }, + } + + let getValue: (channel: Channel, ...deserializedArgs: any[]) => any + try { + const module = await moduleFactory() + if (typeof module.init === 'function') { + await module.init() + } + getValue = module.default + } catch (err) { + try { + await binding.sendTaskMessage( + taskId, + JSON.stringify({ + type: 'error', + ...structuredError(err as Error), + }) + ) + } catch (err) { + // There's nothing we can do about errors that happen after this point, we can't tell anyone + // about them. + console.error('failed to send error back to rust:', err) + } + } + + let isRunning = false + + const run = async () => { + while (queue.length > 0) { + const args = queue.shift()! + try { + const value = await getValue(internalIpc, ...args) + await binding.sendTaskMessage( + taskId, + JSON.stringify({ + type: 'end', + data: value === undefined ? undefined : JSON.stringify(value), + duration: 0, + }) + ) + } catch (err) { + await binding.sendTaskMessage( + taskId, + JSON.stringify({ + type: 'error', + ...structuredError(err as Error), + }) + ) + } + } + isRunning = false + } + + while (true) { + const msg_str = await binding.recvMessageInWorker(workerId) + + const msg = JSON.parse(msg_str) as + | { + type: 'evaluate' + args: string[] + } + | { + type: 'result' + id: number + error?: string + data?: any + } + + switch (msg.type) { + case 'evaluate': { + queue.push(msg.args) + if (!isRunning) { + isRunning = true + run() + } + break + } + case 'result': { + const request = requests.get(msg.id) + if (request) { + requests.delete(msg.id) + if (msg.error) { + request.reject(new Error(msg.error)) + } else { + request.resolve(msg.data) + } + } + break + } + default: { + console.error('unexpected message type', (msg as any).type) + } + } + } +} diff --git a/turbopack/crates/turbopack-node/src/evaluate.rs b/turbopack/crates/turbopack-node/src/evaluate.rs index 9df17e85ceac18..c8244f40c18bdf 100644 --- a/turbopack/crates/turbopack-node/src/evaluate.rs +++ b/turbopack/crates/turbopack-node/src/evaluate.rs @@ -31,10 +31,10 @@ use turbopack_core::{ virtual_source::VirtualSource, }; -#[cfg(feature = "child_process")] -use crate::process_pool::ChildProcessPool as Pool; -#[cfg(feature = "worker_thread")] -use crate::worker_pool::WorkerThreadPool as Pool; +#[cfg(feature = "process_pool")] +use crate::process_pool::ChildProcessPool; +#[cfg(feature = "worker_pool")] +use crate::worker_pool::WorkerThreadPool; use crate::{ AssetsForSourceMapping, embed_js::embed_file_path, emit, emit_package_json, format::FormattingMode, internal_assets_for_source_mapping, source_map::StructuredError, @@ -102,9 +102,9 @@ pub trait EvaluateOperation: Send + Sync { #[async_trait::async_trait] pub trait Operation: Send { - async fn recv(&mut self) -> Result>; + async fn recv(&mut self) -> Result; - async fn send(&mut self, data: Vec) -> Result<()>; + async fn send(&mut self, data: String) -> Result<()>; } #[turbo_tasks::value] @@ -246,7 +246,21 @@ pub async fn get_evaluate_pool( env.read_all().untracked().await? } }; - let pool = Pool::create( + + #[cfg(feature = "process_pool")] + #[allow(unused_variables)] + let pool = ChildProcessPool::create( + cwd.clone(), + entrypoint.clone(), + env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + assets_for_source_mapping, + output_root.clone(), + chunking_context.root_path().owned().await?, + available_parallelism().map_or(1, |v| v.get()), + debug, + ); + #[cfg(feature = "worker_pool")] + let pool = WorkerThreadPool::create( cwd, entrypoint, env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), @@ -355,7 +369,7 @@ pub async fn custom_evaluate(evaluate_context: impl EvaluateContext) -> Result>, runtime_entries: Option>, ) -> Result> { + #[cfg(feature = "process_pool")] + #[allow(unused_variables)] + let runtime_module_path = rcstr!("child_process/evaluate.ts"); + #[cfg(feature = "worker_pool")] + let runtime_module_path = rcstr!("worker_threads/evaluate.ts"); + let runtime_asset = asset_context .process( Vc::upcast(FileSource::new( - embed_file_path(rcstr!("ipc/evaluate.ts")).owned().await?, + embed_file_path(runtime_module_path).owned().await?, )), ReferenceType::Internal(InnerAssets::empty().to_resolved().await?), ) @@ -438,22 +458,29 @@ pub async fn get_evaluate_entries( .await?; let runtime_entries = { - let globals_module = asset_context - .process( - Vc::upcast(FileSource::new( - embed_file_path(rcstr!("globals.ts")).owned().await?, - )), - ReferenceType::Internal(InnerAssets::empty().to_resolved().await?), - ) - .module(); - - let Some(globals_module) = - Vc::try_resolve_sidecast::>(globals_module).await? - else { - bail!("Internal module is not evaluatable"); - }; - - let mut entries = vec![globals_module.to_resolved().await?]; + let mut entries = vec![]; + + #[cfg(feature = "process_pool")] + { + let globals_module = asset_context + .process( + Vc::upcast(FileSource::new( + embed_file_path(rcstr!("child_process/globals.ts")) + .owned() + .await?, + )), + ReferenceType::Internal(InnerAssets::empty().to_resolved().await?), + ) + .module(); + + let Some(globals_module) = + Vc::try_resolve_sidecast::>(globals_module).await? + else { + bail!("Internal module is not evaluatable"); + }; + + entries.push(globals_module.to_resolved().await?); + } if let Some(runtime_entries) = runtime_entries { for &entry in &*runtime_entries.await? { entries.push(entry) @@ -512,8 +539,7 @@ async fn pull_operation( let _guard = duration_span!("Node.js evaluation"); loop { - let buf = operation.recv().await?; - let message = parse_json_with_source_context(std::str::from_utf8(&buf)?)?; + let message = parse_json_with_source_context(&operation.recv().await?)?; match message { EvalJavaScriptIncomingMessage::Error(error) => { @@ -536,7 +562,7 @@ async fn pull_operation( { Ok(response) => { operation - .send(serde_json::to_vec( + .send(serde_json::to_string( &EvalJavaScriptOutgoingMessage::Result { id, error: None, @@ -547,7 +573,7 @@ async fn pull_operation( } Err(e) => { operation - .send(serde_json::to_vec( + .send(serde_json::to_string( &EvalJavaScriptOutgoingMessage::Result { id, error: Some(PrettyPrintError(&e).to_string()), diff --git a/turbopack/crates/turbopack-node/src/lib.rs b/turbopack/crates/turbopack-node/src/lib.rs index 0008c5475d4936..8aa0106456403e 100644 --- a/turbopack/crates/turbopack-node/src/lib.rs +++ b/turbopack/crates/turbopack-node/src/lib.rs @@ -18,11 +18,11 @@ pub mod embed_js; pub mod evaluate; pub mod execution_context; mod format; -#[cfg(feature = "child_process")] +#[cfg(feature = "process_pool")] mod process_pool; pub mod source_map; pub mod transforms; -#[cfg(feature = "worker_thread")] +#[cfg(feature = "worker_pool")] mod worker_pool; #[turbo_tasks::function] diff --git a/turbopack/crates/turbopack-node/src/process_pool/mod.rs b/turbopack/crates/turbopack-node/src/process_pool/mod.rs index 2e852714cae06e..602c8583cec367 100644 --- a/turbopack/crates/turbopack-node/src/process_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/process_pool/mod.rs @@ -730,16 +730,21 @@ impl ChildProcessPool { impl EvaluateOperation for ChildProcessPool { async fn operation(&self) -> Result> { // Acquire a running process (handles concurrency limits, boots up the process) - let (process, permits) = self.acquire_process().await?; - Ok(Box::new(NodeJsOperation { - process: Some(process), - permits, - idle_processes: self.idle_processes.clone(), - start: Instant::now(), - stats: self.stats.clone(), - allow_process_reuse: true, - })) + let operation = { + let _guard = duration_span!("Node.js operation"); + let (process, permits) = self.acquire_process().await?; + NodeJsOperation { + process: Some(process), + permits, + idle_processes: self.idle_processes.clone(), + start: Instant::now(), + stats: self.stats.clone(), + allow_process_reuse: true, + } + }; + + Ok(Box::new(operation)) } } @@ -830,16 +835,18 @@ pub struct NodeJsOperation { #[async_trait::async_trait] impl Operation for NodeJsOperation { - async fn recv(&mut self) -> Result> { - self.with_process(|process| async move { - process.recv().await.context("failed to receive message") - }) - .await + async fn recv(&mut self) -> Result { + let vec = self + .with_process(|process| async move { + process.recv().await.context("failed to receive message") + }) + .await?; + Ok(String::from_utf8(vec)?) } - async fn send(&mut self, data: Vec) -> Result<()> { + async fn send(&mut self, message: String) -> Result<()> { self.with_process(|process| async move { - timeout(Duration::from_secs(30), process.send(data)) + timeout(Duration::from_secs(30), process.send(message.into_bytes())) .await .context("timeout while sending message")? .context("failed to send message")?; diff --git a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs index ad69187022a0d4..78c695c8da9900 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs @@ -2,8 +2,9 @@ use std::path::PathBuf; use anyhow::Result; use rustc_hash::FxHashMap; +use tokio::sync::OnceCell; use turbo_rcstr::RcStr; -use turbo_tasks::ResolvedVc; +use turbo_tasks::{ResolvedVc, duration_span}; use turbo_tasks_fs::FileSystemPath; use crate::{ @@ -13,14 +14,20 @@ use crate::{ }; mod operation; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] mod worker_thread; +#[cfg(all(target_family = "wasm", target_os = "unknown"))] +mod web_worker; + #[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)] pub struct WorkerThreadPool { cwd: PathBuf, entrypoint: PathBuf, env: FxHashMap, concurrency: usize, + #[turbo_tasks(trace_ignore, debug_ignore)] + ready: OnceCell<()>, pub assets_for_source_mapping: ResolvedVc, pub assets_root: FileSystemPath, pub project_dir: FileSystemPath, @@ -44,6 +51,7 @@ impl WorkerThreadPool { entrypoint, env, concurrency, + ready: OnceCell::new(), assets_for_source_mapping, assets_root: assets_root.clone(), project_dir: project_dir.clone(), @@ -58,22 +66,33 @@ impl WorkerThreadPool { #[async_trait::async_trait] impl EvaluateOperation for WorkerThreadPool { async fn operation(&self) -> Result> { - create_pool( - self.entrypoint.to_string_lossy().to_string(), - self.concurrency, - ) - .await?; + let operation = { + let _guard = duration_span!("Node.js operation"); + let entrypoint = self.entrypoint.to_string_lossy().to_string(); + self.ready + .get_or_init(async || { + create_pool( + self.entrypoint.to_string_lossy().to_string(), + self.concurrency, + ) + .await + .unwrap_or_else(|e| { + panic!("failed to create worker pool for {entrypoint} for reason: {e}",) + }) + }) + .await; - let task_id = uuid::Uuid::new_v4().to_string(); + let task_id = uuid::Uuid::new_v4().to_string(); - let worker_id = connect_to_worker( - self.entrypoint.to_string_lossy().to_string(), - task_id.clone(), - ) - .await?; + let worker_id = connect_to_worker( + self.entrypoint.to_string_lossy().to_string(), + task_id.clone(), + ) + .await?; - let worker_operation = WorkerOperation { task_id, worker_id }; + WorkerOperation { task_id, worker_id } + }; - Ok(Box::new(worker_operation)) + Ok(Box::new(operation)) } } diff --git a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs index c6f1bbc4d6a106..3b5065fdb4bc1e 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs @@ -1,11 +1,12 @@ use std::sync::LazyLock; use anyhow::{Context, Result}; -use async_channel::{Receiver, Sender, bounded, unbounded}; +use async_channel::{Receiver, Sender, unbounded}; use dashmap::DashMap; use crate::evaluate::Operation; +#[derive(Clone)] pub(crate) struct MessageChannel { sender: Sender, receiver: Receiver, @@ -16,11 +17,6 @@ impl MessageChannel { let (sender, receiver) = unbounded::(); Self { sender, receiver } } - - pub(super) fn bounded(cap: usize) -> Self { - let (sender, receiver) = bounded::(cap); - Self { sender, receiver } - } } impl MessageChannel { @@ -31,97 +27,203 @@ impl MessageChannel { pub(crate) async fn recv(&self) -> Result { Ok(self.receiver.recv().await?) } + + pub(crate) fn try_recv(&self) -> Result { + Ok(self.receiver.try_recv()?) + } } -pub(crate) static POOL_REQUEST_CHANNEL: LazyLock> = - LazyLock::new(MessageChannel::unbounded); +pub(crate) struct WorkerPoolOperation { + pool_request_channel: MessageChannel<(String, usize)>, + pool_ack_channel: DashMap>, + worker_request_channel: DashMap>, + worker_ack_channel: DashMap>, + worker_routed_channel: DashMap>, + task_routed_channel: DashMap>, +} -pub(crate) static POOL_CREATION_CHANNEL: LazyLock>> = - LazyLock::new(DashMap::new); +impl Default for WorkerPoolOperation { + fn default() -> Self { + Self { + pool_request_channel: MessageChannel::unbounded(), + pool_ack_channel: DashMap::new(), + worker_request_channel: DashMap::new(), + worker_ack_channel: DashMap::new(), + worker_routed_channel: DashMap::new(), + task_routed_channel: DashMap::new(), + } + } +} -pub(crate) async fn create_pool(filename: String, concurrency: usize) -> anyhow::Result<()> { - POOL_REQUEST_CHANNEL - .send(filename.clone()) - .await - .context("failed to send pool request")?; +impl WorkerPoolOperation { + pub(crate) async fn create_pool( + &self, + filename: String, + concurrency: usize, + ) -> anyhow::Result<()> { + self.pool_request_channel + .send((filename.clone(), concurrency)) + .await + .context("failed to send pool request")?; - let mut created_worker_count = 0; + let mut created_worker_count = 0; - { - let channel = POOL_CREATION_CHANNEL - .entry(filename.clone()) - .or_insert_with(|| MessageChannel::bounded(concurrency)); - - while created_worker_count < concurrency { - channel - .recv() - .await - .context("failed to recv worker creation")?; - created_worker_count += 1; - } - }; + { + let channel = self + .pool_ack_channel + .entry(filename.clone()) + .or_insert_with(MessageChannel::unbounded) + .clone(); - POOL_CREATION_CHANNEL.remove(&filename); + while created_worker_count < concurrency { + channel + .recv() + .await + .context("failed to recv worker creation")?; + created_worker_count += 1; + } + }; - Ok(()) -} + self.pool_ack_channel.remove(&filename); -pub(crate) static EVALUATION_REQUEST_CHANNAL: LazyLock>>> = - LazyLock::new(DashMap::new); + Ok(()) + } -pub(crate) static WORKER_REQUEST_CHANNAL: LazyLock>> = - LazyLock::new(DashMap::new); + pub(crate) async fn connect_to_worker(&self, pool_id: String, task_id: String) -> Result { + let channel = self + .worker_request_channel + .entry(pool_id.clone()) + .or_insert_with(MessageChannel::unbounded) + .clone(); + channel + .send(task_id.clone()) + .await + .context("failed to send worker request")?; + let worker_id = async move { + let channel = self + .worker_ack_channel + .entry(task_id.clone()) + .or_insert_with(MessageChannel::unbounded) + .clone(); + channel.recv().await.context("failed to recv worker ack") + } + .await?; + Ok(worker_id) + } -pub(crate) static WORKER_ACK_CHANNAL: LazyLock>> = - LazyLock::new(DashMap::new); + pub(crate) async fn send_message_to_worker(&self, worker_id: u32, data: String) -> Result<()> { + let entry = self + .worker_routed_channel + .entry(worker_id) + .or_insert_with(MessageChannel::unbounded) + .clone(); + entry + .send(data) + .await + .with_context(|| format!("failed to send message to worker {worker_id}"))?; + Ok(()) + } -pub(crate) async fn connect_to_worker(pool_id: String, task_id: String) -> Result { - let channel = WORKER_REQUEST_CHANNAL - .entry(pool_id.clone()) - .or_insert_with(MessageChannel::unbounded); - channel - .send(()) - .await - .context("failed to send evaluation request")?; - let worker_id = async move { - let channel = WORKER_ACK_CHANNAL + pub async fn recv_task_response(&self, task_id: String) -> Result { + let channel = self + .task_routed_channel .entry(task_id.clone()) - .or_insert_with(MessageChannel::unbounded); + .or_insert_with(MessageChannel::unbounded) + .clone(); + let data = channel + .recv() + .await + .with_context(|| format!("failed to recv message for task {task_id}"))?; + Ok(data) + } + + pub(crate) fn try_recv_pool_creation(&self) -> Option<(String, usize)> { + self.pool_request_channel.try_recv().ok() + } + + pub(crate) async fn notify_one_worker_created(&self, filename: String) -> Result<()> { + let channel = self + .pool_ack_channel + .entry(filename.clone()) + .or_insert_with(MessageChannel::unbounded) + .clone(); + channel + .send(()) + .await + .context("failed to notify worker created") + } + + pub(crate) async fn recv_worker_request(&self, pool_id: String) -> Result { + let channel = self + .worker_request_channel + .entry(pool_id.clone()) + .or_insert_with(MessageChannel::unbounded) + .clone(); channel .recv() .await - .context("failed to recv evaluation ack") + .context("failed to recv worker request") + } + + pub(crate) async fn notify_worker_ack(&self, task_id: String, worker_id: u32) -> Result<()> { + let channel = self + .worker_ack_channel + .get(&task_id) + .context(format!("worker ack channel for {task_id} not found"))?; + channel + .send(worker_id) + .await + .context("failed to notify worker ack") + } + + pub(crate) async fn recv_message_in_worker(&self, worker_id: u32) -> Result { + let channel = self + .worker_routed_channel + .entry(worker_id) + .or_insert_with(MessageChannel::unbounded) + .clone(); + channel + .recv() + .await + .with_context(|| format!("failed to recv message in worker {worker_id}")) + } + + pub(crate) async fn send_task_message(&self, task_id: String, data: String) -> Result<()> { + let channel = self + .task_routed_channel + .entry(task_id.clone()) + .or_insert_with(MessageChannel::unbounded) + .clone(); + channel + .send(data) + .await + .with_context(|| format!("failed to send response for task {task_id}")) } - .await?; - Ok(worker_id) } -pub(crate) static WORKER_ROUTED_CHANNEL: LazyLock>>> = - LazyLock::new(DashMap::new); +pub(crate) static WORKER_POOL_OPERATION: LazyLock = + LazyLock::new(WorkerPoolOperation::default); -pub(crate) async fn send_message_to_worker(worker_id: u32, data: Vec) -> Result<()> { - let entry = WORKER_ROUTED_CHANNEL - .entry(worker_id) - .or_insert_with(MessageChannel::unbounded); - entry - .send(data) +pub(crate) async fn create_pool(filename: String, concurrency: usize) -> anyhow::Result<()> { + WORKER_POOL_OPERATION + .create_pool(filename, concurrency) .await - .with_context(|| format!("failed to send message to worker {worker_id}"))?; - Ok(()) } -pub(crate) static TASK_ROUTERD_CHANNEL: LazyLock>>> = - LazyLock::new(DashMap::new); +pub(crate) async fn connect_to_worker(pool_id: String, task_id: String) -> Result { + WORKER_POOL_OPERATION + .connect_to_worker(pool_id, task_id) + .await +} -pub async fn recv_task_response(task_id: String) -> Result> { - let channel = TASK_ROUTERD_CHANNEL - .entry(task_id.clone()) - .or_insert_with(MessageChannel::unbounded); - let data = channel - .recv() +pub(crate) async fn send_message_to_worker(worker_id: u32, data: String) -> Result<()> { + WORKER_POOL_OPERATION + .send_message_to_worker(worker_id, data) .await - .with_context(|| format!("failed to send message to worker {task_id}"))?; - Ok(data) +} + +pub async fn recv_task_response(task_id: String) -> Result { + WORKER_POOL_OPERATION.recv_task_response(task_id).await } pub(crate) struct WorkerOperation { @@ -131,11 +233,11 @@ pub(crate) struct WorkerOperation { #[async_trait::async_trait] impl Operation for WorkerOperation { - async fn recv(&mut self) -> Result> { + async fn recv(&mut self) -> Result { recv_task_response(self.task_id.clone()).await } - async fn send(&mut self, data: Vec) -> Result<()> { + async fn send(&mut self, data: String) -> Result<()> { send_message_to_worker(self.worker_id, data).await } } diff --git a/turbopack/crates/turbopack-node/src/worker_pool/web_worker.rs b/turbopack/crates/turbopack-node/src/worker_pool/web_worker.rs new file mode 100644 index 00000000000000..8b137891791fe9 --- /dev/null +++ b/turbopack/crates/turbopack-node/src/worker_pool/web_worker.rs @@ -0,0 +1 @@ + diff --git a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs index 22be69b3a13f9b..a3cf197cb32063 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs @@ -1,100 +1,59 @@ -use anyhow::Context; use napi_derive::napi; -use crate::worker_pool::operation::{ - EVALUATION_REQUEST_CHANNAL, MessageChannel, POOL_CREATION_CHANNEL, POOL_REQUEST_CHANNEL, - TASK_ROUTERD_CHANNEL, WORKER_REQUEST_CHANNAL, WORKER_ROUTED_CHANNEL, -}; +use crate::worker_pool::operation::WORKER_POOL_OPERATION; -#[napi] -pub async fn recv_pool_request() -> napi::Result { - Ok(POOL_REQUEST_CHANNEL - .recv() - .await - .context("failed to recv pool request")?) +#[napi(object)] +pub struct PoolOptions { + pub filename: String, + pub concurrency: u32, } #[napi] -pub async fn notify_pool_created(filename: String) -> napi::Result<()> { - let channel = if let Some(channel) = POOL_CREATION_CHANNEL.get(&filename) { - channel - } else { - return Err(napi::Error::from_reason(format!( - "pool creation channel for {filename} not found" - ))); - }; - Ok(channel - .send(()) - .await - .context("failed to notify pool created")?) +pub fn recv_pool_creation() -> Option { + WORKER_POOL_OPERATION + .try_recv_pool_creation() + .map(|(filename, concurrency)| PoolOptions { + filename, + concurrency: concurrency as u32, + }) } #[napi] -pub async fn recv_worker_request(pool_id: String) -> napi::Result<()> { - let channel = if let Some(channel) = WORKER_REQUEST_CHANNAL.get(&pool_id) { - channel - } else { - return Err(napi::Error::from_reason(format!( - "worker request channel for {pool_id} not found" - ))); - }; - Ok(channel - .send(()) +pub async fn recv_worker_request(pool_id: String) -> napi::Result { + WORKER_POOL_OPERATION + .recv_worker_request(pool_id) .await - .context("failed to recv worker request")?) + .map_err(|e| napi::Error::from_reason(e.to_string())) } #[napi] -pub async fn notify_worker_ack(pool_id: String) -> napi::Result<()> { - let channel = if let Some(channel) = POOL_CREATION_CHANNEL.get(&pool_id) { - channel - } else { - return Err(napi::Error::from_reason(format!( - "evaluation ack channel for {pool_id} not found" - ))); - }; - Ok(channel - .send(()) - .await - .context("failed to notify evaluation ack")?) +// TODO: use zero-copy externaled type array +pub async fn recv_message_in_worker(worker_id: u32) -> napi::Result { + Ok(WORKER_POOL_OPERATION + .recv_message_in_worker(worker_id) + .await?) } #[napi] -pub async fn recv_evaluation(pool_id: String) -> napi::Result> { - let channel = if let Some(channel) = EVALUATION_REQUEST_CHANNAL.get(&pool_id) { - channel - } else { - return Err(napi::Error::from_reason(format!( - "evaluation request channel for {pool_id} not found" - ))); - }; - - Ok(channel - .recv() +pub async fn notify_one_worker_created(filename: String) -> napi::Result<()> { + WORKER_POOL_OPERATION + .notify_one_worker_created(filename) .await - .context("failed to recv evaluate request")?) + .map_err(|e| napi::Error::from_reason(e.to_string())) } #[napi] -pub async fn recv_message_in_worker(worker_id: u32) -> napi::Result> { - let channel = WORKER_ROUTED_CHANNEL - .entry(worker_id) - .or_insert_with(MessageChannel::unbounded); - let data = channel - .recv() +pub async fn notify_worker_ack(task_id: String, worker_id: u32) -> napi::Result<()> { + WORKER_POOL_OPERATION + .notify_worker_ack(task_id, worker_id) .await - .with_context(|| format!("failed to recv message in worker {worker_id}"))?; - Ok(data) + .map_err(|e| napi::Error::from_reason(e.to_string())) } #[napi] -pub async fn send_task_response(task_id: String, data: Vec) -> napi::Result<()> { - let channel = TASK_ROUTERD_CHANNEL - .entry(task_id.clone()) - .or_insert_with(MessageChannel::unbounded); - channel - .send(data) +pub async fn send_task_message(task_id: String, message: String) -> napi::Result<()> { + WORKER_POOL_OPERATION + .send_task_message(task_id, message) .await - .with_context(|| format!("failed to recv message in worker {task_id}"))?; - Ok(()) + .map_err(|e| napi::Error::from_reason(e.to_string())) } diff --git a/turbopack/crates/turbopack-tests/Cargo.toml b/turbopack/crates/turbopack-tests/Cargo.toml index d173fc14c0de96..42fe67af232842 100644 --- a/turbopack/crates/turbopack-tests/Cargo.toml +++ b/turbopack/crates/turbopack-tests/Cargo.toml @@ -39,7 +39,7 @@ turbopack-ecmascript-plugins = { workspace = true, features = [ ] } turbopack-ecmascript-runtime = { workspace = true } turbopack-env = { workspace = true } -turbopack-node = { workspace = true } +turbopack-node = { workspace = true, default-features = false, features = ["process_pool"] } turbopack-nodejs = { workspace = true, features = ["test"] } turbopack-resolve = { workspace = true } turbopack-test-utils = { workspace = true } diff --git a/turbopack/crates/turbopack/Cargo.toml b/turbopack/crates/turbopack/Cargo.toml index 557ef25204aafa..061a63bb530a52 100644 --- a/turbopack/crates/turbopack/Cargo.toml +++ b/turbopack/crates/turbopack/Cargo.toml @@ -35,7 +35,7 @@ turbopack-ecmascript = { workspace = true } turbopack-env = { workspace = true } turbopack-json = { workspace = true } turbopack-mdx = { workspace = true } -turbopack-node = { workspace = true } +turbopack-node = { workspace = true, default-features = false } turbopack-resolve = { workspace = true } turbopack-static = { workspace = true } turbopack-wasm = { workspace = true } From 01903268824f98619220e4d67b7eb874e611626a Mon Sep 17 00:00:00 2001 From: xusd320 Date: Tue, 25 Nov 2025 14:10:47 +0800 Subject: [PATCH 04/17] chore: remove build.rs in turbopack-node --- turbopack/crates/turbopack-node/build.rs | 6 ------ turbopack/crates/turbopack-node/src/worker_pool/mod.rs | 4 ---- .../crates/turbopack-node/src/worker_pool/web_worker.rs | 1 - 3 files changed, 11 deletions(-) delete mode 100644 turbopack/crates/turbopack-node/build.rs delete mode 100644 turbopack/crates/turbopack-node/src/worker_pool/web_worker.rs diff --git a/turbopack/crates/turbopack-node/build.rs b/turbopack/crates/turbopack-node/build.rs deleted file mode 100644 index 8efb97e10c4e7f..00000000000000 --- a/turbopack/crates/turbopack-node/build.rs +++ /dev/null @@ -1,6 +0,0 @@ -fn main() { - #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] - if std::env::var("CARGO_FEATURE_WORKER_THREAD").is_ok() { - napi_build::setup(); - } -} diff --git a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs index 78c695c8da9900..255b3271488c6d 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs @@ -14,12 +14,8 @@ use crate::{ }; mod operation; -#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] mod worker_thread; -#[cfg(all(target_family = "wasm", target_os = "unknown"))] -mod web_worker; - #[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)] pub struct WorkerThreadPool { cwd: PathBuf, diff --git a/turbopack/crates/turbopack-node/src/worker_pool/web_worker.rs b/turbopack/crates/turbopack-node/src/worker_pool/web_worker.rs deleted file mode 100644 index 8b137891791fe9..00000000000000 --- a/turbopack/crates/turbopack-node/src/worker_pool/web_worker.rs +++ /dev/null @@ -1 +0,0 @@ - From e59a6b7bbe44e814b8f708a67a5d52e1bc36cd2c Mon Sep 17 00:00:00 2001 From: xusd320 Date: Wed, 26 Nov 2025 01:47:27 +0800 Subject: [PATCH 05/17] fix: turbopack ci --- Cargo.lock | 1 + crates/next-build-test/Cargo.toml | 2 +- turbopack/crates/turbopack-cli/Cargo.toml | 2 +- turbopack/crates/turbopack-node/Cargo.toml | 2 +- turbopack/crates/turbopack-tests/Cargo.toml | 3 ++- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6ae932daaf3e6..414df4ace63a6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4341,6 +4341,7 @@ dependencies = [ "turbo-tasks", "turbo-tasks-backend", "turbo-tasks-malloc", + "turbopack-node", "turbopack-trace-utils", ] diff --git a/crates/next-build-test/Cargo.toml b/crates/next-build-test/Cargo.toml index b2fc71e70f6f6a..6c9a51cf34f3e8 100644 --- a/crates/next-build-test/Cargo.toml +++ b/crates/next-build-test/Cargo.toml @@ -27,4 +27,4 @@ turbo-tasks = { workspace = true } turbo-tasks-backend = { workspace = true } turbo-tasks-malloc = { workspace = true } turbopack-trace-utils = { workspace = true } - +turbopack-node = { workspace = true, default-features = false, features = ["process_pool"]} diff --git a/turbopack/crates/turbopack-cli/Cargo.toml b/turbopack/crates/turbopack-cli/Cargo.toml index 0ed62123619d7d..eaacd8e3be96ba 100644 --- a/turbopack/crates/turbopack-cli/Cargo.toml +++ b/turbopack/crates/turbopack-cli/Cargo.toml @@ -66,7 +66,7 @@ turbopack-ecmascript-plugins = { workspace = true, features = [ ] } turbopack-ecmascript-runtime = { workspace = true } turbopack-env = { workspace = true } -turbopack-node = { workspace = true, default-features = false, features = ["process_pool"] } +turbopack-node = { workspace = true, default-features = false, features = ["process_pool"]} turbopack-nodejs = { workspace = true } turbopack-resolve = { workspace = true } turbopack-trace-utils = { workspace = true } diff --git a/turbopack/crates/turbopack-node/Cargo.toml b/turbopack/crates/turbopack-node/Cargo.toml index 405131c08a3557..b90bad9bf3843d 100644 --- a/turbopack/crates/turbopack-node/Cargo.toml +++ b/turbopack/crates/turbopack-node/Cargo.toml @@ -10,7 +10,7 @@ autobenches = false bench = false [features] -default = ["worker_pool"] +default = ["process_pool"] # enable "HMR" for embedded assets dynamic_embed_contents = ["turbo-tasks-fs/dynamic_embed_contents"] process_pool = ["tokio/full"] diff --git a/turbopack/crates/turbopack-tests/Cargo.toml b/turbopack/crates/turbopack-tests/Cargo.toml index 42fe67af232842..2209423c2cb2cc 100644 --- a/turbopack/crates/turbopack-tests/Cargo.toml +++ b/turbopack/crates/turbopack-tests/Cargo.toml @@ -14,6 +14,7 @@ workspace = true [dependencies] turbopack = { workspace = true } +turbopack-node = { workspace = true, default-features = false, features = ["process_pool"]} [dev-dependencies] anyhow = { workspace = true } @@ -39,7 +40,7 @@ turbopack-ecmascript-plugins = { workspace = true, features = [ ] } turbopack-ecmascript-runtime = { workspace = true } turbopack-env = { workspace = true } -turbopack-node = { workspace = true, default-features = false, features = ["process_pool"] } +turbopack-node = { workspace = true, default-features = false, features = ["process_pool"]} turbopack-nodejs = { workspace = true, features = ["test"] } turbopack-resolve = { workspace = true } turbopack-test-utils = { workspace = true } From c388c0dcdaa4ced084322eb7bec99a9a1cef3693 Mon Sep 17 00:00:00 2001 From: xusd320 Date: Wed, 26 Nov 2025 02:01:45 +0800 Subject: [PATCH 06/17] chore: rename types --- .../crates/turbopack-node/src/process_pool/mod.rs | 10 +++++----- .../crates/turbopack-node/src/worker_pool/operation.rs | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/turbopack/crates/turbopack-node/src/process_pool/mod.rs b/turbopack/crates/turbopack-node/src/process_pool/mod.rs index 602c8583cec367..f81a157dc8fe4d 100644 --- a/turbopack/crates/turbopack-node/src/process_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/process_pool/mod.rs @@ -734,7 +734,7 @@ impl EvaluateOperation for ChildProcessPool { let operation = { let _guard = duration_span!("Node.js operation"); let (process, permits) = self.acquire_process().await?; - NodeJsOperation { + ChildProcessOperation { process: Some(process), permits, idle_processes: self.idle_processes.clone(), @@ -822,7 +822,7 @@ impl ChildProcessPool { } } -pub struct NodeJsOperation { +pub struct ChildProcessOperation { process: Option, // This is used for drop #[allow(dead_code)] @@ -834,7 +834,7 @@ pub struct NodeJsOperation { } #[async_trait::async_trait] -impl Operation for NodeJsOperation { +impl Operation for ChildProcessOperation { async fn recv(&mut self) -> Result { let vec = self .with_process(|process| async move { @@ -856,7 +856,7 @@ impl Operation for NodeJsOperation { } } -impl NodeJsOperation { +impl ChildProcessOperation { async fn with_process<'a, F: Future> + Send + 'a, T>( &'a mut self, f: impl FnOnce(&'a mut NodeJsPoolProcess) -> F, @@ -913,7 +913,7 @@ impl NodeJsOperation { } } -impl Drop for NodeJsOperation { +impl Drop for ChildProcessOperation { fn drop(&mut self) { if let Some(mut process) = self.process.take() { let elapsed = self.start.elapsed(); diff --git a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs index 3b5065fdb4bc1e..4b48996ec42f3f 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs @@ -33,7 +33,7 @@ impl MessageChannel { } } -pub(crate) struct WorkerPoolOperation { +pub(crate) struct WorkerThreadOperation { pool_request_channel: MessageChannel<(String, usize)>, pool_ack_channel: DashMap>, worker_request_channel: DashMap>, @@ -42,7 +42,7 @@ pub(crate) struct WorkerPoolOperation { task_routed_channel: DashMap>, } -impl Default for WorkerPoolOperation { +impl Default for WorkerThreadOperation { fn default() -> Self { Self { pool_request_channel: MessageChannel::unbounded(), @@ -55,7 +55,7 @@ impl Default for WorkerPoolOperation { } } -impl WorkerPoolOperation { +impl WorkerThreadOperation { pub(crate) async fn create_pool( &self, filename: String, @@ -201,8 +201,8 @@ impl WorkerPoolOperation { } } -pub(crate) static WORKER_POOL_OPERATION: LazyLock = - LazyLock::new(WorkerPoolOperation::default); +pub(crate) static WORKER_POOL_OPERATION: LazyLock = + LazyLock::new(WorkerThreadOperation::default); pub(crate) async fn create_pool(filename: String, concurrency: usize) -> anyhow::Result<()> { WORKER_POOL_OPERATION From eaf47da0afd205af7079a2e2fcd178ac4f928cdc Mon Sep 17 00:00:00 2001 From: xusd320 Date: Wed, 26 Nov 2025 02:30:37 +0800 Subject: [PATCH 07/17] fix: features conflict when cargo clippy --- .../crates/turbopack-node/src/worker_pool/worker_thread.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs index a3cf197cb32063..d879c6ef287e54 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs @@ -3,12 +3,14 @@ use napi_derive::napi; use crate::worker_pool::operation::WORKER_POOL_OPERATION; #[napi(object)] +#[allow(unused)] pub struct PoolOptions { pub filename: String, pub concurrency: u32, } #[napi] +#[allow(unused)] pub fn recv_pool_creation() -> Option { WORKER_POOL_OPERATION .try_recv_pool_creation() @@ -19,6 +21,7 @@ pub fn recv_pool_creation() -> Option { } #[napi] +#[allow(unused)] pub async fn recv_worker_request(pool_id: String) -> napi::Result { WORKER_POOL_OPERATION .recv_worker_request(pool_id) @@ -28,6 +31,7 @@ pub async fn recv_worker_request(pool_id: String) -> napi::Result { #[napi] // TODO: use zero-copy externaled type array +#[allow(unused)] pub async fn recv_message_in_worker(worker_id: u32) -> napi::Result { Ok(WORKER_POOL_OPERATION .recv_message_in_worker(worker_id) @@ -35,6 +39,7 @@ pub async fn recv_message_in_worker(worker_id: u32) -> napi::Result { } #[napi] +#[allow(unused)] pub async fn notify_one_worker_created(filename: String) -> napi::Result<()> { WORKER_POOL_OPERATION .notify_one_worker_created(filename) @@ -43,6 +48,7 @@ pub async fn notify_one_worker_created(filename: String) -> napi::Result<()> { } #[napi] +#[allow(unused)] pub async fn notify_worker_ack(task_id: String, worker_id: u32) -> napi::Result<()> { WORKER_POOL_OPERATION .notify_worker_ack(task_id, worker_id) @@ -51,6 +57,7 @@ pub async fn notify_worker_ack(task_id: String, worker_id: u32) -> napi::Result< } #[napi] +#[allow(unused)] pub async fn send_task_message(task_id: String, message: String) -> napi::Result<()> { WORKER_POOL_OPERATION .send_task_message(task_id, message) From bc0c59ee6b5c80ea88132d644365b8756e5293a0 Mon Sep 17 00:00:00 2001 From: xusd320 Date: Wed, 26 Nov 2025 10:49:51 +0800 Subject: [PATCH 08/17] chore: rename types --- .../crates/turbopack-node/src/worker_pool/operation.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs index 4b48996ec42f3f..3b5065fdb4bc1e 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs @@ -33,7 +33,7 @@ impl MessageChannel { } } -pub(crate) struct WorkerThreadOperation { +pub(crate) struct WorkerPoolOperation { pool_request_channel: MessageChannel<(String, usize)>, pool_ack_channel: DashMap>, worker_request_channel: DashMap>, @@ -42,7 +42,7 @@ pub(crate) struct WorkerThreadOperation { task_routed_channel: DashMap>, } -impl Default for WorkerThreadOperation { +impl Default for WorkerPoolOperation { fn default() -> Self { Self { pool_request_channel: MessageChannel::unbounded(), @@ -55,7 +55,7 @@ impl Default for WorkerThreadOperation { } } -impl WorkerThreadOperation { +impl WorkerPoolOperation { pub(crate) async fn create_pool( &self, filename: String, @@ -201,8 +201,8 @@ impl WorkerThreadOperation { } } -pub(crate) static WORKER_POOL_OPERATION: LazyLock = - LazyLock::new(WorkerThreadOperation::default); +pub(crate) static WORKER_POOL_OPERATION: LazyLock = + LazyLock::new(WorkerPoolOperation::default); pub(crate) async fn create_pool(filename: String, concurrency: usize) -> anyhow::Result<()> { WORKER_POOL_OPERATION From 54c7f93849910e92cb8bfe2c14a575855d84cdff Mon Sep 17 00:00:00 2001 From: xusd320 Date: Wed, 26 Nov 2025 11:04:03 +0800 Subject: [PATCH 09/17] perf(turbopack-node): use u32 as worker operation task_id --- .../next/src/build/swc/generated-native.d.ts | 6 ++-- .../js/src/worker_threads/evaluate.ts | 6 ++-- .../turbopack-node/src/worker_pool/mod.rs | 17 ++++++---- .../src/worker_pool/operation.rs | 32 +++++++++---------- .../src/worker_pool/worker_thread.rs | 6 ++-- 5 files changed, 35 insertions(+), 32 deletions(-) diff --git a/packages/next/src/build/swc/generated-native.d.ts b/packages/next/src/build/swc/generated-native.d.ts index c6e77dceca0119..f5d0f1d882a081 100644 --- a/packages/next/src/build/swc/generated-native.d.ts +++ b/packages/next/src/build/swc/generated-native.d.ts @@ -39,15 +39,15 @@ export interface PoolOptions { concurrency: number } export declare function recvPoolCreation(): PoolOptions | null -export declare function recvWorkerRequest(poolId: string): Promise +export declare function recvWorkerRequest(poolId: string): Promise export declare function recvMessageInWorker(workerId: number): Promise export declare function notifyOneWorkerCreated(filename: string): Promise export declare function notifyWorkerAck( - taskId: string, + taskId: number, workerId: number ): Promise export declare function sendTaskMessage( - taskId: string, + taskId: number, message: string ): Promise export declare function lockfileTryAcquireSync( diff --git a/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts index ffedf350d55c66..a2caf149028fec 100644 --- a/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts +++ b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts @@ -3,11 +3,11 @@ import { structuredError } from '../error' import type { Channel } from '../types' interface Binding { - recvWorkerRequest(poolId: string): Promise + recvWorkerRequest(poolId: string): Promise recvMessageInWorker(workerId: number): Promise notifyOneWorkerCreated(filename: string): Promise - notifyWorkerAck(taskId: string, workerId: number): Promise - sendTaskMessage(taskId: string, message: string): Promise + notifyWorkerAck(taskId: number, workerId: number): Promise + sendTaskMessage(taskId: number, message: string): Promise } // FIXME: require correct path on diffrent platform diff --git a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs index 255b3271488c6d..7dfe48a9ac2f81 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::{path::PathBuf, sync::atomic::AtomicU32}; use anyhow::Result; use rustc_hash::FxHashMap; @@ -16,6 +16,8 @@ use crate::{ mod operation; mod worker_thread; +static OPERATION_TASK_ID: AtomicU32 = AtomicU32::new(1); + #[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)] pub struct WorkerThreadPool { cwd: PathBuf, @@ -78,13 +80,14 @@ impl EvaluateOperation for WorkerThreadPool { }) .await; - let task_id = uuid::Uuid::new_v4().to_string(); + let task_id = OPERATION_TASK_ID.fetch_add(1, std::sync::atomic::Ordering::Release); + + if task_id == 0 { + panic!("operation task id overflow") + } - let worker_id = connect_to_worker( - self.entrypoint.to_string_lossy().to_string(), - task_id.clone(), - ) - .await?; + let worker_id = + connect_to_worker(self.entrypoint.to_string_lossy().to_string(), task_id).await?; WorkerOperation { task_id, worker_id } }; diff --git a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs index 3b5065fdb4bc1e..195f996f2cd0ab 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs @@ -36,10 +36,10 @@ impl MessageChannel { pub(crate) struct WorkerPoolOperation { pool_request_channel: MessageChannel<(String, usize)>, pool_ack_channel: DashMap>, - worker_request_channel: DashMap>, - worker_ack_channel: DashMap>, + worker_request_channel: DashMap>, + worker_ack_channel: DashMap>, worker_routed_channel: DashMap>, - task_routed_channel: DashMap>, + task_routed_channel: DashMap>, } impl Default for WorkerPoolOperation { @@ -89,20 +89,20 @@ impl WorkerPoolOperation { Ok(()) } - pub(crate) async fn connect_to_worker(&self, pool_id: String, task_id: String) -> Result { + pub(crate) async fn connect_to_worker(&self, pool_id: String, task_id: u32) -> Result { let channel = self .worker_request_channel .entry(pool_id.clone()) .or_insert_with(MessageChannel::unbounded) .clone(); channel - .send(task_id.clone()) + .send(task_id) .await .context("failed to send worker request")?; let worker_id = async move { let channel = self .worker_ack_channel - .entry(task_id.clone()) + .entry(task_id) .or_insert_with(MessageChannel::unbounded) .clone(); channel.recv().await.context("failed to recv worker ack") @@ -124,10 +124,10 @@ impl WorkerPoolOperation { Ok(()) } - pub async fn recv_task_response(&self, task_id: String) -> Result { + pub async fn recv_task_response(&self, task_id: u32) -> Result { let channel = self .task_routed_channel - .entry(task_id.clone()) + .entry(task_id) .or_insert_with(MessageChannel::unbounded) .clone(); let data = channel @@ -153,7 +153,7 @@ impl WorkerPoolOperation { .context("failed to notify worker created") } - pub(crate) async fn recv_worker_request(&self, pool_id: String) -> Result { + pub(crate) async fn recv_worker_request(&self, pool_id: String) -> Result { let channel = self .worker_request_channel .entry(pool_id.clone()) @@ -165,7 +165,7 @@ impl WorkerPoolOperation { .context("failed to recv worker request") } - pub(crate) async fn notify_worker_ack(&self, task_id: String, worker_id: u32) -> Result<()> { + pub(crate) async fn notify_worker_ack(&self, task_id: u32, worker_id: u32) -> Result<()> { let channel = self .worker_ack_channel .get(&task_id) @@ -188,10 +188,10 @@ impl WorkerPoolOperation { .with_context(|| format!("failed to recv message in worker {worker_id}")) } - pub(crate) async fn send_task_message(&self, task_id: String, data: String) -> Result<()> { + pub(crate) async fn send_task_message(&self, task_id: u32, data: String) -> Result<()> { let channel = self .task_routed_channel - .entry(task_id.clone()) + .entry(task_id) .or_insert_with(MessageChannel::unbounded) .clone(); channel @@ -210,7 +210,7 @@ pub(crate) async fn create_pool(filename: String, concurrency: usize) -> anyhow: .await } -pub(crate) async fn connect_to_worker(pool_id: String, task_id: String) -> Result { +pub(crate) async fn connect_to_worker(pool_id: String, task_id: u32) -> Result { WORKER_POOL_OPERATION .connect_to_worker(pool_id, task_id) .await @@ -222,19 +222,19 @@ pub(crate) async fn send_message_to_worker(worker_id: u32, data: String) -> Resu .await } -pub async fn recv_task_response(task_id: String) -> Result { +pub async fn recv_task_response(task_id: u32) -> Result { WORKER_POOL_OPERATION.recv_task_response(task_id).await } pub(crate) struct WorkerOperation { - pub(crate) task_id: String, + pub(crate) task_id: u32, pub(crate) worker_id: u32, } #[async_trait::async_trait] impl Operation for WorkerOperation { async fn recv(&mut self) -> Result { - recv_task_response(self.task_id.clone()).await + recv_task_response(self.task_id).await } async fn send(&mut self, data: String) -> Result<()> { diff --git a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs index d879c6ef287e54..5e184a3b25ac3c 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs @@ -22,7 +22,7 @@ pub fn recv_pool_creation() -> Option { #[napi] #[allow(unused)] -pub async fn recv_worker_request(pool_id: String) -> napi::Result { +pub async fn recv_worker_request(pool_id: String) -> napi::Result { WORKER_POOL_OPERATION .recv_worker_request(pool_id) .await @@ -49,7 +49,7 @@ pub async fn notify_one_worker_created(filename: String) -> napi::Result<()> { #[napi] #[allow(unused)] -pub async fn notify_worker_ack(task_id: String, worker_id: u32) -> napi::Result<()> { +pub async fn notify_worker_ack(task_id: u32, worker_id: u32) -> napi::Result<()> { WORKER_POOL_OPERATION .notify_worker_ack(task_id, worker_id) .await @@ -58,7 +58,7 @@ pub async fn notify_worker_ack(task_id: String, worker_id: u32) -> napi::Result< #[napi] #[allow(unused)] -pub async fn send_task_message(task_id: String, message: String) -> napi::Result<()> { +pub async fn send_task_message(task_id: u32, message: String) -> napi::Result<()> { WORKER_POOL_OPERATION .send_task_message(task_id, message) .await From e2e938741f65bb2352e24c8a04ea035573f5d74f Mon Sep 17 00:00:00 2001 From: xusd320 Date: Wed, 26 Nov 2025 11:17:19 +0800 Subject: [PATCH 10/17] chore: remove unused deps --- Cargo.lock | 2 -- turbopack/crates/turbopack-node/Cargo.toml | 9 +-------- turbopack/crates/turbopack-node/src/worker_pool/mod.rs | 7 +++++-- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 414df4ace63a6d..1c875c53f91f3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9910,7 +9910,6 @@ dependencies = [ "futures-retry", "indoc", "napi", - "napi-build", "napi-derive", "once_cell", "owo-colors", @@ -9931,7 +9930,6 @@ dependencies = [ "turbopack-core", "turbopack-ecmascript", "turbopack-resolve", - "uuid", ] [[package]] diff --git a/turbopack/crates/turbopack-node/Cargo.toml b/turbopack/crates/turbopack-node/Cargo.toml index b90bad9bf3843d..d3429d2bea9956 100644 --- a/turbopack/crates/turbopack-node/Cargo.toml +++ b/turbopack/crates/turbopack-node/Cargo.toml @@ -39,7 +39,6 @@ serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true, features = ["base64"] } tokio = { workspace = true, optional = true } -uuid = { workspace = true, features = ["v4"] } tracing = { workspace = true } turbo-rcstr = { workspace = true } turbo-tasks = { workspace = true } @@ -50,12 +49,6 @@ turbopack-cli-utils = { workspace = true } turbopack-core = { workspace = true } turbopack-ecmascript = { workspace = true } turbopack-resolve = { workspace = true } -napi = { workspace = true, features = ["anyhow"], optional = true } +napi = { workspace = true, features = ["anyhow"], optional = true } napi-derive = { workspace = true, optional = true } async-channel = { version = "2.5.0", optional = true } - -[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] -uuid = { workspace = true, features = ["js"] } - -[build-dependencies] -napi-build = { workspace = true } diff --git a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs index 7dfe48a9ac2f81..073f04887b3903 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs @@ -1,4 +1,7 @@ -use std::{path::PathBuf, sync::atomic::AtomicU32}; +use std::{ + path::PathBuf, + sync::atomic::{AtomicU32, Ordering}, +}; use anyhow::Result; use rustc_hash::FxHashMap; @@ -80,7 +83,7 @@ impl EvaluateOperation for WorkerThreadPool { }) .await; - let task_id = OPERATION_TASK_ID.fetch_add(1, std::sync::atomic::Ordering::Release); + let task_id = OPERATION_TASK_ID.fetch_add(1, Ordering::Release); if task_id == 0 { panic!("operation task id overflow") From 3fb7eeef8eb32eaddf5fe027f550da4856f8cd9c Mon Sep 17 00:00:00 2001 From: xusd320 Date: Wed, 26 Nov 2025 22:14:51 +0800 Subject: [PATCH 11/17] refactor(turbopack-node): using async waiting pool creation message --- .../next/src/build/swc/generated-native.d.ts | 2 +- packages/next/src/build/swc/index.ts | 40 ++++++++---------- .../src/worker_pool/operation.rs | 13 +++--- .../src/worker_pool/worker_thread.rs | 41 +++++++------------ 4 files changed, 39 insertions(+), 57 deletions(-) diff --git a/packages/next/src/build/swc/generated-native.d.ts b/packages/next/src/build/swc/generated-native.d.ts index f5d0f1d882a081..fb7c1a8792b9ec 100644 --- a/packages/next/src/build/swc/generated-native.d.ts +++ b/packages/next/src/build/swc/generated-native.d.ts @@ -38,7 +38,7 @@ export interface PoolOptions { filename: string concurrency: number } -export declare function recvPoolCreation(): PoolOptions | null +export declare function recvPoolCreation(): Promise export declare function recvWorkerRequest(poolId: string): Promise export declare function recvMessageInWorker(workerId: number): Promise export declare function notifyOneWorkerCreated(filename: string): Promise diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index 93699d9924e9e8..3be3be1e2ceaa4 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -672,32 +672,29 @@ function bindingToApi( #poolCreated: Record> = {} - #poolScheduler?: ReturnType - constructor(nativeProject: { __napiType: 'Project' }) { this._nativeProject = nativeProject - if (binding.recvPoolCreation) { - this.#poolScheduler = setInterval(() => { - let poolOptions = binding.recvPoolCreation() - if (poolOptions) { - const { filename, concurrency } = poolOptions - if (!this.#poolCreated[filename]) { - const workers = [] - for (let i = 0; i < concurrency; i++) { - const worker = new Worker(filename, { - workerData: { - poolId: filename, - }, - }) - worker.unref() - workers.push(worker) - } - this.#poolCreated[filename] = workers - } + const createPool = async () => { + let poolOptions = await binding.recvPoolCreation() + const { filename, concurrency } = poolOptions + if (!this.#poolCreated[filename]) { + const workers = [] + for (let i = 0; i < concurrency; i++) { + const worker = new Worker(filename, { + workerData: { + poolId: filename, + bindingPath: require.resolve('./binding.js'), + }, + }) + worker.unref() + workers.push(worker) } - }, 0) + this.#poolCreated[filename] = workers + } + createPool() } + createPool() } async update(options: PartialProjectOptions) { @@ -824,7 +821,6 @@ function bindingToApi( } shutdown(): Promise { - this.#poolScheduler && clearInterval(this.#poolScheduler) return binding.projectShutdown(this._nativeProject) } diff --git a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs index 195f996f2cd0ab..1995d38a6e39c6 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs @@ -27,10 +27,6 @@ impl MessageChannel { pub(crate) async fn recv(&self) -> Result { Ok(self.receiver.recv().await?) } - - pub(crate) fn try_recv(&self) -> Result { - Ok(self.receiver.try_recv()?) - } } pub(crate) struct WorkerPoolOperation { @@ -137,8 +133,11 @@ impl WorkerPoolOperation { Ok(data) } - pub(crate) fn try_recv_pool_creation(&self) -> Option<(String, usize)> { - self.pool_request_channel.try_recv().ok() + pub(crate) async fn recv_pool_creation(&self) -> Result<(String, usize)> { + self.pool_request_channel + .recv() + .await + .context("failed to recv pool creation") } pub(crate) async fn notify_one_worker_created(&self, filename: String) -> Result<()> { @@ -169,7 +168,7 @@ impl WorkerPoolOperation { let channel = self .worker_ack_channel .get(&task_id) - .context(format!("worker ack channel for {task_id} not found"))?; + .with_context(|| format!("worker ack channel for {task_id} not found"))?; channel .send(worker_id) .await diff --git a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs index 5e184a3b25ac3c..429451fd76bbe4 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs @@ -3,35 +3,28 @@ use napi_derive::napi; use crate::worker_pool::operation::WORKER_POOL_OPERATION; #[napi(object)] -#[allow(unused)] pub struct PoolOptions { pub filename: String, pub concurrency: u32, } #[napi] -#[allow(unused)] -pub fn recv_pool_creation() -> Option { - WORKER_POOL_OPERATION - .try_recv_pool_creation() - .map(|(filename, concurrency)| PoolOptions { - filename, - concurrency: concurrency as u32, - }) +pub async fn recv_pool_creation() -> napi::Result { + let (filename, concurrency) = WORKER_POOL_OPERATION.recv_pool_creation().await?; + + Ok(PoolOptions { + filename, + concurrency: concurrency as u32, + }) } #[napi] -#[allow(unused)] pub async fn recv_worker_request(pool_id: String) -> napi::Result { - WORKER_POOL_OPERATION - .recv_worker_request(pool_id) - .await - .map_err(|e| napi::Error::from_reason(e.to_string())) + Ok(WORKER_POOL_OPERATION.recv_worker_request(pool_id).await?) } #[napi] // TODO: use zero-copy externaled type array -#[allow(unused)] pub async fn recv_message_in_worker(worker_id: u32) -> napi::Result { Ok(WORKER_POOL_OPERATION .recv_message_in_worker(worker_id) @@ -39,28 +32,22 @@ pub async fn recv_message_in_worker(worker_id: u32) -> napi::Result { } #[napi] -#[allow(unused)] pub async fn notify_one_worker_created(filename: String) -> napi::Result<()> { - WORKER_POOL_OPERATION + Ok(WORKER_POOL_OPERATION .notify_one_worker_created(filename) - .await - .map_err(|e| napi::Error::from_reason(e.to_string())) + .await?) } #[napi] -#[allow(unused)] pub async fn notify_worker_ack(task_id: u32, worker_id: u32) -> napi::Result<()> { - WORKER_POOL_OPERATION + Ok(WORKER_POOL_OPERATION .notify_worker_ack(task_id, worker_id) - .await - .map_err(|e| napi::Error::from_reason(e.to_string())) + .await?) } #[napi] -#[allow(unused)] pub async fn send_task_message(task_id: u32, message: String) -> napi::Result<()> { - WORKER_POOL_OPERATION + Ok(WORKER_POOL_OPERATION .send_task_message(task_id, message) - .await - .map_err(|e| napi::Error::from_reason(e.to_string())) + .await?) } From 9cc5cc9a67fbcff6036f0df0bbbb38f2f9b6e424 Mon Sep 17 00:00:00 2001 From: xusd320 Date: Thu, 27 Nov 2025 00:28:34 +0800 Subject: [PATCH 12/17] fix(turbopack-node): use correct binding path to create loader pool --- packages/next/src/build/swc/index.ts | 26 +++++++++++++------ .../js/src/worker_threads/evaluate.ts | 5 ++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index 3be3be1e2ceaa4..31b00750efd4f8 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -492,6 +492,7 @@ function rustifyOptionEnv( // TODO(sokra) Support wasm option. function bindingToApi( binding: RawBindings, + bindingPath: string, _wasm: boolean ): Binding['turbo']['createProject'] { type NativeFunction = ( @@ -684,7 +685,7 @@ function bindingToApi( const worker = new Worker(filename, { workerData: { poolId: filename, - bindingPath: require.resolve('./binding.js'), + bindingPath, }, }) worker.unref() @@ -1369,20 +1370,24 @@ function loadNative(importPath?: string) { throw new Error('cannot run loadNative when `NEXT_TEST_WASM` is set') } + const customBindingsPath = !!__INTERNAL_CUSTOM_TURBOPACK_BINDINGS + ? require.resolve(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS) + : null const customBindings: RawBindings = !!__INTERNAL_CUSTOM_TURBOPACK_BINDINGS ? require(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS) : null let bindings: RawBindings = customBindings + let bindingsPath = customBindingsPath let attempts: any[] = [] const NEXT_TEST_NATIVE_DIR = process.env.NEXT_TEST_NATIVE_DIR for (const triple of triples) { if (NEXT_TEST_NATIVE_DIR) { try { + const bindingForTest = `${NEXT_TEST_NATIVE_DIR}/next-swc.${triple.platformArchABI}.node` // Use the binary directly to skip `pnpm pack` for testing as it's slow because of the large native binary. - bindings = require( - `${NEXT_TEST_NATIVE_DIR}/next-swc.${triple.platformArchABI}.node` - ) + bindings = require(bindingForTest) + bindingsPath = require.resolve(bindingForTest) infoLog( 'next-swc build: local built @next/swc from NEXT_TEST_NATIVE_DIR' ) @@ -1390,9 +1395,9 @@ function loadNative(importPath?: string) { } catch (e) {} } else { try { - bindings = require( - `@next/swc/native/next-swc.${triple.platformArchABI}.node` - ) + const normalBinding = `@next/swc/native/next-swc.${triple.platformArchABI}.node` + bindings = require(normalBinding) + bindingsPath = require.resolve(normalBinding) infoLog('next-swc build: local built @next/swc') break } catch (e) {} @@ -1410,6 +1415,7 @@ function loadNative(importPath?: string) { : `@next/swc-${triple.platformArchABI}` try { bindings = require(pkg) + bindingsPath = require.resolve(pkg) if (!importPath) { checkVersionMismatch(require(`${pkg}/package.json`)) } @@ -1488,7 +1494,11 @@ function loadNative(importPath?: string) { initCustomTraceSubscriber: bindings.initCustomTraceSubscriber, teardownTraceSubscriber: bindings.teardownTraceSubscriber, turbo: { - createProject: bindingToApi(customBindings ?? bindings, false), + createProject: bindingToApi( + customBindings ?? bindings, + customBindingsPath ?? bindingsPath!, + false + ), startTurbopackTraceServer(traceFilePath, port) { Log.warn( `Turbopack trace server started. View trace at https://trace.nextjs.org${port != null ? `?port=${port}` : ''}` diff --git a/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts index a2caf149028fec..ac5fbdba80ae4a 100644 --- a/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts +++ b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts @@ -10,8 +10,9 @@ interface Binding { sendTaskMessage(taskId: number, message: string): Promise } -// FIXME: require correct path on diffrent platform -const binding: Binding = require(/* turbopackIgnore: true */ '@next/swc/native/next-swc.darwin-arm64.node') +const binding: Binding = require( + /* turbopackIgnore: true */ workerData.bindingPath +) binding.notifyOneWorkerCreated(workerData.poolId) From 27d3ee47c89e63ed39eca19e6e2fc74707cff610 Mon Sep 17 00:00:00 2001 From: xusd320 Date: Thu, 27 Nov 2025 00:47:20 +0800 Subject: [PATCH 13/17] fix: clippy again --- .../crates/turbopack-node/src/worker_pool/worker_thread.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs index 429451fd76bbe4..b7cca46d96fef8 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs @@ -3,12 +3,14 @@ use napi_derive::napi; use crate::worker_pool::operation::WORKER_POOL_OPERATION; #[napi(object)] +#[allow(unused)] pub struct PoolOptions { pub filename: String, pub concurrency: u32, } #[napi] +#[allow(unused)] pub async fn recv_pool_creation() -> napi::Result { let (filename, concurrency) = WORKER_POOL_OPERATION.recv_pool_creation().await?; @@ -19,11 +21,13 @@ pub async fn recv_pool_creation() -> napi::Result { } #[napi] +#[allow(unused)] pub async fn recv_worker_request(pool_id: String) -> napi::Result { Ok(WORKER_POOL_OPERATION.recv_worker_request(pool_id).await?) } #[napi] +#[allow(unused)] // TODO: use zero-copy externaled type array pub async fn recv_message_in_worker(worker_id: u32) -> napi::Result { Ok(WORKER_POOL_OPERATION @@ -32,6 +36,7 @@ pub async fn recv_message_in_worker(worker_id: u32) -> napi::Result { } #[napi] +#[allow(unused)] pub async fn notify_one_worker_created(filename: String) -> napi::Result<()> { Ok(WORKER_POOL_OPERATION .notify_one_worker_created(filename) @@ -39,6 +44,7 @@ pub async fn notify_one_worker_created(filename: String) -> napi::Result<()> { } #[napi] +#[allow(unused)] pub async fn notify_worker_ack(task_id: u32, worker_id: u32) -> napi::Result<()> { Ok(WORKER_POOL_OPERATION .notify_worker_ack(task_id, worker_id) @@ -46,6 +52,7 @@ pub async fn notify_worker_ack(task_id: u32, worker_id: u32) -> napi::Result<()> } #[napi] +#[allow(unused)] pub async fn send_task_message(task_id: u32, message: String) -> napi::Result<()> { Ok(WORKER_POOL_OPERATION .send_task_message(task_id, message) From f496ba6a0c970b43f81d99f7bee11278ebd697ca Mon Sep 17 00:00:00 2001 From: xusd320 Date: Thu, 27 Nov 2025 12:22:35 +0800 Subject: [PATCH 14/17] feat(turbopack-node): restore wait_or_kill operation for worker thread --- .../next/src/build/swc/generated-native.d.ts | 12 ++- packages/next/src/build/swc/index.ts | 63 +++++++---- turbopack/crates/turbopack-node/Cargo.toml | 2 +- .../js/src/worker_threads/evaluate.ts | 2 - .../crates/turbopack-node/src/evaluate.rs | 25 +++-- .../turbopack-node/src/process_pool/mod.rs | 52 +++++---- .../turbopack-node/src/worker_pool/mod.rs | 37 +++---- .../src/worker_pool/operation.rs | 100 +++++++++--------- .../src/worker_pool/worker_thread.rs | 35 +++--- 9 files changed, 178 insertions(+), 150 deletions(-) diff --git a/packages/next/src/build/swc/generated-native.d.ts b/packages/next/src/build/swc/generated-native.d.ts index fb7c1a8792b9ec..4f9b81b4c8a339 100644 --- a/packages/next/src/build/swc/generated-native.d.ts +++ b/packages/next/src/build/swc/generated-native.d.ts @@ -36,12 +36,16 @@ export declare class ExternalObject { } export interface PoolOptions { filename: string - concurrency: number + maxConcurrency: number } -export declare function recvPoolCreation(): Promise -export declare function recvWorkerRequest(poolId: string): Promise +export interface WorkerTermination { + filename: string + workerId: number +} +export declare function recvPoolRequest(): Promise +export declare function recvWorkerTermination(): Promise +export declare function recvWorkerRequest(filename: string): Promise export declare function recvMessageInWorker(workerId: number): Promise -export declare function notifyOneWorkerCreated(filename: string): Promise export declare function notifyWorkerAck( taskId: number, workerId: number diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index 31b00750efd4f8..ff5b56324e752f 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -668,34 +668,55 @@ function bindingToApi( } } + const loaderWorkers: Record> = {} + + const createOrScalePool = async () => { + let poolOptions = await binding.recvPoolRequest() + const { filename, maxConcurrency } = poolOptions + const workers = loaderWorkers[filename] || (loaderWorkers[filename] = []) + if (workers.length < maxConcurrency) { + for (let i = workers.length; i < maxConcurrency; i++) { + const worker = new Worker(filename, { + workerData: { + poolId: filename, + bindingPath, + }, + }) + worker.unref() + workers.push(worker) + } + } else if (workers.length > maxConcurrency) { + const workersToStop = workers.splice(0, workers.length - maxConcurrency) + workersToStop.forEach((worker) => worker.terminate()) + } + createOrScalePool() + } + + const waitingForWorkerTermination = async () => { + const { filename, workerId } = await binding.recvWorkerTermination() + const workers = loaderWorkers[filename] + const workerIdx = workers.findIndex( + (worker) => worker.threadId === workerId + ) + if (workerIdx > -1) { + const worker = workers.splice(workerIdx, 1) + worker[0].terminate() + } + waitingForWorkerTermination() + } + class ProjectImpl implements Project { private readonly _nativeProject: { __napiType: 'Project' } - #poolCreated: Record> = {} - constructor(nativeProject: { __napiType: 'Project' }) { this._nativeProject = nativeProject - const createPool = async () => { - let poolOptions = await binding.recvPoolCreation() - const { filename, concurrency } = poolOptions - if (!this.#poolCreated[filename]) { - const workers = [] - for (let i = 0; i < concurrency; i++) { - const worker = new Worker(filename, { - workerData: { - poolId: filename, - bindingPath, - }, - }) - worker.unref() - workers.push(worker) - } - this.#poolCreated[filename] = workers - } - createPool() + if (typeof binding.recvPoolRequest === 'function') { + createOrScalePool() + } + if (typeof binding.recvWorkerTermination === 'function') { + waitingForWorkerTermination() } - createPool() } async update(options: PartialProjectOptions) { diff --git a/turbopack/crates/turbopack-node/Cargo.toml b/turbopack/crates/turbopack-node/Cargo.toml index d3429d2bea9956..b5291665270fbf 100644 --- a/turbopack/crates/turbopack-node/Cargo.toml +++ b/turbopack/crates/turbopack-node/Cargo.toml @@ -14,7 +14,7 @@ default = ["process_pool"] # enable "HMR" for embedded assets dynamic_embed_contents = ["turbo-tasks-fs/dynamic_embed_contents"] process_pool = ["tokio/full"] -worker_pool = ["async-channel", "tokio/sync", "napi", "napi-derive"] +worker_pool = ["async-channel", "napi", "napi-derive"] [lints] workspace = true diff --git a/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts index ac5fbdba80ae4a..72fdf4f2597aab 100644 --- a/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts +++ b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts @@ -14,8 +14,6 @@ const binding: Binding = require( /* turbopackIgnore: true */ workerData.bindingPath ) -binding.notifyOneWorkerCreated(workerData.poolId) - const queue: string[][] = [] export const run = async ( diff --git a/turbopack/crates/turbopack-node/src/evaluate.rs b/turbopack/crates/turbopack-node/src/evaluate.rs index c8244f40c18bdf..1864047a642a02 100644 --- a/turbopack/crates/turbopack-node/src/evaluate.rs +++ b/turbopack/crates/turbopack-node/src/evaluate.rs @@ -1,4 +1,7 @@ -use std::{borrow::Cow, iter, sync::Arc, thread::available_parallelism, time::Duration}; +use std::{ + borrow::Cow, iter, process::ExitStatus, sync::Arc, thread::available_parallelism, + time::Duration, +}; use anyhow::{Result, bail}; use futures_retry::{FutureRetry, RetryPolicy}; @@ -42,7 +45,7 @@ use crate::{ #[derive(Serialize)] #[serde(tag = "type", rename_all = "camelCase")] -pub enum EvalJavaScriptOutgoingMessage<'a> { +enum EvalJavaScriptOutgoingMessage<'a> { #[serde(rename_all = "camelCase")] Evaluate { args: Vec<&'a JsonValue> }, Result { @@ -54,7 +57,7 @@ pub enum EvalJavaScriptOutgoingMessage<'a> { #[derive(Deserialize, Debug)] #[serde(tag = "type", rename_all = "camelCase")] -pub enum EvalJavaScriptIncomingMessage { +enum EvalJavaScriptIncomingMessage { Info { data: JsonValue }, Request { id: u64, data: JsonValue }, End { data: Option }, @@ -105,6 +108,10 @@ pub trait Operation: Send { async fn recv(&mut self) -> Result; async fn send(&mut self, data: String) -> Result<()>; + + async fn wait_or_kill(&mut self) -> Result; + + fn disallow_reuse(&mut self) -> (); } #[turbo_tasks::value] @@ -358,7 +365,7 @@ pub async fn custom_evaluate(evaluate_context: impl EvaluateContext) -> Result Result( operation: &mut Box, @@ -545,7 +552,7 @@ async fn pull_operation( EvalJavaScriptIncomingMessage::Error(error) => { evaluate_context.emit_error(error, pool).await?; // Do not reuse the process in case of error - // operation.disallow_reuse(); + operation.disallow_reuse(); // Issue emitted, we want to break but don't want to return an error return Ok(None); } diff --git a/turbopack/crates/turbopack-node/src/process_pool/mod.rs b/turbopack/crates/turbopack-node/src/process_pool/mod.rs index f81a157dc8fe4d..4bf41a16f1ee79 100644 --- a/turbopack/crates/turbopack-node/src/process_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/process_pool/mod.rs @@ -854,32 +854,8 @@ impl Operation for ChildProcessOperation { }) .await } -} - -impl ChildProcessOperation { - async fn with_process<'a, F: Future> + Send + 'a, T>( - &'a mut self, - f: impl FnOnce(&'a mut NodeJsPoolProcess) -> F, - ) -> Result { - let process = self - .process - .as_mut() - .context("Node.js operation already finished")?; - - if !self.allow_process_reuse { - bail!("Node.js process is no longer usable"); - } - - let result = f(process).await; - if result.is_err() && self.allow_process_reuse { - self.stats.lock().remove_worker(); - self.allow_process_reuse = false; - } - result - } - #[allow(dead_code)] - pub async fn wait_or_kill(mut self) -> Result { + async fn wait_or_kill(&mut self) -> Result { let mut process = self .process .take() @@ -904,8 +880,7 @@ impl ChildProcessOperation { Ok(status) } - #[allow(dead_code)] - pub fn disallow_reuse(&mut self) { + fn disallow_reuse(&mut self) { if self.allow_process_reuse { self.stats.lock().remove_worker(); self.allow_process_reuse = false; @@ -913,6 +888,29 @@ impl ChildProcessOperation { } } +impl ChildProcessOperation { + async fn with_process<'a, F: Future> + Send + 'a, T>( + &'a mut self, + f: impl FnOnce(&'a mut NodeJsPoolProcess) -> F, + ) -> Result { + let process = self + .process + .as_mut() + .context("Node.js operation already finished")?; + + if !self.allow_process_reuse { + bail!("Node.js process is no longer usable"); + } + + let result = f(process).await; + if result.is_err() && self.allow_process_reuse { + self.stats.lock().remove_worker(); + self.allow_process_reuse = false; + } + result + } +} + impl Drop for ChildProcessOperation { fn drop(&mut self) { if let Some(mut process) = self.process.take() { diff --git a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs index 073f04887b3903..bbf30b71cbe0b1 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs @@ -5,7 +5,6 @@ use std::{ use anyhow::Result; use rustc_hash::FxHashMap; -use tokio::sync::OnceCell; use turbo_rcstr::RcStr; use turbo_tasks::{ResolvedVc, duration_span}; use turbo_tasks_fs::FileSystemPath; @@ -13,7 +12,7 @@ use turbo_tasks_fs::FileSystemPath; use crate::{ AssetsForSourceMapping, evaluate::{EvaluateOperation, EvaluatePool, Operation}, - worker_pool::operation::{WorkerOperation, connect_to_worker, create_pool}, + worker_pool::operation::{WorkerOperation, connect_to_worker, create_or_scale_pool}, }; mod operation; @@ -21,14 +20,12 @@ mod worker_thread; static OPERATION_TASK_ID: AtomicU32 = AtomicU32::new(1); -#[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)] +#[turbo_tasks::value] pub struct WorkerThreadPool { cwd: PathBuf, entrypoint: PathBuf, env: FxHashMap, concurrency: usize, - #[turbo_tasks(trace_ignore, debug_ignore)] - ready: OnceCell<()>, pub assets_for_source_mapping: ResolvedVc, pub assets_root: FileSystemPath, pub project_dir: FileSystemPath, @@ -43,7 +40,7 @@ impl WorkerThreadPool { assets_root: FileSystemPath, project_dir: FileSystemPath, concurrency: usize, - _debug: bool, + debug: bool, ) -> EvaluatePool { EvaluatePool::new( entrypoint.to_string_lossy().to_string().into(), @@ -51,8 +48,7 @@ impl WorkerThreadPool { cwd, entrypoint, env, - concurrency, - ready: OnceCell::new(), + concurrency: (if debug { 1 } else { concurrency }), assets_for_source_mapping, assets_root: assets_root.clone(), project_dir: project_dir.clone(), @@ -69,19 +65,9 @@ impl EvaluateOperation for WorkerThreadPool { async fn operation(&self) -> Result> { let operation = { let _guard = duration_span!("Node.js operation"); - let entrypoint = self.entrypoint.to_string_lossy().to_string(); - self.ready - .get_or_init(async || { - create_pool( - self.entrypoint.to_string_lossy().to_string(), - self.concurrency, - ) - .await - .unwrap_or_else(|e| { - panic!("failed to create worker pool for {entrypoint} for reason: {e}",) - }) - }) - .await; + let pool_id = self.entrypoint.to_string_lossy().to_string(); + + create_or_scale_pool(pool_id.clone(), self.concurrency).await?; let task_id = OPERATION_TASK_ID.fetch_add(1, Ordering::Release); @@ -89,10 +75,13 @@ impl EvaluateOperation for WorkerThreadPool { panic!("operation task id overflow") } - let worker_id = - connect_to_worker(self.entrypoint.to_string_lossy().to_string(), task_id).await?; + let worker_id = connect_to_worker(pool_id.clone(), task_id).await?; - WorkerOperation { task_id, worker_id } + WorkerOperation { + pool_id, + task_id, + worker_id, + } }; Ok(Box::new(operation)) diff --git a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs index 1995d38a6e39c6..58d33263d9d4d6 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs @@ -1,4 +1,4 @@ -use std::sync::LazyLock; +use std::{process::ExitStatus, sync::LazyLock}; use anyhow::{Context, Result}; use async_channel::{Receiver, Sender, unbounded}; @@ -31,7 +31,7 @@ impl MessageChannel { pub(crate) struct WorkerPoolOperation { pool_request_channel: MessageChannel<(String, usize)>, - pool_ack_channel: DashMap>, + worker_termination_channel: MessageChannel<(String, u32)>, worker_request_channel: DashMap>, worker_ack_channel: DashMap>, worker_routed_channel: DashMap>, @@ -42,7 +42,7 @@ impl Default for WorkerPoolOperation { fn default() -> Self { Self { pool_request_channel: MessageChannel::unbounded(), - pool_ack_channel: DashMap::new(), + worker_termination_channel: MessageChannel::unbounded(), worker_request_channel: DashMap::new(), worker_ack_channel: DashMap::new(), worker_routed_channel: DashMap::new(), @@ -52,36 +52,16 @@ impl Default for WorkerPoolOperation { } impl WorkerPoolOperation { - pub(crate) async fn create_pool( + pub(crate) async fn create_or_scale_pool( &self, filename: String, - concurrency: usize, - ) -> anyhow::Result<()> { + max_concurrency: usize, + ) -> Result<()> { self.pool_request_channel - .send((filename.clone(), concurrency)) + .send((filename.clone(), max_concurrency)) .await .context("failed to send pool request")?; - let mut created_worker_count = 0; - - { - let channel = self - .pool_ack_channel - .entry(filename.clone()) - .or_insert_with(MessageChannel::unbounded) - .clone(); - - while created_worker_count < concurrency { - channel - .recv() - .await - .context("failed to recv worker creation")?; - created_worker_count += 1; - } - }; - - self.pool_ack_channel.remove(&filename); - Ok(()) } @@ -107,16 +87,34 @@ impl WorkerPoolOperation { Ok(worker_id) } + pub(crate) async fn send_worker_termination( + &self, + pool_id: String, + worker_id: u32, + ) -> Result<()> { + self.worker_termination_channel + .send((pool_id, worker_id)) + .await + .context("failed to send worker termination") + } + + pub(crate) async fn recv_worker_termination(&self) -> Result<(String, u32)> { + self.worker_termination_channel + .recv() + .await + .context("failed to recv worker termination") + } + pub(crate) async fn send_message_to_worker(&self, worker_id: u32, data: String) -> Result<()> { - let entry = self + let channel = self .worker_routed_channel .entry(worker_id) .or_insert_with(MessageChannel::unbounded) .clone(); - entry + channel .send(data) .await - .with_context(|| format!("failed to send message to worker {worker_id}"))?; + .context("failed to send message to worker")?; Ok(()) } @@ -129,27 +127,15 @@ impl WorkerPoolOperation { let data = channel .recv() .await - .with_context(|| format!("failed to recv message for task {task_id}"))?; + .context("failed to recv task message")?; Ok(data) } - pub(crate) async fn recv_pool_creation(&self) -> Result<(String, usize)> { + pub(crate) async fn recv_pool_request(&self) -> Result<(String, usize)> { self.pool_request_channel .recv() .await - .context("failed to recv pool creation") - } - - pub(crate) async fn notify_one_worker_created(&self, filename: String) -> Result<()> { - let channel = self - .pool_ack_channel - .entry(filename.clone()) - .or_insert_with(MessageChannel::unbounded) - .clone(); - channel - .send(()) - .await - .context("failed to notify worker created") + .context("failed to recv pool request") } pub(crate) async fn recv_worker_request(&self, pool_id: String) -> Result { @@ -203,9 +189,9 @@ impl WorkerPoolOperation { pub(crate) static WORKER_POOL_OPERATION: LazyLock = LazyLock::new(WorkerPoolOperation::default); -pub(crate) async fn create_pool(filename: String, concurrency: usize) -> anyhow::Result<()> { +pub(crate) async fn create_or_scale_pool(filename: String, max_concurrency: usize) -> Result<()> { WORKER_POOL_OPERATION - .create_pool(filename, concurrency) + .create_or_scale_pool(filename, max_concurrency) .await } @@ -221,11 +207,18 @@ pub(crate) async fn send_message_to_worker(worker_id: u32, data: String) -> Resu .await } -pub async fn recv_task_response(task_id: u32) -> Result { +pub(crate) async fn send_worker_termination(pool_id: String, worker_id: u32) -> Result<()> { + WORKER_POOL_OPERATION + .send_worker_termination(pool_id, worker_id) + .await +} + +pub async fn recv_task_message(task_id: u32) -> Result { WORKER_POOL_OPERATION.recv_task_response(task_id).await } pub(crate) struct WorkerOperation { + pub(crate) pool_id: String, pub(crate) task_id: u32, pub(crate) worker_id: u32, } @@ -233,10 +226,19 @@ pub(crate) struct WorkerOperation { #[async_trait::async_trait] impl Operation for WorkerOperation { async fn recv(&mut self) -> Result { - recv_task_response(self.task_id).await + recv_task_message(self.task_id).await } async fn send(&mut self, data: String) -> Result<()> { send_message_to_worker(self.worker_id, data).await } + + async fn wait_or_kill(&mut self) -> Result { + send_worker_termination(self.pool_id.clone(), self.worker_id).await?; + Ok(ExitStatus::default()) + } + + fn disallow_reuse(&mut self) { + // do nothing + } } diff --git a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs index b7cca46d96fef8..c4f592412043b5 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/worker_thread.rs @@ -6,40 +6,49 @@ use crate::worker_pool::operation::WORKER_POOL_OPERATION; #[allow(unused)] pub struct PoolOptions { pub filename: String, - pub concurrency: u32, + pub max_concurrency: u32, +} + +#[napi(object)] +#[allow(unused)] +pub struct WorkerTermination { + pub filename: String, + pub worker_id: u32, } #[napi] #[allow(unused)] -pub async fn recv_pool_creation() -> napi::Result { - let (filename, concurrency) = WORKER_POOL_OPERATION.recv_pool_creation().await?; +pub async fn recv_pool_request() -> napi::Result { + let (filename, max_concurrency) = WORKER_POOL_OPERATION.recv_pool_request().await?; Ok(PoolOptions { filename, - concurrency: concurrency as u32, + max_concurrency: max_concurrency as u32, }) } #[napi] #[allow(unused)] -pub async fn recv_worker_request(pool_id: String) -> napi::Result { - Ok(WORKER_POOL_OPERATION.recv_worker_request(pool_id).await?) +pub async fn recv_worker_termination() -> napi::Result { + let (filename, worker_id) = WORKER_POOL_OPERATION.recv_worker_termination().await?; + Ok(WorkerTermination { + filename, + worker_id, + }) } #[napi] #[allow(unused)] -// TODO: use zero-copy externaled type array -pub async fn recv_message_in_worker(worker_id: u32) -> napi::Result { - Ok(WORKER_POOL_OPERATION - .recv_message_in_worker(worker_id) - .await?) +pub async fn recv_worker_request(filename: String) -> napi::Result { + Ok(WORKER_POOL_OPERATION.recv_worker_request(filename).await?) } #[napi] #[allow(unused)] -pub async fn notify_one_worker_created(filename: String) -> napi::Result<()> { +// TODO: use zero-copy externaled type array +pub async fn recv_message_in_worker(worker_id: u32) -> napi::Result { Ok(WORKER_POOL_OPERATION - .notify_one_worker_created(filename) + .recv_message_in_worker(worker_id) .await?) } From b4f96c8de31c83aa4dce7c8973129c5ef8868de7 Mon Sep 17 00:00:00 2001 From: xusd320 Date: Thu, 27 Nov 2025 21:15:24 +0800 Subject: [PATCH 15/17] fix(turbopack-node): sass-loader failed via worker thread --- .../crates/turbopack-node/js/src/worker_threads/evaluate.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts index 72fdf4f2597aab..1e854a92a922fe 100644 --- a/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts +++ b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts @@ -121,6 +121,9 @@ export const run = async ( } while (true) { + // need await a micro task, or that if some some request rejected, + // the error will be propergated to schedule threat, then causing panic + await Promise.resolve() const msg_str = await binding.recvMessageInWorker(workerId) const msg = JSON.parse(msg_str) as From b97a5416c56f57e6b73313dab48e56b5933dcc00 Mon Sep 17 00:00:00 2001 From: xusd320 Date: Thu, 27 Nov 2025 21:36:26 +0800 Subject: [PATCH 16/17] chore: fix cargo fmt --- .../crates/turbopack-node/js/src/worker_threads/evaluate.ts | 6 +++--- turbopack/crates/turbopack-node/src/evaluate.rs | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts index 1e854a92a922fe..2a285a00f81f55 100644 --- a/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts +++ b/turbopack/crates/turbopack-node/js/src/worker_threads/evaluate.ts @@ -121,9 +121,6 @@ export const run = async ( } while (true) { - // need await a micro task, or that if some some request rejected, - // the error will be propergated to schedule threat, then causing panic - await Promise.resolve() const msg_str = await binding.recvMessageInWorker(workerId) const msg = JSON.parse(msg_str) as @@ -153,6 +150,9 @@ export const run = async ( requests.delete(msg.id) if (msg.error) { request.reject(new Error(msg.error)) + // need await a micro task, or else if some request rejected, + // the error will be propergated to schedule thread, then causing panic + await Promise.resolve() } else { request.resolve(msg.data) } diff --git a/turbopack/crates/turbopack-node/src/evaluate.rs b/turbopack/crates/turbopack-node/src/evaluate.rs index b59ac8776b0cc7..36e823e2a70bdc 100644 --- a/turbopack/crates/turbopack-node/src/evaluate.rs +++ b/turbopack/crates/turbopack-node/src/evaluate.rs @@ -13,7 +13,9 @@ use turbo_tasks::{ TryJoinIterExt, Vc, duration_span, fxindexmap, get_effects, trace::TraceRawVcs, }; use turbo_tasks_env::{EnvMap, ProcessEnv}; -use turbo_tasks_fs::{File, FileContent, FileSystemPath, json::parse_json_with_source_context, to_sys_path}; +use turbo_tasks_fs::{ + File, FileContent, FileSystemPath, json::parse_json_with_source_context, to_sys_path, +}; use turbopack_core::{ asset::AssetContent, changed::content_changed, From 5124441c4e581bfe746f1c7da1e125d652e6fb4c Mon Sep 17 00:00:00 2001 From: xusd320 Date: Thu, 27 Nov 2025 22:24:50 +0800 Subject: [PATCH 17/17] feat(turbopack-node): restore scale for process pool --- crates/next-api/src/project.rs | 11 +++++------ turbopack/crates/turbopack-node/src/evaluate.rs | 14 ++++++++++++++ .../crates/turbopack-node/src/process_pool/mod.rs | 2 -- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs index f1d7238f2787ce..55f6a89113c28f 100644 --- a/crates/next-api/src/project.rs +++ b/crates/next-api/src/project.rs @@ -1110,12 +1110,11 @@ impl Project { // At this point all modules have been computed and we can get rid of the node.js // process pools - // FIXME : - // if *self.is_watch_enabled().await? { - // turbopack_node::evaluate::scale_down(); - // } else { - // turbopack_node::evaluate::scale_zero(); - // } + if *self.is_watch_enabled().await? { + turbopack_node::evaluate::scale_down(); + } else { + turbopack_node::evaluate::scale_zero(); + } Ok(module_graphs_vc) } diff --git a/turbopack/crates/turbopack-node/src/evaluate.rs b/turbopack/crates/turbopack-node/src/evaluate.rs index 36e823e2a70bdc..56c70a6fa6b8fe 100644 --- a/turbopack/crates/turbopack-node/src/evaluate.rs +++ b/turbopack/crates/turbopack-node/src/evaluate.rs @@ -730,3 +730,17 @@ impl Issue for EvaluationIssue { Vc::cell(Some(self.source)) } } + +pub fn scale_down() { + #[cfg(feature = "process_pool")] + { + ChildProcessPool::scale_down(); + } +} + +pub fn scale_zero() { + #[cfg(feature = "process_pool")] + { + ChildProcessPool::scale_zero(); + } +} diff --git a/turbopack/crates/turbopack-node/src/process_pool/mod.rs b/turbopack/crates/turbopack-node/src/process_pool/mod.rs index 4bf41a16f1ee79..3d229d2287012e 100644 --- a/turbopack/crates/turbopack-node/src/process_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/process_pool/mod.rs @@ -805,7 +805,6 @@ impl ChildProcessPool { Ok((process, start.elapsed())) } - #[allow(dead_code)] pub fn scale_down() { let pools = ACTIVE_POOLS.lock().clone(); for pool in pools { @@ -813,7 +812,6 @@ impl ChildProcessPool { } } - #[allow(dead_code)] pub fn scale_zero() { let pools = ACTIVE_POOLS.lock().clone(); for pool in pools {