Merge 824f34b into 84e30f4

crates/next-core/src/next_client/context.rs

@@ -127,6 +127,7 @@ pub async fn get_client_module_options_context(
             postcss_package: Some(get_postcss_package_mapping(project_path)),
             ..Default::default()
         }),
+        enable_webpack_loaders: next_config.webpack_loaders_options().await?.clone_if(),
         enable_typescript_transform: true,
         rules: vec![(
             foreign_code_context_condition(next_config).await?,

crates/next-core/src/next_config.rs

@@ -1,13 +1,17 @@
 use std::collections::HashMap;
 
 use anyhow::Result;
+use indexmap::IndexMap;
 use serde::{Deserialize, Serialize};
 use turbo_tasks::{
     primitives::{BoolVc, StringsVc},
     trace::TraceRawVcs,
     Value,
 };
-use turbopack::evaluate_context::node_evaluate_asset_context;
+use turbopack::{
+    evaluate_context::node_evaluate_asset_context,
+    module_options::{WebpackLoadersOptions, WebpackLoadersOptionsVc},
+};
 use turbopack_core::{
     asset::Asset,
     reference_type::{EntryReferenceSubType, ReferenceType},
@@ -127,11 +131,12 @@ pub enum RemotePatternProtocal {
     Https,
 }
 
-#[derive(Clone, Debug, Ord, PartialOrd, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs)]
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs)]
 #[serde(rename_all = "camelCase")]
 pub struct ExperimentalConfig {
     pub server_components_external_packages: Option<Vec<String>>,
     pub app_dir: Option<bool>,
+    pub turbopack_webpack_loaders: Option<IndexMap<String, Vec<String>>>,
 }
 
 #[derive(Clone, Debug, Ord, PartialOrd, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs)]
@@ -209,6 +214,23 @@ impl NextConfigVc {
             self.await?.transpile_packages.clone().unwrap_or_default(),
         ))
     }
+
+    #[turbo_tasks::function]
+    pub async fn webpack_loaders_options(self) -> Result<WebpackLoadersOptionsVc> {
+        let this = self.await?;
+        let Some(turbopack_webpack_loaders) = this.experimental.as_ref().and_then(|experimental| experimental.turbopack_webpack_loaders.as_ref()) else {
+            return Ok(WebpackLoadersOptionsVc::cell(WebpackLoadersOptions::default()));
+        };
+        let mut extension_to_loaders = IndexMap::new();
+        for (ext, loaders) in turbopack_webpack_loaders {
+            extension_to_loaders.insert(ext.clone(), StringsVc::cell(loaders.clone()));
+        }
+        Ok(WebpackLoadersOptions {
+            extension_to_loaders,
+            ..Default::default()
+        }
+        .cell())
+    }
 }
 
 fn next_configs() -> StringsVc {

crates/next-core/src/next_server/context.rs

@@ -118,6 +118,7 @@ pub async fn get_server_module_options_context(
                     postcss_package: Some(get_postcss_package_mapping(project_path)),
                     ..Default::default()
                 }),
+                enable_webpack_loaders: next_config.webpack_loaders_options().await?.clone_if(),
                 enable_typescript_transform: true,
                 rules: vec![(
                     foreign_code_context_condition(next_config).await?,
@@ -138,6 +139,7 @@ pub async fn get_server_module_options_context(
                     postcss_package: Some(get_postcss_package_mapping(project_path)),
                     ..Default::default()
                 }),
+                enable_webpack_loaders: next_config.webpack_loaders_options().await?.clone_if(),
                 enable_typescript_transform: true,
                 rules: vec![(
                     foreign_code_context_condition(next_config).await?,
@@ -160,6 +162,7 @@ pub async fn get_server_module_options_context(
                     postcss_package: Some(get_postcss_package_mapping(project_path)),
                     ..Default::default()
                 }),
+                enable_webpack_loaders: next_config.webpack_loaders_options().await?.clone_if(),
                 enable_typescript_transform: true,
                 rules: vec![(
                     foreign_code_context_condition(next_config).await?,

crates/next-dev/src/lib.rs

@@ -35,9 +35,9 @@ use turbopack_core::{
     environment::ServerAddr,
     issue::IssueSeverity,
     resolve::{parse::RequestVc, pattern::QueryMapVc},
+    server_fs::ServerFileSystemVc,
 };
 use turbopack_dev_server::{
-    fs::DevServerFileSystemVc,
     introspect::IntrospectionSource,
     source::{
         combined::CombinedContentSourceVc, router::RouterContentSource,
@@ -279,7 +279,7 @@ async fn source(
     let output_root = output_fs.root().join(".next/server");
     let server_addr = ServerAddr::new(*server_addr).cell();
 
-    let dev_server_fs = DevServerFileSystemVc::new().as_file_system();
+    let dev_server_fs = ServerFileSystemVc::new().as_file_system();
     let dev_server_root = dev_server_fs.root();
     let entry_requests = entry_requests
         .iter()

crates/next-dev/tests/integration/turbopack/basic/webpack-loaders/hello.raw

@@ -0,0 +1 @@
+Hello World

crates/next-dev/tests/integration/turbopack/basic/webpack-loaders/index.js

@@ -0,0 +1,5 @@
+import source from "./hello.raw";
+
+it("runs a simple loader", () => {
+  expect(source).toBe("Hello World");
+});

crates/next-dev/tests/integration/turbopack/basic/webpack-loaders/next.config.js

@@ -0,0 +1,7 @@
+module.exports = {
+  experimental: {
+    turbopackWebpackLoaders: {
+      ".raw": ["raw-loader"],
+    },
+  },
+};

crates/next-dev/tests/package.json

@@ -4,11 +4,12 @@
   "devDependencies": {
     "@turbo/pack-test-harness": "*",
     "autoprefixer": "^10.4.13",
+    "loader-runner": "^4.3.0",
     "next": "13.0.8-canary.2",
     "postcss": "^8.4.20",
-    "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-test-renderer": "^18.2.0",
+    "react": "^18.2.0",
     "styled-jsx": "^5.1.0",
     "tailwindcss": "^3.2.4"
   }

crates/turbopack-core/src/lib.rs

@@ -15,6 +15,7 @@ pub mod issue;
 pub mod reference;
 pub mod reference_type;
 pub mod resolve;
+pub mod server_fs;
 pub mod source_asset;
 pub mod source_map;
 pub mod source_pos;

crates/turbopack-dev-server/src/fs.rs → crates/turbopack-core/src/server_fs.rs

@@ -6,36 +6,36 @@ use turbo_tasks_fs::{
 };
 
 #[turbo_tasks::value]
-pub struct DevServerFileSystem {}
+pub struct ServerFileSystem {}
 
 #[turbo_tasks::value_impl]
-impl DevServerFileSystemVc {
+impl ServerFileSystemVc {
     #[turbo_tasks::function]
     pub fn new() -> Self {
-        Self::cell(DevServerFileSystem {})
+        Self::cell(ServerFileSystem {})
     }
 }
 
 #[turbo_tasks::value_impl]
-impl FileSystem for DevServerFileSystem {
+impl FileSystem for ServerFileSystem {
     #[turbo_tasks::function]
     fn read(&self, _fs_path: FileSystemPathVc) -> Result<FileContentVc> {
-        bail!("Reading is not possible from the virtual filesystem for the dev server")
+        bail!("Reading is not possible from the marker filesystem for the server")
     }
 
     #[turbo_tasks::function]
     fn read_link(&self, _fs_path: FileSystemPathVc) -> Result<LinkContentVc> {
-        bail!("Reading is not possible from the virtual filesystem for the dev server")
+        bail!("Reading is not possible from the marker filesystem for the server")
     }
 
     #[turbo_tasks::function]
     fn read_dir(&self, _fs_path: FileSystemPathVc) -> Result<DirectoryContentVc> {
-        bail!("Reading is not possible from the virtual filesystem for the dev server")
+        bail!("Reading is not possible from the marker filesystem for the server")
     }
 
     #[turbo_tasks::function]
     fn write(&self, _fs_path: FileSystemPathVc, _content: FileContentVc) -> Result<CompletionVc> {
-        bail!("Writing is not possible to the virtual filesystem for the dev server")
+        bail!("Writing is not possible to the marker filesystem for the server")
     }
 
     #[turbo_tasks::function]
@@ -44,19 +44,19 @@ impl FileSystem for DevServerFileSystem {
         _fs_path: FileSystemPathVc,
         _target: LinkContentVc,
     ) -> Result<CompletionVc> {
-        bail!("Writing is not possible to the virtual filesystem for the dev server")
+        bail!("Writing is not possible to the marker filesystem for the  server")
     }
 
     #[turbo_tasks::function]
     fn metadata(&self, _fs_path: FileSystemPathVc) -> Result<FileMetaVc> {
-        bail!("Reading is not possible from the virtual filesystem for the dev server")
+        bail!("Reading is not possible from the marker filesystem for the  server")
     }
 }
 
 #[turbo_tasks::value_impl]
-impl ValueToString for DevServerFileSystem {
+impl ValueToString for ServerFileSystem {
     #[turbo_tasks::function]
     fn to_string(&self) -> StringVc {
-        StringVc::cell("root of the dev server".to_string())
+        StringVc::cell("root of the server".to_string())
     }
 }

crates/turbopack-dev-server/src/lib.rs

@@ -2,7 +2,6 @@
 #![feature(trait_alias)]
 #![feature(array_chunks)]
 
-pub mod fs;
 pub mod html;
 pub mod introspect;
 pub mod source;

crates/turbopack-node/js/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@vercel/turbopack-node",
+  "version": "0.0.0",
+  "description": "turbopack node runtime",
+  "license": "UNLICENSED",
+  "private": true,
+  "scripts": {
+    "check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "loader-runner": "^4.3.0"
+  },
+  "devDependencies": {
+    "@types/loader-runner": "^2.2.4",
+    "@types/node": "^18.11.11"
+  }
+}

crates/turbopack-node/js/src/transforms/webpack-loaders.ts

@@ -0,0 +1,70 @@
+declare const __turbopack_external_require__: (id: string) => any;
+
+import type { Ipc } from "../ipc/evaluate";
+import {
+  relative,
+  isAbsolute,
+  sep,
+  dirname,
+  resolve as pathResolve,
+} from "path";
+
+const { runLoaders } = __turbopack_external_require__(
+  "loader-runner"
+) as typeof import("loader-runner");
+
+const contextDir = process.cwd();
+const toPath = (file: string) => {
+  const relPath = relative(contextDir, file);
+  if (isAbsolute(relPath)) {
+    throw new Error(
+      `Cannot depend on path (${file}) outside of root directory (${contextDir})`
+    );
+  }
+  return sep !== "/" ? relPath.replaceAll(sep, "/") : relPath;
+};
+
+const transform = (ipc: Ipc, content: string, name: string, loaders: any[]) => {
+  return new Promise((resolve, reject) => {
+    const resource = pathResolve(contextDir, name);
+    const resourceDir = dirname(resource);
+    // TODO this should be handled in turbopack instead to ensure it's watched
+    loaders = loaders.map((loader: any) => {
+      return require.resolve(loader, { paths: [resourceDir] });
+    });
+    runLoaders(
+      {
+        resource,
+        context: {
+          rootContext: contextDir,
+        },
+        loaders,
+        readResource: (_filename, callback) => {
+          // TODO assuming the filename === resource, but loaders might change that
+          callback(null, Buffer.from(content, "utf-8"));
+        },
+      },
+      (err, result) => {
+        if (err) return reject(err);
+        for (const dep of result.contextDependencies) {
+          ipc.send({
+            type: "dirDependency",
+            path: toPath(dep),
+            glob: "**",
+          });
+        }
+        for (const dep of result.fileDependencies) {
+          ipc.send({
+            type: "fileDependency",
+            path: toPath(dep),
+          });
+        }
+        if (!result.result) return reject(new Error("No result from loaders"));
+        const [source, map] = result.result;
+        resolve({ source, map });
+      }
+    );
+  });
+};
+
+export { transform as default };

crates/turbopack-node/js/tsconfig.json

@@ -0,0 +1,30 @@
+{
+  "compilerOptions": {
+    // type checking
+    "strict": true,
+    "noFallthroughCasesInSwitch": true,
+    "skipLibCheck": true,
+
+    // interop constraints
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true,
+
+    // js support
+    "allowJs": true,
+    "checkJs": false,
+
+    // environment
+    "lib": ["ESNext"],
+    "target": "esnext",
+
+    // modules
+    "baseUrl": ".",
+    "module": "esnext",
+    "moduleResolution": "node",
+
+    // emit
+    "noEmit": true,
+    "stripInternal": true
+  },
+  "include": ["src"]
+}

crates/turbopack-node/src/transforms/mod.rs

@@ -1 +1,3 @@
 pub mod postcss;
+mod util;
+pub mod webpack;