feat(turbopack): Support export * from 'cjs' (vercel/turborepo#3111)

Many unrelated tests files are changed because this PR adds a new
runtime function, `__turbopack_cjs__`.

Fixes WEB-3

Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com>

crates/turbopack-ecmascript/js/src/runtime.js

@@ -109,6 +109,18 @@ function esm(exports, getters) {
   }
 }
 
+/**
+ * Adds the getters to the exports object
+ *
+ * @param {Exports} exports
+ * @param {Record<string, any>} props
+ */
+function cjs(exports, props) {
+  for (const key in props) {
+    defineProp(exports, key, { get: () => props[key], enumerable: true });
+  }
+}
+
 /**
  * @param {Module} module
  * @param {any} value
@@ -327,6 +339,7 @@ function instantiateModule(id, sourceType, sourceId) {
       x: externalRequire,
       i: esmImport.bind(null, module),
       s: esm.bind(null, module.exports),
+      j: cjs.bind(null, module.exports),
       v: exportValue.bind(null, module),
       m: module,
       c: moduleCache,

crates/turbopack-ecmascript/src/chunk/mod.rs

@@ -515,6 +515,7 @@ async fn module_factory(content: EcmascriptChunkItemContentVc) -> Result<CodeVc>
         "v: __turbopack_export_value__",
         "c: __turbopack_cache__",
         "l: __turbopack_load__",
+        "j: __turbopack_cjs__",
         "p: process",
         "g: global",
         // HACK

crates/turbopack-ecmascript/src/references/esm/base.rs

@@ -37,19 +37,23 @@ pub enum ReferencedAsset {
 impl ReferencedAsset {
     pub async fn get_ident(&self) -> Result<Option<String>> {
         Ok(match self {
-            ReferencedAsset::Some(asset) => {
-                let path = asset.path().to_string().await?;
-                Some(magic_identifier::encode(&format!(
-                    "imported module {}",
-                    path
-                )))
-            }
+            ReferencedAsset::Some(asset) => Some(Self::get_ident_from_placeable(asset).await?),
             ReferencedAsset::OriginalReferenceTypeExternal(request) => {
                 Some(magic_identifier::encode(&format!("external {}", request)))
             }
             ReferencedAsset::None => None,
         })
     }
+
+    pub(crate) async fn get_ident_from_placeable(
+        asset: &EcmascriptChunkPlaceableVc,
+    ) -> Result<String> {
+        let path = asset.path().to_string().await?;
+        Ok(magic_identifier::encode(&format!(
+            "imported module {}",
+            path
+        )))
+    }
 }
 
 #[turbo_tasks::value_impl]

crates/turbopack-ecmascript/src/references/esm/export.rs

@@ -8,16 +8,12 @@ use serde::{Deserialize, Serialize};
 use swc_core::{
     common::DUMMY_SP,
     ecma::ast::{
-        ComputedPropName, Expr, Ident, KeyValueProp, Lit, MemberExpr, MemberProp, Module,
-        ModuleItem, ObjectLit, Program, Prop, PropName, PropOrSpread, Script, Str,
+        ComputedPropName, Expr, ExprStmt, Ident, KeyValueProp, Lit, MemberExpr, MemberProp, Module,
+        ModuleItem, ObjectLit, Program, Prop, PropName, PropOrSpread, Script, Stmt, Str,
     },
-    quote,
-};
-use turbo_tasks::{
-    primitives::{StringVc, StringsVc},
-    trace::TraceRawVcs,
-    ValueToString,
+    quote, quote_expr,
 };
+use turbo_tasks::{primitives::StringVc, trace::TraceRawVcs, ValueToString};
 use turbopack_core::{
     asset::Asset,
     chunk::ChunkingContextVc,
@@ -29,6 +25,7 @@ use crate::{
     chunk::{EcmascriptChunkPlaceableVc, EcmascriptExports},
     code_gen::{CodeGenerateable, CodeGenerateableVc, CodeGeneration, CodeGenerationVc},
     create_visitor,
+    references::esm::base::insert_hoisted_stmt,
 };
 
 #[derive(Clone, Hash, Debug, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs)]
@@ -39,9 +36,16 @@ pub enum EsmExport {
     Error,
 }
 
+#[turbo_tasks::value]
+struct ExpandResults {
+    star_exports: Vec<String>,
+    has_cjs_exports: bool,
+}
+
 #[turbo_tasks::function]
-async fn expand_star_exports(root_asset: EcmascriptChunkPlaceableVc) -> Result<StringsVc> {
+async fn expand_star_exports(root_asset: EcmascriptChunkPlaceableVc) -> Result<ExpandResultsVc> {
     let mut set = HashSet::new();
+    let mut has_cjs_exports = false;
     let mut checked_assets = HashSet::new();
     checked_assets.insert(root_asset);
     let mut queue = vec![(root_asset, root_asset.get_exports())];
@@ -92,26 +96,33 @@ async fn expand_star_exports(root_asset: EcmascriptChunkPlaceableVc) -> Result<S
             .cell()
             .as_issue()
             .emit(),
-            EcmascriptExports::CommonJs => AnalyzeIssue {
-                code: None,
-                category: StringVc::cell("analyze".to_string()),
-                message: StringVc::cell(format!(
-                    "export * used with module {} which is a CommonJS module with exports only \
-                     available at runtime\nList all export names manually (`export {{ a, b, c }} \
-                     from \"...\") or rewrite the module to ESM.`",
-                    asset.path().to_string().await?
-                )),
-                path: asset.path(),
-                severity: IssueSeverity::Warning.into(),
-                source: None,
-                title: StringVc::cell("unexpected export *".to_string()),
+            EcmascriptExports::CommonJs => {
+                has_cjs_exports = true;
+                AnalyzeIssue {
+                    code: None,
+                    category: StringVc::cell("analyze".to_string()),
+                    message: StringVc::cell(format!(
+                        "export * used with module {} which is a CommonJS module with exports \
+                         only available at runtime\nList all export names manually (`export {{ a, \
+                         b, c }} from \"...\") or rewrite the module to ESM, to avoid the \
+                         additional runtime code.`",
+                        asset.path().to_string().await?
+                    )),
+                    path: asset.path(),
+                    severity: IssueSeverity::Warning.into(),
+                    source: None,
+                    title: StringVc::cell("unexpected export *".to_string()),
+                }
+                .cell()
+                .as_issue()
+                .emit()
             }
-            .cell()
-            .as_issue()
-            .emit(),
         }
     }
-    Ok(StringsVc::cell(set.into_iter().collect()))
+    Ok(ExpandResultsVc::cell(ExpandResults {
+        star_exports: set.into_iter().collect(),
+        has_cjs_exports,
+    }))
 }
 
 #[turbo_tasks::value(shared)]
@@ -137,9 +148,12 @@ impl CodeGenerateable for EsmExports {
             .map(|(k, v)| (Cow::<str>::Borrowed(k), Cow::Borrowed(v)))
             .collect();
         let mut props = Vec::new();
+        let mut cjs_exports = Vec::<Box<Expr>>::new();
+
         for esm_ref in this.star_exports.iter() {
             if let ReferencedAsset::Some(asset) = &*esm_ref.get_referenced_asset().await? {
-                let export_names = expand_star_exports(*asset).await?;
+                let export_info = expand_star_exports(*asset).await?;
+                let export_names = &export_info.star_exports;
                 for export in export_names.iter() {
                     if !all_exports.contains_key(&Cow::<str>::Borrowed(export)) {
                         all_exports.insert(
@@ -148,6 +162,15 @@ impl CodeGenerateable for EsmExports {
                         );
                     }
                 }
+
+                if export_info.has_cjs_exports {
+                    let ident = ReferencedAsset::get_ident_from_placeable(asset).await?;
+
+                    cjs_exports.push(quote_expr!(
+                        "__turbopack__cjs__($arg)",
+                        arg: Expr = Ident::new(ident.into(), DUMMY_SP).into()
+                    ));
+                }
             }
         }
         for (exported, local) in all_exports.into_iter() {
@@ -204,6 +227,14 @@ impl CodeGenerateable for EsmExports {
             span: DUMMY_SP,
             props,
         });
+        let cjs_stmt = if !cjs_exports.is_empty() {
+            Some(Stmt::Expr(ExprStmt {
+                span: DUMMY_SP,
+                expr: Expr::from_exprs(cjs_exports),
+            }))
+        } else {
+            None
+        };
 
         visitors.push(create_visitor!(visit_mut_program(program: &mut Program) {
             let stmt = quote!("__turbopack_esm__($getters);" as Stmt,
@@ -217,6 +248,9 @@ impl CodeGenerateable for EsmExports {
                     body.insert(0, stmt);
                 }
             }
+            if let Some(cjs_stmt) = cjs_stmt.clone() {
+                insert_hoisted_stmt(program, cjs_stmt);
+            }
         }));
 
         Ok(CodeGeneration { visitors }.into())

crates/turbopack-ecmascript/src/references/mod.rs

@@ -401,10 +401,10 @@ pub(crate) async fn analyze_ecmascript_module(
                 .into();
                 analysis.add_code_gen(esm_exports);
                 EcmascriptExports::EsmExports(esm_exports)
-            } else if let Program::Module(_) = program {
-                EcmascriptExports::None
-            } else {
+            } else if has_cjs_export(program) {
                 EcmascriptExports::CommonJs
+            } else {
+                EcmascriptExports::None
             };
 
             analysis.set_exports(exports);
@@ -1933,3 +1933,45 @@ async fn resolve_as_webpack_runtime(
 // TODO enable serialization
 #[turbo_tasks::value(transparent, serialization = "none")]
 pub struct AstPath(#[turbo_tasks(trace_ignore)] Vec<AstParentKind>);
+
+fn has_cjs_export(p: &Program) -> bool {
+    use swc_core::ecma::visit::{visit_obj_and_computed, Visit, VisitWith};
+
+    if let Program::Module(m) = p {
+        // Check for imports/exports
+        if m.body.iter().any(ModuleItem::is_module_decl) {
+            return false;
+        }
+    }
+
+    struct Visitor {
+        found: bool,
+    }
+
+    impl Visit for Visitor {
+        visit_obj_and_computed!();
+
+        fn visit_ident(&mut self, i: &Ident) {
+            if &*i.sym == "module" || &*i.sym == "exports" {
+                self.found = true;
+            }
+        }
+        fn visit_expr(&mut self, n: &Expr) {
+            if self.found {
+                return;
+            }
+            n.visit_children_with(self);
+        }
+
+        fn visit_stmt(&mut self, n: &Stmt) {
+            if self.found {
+                return;
+            }
+            n.visit_children_with(self);
+        }
+    }
+
+    let mut v = Visitor { found: false };
+    p.visit_with(&mut v);
+    v.found
+}

crates/turbopack-tests/tests/snapshot.rs

@@ -412,7 +412,12 @@ async fn handle_issues(source: FileSystemPathVc) -> Result<()> {
         let plain_issue = issue.into_plain();
         let hash = encode_hex(*plain_issue.internal_hash().await?);
 
-        let path = issues_path.join(&format!("{}-{}.txt", plain_issue.await?.title, &hash[0..6]));
+        // We replace "*" because it's not allowed for filename on Windows.
+        let path = issues_path.join(&format!(
+            "{}-{}.txt",
+            plain_issue.await?.title.replace('*', "__star__"),
+            &hash[0..6]
+        ));
         seen.insert(path);
 
         // Annoyingly, the PlainIssue.source -> PlainIssueSource.asset ->

crates/turbopack-tests/tests/snapshot/basic/async_chunk/output/79fb1_turbopack-tests_tests_snapshot_basic_async_chunk_input_import.js_manifest-chunk.js

@@ -1,6 +1,6 @@
 (self.TURBOPACK = self.TURBOPACK || []).push(["output/79fb1_turbopack-tests_tests_snapshot_basic_async_chunk_input_import.js_manifest-chunk.js", {
 
-"[project]/crates/turbopack-tests/tests/snapshot/basic/async_chunk/input/import.js/manifest-chunk.js": (({ r: __turbopack_require__, x: __turbopack_external_require__, i: __turbopack_import__, s: __turbopack_esm__, v: __turbopack_export_value__, c: __turbopack_cache__, l: __turbopack_load__, p: process, g: global, __dirname }) => (() => {
+"[project]/crates/turbopack-tests/tests/snapshot/basic/async_chunk/input/import.js/manifest-chunk.js": (({ r: __turbopack_require__, x: __turbopack_external_require__, i: __turbopack_import__, s: __turbopack_esm__, v: __turbopack_export_value__, c: __turbopack_cache__, l: __turbopack_load__, j: __turbopack_cjs__, p: process, g: global, __dirname }) => (() => {
 
 const chunks = [
     "output/crates_turbopack-tests_tests_snapshot_basic_async_chunk_input_import.js",