From b3bbcea1f9623594b9c6f5155afa20c7ea57051e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Sun, 14 Mar 2021 00:08:51 +0000 Subject: [PATCH 1/3] Add externals to metafile --- internal/bundler/bundler.go | 27 +++++++++++++ internal/resolver/resolver.go | 71 +++++++++++++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/internal/bundler/bundler.go b/internal/bundler/bundler.go index 213cf0ed4e5..e660ed1538b 100644 --- a/internal/bundler/bundler.go +++ b/internal/bundler/bundler.go @@ -1458,6 +1458,33 @@ func (s *scanner) processScannedFiles() []file { if !isFirstImport { sb.WriteString("\n ") } + sb.WriteString("]") + + // Write any externals + sb.WriteString(",\n \"external\": [") + + hasExternalImports := false + + for _, resolveResult := range result.resolveResults { + if resolveResult.IsExternal { + if hasExternalImports { + sb.WriteString(",") + } + hasExternalImports = true + sb.WriteString(fmt.Sprintf("\n {\n \"path\": %s,", js_printer.QuoteForJSON(resolveResult.PathPair.Primary.Text, s.options.ASCIIOnly))) + + if resolveResult.External.Source != "" { + sb.WriteString(fmt.Sprintf("\n \"source\": %s,", js_printer.QuoteForJSON(resolveResult.External.Source, s.options.ASCIIOnly))) + } + + sb.WriteString(fmt.Sprintf("\n \"kind\": %s\n }", js_printer.QuoteForJSON(resolveResult.External.Kind.StringForMetafile(), s.options.ASCIIOnly))) + } + } + + if hasExternalImports { + sb.WriteString("\n ") + } + sb.WriteString("]\n }") } diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index dfb25c55d74..02270601482 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -83,6 +83,42 @@ type IgnoreIfUnusedData struct { IsSideEffectsArrayInJSON bool } +type ExternalKind uint8 + +const ( + // Marked as external by the user (i.e. using the "External" config option) + UserExternal ExternalKind = iota + + // Marked as external by an internal routine + SystemExternal + + // Part of the Node built-in modules + NodeBuiltInExternal + + // Marked as external due to the presence of native bindings + NativeExternal +) + +func (kind ExternalKind) StringForMetafile() string { + switch kind { + case UserExternal: + return "user" + case SystemExternal: + return "system" + case NodeBuiltInExternal: + return "node-built-in" + case NativeExternal: + return "native-module" + default: + panic("Internal error") + } +} + +type ExternalData struct { + Kind ExternalKind + Source string +} + type ResolveResult struct { PathPair PathPair @@ -94,6 +130,7 @@ type ResolveResult struct { JSXFragment []string // Default if empty: "React.Fragment" IsExternal bool + External *ExternalData DifferentCase *fs.DifferentCase // If true, any ES6 imports to this file can be considered to have no side @@ -244,6 +281,7 @@ func (r *resolver) Resolve(sourceDir string, importPath string, kind ast.ImportK return &ResolveResult{ PathPair: PathPair{Primary: logger.Path{Text: importPath}}, IsExternal: true, + External: &ExternalData{Kind: SystemExternal, Source: sourceDir}, }, nil } @@ -260,6 +298,7 @@ func (r *resolver) Resolve(sourceDir string, importPath string, kind ast.ImportK return &ResolveResult{ PathPair: PathPair{Primary: logger.Path{Text: importPath}}, IsExternal: true, + External: &ExternalData{Kind: SystemExternal, Source: sourceDir}, }, nil } @@ -442,7 +481,11 @@ func (r *resolver) resolveWithoutSymlinks(sourceDir string, importPath string, k // been marked as an external module, mark it as *not* an absolute path. // That way we preserve the literal text in the output and don't generate // a relative path from the output directory to that path. - return &ResolveResult{PathPair: PathPair{Primary: logger.Path{Text: importPath}}, IsExternal: true}, nil + return &ResolveResult{ + PathPair: PathPair{Primary: logger.Path{Text: importPath}}, + IsExternal: true, + External: &ExternalData{Kind: UserExternal, Source: sourceDir}, + }, nil } // Run node's resolution rules (e.g. adding ".js") @@ -463,7 +506,11 @@ func (r *resolver) resolveWithoutSymlinks(sourceDir string, importPath string, k // Check for external packages first if r.options.ExternalModules.AbsPaths != nil && r.options.ExternalModules.AbsPaths[absPath] { - return &ResolveResult{PathPair: PathPair{Primary: logger.Path{Text: absPath, Namespace: "file"}}, IsExternal: true}, nil + return &ResolveResult{ + PathPair: PathPair{Primary: logger.Path{Text: absPath, Namespace: "file"}}, + IsExternal: true, + External: &ExternalData{Kind: UserExternal, Source: sourceDir}, + }, nil } // Check the non-package "browser" map for the first time (1 out of 2) @@ -498,7 +545,17 @@ func (r *resolver) resolveWithoutSymlinks(sourceDir string, importPath string, k query := importPath for { if r.options.ExternalModules.NodeModules[query] { - return &ResolveResult{PathPair: PathPair{Primary: logger.Path{Text: importPath}}, IsExternal: true}, nil + externalType := UserExternal + + if BuiltInNodeModules[importPath] { + externalType = NodeBuiltInExternal + } + + return &ResolveResult{ + PathPair: PathPair{Primary: logger.Path{Text: importPath}}, + IsExternal: true, + External: &ExternalData{Kind: externalType, Source: sourceDir}, + }, nil } // If the module "foo" has been marked as external, we also want to treat @@ -556,6 +613,14 @@ func (r *resolver) resolveWithoutSymlinks(sourceDir string, importPath string, k resultDir := r.fs.Dir(path.Text) resultDirInfo := r.dirInfoCached(resultDir) + if resultDirInfo.packageJSON.hasNativeBindings && r.options.ExternalizeNativeModules { + return &ResolveResult{ + PathPair: PathPair{Primary: logger.Path{Text: importPath}}, + IsExternal: true, + External: &ExternalData{Kind: NativeExternal, Source: sourceDir}, + }, nil + } + // Check the non-package "browser" map for the second time (2 out of 2) if resultDirInfo != nil && resultDirInfo.enclosingBrowserScope != nil { packageJSON := resultDirInfo.enclosingBrowserScope.packageJSON From 0962b7d4a6b2e6cb1999f5b71312bea36f702940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Sun, 14 Mar 2021 00:09:23 +0000 Subject: [PATCH 2/3] Add -externalize-native-modules flag --- internal/config/config.go | 11 +++-- internal/resolver/package_json.go | 43 +++++++++++++++++ pkg/api/api.go | 47 ++++++++++--------- pkg/api/api_impl.go | 77 ++++++++++++++++--------------- pkg/cli/cli_impl.go | 3 ++ 5 files changed, 115 insertions(+), 66 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index e0eaac0e985..31fd94309a7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -199,11 +199,12 @@ type Options struct { UnsupportedJSFeatures compat.JSFeature UnsupportedCSSFeatures compat.CSSFeature - ExtensionOrder []string - MainFields []string - Conditions []string - AbsNodePaths []string // The "NODE_PATH" variable from Node.js - ExternalModules ExternalModules + ExtensionOrder []string + MainFields []string + Conditions []string + AbsNodePaths []string // The "NODE_PATH" variable from Node.js + ExternalModules ExternalModules + ExternalizeNativeModules bool AbsOutputFile string AbsOutputDir string diff --git a/internal/resolver/package_json.go b/internal/resolver/package_json.go index 4d4945c616a..3723298bac7 100644 --- a/internal/resolver/package_json.go +++ b/internal/resolver/package_json.go @@ -64,6 +64,8 @@ type packageJSON struct { // This represents the "exports" field in this package.json file. exportsMap *peMap + + hasNativeBindings bool } func (r *resolver) parsePackageJSON(path string) *packageJSON { @@ -128,6 +130,38 @@ func (r *resolver) parsePackageJSON(path string) *packageJSON { } } + // Look for native module markers in "devDependencies" + if devDependenciesJSON, _, ok := getProperty(json, "devDependencies"); ok { + if devDependencies, ok := devDependenciesJSON.Data.(*js_ast.EObject); ok { + for _, prop := range devDependencies.Properties { + if dependencyName, ok := getString(prop.Key); ok { + if NativeModuleMarkers[dependencyName] { + packageJSON.hasNativeBindings = true + + break + } + } + } + } + } + + if !packageJSON.hasNativeBindings { + // Look for native module markers in "dependencies" + if dependenciesJSON, _, ok := getProperty(json, "dependencies"); ok { + if dependencies, ok := dependenciesJSON.Data.(*js_ast.EObject); ok { + for _, prop := range dependencies.Properties { + if dependencyName, ok := getString(prop.Key); ok { + if NativeModuleMarkers[dependencyName] { + packageJSON.hasNativeBindings = true + + break + } + } + } + } + } + } + // Read the "browser" property, but only when targeting the browser if browserJSON, _, ok := getProperty(json, "browser"); ok && r.options.Platform == config.PlatformBrowser { // We both want the ability to have the option of CJS vs. ESM and the @@ -701,3 +735,12 @@ func esmParsePackageName(packageSpecifier string) (packageName string, packageSu ok = true return } + +// If a module has any of these as dependencies, it likely has native bindings +var NativeModuleMarkers = map[string]bool{ + "bindings": true, + "nan": true, + "node-gyp-build": true, + "node-pre-gyp": true, + "prebuild": true, +} diff --git a/pkg/api/api.go b/pkg/api/api.go index 99820b87c5f..6e05922d4a9 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -241,29 +241,30 @@ type BuildOptions struct { Pure []string KeepNames bool - GlobalName string - Bundle bool - PreserveSymlinks bool - Splitting bool - Outfile string - Metafile bool - Outdir string - Outbase string - AbsWorkingDir string - Platform Platform - Format Format - External []string - MainFields []string - Conditions []string // For the "exports" field in "package.json" - Loader map[string]Loader - ResolveExtensions []string - Tsconfig string - OutExtensions map[string]string - PublicPath string - Inject []string - Banner map[string]string - Footer map[string]string - NodePaths []string // The "NODE_PATH" variable from Node.js + GlobalName string + Bundle bool + PreserveSymlinks bool + Splitting bool + Outfile string + Metafile bool + Outdir string + Outbase string + AbsWorkingDir string + Platform Platform + Format Format + External []string + ExternalizeNativeModules bool + MainFields []string + Conditions []string // For the "exports" field in "package.json" + Loader map[string]Loader + ResolveExtensions []string + Tsconfig string + OutExtensions map[string]string + PublicPath string + Inject []string + Banner map[string]string + Footer map[string]string + NodePaths []string // The "NODE_PATH" variable from Node.js ChunkNames string AssetNames string diff --git a/pkg/api/api_impl.go b/pkg/api/api_impl.go index 5214fca9223..95967857bb0 100644 --- a/pkg/api/api_impl.go +++ b/pkg/api/api_impl.go @@ -711,44 +711,45 @@ func rebuildImpl( Factory: validateJSX(log, buildOpts.JSXFactory, "factory"), Fragment: validateJSX(log, buildOpts.JSXFragment, "fragment"), }, - Defines: defines, - InjectedDefines: injectedDefines, - Platform: validatePlatform(buildOpts.Platform), - SourceMap: validateSourceMap(buildOpts.Sourcemap), - ExcludeSourcesContent: buildOpts.SourcesContent == SourcesContentExclude, - MangleSyntax: buildOpts.MinifySyntax, - RemoveWhitespace: buildOpts.MinifyWhitespace, - MinifyIdentifiers: buildOpts.MinifyIdentifiers, - ASCIIOnly: validateASCIIOnly(buildOpts.Charset), - IgnoreDCEAnnotations: validateIgnoreDCEAnnotations(buildOpts.TreeShaking), - GlobalName: validateGlobalName(log, buildOpts.GlobalName), - CodeSplitting: buildOpts.Splitting, - OutputFormat: validateFormat(buildOpts.Format), - AbsOutputFile: validatePath(log, realFS, buildOpts.Outfile, "outfile path"), - AbsOutputDir: validatePath(log, realFS, buildOpts.Outdir, "outdir path"), - AbsOutputBase: validatePath(log, realFS, buildOpts.Outbase, "outbase path"), - NeedsMetafile: buildOpts.Metafile, - ChunkPathTemplate: validatePathTemplate(buildOpts.ChunkNames), - AssetPathTemplate: validatePathTemplate(buildOpts.AssetNames), - OutputExtensionJS: outJS, - OutputExtensionCSS: outCSS, - ExtensionToLoader: validateLoaders(log, buildOpts.Loader), - ExtensionOrder: validateResolveExtensions(log, buildOpts.ResolveExtensions), - ExternalModules: validateExternals(log, realFS, buildOpts.External), - TsConfigOverride: validatePath(log, realFS, buildOpts.Tsconfig, "tsconfig path"), - MainFields: buildOpts.MainFields, - Conditions: append([]string{}, buildOpts.Conditions...), - PublicPath: buildOpts.PublicPath, - KeepNames: buildOpts.KeepNames, - InjectAbsPaths: make([]string, len(buildOpts.Inject)), - AbsNodePaths: make([]string, len(buildOpts.NodePaths)), - JSBanner: bannerJS, - JSFooter: footerJS, - CSSBanner: bannerCSS, - CSSFooter: footerCSS, - PreserveSymlinks: buildOpts.PreserveSymlinks, - WatchMode: buildOpts.Watch != nil, - Plugins: plugins, + Defines: defines, + InjectedDefines: injectedDefines, + Platform: validatePlatform(buildOpts.Platform), + SourceMap: validateSourceMap(buildOpts.Sourcemap), + ExcludeSourcesContent: buildOpts.SourcesContent == SourcesContentExclude, + MangleSyntax: buildOpts.MinifySyntax, + RemoveWhitespace: buildOpts.MinifyWhitespace, + MinifyIdentifiers: buildOpts.MinifyIdentifiers, + ASCIIOnly: validateASCIIOnly(buildOpts.Charset), + IgnoreDCEAnnotations: validateIgnoreDCEAnnotations(buildOpts.TreeShaking), + GlobalName: validateGlobalName(log, buildOpts.GlobalName), + CodeSplitting: buildOpts.Splitting, + OutputFormat: validateFormat(buildOpts.Format), + AbsOutputFile: validatePath(log, realFS, buildOpts.Outfile, "outfile path"), + AbsOutputDir: validatePath(log, realFS, buildOpts.Outdir, "outdir path"), + AbsOutputBase: validatePath(log, realFS, buildOpts.Outbase, "outbase path"), + NeedsMetafile: buildOpts.Metafile, + ChunkPathTemplate: validatePathTemplate(buildOpts.ChunkNames), + AssetPathTemplate: validatePathTemplate(buildOpts.AssetNames), + OutputExtensionJS: outJS, + OutputExtensionCSS: outCSS, + ExtensionToLoader: validateLoaders(log, buildOpts.Loader), + ExtensionOrder: validateResolveExtensions(log, buildOpts.ResolveExtensions), + ExternalModules: validateExternals(log, realFS, buildOpts.External), + ExternalizeNativeModules: buildOpts.ExternalizeNativeModules, + TsConfigOverride: validatePath(log, realFS, buildOpts.Tsconfig, "tsconfig path"), + MainFields: buildOpts.MainFields, + Conditions: append([]string{}, buildOpts.Conditions...), + PublicPath: buildOpts.PublicPath, + KeepNames: buildOpts.KeepNames, + InjectAbsPaths: make([]string, len(buildOpts.Inject)), + AbsNodePaths: make([]string, len(buildOpts.NodePaths)), + JSBanner: bannerJS, + JSFooter: footerJS, + CSSBanner: bannerCSS, + CSSFooter: footerCSS, + PreserveSymlinks: buildOpts.PreserveSymlinks, + WatchMode: buildOpts.Watch != nil, + Plugins: plugins, } if options.MainFields != nil { options.MainFields = append([]string{}, options.MainFields...) diff --git a/pkg/cli/cli_impl.go b/pkg/cli/cli_impl.go index 9d62a9c0b70..013484134e5 100644 --- a/pkg/cli/cli_impl.go +++ b/pkg/cli/cli_impl.go @@ -51,6 +51,9 @@ func parseOptionsImpl( case arg == "--bundle" && buildOpts != nil: buildOpts.Bundle = true + case arg == "--externalize-native-modules" && buildOpts != nil: + buildOpts.ExternalizeNativeModules = true + case arg == "--preserve-symlinks" && buildOpts != nil: buildOpts.PreserveSymlinks = true From 80c02c940fb1ab3f24d37518c166c3768f72ea10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Sun, 14 Mar 2021 11:20:03 +0000 Subject: [PATCH 3/3] Check that packageJSON exists --- internal/resolver/resolver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 02270601482..0a6c131e316 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -613,7 +613,7 @@ func (r *resolver) resolveWithoutSymlinks(sourceDir string, importPath string, k resultDir := r.fs.Dir(path.Text) resultDirInfo := r.dirInfoCached(resultDir) - if resultDirInfo.packageJSON.hasNativeBindings && r.options.ExternalizeNativeModules { + if resultDirInfo.packageJSON != nil && resultDirInfo.packageJSON.hasNativeBindings && r.options.ExternalizeNativeModules { return &ResolveResult{ PathPair: PathPair{Primary: logger.Path{Text: importPath}}, IsExternal: true,