Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add externals to metafile + externalize native modules #972

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions internal/bundler/bundler.go
Original file line number Diff line number Diff line change
Expand Up @@ -1458,6 +1458,33 @@ func (s *scanner) processScannedFiles() []file {
if !isFirstImport {
sb.WriteString("\n ")
}
sb.WriteString("]")

// Write any externals
sb.WriteString(",\n \"external\": [")

hasExternalImports := false

for _, resolveResult := range result.resolveResults {
if resolveResult.IsExternal {
if hasExternalImports {
sb.WriteString(",")
}
hasExternalImports = true
sb.WriteString(fmt.Sprintf("\n {\n \"path\": %s,", js_printer.QuoteForJSON(resolveResult.PathPair.Primary.Text, s.options.ASCIIOnly)))

if resolveResult.External.Source != "" {
sb.WriteString(fmt.Sprintf("\n \"source\": %s,", js_printer.QuoteForJSON(resolveResult.External.Source, s.options.ASCIIOnly)))
}

sb.WriteString(fmt.Sprintf("\n \"kind\": %s\n }", js_printer.QuoteForJSON(resolveResult.External.Kind.StringForMetafile(), s.options.ASCIIOnly)))
}
}

if hasExternalImports {
sb.WriteString("\n ")
}

sb.WriteString("]\n }")
}

Expand Down
11 changes: 6 additions & 5 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,11 +199,12 @@ type Options struct {
UnsupportedJSFeatures compat.JSFeature
UnsupportedCSSFeatures compat.CSSFeature

ExtensionOrder []string
MainFields []string
Conditions []string
AbsNodePaths []string // The "NODE_PATH" variable from Node.js
ExternalModules ExternalModules
ExtensionOrder []string
MainFields []string
Conditions []string
AbsNodePaths []string // The "NODE_PATH" variable from Node.js
ExternalModules ExternalModules
ExternalizeNativeModules bool

AbsOutputFile string
AbsOutputDir string
Expand Down
43 changes: 43 additions & 0 deletions internal/resolver/package_json.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ type packageJSON struct {

// This represents the "exports" field in this package.json file.
exportsMap *peMap

hasNativeBindings bool
}

func (r *resolver) parsePackageJSON(path string) *packageJSON {
Expand Down Expand Up @@ -128,6 +130,38 @@ func (r *resolver) parsePackageJSON(path string) *packageJSON {
}
}

// Look for native module markers in "devDependencies"
if devDependenciesJSON, _, ok := getProperty(json, "devDependencies"); ok {
if devDependencies, ok := devDependenciesJSON.Data.(*js_ast.EObject); ok {
for _, prop := range devDependencies.Properties {
if dependencyName, ok := getString(prop.Key); ok {
if NativeModuleMarkers[dependencyName] {
packageJSON.hasNativeBindings = true

break
}
}
}
}
}

if !packageJSON.hasNativeBindings {
// Look for native module markers in "dependencies"
if dependenciesJSON, _, ok := getProperty(json, "dependencies"); ok {
if dependencies, ok := dependenciesJSON.Data.(*js_ast.EObject); ok {
for _, prop := range dependencies.Properties {
if dependencyName, ok := getString(prop.Key); ok {
if NativeModuleMarkers[dependencyName] {
packageJSON.hasNativeBindings = true

break
}
}
}
}
}
}

// Read the "browser" property, but only when targeting the browser
if browserJSON, _, ok := getProperty(json, "browser"); ok && r.options.Platform == config.PlatformBrowser {
// We both want the ability to have the option of CJS vs. ESM and the
Expand Down Expand Up @@ -701,3 +735,12 @@ func esmParsePackageName(packageSpecifier string) (packageName string, packageSu
ok = true
return
}

// If a module has any of these as dependencies, it likely has native bindings
var NativeModuleMarkers = map[string]bool{
"bindings": true,
"nan": true,
"node-gyp-build": true,
"node-pre-gyp": true,
"prebuild": true,
}
71 changes: 68 additions & 3 deletions internal/resolver/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,42 @@ type IgnoreIfUnusedData struct {
IsSideEffectsArrayInJSON bool
}

type ExternalKind uint8

const (
// Marked as external by the user (i.e. using the "External" config option)
UserExternal ExternalKind = iota

// Marked as external by an internal routine
SystemExternal

// Part of the Node built-in modules
NodeBuiltInExternal

// Marked as external due to the presence of native bindings
NativeExternal
)

func (kind ExternalKind) StringForMetafile() string {
switch kind {
case UserExternal:
return "user"
case SystemExternal:
return "system"
case NodeBuiltInExternal:
return "node-built-in"
case NativeExternal:
return "native-module"
default:
panic("Internal error")
}
}

type ExternalData struct {
Kind ExternalKind
Source string
}

type ResolveResult struct {
PathPair PathPair

Expand All @@ -94,6 +130,7 @@ type ResolveResult struct {
JSXFragment []string // Default if empty: "React.Fragment"

IsExternal bool
External *ExternalData
DifferentCase *fs.DifferentCase

// If true, any ES6 imports to this file can be considered to have no side
Expand Down Expand Up @@ -244,6 +281,7 @@ func (r *resolver) Resolve(sourceDir string, importPath string, kind ast.ImportK
return &ResolveResult{
PathPair: PathPair{Primary: logger.Path{Text: importPath}},
IsExternal: true,
External: &ExternalData{Kind: SystemExternal, Source: sourceDir},
}, nil
}

Expand All @@ -260,6 +298,7 @@ func (r *resolver) Resolve(sourceDir string, importPath string, kind ast.ImportK
return &ResolveResult{
PathPair: PathPair{Primary: logger.Path{Text: importPath}},
IsExternal: true,
External: &ExternalData{Kind: SystemExternal, Source: sourceDir},
}, nil
}

Expand Down Expand Up @@ -442,7 +481,11 @@ func (r *resolver) resolveWithoutSymlinks(sourceDir string, importPath string, k
// been marked as an external module, mark it as *not* an absolute path.
// That way we preserve the literal text in the output and don't generate
// a relative path from the output directory to that path.
return &ResolveResult{PathPair: PathPair{Primary: logger.Path{Text: importPath}}, IsExternal: true}, nil
return &ResolveResult{
PathPair: PathPair{Primary: logger.Path{Text: importPath}},
IsExternal: true,
External: &ExternalData{Kind: UserExternal, Source: sourceDir},
}, nil
}

// Run node's resolution rules (e.g. adding ".js")
Expand All @@ -463,7 +506,11 @@ func (r *resolver) resolveWithoutSymlinks(sourceDir string, importPath string, k

// Check for external packages first
if r.options.ExternalModules.AbsPaths != nil && r.options.ExternalModules.AbsPaths[absPath] {
return &ResolveResult{PathPair: PathPair{Primary: logger.Path{Text: absPath, Namespace: "file"}}, IsExternal: true}, nil
return &ResolveResult{
PathPair: PathPair{Primary: logger.Path{Text: absPath, Namespace: "file"}},
IsExternal: true,
External: &ExternalData{Kind: UserExternal, Source: sourceDir},
}, nil
}

// Check the non-package "browser" map for the first time (1 out of 2)
Expand Down Expand Up @@ -498,7 +545,17 @@ func (r *resolver) resolveWithoutSymlinks(sourceDir string, importPath string, k
query := importPath
for {
if r.options.ExternalModules.NodeModules[query] {
return &ResolveResult{PathPair: PathPair{Primary: logger.Path{Text: importPath}}, IsExternal: true}, nil
externalType := UserExternal

if BuiltInNodeModules[importPath] {
externalType = NodeBuiltInExternal
}

return &ResolveResult{
PathPair: PathPair{Primary: logger.Path{Text: importPath}},
IsExternal: true,
External: &ExternalData{Kind: externalType, Source: sourceDir},
}, nil
}

// If the module "foo" has been marked as external, we also want to treat
Expand Down Expand Up @@ -556,6 +613,14 @@ func (r *resolver) resolveWithoutSymlinks(sourceDir string, importPath string, k
resultDir := r.fs.Dir(path.Text)
resultDirInfo := r.dirInfoCached(resultDir)

if resultDirInfo.packageJSON != nil && resultDirInfo.packageJSON.hasNativeBindings && r.options.ExternalizeNativeModules {
return &ResolveResult{
PathPair: PathPair{Primary: logger.Path{Text: importPath}},
IsExternal: true,
External: &ExternalData{Kind: NativeExternal, Source: sourceDir},
}, nil
}

// Check the non-package "browser" map for the second time (2 out of 2)
if resultDirInfo != nil && resultDirInfo.enclosingBrowserScope != nil {
packageJSON := resultDirInfo.enclosingBrowserScope.packageJSON
Expand Down
47 changes: 24 additions & 23 deletions pkg/api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -241,29 +241,30 @@ type BuildOptions struct {
Pure []string
KeepNames bool

GlobalName string
Bundle bool
PreserveSymlinks bool
Splitting bool
Outfile string
Metafile bool
Outdir string
Outbase string
AbsWorkingDir string
Platform Platform
Format Format
External []string
MainFields []string
Conditions []string // For the "exports" field in "package.json"
Loader map[string]Loader
ResolveExtensions []string
Tsconfig string
OutExtensions map[string]string
PublicPath string
Inject []string
Banner map[string]string
Footer map[string]string
NodePaths []string // The "NODE_PATH" variable from Node.js
GlobalName string
Bundle bool
PreserveSymlinks bool
Splitting bool
Outfile string
Metafile bool
Outdir string
Outbase string
AbsWorkingDir string
Platform Platform
Format Format
External []string
ExternalizeNativeModules bool
MainFields []string
Conditions []string // For the "exports" field in "package.json"
Loader map[string]Loader
ResolveExtensions []string
Tsconfig string
OutExtensions map[string]string
PublicPath string
Inject []string
Banner map[string]string
Footer map[string]string
NodePaths []string // The "NODE_PATH" variable from Node.js

ChunkNames string
AssetNames string
Expand Down
77 changes: 39 additions & 38 deletions pkg/api/api_impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -711,44 +711,45 @@ func rebuildImpl(
Factory: validateJSX(log, buildOpts.JSXFactory, "factory"),
Fragment: validateJSX(log, buildOpts.JSXFragment, "fragment"),
},
Defines: defines,
InjectedDefines: injectedDefines,
Platform: validatePlatform(buildOpts.Platform),
SourceMap: validateSourceMap(buildOpts.Sourcemap),
ExcludeSourcesContent: buildOpts.SourcesContent == SourcesContentExclude,
MangleSyntax: buildOpts.MinifySyntax,
RemoveWhitespace: buildOpts.MinifyWhitespace,
MinifyIdentifiers: buildOpts.MinifyIdentifiers,
ASCIIOnly: validateASCIIOnly(buildOpts.Charset),
IgnoreDCEAnnotations: validateIgnoreDCEAnnotations(buildOpts.TreeShaking),
GlobalName: validateGlobalName(log, buildOpts.GlobalName),
CodeSplitting: buildOpts.Splitting,
OutputFormat: validateFormat(buildOpts.Format),
AbsOutputFile: validatePath(log, realFS, buildOpts.Outfile, "outfile path"),
AbsOutputDir: validatePath(log, realFS, buildOpts.Outdir, "outdir path"),
AbsOutputBase: validatePath(log, realFS, buildOpts.Outbase, "outbase path"),
NeedsMetafile: buildOpts.Metafile,
ChunkPathTemplate: validatePathTemplate(buildOpts.ChunkNames),
AssetPathTemplate: validatePathTemplate(buildOpts.AssetNames),
OutputExtensionJS: outJS,
OutputExtensionCSS: outCSS,
ExtensionToLoader: validateLoaders(log, buildOpts.Loader),
ExtensionOrder: validateResolveExtensions(log, buildOpts.ResolveExtensions),
ExternalModules: validateExternals(log, realFS, buildOpts.External),
TsConfigOverride: validatePath(log, realFS, buildOpts.Tsconfig, "tsconfig path"),
MainFields: buildOpts.MainFields,
Conditions: append([]string{}, buildOpts.Conditions...),
PublicPath: buildOpts.PublicPath,
KeepNames: buildOpts.KeepNames,
InjectAbsPaths: make([]string, len(buildOpts.Inject)),
AbsNodePaths: make([]string, len(buildOpts.NodePaths)),
JSBanner: bannerJS,
JSFooter: footerJS,
CSSBanner: bannerCSS,
CSSFooter: footerCSS,
PreserveSymlinks: buildOpts.PreserveSymlinks,
WatchMode: buildOpts.Watch != nil,
Plugins: plugins,
Defines: defines,
InjectedDefines: injectedDefines,
Platform: validatePlatform(buildOpts.Platform),
SourceMap: validateSourceMap(buildOpts.Sourcemap),
ExcludeSourcesContent: buildOpts.SourcesContent == SourcesContentExclude,
MangleSyntax: buildOpts.MinifySyntax,
RemoveWhitespace: buildOpts.MinifyWhitespace,
MinifyIdentifiers: buildOpts.MinifyIdentifiers,
ASCIIOnly: validateASCIIOnly(buildOpts.Charset),
IgnoreDCEAnnotations: validateIgnoreDCEAnnotations(buildOpts.TreeShaking),
GlobalName: validateGlobalName(log, buildOpts.GlobalName),
CodeSplitting: buildOpts.Splitting,
OutputFormat: validateFormat(buildOpts.Format),
AbsOutputFile: validatePath(log, realFS, buildOpts.Outfile, "outfile path"),
AbsOutputDir: validatePath(log, realFS, buildOpts.Outdir, "outdir path"),
AbsOutputBase: validatePath(log, realFS, buildOpts.Outbase, "outbase path"),
NeedsMetafile: buildOpts.Metafile,
ChunkPathTemplate: validatePathTemplate(buildOpts.ChunkNames),
AssetPathTemplate: validatePathTemplate(buildOpts.AssetNames),
OutputExtensionJS: outJS,
OutputExtensionCSS: outCSS,
ExtensionToLoader: validateLoaders(log, buildOpts.Loader),
ExtensionOrder: validateResolveExtensions(log, buildOpts.ResolveExtensions),
ExternalModules: validateExternals(log, realFS, buildOpts.External),
ExternalizeNativeModules: buildOpts.ExternalizeNativeModules,
TsConfigOverride: validatePath(log, realFS, buildOpts.Tsconfig, "tsconfig path"),
MainFields: buildOpts.MainFields,
Conditions: append([]string{}, buildOpts.Conditions...),
PublicPath: buildOpts.PublicPath,
KeepNames: buildOpts.KeepNames,
InjectAbsPaths: make([]string, len(buildOpts.Inject)),
AbsNodePaths: make([]string, len(buildOpts.NodePaths)),
JSBanner: bannerJS,
JSFooter: footerJS,
CSSBanner: bannerCSS,
CSSFooter: footerCSS,
PreserveSymlinks: buildOpts.PreserveSymlinks,
WatchMode: buildOpts.Watch != nil,
Plugins: plugins,
}
if options.MainFields != nil {
options.MainFields = append([]string{}, options.MainFields...)
Expand Down
3 changes: 3 additions & 0 deletions pkg/cli/cli_impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ func parseOptionsImpl(
case arg == "--bundle" && buildOpts != nil:
buildOpts.Bundle = true

case arg == "--externalize-native-modules" && buildOpts != nil:
buildOpts.ExternalizeNativeModules = true

case arg == "--preserve-symlinks" && buildOpts != nil:
buildOpts.PreserveSymlinks = true

Expand Down