fix #70: implement the "--ascii-only" flag

CHANGELOG.md

@@ -2,6 +2,22 @@
 
 ## Unreleased
 
+* Add the `--ascii-only` flag ([#70](https://github.com/evanw/esbuild/issues/70))
+
+    While esbuild's output is encoded using UTF-8 encoding, there are many other character encodings in the wild (e.g. [Windows-1250](https://en.wikipedia.org/wiki/Windows-1250)). You can explicitly mark the output files as UTF-8 by adding `<meta charset="utf-8">` to your HTML page or by including `charset=utf-8` in the `Content-Type` header sent by your server. This is probably a good idea regardless of the contents of esbuild's output since information being displayed to users is probably also encoded using UTF-8.
+
+    However, sometimes it's not possible to guarantee that your users will be running your code as UTF-8. For example, you may not control the server response or the contents of the HTML page that loads your script. Also, if your code needs to run in IE, there are [certain cases](https://docs.microsoft.com/en-us/troubleshoot/browsers/wrong-character-set-for-html-page) where IE may ignore the `<meta charset="utf-8">` tag and make up another encoding instead.
+
+    In that case, you can now enable esbuild's ASCII-only output mode using the `--ascii-only` flag. This escapes all Unicode code points in identifiers and strings that are outside of the printable ASCII range (`\x20-\x7E` inclusive).
+
+    Further details:
+
+    * This does not yet escape non-ASCII characters embedded in regular expressions. This is because esbuild does not currently parse the contents of regular expressions at all. The flag was added despite this limitation because it's still useful for code that doesn't contain cases like this.
+
+    * This flag does not apply to comments. I believe preserving non-ASCII data in comments should be fine because even if the encoding is wrong, the run time environment should completely ignore the contents of all comments.
+
+    * This ASCII-only flag simultaneously applies to all output file types (JavaScript, CSS, and JSON).
+
 * Interpret escape sequences in CSS tokens
 
     Escape sequences in CSS tokens are now interpreted. This was already the case for string and URL tokens before, but this is now the case for all identifier-like tokens as well. For example, `c\6flor: #\66 00` is now correctly recognized as `color: #f00`.

README.md

@@ -405,6 +405,7 @@ Advanced options:
                             browser and "main,module" when platform is node)
   --public-path=...         Set the base URL for the "file" loader
   --color=...               Force use of color terminal escapes (true | false)
+  --ascii-only              Escape all non-ASCII characters (makes code bigger)
   --avoid-tdz               An optimization for large bundles in Safari
 
 Examples:

cmd/esbuild/main.go

@@ -69,6 +69,7 @@ Advanced options:
                             browser and "main,module" when platform is node)
   --public-path=...         Set the base URL for the "file" loader
   --color=...               Force use of color terminal escapes (true | false)
+  --ascii-only              Escape all non-ASCII characters (makes code bigger)
   --avoid-tdz               An optimization for large bundles in Safari
 
 Examples:

internal/bundler/bundler.go

@@ -753,7 +753,7 @@ func ScanBundle(log logger.Log, fs fs.FS, res resolver.Resolver, entryPaths []st
 
 		// Begin the metadata chunk
 		if options.AbsMetadataFile != "" {
-			j.AddBytes(js_printer.QuoteForJSON(result.file.source.PrettyPath))
+			j.AddBytes(js_printer.QuoteForJSON(result.file.source.PrettyPath, options.ASCIIOnly))
 			j.AddString(fmt.Sprintf(": {\n      \"bytes\": %d,\n      \"imports\": [", len(result.file.source.Contents)))
 		}
 
@@ -797,7 +797,7 @@ func ScanBundle(log logger.Log, fs fs.FS, res resolver.Resolver, entryPaths []st
 						j.AddString(",\n        ")
 					}
 					j.AddString(fmt.Sprintf("{\n          \"path\": %s\n        }",
-						js_printer.QuoteForJSON(results[*record.SourceIndex].file.source.PrettyPath)))
+						js_printer.QuoteForJSON(results[*record.SourceIndex].file.source.PrettyPath, options.ASCIIOnly)))
 				}
 
 				// Importing a JavaScript file from a CSS file is not allowed.
@@ -957,7 +957,7 @@ func (b *Bundle) Compile(log logger.Log, options config.Options) []OutputFile {
 	if options.AbsMetadataFile != "" {
 		outputFiles = append(outputFiles, OutputFile{
 			AbsPath:  options.AbsMetadataFile,
-			Contents: b.generateMetadataJSON(outputFiles),
+			Contents: b.generateMetadataJSON(outputFiles, options.ASCIIOnly),
 		})
 	}
 
@@ -1088,7 +1088,7 @@ func (b *Bundle) lowestCommonAncestorDirectory(codeSplitting bool) string {
 	return lowestAbsDir
 }
 
-func (b *Bundle) generateMetadataJSON(results []OutputFile) []byte {
+func (b *Bundle) generateMetadataJSON(results []OutputFile, asciiOnly bool) []byte {
 	// Sort files by key path for determinism
 	sorted := make(indexAndPathArray, 0, len(b.files))
 	for sourceIndex, file := range b.files {
@@ -1124,7 +1124,7 @@ func (b *Bundle) generateMetadataJSON(results []OutputFile) []byte {
 				j.AddString(",\n    ")
 			}
 			j.AddString(fmt.Sprintf("%s: ", js_printer.QuoteForJSON(b.res.PrettyPath(
-				logger.Path{Text: result.AbsPath, Namespace: "file"}))))
+				logger.Path{Text: result.AbsPath, Namespace: "file"}), asciiOnly)))
 			j.AddBytes(result.jsonMetadataChunk)
 		}
 	}

internal/bundler/linker.go

@@ -3123,6 +3123,7 @@ func (c *linkerContext) generateCodeForFileInChunkJS(
 		OutputFormat:        c.options.OutputFormat,
 		RemoveWhitespace:    c.options.RemoveWhitespace,
 		MangleSyntax:        c.options.MangleSyntax,
+		ASCIIOnly:           c.options.ASCIIOnly,
 		ToModuleRef:         toModuleRef,
 		ExtractComments:     c.options.Mode == config.ModeBundle && c.options.RemoveWhitespace,
 		UnsupportedFeatures: c.options.UnsupportedJSFeatures,
@@ -3384,7 +3385,7 @@ func (repr *chunkReprJS) generate(c *linkerContext, chunk *chunkInfo) func([]ast
 
 			// Add the top-level directive if present
 			if repr.ast.Directive != "" {
-				quoted := string(js_printer.QuoteForJSON(repr.ast.Directive)) + ";" + newline
+				quoted := string(js_printer.QuoteForJSON(repr.ast.Directive, c.options.ASCIIOnly)) + ";" + newline
 				prevOffset.advanceString(quoted)
 				j.AddString(quoted)
 				newlineBeforeComment = true
@@ -3428,7 +3429,7 @@ func (repr *chunkReprJS) generate(c *linkerContext, chunk *chunkInfo) func([]ast
 				}
 				importAbsPath := c.fs.Join(c.options.AbsOutputDir, chunk.relDir, record.Path.Text)
 				jMeta.AddString(fmt.Sprintf("\n        {\n          \"path\": %s\n        }",
-					js_printer.QuoteForJSON(c.res.PrettyPath(logger.Path{Text: importAbsPath, Namespace: "file"}))))
+					js_printer.QuoteForJSON(c.res.PrettyPath(logger.Path{Text: importAbsPath, Namespace: "file"}), c.options.ASCIIOnly)))
 			}
 			if !isFirstMeta {
 				jMeta.AddString("\n      ")
@@ -3535,7 +3536,7 @@ func (repr *chunkReprJS) generate(c *linkerContext, chunk *chunkInfo) func([]ast
 						jMeta.AddString(",")
 					}
 					jMeta.AddString(fmt.Sprintf("\n        %s: {\n          \"bytesInOutput\": %d\n        }",
-						js_printer.QuoteForJSON(c.files[compileResult.sourceIndex].source.PrettyPath),
+						js_printer.QuoteForJSON(c.files[compileResult.sourceIndex].source.PrettyPath, c.options.ASCIIOnly),
 						len(js)))
 				}
 			}
@@ -3697,6 +3698,7 @@ func (repr *chunkReprCSS) generate(c *linkerContext, chunk *chunkInfo) func([]as
 
 			compileResult.printedCSS = css_printer.Print(ast, css_printer.Options{
 				RemoveWhitespace: c.options.RemoveWhitespace,
+				ASCIIOnly:        c.options.ASCIIOnly,
 			})
 			compileResult.sourceIndex = sourceIndex
 			waitGroup.Done()
@@ -3754,7 +3756,7 @@ func (repr *chunkReprCSS) generate(c *linkerContext, chunk *chunkInfo) func([]as
 				}
 				importAbsPath := c.fs.Join(c.options.AbsOutputDir, chunk.relDir, record.Path.Text)
 				jMeta.AddString(fmt.Sprintf("\n        {\n          \"path\": %s\n        }",
-					js_printer.QuoteForJSON(c.res.PrettyPath(logger.Path{Text: importAbsPath, Namespace: "file"}))))
+					js_printer.QuoteForJSON(c.res.PrettyPath(logger.Path{Text: importAbsPath, Namespace: "file"}), c.options.ASCIIOnly)))
 			}
 			if !isFirstMeta {
 				jMeta.AddString("\n      ")
@@ -3784,7 +3786,7 @@ func (repr *chunkReprCSS) generate(c *linkerContext, chunk *chunkInfo) func([]as
 					jMeta.AddString(",")
 				}
 				jMeta.AddString(fmt.Sprintf("\n        %s: {\n          \"bytesInOutput\": %d\n        }",
-					js_printer.QuoteForJSON(c.files[compileResult.sourceIndex].source.PrettyPath),
+					js_printer.QuoteForJSON(c.files[compileResult.sourceIndex].source.PrettyPath, c.options.ASCIIOnly),
 					len(compileResult.printedCSS)))
 			}
 		}

internal/config/config.go

@@ -188,6 +188,7 @@ type Options struct {
 	OmitRuntimeForTests     bool
 	PreserveUnusedImportsTS bool
 	AvoidTDZ                bool
+	ASCIIOnly               bool
 
 	Strict   StrictOptions
 	Defines  *ProcessedDefines

internal/css_parser/css_parser_test.go

@@ -171,11 +171,20 @@ func TestEscapes(t *testing.T) {
 	expectPrinted(t, "*|div {}", "*|div {\n}\n")
 	expectPrinted(t, "\\2a {}", "\\* {\n}\n")
 	expectPrinted(t, "\\2a|div {}", "\\*|div {\n}\n")
+	expectPrinted(t, "\\2d {}", "- {\n}\n")
+	expectPrinted(t, "\\2d- {}", "-- {\n}\n")
+	expectPrinted(t, "-\\2d {}", "-- {\n}\n")
+	expectPrinted(t, "\\2d 123 {}", "\\-123 {\n}\n")
 
 	// SSHash
 	expectPrinted(t, "#h\\61sh {}", "#hash {\n}\n")
 	expectPrinted(t, "#\\2chash {}", "#\\,hash {\n}\n")
 	expectPrinted(t, "#\\,hash {}", "#\\,hash {\n}\n")
+	expectPrinted(t, "#\\2d {}", "#- {\n}\n")
+	expectPrinted(t, "#\\2d- {}", "#-- {\n}\n")
+	expectPrinted(t, "#-\\2d {}", "#-- {\n}\n")
+	expectPrinted(t, "#\\2d 123 {}", "#\\-123 {\n}\n")
+	expectPrinted(t, "#\\61hash {}", "#ahash {\n}\n")
 	expectPrinted(t, "#\\30hash {}", "#\\30hash {\n}\n")
 	expectPrinted(t, "#0\\2chash {}", "#0\\,hash {\n}\n")
 	expectPrinted(t, "#0\\,hash {}", "#0\\,hash {\n}\n")