Use remapping to remap minified sourcemap into source code (#37238)

* Use remapping to remap minified sourcemap into source code

* Cleanup massaging

* Eliminate sourcesContent at the esbuild level

* Fix undeclared variable

* Store JSON messages in TransformCache

* Extract babel sourcemaps during esbuild transformation

* Remap sourcemaps through terser -> esbuild -> babel

* Remove TODO

* Fix merge conflict

* Preserve original key name

* Make sourceRoot include the full destFile

Some build targets have directory parts in the `destFilename` as well as the `destDir`. We need the full dest directory path for remapping to work.

build-system/babel-config/minified-config.js

@@ -71,7 +71,7 @@ function getMinifiedConfig() {
   return {
     compact: false,
     plugins,
-    sourceMaps: 'inline',
+    sourceMaps: true,
     presets: [presetEnv],
     retainLines: true,
     assumptions: {

build-system/babel-config/unminified-config.js

@@ -51,7 +51,7 @@ function getUnminifiedConfig() {
     compact: false,
     plugins: unminifiedPlugins,
     presets: unminifiedPresets,
-    sourceMaps: 'inline',
+    sourceMaps: true,
     assumptions: {
       constantSuper: true,
       noClassCalls: true,

build-system/babel-plugins/babel-plugin-transform-rename-privates/index.js

@@ -28,15 +28,16 @@ module.exports = function (babel) {
         return;
       }
 
-      const {name} = key.node;
+      const {node} = key;
+      const {name} = node;
       if (name.endsWith('_AMP_PRIVATE_')) {
         return;
       }
 
-      if (!name.endsWith('_')) {
+      if (!name.endsWith('_') || name === '__proto__') {
         return;
       }
-      key.replaceWith(t.identifier(`${name}AMP_PRIVATE_`));
+      key.replaceWith(t.inherits(t.identifier(`${name}AMP_PRIVATE_`), node));
     };
   }
 

build-system/common/esbuild-babel.js

@@ -3,9 +3,18 @@ const path = require('path');
 const {debug} = require('../compile/debug-compilation-lifecycle');
 const {TransformCache, batchedRead, md5} = require('./transform-cache');
 
+/**
+ * @typedef {{
+ *   filename: string,
+ *   code: string,
+ *   map: *,
+ * }}
+ */
+let CacheMessageDef;
+
 /**
  * Used to cache babel transforms done by esbuild.
- * @const {TransformCache}
+ * @type {TransformCache<!CacheMessageDef>}
  */
 let transformCache;
 
@@ -14,27 +23,29 @@ let transformCache;
  * caching to speed up transforms.
  * @param {string} callerName
  * @param {boolean} enableCache
- * @param {function(): void} preSetup
- * @param {function(): void} postLoad
+ * @param {{
+ *   preSetup?: function():void,
+ *   postLoad?: function():void,
+ *   babelMaps?: Map<string, *>,
+ * }} callbacks
  * @return {!Object}
  */
 function getEsbuildBabelPlugin(
   callerName,
   enableCache,
-  preSetup = () => {},
-  postLoad = () => {}
+  {preSetup = () => {}, postLoad = () => {}, babelMaps} = {}
 ) {
   /**
    * @param {string} filename
    * @param {string} contents
    * @param {string} hash
    * @param {Object} babelOptions
-   * @return {Promise}
+   * @return {!Promise<!CacheMessageDef>}
    */
   async function transformContents(filename, contents, hash, babelOptions) {
     if (enableCache) {
       if (!transformCache) {
-        transformCache = new TransformCache('.babel-cache', '.js');
+        transformCache = new TransformCache('.babel-cache');
       }
       const cached = transformCache.get(hash);
       if (cached) {
@@ -46,9 +57,9 @@ function getEsbuildBabelPlugin(
     const promise = babel
       .transformAsync(contents, babelOptions)
       .then((result) => {
-        const {code, map} = result || {};
+        const {code, map} = /** @type {!babel.BabelFileResult} */ (result);
         debug('post-babel', filename, code, map);
-        return code + `\n// ${filename}`;
+        return {filename, code: code || '', map};
       });
 
     if (enableCache) {
@@ -86,7 +97,8 @@ function getEsbuildBabelPlugin(
           rehash,
           getFileBabelOptions(babelOptions, filename)
         );
-        return {contents: transformed};
+        babelMaps?.set(filename, transformed.map);
+        return {contents: transformed.code};
       });
     },
   };
@@ -119,10 +131,12 @@ function getFileBabelOptions(babelOptions, filename) {
 
   // The amp runner automatically sets cwd to the `amphtml` directory.
   const root = process.cwd();
+  const filenameRelative = path.relative(root, filename);
+
   return {
     ...babelOptions,
     filename,
-    filenameRelative: path.relative(root, filename),
+    filenameRelative,
   };
 }
 

build-system/common/transform-cache.js

@@ -2,23 +2,25 @@ const crypto = require('crypto');
 const fs = require('fs-extra');
 const path = require('path');
 
+/**
+ * Used to bust caches when the TransformCache makes a breaking change to the API.
+ */
+const API_VERSION = 2;
+
 /**
  * Cache for storing transformed files on both memory and on disk.
+ * @template T
  */
 class TransformCache {
   /**
    * @param {string} cacheName
-   * @param {string} fileExtension
    */
-  constructor(cacheName, fileExtension) {
-    /** @type {string} */
-    this.fileExtension = fileExtension;
-
+  constructor(cacheName) {
     /** @type {string} */
     this.cacheDir = path.resolve(__dirname, '..', '..', cacheName);
     fs.ensureDirSync(this.cacheDir);
 
-    /** @type {Map<string, Promise<Buffer|string>>} */
+    /** @type {Map<string, Promise<T>>} */
     this.transformMap = new Map();
 
     /** @type {Set<string>} */
@@ -27,35 +29,45 @@ class TransformCache {
 
   /**
    * @param {string} hash
-   * @return {null|Promise<Buffer|string>}
+   * @return {null|Promise<T>}
    */
   get(hash) {
     const cached = this.transformMap.get(hash);
     if (cached) {
       return cached;
     }
-    const filename = hash + this.fileExtension;
+
+    const filename = this.key_(hash);
     if (this.fsCache.has(filename)) {
-      const transformedPromise = fs.readFile(
-        path.join(this.cacheDir, filename)
-      );
-      this.transformMap.set(hash, transformedPromise);
-      return transformedPromise;
+      const persisted = fs.readJson(path.join(this.cacheDir, filename));
+      this.transformMap.set(hash, persisted);
+      return persisted;
     }
+
     return null;
   }
 
   /**
    * @param {string} hash
-   * @param {Promise<string>} transformPromise
+   * @param {Promise<T>} transformPromise
    */
   set(hash, transformPromise) {
     if (this.transformMap.has(hash)) {
       throw new Error(`Read race: Attempting to transform ${hash} file twice.`);
     }
     this.transformMap.set(hash, transformPromise);
-    const filepath = path.join(this.cacheDir, hash) + this.fileExtension;
-    transformPromise.then((contents) => fs.outputFile(filepath, contents));
+
+    const filepath = path.join(this.cacheDir, this.key_(hash));
+    transformPromise.then((contents) => fs.outputJson(filepath, contents));
+  }
+
+  /**
+   * @param {string} hash
+   * @return {string}
+   * @private
+   */
+  key_(hash) {
+    return `${API_VERSION}_${hash}.json`;
   }
 }
 

build-system/tasks/css/jsify-css.js

@@ -66,15 +66,15 @@ function getEnvironmentHash() {
 }
 
 /**
- * Used to cache css transforms done by postcss.
- * @const {TransformCache}
+ * @typedef {{css: string, warnings: string[]}}:
  */
-let transformCache;
+let CssTransformResultDef;
 
 /**
- * @typedef {{css: string, warnings: string[]}}:
+ * Used to cache css transforms done by postcss.
+ * @type {TransformCache<CssTransformResultDef>}
  */
-let CssTransformResultDef;
+let transformCache;
 
 /**
  * Transform a css string using postcss.
@@ -139,18 +139,15 @@ async function getCssImports(cssFile) {
  */
 async function transformCss(contents, hash, opt_filename) {
   if (!transformCache) {
-    transformCache = new TransformCache('.css-cache', '.css');
+    transformCache = new TransformCache('.css-cache');
   }
   const cached = transformCache.get(hash);
   if (cached) {
-    return JSON.parse((await cached).toString());
+    return cached;
   }
 
   const transformed = transform(contents, opt_filename);
-  transformCache.set(
-    hash,
-    transformed.then((r) => JSON.stringify(r))
-  );
+  transformCache.set(hash, transformed);
   return transformed;
 }
 

build-system/tasks/extension-helpers.js

@@ -488,19 +488,19 @@ async function buildNpmCss(extDir, options) {
   endBuildStep('Wrote CSS', `${options.name} → styles.css`, startCssTime);
 }
 
-/** @type {TransformCache} */
+/** @type {TransformCache<string>} */
 let jssCache;
 
 /**
  * Returns the minified CSS for a .jss.js file.
  *
  * @param {string} jssFile
- * @return {Promise<string|Buffer>}
+ * @return {Promise<string>}
  */
 async function getCssForJssFile(jssFile) {
   // Lazily instantiate the TransformCache
   if (!jssCache) {
-    jssCache = new TransformCache('.jss-cache', '.css');
+    jssCache = new TransformCache('.jss-cache');
   }
 
   const {contents, hash} = await batchedRead(jssFile);