Rewrite JS/HTML using AST-based approach (#5273)

* Add winPropAccessor to security.js, remove other replacers

* Add start of Cypress.resolveWindowReference

* Add regexes for dot and bracket access

* Some security_spec tests pass with new injection

* Add resolveWindowReference unit tests

* Old security_spec now passes with resolveWindowReference

* Inject stub resolveWindowReference so proxy still works outside of Cypress

* wip: rewrite HTML + JS with tokenizer

* Move to using esprima + hyntax to rewrite JS + HTML

* remove comment; oneLine makes the whole thing commented

* Fix tests, apple.com edge case

* wip: add getOrSet

* Revert "wip: add getOrSet"

This reverts commit a5c647c.

* release 3.5.0 [skip ci]

* use recast to replace window property accesses

* replace assignments to top properly

* fix yarn.lock

* bump deps

* update integration tests

* remove old security ts?

* fix integration spec

* always ignore js interception failure

* use globalThis instead of window

* add experimentalSourceRewriting flag

* restore regex-writer spec

* fix types

* update config_spec

* add source rewriting spec

* cleanup

* simplify rewriting logic, move rules into rewriter package

* create threaded rewriting tool for non-streaming use

* update @packages/rewriter to use threads for async

* use async rewriting where convenient

* add worker-shim.js

* add performance info to debug logs

* properly handle +=, -=, ...

* add proxy, rewriter to unit-tests stage

* cleanup

* use parse5 to rewrite HTML, strip SRI

* update tests

* reorganization, cleanup

* rewrite ALL parent, top identifiers except in a few cases

* handle many JS edge cases

* ensure parse5@5.1.1 is installed

* update yarn.lock

* update tests

* add debugging, add tests

* add attempted repro for .href issue

* implement source maps + extending inline source maps

* update opts passing in proxy layer

* fix sourcemap naming structure

* update tests to account for sourcemaps

* sourcemap tests

* remote source maps work

* comment

* update rewriter tests

* clean up TODOs in resolveWindowReference

* remove @types/nock

* clean up todos in deferred-source-map-cache

* fix rewriter build script

* fix concatStream import

* bump expectedresultcount

* clean up js-rules

* threading improvements, workaround for Electron segfault

* no visit_spec for now

* fix 6_visit_spec

* update MAX_WORKER_THREADS

* add repro for #3975

* cleanup

* cleanup

* make better use of namedTypes and builders

* get rid of the horrific closureDetectionTernary

ast-types keeps track of scope, so it is unneeded

* fix #3975, #3994

* add x-sourcemap, sourcemap header support

* snap-shot-it 7.9.3

* add deferred-source-map-cache-spec

* add tests

* Throw error in driver if AST rewriting fails

* Fix "location = 'relative-url'"

* fix max recursion depth

* slim down some fixtures

* fix window.location usage

* don't mess with `frames` at all

* no integration tests

* skip testing apple.com for now

* update wording: regex-based vs. ast-based

* skip real-world tests for now

* add some padding to process.exit workaround

* fix resolvers_spec

* fix html-spec

* cleanup

* Update packages/rewriter/lib/js-rules.ts

* Update packages/driver/src/cypress/resolvers.ts

* just import find by itself

* privatize typedefs for Cypress.state, remove .gitignore, remove dead code

Co-authored-by: Ben Kucera <14625260+Bkucera@users.noreply.github.com>

circle.yml

@@ -330,7 +330,7 @@ jobs:
       # run unit tests from each individual package
       - run: yarn test
       - verify-mocha-results:
-          expectedResultCount: 6
+          expectedResultCount: 8
       - store_test_results:
           path: /tmp/cypress
       # CLI tests generate HTML files with sample CLI command output

cli/schema/cypress.schema.json

@@ -228,6 +228,11 @@
       "type": "boolean",
       "default": false,
       "description": "If `true`, Cypress will add `sameSite` values to the objects yielded from `cy.setCookie()`, `cy.getCookie()`, and `cy.getCookies()`. This will become the default behavior in Cypress 5.0."
+    },
+    "experimentalSourceRewriting": {
+      "type": "boolean",
+      "default": false,
+      "description": "Enables AST-based JS/HTML rewriting. This may fix issues caused by the existing regex-based JS/HTML replacement algorithm."
     }
   }
 }

cli/types/index.d.ts

@@ -478,11 +478,11 @@ declare namespace Cypress {
       /**
        * Returns a boolean indicating whether an object is a window object.
        */
-      isWindow(obj: any): boolean
+      isWindow(obj: any): obj is Window
       /**
        * Returns a boolean indicating whether an object is a jQuery object.
        */
-      isJquery(obj: any): boolean
+      isJquery(obj: any): obj is JQuery
       isInputType(element: JQuery | HTMLElement, type: string | string[]): boolean
       stringify(element: JQuery | HTMLElement, form: string): string
       getElements(element: JQuery): JQuery | HTMLElement[]
@@ -2456,6 +2456,12 @@ declare namespace Cypress {
      * @default false
      */
     experimentalGetCookiesSameSite: boolean
+    /**
+     * Enables AST-based JS/HTML rewriting. This may fix issues caused by the existing regex-based JS/HTML replacement
+     * algorithm.
+     * @default false
+     */
+    experimentalSourceRewriting: boolean
   }
 
   /**

package.json

@@ -51,7 +51,7 @@
     "stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,src --exclude e2e.coffee,e2e.js",
     "stop-only-all": "yarn stop-only --folder packages",
     "pretest": "yarn ensure-deps",
-    "test": "yarn lerna exec yarn test --scope cypress --scope \"'@packages/{electron,extension,https-proxy,launcher,network,reporter,runner,socket}'\"",
+    "test": "yarn lerna exec yarn test --scope cypress --scope \"'@packages/{electron,extension,https-proxy,launcher,network,proxy,rewriter,reporter,runner,socket}'\"",
     "test-debug": "lerna exec yarn test-debug --ignore \"'@packages/{coffee,desktop-gui,driver,root,static,web-config}'\"",
     "pretest-e2e": "yarn ensure-deps",
     "test-e2e": "lerna exec yarn test-e2e --ignore \"'@packages/{coffee,desktop-gui,driver,root,static,web-config}'\"",

packages/driver/src/cypress.js

@@ -32,6 +32,7 @@ const $utils = require('./cypress/utils')
 const $errUtils = require('./cypress/error_utils')
 const $scriptUtils = require('./cypress/script_utils')
 const browserInfo = require('./cypress/browser')
+const resolvers = require('./cypress/resolvers')
 const debug = require('debug')('cypress:driver:cypress')
 
 const proxies = {
@@ -607,6 +608,8 @@ $Cypress.prototype.Location = $Location
 $Cypress.prototype.Log = $Log
 $Cypress.prototype.LocalStorage = $LocalStorage
 $Cypress.prototype.Mocha = $Mocha
+$Cypress.prototype.resolveWindowReference = resolvers.resolveWindowReference
+$Cypress.prototype.resolveLocationReference = resolvers.resolveLocationReference
 $Cypress.prototype.Mouse = $Mouse
 $Cypress.prototype.Runner = $Runner
 $Cypress.prototype.Server = $Server

packages/driver/src/cypress/error_messages.coffee

@@ -841,6 +841,18 @@ module.exports = {
   ng:
     no_global: "Angular global (`window.angular`) was not found in your window. You cannot use #{cmd('ng')} methods without angular."
 
+  proxy:
+    js_rewriting_failed: """
+    An error occurred in the Cypress proxy layer while rewriting your source code. This is a bug in Cypress.
+
+    JS URL: {{url}}
+
+    Original error:
+
+    {{errMessage}}
+    {{errStack}}
+    """
+
   reload:
     invalid_arguments: {
       message: "#{cmd('reload')} can only accept a boolean or `options` as its arguments."

packages/driver/src/cypress/resolvers.ts

@@ -0,0 +1,159 @@
+import _ from 'lodash'
+import $Cypress from '../..'
+
+/**
+ * Fix property reads and writes that could potentially help the AUT to break out of its iframe.
+ *
+ * @param currentWindow the value of `globalThis` from the scope of the window reference in question
+ * @param accessedObject a reference to the object being accessed
+ * @param accessedProp the property name being accessed (Symbol/number properties are not intercepted)
+ * @param value the right-hand side of an assignment operation (accessedObject.accessedProp = value)
+ */
+export function resolveWindowReference (this: typeof $Cypress, currentWindow: Window, accessedObject: Window | any, accessedProp: string, value?: any) {
+  const { dom, state } = this
+
+  const getTargetValue = () => {
+    const targetValue = accessedObject[accessedProp]
+
+    if (dom.isWindow(accessedObject) && accessedProp === 'location') {
+      const targetLocation = resolveLocationReference(accessedObject)
+
+      if (isValPassed) {
+        return targetLocation.href = value
+      }
+
+      return targetLocation
+    }
+
+    if (_.isFunction(targetValue)) {
+      return targetValue.bind(accessedObject)
+    }
+
+    return targetValue
+  }
+
+  const setTargetValue = () => {
+    if (dom.isWindow(accessedObject) && accessedProp === 'location') {
+      const targetLocation = resolveLocationReference(accessedObject)
+
+      return targetLocation.href = value
+    }
+
+    return (accessedObject[accessedProp] = value)
+  }
+
+  const isValPassed = arguments.length === 4
+
+  const $autIframe = state('$autIframe')
+
+  if (!$autIframe) {
+    // missing AUT iframe, resolve the property access normally
+    if (isValPassed) {
+      return setTargetValue()
+    }
+
+    return getTargetValue()
+  }
+
+  const contentWindow = $autIframe.prop('contentWindow')
+
+  if (accessedObject === currentWindow.top) {
+    // doing a property access on topmost window, adjust accessedObject
+    accessedObject = contentWindow
+  }
+
+  const targetValue = getTargetValue()
+
+  if (!dom.isWindow(targetValue) || dom.isJquery(targetValue)) {
+    if (isValPassed) {
+      return setTargetValue()
+    }
+
+    return targetValue
+  }
+
+  // targetValue is a reference to a Window object
+
+  if (accessedProp === 'top') {
+    // note: `isValPassed` is not considered here because `window.top` is readonly
+    return contentWindow
+  }
+
+  if (accessedProp === 'parent') {
+    // note: `isValPassed` is not considered here because `window.parent` is readonly
+    if (targetValue === currentWindow.top) {
+      return contentWindow
+    }
+
+    return targetValue
+  }
+
+  throw new Error('unhandled resolveWindowReference')
+}
+
+/**
+ * Fix `window.location` usages that would otherwise navigate to the wrong URL.
+ *
+ * @param currentWindow the value of `globalThis` from the scope of the location reference in question
+ */
+export function resolveLocationReference (currentWindow: Window) {
+  // @ts-ignore
+  if (currentWindow.__cypressFakeLocation) {
+    // @ts-ignore
+    return currentWindow.__cypressFakeLocation
+  }
+
+  function _resolveHref (href: string) {
+    const a = currentWindow.document.createElement('a')
+
+    a.href = href
+
+    // a.href will be resolved into the correct fully-qualified URL
+    return a.href
+  }
+
+  function assign (href: string) {
+    return currentWindow.location.assign(_resolveHref(href))
+  }
+
+  function replace (href: string) {
+    return currentWindow.location.replace(_resolveHref(href))
+  }
+
+  function setHref (href: string) {
+    return currentWindow.location.href = _resolveHref(href)
+  }
+
+  const locationKeys = Object.keys(currentWindow.location)
+
+  const fakeLocation = {}
+
+  _.reduce(locationKeys, (acc, cur) => {
+    // set a dummy value, the proxy will handle sets/gets
+    acc[cur] = Symbol.for('Proxied')
+
+    return acc
+  }, {})
+
+  // @ts-ignore
+  return currentWindow.__cypressFakeLocation = new Proxy(fakeLocation, {
+    get (_target, prop, _receiver) {
+      if (prop === 'assign') {
+        return assign
+      }
+
+      if (prop === 'replace') {
+        return replace
+      }
+
+      return currentWindow.location[prop]
+    },
+    set (_obj, prop, value) {
+      if (prop === 'href') {
+        return setHref(value)
+      }
+
+      return currentWindow.location[prop] = value
+    },
+  })
+}