chore: Move retry tests that are intended to retry to system tests to…

… reduce flake (#28881)

* Move currentRetries tests to a system test to reduce flake

* empty commit

* remove typo on test title

* Migrate origin retries test, which purposefully retries, into a system test

* Snapshot updates around the 'errors errors' typo

* Fix port issue

* Update origin test to reuse existing html files in directory

packages/driver/cypress/e2e/cypress/cypress.cy.js

@@ -86,96 +86,6 @@ describe('driver/src/cypress/index', () => {
     })
   })
 
-  context('.currentRetry', () => {
-    describe('test is not retried', () => {
-      before(() => {
-        expect(Cypress.currentRetry).to.eq(0)
-      })
-
-      beforeEach(() => {
-        expect(Cypress.currentRetry).to.eq(0)
-      })
-
-      afterEach(() => {
-        expect(Cypress.currentRetry).to.eq(0)
-      })
-
-      after(() => {
-        expect(Cypress.currentRetry).to.eq(0)
-      })
-
-      it('correctly returns currentRetry', () => {
-        expect(Cypress.currentRetry).to.eq(0)
-      })
-    })
-
-    describe('test is retried due to beforeEach hook failure', { retries: 1 }, () => {
-      before(() => {
-        expect(Cypress.currentRetry).to.be.oneOf([0, 1])
-      })
-
-      beforeEach(() => {
-        expect(Cypress.currentRetry).to.eq(1)
-      })
-
-      it('correctly returns currentRetry', () => {
-        expect(Cypress.currentRetry).to.eq(1)
-      })
-
-      afterEach(() => {
-        expect(Cypress.currentRetry).to.eq(1)
-      })
-
-      after(() => {
-        expect(Cypress.currentRetry).to.eq(1)
-      })
-    })
-
-    describe('test is retried due to test failure', { retries: 1 }, () => {
-      before(() => {
-        expect(Cypress.currentRetry).to.be.oneOf([0, 1])
-      })
-
-      beforeEach(() => {
-        expect(Cypress.currentRetry).to.be.oneOf([0, 1])
-      })
-
-      it('correctly returns currentRetry', () => {
-        expect(Cypress.currentRetry).to.eq(1)
-      })
-
-      afterEach(() => {
-        expect(Cypress.currentRetry).to.eq(1)
-      })
-
-      after(() => {
-        expect(Cypress.currentRetry).to.eq(1)
-      })
-    })
-
-    describe('test is retried due to afterEach hook failure', { retries: 1 }, () => {
-      before(() => {
-        expect(Cypress.currentRetry).to.be.oneOf([0, 1])
-      })
-
-      beforeEach(() => {
-        expect(Cypress.currentRetry).to.be.oneOf([0, 1])
-      })
-
-      it('correctly returns currentRetry', () => {
-        expect(Cypress.currentRetry).to.be.oneOf([0, 1])
-      })
-
-      afterEach(() => {
-        expect(Cypress.currentRetry).to.eq(1)
-      })
-
-      after(() => {
-        expect(Cypress.currentRetry).to.eq(1)
-      })
-    })
-  })
-
   context('.isCy', () => {
     it('returns true on cy, cy chainable', () => {
       expect(Cypress.isCy(cy)).to.be.true

packages/driver/cypress/e2e/e2e/origin/origin.cy.ts

@@ -154,69 +154,6 @@ describe('cy.origin', { browser: '!webkit' }, () => {
       cy.log('after cy.origin')
     })
 
-    ;[{
-      title: 'first attempt',
-      retries: 0,
-    }, {
-      title: 'on retry',
-      retries: 1,
-    }].forEach(({ title, retries }) => {
-      it(`passes runnable state to the secondary origin ${title}`, { retries }, () => {
-        const runnable = cy.state('runnable')
-        const expectedRunnable = {
-          clearTimeout: null,
-          isPending: null,
-          resetTimeout: null,
-          timeout: null,
-          id: runnable.id,
-          _currentRetry: retries,
-          _timeout: 4000,
-          type: 'test',
-          title: `passes runnable state to the secondary origin ${title}`,
-          titlePath: [
-            'cy.origin',
-            'withBeforeEach',
-            `passes runnable state to the secondary origin ${title}`,
-          ],
-          parent: {
-            id: runnable.parent.id,
-            type: 'suite',
-            title: 'withBeforeEach',
-            titlePath: [
-              'withBeforeEach',
-            ],
-            parent: {
-              id: runnable.parent.parent.id,
-              type: 'suite',
-              title: '',
-              titlePath: undefined,
-              ctx: {},
-            },
-            ctx: {},
-          },
-          ctx: {},
-        }
-
-        cy.origin('http://www.foobar.com:3500', { args: expectedRunnable }, (expectedRunnable) => {
-          const actualRunnable = cy.state('runnable')
-
-          expect(actualRunnable.titlePath()).to.deep.equal(expectedRunnable.titlePath)
-          expectedRunnable.titlePath = actualRunnable.titlePath
-
-          expect(actualRunnable.title).to.equal(expectedRunnable.title)
-          expect(actualRunnable.id).to.equal(expectedRunnable.id)
-          expect(actualRunnable.ctx).to.deep.equal(expectedRunnable.ctx)
-          expect(actualRunnable._currentRetry).to.equal(expectedRunnable._currentRetry)
-          expect(actualRunnable._timeout).to.equal(expectedRunnable._timeout)
-          expect(actualRunnable.type).to.equal(expectedRunnable.type)
-          expect(actualRunnable.callback).to.exist
-          expect(actualRunnable.timeout).to.exist
-          expect(actualRunnable.parent.title).to.equal(expectedRunnable.parent.title)
-          expect(actualRunnable.parent.type).to.equal(expectedRunnable.parent.type)
-        })
-      })
-    })
-
     it('handles querying nested elements', () => {
       cy.origin('http://www.foobar.com:3500', () => {
         cy

system-tests/__snapshots__/cy_origin_retries_spec.ts.js

@@ -18,16 +18,16 @@ exports['e2e cy.origin retries / Appropriately displays test retry errors withou
 
 
   cy.origin test retries
-    (Attempt 1 of 3) appropriately retries test within "cy.origin" without propagating other errors errors
-    (Attempt 2 of 3) appropriately retries test within "cy.origin" without propagating other errors errors
-    1) appropriately retries test within "cy.origin" without propagating other errors errors
+    (Attempt 1 of 3) appropriately retries test within "cy.origin" without propagating other errors
+    (Attempt 2 of 3) appropriately retries test within "cy.origin" without propagating other errors
+    1) appropriately retries test within "cy.origin" without propagating other errors
 
 
   0 passing
   1 failing
 
   1) cy.origin test retries
-       appropriately retries test within "cy.origin" without propagating other errors errors:
+       appropriately retries test within "cy.origin" without propagating other errors:
      AssertionError: expected true to be false
       [stack trace lines]
 
@@ -53,13 +53,13 @@ exports['e2e cy.origin retries / Appropriately displays test retry errors withou
 
   -  /XXX/XXX/XXX/cypress/screenshots/cy_origin_retries.cy.ts/cy.origin test retries      (1280x720)
      -- appropriately retries test within cy.origin without propagating other errors                
-     errors (failed).png                                                                            
+     (failed).png                                                                                   
   -  /XXX/XXX/XXX/cypress/screenshots/cy_origin_retries.cy.ts/cy.origin test retries      (1280x720)
      -- appropriately retries test within cy.origin without propagating other errors                
-     errors (failed) (attempt 2).png                                                                
+     (failed) (attempt 2).png                                                                       
   -  /XXX/XXX/XXX/cypress/screenshots/cy_origin_retries.cy.ts/cy.origin test retries      (1280x720)
      -- appropriately retries test within cy.origin without propagating other errors                
-     errors (failed) (attempt 3).png                                                                
+     (failed) (attempt 3).png                                                                       
 
 
 ====================================================================================================

system-tests/projects/e2e/cypress/e2e/current_retries.cy.js

@@ -0,0 +1,89 @@
+context('.currentRetry', () => {
+  describe('test is not retried', () => {
+    before(() => {
+      expect(Cypress.currentRetry).to.eq(0)
+    })
+
+    beforeEach(() => {
+      expect(Cypress.currentRetry).to.eq(0)
+    })
+
+    afterEach(() => {
+      expect(Cypress.currentRetry).to.eq(0)
+    })
+
+    after(() => {
+      expect(Cypress.currentRetry).to.eq(0)
+    })
+
+    it('correctly returns currentRetry', () => {
+      expect(Cypress.currentRetry).to.eq(0)
+    })
+  })
+
+  describe('test is retried due to beforeEach hook failure', { retries: 1 }, () => {
+    before(() => {
+      expect(Cypress.currentRetry).to.be.oneOf([0, 1])
+    })
+
+    beforeEach(() => {
+      expect(Cypress.currentRetry).to.eq(1)
+    })
+
+    it('correctly returns currentRetry', () => {
+      expect(Cypress.currentRetry).to.eq(1)
+    })
+
+    afterEach(() => {
+      expect(Cypress.currentRetry).to.eq(1)
+    })
+
+    after(() => {
+      expect(Cypress.currentRetry).to.eq(1)
+    })
+  })
+
+  describe('test is retried due to test failure', { retries: 1 }, () => {
+    before(() => {
+      expect(Cypress.currentRetry).to.be.oneOf([0, 1])
+    })
+
+    beforeEach(() => {
+      expect(Cypress.currentRetry).to.be.oneOf([0, 1])
+    })
+
+    it('correctly returns currentRetry', () => {
+      expect(Cypress.currentRetry).to.eq(1)
+    })
+
+    afterEach(() => {
+      expect(Cypress.currentRetry).to.eq(1)
+    })
+
+    after(() => {
+      expect(Cypress.currentRetry).to.eq(1)
+    })
+  })
+
+  describe('test is retried due to afterEach hook failure', { retries: 1 }, () => {
+    before(() => {
+      expect(Cypress.currentRetry).to.be.oneOf([0, 1])
+    })
+
+    beforeEach(() => {
+      expect(Cypress.currentRetry).to.be.oneOf([0, 1])
+    })
+
+    it('correctly returns currentRetry', () => {
+      expect(Cypress.currentRetry).to.be.oneOf([0, 1])
+    })
+
+    afterEach(() => {
+      expect(Cypress.currentRetry).to.eq(1)
+    })
+
+    after(() => {
+      expect(Cypress.currentRetry).to.eq(1)
+    })
+  })
+})

system-tests/projects/e2e/cypress/e2e/cy_origin_retries.cy.ts

@@ -4,7 +4,7 @@ describe('cy.origin test retries', () => {
     cy.get('a[data-cy="cross_origin_secondary_link"]').click()
   })
 
-  it('appropriately retries test within "cy.origin" without propagating other errors errors', function () {
+  it('appropriately retries test within "cy.origin" without propagating other errors', function () {
     cy.origin('http://foobar.com:4466', () => {
       cy.then(() => {
         expect(true).to.be.false