test: change some skipped tests to use retries (#24124)

Co-authored-by: Matt Henkes <mjhenkes@gmail.com>
cypress-io · Dec 20, 2022 · ebf57a8 · ebf57a8 · cypress-bot · Dec 20, 2022
1 parent 0454caf
commit ebf57a8
Show file tree

Hide file tree

Showing 37 changed files with 105 additions and 184 deletions.
diff --git a/.circleci/workflows.yml b/.circleci/workflows.yml
@@ -28,7 +28,7 @@ mainBuildFilters: &mainBuildFilters
       only:
         - develop
         - /^release\/\d+\.\d+\.\d+$/
-        - 'feature/create-from-react-component'
+        - 'retry-flake'
 
 # usually we don't build Mac app - it takes a long time
 # but sometimes we want to really confirm we are doing the right thing
@@ -37,15 +37,15 @@ macWorkflowFilters: &darwin-workflow-filters
   when:
     or:
     - equal: [ develop, << pipeline.git.branch >> ]
-    - equal: [ 'feature/create-from-react-component', << pipeline.git.branch >> ]
+    - equal: [ 'retry-flake', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
 linuxArm64WorkflowFilters: &linux-arm64-workflow-filters
   when:
     or:
     - equal: [ develop, << pipeline.git.branch >> ]
-    - equal: [ 'feature/create-from-react-component', << pipeline.git.branch >> ]
+    - equal: [ 'retry-flake', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -63,7 +63,7 @@ windowsWorkflowFilters: &windows-workflow-filters
   when:
     or:
     - equal: [ develop, << pipeline.git.branch >> ]
-    - equal: [ 'feature/create-from-react-component', << pipeline.git.branch >> ]
+    - equal: [ 'retry-flake', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -129,7 +129,7 @@ commands:
       - run:
           name: Check current branch to persist artifacts
           command: |
-            if [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "feature/create-from-react-component" ]]; then
+            if [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "retry-flake" ]]; then
               echo "Not uploading artifacts or posting install comment for this branch."
               circleci-agent step halt
             fi

diff --git a/npm/webpack-dev-server/cypress/e2e/angular.cy.ts b/npm/webpack-dev-server/cypress/e2e/angular.cy.ts
@@ -77,7 +77,7 @@ for (const project of WEBPACK_REACT) {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23455
-    it.skip('should detect new spec', () => {
+    it('should detect new spec', { retries: 15 }, () => {
       cy.visitApp()
 
       cy.withCtx(async (ctx) => {

diff --git a/npm/webpack-dev-server/cypress/e2e/next.cy.ts b/npm/webpack-dev-server/cypress/e2e/next.cy.ts
@@ -76,7 +76,7 @@ for (const project of WEBPACK_REACT) {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23417
-    it.skip('should detect new spec', () => {
+    it('should detect new spec', { retries: 15 }, () => {
       cy.visitApp()
 
       cy.withCtx(async (ctx) => {
@@ -94,7 +94,7 @@ for (const project of WEBPACK_REACT) {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23417
-    it.skip('should allow import of global styles in support file', () => {
+    it('should allow import of global styles in support file', { retries: 15 }, () => {
       cy.visitApp()
       cy.contains('styles.cy.js').click()
       cy.waitForSpecToFinish({ passCount: 1 })

diff --git a/npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts b/npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts
@@ -71,7 +71,7 @@ for (const project of PROJECTS) {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23455
-    it.skip('should detect new spec', () => {
+    it('should detect new spec', { retries: 15 }, () => {
       cy.visitApp()
 
       cy.withCtx(async (ctx) => {

diff --git a/packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts b/packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts
@@ -82,7 +82,7 @@ describe('Cypress In Cypress CT', { viewportWidth: 1500, defaultCommandTimeout:
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23159
-    it.skip('redirects to the specs list with error if a spec is not found', () => {
+    it('redirects to the specs list with error if a spec is not found', { retries: 15 }, () => {
       cy.visitApp()
       const { title, intro, explainer } = defaultMessages.specPage.noSpecError
       const badFilePath = 'src/DoesNotExist.spec.js'

diff --git a/packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts b/packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts
@@ -34,7 +34,7 @@ describe('Cypress In Cypress - run mode', { viewportWidth: 1200 }, () => {
   })
 
   // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23306
-  it.skip('component testing run mode spec runner header is correct', () => {
+  it('component testing run mode spec runner header is correct', { retries: 15 }, () => {
     cy.scaffoldProject('cypress-in-cypress')
     cy.findBrowsers()
     cy.openProject('cypress-in-cypress')

diff --git a/packages/app/cypress/e2e/cypress-in-cypress.cy.ts b/packages/app/cypress/e2e/cypress-in-cypress.cy.ts
@@ -93,7 +93,7 @@ describe('Cypress in Cypress', { viewportWidth: 1500, defaultCommandTimeout: 100
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23307
-    it.skip(`scales the AUT correctly in ${testingType}`, () => {
+    it(`scales the AUT correctly in ${testingType}`, { retries: 15 }, () => {
       const assertNoScaleShown = () => {
         // check that no message about scale % is shown,
         // meaning the AUT is at 100% scale

diff --git a/packages/app/cypress/e2e/specs_list_latest_runs.cy.ts b/packages/app/cypress/e2e/specs_list_latest_runs.cy.ts
@@ -671,7 +671,7 @@ describe('App/Cloud Integration - Latest runs and Average duration', { viewportW
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23419
-    it.skip('shows offline alert then hides it after coming online', () => {
+    it('shows offline alert then hides it after coming online', { retries: 15 }, () => {
       cy.findByTestId('offline-alert')
       .should('contain.text', defaultMessages.specPage.offlineWarning.title)
       .and('contain.text', defaultMessages.specPage.offlineWarning.explainer)

diff --git a/packages/data-context/test/unit/sources/GitDataSource.spec.ts b/packages/data-context/test/unit/sources/GitDataSource.spec.ts
@@ -43,8 +43,10 @@ describe('GitDataSource', () => {
     sinon.restore()
   })
 
-  // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317
-  it.skip(`gets correct status for files on ${os.platform()}`, async () => {
+  it(`gets correct status for files on ${os.platform()}`, async function () {
+    // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317
+    this.retries(15)
+
     const onBranchChange = sinon.stub()
     const onGitInfoChange = sinon.stub()
     const onError = sinon.stub()

diff --git a/packages/driver/cypress/e2e/commands/actions/type_special_chars.cy.js b/packages/driver/cypress/e2e/commands/actions/type_special_chars.cy.js
@@ -85,7 +85,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
 
   context('{{}', () => {
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('sets which and keyCode to 219', (done) => {
+    it('sets which and keyCode to 219', { retries: 15 }, (done) => {
       cy.$$(':text:first').on('keydown', (e) => {
         expect(e.which).to.eq(219)
         expect(e.keyCode).to.eq(219)
@@ -238,7 +238,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('sets which and keyCode to 8 and does not fire keypress events', (done) => {
+    it('sets which and keyCode to 8 and does not fire keypress events', { retries: 15 }, (done) => {
       cy.$$(':text:first').on('keypress', () => {
         done('should not have received keypress')
       })
@@ -255,7 +255,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('does not fire textInput event', (done) => {
+    it('does not fire textInput event', { retries: 15 }, (done) => {
       cy.$$(':text:first').on('textInput', (e) => {
         done(new Error('textInput should not have fired'))
       })
@@ -266,7 +266,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('can prevent default backspace movement', (done) => {
+    it('can prevent default backspace movement', { retries: 15 }, (done) => {
       cy.$$(':text:first').on('keydown', (e) => {
         if (e.keyCode === 8) {
           e.preventDefault()
@@ -581,7 +581,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('can prevent default del movement', (done) => {
+    it('can prevent default del movement', { retries: 15 }, (done) => {
       cy.$$(':text:first').on('keydown', (e) => {
         if (e.keyCode === 46) {
           e.preventDefault()
@@ -1240,7 +1240,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
 
     // https://github.com/cypress-io/cypress/issues/3405
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('does fire input event when text inserted', (done) => {
+    it('does fire input event when text inserted', { retries: 15 }, (done) => {
       cy.$$('[contenteditable]:first').on('input', (e) => {
         done()
       })
@@ -1281,7 +1281,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
 
     context('1 input, no \'submit\' elements', () => {
       // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-      it.skip('triggers form submit', function (done) {
+      it('triggers form submit', { retries: 15 }, function (done) {
         this.foo = {}
 
         this.$forms.find('#single-input').submit((e) => {
@@ -1481,7 +1481,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
 
     context('2 inputs, 1 \'submit\' element input[type=submit]', () => {
       // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-      it.skip('triggers form submit', function (done) {
+      it('triggers form submit', { retries: 15 }, function (done) {
         this.$forms.find('#multiple-inputs-and-input-submit').submit((e) => {
           e.preventDefault()
 

diff --git a/packages/driver/cypress/e2e/commands/navigation.cy.js b/packages/driver/cypress/e2e/commands/navigation.cy.js
@@ -273,7 +273,6 @@ describe('src/cy/commands/navigation', () => {
 
   // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23308
   context.skip('#go', () => {
-    // TODO: fix this
     it('sets timeout to Cypress.config(pageLoadTimeout)', {
       pageLoadTimeout: 4567,
     }, () => {
@@ -816,7 +815,7 @@ describe('src/cy/commands/navigation', () => {
 
     // https://github.com/cypress-io/cypress/issues/1311
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23201
-    it.skip('window immediately resolves and doesn\'t reload when visiting the same URL with hashes', () => {
+    it('window immediately resolves and doesn\'t reload when visiting the same URL with hashes', { retries: 15 }, () => {
       const onLoad = cy.stub()
 
       cy
@@ -1803,7 +1802,7 @@ describe('src/cy/commands/navigation', () => {
     })
   })
 
-  // TODO(webkit): fix+unskip for experimental webkit release
+  // TODO(webkit): fix+unskip for webkit release
   context('#page load', { browser: '!webkit' }, () => {
     it('sets initial=true and then removes', () => {
       Cookie.remove('__cypress.initial')
@@ -2200,7 +2199,7 @@ describe('src/cy/commands/navigation', () => {
     })
   })
 
-  // TODO(webkit): fix+unskip for experimental webkit release
+  // TODO(webkit): fix+unskip for webkit release
   context('#url:changed', { browser: '!webkit' }, () => {
     beforeEach(function () {
       this.logs = []

diff --git a/packages/driver/cypress/e2e/commands/net_stubbing.cy.ts b/packages/driver/cypress/e2e/commands/net_stubbing.cy.ts
@@ -1639,7 +1639,7 @@ describe('network stubbing', { retries: 15 }, function () {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23422
-    it.skip('can add a body to a request that does not have one', function () {
+    it('can add a body to a request that does not have one', { retries: 15 }, function () {
       const body = '{"foo":"bar"}'
 
       cy.intercept('/post-only', function (req) {
@@ -1670,7 +1670,7 @@ describe('network stubbing', { retries: 15 }, function () {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23303
-    it.skip('can delay and throttle a StaticResponse', function (done) {
+    it('can delay and throttle a StaticResponse', { retries: 15 }, function (done) {
       const payload = 'A'.repeat(10 * 1024)
       const throttleKbps = 10
       const delay = 250
@@ -1696,7 +1696,7 @@ describe('network stubbing', { retries: 15 }, function () {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23404
-    it.skip('can delay with deprecated delayMs param', function () {
+    it('can delay with deprecated delayMs param', { retries: 15 }, function () {
       const delayMs = 250
 
       cy.intercept('/timeout*', (req) => {
@@ -1714,7 +1714,7 @@ describe('network stubbing', { retries: 15 }, function () {
 
     // @see https://github.com/cypress-io/cypress/issues/14446
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23406
-    it.skip('should delay the same amount on every response', () => {
+    it('should delay the same amount on every response', { retries: 15 }, () => {
       const delay = 250
 
       const testDelay = () => {
@@ -1829,7 +1829,7 @@ describe('network stubbing', { retries: 15 }, function () {
         })
 
         // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23407
-        it.skip('by doing both', () => {
+        it('by doing both', { retries: 15 }, () => {
           cy.intercept({ url: '/users*' }, (req) => {
             req.query = {
               a: 'b',
@@ -1852,7 +1852,7 @@ describe('network stubbing', { retries: 15 }, function () {
         })
 
         // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23414
-        it.skip('by deleting query member', () => {
+        it('by deleting query member', { retries: 15 }, () => {
           cy.intercept({ url: '/users*' }, (req) => {
             req.query = {
               a: 'b',
@@ -1876,7 +1876,7 @@ describe('network stubbing', { retries: 15 }, function () {
 
         context('by setting new url', () => {
           // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23415
-          it.skip('absolute path', () => {
+          it('absolute path', { retries: 15 }, () => {
             cy.intercept({ url: '/users*' }, (req) => {
               req.url = 'http://localhost:3500/users?a=b'
 
@@ -1894,7 +1894,7 @@ describe('network stubbing', { retries: 15 }, function () {
           })
 
           // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23433
-          it.skip('relative path', () => {
+          it('relative path', { retries: 15 }, () => {
             cy.intercept({ url: '/users*' }, (req) => {
               req.url = '/users?a=b'
 
@@ -1913,7 +1913,7 @@ describe('network stubbing', { retries: 15 }, function () {
           })
 
           // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23434
-          it.skip('empty string', () => {
+          it('empty string', { retries: 15 }, () => {
             cy.intercept({ url: '/users*' }, (req) => {
               req.url = ''
 
@@ -1933,7 +1933,7 @@ describe('network stubbing', { retries: 15 }, function () {
 
         context('throwing errors correctly', () => {
           // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23423
-          it.skip('defineproperty', (done) => {
+          it('defineproperty', { retries: 15 }, (done) => {
             cy.on('fail', (err) => {
               expect(err.message).to.eq('`defineProperty()` is not allowed.')
 
@@ -1962,7 +1962,7 @@ describe('network stubbing', { retries: 15 }, function () {
           })
 
           // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23147
-          it.skip('setPrototypeOf', (done) => {
+          it('setPrototypeOf', { retries: 15 }, (done) => {
             cy.on('fail', (err) => {
               expect(err.message).to.eq('`setPrototypeOf()` is not allowed.')
 
@@ -1992,7 +1992,7 @@ describe('network stubbing', { retries: 15 }, function () {
       context('can end response', () => {
         for (const eventName of ['before:response', 'response']) {
           // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23434
-          it.skip(`in \`${eventName}\``, () => {
+          it(`in \`${eventName}\``, { retries: 15 }, () => {
             const url = uniqueRoute('/foo')
             const expectBeforeResponse = eventName === 'response'
             let beforeResponseCalled = false
@@ -2203,7 +2203,7 @@ describe('network stubbing', { retries: 15 }, function () {
 
       context('with `times`', function () {
         // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23434
-        it.skip('only uses each handler N times', function () {
+        it('only uses each handler N times', { retries: 15 }, function () {
           const url = uniqueRoute('/foo')
           const third = sinon.stub()