feat: Debug page [IATR]

cli/CHANGELOG.md

@@ -3,6 +3,10 @@
 
 _Released 01/31/2023 (PENDING)_
 
+**Features:**
+
+- Easily debug failed CI test runs right from your local Cypress app with the new Debug page, powered by Cypress Cloud.  For more details, see the [Debug documentation](https://on.cypress.io/debug-page). Addressed in [#25488](https://github.com/cypress-io/cypress/pull/25488).
+
 **Bugfixes:**
 
 - Fixed an issue where alternative Microsoft Edge Beta, Canary, and Dev binary versions were not being discovered by Cypress. 

packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts

@@ -1,14 +1,13 @@
+import { CY_IN_CY_SIMULATE_RUN_MODE } from '@packages/types/src/constants'
+
 describe('Cypress In Cypress - run mode', { viewportWidth: 1200 }, () => {
   it('e2e run mode spec runner header is correct', () => {
     cy.scaffoldProject('cypress-in-cypress')
     cy.findBrowsers()
     cy.openProject('cypress-in-cypress')
     cy.startAppServer()
-    cy.visitApp()
-
-    simulateRunModeInUI()
+    cy.visitApp(`/specs/runner?file=cypress/e2e/dom-content.spec.js&${CY_IN_CY_SIMULATE_RUN_MODE}`)
 
-    cy.contains('dom-content.spec').click()
     cy.waitForSpecToFinish()
 
     cy.findByTestId('aut-url').should('be.visible')
@@ -33,17 +32,14 @@ describe('Cypress In Cypress - run mode', { viewportWidth: 1200 }, () => {
     // cy.percySnapshot() // TODO: restore when Percy CSS is fixed. See https://github.com/cypress-io/cypress/issues/23435
   })
 
-  // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23306
-  it('component testing run mode spec runner header is correct', { retries: 15 }, () => {
+  it('component testing run mode spec runner header is correct', () => {
     cy.scaffoldProject('cypress-in-cypress')
     cy.findBrowsers()
     cy.openProject('cypress-in-cypress')
     cy.startAppServer('component')
-    cy.visitApp()
-    simulateRunModeInUI()
-    cy.contains('TestComponent.spec').click()
-    cy.waitForSpecToFinish()
+    cy.visitApp(`/specs/runner?file=src/TestComponent.spec.jsx&${CY_IN_CY_SIMULATE_RUN_MODE}`)
 
+    cy.waitForSpecToFinish()
     cy.findByTestId('aut-url').should('not.exist')
     cy.findByTestId('playground-activator').should('not.exist')
 
@@ -81,9 +77,7 @@ describe('Cypress In Cypress - run mode', { viewportWidth: 1200 }, () => {
       })
     })
 
-    cy.visitApp()
-    simulateRunModeInUI()
-    cy.contains('dom-content.spec').click()
+    cy.visitApp(`/specs/runner?file=cypress/e2e/dom-content.spec.js&${CY_IN_CY_SIMULATE_RUN_MODE}`)
 
     cy.contains('http://localhost:4455/cypress/e2e/dom-content.html').should('be.visible')
     cy.findByLabelText('Stats').should('not.exist')
@@ -92,13 +86,3 @@ describe('Cypress In Cypress - run mode', { viewportWidth: 1200 }, () => {
     cy.findByTestId('sidebar').should('not.exist')
   })
 })
-
-function simulateRunModeInUI () {
-  // this simulates run mode enough for this test
-  cy.window().then((win) => {
-    win.__CYPRESS_MODE__ = 'run'
-    cy.get('body').then(($el) => {
-      $el.find('[data-cy="sidebar"]')?.remove()
-    })
-  })
-}

packages/app/cypress/e2e/debug.cy.ts

@@ -0,0 +1,151 @@
+import RelevantRunsDataSource_RunsByCommitShas from '../fixtures/gql-RelevantRunsDataSource_RunsByCommitShas.json'
+
+Cypress.on('window:before:load', (win) => {
+  win.__CYPRESS_GQL_NO_SOCKET__ = 'true'
+})
+
+// These mocks all the responses so we can get deterministic
+// results to test the debug page.
+// The JSON fixtures were generated by using a real app and capturing
+// the responses.
+describe('App - Debug Page', () => {
+  beforeEach(() => {
+    cy.scaffoldProject('cypress-in-cypress')
+    cy.openProject('cypress-in-cypress')
+    cy.startAppServer('component')
+
+    cy.loginUser()
+    cy.withCtx((ctx) => {
+      ctx.git?.__setGitHashesForTesting(['commit1', 'commit2'])
+    })
+
+    cy.remoteGraphQLIntercept((obj, _testState, options) => {
+      if (obj.operationName === 'RelevantRunsDataSource_RunsByCommitShas') {
+        obj.result.data = options.RelevantRunsDataSource_RunsByCommitShas.data
+      }
+
+      return obj.result
+    }, { RelevantRunsDataSource_RunsByCommitShas })
+  })
+
+  it('all tests passed', () => {
+    // This mocks all the responses so we can get deterministic
+    // results to test the debug page.
+    cy.intercept('POST', '/__cypress/graphql/query-Debug', {
+      fixture: 'debug-Passing/gql-Debug.json',
+    })
+
+    cy.intercept('POST', '/__cypress/graphql/query-CloudViewerAndProject_RequiredData', {
+      fixture: 'debug-Passing/gql-CloudViewerAndProject_RequiredData.json',
+    })
+
+    cy.intercept('POST', '/__cypress/graphql/query-MainAppQuery', {
+      fixture: 'debug-Passing/gql-MainAppQuery.json',
+    })
+
+    cy.intercept('POST', '/__cypress/graphql/query-SideBarNavigationContainer', {
+      fixture: 'debug-Passing/gql-SideBarNavigationContainer',
+    })
+
+    cy.intercept('POST', '/__cypress/graphql/query-HeaderBar_HeaderBarQuery', {
+      fixture: 'debug-Passing/gql-HeaderBar_HeaderBarQuery',
+    })
+
+    cy.intercept('POST', '/__cypress/graphql/query-SpecsPageContainer', {
+      fixture: 'debug-Passing/gql-SpecsPageContainer',
+    })
+
+    cy.visitApp()
+
+    cy.findByTestId('sidebar-link-debug-page').click()
+    cy.findByTestId('debug-container').should('be.visible')
+
+    cy.findByTestId('header-top').contains('update projectId')
+    cy.findByTestId('debug-header-dashboard-link')
+    .contains('View in Cypress Cloud')
+    .should('have.attr', 'href', 'https://cloud.cypress.io/projects/7p5uce/runs/2')
+
+    cy.findByTestId('debug-runNumber-PASSED').contains('#2')
+    cy.findByTestId('debug-commitsAhead').contains('You are 1 commit ahead')
+
+    cy.findByTestId('metadata').within(() => {
+      cy.get('[title="passed"]').contains('2')
+      cy.get('[title="failed"]').contains('0')
+      cy.get('[title="skipped"]').contains('0')
+      cy.get('[title="pending"]').contains('2')
+      cy.findByTestId('debug-header-branch').contains('main')
+      cy.findByTestId('debug-header-commitHash').contains('e9d176f')
+      cy.findByTestId('debug-header-author').contains('Lachlan Miller')
+      cy.findByTestId('debug-header-createdAt').contains('01:18')
+    })
+
+    cy.findByTestId('debug-passed').contains('Well Done!')
+    cy.findByTestId('debug-passed').contains('All your tests passed.')
+    cy.findByLabelText('Relevant run passed').should('be.visible').contains('0')
+    cy.findByTestId('run-failures').should('not.exist')
+  })
+
+  it('shows information about a failed spec', () => {
+    cy.intercept('POST', '/__cypress/graphql/query-Debug', {
+      fixture: 'debug-Failing/gql-Debug.json',
+    })
+
+    cy.intercept('POST', '/__cypress/graphql/query-CloudViewerAndProject_RequiredData', {
+      fixture: 'debug-Failing/gql-CloudViewerAndProject_RequiredData.json',
+    })
+
+    cy.intercept('POST', '/__cypress/graphql/query-MainAppQuery', {
+      fixture: 'debug-Failing/gql-MainAppQuery.json',
+    })
+
+    cy.intercept('POST', '/__cypress/graphql/query-SideBarNavigationContainer', {
+      fixture: 'debug-Failing/gql-SideBarNavigationContainer',
+    })
+
+    cy.intercept('POST', '/__cypress/graphql/query-HeaderBar_HeaderBarQuery', {
+      fixture: 'debug-Failing/gql-HeaderBar_HeaderBarQuery',
+    })
+
+    cy.intercept('POST', '/__cypress/graphql/query-SpecsPageContainer', {
+      fixture: 'debug-Failing/gql-SpecsPageContainer',
+    })
+
+    cy.visitApp()
+
+    cy.findByTestId('sidebar-link-debug-page').click()
+    cy.findByTestId('debug-container').should('be.visible')
+
+    cy.findByTestId('header-top').contains('chore: testing cypress')
+    cy.findByTestId('debug-header-dashboard-link')
+    .contains('View in Cypress Cloud')
+    .should('have.attr', 'href', 'https://cloud.cypress.io/projects/vgqrwp/runs/136')
+
+    cy.findByLabelText('Relevant run had 1 test failure').should('be.visible').contains('1')
+
+    cy.findByTestId('debug-runNumber-FAILED').contains('#136')
+    cy.findByTestId('debug-commitsAhead').contains('You are 1 commit ahead')
+
+    cy.findByTestId('metadata').within(() => {
+      cy.get('[title="passed"]').contains('1')
+      cy.get('[title="failed"]').contains('1')
+      cy.get('[title="skipped"]').contains('0')
+      cy.get('[title="pending"]').contains('0')
+      cy.findByTestId('debug-header-branch').contains('main')
+      cy.findByTestId('debug-header-commitHash').contains('commit1')
+      cy.findByTestId('debug-header-author').contains('Lachlan Miller')
+      cy.findByTestId('debug-header-createdAt').contains('00:19')
+    })
+
+    cy.findByTestId('spec-contents').within(() => {
+      cy.contains('src/components/InfoPanel/InfoPanel.cy.ts')
+      cy.findByTestId('metaData-Results-spec-duration').contains('00:04')
+      cy.findByTestId('metaData-Results-operating-system').contains('Linux Ubuntu')
+      cy.findByTestId('metaData-Results-browser').contains('Electron 106')
+      cy.findByTestId('metaData-Results-testing-type').contains('Component')
+    })
+
+    cy.findByTestId('test-row').contains('InfoPanel')
+    cy.findByTestId('test-row').contains('renders')
+    cy.findByTestId('run-failures').should('exist').should('have.attr', 'href', '#/specs/runner?file=src/components/InfoPanel/InfoPanel.cy.ts&mode=debug')
+  })
+})

packages/app/cypress/e2e/runner/cloud-debug-filter.cy.ts

@@ -0,0 +1,157 @@
+describe('cloud debug test filtering', () => {
+  beforeEach(() => {
+    cy.scaffoldProject('cloud-debug-filter')
+    cy.openProject('cloud-debug-filter')
+    cy.startAppServer('e2e')
+  })
+
+  it('works with nested suites', () => {
+    cy.visitApp(`specs/runner?file=cypress/e2e/test.cy.js`)
+
+    cy.waitForSpecToFinish()
+
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = {
+        'cypress/e2e/test.cy.js': ['t2'],
+      }
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/test.cy.js&mode=debug`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+
+    cy.get('.runnable-title').contains('t2')
+
+    cy.get('.debug-dismiss').contains('1 / 4 tests').click()
+    cy.waitForSpecToFinish({ passCount: 2, failCount: 2 })
+
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = {
+        'cypress/e2e/test.cy.js': ['s1 t4'],
+      }
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/test.cy.js&mode=debug`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+
+    cy.get('.runnable-title').contains('t4')
+  })
+
+  it('wraps filter UI with large number of tests', () => {
+    cy.visitApp(`specs/runner?file=cypress/e2e/lots-of-tests.cy.js`)
+
+    cy.get('[data-cy="reporter-panel"]').as('reporterPanel')
+
+    cy.waitForSpecToFinish()
+
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = {
+        'cypress/e2e/lots-of-tests.cy.js': ['test1'],
+      }
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/lots-of-tests.cy.js&mode=debug`)
+    cy.waitForSpecToFinish({ passCount: 50 })
+
+    cy.get('@reporterPanel').then((el) => el.width(500))
+    cy.get('@reporterPanel').percySnapshot('wide')
+
+    cy.get('@reporterPanel').then((el) => el.width(350))
+    cy.get('@reporterPanel').percySnapshot('medium')
+
+    cy.get('@reporterPanel').then((el) => el.width(250))
+    cy.get('@reporterPanel').percySnapshot('narrow')
+
+    cy.get('@reporterPanel').then((el) => el.width(150))
+    cy.get('@reporterPanel').percySnapshot('skinny')
+  })
+
+  it('works with skips and onlys', () => {
+    cy.visitApp(`specs/runner?file=cypress/e2e/skip-and-only.cy.js`)
+
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+
+    // .only is respected
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = {
+        'cypress/e2e/skip-and-only.cy.js': ['t1', 't3'],
+      }
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/skip-and-only.cy.js&mode=debug`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+
+    cy.get('.runnable-title').contains('t1')
+
+    cy.get('.debug-dismiss').click().waitForSpecToFinish()
+
+    // .only is ignored as it is not in set of filtered tests
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = {
+        'cypress/e2e/skip-and-only.cy.js': ['t3'],
+      }
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/skip-and-only.cy.js&mode=debug`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+
+    cy.get('.runnable-title').contains('t3')
+
+    cy.get('.debug-dismiss').click().waitForSpecToFinish()
+
+    // .skip is respected
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = {
+        'cypress/e2e/skip-and-only.cy.js': ['t2', 't3'],
+      }
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/skip-and-only.cy.js&mode=debug`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1, pendingCount: 1 })
+    cy.get('.runnable-title').first().contains('t2')
+    cy.get('.runnable-title').last().contains('t3')
+
+    cy.get('.debug-dismiss').contains('2 / 4 tests').click().waitForSpecToFinish()
+
+    // suite.only is respected
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = {
+        'cypress/e2e/skip-and-only.cy.js': ['t3', 's1 t4'],
+      }
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/skip-and-only.cy.js&mode=debug`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+    cy.get('.runnable-title').contains('t4')
+  })
+
+  it('works with browser filter', () => {
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = {
+        'cypress/e2e/lots-of-tests.cy.j': ['t1', 's1 t2'],
+      }
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/browsers.cy.js&mode=debug`)
+
+    cy.get('.runnable-title').eq(0).contains('t1 (skipped due to browser)')
+    cy.get('.runnable-title').eq(1).contains('s1 (skipped due to browser)')
+    cy.get('.runnable-title').eq(2).contains('t2')
+  })
+
+  it('filter is maintained across cross-domain reinitialization', () => {
+    cy.visitApp(`specs/runner?file=cypress/e2e/domain-change.cy.js`)
+
+    cy.get('[data-cy="reporter-panel"]').as('reporterPanel')
+
+    cy.waitForSpecToFinish()
+
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = {
+        'cypress/e2e/lots-of-tests.cy.j': ['t2', 't3'],
+      }
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/domain-change.cy.js&mode=debug`)
+    cy.waitForSpecToFinish({ failCount: 2 })
+  })
+})