From 23a66b91393820c25ea809635b5388aeebe8d35a Mon Sep 17 00:00:00 2001 From: Zach Panzarino Date: Thu, 11 Feb 2021 00:48:43 -0500 Subject: [PATCH 1/3] fix: regression where failed events could cause a passing test to display as failed --- .../reporter/src/attempts/attempt-model.ts | 6 +----- packages/reporter/src/test/test-model.ts | 2 +- packages/runner/src/lib/event-manager.js | 21 +++++++++++++------ packages/runner/src/studio/studio-recorder.js | 7 +++++++ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/reporter/src/attempts/attempt-model.ts b/packages/reporter/src/attempts/attempt-model.ts index 914fa01e8b32..d3a1a66f3626 100644 --- a/packages/reporter/src/attempts/attempt-model.ts +++ b/packages/reporter/src/attempts/attempt-model.ts @@ -109,15 +109,11 @@ export default class Attempt { } } - @action updateLog (props: LogProps) { + updateLog (props: LogProps) { const log = this._logs[props.id] if (log) { log.update(props) - - if (log.state === 'failed') { - this._state = 'failed' - } } } diff --git a/packages/reporter/src/test/test-model.ts b/packages/reporter/src/test/test-model.ts index 6436e9af9131..b1795f391749 100644 --- a/packages/reporter/src/test/test-model.ts +++ b/packages/reporter/src/test/test-model.ts @@ -168,7 +168,7 @@ export default class Test extends Runnable { } } - if (props.err) { + if (props.err || props.state) { this._withAttempt(this.currentRetry, (attempt: Attempt) => { attempt.update(props) }) diff --git a/packages/runner/src/lib/event-manager.js b/packages/runner/src/lib/event-manager.js index b31a1abc26ff..154f81fc6dd1 100644 --- a/packages/runner/src/lib/event-manager.js +++ b/packages/runner/src/lib/event-manager.js @@ -388,9 +388,7 @@ const eventManager = { Cypress.on('log:added', (log) => { const displayProps = Cypress.runner.getDisplayPropsForLog(log) - if (studioRecorder.isActive) { - displayProps.hookId = studioRecorder.hookId - } + this._interceptStudio(displayProps) reporterBus.emit('reporter:log:add', displayProps) }) @@ -398,9 +396,7 @@ const eventManager = { Cypress.on('log:changed', (log) => { const displayProps = Cypress.runner.getDisplayPropsForLog(log) - if (studioRecorder.isActive) { - displayProps.hookId = studioRecorder.hookId - } + this._interceptStudio(displayProps) reporterBus.emit('reporter:log:state:changed', displayProps) }) @@ -556,6 +552,19 @@ const eventManager = { } }, + _interceptStudio (displayProps) { + if (studioRecorder.isActive) { + displayProps.hookId = studioRecorder.hookId + + if (displayProps.name === 'visit' && displayProps.state === 'failed') { + studioRecorder.testFailed() + reporterBus.emit('test:set:state', studioRecorder.testError, _.noop) + } + } + + return displayProps + }, + emit (event, ...args) { localBus.emit(event, ...args) }, diff --git a/packages/runner/src/studio/studio-recorder.js b/packages/runner/src/studio/studio-recorder.js index 2ca86a4ae36e..bef144f3d784 100644 --- a/packages/runner/src/studio/studio-recorder.js +++ b/packages/runner/src/studio/studio-recorder.js @@ -68,6 +68,13 @@ export class StudioRecorder { return this.isActive && !this.url && !this.isFailed } + @computed get testError () { + return { + id: this.testId, + state: 'failed', + } + } + @computed get saveError () { return { id: this.testId, From 097e94d6786d3e39e4785b278d1cd354b4b4ba3a Mon Sep 17 00:00:00 2001 From: Zach Panzarino Date: Thu, 11 Feb 2021 02:34:56 -0500 Subject: [PATCH 2/3] add tests --- .../integration/unit/test_model_spec.ts | 14 +++- .../fixtures/studio/error_hooks_spec.js | 13 +++ .../fixtures/studio/error_test_spec.js | 11 +++ .../cypress/integration/studio.ui.spec.js | 79 +++++++++++++++++++ packages/runner/cypress/support/helpers.js | 3 +- 5 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 packages/runner/cypress/fixtures/studio/error_hooks_spec.js create mode 100644 packages/runner/cypress/fixtures/studio/error_test_spec.js diff --git a/packages/reporter/cypress/integration/unit/test_model_spec.ts b/packages/reporter/cypress/integration/unit/test_model_spec.ts index c4461f920141..a1b872b90031 100644 --- a/packages/reporter/cypress/integration/unit/test_model_spec.ts +++ b/packages/reporter/cypress/integration/unit/test_model_spec.ts @@ -29,7 +29,6 @@ const createCommand = (props: Partial = {}) => { testId: 'r3', timeout: 4000, wallClockStartedAt: new Date().toString(), - } as CommandProps return _.defaults(props, defaults) @@ -262,6 +261,19 @@ describe('Test model', () => { test.updateLog(createCommand({ timeout: 6000 })) expect(test.lastAttempt.commands[0].timeout).to.equal(6000) }) + + // https://github.com/cypress-io/cypress/issues/14978 + it('does not change test state based on log state', () => { + const test = createTest() + + test.addLog(createCommand({ state: 'active' })) + expect(test.lastAttempt.commands[0].state).to.equal('active') + expect(test.state).to.equal('processing') + + test.updateLog(createCommand({ state: 'failed' })) + expect(test.lastAttempt.commands[0].state).to.equal('failed') + expect(test.state).to.equal('processing') + }) }) context('#removeLog', () => { diff --git a/packages/runner/cypress/fixtures/studio/error_hooks_spec.js b/packages/runner/cypress/fixtures/studio/error_hooks_spec.js new file mode 100644 index 000000000000..b0878394ace6 --- /dev/null +++ b/packages/runner/cypress/fixtures/studio/error_hooks_spec.js @@ -0,0 +1,13 @@ +describe('suite', () => { + beforeEach(() => { + cy.visit('the://url') + + cy.get('body').then(() => { + throw new Error('Failing Test') + }) + }) + + it('test', () => { + cy.get('body') + }) +}) diff --git a/packages/runner/cypress/fixtures/studio/error_test_spec.js b/packages/runner/cypress/fixtures/studio/error_test_spec.js new file mode 100644 index 000000000000..77096cc0f593 --- /dev/null +++ b/packages/runner/cypress/fixtures/studio/error_test_spec.js @@ -0,0 +1,11 @@ +describe('suite', () => { + beforeEach(() => { + cy.visit('the://url') + }) + + it('test', () => { + cy.get('body').then(() => { + throw new Error('Failing Test') + }) + }) +}) diff --git a/packages/runner/cypress/integration/studio.ui.spec.js b/packages/runner/cypress/integration/studio.ui.spec.js index 042c250c81e9..b154facc4982 100644 --- a/packages/runner/cypress/integration/studio.ui.spec.js +++ b/packages/runner/cypress/integration/studio.ui.spec.js @@ -42,6 +42,7 @@ describe('studio ui', () => { cy.get('.runner').find('.url').should('have.value', '') cy.get('.runner').find('.url-menu').should('be.visible') + cy.get('.runner').find('.url-menu').find('.btn-submit').should('be.disabled') cy.percySnapshot() }) @@ -63,6 +64,27 @@ describe('studio ui', () => { }) }) + // doesn't actually test visiting, just ui state + it('allows user to visit inputted url and prompts for interaction after visit', () => { + runIsolatedCypress('cypress/fixtures/studio/basic_spec.js', { + config: { + baseUrl: null, + }, + state: { + studioTestId: 'r5', + }, + }) + .then(() => { + cy.get('.runner').find('.url').type('the://url') + cy.get('.runner').find('.url-menu').find('.btn-submit').click() + + cy.get('.reporter').contains('the://url').closest('.command-wrapper-text').contains('visit') + cy.get('.reporter').contains('Interact with your site to add test commands.') + + cy.percySnapshot() + }) + }) + it('displays modal when available commands is clicked', () => { runIsolatedCypress('cypress/fixtures/studio/basic_spec.js', { state: { @@ -76,4 +98,61 @@ describe('studio ui', () => { cy.get('reach-portal').should('not.exist') }) }) + + describe('error state', () => { + it('displays error state when extending a failed test', () => { + runIsolatedCypress('cypress/fixtures/studio/error_test_spec.js', { + state: { + studioTestId: 'r3', + }, + }) + .then(() => { + cy.get('.reporter').contains('test').closest('.runnable').should('have.class', 'runnable-failed') + cy.get('.reporter').contains('Studio cannot add commands to a failing test.').should('exist') + + cy.get('.runner').find('.iframes-container').should('have.class', 'studio-is-failed') + + cy.percySnapshot() + }) + }) + + it('displays error state when a before hook fails', () => { + runIsolatedCypress('cypress/fixtures/studio/error_hooks_spec.js', { + state: { + studioTestId: 'r3', + }, + }) + .then(() => { + cy.get('.reporter').contains('test').closest('.runnable').should('have.class', 'runnable-failed') + cy.get('.reporter').contains('Studio cannot add commands to a failing test.').should('exist') + + cy.get('.runner').find('.iframes-container').should('have.class', 'studio-is-failed') + }) + }) + + it('displays error state when cy.visit() fails on user inputted url', () => { + runIsolatedCypress('cypress/fixtures/studio/basic_spec.js', { + config: { + baseUrl: null, + }, + state: { + studioTestId: 'r5', + }, + visitUrl: 'http://localhost:3500/foo', + visitSuccess: false, + }) + .then(() => { + cy.get('.runner').find('.url').type('the://url') + cy.get('.runner').find('.url-menu').find('.btn-submit').click() + + cy.get('.reporter').contains('test 3').closest('.runnable').should('have.class', 'runnable-failed') + cy.get('.reporter').contains('the://url').closest('.command-wrapper-text').contains('visit') + cy.get('.reporter').contains('Studio cannot add commands to a failing test.').should('exist') + + cy.get('.runner').find('.iframes-container').should('have.class', 'studio-is-failed') + + cy.percySnapshot() + }) + }) + }) }) diff --git a/packages/runner/cypress/support/helpers.js b/packages/runner/cypress/support/helpers.js index db96c46b4f76..56dedcae05b7 100644 --- a/packages/runner/cypress/support/helpers.js +++ b/packages/runner/cypress/support/helpers.js @@ -106,6 +106,7 @@ function createCypress (defaultOptions = {}) { config: { video: false }, onBeforeRun () {}, visitUrl: 'http://localhost:3500/fixtures/dom.html', + visitSuccess: true, }) return cy.visit('/fixtures/isolated-runner.html#/tests/cypress/fixtures/empty_spec.js') @@ -254,7 +255,7 @@ function createCypress (defaultOptions = {}) { .withArgs('backend:request', 'resolve:url') .yieldsAsync({ response: { - isOkStatusCode: true, + isOkStatusCode: opts.visitSuccess, isHtml: true, url: opts.visitUrl, } }) From 5ad3b577761c9d0fc84a727b7f4b0e64fd97ce40 Mon Sep 17 00:00:00 2001 From: Zach Panzarino Date: Thu, 11 Feb 2021 03:29:22 -0500 Subject: [PATCH 3/3] Remove those percy snapshots since they'll flake --- packages/runner/cypress/integration/studio.ui.spec.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/runner/cypress/integration/studio.ui.spec.js b/packages/runner/cypress/integration/studio.ui.spec.js index b154facc4982..b27949470319 100644 --- a/packages/runner/cypress/integration/studio.ui.spec.js +++ b/packages/runner/cypress/integration/studio.ui.spec.js @@ -80,8 +80,6 @@ describe('studio ui', () => { cy.get('.reporter').contains('the://url').closest('.command-wrapper-text').contains('visit') cy.get('.reporter').contains('Interact with your site to add test commands.') - - cy.percySnapshot() }) }) @@ -150,8 +148,6 @@ describe('studio ui', () => { cy.get('.reporter').contains('Studio cannot add commands to a failing test.').should('exist') cy.get('.runner').find('.iframes-container').should('have.class', 'studio-is-failed') - - cy.percySnapshot() }) }) })