Skip to content

Commit

Permalink
test: fix flaky tests (#26505)
Browse files Browse the repository at this point in the history 
* test(launchpad): fix flaky tests

Closes #23153

Closes #23154

---------

Co-authored-by: astone123 <adams@cypress.io>
  • Loading branch information
@jordanpowell88 @astone123
jordanpowell88 and astone123 committed Apr 17, 2023
1 parent 6209b91 commit 067e4fa
Show file tree
Hide file tree
Showing 8 changed files with 52 additions and 46 deletions.
2 changes: 1 addition & 1 deletion .circleci/workflows.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ windowsWorkflowFilters: &windows-workflow-filters
or:
- equal: [ develop, << pipeline.git.branch >> ]
# use the following branch as well to ensure that v8 snapshot cache updates are fully tested
- equal: [ 'lmiller/fixing-vite-windows', << pipeline.git.branch >> ]
- equal: [ 'windows-flake', << pipeline.git.branch >> ]
- equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
- matches:
pattern: /^release\/\d+\.\d+\.\d+$/
Expand Down
2 changes: 1 addition & 1 deletion packages/app/cypress/e2e/runs.cy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ describe('App: Runs', { viewportWidth: 1200 }, () => {
moveToRunsPage()
cy.findByText(defaultMessages.runs.connect.buttonProject).click()
cy.contains('button', defaultMessages.runs.connect.modal.selectProject.createProject).click()
cy.findByText(defaultMessages.runs.connectSuccessAlert.title).should('be.visible')
cy.findByText(defaultMessages.runs.connectSuccessAlert.title, { timeout: 10000 }).should('be.visible')

cy.withCtx(async (ctx) => {
const config = await ctx.project.getConfig()
Expand Down
46 changes: 23 additions & 23 deletions packages/launchpad/cypress/e2e/config-files-error-handling.cy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ describe('Config files error handling', () => {

cy.findByRole('button', { name: 'Try again' }).click()

cy.get('h1').should('contain', 'Welcome to Cypress')
cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 })
})

it('shows the upgrade screen if there is a legacy config file', () => {
Expand Down Expand Up @@ -70,7 +70,7 @@ describe('Config files error handling', () => {

cy.findByRole('button', { name: 'Try again' }).click()

cy.get('h1').should('contain', 'Welcome to Cypress')
cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 })
})

it('handles deprecated config fields', () => {
Expand All @@ -92,7 +92,7 @@ describe('Config files error handling', () => {

cy.findByRole('button', { name: 'Try again' }).click()

cy.get('h1').should('contain', 'Choose a browser')
cy.contains('h1', 'Choose a browser', { timeout: 10000 })
})
})

Expand All @@ -102,7 +102,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('plugins-root-sync-error', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
expectStackToBe('open')
})

Expand All @@ -111,7 +111,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('plugins-root-syntax-error', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
expectStackToBe('open')
})

Expand All @@ -120,7 +120,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('plugins-root-async-error', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', 'Unexpected Error')
cy.contains('h1', 'Unexpected Error', { timeout: 10000 })
expectStackToBe('open')
})

Expand All @@ -129,7 +129,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('plugins-function-sync-error', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
expectStackToBe('open')
})

Expand All @@ -138,7 +138,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('config-with-invalid-browser', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
expectStackToBe('closed')
})

Expand All @@ -147,7 +147,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('plugins-function-sync-error', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
expectStackToBe('open')
})

Expand All @@ -156,7 +156,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('config-with-ts-syntax-error')
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()

cy.withCtx(async (ctx) => {
Expand All @@ -165,7 +165,7 @@ describe('Launchpad: Error System Tests', () => {

cy.findByRole('button', { name: 'Try again' }).click()

cy.get('h1').should('contain', 'Welcome to Cypress')
cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 })
})

it(`clears the error correctly after first 'try again' attempt`, () => {
Expand All @@ -174,31 +174,31 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('config-with-ts-syntax-error')
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })

// Try again while the config is still invalid
cy.findByRole('button', { name: 'Try again' }).click()

cy.wait('@resetErrorsAndLoadConfig')

// Wait until config error is on screen again
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })

cy.withCtx(async (ctx) => {
await ctx.actions.file.writeFileInProject('cypress.config.ts', 'export default { e2e: { supportFile: false } }')
})

cy.findByRole('button', { name: 'Try again' }).click()

cy.get('h1').should('contain', 'Welcome to Cypress')
cy.contains('h1', 'Welcome to Cypress')
})

it('shows correct user file instead of node file', () => {
cy.scaffoldProject('config-with-import-error')
cy.openProject('config-with-import-error')
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()

cy.get('[data-testid="error-code-frame"]').should('contain', 'cypress.config.js:3:23')
Expand All @@ -209,7 +209,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('config-with-ts-module-error')
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()

cy.get('[data-testid="error-code-frame"]').should('contain', 'cypress.config.ts:6:10')
Expand All @@ -223,7 +223,7 @@ describe('setupNodeEvents', () => {
cy.visitLaunchpad()
cy.skipWelcome()
cy.findByText('E2E Testing').click()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()
})

Expand All @@ -233,7 +233,7 @@ describe('setupNodeEvents', () => {
cy.visitLaunchpad()
cy.skipWelcome()
cy.findByText('E2E Testing').click()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()

cy.get('[data-cy="alert-body"]').should('contain', 'integrationFolder')
Expand All @@ -245,7 +245,7 @@ describe('setupNodeEvents', () => {
cy.visitLaunchpad()
cy.skipWelcome()
cy.findByText('E2E Testing').click()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()
})

Expand Down Expand Up @@ -279,7 +279,7 @@ describe('setupNodeEvents', () => {

cy.findByRole('button', { name: 'Try again' }).click()

cy.get('h1').should('contain', 'Choose a browser')
cy.contains('h1', 'Choose a browser', { timeout: 10000 })
})

it('handles multiple config errors and then recovers', () => {
Expand All @@ -293,7 +293,7 @@ describe('setupNodeEvents', () => {

cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.get('[data-cy="alert-body"]').should('contain', 'Expected baseUrl to be a fully qualified URL')

cy.withCtx(async (ctx) => {
Expand All @@ -302,15 +302,15 @@ describe('setupNodeEvents', () => {

cy.findByRole('button', { name: 'Try again' }).click()
cy.get('[data-cy-testingType=e2e]').click()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.get('[data-cy="alert-body"]').should('contain', 'The baseUrl configuration option is now invalid when set from the root of the config object')

cy.withCtx(async (ctx) => {
await ctx.actions.file.writeFileInProject('cypress.config.js', `module.exports = { e2e: { baseUrl: 'http://localhost:3000', supportFile: false } }`)
})

cy.findByRole('button', { name: 'Try again' }).click()
cy.get('h1').should('contain', 'Choose a browser')
cy.contains('h1', 'Choose a browser', { timeout: 10000 })
cy.get('[data-cy="alert"]').should('contain', 'Warning: Cannot Connect Base Url Warning')
})

Expand Down
11 changes: 5 additions & 6 deletions packages/launchpad/cypress/e2e/config-warning.cy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ describe('component testing dependency warnings', () => {
cy.skipWelcome()
cy.contains('component-testing-outdated-dependencies').click()
cy.get('[data-cy="warning-alert"]').should('not.exist')
cy.get('a').contains('Projects').click()
cy.contains('a', 'Projects').click()
cy.get('[data-cy-testingtype="component"]').click()
cy.get('[data-cy="warning-alert"]', { timeout: 12000 }).should('exist')
.should('contain.text', 'Warning: Component Testing Mismatched Dependencies')
Expand All @@ -243,17 +243,16 @@ describe('component testing dependency warnings', () => {
cy.get('.warning-markdown').find('li').should('have.length', 3)
})

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23154
it('warns against outdated @vue/cli dependency', { retries: 15 }, () => {
it('warns against outdated @vue/cli dependency', () => {
cy.scaffoldProject('outdated-deps-vuecli3')
cy.addProject('outdated-deps-vuecli3')
cy.openGlobalMode()
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('outdated-deps-vuecli3').click()
cy.get('[data-cy="warning-alert"]').should('not.exist')
cy.get('a').contains('Projects').click()
cy.get('[data-cy-testingtype="component"]').click()
cy.contains('a', 'Projects').click()
cy.get('[data-cy-testingtype="component"]', { timeout: 12000 }).click()
cy.get('[data-cy="warning-alert"]', { timeout: 12000 }).should('exist')
.should('contain.text', 'Warning: Component Testing Mismatched Dependencies')
.should('contain.text', '@vue/cli-service. Expected ^=4.0.0 || ^=5.0.0, found 3.12.1.')
Expand All @@ -271,7 +270,7 @@ describe('component testing dependency warnings', () => {
cy.contains('vueclivue3-configured').click()
cy.get('[data-cy="warning-alert"]').should('not.exist')
cy.get('a').contains('Projects').click()
cy.get('[data-cy-testingtype="component"]').click()
cy.get('[data-cy-testingtype="component"]', { timeout: 10000 }).click()

// Wait until launch browser screen and assert warning does not exist
cy.contains('Choose a browser', { timeout: 12000 })
Expand Down
4 changes: 2 additions & 2 deletions packages/launchpad/cypress/e2e/migration.cy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ function finishMigrationAndContinue () {
}

function checkOutcome () {
cy.contains('Welcome to Cypress!').should('be.visible')
cy.contains('Welcome to Cypress!', { timeout: 10000 }).should('be.visible')
}

function runAutoRename () {
Expand Down Expand Up @@ -95,7 +95,7 @@ describe('global mode', () => {
cy.contains('migration-e2e-export-default').click()

// rename integration->e2e
cy.get(renameAutoStep).should('exist')
cy.get(renameAutoStep, { timeout: 10000 }).should('exist')
cy.get(renameManualStep).should('not.exist')

// cypress/support/index.ts -> cypress/support/e2e.ts
Expand Down
15 changes: 7 additions & 8 deletions packages/launchpad/cypress/e2e/project-setup.cy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -556,8 +556,7 @@ describe('Launchpad: Setup Project', () => {
})
})

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23153
describe('Command for package managers', { retries: 15 }, () => {
describe('Command for package managers', () => {
it('makes the right command for yarn', () => {
scaffoldAndOpenProject('pristine-yarn')

Expand All @@ -566,7 +565,7 @@ describe('Launchpad: Setup Project', () => {
cy.get('[data-cy-testingtype="component"]').click()
cy.get('[data-testid="select-framework"]').click()
cy.findByText('Create React App').click()
cy.findByText('Next step').click()
cy.contains('button', 'Next step').should('not.be.disabled').click()
cy.findByDisplayValue('yarn add -D react-scripts react-dom react').should('be.visible')
})

Expand All @@ -578,7 +577,7 @@ describe('Launchpad: Setup Project', () => {
cy.get('[data-cy-testingtype="component"]').click()
cy.get('[data-testid="select-framework"]').click()
cy.findByText('Create React App').click()
cy.findByText('Next step').click()
cy.contains('button', 'Next step').should('not.be.disabled').click()
cy.findByDisplayValue('pnpm install -D react-scripts react-dom react')
})

Expand All @@ -588,9 +587,9 @@ describe('Launchpad: Setup Project', () => {
cy.visitLaunchpad()

cy.get('[data-cy-testingtype="component"]').click()
cy.get('button').should('be.visible').contains('Vue.js 3(detected)')
cy.get('button').should('be.visible').contains('Vite(detected)')
cy.findByText('Next step').click()
cy.contains('button', 'Vue.js 3(detected)').should('be.visible')
cy.contains('button', 'Vite(detected)').should('be.visible')
cy.contains('button', 'Next step').should('not.be.disabled').click()
cy.findByTestId('alert').contains(`You've successfully installed all required dependencies.`)
})

Expand All @@ -602,7 +601,7 @@ describe('Launchpad: Setup Project', () => {
cy.get('[data-cy-testingtype="component"]').click()
cy.get('[data-testid="select-framework"]').click()
cy.findByText('Create React App').click()
cy.findByText('Next step').click()
cy.contains('button', 'Next step').should('not.be.disabled').click()
cy.findByDisplayValue('npm install -D react-scripts react-dom react')
})
})
Expand Down
10 changes: 9 additions & 1 deletion packages/launchpad/cypress/e2e/scaffold-project.cy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,10 @@ describe('scaffolding new projects', { defaultCommandTimeout: 7000 }, () => {
})

it('generates valid config file for pristine project without cypress installed', () => {
cy.intercept('mutation-ScaffoldedFiles_completeSetup').as('mutationScaffoldedFiles')
cy.intercept('query-MainLaunchpadQuery').as('mainLaunchpadQuery')
cy.intercept('query-HeaderBar_HeaderBarQuery').as('headerBarQuery')
cy.intercept('query-CloudViewerAndProject_RequiredData').as('cloudViewerAndProjectRequiredData')
cy.scaffoldProject('pristine')
cy.openProject('pristine')
cy.withCtx((ctx) => ctx.currentProject).then((currentProject) => {
Expand All @@ -178,6 +182,10 @@ describe('scaffolding new projects', { defaultCommandTimeout: 7000 }, () => {
cy.skipWelcome()
cy.contains('button', cy.i18n.testingType.e2e.name).click()
cy.contains('button', cy.i18n.setupPage.step.continue).click()
cy.contains('h1', cy.i18n.setupPage.testingCard.chooseABrowser).should('be.visible')
cy.wait('@mutationScaffoldedFiles')
cy.wait('@mainLaunchpadQuery')
cy.wait('@headerBarQuery')
cy.wait('@cloudViewerAndProjectRequiredData')
cy.get('h1').contains(cy.i18n.setupPage.testingCard.chooseABrowser).should('be.visible')
})
})
8 changes: 4 additions & 4 deletions packages/launchpad/cypress/e2e/top-nav-launchpad.cy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -366,12 +366,12 @@ describe('Launchpad Top Nav Workflows', () => {
options.sinon.stub(ctx._apis.authApi, 'logIn').callsFake(async (onMessage) => {
setTimeout(() => {
onMessage({ browserOpened: true })
}, 500)
}, 2000)

return new Promise((resolve) => {
setTimeout(() => {
resolve(options.user)
}, 2000)
}, 3000)
})
})
}, { user })
Expand Down Expand Up @@ -687,7 +687,7 @@ describe('Launchpad Top Nav Workflows', () => {

cy.get('[data-cy="project-card"]').click()

cy.contains('E2E Testing').click()
cy.contains('E2E Testing', { timeout: 10000 }).click()

mockLogInActionsForUser(mockUser)
logIn({ expectedNextStepText: 'Continue', displayName: mockUser.name })
Expand All @@ -707,7 +707,7 @@ describe('Launchpad Top Nav Workflows', () => {

cy.get('[data-cy="project-card"]').click()

cy.contains('E2E Testing').click()
cy.contains('E2E Testing', { timeout: 10000 }).click()

mockLogInActionsForUser(mockUser)
logIn({ expectedNextStepText: 'Connect project', displayName: mockUser.name })
Expand Down

5 comments on commit 067e4fa

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 067e4fa Apr 17, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the linux arm64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/12.10.0/linux-arm64/develop-067e4fa304ef16b6efcc0ff2ff7b64c6ce65a01d/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 067e4fa Apr 17, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the linux x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/12.10.0/linux-x64/develop-067e4fa304ef16b6efcc0ff2ff7b64c6ce65a01d/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 067e4fa Apr 17, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the darwin arm64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/12.10.0/darwin-arm64/develop-067e4fa304ef16b6efcc0ff2ff7b64c6ce65a01d/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 067e4fa Apr 17, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the darwin x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/12.10.0/darwin-x64/develop-067e4fa304ef16b6efcc0ff2ff7b64c6ce65a01d/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 067e4fa Apr 17, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the win32 x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/12.10.0/win32-x64/develop-067e4fa304ef16b6efcc0ff2ff7b64c6ce65a01d/cypress.tgz

Please sign in to comment.