Skip to content

Commit

Permalink
ensure system test snapshots
Browse files Browse the repository at this point in the history
  • Loading branch information
@cacieprins
cacieprins committed Mar 26, 2024
1 parent 8aaacca commit 967e876
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 12 deletions.
5 changes: 4 additions & 1 deletion packages/server/lib/cloud/api/putProtocolArtifact.ts
View file
@@ -1,10 +1,13 @@
import fsAsync from 'fs/promises'
import fs from 'fs'

import Debug from 'debug'
import { uploadStream, geometricRetry } from '../upload/uploadStream'
import { StreamActivityMonitor } from '../upload/StreamActivityMonitor'

const debug = Debug('cypress:server:cloud:api:protocol-artifact')

export const putProtocolArtifact = async (artifactPath: string, maxFileSize: number, destinationUrl: string) => {
debug(`Atttempting to upload Test Replay archive from ${artifactPath} to ${destinationUrl})`)
const { size } = await fsAsync.stat(artifactPath)

if (size > maxFileSize) {
Expand Down
8 changes: 7 additions & 1 deletion packages/server/lib/modes/record.js
View file
Expand Up @@ -373,9 +373,15 @@ const uploadArtifactBatch = async (artifacts, protocolManager, quiet) => {
if (key === 'protocol') {
let { error, errorStack, allErrors } = report

debug('reducing uploadResuls: %O', report)
if (allErrors) {
error = `Failed to upload after ${allErrors.length} attempts. Errors: ${allErrors.map((error) => error.message).join(', ')}`
error = allErrors.length > 1 ?
`Failed to upload Test Replay after ${allErrors.length} attempts. Errors: ${allErrors.map((error) => error.message).join(', ')}` :
`Failed to upload Test Replay: ${allErrors[0]}`

errorStack = allErrors.map((error) => error.stack).join(', ')
} else if (error) {
error = `Failed to upload Test Replay: ${error}`
}

return skipped && !report.error ? acc : {
Expand Down
2 changes: 1 addition & 1 deletion packages/server/lib/util/print-run.ts
View file
Expand Up @@ -600,7 +600,7 @@ export const printPendingArtifactUpload = <T extends ArtifactLike> (artifact: T,
process.stdout.write(`- ${formatFileSize(Number(artifact.fileSize))}`)
}

if (artifact.filePath) {
if (artifact.filePath && artifact.reportKey !== 'protocol') {
process.stdout.write(` ${formatPath(artifact.filePath, undefined, 'cyan')}`)
}

Expand Down
90 changes: 87 additions & 3 deletions system-tests/__snapshots__/record_spec.js
View file
Expand Up @@ -3047,7 +3047,7 @@ exports['e2e record capture-protocol enabled protocol runtime errors db size too
(Uploaded Cloud Artifacts)
- Screenshot - Done Uploading 1 kB in Xm, Ys ZZ.ZZms 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png
- Test Replay - Failed Uploading after Xm, Ys ZZ.ZZms 2/2 - Spec recording too large: db is 1024 bytes, limit is 200 bytes
- Test Replay - Failed Uploading after Xm, Ys ZZ.ZZms 2/2 - Spec recording too large: artifact is 1024 bytes, limit is 200 bytes
====================================================================================================
Expand Down Expand Up @@ -3658,7 +3658,7 @@ exports['e2e record capture-protocol enabled passing retrieves the capture proto
`

exports['capture-protocol api errors upload 500 - retries 3 times and fails continues 1'] = `
exports['capture-protocol api errors upload 500 - does not retry continues 1'] = `
====================================================================================================
Expand Down Expand Up @@ -3742,7 +3742,7 @@ exports['capture-protocol api errors upload 500 - retries 3 times and fails cont
`

exports['capture-protocol api errors upload 500 - retries 2 times and succeeds on the last call continues 1'] = `
exports['capture-protocol api errors upload 503 - retries 2 times and succeeds on the last call continues 1'] = `
====================================================================================================
Expand Down Expand Up @@ -3825,3 +3825,87 @@ exports['capture-protocol api errors upload 500 - retries 2 times and succeeds o
`

exports['capture-protocol api errors upload 503 - tries 3 times and fails continues 1'] = `
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 1 found (record_pass.cy.js) │
│ Searched: cypress/e2e/record_pass* │
│ Params: Tag: false, Group: false, Parallel: false │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: record_pass.cy.js (1 of 1)
Estimated: X second(s)
record pass
✓ passes
- is pending
1 passing
1 pending
(Results)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Tests: 2 │
│ Passing: 1 │
│ Failing: 0 │
│ Pending: 1 │
│ Skipped: 0 │
│ Screenshots: 1 │
│ Video: false │
│ Duration: X seconds │
│ Estimated: X second(s) │
│ Spec Ran: record_pass.cy.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
(Screenshots)
- /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022)
(Uploading Cloud Artifacts)
- Video - Nothing to upload
- Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png
- Test Replay
Uploading Cloud Artifacts: . . . . .
(Uploaded Cloud Artifacts)
- Screenshot - Done Uploading 1 kB in Xm, Ys ZZ.ZZms 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png
- Test Replay - Failed Uploading after Xm, Ys ZZ.ZZms 2/2 - 503 Service Unavailable (http://localhost:1234/capture-protocol/upload/?x-amz-credential=XXXXXXXX&x-amz-signature=XXXXXXXXXXXXX)
====================================================================================================
(Run Finished)
Spec Tests Passing Failing Pending Skipped
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
✔ All specs passed! XX:XX 2 1 - 1 -
───────────────────────────────────────────────────────────────────────────────────────────────────────
Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12
`
38 changes: 32 additions & 6 deletions system-tests/test/record_spec.js
View file
Expand Up @@ -2524,15 +2524,15 @@ describe('capture-protocol api errors', () => {

enableCaptureProtocol()

const stubbedServerWithErrorOn = (endpoint, numberOfFailuresBeforeSuccess = Number.MAX_SAFE_INTEGER) => {
const stubbedServerWithErrorOn = (endpoint, numberOfFailuresBeforeSuccess = Number.MAX_SAFE_INTEGER, status = 500, statusText = 'Internal Server Error') => {
let failures = 0

return setupStubbedServer(createRoutes({
[endpoint]: {
res: (req, res) => {
if (failures < numberOfFailuresBeforeSuccess) {
failures += 1
res.status(500).send('500 - Internal Server Error')
res.status(status).send(`${status} - ${statusText}`)
} else {
routeHandlers[endpoint].res(req, res)
}
Expand All @@ -2541,7 +2541,7 @@ describe('capture-protocol api errors', () => {
}))
}

describe('upload 500 - retries 3 times and fails', () => {
describe('upload 500 - does not retry', () => {
stubbedServerWithErrorOn('putCaptureProtocolUpload')
it('continues', function () {
process.env.API_RETRY_INTERVALS = '1000'
Expand All @@ -2561,16 +2561,42 @@ describe('capture-protocol api errors', () => {

expect(artifactReport?.protocol).to.exist()
expect(artifactReport?.protocol?.error).to.equal(
'Failed to upload after 3 attempts. Errors: 500 Internal Server Error (http://localhost:1234/capture-protocol/upload/?x-amz-credential=XXXXXXXX&x-amz-signature=XXXXXXXXXXXXX), 500 Internal Server Error (http://localhost:1234/capture-protocol/upload/?x-amz-credential=XXXXXXXX&x-amz-signature=XXXXXXXXXXXXX), 500 Internal Server Error (http://localhost:1234/capture-protocol/upload/?x-amz-credential=XXXXXXXX&x-amz-signature=XXXXXXXXXXXXX)',
'Failed to upload Test Replay: 500 Internal Server Error (http://localhost:1234/capture-protocol/upload/?x-amz-credential=XXXXXXXX&x-amz-signature=XXXXXXXXXXXXX)',
)
})
})
})

describe('upload 503 - tries 3 times and fails', () => {
stubbedServerWithErrorOn('putCaptureProtocolUpload', Number.MAX_SAFE_INTEGER, 503, 'Service Unavailable')
it('continues', function () {
process.env.API_RETRY_INTERVALS = '1000'

return systemTests.exec(this, {
key: 'f858a2bc-b469-4e48-be67-0876339ee7e1',
configFile: 'cypress-with-project-id.config.js',
spec: 'record_pass*',
record: true,
snapshot: true,
}).then(() => {
const urls = getRequestUrls()

expect(urls).to.include.members([`PUT /instances/${instanceId}/artifacts`])

const artifactReport = getRequests().find(({ url }) => url === `PUT /instances/${instanceId}/artifacts`)?.body

expect(artifactReport?.protocol).to.exist()
expect(artifactReport?.protocol?.error).to.equal(
'Failed to upload Test Replay after 3 attempts. Errors: 503 Service Unavailable (http://localhost:1234/capture-protocol/upload/?x-amz-credential=XXXXXXXX&x-amz-signature=XXXXXXXXXXXXX), 503 Service Unavailable (http://localhost:1234/capture-protocol/upload/?x-amz-credential=XXXXXXXX&x-amz-signature=XXXXXXXXXXXXX), 503 Service Unavailable (http://localhost:1234/capture-protocol/upload/?x-amz-credential=XXXXXXXX&x-amz-signature=XXXXXXXXXXXXX)',
)

expect(artifactReport?.protocol?.errorStack).to.exist().and.not.to.be.empty()
})
})
})

describe('upload 500 - retries 2 times and succeeds on the last call', () => {
stubbedServerWithErrorOn('putCaptureProtocolUpload', 2)
describe('upload 503 - retries 2 times and succeeds on the last call', () => {
stubbedServerWithErrorOn('putCaptureProtocolUpload', 2, 503, 'Internal Server Error')

let archiveFile = ''

Expand Down

0 comments on commit 967e876

Please sign in to comment.