merged v5.0-release

.eslintignore

@@ -26,7 +26,7 @@ packages/server/lib/scaffold/plugins/index.js
 packages/server/lib/scaffold/support/index.js
 packages/server/lib/scaffold/support/commands.js
 packages/server/test/support/fixtures/projects/e2e/cypress/integration/stdout_exit_early_failing_spec.js
-packages/server/test/support/fixtures/projects/e2e/cypress/integration/browserify_typescript_failing_spec.ts
+packages/server/test/support/fixtures/projects/e2e/cypress/integration/typescript_syntax_error_spec.ts
 
 **/.projects
 **/*.d.ts

.eslintrc.json

@@ -6,7 +6,8 @@
     "plugin:@cypress/dev/general"
   ],
   "rules": {
-    "prefer-spread": "off"
+    "prefer-spread": "off",
+    "prefer-rest-params": "off"
   },
   "settings": {
     "react": {

circle.yml

@@ -155,14 +155,22 @@ commands:
       browser:
         description: browser shortname to target
         type: string
+      percy:
+        description: enable percy
+        type: boolean
+        default: false
     steps:
       - attach_workspace:
           at: ~/
       - run:
           command: |
+            cmd=$([[ <<parameters.percy>> == 'true' ]] && echo 'yarn percy exec --') || true
+
             CYPRESS_KONFIG_ENV=production \
             CYPRESS_RECORD_KEY=$PACKAGES_RECORD_KEY \
-            yarn workspace @packages/runner cypress:run --record --parallel --group runner-integration-<<parameters.browser>> --browser <<parameters.browser>>
+            PERCY_PARALLEL_NONCE=$CIRCLE_WORKFLOW_ID \
+            PERCY_PARALLEL_TOTAL=-1 \
+            $cmd yarn workspace @packages/runner cypress:run --record --parallel --group runner-integration-<<parameters.browser>> --browser <<parameters.browser>>
       - store_test_results:
           path: /tmp/cypress
       - store_artifacts:
@@ -237,6 +245,7 @@ commands:
           command: |
             git clone --depth 1 --no-single-branch https://github.com/cypress-io/<<parameters.repo>>.git /tmp/<<parameters.repo>>
             cd /tmp/<<parameters.repo>> && (git checkout $NEXT_DEV_VERSION || true)
+
   test-binary-against-repo:
     description: |
       Takes the built binary and NPM package, clones given example repo
@@ -271,6 +280,10 @@ commands:
         description: Whether to use wait-on to wait on a server to be booted
         type: string
         default: ""
+      server-start-command:
+        description: Server start command for repo
+        type: string
+        default: "npm start --if-present"
     steps:
       - attach_workspace:
           at: ~/
@@ -307,7 +320,7 @@ commands:
           command: npm run build --if-present
       - run:
           working_directory: /tmp/<<parameters.repo>>
-          command: npm start --if-present
+          command: <<parameters.server-start-command>>
           background: true
       - run:
           condition: <<parameters.wait-on>>
@@ -363,6 +376,17 @@ commands:
                 path: /tmp/<<parameters.repo>>/cypress/videos
       - store-npm-logs
 
+  wait-on-circle-jobs:
+    description: Polls certain Circle CI jobs until they finish
+    parameters:
+      job-names:
+        description: comma separated list of circle ci job names to wait for
+        type: string
+    steps:
+      - run:
+          name: "Waiting on Circle CI jobs: <<parameters.job-names>>"
+          command: node ./scripts/wait-on-circle-jobs.js --job-names="<<parameters.job-names>>"
+
 jobs:
   ## code checkout and yarn installs
   build:
@@ -452,6 +476,37 @@ jobs:
           command: node cli/bin/cypress info --dev
       - store-npm-logs
 
+  # a special job that keeps polling Circle and when all
+  # individual jobs are finished, it closes the Percy build
+  percy-finalize:
+    <<: *defaults
+    executor: cy-doc
+    parameters:
+      required_env_var:
+        type: env_var_name
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          # if this is an external pull request, the environment variables
+          # are NOT set for security reasons, thus no need to poll -
+          # and no need to finalize Percy, since there will be no visual tests
+          name: Check if <<parameters.required_env_var>> is set
+          command: |
+            if [[ -v <<parameters.required_env_var>> ]]; then
+              echo "Internal PR, good to go"
+            else
+              echo "This is an external PR, cannot access other services"
+              circleci-agent step halt
+            fi
+      - wait-on-circle-jobs:
+          job-names: >
+            desktop-gui-integration-tests-2x,
+            desktop-gui-component-tests,
+            cli-visual-tests,
+            runner-integration-tests-chrome,
+      - run: npx percy finalize --all
+
   cli-visual-tests:
     <<: *defaults
     parallelism: 1
@@ -761,6 +816,7 @@ jobs:
     steps:
       - run-runner-integration-tests:
           browser: chrome
+          percy: true
 
   runner-integration-tests-firefox:
     <<: *defaults
@@ -939,13 +995,6 @@ jobs:
           command: node index.js
           working_directory: packages/launcher
 
-  percy-finalize:
-    <<: *defaults
-    steps:
-      - run:
-          name: "finalizes percy builds"
-          command: npx percy finalize --all
-
   build-binary:
     <<: *defaults
     shell: /bin/bash --login
@@ -1447,6 +1496,11 @@ jobs:
           command: npm run test:ci
           pull_request_id: 515
           folder: examples/fundamentals__typescript
+      - test-binary-against-repo:
+          repo: cypress-example-recipes
+          command: npm test
+          pull_request_id: 513
+          folder: examples/fundamentals__module-api-wrap
 
   "test-binary-against-kitchensink":
     <<: *defaults
@@ -1516,6 +1570,16 @@ jobs:
           browser: firefox
           command: "npm run cypress:run"
 
+  "test-binary-against-cypress-realworld-app":
+    <<: *defaults
+    steps:
+      - test-binary-against-repo:
+          repo: cypress-realworld-app
+          browser: chrome
+          server-start-command: "npm run start:ci"
+          command: "npm run cypress:run"
+          wait-on: http://localhost:3000
+
   test-binary-as-specific-user:
     <<: *defaults
     steps:
@@ -1571,6 +1635,11 @@ linux-workflow: &linux-workflow
         name: Linux lint
         requires:
           - build
+    - percy-finalize:
+        context: test-runner:poll-circle-workflow
+        required_env_var: PERCY_TOKEN # skips job if not defined (external PR)
+        requires:
+          - build
     - lint-types:
         requires:
           - build
@@ -1702,12 +1771,6 @@ linux-workflow: &linux-workflow
         requires:
           - build
 
-    - percy-finalize:
-        requires:
-          - desktop-gui-integration-tests-2x
-          - desktop-gui-component-tests
-          - cli-visual-tests
-
     # various testing scenarios, like building full binary
     # and testing it on a real project
     - test-against-staging:
@@ -1744,6 +1807,7 @@ linux-workflow: &linux-workflow
           branches:
             only:
               - develop
+              - test-retries
         requires:
           - build-npm-package
     - build-binary:
@@ -1755,6 +1819,7 @@ linux-workflow: &linux-workflow
           branches:
             only:
               - develop
+              - test-retries
         requires:
           - build-binary
     - test-npm-module-on-minimum-node-version:
@@ -1811,6 +1876,7 @@ linux-workflow: &linux-workflow
           branches:
             only:
               - develop
+              - test-retries
         requires:
           - upload-npm-package
           - upload-binary
@@ -1857,6 +1923,16 @@ linux-workflow: &linux-workflow
         <<: *testBinaryFirefox
     - test-binary-against-piechopper-firefox:
         <<: *testBinaryFirefox
+    - test-binary-against-cypress-realworld-app:
+        filters:
+          branches:
+            only:
+              - develop
+              - kevin-v5.0-release-rwa
+              - v5.0-release
+        requires:
+          - build-npm-package
+          - build-binary
 
     - test-binary-as-specific-user:
         name: "test binary as a non-root user"

cli/schema/cypress.schema.json

@@ -243,6 +243,18 @@
       "type": "boolean",
       "default": false,
       "description": "Polyfills `window.fetch` to enable Network spying and stubbing"
+    },
+    "retries": {
+      "type": [
+        "object",
+        "number",
+        "null"
+      ],
+      "default": {
+        "runMode": 0,
+        "openMode": 0
+      },
+      "description": "The number of times to retry a failing. Can be configured to apply only in runMode or openMode"
     }
   }
 }

cli/types/cypress-npm-api.d.ts

@@ -7,6 +7,12 @@
 // but for now describe it as an ambient module
 
 declare namespace CypressCommandLine {
+  type HookName = 'before' | 'beforeEach' | 'afterEach' | 'after'
+  interface TestError {
+    name: string
+    message: string
+    stack: string
+  }
   /**
    * All options that one can pass to "cypress.run"
    * @see https://on.cypress.io/module-api#cypress-run
@@ -154,39 +160,36 @@ declare namespace CypressCommandLine {
   // small utility types to better express meaning of other types
   type dateTimeISO = string
   type ms = number
-  type hookId = string
-  type testId = string
   type pixels = number
 
   /**
    * Cypress single test result
    */
   interface TestResult {
-    testId: testId
     title: string[]
     state: string
     body: string
-    /**
-     * Error stack string if there is an error
-     */
-    stack: string | null
-    /**
-     * Error message if there is an error
+     /**
+     * Error string as it's presented in console if the test fails
      */
-    error: string | null
-    timings: any
-    failedFromHookId: hookId | null
-    wallClockStartedAt: dateTimeISO
-    wallClockDuration: ms
+    displayError: string | null
+    attempts: AttemptResult[]
+  }
+
+  interface AttemptResult {
+    state: string
+    error: TestError | null
+    startedAt: dateTimeISO
+    duration: ms
     videoTimestamp: ms
+    screenshots: ScreenshotInformation[]
   }
 
   /**
    * Information about a single "before", "beforeEach", "afterEach" and "after" hook.
   */
   interface HookInformation {
-    hookId: hookId
-    hookName: 'before' | 'beforeEach' | 'afterEach' | 'after'
+    hookName: HookName
     title: string[]
     body: string
   }
@@ -195,9 +198,7 @@ declare namespace CypressCommandLine {
    * Information about a single screenshot.
    */
   interface ScreenshotInformation {
-    screenshotId: string
     name: string
-    testId: testId
     takenAt: dateTimeISO
     /**
      * Absolute path to the saved image
@@ -221,9 +222,9 @@ declare namespace CypressCommandLine {
       pending: number
       skipped: number
       failures: number
-      wallClockStartedAt: dateTimeISO
-      wallClockEndedAt: dateTimeISO
-      wallClockDuration: ms
+      startedAt: dateTimeISO
+      endedAt: dateTimeISO
+      duration: ms
     },
     /**
      * Reporter name like "spec"
@@ -238,7 +239,6 @@ declare namespace CypressCommandLine {
     tests: TestResult[]
     error: string | null
     video: string | null
-    screenshots: ScreenshotInformation[]
     /**
      * information about the spec test file.
     */

cli/types/cypress.d.ts

@@ -334,6 +334,11 @@ declare namespace Cypress {
      */
     getFirefoxGcInterval(): number | null | undefined
 
+    /**
+     * @returns the number of test retries currently enabled for the run
+     */
+    getTestRetries(): number | null
+
     /**
      * Checks if a variable is a valid instance of `cy` or a `cy` chainable.
      *
@@ -2563,6 +2568,13 @@ declare namespace Cypress {
      * the `includeShadowDom` option to some DOM commands.
      */
     experimentalShadowDomSupport: boolean
+    /**
+     * Number of times to retry a failed test.
+     * If a number is set, tests will retry in both runMode and openMode.
+     * To enable test retries only in runMode, set e.g. `{ openMode: null, runMode: 2 }`
+     * @default null
+     */
+    retries: Nullable<number | {runMode: Nullable<number>, openMode: Nullable<number>}>
   }
 
   interface TestConfigOverrides extends Partial<Pick<ConfigOptions, 'baseUrl' | 'defaultCommandTimeout' | 'taskTimeout' | 'animationDistanceThreshold' | 'waitForAnimations' | 'viewportHeight' | 'viewportWidth' | 'requestTimeout' | 'execTimeout' | 'env' | 'responseTimeout'>> {

package.json

@@ -136,6 +136,7 @@
     "fs-extra": "8.1.0",
     "gift": "0.10.2",
     "globby": "10.0.1",
+    "got": "11.5.1",
     "gulp": "4.0.2",
     "gulp-awspublish": "4.0.0",
     "gulp-debug": "4.0.0",