[test optimization] Move plugin cypress tests to integration tests (#6876)

Author: juan-fernandez

.github/workflows/test-optimization.yml

@@ -194,28 +194,6 @@ jobs:
         with:
           dd_api_key: ${{ secrets.DD_API_KEY }}
 
-  # TODO: fix performance issues and test more Node versions
-  plugin-cypress:
-    runs-on: ubuntu-latest
-    env:
-      PLUGINS: cypress
-    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      - uses: ./.github/actions/testagent/start
-      - uses: ./.github/actions/node/latest
-      - uses: ./.github/actions/install
-      - run: yarn test:plugins:ci
-      - if: always()
-        uses: ./.github/actions/testagent/logs
-        with:
-          suffix: plugins-${{ github.job }}
-      - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
-      - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a  # v2.0.0
-        if: always()
-        with:
-          api_key: ${{ secrets.DD_API_KEY }}
-          service: dd-trace-js-tests
-
   # TODO: fix performance issues and test more Node versions
   plugin-jest:
     runs-on: ubuntu-latest

integration-tests/CODEOWNERS

@@ -1,3 +1,4 @@
 # CODEOWNERS for testing purposes
 ci-visibility/subproject @datadog-dd-trace-js
 ci-visibility/ @datadog-dd-trace-js
+cypress/ @datadog-dd-trace-js

integration-tests/cypress/cypress.spec.js

@@ -23,7 +23,9 @@ const {
   TEST_STATUS,
   TEST_COMMAND,
   TEST_MODULE,
+  TEST_FRAMEWORK,
   TEST_FRAMEWORK_VERSION,
+  TEST_TYPE,
   TEST_TOOLCHAIN,
   TEST_CODE_COVERAGE_ENABLED,
   TEST_ITR_SKIPPING_ENABLED,
@@ -63,7 +65,7 @@ const {
   TEST_IS_MODIFIED
 } = require('../../packages/dd-trace/src/plugins/util/test')
 const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
-const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
+const { ERROR_MESSAGE, ERROR_TYPE, COMPONENT } = require('../../packages/dd-trace/src/constants')
 const { DD_MAJOR, NODE_MAJOR } = require('../../version')
 
 const RECEIVER_STOP_TIMEOUT = 20000
@@ -129,10 +131,10 @@ moduleTypes.forEach(({
       testCommand = testCommand(version)
     }
 
+    // cypress-fail-fast is required as an incompatible plugin
     useSandbox([`cypress@${version}`, 'cypress-fail-fast@7.1.0'], true)
 
     before(async () => {
-      // cypress-fail-fast is required as an incompatible plugin
       cwd = sandboxCwd()
 
       const { NODE_OPTIONS, ...restOfEnv } = process.env
@@ -176,6 +178,95 @@ moduleTypes.forEach(({
       }
     })
 
+    it('instruments tests with the APM protocol (old agents)', async () => {
+      receiver.setInfoResponse({ endpoints: [] })
+
+      const receiverPromise = receiver
+        .gatherPayloadsMaxTimeout(({ url }) => url === '/v0.4/traces', (payloads) => {
+          const testSpans = payloads.flatMap(({ payload }) => payload.flatMap(trace => trace))
+
+          const passedTestSpan = testSpans.find(span =>
+            span.resource === 'cypress/e2e/basic-pass.js.basic pass suite can pass'
+          )
+          const failedTestSpan = testSpans.find(span =>
+            span.resource === 'cypress/e2e/basic-fail.js.basic fail suite can fail'
+          )
+
+          assert.exists(passedTestSpan, 'passed test span should exist')
+          assert.equal(passedTestSpan.name, 'cypress.test')
+          assert.equal(passedTestSpan.resource, 'cypress/e2e/basic-pass.js.basic pass suite can pass')
+          assert.equal(passedTestSpan.type, 'test')
+          assert.equal(passedTestSpan.meta[TEST_STATUS], 'pass')
+          assert.equal(passedTestSpan.meta[TEST_NAME], 'basic pass suite can pass')
+          assert.equal(passedTestSpan.meta[TEST_SUITE], 'cypress/e2e/basic-pass.js')
+          assert.equal(passedTestSpan.meta[TEST_FRAMEWORK], 'cypress')
+          assert.equal(passedTestSpan.meta[TEST_TYPE], 'browser')
+          assert.exists(passedTestSpan.meta[TEST_SOURCE_FILE])
+          assert.include(passedTestSpan.meta[TEST_SOURCE_FILE], 'cypress/e2e/basic-pass.js')
+          assert.exists(passedTestSpan.meta[TEST_FRAMEWORK_VERSION])
+          assert.exists(passedTestSpan.meta[COMPONENT])
+          assert.exists(passedTestSpan.metrics[TEST_SOURCE_START])
+          assert.equal(passedTestSpan.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
+          assert.equal(passedTestSpan.meta.customTag, 'customValue')
+          assert.equal(passedTestSpan.meta.addTagsBeforeEach, 'customBeforeEach')
+          assert.equal(passedTestSpan.meta.addTagsAfterEach, 'customAfterEach')
+
+          assert.exists(failedTestSpan, 'failed test span should exist')
+          assert.equal(failedTestSpan.name, 'cypress.test')
+          assert.equal(failedTestSpan.resource, 'cypress/e2e/basic-fail.js.basic fail suite can fail')
+          assert.equal(failedTestSpan.type, 'test')
+          assert.equal(failedTestSpan.meta[TEST_STATUS], 'fail')
+          assert.equal(failedTestSpan.meta[TEST_NAME], 'basic fail suite can fail')
+          assert.equal(failedTestSpan.meta[TEST_SUITE], 'cypress/e2e/basic-fail.js')
+          assert.equal(failedTestSpan.meta[TEST_FRAMEWORK], 'cypress')
+          assert.equal(failedTestSpan.meta[TEST_TYPE], 'browser')
+          assert.exists(failedTestSpan.meta[TEST_SOURCE_FILE])
+          assert.include(failedTestSpan.meta[TEST_SOURCE_FILE], 'cypress/e2e/basic-fail.js')
+          assert.exists(failedTestSpan.meta[TEST_FRAMEWORK_VERSION])
+          assert.exists(failedTestSpan.meta[COMPONENT])
+          assert.exists(failedTestSpan.meta[ERROR_MESSAGE])
+          assert.include(failedTestSpan.meta[ERROR_MESSAGE], 'expected')
+          assert.exists(failedTestSpan.meta[ERROR_TYPE])
+          assert.exists(failedTestSpan.metrics[TEST_SOURCE_START])
+          assert.equal(passedTestSpan.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
+          assert.equal(failedTestSpan.meta.customTag, 'customValue')
+          assert.equal(failedTestSpan.meta.addTagsBeforeEach, 'customBeforeEach')
+          assert.equal(failedTestSpan.meta.addTagsAfterEach, 'customAfterEach')
+          // Tags added after failure should not be present because test failed
+          assert.notProperty(failedTestSpan.meta, 'addTagsAfterFailure')
+        }, 60000)
+
+      const {
+        NODE_OPTIONS,
+        ...restEnvVars
+      } = getCiVisEvpProxyConfig(receiver.port)
+
+      const specToRun = 'cypress/e2e/basic-*.js'
+
+      // For Cypress 6.7.0, we need to override the --spec flag that's hardcoded in testCommand
+      const command = version === '6.7.0'
+        ? `./node_modules/.bin/cypress run --config-file cypress-config.json --spec "${specToRun}"`
+        : testCommand
+
+      childProcess = exec(
+        command,
+        {
+          cwd,
+          env: {
+            ...restEnvVars,
+            CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
+            SPEC_PATTERN: specToRun
+          },
+          stdio: 'pipe'
+        }
+      )
+
+      await Promise.all([
+        once(childProcess, 'exit'),
+        receiverPromise
+      ])
+    })
+
     if (version === '6.7.0') {
       // to be removed when we drop support for cypress@6.7.0
       it('logs a warning if using a deprecated version of cypress', async () => {

integration-tests/cypress/e2e/basic-fail.js

@@ -0,0 +1,19 @@
+/* eslint-disable */
+describe('basic fail suite', () => {
+  beforeEach(() => {
+    cy.visit('/')
+    cy.task('dd:addTags', { addTagsBeforeEach: 'customBeforeEach' })
+  })
+
+  afterEach(() => {
+    cy.task('dd:addTags', { addTagsAfterEach: 'customAfterEach' })
+  })
+
+  it('can fail', () => {
+    cy.task('dd:addTags', { customTag: 'customValue' })
+    cy.get('.hello-world')
+      .should('have.text', 'Hello warld')
+    cy.task('dd:addTags', { addTagsAfterFailure: 'customAfterFailure' })
+  })
+})
+

integration-tests/cypress/e2e/basic-pass.js

@@ -0,0 +1,18 @@
+/* eslint-disable */
+describe('basic pass suite', () => {
+  beforeEach(() => {
+    cy.visit('/')
+    cy.task('dd:addTags', { addTagsBeforeEach: 'customBeforeEach' })
+  })
+
+  afterEach(() => {
+    cy.task('dd:addTags', { addTagsAfterEach: 'customAfterEach' })
+  })
+
+  it('can pass', () => {
+    cy.task('dd:addTags', { customTag: 'customValue' })
+    cy.get('.hello-world')
+      .should('have.text', 'Hello World')
+  })
+})
+