[test optimization] Do not enable Test Optimization plugins when isCiVisibility is false (#6404)

Author: juan-fernandez

integration-tests/ci-visibility/test-optimization-wrong-init-cucumber/sum.feature

@@ -0,0 +1,9 @@
+Feature: Sum calculation
+  As a developer
+  I want to test sum functionality
+  So that I can verify basic math operations
+
+  Scenario: Add two numbers
+    Given I have two numbers 1 and 2
+    When I add them together
+    Then the result should be 3

integration-tests/ci-visibility/test-optimization-wrong-init-cucumber/support/steps.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const tracer = require('dd-trace')
+const { Given, When, Then } = require('@cucumber/cucumber')
+const { expect } = require('chai')
+
+let num1, num2, result
+
+Given('I have two numbers {int} and {int}', function (first, second) {
+  tracer.startSpan('custom').finish()
+  num1 = first
+  num2 = second
+})
+
+When('I add them together', function () {
+  result = num1 + num2
+})
+
+Then('the result should be {int}', function (expected) {
+  expect(result).to.equal(expected)
+})

integration-tests/ci-visibility/test-optimization-wrong-init.spec.js

@@ -0,0 +1,131 @@
+'use strict'
+
+const { once } = require('node:events')
+const assert = require('node:assert')
+const { exec } = require('child_process')
+
+const { createSandbox, getCiVisAgentlessConfig } = require('../helpers')
+const { FakeCiVisIntake } = require('../ci-visibility-intake')
+const { NODE_MAJOR } = require('../../version')
+
+// no playwright because it has no programmatic API
+// no cypress because it's not a proper dd-trace plugin
+const testFrameworks = [
+  {
+    testFramework: 'mocha',
+    command: 'node ./ci-visibility/test-optimization-wrong-init/run-mocha.js',
+    expectedOutput: '1 passing'
+  },
+  {
+    testFramework: 'jest',
+    command: 'node ./ci-visibility/test-optimization-wrong-init/run-jest.js',
+    expectedOutput: 'PASS ci-visibility/test-optimization-wrong-init/sum-wrong-init-test.js'
+  },
+  {
+    testFramework: 'vitest',
+    command: 'node ./ci-visibility/test-optimization-wrong-init/run-vitest.mjs',
+    expectedOutput: '1 passed',
+    extraTestContext: {
+      TEST_DIR: 'ci-visibility/test-optimization-wrong-init/vitest-sum-wrong-init*',
+      NODE_OPTIONS: '--import dd-trace/register.js'
+    }
+  },
+  {
+    testFramework: 'cucumber',
+    command: './node_modules/.bin/cucumber-js ci-visibility/test-optimization-wrong-init-cucumber/*.feature',
+    expectedOutput: '1 passed',
+    extraTestContext: {
+      NODE_OPTIONS: '-r dd-trace/init'
+    }
+  }
+]
+
+testFrameworks.forEach(({ testFramework, command, expectedOutput, extraTestContext }) => {
+  describe(`test optimization wrong init for ${testFramework}`, () => {
+    let sandbox, cwd, receiver, childProcess, processOutput
+
+    // cucumber does not support Node.js@18 anymore
+    if (NODE_MAJOR <= 18 && testFramework === 'cucumber') return
+
+    before(async () => {
+      const testFrameworks = ['jest', 'mocha', 'vitest']
+
+      // Remove once we drop support for Node.js@18
+      if (NODE_MAJOR > 18) {
+        testFrameworks.push('@cucumber/cucumber')
+      }
+
+      sandbox = await createSandbox(testFrameworks, true)
+      cwd = sandbox.folder
+    })
+
+    after(async () => {
+      await sandbox.remove()
+    })
+
+    beforeEach(async function () {
+      processOutput = ''
+      receiver = await new FakeCiVisIntake().start()
+    })
+
+    afterEach(async () => {
+      childProcess.kill()
+      await receiver.stop()
+    })
+
+    it('does not initialize test optimization plugins if Test Optimization mode is not enabled', async () => {
+      const eventsPromise = receiver
+        .gatherPayloadsMaxTimeout(({ url }) => url === '/v0.4/traces', (tracesRequests) => {
+          const spans = tracesRequests.flatMap(trace => trace.payload).flatMap(request => request)
+          const includesTestOptimizationSpans = spans.some(span => span.name.includes(testFramework))
+          if (spans.length === 0) {
+            throw new Error('No spans were sent')
+          }
+          if (includesTestOptimizationSpans) {
+            throw new Error('Test Optimization spans should not be sent')
+          }
+        }, 10000)
+
+      const envVars = getCiVisAgentlessConfig(receiver.port)
+
+      const {
+        NODE_OPTIONS, // we don't want to initialize dd-trace in Test Optimization mode
+        ...restEnvVars
+      } = envVars
+
+      childProcess = exec(command,
+        {
+          cwd,
+          env: {
+            ...process.env,
+            ...restEnvVars,
+            DD_TRACE_DISABLED_INSTRUMENTATIONS: 'child_process',
+            DD_TRACE_DEBUG: '1',
+            ...extraTestContext
+          },
+          stdio: 'pipe'
+        }
+      )
+
+      childProcess.stderr.on('data', (chunk) => {
+        processOutput += chunk.toString()
+      })
+
+      childProcess.stdout.on('data', (chunk) => {
+        processOutput += chunk.toString()
+      })
+
+      await Promise.all([
+        once(childProcess, 'exit'),
+        eventsPromise
+      ])
+
+      assert.ok(
+        processOutput.includes(
+          `Plugin "${testFramework}" is not initialized because Test Optimization mode is not enabled.`
+        )
+      )
+      assert.ok(processOutput.includes(expectedOutput))
+    })
+  })
+})

integration-tests/ci-visibility/test-optimization-wrong-init/run-jest.js

@@ -0,0 +1,20 @@
+'use strict'
+
+require('dd-trace').init({
+  service: 'sum-service-tests',
+})
+
+const { runCLI } = require('jest')
+
+async function main () {
+  const projectRoot = process.cwd()
+
+  await runCLI(
+    {
+      testMatch: ['**/sum-wrong-init-test.js'],
+    },
+    [projectRoot]
+  )
+}
+
+main()

integration-tests/ci-visibility/test-optimization-wrong-init/run-mocha.js

@@ -0,0 +1,27 @@
+'use strict'
+
+require('dd-trace').init({
+  service: 'sum-service-tests',
+})
+
+const Mocha = require('mocha')
+
+async function main () {
+  const mocha = new Mocha()
+
+  mocha.addFile(require.resolve('./sum-wrong-init-test.js'))
+
+  await new Promise((resolve, reject) => {
+    mocha.run(failures => {
+      if (failures > 0) {
+        reject(new Error(`${failures} test(s) failed.`))
+      } else {
+        resolve()
+      }
+    })
+  })
+}
+
+main().catch(() => {
+  process.exit(1)
+})

integration-tests/ci-visibility/test-optimization-wrong-init/run-vitest.mjs

@@ -0,0 +1,23 @@
+import tracer from 'dd-trace'
+import { startVitest } from 'vitest/node'
+
+// The tracer needs to be initialized both in the main process and in the worker process.
+// This is normally taken care of by using NODE_OPTIONS, but we can't set
+// flushInterval through an env var
+tracer.init({
+  flushInterval: 0
+})
+
+async function main () {
+  return startVitest(
+    'test',
+    [],
+    { watch: false },
+    {},
+    {}
+  )
+}
+
+main().catch(() => {
+  process.exit(1)
+})

integration-tests/ci-visibility/test-optimization-wrong-init/sum-wrong-init-test.js

@@ -0,0 +1,12 @@
+'use strict'
+
+const assert = require('node:assert')
+const tracer = require('dd-trace')
+
+tracer.trace('sum.test', { resource: 'sum.test.js' }, () => {
+  describe('sum', () => {
+    it('should return the sum of two numbers', () => {
+      assert.equal(1 + 2, 3)
+    })
+  })
+})

integration-tests/ci-visibility/test-optimization-wrong-init/vitest-sum-wrong-init-test.mjs

@@ -0,0 +1,17 @@
+import tracer from 'dd-trace'
+import { describe, it, expect } from 'vitest'
+
+// The tracer needs to be initialized both in the main process and in the worker process.
+// This is normally taken care of by using NODE_OPTIONS, but we can't set
+// flushInterval through an env var
+tracer.init({
+  flushInterval: 0
+})
+
+tracer.trace('sum.test', { resource: 'sum.test.js' }, () => {
+  describe('sum', () => {
+    it('should return the sum of two numbers', async () => {
+      expect(1 + 2).toBe(3)
+    })
+  })
+})

packages/datadog-instrumentations/src/cucumber.js

@@ -599,6 +599,9 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin
 // Handles EFD in both the main process and the worker process.
 function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = false, isWorker = false) {
   return async function () {
+    if (!testSuiteFinishCh.hasSubscribers) {
+      return runTestCaseFunction.apply(this, arguments)
+    }
     const pickle = isNewerCucumberVersion
       ? arguments[0].pickle
       : this.eventDataCollector.getPickle(arguments[0])
@@ -754,6 +757,9 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa
 
 function getWrappedParseWorkerMessage (parseWorkerMessageFunction, isNewVersion) {
   return function (worker, message) {
+    if (!testSuiteFinishCh.hasSubscribers) {
+      return parseWorkerMessageFunction.apply(this, arguments)
+    }
     // If the message is an array, it's a dd-trace message, so we need to stop cucumber processing,
     // or cucumber will throw an error
     // TODO: identify the message better

packages/datadog-plugin-cucumber/test/index.spec.js

@@ -70,7 +70,8 @@ describe('Plugin', function () {
         .get('/')
         .reply(200, 'OK')
 
-      return agent.load(['cucumber', 'http']).then(() => {
+      return agent.load(
+        ['cucumber', 'http'], { service: 'test' }, { isCiVisibility: true }).then(() => {
         Cucumber = proxyquire(`../../../versions/@cucumber/cucumber@${version}`, {}).get()
       })
     })