Implement flaky tests retry for Cucumber JUnit 5

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/CucumberUtils.java

@@ -18,6 +18,10 @@
 
 public abstract class CucumberUtils {
 
+  static {
+    TestIdentifierFactory.register("cucumber", CucumberUtils::toTestIdentifier);
+  }
+
   public static @Nullable String getCucumberVersion(TestEngine cucumberEngine) {
     try (InputStream cucumberPropsStream =
         cucumberEngine
@@ -92,7 +96,8 @@ public static boolean isFeature(UniqueId uniqueId) {
     return "feature".equals(lastSegment.getType());
   }
 
-  public static TestIdentifier toTestIdentifier(TestDescriptor testDescriptor) {
+  public static TestIdentifier toTestIdentifier(
+      TestDescriptor testDescriptor, boolean includeParameters) {
     TestSource testSource = testDescriptor.getSource().orElse(null);
     if (testSource instanceof ClasspathResourceSource) {
       ClasspathResourceSource classpathResourceSource = (ClasspathResourceSource) testSource;

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java

@@ -36,6 +36,7 @@ public String instrumentedType() {
   @Override
   public String[] helperClassNames() {
     return new String[] {
+      packageName + ".TestIdentifierFactory",
       packageName + ".JUnitPlatformUtils",
       packageName + ".CucumberUtils",
       packageName + ".TestEventsHandlerHolder",

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberItrInstrumentation.java

@@ -54,7 +54,10 @@ public ElementMatcher<TypeDescription> hierarchyMatcher() {
   @Override
   public String[] helperClassNames() {
     return new String[] {
-      packageName + ".CucumberUtils", packageName + ".TestEventsHandlerHolder",
+      packageName + ".TestIdentifierFactory",
+      packageName + ".JUnitPlatformUtils",
+      packageName + ".CucumberUtils",
+      packageName + ".TestEventsHandlerHolder",
     };
   }
 
@@ -96,7 +99,7 @@ public static void shouldBeSkipped(
         }
       }
 
-      TestIdentifier test = CucumberUtils.toTestIdentifier(testDescriptor);
+      TestIdentifier test = CucumberUtils.toTestIdentifier(testDescriptor, true);
       if (test != null && TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skip(test)) {
         skipResult = Node.SkipResult.skip(InstrumentationBridge.ITR_SKIP_REASON);
       }

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy

@@ -18,26 +18,65 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
 
   def "test #testcaseName"() {
     setup:
-    givenSkippableTests(skippedTests)
     runFeatures(features, parallel)
 
     expect:
     assertSpansData(testcaseName, expectedTracesCount)
 
     where:
-    testcaseName                                 | features                                                                                                                         | parallel | expectedTracesCount | skippedTests
-    "test-succeed"                               | ["org/example/cucumber/calculator/basic_arithmetic.feature"]                                                                     | false    | 2                   | []
-    "test-scenario-outline-${version()}"         | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"]                                                       | false    | 5                   | []
-    "test-skipped"                               | ["org/example/cucumber/calculator/basic_arithmetic_skipped.feature"]                                                             | false    | 3                   | []
-    "test-skipped-feature"                       | ["org/example/cucumber/calculator/basic_arithmetic_skipped_feature.feature"]                                                     | false    | 3                   | []
-    "test-skipped-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_skipped.feature"]                                               | false    | 5                   | []
-    "test-itr-skipping"                          | ["org/example/cucumber/calculator/basic_arithmetic.feature"]                                                                     | false    | 2                   | [new TestIdentifier("Basic Arithmetic", "Addition", null, null)]
-    "test-itr-unskippable"                       | ["org/example/cucumber/calculator/basic_arithmetic_unskippable.feature"]                                                         | false    | 2                   | [new TestIdentifier("Basic Arithmetic", "Addition", null, null)]
-    "test-itr-unskippable-suite"                 | ["org/example/cucumber/calculator/basic_arithmetic_unskippable_suite.feature"]                                                   | false    | 2                   | [new TestIdentifier("Basic Arithmetic", "Addition", null, null)]
+    testcaseName                                 | features                                                                                                                         | parallel | expectedTracesCount
+    "test-succeed"                               | ["org/example/cucumber/calculator/basic_arithmetic.feature"]                                                                     | false    | 2
+    "test-scenario-outline-${version()}"         | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"]                                                       | false    | 5
+    "test-skipped"                               | ["org/example/cucumber/calculator/basic_arithmetic_skipped.feature"]                                                             | false    | 3
+    "test-skipped-feature"                       | ["org/example/cucumber/calculator/basic_arithmetic_skipped_feature.feature"]                                                     | false    | 3
+    "test-skipped-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_skipped.feature"]                                               | false    | 5
     "test-parallel"                              | [
       "org/example/cucumber/calculator/basic_arithmetic.feature",
       "org/example/cucumber/calculator/basic_arithmetic_skipped.feature"
-    ] | true     | 4                   | []
+    ] | true     | 4
+  }
+
+  def "test ITR #testcaseName"() {
+    setup:
+    givenSkippableTests(skippedTests)
+    runFeatures(features, false)
+
+    expect:
+    assertSpansData(testcaseName, expectedTracesCount)
+
+    where:
+    testcaseName                 | features                                                                       | expectedTracesCount | skippedTests
+    "test-itr-skipping"          | ["org/example/cucumber/calculator/basic_arithmetic.feature"]                   | 2                   | [new TestIdentifier("Basic Arithmetic", "Addition", null, null)]
+    "test-itr-unskippable"       | ["org/example/cucumber/calculator/basic_arithmetic_unskippable.feature"]       | 2                   | [new TestIdentifier("Basic Arithmetic", "Addition", null, null)]
+    "test-itr-unskippable-suite" | ["org/example/cucumber/calculator/basic_arithmetic_unskippable_suite.feature"] | 2                   | [new TestIdentifier("Basic Arithmetic", "Addition", null, null)]
+  }
+
+  def "test flaky retries #testcaseName"() {
+    setup:
+    givenFlakyTests(retriedTests)
+
+    runFeatures(features, false)
+
+    expect:
+    assertSpansData(testcaseName, expectedTracesCount)
+
+    where:
+    testcaseName                                | features                                                                          | expectedTracesCount | retriedTests
+    "test-failed"                               | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"]               | 2                   | []
+    "test-retry-failed"                         | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"]               | 5                   | [
+      new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null, null)
+    ]
+    "test-failed-then-succeed"                  | ["org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature"]  | 4                   | [
+      new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature:Basic Arithmetic", "Addition", null, null)
+    ]
+    "test-failed-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature"] | 2                   | [
+      new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature:Basic Arithmetic With Examples", "Many additions.Single digits.${parameterizedTestNameSuffix()}", null, null)
+    ]
+  }
+
+  private String parameterizedTestNameSuffix() {
+    // older releases report different example names
+    version() == "5.4.0" ? "Example #1" : "Example #1.1"
   }
 
   private String version() {

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/java/org/example/cucumber/calculator/CalculatorSteps.java

@@ -14,11 +14,18 @@ public class CalculatorSteps {
 
   private Calculator calc;
 
+  private static int flakyCounter = 0;
+
   @Given("a calculator I just turned on")
   public void a_calculator_I_just_turned_on() {
     calc = new Calculator();
   }
 
+  @Given("a flaky calculator I just turned on")
+  public void a_flaky_calculator_I_just_turned_on() {
+    calc = ++flakyCounter >= 3 ? new Calculator() : null;
+  }
+
   @When("I add {int} and {int}")
   public void adding(int arg1, int arg2) {
     calc.push(arg1);

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/org/example/cucumber/calculator/basic_arithmetic.feature

@@ -5,6 +5,5 @@ Feature: Basic Arithmetic
     Given a calculator I just turned on
 
   Scenario: Addition
-  # Try to change one of the values below to provoke a failure
     When I add 4 and 5
     Then the result is 9

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/org/example/cucumber/calculator/basic_arithmetic_failed.feature

@@ -0,0 +1,9 @@
+@foo
+Feature: Basic Arithmetic
+
+  Background: A Calculator
+    Given a calculator I just turned on
+
+  Scenario: Addition
+    When I add 4 and 5
+    Then the result is 45

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature

@@ -0,0 +1,9 @@
+@foo
+Feature: Basic Arithmetic
+
+  Background: A Calculator
+    Given a flaky calculator I just turned on
+
+  Scenario: Addition
+    When I add 4 and 5
+    Then the result is 9

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/org/example/cucumber/calculator/basic_arithmetic_skipped.feature

@@ -6,11 +6,9 @@ Feature: Basic Arithmetic
 
   @Disabled
   Scenario: Addition
-  # Try to change one of the values below to provoke a failure
     When I add 4 and 5
     Then the result is 9
 
   Scenario: Subtraction
-  # Try to change one of the values below to provoke a failure
     When I add 4 and -5
     Then the result is -1

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/org/example/cucumber/calculator/basic_arithmetic_skipped_feature.feature

@@ -6,11 +6,9 @@ Feature: Basic Arithmetic
     Given a calculator I just turned on
 
   Scenario: Addition
-  # Try to change one of the values below to provoke a failure
     When I add 4 and 5
     Then the result is 9
 
   Scenario: Subtraction
-  # Try to change one of the values below to provoke a failure
     When I add 4 and -5
     Then the result is -1

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/org/example/cucumber/calculator/basic_arithmetic_unskippable.feature

@@ -5,6 +5,5 @@ Feature: Basic Arithmetic
 
   @datadog_itr_unskippable
   Scenario: Addition
-  # Try to change one of the values below to provoke a failure
     When I add 4 and 5
     Then the result is 9

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/org/example/cucumber/calculator/basic_arithmetic_unskippable_suite.feature

@@ -5,6 +5,5 @@ Feature: Basic Arithmetic
     Given a calculator I just turned on
 
   Scenario: Addition
-  # Try to change one of the values below to provoke a failure
     When I add 4 and 5
     Then the result is 9

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature

@@ -0,0 +1,21 @@
+@foo
+Feature: Basic Arithmetic With Examples
+
+  Background: A Calculator
+    Given a calculator I just turned on
+
+  Scenario Outline: Many additions
+    Given the previous entries:
+      | first | second | operation |
+      | 1     | 1      | +         |
+      | 2     | 1      | +         |
+    When I press +
+    And I add <a> and <b>
+    And I press +
+    Then the result is <c>
+
+    Examples: Single digits
+      | a | b | c  |
+      | 1 | 2 | 12  |
+      | 2 | 3 | 23 |
+

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-scenario-outline-5.4.0/coverages.ftl

@@ -0,0 +1,49 @@
+[ {
+  "test_session_id" : ${content_test_session_id},
+  "test_suite_id" : ${content_test_suite_id},
+  "span_id" : ${content_span_id},
+  "files" : [ {
+    "filename" : "org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature",
+    "segments" : [ ]
+  } ]
+}, {
+  "test_session_id" : ${content_test_session_id},
+  "test_suite_id" : ${content_test_suite_id},
+  "span_id" : ${content_span_id_2},
+  "files" : [ {
+    "filename" : "org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature",
+    "segments" : [ ]
+  } ]
+}, {
+  "test_session_id" : ${content_test_session_id},
+  "test_suite_id" : ${content_test_suite_id},
+  "span_id" : ${content_span_id_3},
+  "files" : [ {
+    "filename" : "org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature",
+    "segments" : [ ]
+  } ]
+}, {
+  "test_session_id" : ${content_test_session_id},
+  "test_suite_id" : ${content_test_suite_id},
+  "span_id" : ${content_span_id_4},
+  "files" : [ {
+    "filename" : "org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature",
+    "segments" : [ ]
+  } ]
+}, {
+  "test_session_id" : ${content_test_session_id},
+  "test_suite_id" : ${content_test_suite_id},
+  "span_id" : ${content_span_id_5},
+  "files" : [ {
+    "filename" : "org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature",
+    "segments" : [ ]
+  } ]
+}, {
+  "test_session_id" : ${content_test_session_id},
+  "test_suite_id" : ${content_test_suite_id},
+  "span_id" : ${content_span_id_6},
+  "files" : [ {
+    "filename" : "org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature",
+    "segments" : [ ]
+  } ]
+} ]