Fix handling skipped features in Karate

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestSuiteImpl.java

@@ -165,7 +165,10 @@ public void end(@Nullable Long endTime) {
                 + "it is possible that end() was called multiple times "
                 + "or an operation that was started by the suite is still in progress; "
                 + "active scope span is: "
-                + scopeSpan);
+                + scopeSpan
+                + "; "
+                + "expected span is: "
+                + span);
       }
 
       scope.close();

dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java

@@ -48,6 +48,13 @@ public boolean beforeFeature(FeatureRuntime fr) {
         KarateUtils.getCategories(feature.getTags()),
         suite.parallel,
         TestFrameworkInstrumentation.KARATE);
+
+    if (!fr.scenarios.hasNext()) {
+      // Karate will not trigger the afterFeature hook if suite has no scenarios
+      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, null);
+      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor);
+    }
+
     return true;
   }
 

dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy

@@ -8,6 +8,7 @@ import org.example.TestFailedParameterizedKarate
 import org.example.TestFailedThenSucceedKarate
 import org.example.TestParameterizedKarate
 import org.example.TestParameterizedMoreCasesKarate
+import org.example.TestSkippedFeatureKarate
 import org.example.TestSucceedKarate
 import org.example.TestSucceedKarateSlow
 import org.example.TestSucceedOneCaseKarate
@@ -34,11 +35,12 @@ class KarateTest extends CiVisibilityInstrumentationTest {
     assertSpansData(testcaseName, expectedTracesCount)
 
     where:
-    testcaseName         | tests                     | expectedTracesCount | assumption
-    "test-succeed"       | [TestSucceedKarate]       | 3                   | true
-    "test-with-setup"    | [TestWithSetupKarate]     | 3                   | isSetupTagSupported(FileUtils.KARATE_VERSION)
-    "test-parameterized" | [TestParameterizedKarate] | 3                   | true
-    "test-failed"        | [TestFailedKarate]        | 3                   | true
+    testcaseName           | tests                      | expectedTracesCount | assumption
+    "test-succeed"         | [TestSucceedKarate]        | 3                   | true
+    "test-with-setup"      | [TestWithSetupKarate]      | 3                   | isSetupTagSupported(FileUtils.KARATE_VERSION)
+    "test-parameterized"   | [TestParameterizedKarate]  | 3                   | true
+    "test-failed"          | [TestFailedKarate]         | 3                   | true
+    "test-skipped-feature" | [TestSkippedFeatureKarate] | 1                   | true
   }
 
   def "test ITR #testcaseName"() {

dd-java-agent/instrumentation/karate/src/test/java/org/example/TestSkippedFeatureKarate.java

@@ -0,0 +1,19 @@
+package org.example;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.intuit.karate.Results;
+import com.intuit.karate.Runner;
+import org.junit.jupiter.api.Test;
+
+public class TestSkippedFeatureKarate {
+
+  @Test
+  void testParallel() {
+    Results results =
+        Runner.path("classpath:org/example/test_succeed.feature")
+            .systemProperty("karate.options", "--tags ~@foo")
+            .parallel(1);
+    assertEquals(0, results.getFailCount(), results.getErrorMessages());
+  }
+}

dd-java-agent/instrumentation/karate/src/test/resources/test-skipped-feature/coverages.ftl

@@ -0,0 +1 @@
+[ ]

dd-java-agent/instrumentation/karate/src/test/resources/test-skipped-feature/events.ftl

@@ -0,0 +1,106 @@
+[ {
+  "type" : "test_suite_end",
+  "version" : 1,
+  "content" : {
+    "test_session_id" : ${content_test_session_id},
+    "test_module_id" : ${content_test_module_id},
+    "test_suite_id" : ${content_test_suite_id},
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "name" : "karate.test_suite",
+    "resource" : "[org/example/test_succeed] test succeed",
+    "start" : ${content_start},
+    "duration" : ${content_duration},
+    "error" : 0,
+    "metrics" : { },
+    "meta" : {
+      "test.type" : "test",
+      "os.architecture" : ${content_meta_os_architecture},
+      "test.module" : "karate",
+      "test.status" : "skip",
+      "test.traits" : "{\"category\":[\"foo\"]}",
+      "runtime.name" : ${content_meta_runtime_name},
+      "runtime.vendor" : ${content_meta_runtime_vendor},
+      "env" : "none",
+      "os.platform" : ${content_meta_os_platform},
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "os.version" : ${content_meta_os_version},
+      "library_version" : ${content_meta_library_version},
+      "component" : "karate",
+      "span.kind" : "test_suite_end",
+      "test.suite" : "[org/example/test_succeed] test succeed",
+      "runtime.version" : ${content_meta_runtime_version},
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.framework" : "karate"
+    }
+  }
+}, {
+  "type" : "test_session_end",
+  "version" : 1,
+  "content" : {
+    "test_session_id" : ${content_test_session_id},
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "name" : "karate.test_session",
+    "resource" : "karate",
+    "start" : ${content_start_2},
+    "duration" : ${content_duration_2},
+    "error" : 0,
+    "metrics" : {
+      "process_id" : ${content_metrics_process_id},
+      "_dd.profiling.enabled" : 0,
+      "_dd.trace_span_attribute_schema" : 0
+    },
+    "meta" : {
+      "test.type" : "test",
+      "os.architecture" : ${content_meta_os_architecture},
+      "test.status" : "skip",
+      "language" : "jvm",
+      "runtime.name" : ${content_meta_runtime_name},
+      "runtime.vendor" : ${content_meta_runtime_vendor},
+      "env" : "none",
+      "os.platform" : ${content_meta_os_platform},
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "os.version" : ${content_meta_os_version},
+      "library_version" : ${content_meta_library_version},
+      "component" : "karate",
+      "_dd.profiling.ctx" : "test",
+      "span.kind" : "test_session_end",
+      "runtime.version" : ${content_meta_runtime_version},
+      "runtime-id" : ${content_meta_runtime_id},
+      "test.command" : "karate",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.framework" : "karate"
+    }
+  }
+}, {
+  "type" : "test_module_end",
+  "version" : 1,
+  "content" : {
+    "test_session_id" : ${content_test_session_id},
+    "test_module_id" : ${content_test_module_id},
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "name" : "karate.test_module",
+    "resource" : "karate",
+    "start" : ${content_start_3},
+    "duration" : ${content_duration_3},
+    "error" : 0,
+    "metrics" : { },
+    "meta" : {
+      "test.type" : "test",
+      "os.architecture" : ${content_meta_os_architecture},
+      "test.module" : "karate",
+      "test.status" : "skip",
+      "runtime.name" : ${content_meta_runtime_name},
+      "runtime.vendor" : ${content_meta_runtime_vendor},
+      "env" : "none",
+      "os.platform" : ${content_meta_os_platform},
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "os.version" : ${content_meta_os_version},
+      "library_version" : ${content_meta_library_version},
+      "component" : "karate",
+      "span.kind" : "test_module_end",
+      "runtime.version" : ${content_meta_runtime_version},
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.framework" : "karate"
+    }
+  }
+} ]