Report X-Datadog-Trace-Count header with traces

.circleci/collect_libs.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Save all important libs into (project-root)/libs
+# This folder will be saved by circleci and available after test runs.
+
+set -e
+
+LIBS_DIR=./libs/
+mkdir -p $LIBS_DIR >/dev/null 2>&1
+
+for lib_path in workspace/**/build/libs; do
+    echo "saving libs in $lib_path"
+    cp $lib_path/*.jar $LIBS_DIR/
+done

.circleci/collect_reports.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# Save all important reports into (project-root)/reports
+# This folder will be saved by circleci and available after test runs.
+
+set -e
+
+REPORTS_DIR=./reports
+mkdir -p $REPORTS_DIR >/dev/null 2>&1
+
+function save_reports () {
+    project_to_save=$1
+    echo "saving reports for $project_to_save"
+
+    report_path=$REPORTS_DIR/$project_to_save
+    mkdir -p $report_path
+    cp -r workspace/$project_to_save/build/reports/* $report_path/
+}
+
+shopt -s globstar
+
+for report_path in workspace/**/build/reports; do
+    report_path=${report_path//workspace\//}
+    report_path=${report_path//\/build\/reports/}
+    save_reports $report_path
+done

.circleci/collect_results.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Save all important reports and artifacts into (project-root)/results
+# This folder will be saved by circleci and available after test runs.
+
+set -e
+
+TEST_RESULTS_DIR=./results
+mkdir -p $TEST_RESULTS_DIR >/dev/null 2>&1
+
+echo "saving test results"
+mkdir -p $TEST_RESULTS_DIR/results
+find workspace/**/build/test-results -name \*.xml -exec cp {} $TEST_RESULTS_DIR \;

.circleci/config.yml

@@ -29,7 +29,23 @@ jobs:
 
       - run:
           name: Build Project
-          command: GRADLE_OPTS="-Dorg.gradle.jvmargs=-Xmx2G -Xms512M" ./gradlew clean compileTestGroovy compileTestScala compileTestJava check -x test -x traceAgentTest --build-cache --parallel --stacktrace --no-daemon --max-workers=6
+          command: GRADLE_OPTS="-Dorg.gradle.jvmargs=-Xmx2G -Xms512M" ./gradlew clean :dd-java-agent:shadowJar compileTestGroovy compileTestScala compileTestJava check -x test -x traceAgentTest --build-cache --parallel --stacktrace --no-daemon --max-workers=6
+
+      - run:
+          name: Collect Libs
+          when: always
+          command: .circleci/collect_libs.sh
+
+      - store_artifacts:
+          path: ./libs
+
+      - run:
+          name: Collect Reports
+          when: always
+          command: .circleci/collect_reports.sh
+
+      - store_artifacts:
+          path: ./reports
 
       - persist_to_workspace:
           root: .
@@ -69,15 +85,20 @@ jobs:
           command: GRADLE_OPTS="-Dorg.gradle.jvmargs=-Xmx2G -Xms512M" ./gradlew $TEST_TASK --build-cache --parallel --stacktrace --no-daemon --max-workers=3
 
       - run:
-          name: Collect Artifacts
+          name: Collect Reports
           when: always
-          command: .circleci/collect_artifacts.sh
+          command: .circleci/collect_reports.sh
 
-      - store_test_results:
+      - store_artifacts:
           path: ./reports
 
-      - store_artifacts:
-          path: ./artifacts
+      - run:
+          name: Collect Test Results
+          when: always
+          command: .circleci/collect_results.sh
+
+      - store_test_results:
+          path: ./results
 
   test_7:
     <<: *default_test_job
@@ -128,13 +149,21 @@ jobs:
           command: ./gradlew traceAgentTest --build-cache --parallel --stacktrace --no-daemon --max-workers=6
 
       - run:
-          name: Collect Artifacts
+          name: Collect Reports
           when: always
-          command: .circleci/collect_artifacts.sh
+          command: .circleci/collect_reports.sh
 
-      - store_test_results:
+      - store_artifacts:
           path: ./reports
 
+      - run:
+          name: Collect Test Results
+          when: always
+          command: .circleci/collect_results.sh
+
+      - store_test_results:
+          path: ./results
+
   scan_versions:
     <<: *defaults
     steps:
@@ -150,17 +179,6 @@ jobs:
           name: Verify Version Scan
           command: ./gradlew verifyVersionScan --parallel --stacktrace --no-daemon --max-workers=6
 
-      - run:
-          name: Collect Artifacts
-          when: always
-          command: .circleci/collect_artifacts.sh
-
-      - store_test_results:
-          path: ./reports
-
-      - store_artifacts:
-          path: ./artifacts
-
       - save_cache:
           key: dd-trace-java-version-scan-{{ checksum "dd-trace-java.gradle" }}
           paths: ~/.gradle

dd-java-agent/instrumentation/servlet-2/servlet-2.gradle

@@ -28,6 +28,5 @@ dependencies {
   testCompile group: 'org.eclipse.jetty', name: 'jetty-server', version: '7.0.0.v20091005'
   testCompile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: '7.0.0.v20091005'
 
-  testCompile project(':dd-java-agent:instrumentation:okhttp-3') // used in the tests
   testCompile group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.6.0'
 }

dd-java-agent/instrumentation/servlet-2/src/test/groovy/JettyServletTest.groovy

@@ -93,7 +93,8 @@ class JettyServletTest extends AgentTestRunner {
 
     expect:
     response.body().string().trim() == expectedResponse
-    writer.size() == 2 // second (parent) trace is the okhttp call above...
+    writer.waitForTraces(1)
+    writer.size() == 1
     def trace = writer.firstTrace()
     trace.size() == 1
     def span = trace[0]
@@ -103,7 +104,7 @@ class JettyServletTest extends AgentTestRunner {
     span.context().resourceName == "GET /$path"
     span.context().spanType == DDSpanTypes.WEB_SERVLET
     !span.context().getErrorFlag()
-    span.context().parentId != 0 // parent should be the okhttp call.
+    span.context().parentId == 0
     span.context().tags["http.url"] == "http://localhost:$PORT/$path"
     span.context().tags["http.method"] == "GET"
     span.context().tags["span.kind"] == "server"
@@ -129,7 +130,8 @@ class JettyServletTest extends AgentTestRunner {
 
     expect:
     response.body().string().trim() != expectedResponse
-    writer.size() == 2 // second (parent) trace is the okhttp call above...
+    writer.waitForTraces(1)
+    writer.size() == 1
     def trace = writer.firstTrace()
     trace.size() == 1
     def span = trace[0]
@@ -138,7 +140,7 @@ class JettyServletTest extends AgentTestRunner {
     span.context().resourceName == "GET /$path"
     span.context().spanType == DDSpanTypes.WEB_SERVLET
     span.context().getErrorFlag()
-    span.context().parentId != 0 // parent should be the okhttp call.
+    span.context().parentId == 0
     span.context().tags["http.url"] == "http://localhost:$PORT/$path"
     span.context().tags["http.method"] == "GET"
     span.context().tags["span.kind"] == "server"

dd-java-agent/instrumentation/servlet-3/src/test/groovy/JettyServletTest.groovy

@@ -94,6 +94,7 @@ class JettyServletTest extends AgentTestRunner {
 
     expect:
     response.body().string().trim() == expectedResponse
+    writer.waitForTraces(1)
     writer.size() == 1
     def trace = writer.firstTrace()
     trace.size() == 1
@@ -131,6 +132,7 @@ class JettyServletTest extends AgentTestRunner {
 
     expect:
     response.body().string().trim() != expectedResponse
+    writer.waitForTraces(1)
     writer.size() == 1
     def trace = writer.firstTrace()
     trace.size() == 1
@@ -172,6 +174,7 @@ class JettyServletTest extends AgentTestRunner {
 
     expect:
     response.body().string().trim() != expectedResponse
+    writer.waitForTraces(1)
     writer.size() == 1
     def trace = writer.firstTrace()
     trace.size() == 1

dd-java-agent/instrumentation/servlet-3/src/test/groovy/TomcatServletTest.groovy

@@ -93,6 +93,7 @@ class TomcatServletTest extends AgentTestRunner {
 
     expect:
     response.body().string().trim() == expectedResponse
+    writer.waitForTraces(1)
     writer.size() == 1
     def trace = writer.firstTrace()
     trace.size() == 1
@@ -130,6 +131,7 @@ class TomcatServletTest extends AgentTestRunner {
 
     expect:
     response.body().string().trim() != expectedResponse
+    writer.waitForTraces(1)
     writer.size() == 1
     def trace = writer.firstTrace()
     trace.size() == 1
@@ -171,6 +173,7 @@ class TomcatServletTest extends AgentTestRunner {
 
     expect:
     response.body().string().trim() != expectedResponse
+    writer.waitForTraces(1)
     writer.size() == 1
     def trace = writer.firstTrace()
     trace.size() == 1

dd-trace-ot/src/main/java/datadog/opentracing/DDTracer.java

@@ -1,6 +1,5 @@
 package datadog.opentracing;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import datadog.opentracing.decorators.AbstractDecorator;
@@ -15,7 +14,6 @@
 import datadog.trace.api.interceptor.TraceInterceptor;
 import datadog.trace.api.sampling.PrioritySampling;
 import datadog.trace.common.DDTraceConfig;
-import datadog.trace.common.Service;
 import datadog.trace.common.sampling.AllSampler;
 import datadog.trace.common.sampling.RateByServiceSampler;
 import datadog.trace.common.sampling.Sampler;
@@ -41,6 +39,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentSkipListSet;
 import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
 import lombok.extern.slf4j.Slf4j;
 
 /** DDTracer makes it easy to send traces and span to DD using the OpenTracing API. */
@@ -74,7 +73,8 @@ public int compare(final TraceInterceptor o1, final TraceInterceptor o2) {
             }
           });
   private final CodecRegistry registry;
-  private final Map<String, Service> services = new HashMap<>();
+
+  private final AtomicInteger traceCount = new AtomicInteger(0);
 
   /** By default, report to local agent and collect all traces. */
   public DDTracer() {
@@ -115,6 +115,7 @@ public DDTracer(
     if (this.writer instanceof DDAgentWriter && sampler instanceof DDApi.ResponseListener) {
       final DDApi api = ((DDAgentWriter) this.writer).getApi();
       api.addResponseListener((DDApi.ResponseListener) this.sampler);
+      api.addTraceCounter(traceCount);
     }
 
     registerClassLoader(ClassLoader.getSystemClassLoader());
@@ -256,6 +257,7 @@ void write(final PendingTrace trace) {
         }
       }
     }
+    traceCount.incrementAndGet();
     if (!writtenTrace.isEmpty() && this.sampler.sample(writtenTrace.get(0))) {
       this.writer.write(writtenTrace);
     }
@@ -280,33 +282,6 @@ public String toString() {
         + '}';
   }
 
-  /**
-   * Register additional information about a service. Service additional information are a Datadog
-   * feature only. Services are reported through a specific Datadog endpoint.
-   *
-   * @param service additional service information
-   */
-  public void addServiceInfo(final Service service) {
-    services.put(service.getName(), service);
-    // Update the writer
-    try {
-      // We don't bother to send multiple times the list of services at this time
-      writer.writeServices(services);
-    } catch (final Throwable ex) {
-      log.warn("Failed to report additional service information, reason: {}", ex.getMessage());
-    }
-  }
-
-  /**
-   * Return the list of additional service information registered
-   *
-   * @return the list of additional service information
-   */
-  @JsonIgnore
-  public Map<String, Service> getServiceInfo() {
-    return services;
-  }
-
   private static class CodecRegistry {
 
     private final Map<Format<?>, Codec<?>> codecs = new HashMap<>();