tracing: add datadog extension (#4699)

Signed-off-by: Caleb Gilmour <caleb.gilmour@datadoghq.com>

api/envoy/config/trace/v2/trace.proto

@@ -23,15 +23,21 @@ import "validate/validate.proto";
 message Tracing {
   message Http {
     // The name of the HTTP trace driver to instantiate. The name must match a
-    // supported HTTP trace driver. *envoy.lightstep*, *envoy.zipkin*, and
-    // *envoy.dynamic.ot* are built-in trace drivers.
+    // supported HTTP trace driver. Built-in trace drivers:
+    //
+    // - *envoy.lightstep*
+    // - *envoy.zipkin*
+    // - *envoy.dynamic.ot*
+    // - *envoy.tracers.datadog*
     string name = 1 [(validate.rules).string.min_bytes = 1];
 
     // Trace driver specific configuration which depends on the driver being instantiated.
-    // See the :ref:`LightstepConfig
-    // <envoy_api_msg_config.trace.v2.LightstepConfig>`, :ref:`ZipkinConfig
-    // <envoy_api_msg_config.trace.v2.ZipkinConfig>`, and :ref:`DynamicOtConfig
-    // <envoy_api_msg_config.trace.v2.DynamicOtConfig>` trace drivers for examples.
+    // See the trace drivers for examples:
+    //
+    // - :ref:`LightstepConfig <envoy_api_msg_config.trace.v2.LightstepConfig>`
+    // - :ref:`ZipkinConfig <envoy_api_msg_config.trace.v2.ZipkinConfig>`
+    // - :ref:`DynamicOtConfig <envoy_api_msg_config.trace.v2.DynamicOtConfig>`
+    // - :ref:`DatadogConfig <envoy_api_msg_config.trace.v2.DatadogConfig>`
     oneof config_type {
       google.protobuf.Struct config = 2;
 
@@ -86,6 +92,14 @@ message DynamicOtConfig {
   google.protobuf.Struct config = 2;
 }
 
+// Configuration for the Datadog tracer.
+message DatadogConfig {
+  // The cluster to use for submitting traces to the Datadog agent.
+  string collector_cluster = 1 [(validate.rules).string.min_bytes = 1];
+  // The name used for the service when traces are generated by envoy.
+  string service_name = 2 [(validate.rules).string.min_bytes = 1];
+}
+
 // Configuration structure.
 message TraceServiceConfig {
   // The upstream gRPC cluster that hosts the metrics service.

bazel/external/BUILD

@@ -6,6 +6,7 @@ cc_library(
     deps = [
         "@com_google_googletest//:gtest",
         "@com_lightstep_tracer_cpp//:lightstep_tracer",
+        "@com_github_datadog_dd_opentracing_cpp//:dd_opentracing_cpp",
         "@io_opentracing_cpp//:opentracing",
     ],
 )

bazel/repositories.bzl

@@ -280,6 +280,7 @@ def envoy_dependencies(path = "@envoy_deps//", skip_targets = []):
     _com_github_google_libprotobuf_mutator()
     _io_opentracing_cpp()
     _com_lightstep_tracer_cpp()
+    _com_github_datadog_dd_opentracing_cpp()
     _com_github_grpc_grpc()
     _com_github_google_jwt_verify()
     _com_github_nanopb_nanopb()
@@ -401,6 +402,17 @@ def _com_lightstep_tracer_cpp():
         actual = "@com_lightstep_tracer_cpp//:lightstep_tracer",
     )
 
+def _com_github_datadog_dd_opentracing_cpp():
+    _repository_impl("com_github_datadog_dd_opentracing_cpp")
+    _repository_impl(
+        name = "com_github_msgpack_msgpack_c",
+        build_file = "@com_github_datadog_dd_opentracing_cpp//:bazel/external/msgpack.BUILD",
+    )
+    native.bind(
+        name = "dd_opentracing_cpp",
+        actual = "@com_github_datadog_dd_opentracing_cpp//:dd_opentracing_cpp",
+    )
+
 def _com_github_tencent_rapidjson():
     _repository_impl(
         name = "com_github_tencent_rapidjson",

bazel/repository_locations.bzl

@@ -90,6 +90,16 @@ REPOSITORY_LOCATIONS = dict(
         # From: https://github.com/lightstep/lightstep-tracer-cpp/blob/v0.8.0/lightstep-tracer-common/third_party/googleapis/README.lightstep-tracer-common#L6
         urls = ["https://github.com/googleapis/googleapis/archive/d6f78d948c53f3b400bb46996eb3084359914f9b.tar.gz"],
     ),
+    com_github_datadog_dd_opentracing_cpp = dict(
+        sha256 = "733e9b698a232cfd3aa35b4e27c59641bf1fa78e52e71d29e230af4f2070cdf5",
+        strip_prefix = "dd-opentracing-cpp-0.3.5",
+        urls = ["https://github.com/DataDog/dd-opentracing-cpp/archive/v0.3.5.tar.gz"],
+    ),
+    com_github_msgpack_msgpack_c = dict(
+        sha256 = "bda49f996a73d2c6080ff0523e7b535917cd28c8a79c3a5da54fc29332d61d1e",
+        strip_prefix = "msgpack-c-cpp-3.1.1",
+        urls = ["https://github.com/msgpack/msgpack-c/archive/cpp-3.1.1.tar.gz"],
+    ),
     com_github_google_jwt_verify = dict(
         sha256 = "499f1e145c19f33031eb8fc6452d5d391b4cecfdeda23e2055386a3b33be4d41",
         strip_prefix = "jwt_verify_lib-66792a057ec54e4b75c6a2eeda4e98220bd12a9a",

docs/root/configuration/http_conn_man/headers.rst

@@ -437,6 +437,34 @@ The *b3* HTTP header is used by the Zipkin tracer in Envoy.
 Is a more compressed header format. See more on zipkin tracing
 `here <https://github.com/openzipkin/b3-propagation#single-header>`.
 
+.. _config_http_conn_man_headers_x-datadog-trace-id:
+
+x-datadog-trace-id
+------------------
+
+The *x-datadog-trace-id* HTTP header is used by the Datadog tracer in Envoy.
+The 64-bit value represents the ID of the overall trace, and is used to correlate
+the spans.
+
+.. _config_http_conn_man_headers_x-datadog-parent-id:
+
+x-datadog-parent-id
+-------------------
+
+The *x-datadog-parent-id* HTTP header is used by the Datadog tracer in Envoy.
+The 64-bit value uniquely identifies the span within the trace, and is used to
+create parent-child relationships between spans.
+
+.. _config_http_conn_man_headers_x-datadog-sampling-priority:
+
+x-datadog-sampling-priority
+---------------------------
+
+The *x-datadog-sampling-priority* HTTP header is used by the Datadog tracer in Envoy.
+The integer value indicates the sampling decision that has been made for this trace.
+A value of 0 indicates that the trace should not be collected, and a value of 1
+requests that spans are sampled and reported.
+
 .. _config_http_conn_man_headers_custom_request_headers:
 
 Custom request/response headers

docs/root/install/ref_configs.rst

@@ -47,7 +47,7 @@ A few notes about the example configurations:
 * DNS for `yourcompany.net` is assumed to be setup for various things. Search the configuration
   templates for different instances of this.
 * Tracing is configured for `LightStep <http://lightstep.com/>`_. To
-  disable this or enable `Zipkin <http://zipkin.io>` tracing, delete or
+  disable this or enable `Zipkin <http://zipkin.io>` or `Datadog <https://datadoghq.com>` tracing, delete or
   change the :ref:`tracing configuration <envoy_api_file_envoy/config/trace/v2/trace.proto>` accordingly.
 * The configuration demonstrates the use of a :ref:`global rate limiting service
   <arch_overview_rate_limit>`. To disable this delete the :ref:`rate limit configuration

docs/root/intro/arch_overview/tracing.rst

@@ -14,7 +14,8 @@ sources of latency. Envoy supports three features related to system wide tracing
   x-request-id header for unified logging as well as tracing.
 * **External trace service integration**: Envoy supports pluggable external trace visualization
   providers. Currently Envoy supports `LightStep <http://lightstep.com/>`_, `Zipkin <http://zipkin.io/>`_
-  or any Zipkin compatible backends (e.g. `Jaeger <https://github.com/jaegertracing/>`_).
+  or any Zipkin compatible backends (e.g. `Jaeger <https://github.com/jaegertracing/>`_), and
+  `Datadog <https://datadoghq.com>`_.
   However, support for other tracing providers would not be difficult to add.
 * **Client trace ID joining**: The :ref:`config_http_conn_man_headers_x-client-trace-id` header can
   be used to join untrusted request IDs to the trusted internal
@@ -71,6 +72,12 @@ Alternatively the trace context can be manually propagated by the service:
   request. In addition, the single :ref:`config_http_conn_man_headers_b3` header propagation format is
   supported, which is a more compressed format.
 
+* When using the Datadog tracer, Envoy relies on the service to propagate the
+  Datadog-specific HTTP headers (
+  :ref:`config_http_conn_man_headers_x-datadog-trace-id`,
+  :ref:`config_http_conn_man_headers_x-datadog-parent-id`,
+  :ref:`config_http_conn_man_headers_x-datadog-sampling-priority`).
+
 What data each trace contains
 -----------------------------
 An end-to-end trace is comprised of one or more spans. A
@@ -95,7 +102,7 @@ the route. The name can also be overridden using the
 Envoy automatically sends spans to tracing collectors. Depending on the tracing collector,
 multiple spans are stitched together using common information such as the globally unique
 request ID :ref:`config_http_conn_man_headers_x-request-id` (LightStep) or
-the trace ID configuration (Zipkin). See
+the trace ID configuration (Zipkin and Datadog). See
 
 * :ref:`v2 API reference <envoy_api_msg_config.trace.v2.Tracing>`
 

docs/root/intro/version_history.rst

@@ -41,6 +41,7 @@ Version history
 * thrift_proxy: introduced thrift rate limiter filter
 * tls: add support for CRLs in :ref:`trusted_ca <envoy_api_field_auth.CertificateValidationContext.trusted_ca>`.
 * tracing: added support to the Zipkin tracer for the :ref:`b3 <config_http_conn_man_headers_b3>` single header format.
+* tracing: added support for :ref:`Datadog <arch_overview_tracing>` tracer.
 * upstream: changed how load calculation for :ref:`priority levels<arch_overview_load_balancing_priority_levels>` and :ref:`panic thresholds<arch_overview_load_balancing_panic_threshold>` interact. As long as normalized total health is 100% panic thresholds are disregarded.
 * upstream: changed the default hash for :ref:`ring hash <envoy_api_msg_Cluster.RingHashLbConfig>` from std::hash to `xxHash <https://github.com/Cyan4973/xxHash>`_.
 

source/extensions/extensions_build_config.bzl

@@ -102,6 +102,7 @@ EXTENSIONS = {
 
     "envoy.tracers.dynamic_ot":                         "//source/extensions/tracers/dynamic_ot:config",
     "envoy.tracers.lightstep":                          "//source/extensions/tracers/lightstep:config",
+    "envoy.tracers.datadog":                            "//source/extensions/tracers/datadog:config",
     "envoy.tracers.zipkin":                             "//source/extensions/tracers/zipkin:config",
 
     #

source/extensions/tracers/datadog/BUILD

@@ -0,0 +1,39 @@
+licenses(["notice"])  # Apache 2
+
+# Trace driver for Datadog (https://datadoghq.com/)
+
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_library",
+    "envoy_package",
+)
+
+envoy_package()
+
+envoy_cc_library(
+    name = "datadog_tracer_lib",
+    srcs = [
+        "datadog_tracer_impl.cc",
+    ],
+    hdrs = [
+        "datadog_tracer_impl.h",
+    ],
+    external_deps = ["dd_opentracing_cpp"],
+    deps = [
+        "//source/common/tracing:http_tracer_lib",
+        "//source/extensions/tracers:well_known_names",
+        "//source/extensions/tracers/common/ot:opentracing_driver_lib",
+        "//source/server:configuration_lib",
+    ],
+)
+
+envoy_cc_library(
+    name = "config",
+    srcs = ["config.cc"],
+    hdrs = ["config.h"],
+    deps = [
+        ":datadog_tracer_lib",
+        "//source/extensions/tracers:well_known_names",
+        "//source/server:configuration_lib",
+    ],
+)

source/extensions/tracers/datadog/config.cc

@@ -0,0 +1,48 @@
+#include "extensions/tracers/datadog/config.h"
+
+#include "envoy/registry/registry.h"
+
+#include "common/common/utility.h"
+#include "common/tracing/http_tracer_impl.h"
+
+#include "extensions/tracers/datadog/datadog_tracer_impl.h"
+#include "extensions/tracers/well_known_names.h"
+
+#include "datadog/opentracing.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Tracers {
+namespace Datadog {
+
+Tracing::HttpTracerPtr
+DatadogTracerFactory::createHttpTracer(const envoy::config::trace::v2::Tracing& configuration,
+                                       Server::Instance& server) {
+
+  ProtobufTypes::MessagePtr config_ptr = createEmptyConfigProto();
+
+  if (configuration.http().has_config()) {
+    MessageUtil::jsonConvert(configuration.http().config(), *config_ptr);
+  }
+
+  const auto& datadog_config =
+      dynamic_cast<const envoy::config::trace::v2::DatadogConfig&>(*config_ptr);
+
+  Tracing::DriverPtr datadog_driver{
+      std::make_unique<Driver>(datadog_config, server.clusterManager(), server.stats(),
+                               server.threadLocal(), server.runtime())};
+  return std::make_unique<Tracing::HttpTracerImpl>(std::move(datadog_driver), server.localInfo());
+}
+
+std::string DatadogTracerFactory::name() { return TracerNames::get().Datadog; }
+
+/**
+ * Static registration for the Datadog tracer. @see RegisterFactory.
+ */
+static Registry::RegisterFactory<DatadogTracerFactory, Server::Configuration::TracerFactory>
+    register_;
+
+} // namespace Datadog
+} // namespace Tracers
+} // namespace Extensions
+} // namespace Envoy

source/extensions/tracers/datadog/config.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <string>
+
+#include "envoy/server/instance.h"
+
+#include "server/configuration_impl.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Tracers {
+namespace Datadog {
+
+/**
+ * Config registration for the Datadog tracer. @see TracerFactory.
+ */
+class DatadogTracerFactory : public Server::Configuration::TracerFactory {
+public:
+  // TracerFactory
+  Tracing::HttpTracerPtr createHttpTracer(const envoy::config::trace::v2::Tracing& configuration,
+                                          Server::Instance& server) override;
+
+  ProtobufTypes::MessagePtr createEmptyConfigProto() override {
+    return std::make_unique<envoy::config::trace::v2::DatadogConfig>();
+  }
+
+  std::string name() override;
+};
+
+} // namespace Datadog
+} // namespace Tracers
+} // namespace Extensions
+} // namespace Envoy