tracing: Support dynamically loading tracing libraries.

bazel/repository_locations.bzl

@@ -34,8 +34,8 @@ REPOSITORY_LOCATIONS = dict(
         remote = "https://github.com/grpc/grpc.git",
     ),
     io_opentracing_cpp = dict(
-        commit = "e57161e2a4bd1f9d3a8d3edf23185f033bb45f17",
-        remote = "https://github.com/opentracing/opentracing-cpp", # v1.2.0
+        commit = "1368b7de0343388a0e217643481ec4d475ed22e5",
+        remote = "https://github.com/rnburn/opentracing-cpp",
     ),
     com_lightstep_tracer_cpp = dict(
         commit = "6a198acd328f976984699f7272bbec7c8b220f65",

source/common/config/well_known_names.h

@@ -156,6 +156,8 @@ class HttpTracerNameValues {
   const std::string LIGHTSTEP = "envoy.lightstep";
   // Zipkin tracer
   const std::string ZIPKIN = "envoy.zipkin";
+  // Dynamic tracer
+  const std::string DYNAMIC_OT = "envoy.dynamic.ot";
 };
 
 typedef ConstSingleton<HttpTracerNameValues> HttpTracerNames;

source/common/tracing/BUILD

@@ -53,6 +53,20 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "dynamic_opentracing_driver_lib",
+    srcs = [
+        "dynamic_opentracing_driver_impl.cc",
+    ],
+    hdrs = [
+        "dynamic_opentracing_driver_impl.h",
+    ],
+    deps = [
+        ":http_tracer_lib",
+        ":opentracing_driver_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "lightstep_tracer_lib",
     srcs = [

source/common/tracing/dynamic_opentracing_driver_impl.cc

@@ -0,0 +1,38 @@
+#include "common/tracing/dynamic_opentracing_driver_impl.h"
+
+#include "common/common/assert.h"
+
+namespace Envoy {
+namespace Tracing {
+
+DynamicOpenTracingDriver::DynamicOpenTracingDriver(Stats::Store& stats, const std::string& library,
+                                                   const std::string& tracer_config)
+    : OpenTracingDriver{stats} {
+  std::string error_message;
+  opentracing::expected<opentracing::DynamicTracingLibraryHandle> library_handle_maybe =
+      opentracing::DynamicallyLoadTracingLibrary(library.c_str(), error_message);
+  if (!library_handle_maybe) {
+    if (error_message.empty()) {
+      throw EnvoyException{fmt::format("{}", library_handle_maybe.error().message())};
+    } else {
+      throw EnvoyException{
+          fmt::format("{}: {}", library_handle_maybe.error().message(), error_message)};
+    }
+  }
+  library_handle_ = std::move(*library_handle_maybe);
+
+  opentracing::expected<std::shared_ptr<opentracing::Tracer>> tracer_maybe =
+      library_handle_.tracer_factory().MakeTracer(tracer_config.c_str(), error_message);
+  if (!tracer_maybe) {
+    if (error_message.empty()) {
+      throw EnvoyException{fmt::format("{}", tracer_maybe.error().message())};
+    } else {
+      throw EnvoyException{fmt::format("{}: {}", tracer_maybe.error().message(), error_message)};
+    }
+  }
+  tracer_ = std::move(*tracer_maybe);
+  RELEASE_ASSERT(tracer_ != nullptr);
+}
+
+} // namespace Tracing
+} // namespace Envoy

source/common/tracing/dynamic_opentracing_driver_impl.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "envoy/runtime/runtime.h"
+#include "envoy/thread_local/thread_local.h"
+#include "envoy/tracing/http_tracer.h"
+#include "envoy/upstream/cluster_manager.h"
+
+#include "common/tracing/opentracing_driver_impl.h"
+
+#include "opentracing/dynamic_load.h"
+
+namespace Envoy {
+namespace Tracing {
+
+/**
+ * This driver provides support for dynamically loading tracing libraries into Envoy that provide an
+ * implementation of the OpenTracing API (see https://github.com/opentracing/opentracing-cpp).
+ */
+class DynamicOpenTracingDriver : public OpenTracingDriver {
+public:
+  DynamicOpenTracingDriver(Stats::Store& stats, const std::string& library,
+                           const std::string& tracer_config);
+
+  // Tracer::OpenTracingDriver
+  opentracing::Tracer& tracer() override { return *tracer_; }
+
+  PropagationMode propagationMode() const override {
+    return OpenTracingDriver::PropagationMode::TracerNative;
+  }
+
+private:
+  opentracing::DynamicTracingLibraryHandle library_handle_;
+  std::shared_ptr<opentracing::Tracer> tracer_;
+};
+
+} // namespace Tracing
+} // namespace Envoy

source/common/tracing/lightstep_tracer_impl.cc

@@ -112,7 +112,7 @@ LightStepDriver::TlsLightStepTracer::TlsLightStepTracer(
   enableTimer();
 }
 
-const opentracing::Tracer& LightStepDriver::TlsLightStepTracer::tracer() const { return *tracer_; }
+opentracing::Tracer& LightStepDriver::TlsLightStepTracer::tracer() { return *tracer_; }
 
 void LightStepDriver::TlsLightStepTracer::enableTimer() {
   const uint64_t flush_interval =
@@ -160,7 +160,7 @@ LightStepDriver::LightStepDriver(const Json::Object& config,
   });
 }
 
-const opentracing::Tracer& LightStepDriver::tracer() const {
+opentracing::Tracer& LightStepDriver::tracer() {
   return tls_->getTyped<TlsLightStepTracer>().tracer();
 }
 

source/common/tracing/lightstep_tracer_impl.h

@@ -50,7 +50,7 @@ class LightStepDriver : public OpenTracingDriver {
   LightstepTracerStats& tracerStats() { return tracer_stats_; }
 
   // Tracer::OpenTracingDriver
-  const opentracing::Tracer& tracer() const override;
+  opentracing::Tracer& tracer() override;
   PropagationMode propagationMode() const override { return propagation_mode_; }
 
 private:
@@ -90,7 +90,7 @@ class LightStepDriver : public OpenTracingDriver {
     TlsLightStepTracer(const std::shared_ptr<lightstep::LightStepTracer>& tracer,
                        LightStepDriver& driver, Event::Dispatcher& dispatcher);
 
-    const opentracing::Tracer& tracer() const;
+    opentracing::Tracer& tracer();
 
   private:
     void enableTimer();

source/common/tracing/opentracing_driver_impl.h

@@ -51,7 +51,7 @@ class OpenTracingDriver : public Driver, protected Logger::Loggable<Logger::Id::
   SpanPtr startSpan(const Config& config, Http::HeaderMap& request_headers,
                     const std::string& operation_name, SystemTime start_time) override;
 
-  virtual const opentracing::Tracer& tracer() const PURE;
+  virtual opentracing::Tracer& tracer() PURE;
 
   enum class PropagationMode { SingleHeader, TracerNative };
 

source/exe/BUILD

@@ -81,6 +81,7 @@ envoy_cc_library(
         "//source/server:hot_restart_lib",
         "//source/server:options_lib",
         "//source/server/config/http:lightstep_lib",
+        "//source/server/config/http:dynamic_opentracing_lib",
         "//source/server/config/http:zipkin_lib",
     ] + select({
         "//bazel:disable_signal_trace": [],

source/server/config/http/BUILD

@@ -148,6 +148,18 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "dynamic_opentracing_lib",
+    srcs = ["dynamic_opentracing_http_tracer.cc"],
+    hdrs = ["dynamic_opentracing_http_tracer.h"],
+    deps = [
+        "//source/common/config:well_known_names",
+        "//source/common/tracing:dynamic_opentracing_driver_lib",
+        "//source/common/tracing:http_tracer_lib",
+        "//source/server:configuration_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "ratelimit_lib",
     srcs = ["ratelimit.cc"],

source/server/config/http/dynamic_opentracing_http_tracer.cc

@@ -0,0 +1,37 @@
+#include "server/config/http/dynamic_opentracing_http_tracer.h"
+
+#include <string>
+
+#include "envoy/registry/registry.h"
+
+#include "common/common/utility.h"
+#include "common/config/well_known_names.h"
+#include "common/tracing/dynamic_opentracing_driver_impl.h"
+#include "common/tracing/http_tracer_impl.h"
+
+namespace Envoy {
+namespace Server {
+namespace Configuration {
+
+Tracing::HttpTracerPtr DynamicOpenTracingHttpTracerFactory::createHttpTracer(
+    const Json::Object& json_config, Server::Instance& server,
+    Upstream::ClusterManager& /*cluster_manager*/) {
+  const std::string library = json_config.getString("library");
+  const std::string config = json_config.getObject("config")->asJsonString();
+  Tracing::DriverPtr dynamic_driver{
+      std::make_unique<Tracing::DynamicOpenTracingDriver>(server.stats(), library, config)};
+  return std::make_unique<Tracing::HttpTracerImpl>(std::move(dynamic_driver), server.localInfo());
+}
+
+std::string DynamicOpenTracingHttpTracerFactory::name() {
+  return Config::HttpTracerNames::get().DYNAMIC_OT;
+}
+
+/**
+ * Static registration for the dynamic opentracing http tracer. @see RegisterFactory.
+ */
+static Registry::RegisterFactory<DynamicOpenTracingHttpTracerFactory, HttpTracerFactory> register_;
+
+} // namespace Configuration
+} // namespace Server
+} // namespace Envoy

source/server/config/http/dynamic_opentracing_http_tracer.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <string>
+
+#include "envoy/server/instance.h"
+
+#include "server/configuration_impl.h"
+
+namespace Envoy {
+namespace Server {
+namespace Configuration {
+
+/**
+ * Config registration for the dynamic opentracing tracer. @see HttpTracerFactory.
+ */
+class DynamicOpenTracingHttpTracerFactory : public HttpTracerFactory {
+public:
+  // HttpTracerFactory
+  Tracing::HttpTracerPtr createHttpTracer(const Json::Object& json_config, Server::Instance& server,
+                                          Upstream::ClusterManager& cluster_manager) override;
+
+  std::string name() override;
+
+  bool requiresClusterName() override { return false; }
+};
+
+} // namespace Configuration
+} // namespace Server
+} // namespace Envoy

source/server/configuration_impl.cc

@@ -96,11 +96,6 @@ void MainImpl::initializeTracers(const envoy::config::trace::v2::Tracing& config
     return;
   }
 
-  if (server.localInfo().clusterName().empty()) {
-    throw EnvoyException("cluster name must be defined if tracing is enabled. See "
-                         "--service-cluster option.");
-  }
-
   // Initialize tracing driver.
   std::string type = configuration.http().name();
   ENVOY_LOG(info, "  loading tracing driver: {}", type);
@@ -111,6 +106,11 @@ void MainImpl::initializeTracers(const envoy::config::trace::v2::Tracing& config
 
   // Now see if there is a factory that will accept the config.
   auto& factory = Config::Utility::getAndCheckFactory<HttpTracerFactory>(type);
+  if (factory.requiresClusterName() && server.localInfo().clusterName().empty()) {
+    throw EnvoyException(fmt::format("cluster name must be defined for the tracing driver {}. See "
+                                     "--service-cluster option.",
+                                     type));
+  }
   http_tracer_ = factory.createHttpTracer(*driver_config, server, *cluster_manager_);
 }
 

source/server/configuration_impl.h

@@ -54,6 +54,11 @@ class HttpTracerFactory {
    * factory.
    */
   virtual std::string name() PURE;
+
+  /**
+   * Returns true if the tracing driver requires cluster name to be defined.
+   */
+  virtual bool requiresClusterName() { return true; }
 };
 
 /**

test/common/tracing/BUILD

@@ -55,3 +55,21 @@ envoy_cc_test(
         "//test/test_common:utility_lib",
     ],
 )
+
+envoy_cc_test(
+    name = "dynamic_opentracing_driver_impl_test",
+    srcs = [
+        "dynamic_opentracing_driver_impl_test.cc",
+    ],
+    data = [
+        "@io_opentracing_cpp//mocktracer:libmocktracer_plugin.so",
+    ],
+    deps = [
+        "//source/common/http:header_map_lib",
+        "//source/common/tracing:dynamic_opentracing_driver_lib",
+        "//test/mocks/http:http_mocks",
+        "//test/mocks/stats:stats_mocks",
+        "//test/mocks/tracing:tracing_mocks",
+        "//test/test_common:environment_lib",
+    ],
+)

test/common/tracing/dynamic_opentracing_driver_impl_test.cc

@@ -0,0 +1,77 @@
+#include "common/http/header_map_impl.h"
+#include "common/tracing/dynamic_opentracing_driver_impl.h"
+
+#include "test/mocks/http/mocks.h"
+#include "test/mocks/stats/mocks.h"
+#include "test/mocks/tracing/mocks.h"
+#include "test/test_common/environment.h"
+
+#include "fmt/printf.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using testing::Test;
+
+namespace Envoy {
+namespace Tracing {
+
+class DynamicOpenTracingDriverTest : public Test {
+public:
+  void setup(const std::string& library, const std::string& tracer_config) {
+    driver_.reset(new DynamicOpenTracingDriver{stats_, library, tracer_config});
+  }
+
+  void setupValidDriver() { setup(library_path_, tracer_config_); }
+
+  const std::string library_path_ =
+      TestEnvironment::runfilesDirectory() +
+      "/external/io_opentracing_cpp/mocktracer/libmocktracer_plugin.so";
+  const std::string spans_file_ = TestEnvironment::temporaryDirectory() + "/spans.json";
+  const std::string tracer_config_ = fmt::sprintf(R"EOF(
+      {
+        "output_file": "%s"
+      }
+    )EOF",
+                                                  spans_file_);
+  std::unique_ptr<DynamicOpenTracingDriver> driver_;
+  Stats::IsolatedStoreImpl stats_;
+
+  const std::string operation_name_{"test"};
+  Http::TestHeaderMapImpl request_headers_{
+      {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}};
+  SystemTime start_time_;
+  NiceMock<Tracing::MockConfig> config_;
+};
+
+TEST_F(DynamicOpenTracingDriverTest, InitializeDriver) {
+  {
+    std::string invalid_library = "abc123";
+    std::string invalid_config = R"EOF(
+      {"fake" : "fake"}
+    )EOF";
+
+    EXPECT_THROW(setup(invalid_library, invalid_config), EnvoyException);
+  }
+
+  {
+    std::string empty_config = "{}";
+
+    EXPECT_THROW(setup(library_path_, empty_config), EnvoyException);
+  }
+}
+
+TEST_F(DynamicOpenTracingDriverTest, FlushSpans) {
+  setupValidDriver();
+
+  SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, start_time_);
+  first_span->finishSpan();
+  driver_->tracer().Close();
+
+  const Json::ObjectSharedPtr spans_json =
+      TestEnvironment::jsonLoadFromString(TestEnvironment::readFileToStringForTest(spans_file_));
+  EXPECT_NE(spans_json, nullptr);
+  EXPECT_EQ(spans_json->asObjectArray().size(), 1);
+}
+
+} // namespace Tracing
+} // namespace Envoy