feat: add telemetry logs (#185)

Author: dmehala

BUILD.bazel

@@ -3,6 +3,7 @@ cc_library(
     srcs = [
       "src/datadog/telemetry/configuration.cpp",
       "src/datadog/telemetry/metrics.cpp",
+      "src/datadog/telemetry/log.h",
       "src/datadog/telemetry/telemetry.cpp",
       "src/datadog/base64.cpp",
       "src/datadog/cerr_logger.cpp",

include/datadog/environment.h

@@ -54,7 +54,8 @@ namespace environment {
   MACRO(DD_TELEMETRY_HEARTBEAT_INTERVAL)             \
   MACRO(DD_TELEMETRY_METRICS_ENABLED)                \
   MACRO(DD_TELEMETRY_METRICS_INTERVAL_SECONDS)       \
-  MACRO(DD_TELEMETRY_DEBUG)
+  MACRO(DD_TELEMETRY_DEBUG)                          \
+  MACRO(DD_TELEMETRY_LOG_COLLECTION_ENABLED)
 
 #define WITH_COMMA(ARG) ARG,
 

include/datadog/telemetry/configuration.h

@@ -33,12 +33,18 @@ struct Configuration {
   // library.
   // Example: "1.2.3", "6c44da20", "2020.02.13"
   tracing::Optional<std::string> integration_version;
+  // Enable or disable telemetry logs collection.
+  // Default: enabled.
+  // Can be overriden by the `DD_TELEMETRY_LOG_COLLECTION_ENABLED` environment
+  // variable.
+  tracing::Optional<bool> report_logs;
 };
 
 struct FinalizedConfiguration {
   bool debug;
   bool enabled;
   bool report_metrics;
+  bool report_logs;
   std::chrono::steady_clock::duration metrics_interval;
   std::chrono::steady_clock::duration heartbeat_interval;
   std::string integration_name;

include/datadog/telemetry/telemetry.h

@@ -16,19 +16,40 @@ class TracerTelemetry;
 
 namespace telemetry {
 
-class Telemetry {
+/// The telemetry class is responsible for handling internal telemetry data to
+/// track Datadog product usage. It _can_ collect and report logs and metrics.
+///
+/// IMPORTANT: This is intended for use only by Datadog Engineers.
+class Telemetry final {
+  /// Configuration object containing the validated settings for telemetry
   FinalizedConfiguration config_;
+  /// Shared pointer to the user logger instance.
   std::shared_ptr<tracing::Logger> logger_;
-
+  /// TODO(@dmehala): Legacy dependency.
   std::shared_ptr<tracing::DatadogAgent> datadog_agent_;
   std::shared_ptr<tracing::TracerTelemetry> tracer_telemetry_;
 
  public:
+  /// Constructor for the Telemetry class
+  ///
+  /// @param configuration The finalized configuration settings.
+  /// @param logger User logger instance.
+  /// @param metrics A vector user metrics to report.
   Telemetry(FinalizedConfiguration configuration,
             std::shared_ptr<tracing::Logger> logger,
             std::vector<std::shared_ptr<Metric>> metrics);
 
   ~Telemetry() = default;
+
+  /// Capture and report internal error message to Datadog.
+  ///
+  /// @param message The error message.
+  void log_error(std::string message);
+
+  /// capture and report internal warning message to Datadog.
+  ///
+  /// @param message The warning message to log.
+  void log_warning(std::string message);
 };
 
 }  // namespace telemetry

src/datadog/telemetry/configuration.cpp

@@ -24,6 +24,11 @@ tracing::Expected<Configuration> load_telemetry_env_config() {
     env_cfg.report_metrics = !falsy(*metrics_enabled);
   }
 
+  if (auto logs_enabled =
+          lookup(environment::DD_TELEMETRY_LOG_COLLECTION_ENABLED)) {
+    env_cfg.report_logs = !falsy(*logs_enabled);
+  }
+
   if (auto metrics_interval_seconds =
           lookup(environment::DD_TELEMETRY_METRICS_INTERVAL_SECONDS)) {
     auto maybe_value = parse_double(*metrics_interval_seconds);
@@ -66,10 +71,15 @@ tracing::Expected<FinalizedConfiguration> finalize_config(
     // NOTE(@dmehala): if the telemetry module is disabled then report metrics
     // is also disabled.
     result.report_metrics = false;
+    result.report_logs = false;
   } else {
     // report_metrics
     std::tie(origin, result.report_metrics) =
         pick(env_config->report_metrics, user_config.report_metrics, true);
+
+    // report_logs
+    std::tie(origin, result.report_logs) =
+        pick(env_config->report_logs, user_config.report_logs, true);
   }
 
   // debug

src/datadog/telemetry/log.h

@@ -0,0 +1,12 @@
+#include <string>
+
+namespace datadog::telemetry {
+
+enum class LogLevel : char { ERROR, WARNING };
+
+struct LogMessage final {
+  std::string message;
+  LogLevel level;
+};
+
+}  // namespace datadog::telemetry

src/datadog/telemetry/telemetry.cpp

@@ -39,5 +39,13 @@ Telemetry::Telemetry(FinalizedConfiguration config,
       std::vector<std::shared_ptr<remote_config::Listener>>{});
 }
 
+void Telemetry::log_error(std::string message) {
+  tracer_telemetry_->log(std::move(message), LogLevel::ERROR);
+}
+
+void Telemetry::log_warning(std::string message) {
+  tracer_telemetry_->log(std::move(message), LogLevel::WARNING);
+}
+
 }  // namespace telemetry
 }  // namespace datadog

src/datadog/tracer_telemetry.cpp

@@ -292,6 +292,27 @@ std::string TracerTelemetry::heartbeat_and_telemetry() {
     batch_payloads.emplace_back(std::move(generate_metrics));
   }
 
+  if (!logs_.empty()) {
+    auto encoded_logs = nlohmann::json::array();
+    for (const auto& log : logs_) {
+      auto encoded =
+          nlohmann::json{{"message", log.message}, {"level", log.level}};
+      encoded_logs.emplace_back(std::move(encoded));
+    }
+
+    assert(!encoded_logs.empty());
+
+    auto logs_payload = nlohmann::json::object({
+        {"request_type", "logs"},
+        {"payload",
+         nlohmann::json{
+             {"logs", encoded_logs},
+         }},
+    });
+
+    batch_payloads.emplace_back(std::move(logs_payload));
+  }
+
   auto telemetry_body = generate_telemetry_body("message-batch");
   telemetry_body["payload"] = batch_payloads;
   auto message_batch_payload = telemetry_body.dump();
@@ -348,6 +369,27 @@ std::string TracerTelemetry::app_closing() {
     batch_payloads.emplace_back(std::move(generate_metrics));
   }
 
+  if (!logs_.empty()) {
+    auto encoded_logs = nlohmann::json::array();
+    for (const auto& log : logs_) {
+      auto encoded =
+          nlohmann::json{{"message", log.message}, {"level", log.level}};
+      encoded_logs.emplace_back(std::move(encoded));
+    }
+
+    assert(!encoded_logs.empty());
+
+    auto logs_payload = nlohmann::json::object({
+        {"request_type", "logs"},
+        {"payload",
+         nlohmann::json{
+             {"logs", encoded_logs},
+         }},
+    });
+
+    batch_payloads.emplace_back(std::move(logs_payload));
+  }
+
   auto telemetry_body = generate_telemetry_body("message-batch");
   telemetry_body["payload"] = batch_payloads;
   auto message_batch_payload = telemetry_body.dump();

src/datadog/tracer_telemetry.h

@@ -38,6 +38,7 @@
 
 #include "json.hpp"
 #include "platform_util.h"
+#include "telemetry/log.h"
 
 namespace datadog {
 namespace tracing {
@@ -47,7 +48,7 @@ struct SpanDefaults;
 
 class TracerTelemetry {
   bool enabled_ = false;
-  bool debug_ = false;
+  bool debug_ = true;
   Clock clock_;
   std::shared_ptr<Logger> logger_;
   HostInfo host_info_;
@@ -120,6 +121,8 @@ class TracerTelemetry {
 
   std::vector<std::shared_ptr<telemetry::Metric>> user_metrics_;
 
+  std::vector<telemetry::LogMessage> logs_;
+
  public:
   TracerTelemetry(
       bool enabled, const Clock& clock, const std::shared_ptr<Logger>& logger,
@@ -151,6 +154,10 @@ class TracerTelemetry {
   std::string app_closing();
   // Construct an `app-client-configuration-change` message.
   Optional<std::string> configuration_change();
+
+  inline void log(std::string message, telemetry::LogLevel level) {
+    logs_.emplace_back(telemetry::LogMessage{std::move(message), level});
+  }
 };
 
 }  // namespace tracing

test/telemetry/test_configuration.cpp

@@ -1,8 +1,6 @@
 #include <datadog/environment.h>
 #include <datadog/telemetry/configuration.h>
 
-#include <chrono>
-
 #include "../common/environment.h"
 #include "../test.h"
 
@@ -19,6 +17,7 @@ TELEMETRY_CONFIGURATION_TEST("defaults") {
   REQUIRE(cfg);
   CHECK(cfg->debug == false);
   CHECK(cfg->enabled == true);
+  CHECK(cfg->report_logs == true);
   CHECK(cfg->report_metrics == true);
   CHECK(cfg->metrics_interval == 60s);
   CHECK(cfg->heartbeat_interval == 10s);
@@ -27,6 +26,7 @@ TELEMETRY_CONFIGURATION_TEST("defaults") {
 TELEMETRY_CONFIGURATION_TEST("code override") {
   telemetry::Configuration cfg;
   cfg.enabled = false;
+  cfg.report_logs = false;
   cfg.report_metrics = false;
   cfg.metrics_interval_seconds = 1;
   cfg.heartbeat_interval_seconds = 2;
@@ -37,6 +37,7 @@ TELEMETRY_CONFIGURATION_TEST("code override") {
   REQUIRE(final_cfg);
   CHECK(final_cfg->enabled == false);
   CHECK(final_cfg->debug == false);
+  CHECK(final_cfg->report_logs == false);
   CHECK(final_cfg->report_metrics == false);
   CHECK(final_cfg->metrics_interval == 1s);
   CHECK(final_cfg->heartbeat_interval == 2s);
@@ -48,11 +49,13 @@ TELEMETRY_CONFIGURATION_TEST("enabled and report metrics precedence") {
   SECTION("enabled takes precedence over metrics enabled") {
     telemetry::Configuration cfg;
     cfg.enabled = false;
+    cfg.report_logs = true;
     cfg.report_metrics = true;
 
     auto final_cfg = finalize_config(cfg);
     REQUIRE(final_cfg);
     CHECK(final_cfg->enabled == false);
+    CHECK(final_cfg->report_logs == false);
     CHECK(final_cfg->report_metrics == false);
   }
 }
@@ -84,6 +87,14 @@ TELEMETRY_CONFIGURATION_TEST("environment environment override") {
     CHECK(final_cfg->report_metrics == false);
   }
 
+  SECTION("Override `report_logs` field") {
+    cfg.report_logs = true;
+    ddtest::EnvGuard env("DD_TELEMETRY_LOG_COLLECTION_ENABLED", "false");
+    auto final_cfg = telemetry::finalize_config(cfg);
+    REQUIRE(final_cfg);
+    CHECK(final_cfg->report_logs == false);
+  }
+
   SECTION("Override metrics interval") {
     cfg.metrics_interval_seconds = 88;
     ddtest::EnvGuard env("DD_TELEMETRY_METRICS_INTERVAL_SECONDS", "15");