feat: otel collector integration (#4769)

* chore: WIP for an otel-collector provider * feat: create exporter that can be configured at a later point and then exports spans * feat: send traces to OTLP exporter once collector is spawned * feat: wait for initialization of collector * chore: remove redundant createModuleTypes * feat: create temporary config if it does not exist * wip: start creating config for datadog * feat: ensure that async hooks are run when the process is shut down * feat: log logged processes output with silly loglevel * feat: ensure collector has enough time to terminate * feat: wait for datadog export log line explicitly * refactor: simplify config generation and get actual hostname * feat: make datadog configurable from the API secrets * feat: support multiple concurrent http exporters, refactor config * refactor: extract out base config * feat: trace the otel collector startup times * feat: configure exporters in provider config * feat: timeout for final datadog sync * feat: wait for collector process to terminate * feat: add honeycomb support * fix: bind to localhost, handle empty exporters * fix: otel provider example and added the logging exporter * chore: remove unused import * fix: work around anyOf validation to schema conversion Should receive the same follow up fix as `oneOf` * fix: license headers * chore: code formatting * docs: update docs * chore: remove two todos * chore: remove projectId from plugin context * refactor: move validators into exporter modules and use those for the config types * feat: always enable tracing by default but no-op when it's not configured * fix: remove console.log statements * feat: address review comments * fix: make sure when there are no active otel exporters the no-op exporter is used * test: e2e smoke test for otel exporter * fix: correct hostname for test * fix: explicitly forward log events from provider instead of having `streamLogs` do it implicitly --------- Co-authored-by: Mikael Hoegqvist Tabor <mikael@garden.io>
garden-io · Jul 17, 2023 · 9c44055 · 9c44055
1 parent 135ea04
commit 9c44055
Show file tree

Hide file tree

Showing 68 changed files with 4,544 additions and 116 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -933,6 +933,11 @@ workflows:
           name: e2e-vote-helm
           project: vote-helm
           requires: [build]
+      - e2e-project:
+          <<: *only-internal-prs
+          name: e2e-open-telemetry
+          project: open-telemetry
+          requires: [build]
       # - e2e-project:
       #     <<: *only-internal-prs
       #     name: e2e-vote-helm-modules

diff --git a/cli/bin/garden b/cli/bin/garden
@@ -35,7 +35,7 @@ if (process.env.GARDEN_ENABLE_PROFILING === "1" || process.env.GARDEN_ENABLE_PRO
   }
 }
 
-const { initTracing } = require("@garden-io/core/build/src/util/tracing/tracing")
+const { initTracing } = require("@garden-io/core/build/src/util/open-telemetry/tracing")
 initTracing()
 
 require("source-map-support").install()

diff --git a/cli/src/cli.ts b/cli/src/cli.ts
@@ -10,9 +10,9 @@ import { shutdown } from "@garden-io/core/build/src/util/util"
 import { GardenCli, RunOutput } from "@garden-io/core/build/src/cli/cli"
 import { GardenPluginReference } from "@garden-io/core/build/src/plugin/plugin"
 import { GlobalConfigStore } from "@garden-io/core/build/src/config-store/global"
-import { getOtelSDK } from "@garden-io/core/build/src/util/tracing/tracing"
-import { withContextFromEnv } from "@garden-io/core/build/src/util/tracing/propagation"
-import { wrapActiveSpan } from "@garden-io/core/build/src/util/tracing/spans"
+import { getOtelSDK } from "@garden-io/core/build/src/util/open-telemetry/tracing"
+import { withContextFromEnv } from "@garden-io/core/build/src/util/open-telemetry/propagation"
+import { wrapActiveSpan } from "@garden-io/core/build/src/util/open-telemetry/spans"
 
 // These plugins are always registered
 export const getBundledPlugins = (): GardenPluginReference[] => [

diff --git a/core/package.json b/core/package.json
@@ -34,12 +34,13 @@
     "@opentelemetry/semantic-conventions": "^1.14.0",
     "@opentelemetry/instrumentation-http": "^0.40.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.40.0",
+    "@opentelemetry/otlp-exporter-base": "^0.40.0",
     "ajv": "^8.12.0",
     "ajv-formats": "^2.1.1",
     "analytics-node": "3.5.0",
     "archiver": "^5.3.1",
     "async": "^3.2.4",
-    "async-exit-hook": "^2.0.1",
+    "@scg82/exit-hook": "^3.4.1",
     "async-lock": "^1.4.0",
     "bluebird": "^3.7.2",
     "certpem": "^1.1.3",
@@ -248,7 +249,7 @@
     "testdouble-chai": "^0.5.0",
     "timekeeper": "^2.2.0",
     "touch": "^3.1.0",
-    "type-fest": "^2.19.0",
+    "type-fest": "^3.12.0",
     "typescript": "^5.1.3",
     "utility-types": "^3.10.0"
   },

diff --git a/core/src/cli/cli.ts b/core/src/cli/cli.ts
@@ -49,8 +49,8 @@ import { dedent } from "../util/string"
 import { GardenProcess, GlobalConfigStore } from "../config-store/global"
 import { registerProcess, waitForOutputFlush } from "../process"
 import { uuidv4 } from "../util/random"
-import { withSessionContext } from "../util/tracing/context"
-import { wrapActiveSpan } from "../util/tracing/spans"
+import { withSessionContext } from "../util/open-telemetry/context"
+import { wrapActiveSpan } from "../util/open-telemetry/spans"
 
 export interface RunOutput {
   argv: any

diff --git a/core/src/cli/command-line.ts b/core/src/cli/command-line.ts
@@ -36,8 +36,8 @@ import {
   renderCommands,
 } from "./helpers"
 import type { GlobalOptions, ParameterValues } from "./params"
-import { bindActiveContext, withSessionContext } from "../util/tracing/context"
-import { wrapActiveSpan } from "../util/tracing/spans"
+import { bindActiveContext, withSessionContext } from "../util/open-telemetry/context"
+import { wrapActiveSpan } from "../util/open-telemetry/spans"
 
 const defaultMessageDuration = 3000
 const commandLinePrefix = chalk.yellow("🌼  > ")

diff --git a/core/src/commands/base.ts b/core/src/commands/base.ts
@@ -55,8 +55,8 @@ import { GraphResultMapWithoutTask, GraphResultWithoutTask, GraphResults } from
 import { splitFirst } from "../util/string"
 import { ActionMode } from "../actions/types"
 import { AnalyticsHandler } from "../analytics/analytics"
-import { withSessionContext } from "../util/tracing/context"
-import { wrapActiveSpan } from "../util/tracing/spans"
+import { withSessionContext } from "../util/open-telemetry/context"
+import { wrapActiveSpan } from "../util/open-telemetry/spans"
 
 export interface CommandConstructor {
   new (parent?: CommandGroup): Command

diff --git a/core/src/commands/custom.ts b/core/src/commands/custom.ts
@@ -33,7 +33,7 @@ import { removeSlice } from "../util/util"
 import { join } from "path"
 import { getBuiltinCommands } from "./commands"
 import { Log } from "../logger/log-entry"
-import { getTracePropagationEnvVars } from "../util/tracing/propagation"
+import { getTracePropagationEnvVars } from "../util/open-telemetry/propagation"
 
 function convertArgSpec(spec: CustomCommandOption) {
   const params = {

diff --git a/core/src/commands/dev.tsx b/core/src/commands/dev.tsx
@@ -23,7 +23,7 @@ import moment from "moment"
 import { dedent } from "../util/string"
 import Spinner from "ink-spinner"
 import type { Log } from "../logger/log-entry"
-import { bindActiveContext } from "../util/tracing/context"
+import { bindActiveContext } from "../util/open-telemetry/context"
 
 const devCommandArgs = {
   ...serveArgs,

diff --git a/core/src/config/zod.ts b/core/src/config/zod.ts
@@ -6,7 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import { Schema, z } from "zod"
+import { Schema, z, infer as inferZodType, ZodType } from "zod"
 import { envVarRegex, identifierRegex, joiIdentifierDescription, userIdentifierRegex } from "./constants"
 
 // Add metadata support to schemas. See https://github.com/colinhacks/zod/issues/273#issuecomment-1434077058
@@ -76,6 +76,7 @@ type GardenSchema = typeof z & {
 
 // This should be imported instead of z because we augment zod with custom methods
 export const s = z as GardenSchema
+export type inferType<T extends ZodType<any, any, any>> = inferZodType<T>
 
 s.envVars = () => s.record(s.string().regex(envVarRegex).min(1), z.string())
 

diff --git a/core/src/constants.ts b/core/src/constants.ts
@@ -77,5 +77,5 @@ export const gardenEnv = {
   GARDEN_HARD_CONCURRENCY_LIMIT: env.get("GARDEN_HARD_CONCURRENCY_LIMIT").required(false).default(50).asInt(),
   GARDEN_WORKFLOW_RUN_UID: env.get("GARDEN_WORKFLOW_RUN_UID").required(false).asString(),
   GARDEN_CLOUD_DOMAIN: env.get("GARDEN_CLOUD_DOMAIN").required(false).asUrlString(),
-  GARDEN_ENABLE_TRACING: env.get("GARDEN_ENABLE_TRACING").required(false).asBool(),
+  GARDEN_ENABLE_TRACING: env.get("GARDEN_ENABLE_TRACING").required(false).default("true").asBool(),
 }
diff --git a/core/src/docs/json-schema.ts b/core/src/docs/json-schema.ts
@@ -39,6 +39,11 @@ export class JsonKeyDescription<T = any> extends BaseKeyDescription<T> {
       schema = schema.oneOf[0]
     }
 
+    // FIXME: We only use the first type if there are multiple possible schemas
+    if (schema.anyOf) {
+      schema = schema.anyOf[0]
+    }
+
     this.schema = schema
     this.type = getType(schema)
 
@@ -119,6 +124,10 @@ export class JsonKeyDescription<T = any> extends BaseKeyDescription<T> {
         itemsSchema = itemsSchema.oneOf[0]
       }
 
+      if (itemsSchema.anyOf) {
+        itemsSchema = itemsSchema.anyOf[0]
+      }
+
       return [
         new JsonKeyDescription({
           schema: itemsSchema,

diff --git a/core/src/garden.ts b/core/src/garden.ts
@@ -149,9 +149,10 @@ import { MonitorManager } from "./monitors/manager"
 import { AnalyticsHandler } from "./analytics/analytics"
 import { getGardenInstanceKey } from "./server/helpers"
 import { SuggestedCommand } from "./commands/base"
-import { OtelTraced } from "./util/tracing/decorators"
-import { wrapActiveSpan } from "./util/tracing/spans"
+import { OtelTraced } from "./util/open-telemetry/decorators"
+import { wrapActiveSpan } from "./util/open-telemetry/spans"
 import { GitRepoHandler } from "./vcs/git-repo"
+import { configureNoOpExporter } from "./util/open-telemetry/tracing"
 
 const defaultLocalAddress = "localhost"
 
@@ -397,6 +398,24 @@ export class Garden {
     this.version = getPackageVersion()
     this.monitors = params.monitors || new MonitorManager(this.log, this.events)
     this.solver = new GraphSolver(this)
+
+    // In order not to leak memory, we should ensure that there's always a collector for the OTEL data
+    // Here we check if the otel-collector was configured and we set a NoOp exporter if it was not
+    // This is of course not entirely ideal since this puts into this class some level of coupling
+    // with the plugin based otel-collector.
+    // Since we don't have the ability to hook into the post provider init stage from within the provider plugin
+    // especially because it's the absence of said provider that needs to trigger this case,
+    // there isn't really a cleaner way around this for now.
+    const providerConfigs = this.getRawProviderConfigs()
+
+    const hasOtelCollectorProvider = providerConfigs.some((providerConfig) => {
+      return providerConfig.name === "otel-collector"
+    })
+
+    if (!hasOtelCollectorProvider) {
+      this.log.silly("No OTEL collector configured, setting no-op exporter")
+      configureNoOpExporter()
+    }
   }
 
   static async factory<T extends typeof Garden>(

diff --git a/core/src/index.ts b/core/src/index.ts
@@ -6,7 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import { initTracing } from "./util/tracing/tracing"
+import { initTracing } from "./util/open-telemetry/tracing"
 initTracing()
 
 export { Garden } from "./garden"
diff --git a/core/src/plugin/handlers/Build/get-status.ts b/core/src/plugin/handlers/Build/get-status.ts
@@ -48,7 +48,8 @@ export const buildResultSchema = createSchema({
   }),
 })
 
-export interface BuildStatus<T extends BuildAction = BuildAction, D extends {} = BuildResult> extends ActionStatus<T, D> {}
+export interface BuildStatus<T extends BuildAction = BuildAction, D extends {} = BuildResult>
+  extends ActionStatus<T, D> {}
 
 export interface BuildStatusMap extends ActionStatusMap<BuildAction> {
   [key: string]: BuildStatus

diff --git a/core/src/plugins/exec/deploy.ts b/core/src/plugins/exec/deploy.ts
@@ -36,7 +36,7 @@ import { convertCommandSpec, execRunCommand, getDefaultEnvVars } from "./common"
 import { isRunning, killRecursive } from "../../process"
 import { sdk } from "../../plugin/sdk"
 import { execProvider } from "./exec"
-import { getTracePropagationEnvVars } from "../../util/tracing/propagation"
+import { getTracePropagationEnvVars } from "../../util/open-telemetry/propagation"
 import { DeployState } from "../../types/service"
 
 const persistentLocalProcRetryIntervalMs = 2500

diff --git a/core/src/plugins/kubernetes/container/build/buildkit.ts b/core/src/plugins/kubernetes/container/build/buildkit.ts
@@ -10,7 +10,14 @@ import AsyncLock from "async-lock"
 import chalk from "chalk"
 import split2 = require("split2")
 import { isEmpty } from "lodash"
-import { buildSyncVolumeName, buildkitContainerName, buildkitDeploymentName, buildkitImageName, buildkitRootlessImageName, dockerAuthSecretKey } from "../../constants"
+import {
+  buildSyncVolumeName,
+  buildkitContainerName,
+  buildkitDeploymentName,
+  buildkitImageName,
+  buildkitRootlessImageName,
+  dockerAuthSecretKey,
+} from "../../constants"
 import { KubeApi } from "../../api"
 import { KubernetesDeployment } from "../../types"
 import { Log } from "../../../../logger/log-entry"

diff --git a/core/src/plugins/otel-collector/config.ts b/core/src/plugins/otel-collector/config.ts
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2018-2023 Garden Technologies, Inc. <info@garden.io>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import { mergeWith } from "lodash"
+import { MergeDeep } from "type-fest"
+import {
+  OtlpHttpExporterConfigPartial,
+  OtelCollectorOtlpHttpConfiguration,
+  makeOtlpHttpPartialConfig,
+} from "./config/otlphttp"
+import {
+  DatadogExporterConfigPartial,
+  OtelCollectorDatadogConfiguration,
+  makeDatadogPartialConfig,
+} from "./config/datadog"
+import { OtelCollectorNewRelicConfiguration, makeNewRelicPartialConfig } from "./config/newrelic"
+import { OtelCollectorBaseConfig, getOtelCollectorBaseConfig } from "./config/base"
+import { OtelCollectorHoneycombConfiguration, makeHoneycombPartialConfig } from "./config/honeycomb"
+import { OtelCollectorLoggingConfiguration, makeLoggingPartialConfig } from "./config/logging"
+
+export type OtelConfigFile = MergeDeep<
+  OtelCollectorBaseConfig,
+  MergeDeep<OtlpHttpExporterConfigPartial, DatadogExporterConfigPartial, { arrayMergeMode: "spread" }>,
+  { arrayMergeMode: "spread" }
+>
+
+export type OtelExportersConfig =
+  | OtelCollectorLoggingConfiguration
+  | OtelCollectorDatadogConfiguration
+  | OtelCollectorNewRelicConfiguration
+  | OtelCollectorOtlpHttpConfiguration
+  | OtelCollectorHoneycombConfiguration
+
+export type OtelCollectorConfigFileOptions = {
+  exporters: OtelExportersConfig[]
+}
+
+function mergeArrays(objValue, srcValue) {
+  if (Array.isArray(objValue)) {
+    return objValue.concat(srcValue)
+  }
+  return undefined
+}
+
+export function getOtelCollectorConfigFile({ exporters }: OtelCollectorConfigFileOptions) {
+  let config: OtelConfigFile = getOtelCollectorBaseConfig()
+
+  for (const exporter of exporters) {
+    if (exporter.enabled) {
+      if (exporter.name === "datadog") {
+        config = mergeWith(config, makeDatadogPartialConfig(exporter), mergeArrays)
+      }
+      if (exporter.name === "newrelic") {
+        config = mergeWith(config, makeNewRelicPartialConfig(exporter), mergeArrays)
+      }
+      if (exporter.name === "otlphttp") {
+        config = mergeWith(config, makeOtlpHttpPartialConfig(exporter), mergeArrays)
+      }
+      if (exporter.name === "honeycomb") {
+        config = mergeWith(config, makeHoneycombPartialConfig(exporter), mergeArrays)
+      }
+      if (exporter.name === "logging") {
+        config = mergeWith(config, makeLoggingPartialConfig(exporter), mergeArrays)
+      }
+    }
+  }
+
+  return config
+}
diff --git a/core/src/plugins/otel-collector/config/base.ts b/core/src/plugins/otel-collector/config/base.ts
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2018-2023 Garden Technologies, Inc. <info@garden.io>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import { sdk } from "../../../plugin/sdk"
+
+const s = sdk.schema
+
+export const baseValidator = s.object({
+  name: s.string(),
+  enabled: s.boolean(),
+})
+
+export type OtelCollectorBaseConfig = {
+  processors: {
+    batch: null | {
+      send_batch_max_size?: number
+      timeout?: string
+    }
+  }
+  exporters: {}
+  extensions: Record<string, null>
+  service: {
+    extensions: string[]
+    pipelines: {
+      traces: {
+        receivers: ["otlp"]
+        processors: ["batch"]
+        exporters: string[]
+      }
+    }
+    telemetry: {
+      logs: {
+        level: string
+      }
+    }
+  }
+}
+
+export function getOtelCollectorBaseConfig(): OtelCollectorBaseConfig {
+  return {
+    processors: {
+      batch: null,
+    },
+    exporters: {},
+    extensions: {
+      health_check: null,
+      pprof: null,
+      zpages: null,
+    },
+    service: {
+      extensions: [],
+      pipelines: {
+        traces: {
+          receivers: ["otlp"],
+          processors: ["batch"],
+          exporters: [],
+        },
+      },
+      telemetry: {
+        logs: {
+          level: "debug",
+        },
+      },
+    },
+  }
+}