Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add logsagentpipeline component for OTLP ingest paths #25032

Merged
merged 21 commits into from
Apr 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions cmd/agent/subcommands/run/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ import (
netflowServer "github.com/DataDog/datadog-agent/comp/netflow/server"
"github.com/DataDog/datadog-agent/comp/otelcol"
otelcollector "github.com/DataDog/datadog-agent/comp/otelcol/collector"
"github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline"
processAgent "github.com/DataDog/datadog-agent/comp/process/agent"
processagentStatusImpl "github.com/DataDog/datadog-agent/comp/process/status/statusimpl"
remoteconfig "github.com/DataDog/datadog-agent/comp/remote-config"
Expand Down Expand Up @@ -370,6 +371,12 @@ func getSharedFxOption() fx.Option {
compressionimpl.Module(),
demultiplexerimpl.Module(),
dogstatsd.Bundle(),
fx.Provide(func(logsagent optional.Option[logsAgent.Component]) optional.Option[logsagentpipeline.Component] {
liustanley marked this conversation as resolved.
Show resolved Hide resolved
if la, ok := logsagent.Get(); ok {
return optional.NewOption[logsagentpipeline.Component](la)
}
return optional.NewNoneOption[logsagentpipeline.Component]()
}),
otelcol.Bundle(),
rctelemetryreporterimpl.Module(),
rcserviceimpl.Module(),
Expand Down
8 changes: 4 additions & 4 deletions cmd/otel-agent/subcommands/run/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@ import (
"github.com/DataDog/datadog-agent/comp/forwarder"
"github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder"
"github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorinterface"
"github.com/DataDog/datadog-agent/comp/logs"
logsAgent "github.com/DataDog/datadog-agent/comp/logs/agent"
"github.com/DataDog/datadog-agent/comp/metadata/inventoryagent/inventoryagentimpl"
"github.com/DataDog/datadog-agent/comp/otelcol"
"github.com/DataDog/datadog-agent/comp/otelcol/collector"
collectorcontribFx "github.com/DataDog/datadog-agent/comp/otelcol/collector-contrib/fx"
"github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline"
"github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline/logsagentpipelineimpl"
"github.com/DataDog/datadog-agent/comp/serializer/compression"
"github.com/DataDog/datadog-agent/comp/serializer/compression/compressionimpl/strategy"
"github.com/DataDog/datadog-agent/pkg/serializer"
Expand Down Expand Up @@ -96,7 +96,7 @@ func runOTelAgentCommand(_ context.Context, params *subcommands.GlobalParams) er
// TODO configure the log level from collector config
return corelogimpl.ForOneShot(params.LoggerName, "debug", true)
}),
logs.Bundle(),
logsagentpipelineimpl.Module(),
// We create strategy.ZlibStrategy directly to avoid build tags
fx.Provide(strategy.NewZlibStrategy),
fx.Provide(func(s *strategy.ZlibStrategy) compression.Component {
Expand All @@ -117,7 +117,7 @@ func runOTelAgentCommand(_ context.Context, params *subcommands.GlobalParams) er
return defaultforwarder.Forwarder(c), nil
}),
fx.Provide(newOrchestratorinterfaceimpl),
fx.Invoke(func(_ collector.Component, _ defaultforwarder.Forwarder, _ optional.Option[logsAgent.Component]) {
fx.Invoke(func(_ collector.Component, _ defaultforwarder.Forwarder, _ optional.Option[logsagentpipeline.Component]) {
}),
)
if err != nil {
Expand Down
4 changes: 4 additions & 0 deletions comp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,10 @@ Package collector implements the OpenTelemetry Collector component.

Package collectorcontrib defines the OTel collector-contrib component

### [comp/otelcol/logsagentpipeline](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline)

Package logsagentpipeline contains logs agent pipeline component

## [comp/process](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/process) (Component Bundle)

*Datadog Team*: processes
Expand Down
4 changes: 2 additions & 2 deletions comp/otelcol/collector/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ import (
"github.com/DataDog/datadog-agent/comp/core/config"
corelog "github.com/DataDog/datadog-agent/comp/core/log"
"github.com/DataDog/datadog-agent/comp/core/status"
logsagent "github.com/DataDog/datadog-agent/comp/logs/agent"
"github.com/DataDog/datadog-agent/comp/metadata/inventoryagent"
"github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline"
"github.com/DataDog/datadog-agent/comp/otelcol/otlp"
"github.com/DataDog/datadog-agent/pkg/logs/message"
"github.com/DataDog/datadog-agent/pkg/serializer"
Expand Down Expand Up @@ -47,7 +47,7 @@ type dependencies struct {
Serializer serializer.MetricSerializer

// LogsAgent specifies a logs agent
LogsAgent optional.Option[logsagent.Component]
LogsAgent optional.Option[logsagentpipeline.Component]

// InventoryAgent require the inventory metadata payload, allowing otelcol to add data to it.
InventoryAgent inventoryagent.Component
Expand Down
32 changes: 32 additions & 0 deletions comp/otelcol/logsagentpipeline/component.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// Unless explicitly stated otherwise all files in this repository are licensed
// under the Apache License Version 2.0.
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2024-present Datadog, Inc.

// Package logsagentpipeline contains logs agent pipeline component
package logsagentpipeline

import (
"context"

"github.com/DataDog/datadog-agent/pkg/logs/pipeline"
)

// team: opentelemetry

// Component is the component type.
type Component interface {
// GetPipelineProvider gets the pipeline provider
GetPipelineProvider() pipeline.Provider
}

// LogsAgent is a compat version of component for non fx usage
type LogsAgent interface {
Component

// Start sets up the logs agent and starts its pipelines
Start(context.Context) error

// Stop stops the logs agent and all elements of the data pipeline
Stop(context.Context) error
}
159 changes: 159 additions & 0 deletions comp/otelcol/logsagentpipeline/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
module github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline

go 1.21.0

replace (
github.com/DataDog/datadog-agent/cmd/agent/common/path => ../../../cmd/agent/common/path
github.com/DataDog/datadog-agent/comp/core/config => ../../core/config
github.com/DataDog/datadog-agent/comp/core/flare/types => ../../core/flare/types
github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface => ../../core/hostname/hostnameinterface
github.com/DataDog/datadog-agent/comp/core/log => ../../core/log
github.com/DataDog/datadog-agent/comp/core/secrets => ../../core/secrets
github.com/DataDog/datadog-agent/comp/core/telemetry => ../../core/telemetry
github.com/DataDog/datadog-agent/comp/def => ../../def
github.com/DataDog/datadog-agent/comp/logs/agent/config => ../../logs/agent/config
github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../../pkg/collector/check/defaults
github.com/DataDog/datadog-agent/pkg/config/env => ../../../pkg/config/env
github.com/DataDog/datadog-agent/pkg/config/logs => ../../../pkg/config/logs
github.com/DataDog/datadog-agent/pkg/config/model => ../../../pkg/config/model
github.com/DataDog/datadog-agent/pkg/config/setup => ../../../pkg/config/setup
github.com/DataDog/datadog-agent/pkg/config/utils => ../../../pkg/config/utils
github.com/DataDog/datadog-agent/pkg/logs/auditor => ../../../pkg/logs/auditor
github.com/DataDog/datadog-agent/pkg/logs/client => ../../../pkg/logs/client
github.com/DataDog/datadog-agent/pkg/logs/diagnostic => ../../../pkg/logs/diagnostic
github.com/DataDog/datadog-agent/pkg/logs/message => ../../../pkg/logs/message
github.com/DataDog/datadog-agent/pkg/logs/metrics => ../../../pkg/logs/metrics
github.com/DataDog/datadog-agent/pkg/logs/pipeline => ../../../pkg/logs/pipeline
github.com/DataDog/datadog-agent/pkg/logs/processor => ../../../pkg/logs/processor
github.com/DataDog/datadog-agent/pkg/logs/sds => ../../../pkg/logs/sds
github.com/DataDog/datadog-agent/pkg/logs/sender => ../../../pkg/logs/sender
github.com/DataDog/datadog-agent/pkg/logs/sources => ../../../pkg/logs/sources
github.com/DataDog/datadog-agent/pkg/logs/status/statusinterface => ../../../pkg/logs/status/statusinterface
github.com/DataDog/datadog-agent/pkg/logs/status/utils => ../../../pkg/logs/status/utils
github.com/DataDog/datadog-agent/pkg/logs/util/testutils => ../../../pkg/logs/util/testutils
github.com/DataDog/datadog-agent/pkg/status/health => ../../../pkg/status/health
github.com/DataDog/datadog-agent/pkg/telemetry => ../../../pkg/telemetry
github.com/DataDog/datadog-agent/pkg/util/backoff => ../../../pkg/util/backoff
github.com/DataDog/datadog-agent/pkg/util/executable => ../../../pkg/util/executable
github.com/DataDog/datadog-agent/pkg/util/filesystem => ../../../pkg/util/filesystem
github.com/DataDog/datadog-agent/pkg/util/fxutil => ../../../pkg/util/fxutil
github.com/DataDog/datadog-agent/pkg/util/hostname/validate => ../../../pkg/util/hostname/validate
github.com/DataDog/datadog-agent/pkg/util/http => ../../../pkg/util/http
github.com/DataDog/datadog-agent/pkg/util/log => ../../../pkg/util/log
github.com/DataDog/datadog-agent/pkg/util/optional => ../../../pkg/util/optional
github.com/DataDog/datadog-agent/pkg/util/pointer => ../../../pkg/util/pointer
github.com/DataDog/datadog-agent/pkg/util/scrubber => ../../../pkg/util/scrubber
github.com/DataDog/datadog-agent/pkg/util/startstop => ../../../pkg/util/startstop
github.com/DataDog/datadog-agent/pkg/util/statstracker => ../../../pkg/util/statstracker
github.com/DataDog/datadog-agent/pkg/util/system => ../../../pkg/util/system
github.com/DataDog/datadog-agent/pkg/util/system/socket => ../../../pkg/util/system/socket
github.com/DataDog/datadog-agent/pkg/util/testutil => ../../../pkg/util/testutil
github.com/DataDog/datadog-agent/pkg/util/winutil => ../../../pkg/util/winutil
github.com/DataDog/datadog-agent/pkg/version => ../../../pkg/version
)

require github.com/DataDog/datadog-agent/pkg/logs/pipeline v0.0.0-00010101000000-000000000000

require (
github.com/DataDog/agent-payload/v5 v5.0.106 // indirect
github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/comp/core/secrets v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/comp/core/telemetry v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/comp/def v0.0.0-00010101000000-000000000000 // indirect
github.com/DataDog/datadog-agent/comp/logs/agent/config v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/config/env v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/config/model v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/config/setup v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/config/utils v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/logs/auditor v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/logs/client v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/logs/diagnostic v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/logs/message v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/logs/metrics v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/logs/processor v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/logs/sds v0.0.0-00010101000000-000000000000 // indirect
github.com/DataDog/datadog-agent/pkg/logs/sender v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/logs/sources v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/logs/status/statusinterface v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/logs/status/utils v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/status/health v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/telemetry v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/backoff v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/executable v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/filesystem v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/fxutil v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/http v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/log v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/optional v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/pointer v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/scrubber v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/startstop v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/statstracker v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/system v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/system/socket v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/util/winutil v0.53.0-rc.2 // indirect
github.com/DataDog/datadog-agent/pkg/version v0.53.0-rc.2 // indirect
github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe // indirect
github.com/DataDog/viper v1.13.2 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect
github.com/magiconair/properties v1.8.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/pelletier/go-toml v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
github.com/prometheus/client_golang v1.17.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/shirou/gopsutil/v3 v3.24.1 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/cobra v1.7.0 // indirect
github.com/spf13/jwalterweatherman v1.0.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.9.0 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/exporters/prometheus v0.42.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/sdk v1.20.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.20.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/dig v1.17.0 // indirect
go.uber.org/fx v1.18.2 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.18.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Loading
Loading