Skip to content

Commit

Permalink
feat: use aws s3 exporter from contrib (#966)
Browse files Browse the repository at this point in the history
- This PR migrates the aws s3 destination to use the exporter from
[opentelemetery collector
contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/awss3exporter),
instead of the one in this codebase.

- I also introduced 2 new configuration options for this destination,
which are supported in the contrib exporter: `s3_partition` and
`marshaler`.

- Added docs which fix #903 

- Updated our collector to use the latest components `v0.94.0`
  • Loading branch information
blumamir committed Feb 12, 2024
1 parent f34b1db commit ba16a79
Show file tree
Hide file tree
Showing 26 changed files with 966 additions and 1,768 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,10 @@ For more details, see our [quickstart guide](https://docs.odigos.io/intro).
| qryn.cloud ||||
| OpsVerse ||||
| Dynatrace ||||
| AWS S3 ||||
| Google Cloud Monitoring || ||
| Google Cloud Storage || ||
| Azure Blob Storage || ||
| AWS S3 || ||
| Splunk || | |
| Lightstep || | |
| Sentry || | |
Expand Down
72 changes: 51 additions & 21 deletions autoscaler/controllers/gateway/config/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@ import (
commonconf "github.com/keyval-dev/odigos/autoscaler/controllers/common"
"github.com/keyval-dev/odigos/common"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/log"
)

const (
s3BucketKey = "S3_BUCKET"
s3RegionKey = "S3_REGION"
s3BucketKey = "S3_BUCKET"
s3RegionKey = "S3_REGION"
s3PartitionKey = "S3_PARTITION"
s3Marshaller = "S3_MARSHALER"
)

var (
Expand All @@ -26,45 +29,72 @@ func (s *AWSS3) DestType() common.DestinationType {
}

func (s *AWSS3) ModifyConfig(dest *odigosv1.Destination, currentConfig *commonconf.Config) {

if !isLoggingEnabled(dest) && !isTracingEnabled(dest) && !isMetricsEnabled(dest) {
log.Log.V(0).Info("No metrics, logs or traces enabled, gateway will not be configured for AWS S3")
return
}

bucket, ok := dest.Spec.Data[s3BucketKey]
if !ok {
ctrl.Log.Error(ErrS3BucketNotSpecified, "bucket not specified")
ctrl.Log.Error(ErrS3BucketNotSpecified, "s3 bucket not specified")
return
}

region, ok := dest.Spec.Data[s3RegionKey]
if !ok {
ctrl.Log.Error(ErrS3RegionNotSpecified, "region not specified")
ctrl.Log.Error(ErrS3RegionNotSpecified, "s3 region not specified")
return
}

if isLoggingEnabled(dest) {
exporterName := "s3/" + dest.Name
currentConfig.Exporters[exporterName] = commonconf.GenericMap{
"settings": commonconf.GenericMap{
"bucket": bucket,
"region": region,
},
}
partition, ok := dest.Spec.Data[s3PartitionKey]
if !ok {
partition = "minute"
}
if partition != "minute" && partition != "hour" {
log.Log.V(0).Info("Invalid partition specified, gateway will not be configured for AWS S3")
return
}

marshaler, ok := dest.Spec.Data[s3Marshaller]
if !ok {
marshaler = "otlp_json"
}
if marshaler != "otlp_json" && marshaler != "otlp_proto" {
log.Log.V(0).Info("Invalid marshaller specified, gateway will not be configured for AWS S3")
return
}

logsPipelineName := "logs/s3-" + dest.Name
exporterName := "awss3/" + dest.Name
currentConfig.Exporters[exporterName] = commonconf.GenericMap{
"s3uploader": commonconf.GenericMap{
"region": region,
"s3_bucket": bucket,
"s3_partition": partition,
},
"marshaler": marshaler,
}

if isLoggingEnabled(dest) {
logsPipelineName := "logs/awss3-" + dest.Name
currentConfig.Service.Pipelines[logsPipelineName] = commonconf.Pipeline{
Receivers: []string{"otlp"},
Processors: []string{"batch"},
Exporters: []string{exporterName},
}
}

if isTracingEnabled(dest) {
exporterName := "s3/" + dest.Name
currentConfig.Exporters[exporterName] = commonconf.GenericMap{
"settings": commonconf.GenericMap{
"bucket": bucket,
"region": region,
},
if isMetricsEnabled(dest) {
metricsPipelineName := "metrics/awss3-" + dest.Name
currentConfig.Service.Pipelines[metricsPipelineName] = commonconf.Pipeline{
Receivers: []string{"otlp"},
Processors: []string{"batch"},
Exporters: []string{exporterName},
}
}

tracesPipelineName := "traces/s3-" + dest.Name
if isTracingEnabled(dest) {
tracesPipelineName := "traces/awss3-" + dest.Name
currentConfig.Service.Pipelines[tracesPipelineName] = commonconf.Pipeline{
Receivers: []string{"otlp"},
Processors: []string{"batch"},
Expand Down
173 changes: 84 additions & 89 deletions collector/builder-config.yaml

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion collector/exporters/awss3exporter/Makefile

This file was deleted.

2 changes: 0 additions & 2 deletions collector/exporters/awss3exporter/README.md

This file was deleted.

21 changes: 0 additions & 21 deletions collector/exporters/awss3exporter/config.go

This file was deleted.

7 changes: 0 additions & 7 deletions collector/exporters/awss3exporter/data_writer.go

This file was deleted.

79 changes: 0 additions & 79 deletions collector/exporters/awss3exporter/exporter.go

This file was deleted.

74 changes: 0 additions & 74 deletions collector/exporters/awss3exporter/factory.go

This file was deleted.

60 changes: 0 additions & 60 deletions collector/exporters/awss3exporter/go.mod

This file was deleted.

Loading

0 comments on commit ba16a79

Please sign in to comment.