Server: Add support for shipping traces to an otel collector

Makefile

@@ -17,6 +17,7 @@ GOLANGCI_LINT_VERSION=v1.51.0
 EMBEDMD_BIN=$(BIN_DIR)/embedmd
 THANOS_BIN=$(BIN_DIR)/thanos
 UP_BIN=$(BIN_DIR)/up
+OTELCOL_BIN=$(BIN_DIR)/otelcol
 MEMCACHED_BIN=$(BIN_DIR)/memcached
 PROMETHEUS_BIN=$(BIN_DIR)/prometheus
 GOJSONTOYAML_BIN=$(BIN_DIR)/gojsontoyaml
@@ -202,7 +203,7 @@ test/timeseries.txt:
 test-rhel-generate-e2e-certs:
 	./test/generate-e2e-certs.sh $(TEST_E2E_CERTS_DIR)
 
-test-rhelemeter-integration: build | $(THANOS_BIN) $(UP_BIN) $(PROMETHEUS_BIN)
+test-rhelemeter-integration: build | $(THANOS_BIN) $(UP_BIN) $(PROMETHEUS_BIN) $(OTELCOL_BIN)
 	@echo "Running rhelemeter integration tests"
 	PATH=$$PATH:$$(pwd)/$(BIN_DIR) LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$$(pwd)/$(LIB_DIR) ./test/integration-rhelemeter.sh
 
@@ -227,6 +228,10 @@ $(PROMETHEUS_BIN): $(BIN_DIR)
 	@echo "Downloading Prometheus"
 	curl -L "https://github.com/prometheus/prometheus/releases/download/v2.3.2/prometheus-2.3.2.$$(go env GOOS)-$$(go env GOARCH).tar.gz" | tar --strip-components=1 -xzf - -C $(BIN_DIR)
 
+$(OTELCOL_BIN): $(BIN_DIR)
+	@echo "Downloading the OTEL collector"
+	curl -L "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.86.0/otelcol_0.86.0_$$(go env GOOS)_$$(go env GOARCH).tar.gz" | tar -xzf - -C $(BIN_DIR)
+
 $(GOLANGCI_LINT_BIN): $(BIN_DIR)
 	curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/$(GOLANGCI_LINT_VERSION)/install.sh \
 		| sed -e '/install -d/d' \

cmd/rhelemeter-server/main.go

@@ -18,18 +18,18 @@ import (
 
 	"github.com/coreos/go-oidc"
 	"github.com/go-chi/chi"
-	"github.com/go-chi/chi/middleware"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/oklog/run"
-	"github.com/openshift/telemeter/pkg/tracing"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/cobra"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.opentelemetry.io/otel"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
 
+	"github.com/openshift/telemeter/pkg/tracing"
+
 	"github.com/openshift/telemeter/pkg/authorize/ssl"
 	telemeter_http "github.com/openshift/telemeter/pkg/http"
 	"github.com/openshift/telemeter/pkg/logger"
@@ -38,7 +38,7 @@ import (
 )
 
 const desc = `
-Server for receiving Prometheus metrics through the remote_write API. Clients are authenticated 
+Server for receiving Prometheus metrics through the remote_write API. Clients are authenticated
 with mTLS.
 `
 
@@ -111,6 +111,8 @@ func main() {
 		"The service name to report to the tracing backend.")
 	cmd.Flags().StringVar(&opt.TracingEndpoint, "internal.tracing.endpoint", "",
 		"The full URL of the trace collector. If it's not set, tracing will be disabled.")
+	cmd.Flags().BoolVar(&opt.TracingInsecure, "internal.tracing.insecure", false,
+		"Allow insecure connections to the tracing endpoint.")
 	cmd.Flags().Float64Var(&opt.TracingSamplingFraction, "internal.tracing.sampling-fraction", 0.1,
 		"The fraction of traces to sample. Thus, if you set this to .5, half of traces will be sampled.")
 	cmd.Flags().StringVar(&opt.TracingEndpointType, "internal.tracing.endpoint-type", string(tracing.EndpointTypeAgent),
@@ -170,6 +172,7 @@ type Options struct {
 	TracingEndpoint         string
 	TracingEndpointType     string
 	TracingSamplingFraction float64
+	TracingInsecure         bool
 
 	Verbose bool
 }
@@ -309,7 +312,7 @@ func (o *Options) Run(ctx context.Context, externalListener, internalListener ne
 	{
 		var hasClientCertConfig bool
 		external := chi.NewRouter()
-		external.Use(middleware.RequestID)
+		external.Use(logger.RequestLoggerWithTraceInfo(o.Logger))
 
 		if o.ClientInfoFromRequestConfigFile != "" {
 			hasClientCertConfig = true

cmd/telemeter-server/main.go

@@ -23,19 +23,19 @@ import (
 
 	"github.com/coreos/go-oidc"
 	"github.com/go-chi/chi"
-	"github.com/go-chi/chi/middleware"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/oklog/run"
-	"github.com/openshift/telemeter/pkg/runutil"
-	"github.com/openshift/telemeter/pkg/tracing"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/cobra"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.opentelemetry.io/otel"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
 
+	"github.com/openshift/telemeter/pkg/runutil"
+	"github.com/openshift/telemeter/pkg/tracing"
+
 	"github.com/openshift/telemeter/pkg/authorize"
 	"github.com/openshift/telemeter/pkg/authorize/jwt"
 	"github.com/openshift/telemeter/pkg/authorize/stub"
@@ -388,8 +388,7 @@ func (o *Options) Run(ctx context.Context, externalListener, internalListener ne
 	}
 	{
 		external := chi.NewRouter()
-		external.Use(middleware.RequestID)
-		external.Use(server.RequestLogger(o.Logger)) // Added after RequestID to log the request ID.
+		external.Use(logger.RequestLoggerWithTraceInfo(o.Logger))
 
 		// TODO: Refactor HealthRoutes to not take *http.Mux
 		mux := http.NewServeMux()

go.mod

@@ -23,6 +23,8 @@ require (
 	go.opentelemetry.io/contrib/propagators/jaeger v1.16.0
 	go.opentelemetry.io/otel v1.16.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.16.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0
 	go.opentelemetry.io/otel/sdk v1.16.0
 	go.opentelemetry.io/otel/trace v1.16.0
 	go.uber.org/goleak v1.2.1
@@ -33,6 +35,7 @@ require (
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dennwc/varint v1.0.0 // indirect
@@ -42,21 +45,26 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/kr/text v0.1.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/crypto v0.8.0 // indirect
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect
+	google.golang.org/grpc v1.55.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )