Skip to content
Permalink
Browse files
feat(telemetry): experiments with opentelemetry, traces and clickhous…
…e storage

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Jun 30, 2022
1 parent 9c8168e commit 2e404a963be263498244a473418c025f5646ef10
Show file tree
Hide file tree
Showing 9 changed files with 176 additions and 55 deletions.
@@ -50,13 +50,16 @@ import (
"github.com/werf/werf/cmd/werf/synchronization"
"github.com/werf/werf/cmd/werf/version"
"github.com/werf/werf/pkg/process_exterminator"
"github.com/werf/werf/pkg/telemetry"
)

func main() {
ctx := context.Background()

common.InitTelemetry(ctx)

telemetry.MessageEvent(ctx, "command started")

shouldTerminate, err := common.ContainerBackendProcessStartupHook()
if err != nil {
common.ShutdownTelemetry(ctx, 1)
12 go.mod
@@ -62,6 +62,13 @@ require (
github.com/werf/kubedog v0.6.5-0.20220608151530-2555924b0ce9
github.com/werf/lockgate v0.0.0-20200729113342-ec2c142f71ea
github.com/werf/logboek v0.5.4
go.opentelemetry.io/otel v1.7.1-0.20220624212736-ef6c0da0de3b
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.30.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0
go.opentelemetry.io/otel/metric v0.30.0
go.opentelemetry.io/otel/sdk/metric v0.30.0
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
gopkg.in/errgo.v2 v2.1.0
@@ -270,13 +277,8 @@ require (
go.mongodb.org/mongo-driver v1.5.1 // indirect
go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/otel v1.7.1-0.20220624212736-ef6c0da0de3b // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.30.0 // indirect
go.opentelemetry.io/otel/metric v0.30.0 // indirect
go.opentelemetry.io/otel/sdk v1.7.0 // indirect
go.opentelemetry.io/otel/sdk/metric v0.30.0 // indirect
go.opentelemetry.io/otel/trace v1.7.0 // indirect
go.opentelemetry.io/proto/otlp v0.16.0 // indirect
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
12 go.sum
@@ -310,6 +310,7 @@ github.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYX
github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -781,7 +782,6 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -940,6 +940,7 @@ github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -1140,7 +1141,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.2/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
@@ -2154,12 +2154,9 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.2
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs=
go.opentelemetry.io/otel v1.7.0 h1:Z2lA3Tdch0iDcrhJXDIlC94XE+bxok1F9B+4Lz/lGsM=
go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk=
go.opentelemetry.io/otel v1.7.1-0.20220624212736-ef6c0da0de3b h1:wZT15nCrRJsT1PJpZ3rYPTsPH2LnkDq8s6CNpgIsyiw=
go.opentelemetry.io/otel v1.7.1-0.20220624212736-ef6c0da0de3b/go.mod h1:lnHkMUGUQfY3wBpkzmPQH6RuCyURze1hKrrF7xTxJ60=
go.opentelemetry.io/otel/exporters/jaeger v1.7.0 h1:wXgjiRldljksZkZrldGVe6XrG9u3kYDyQmkZwmm5dI0=
go.opentelemetry.io/otel/exporters/jaeger v1.7.0/go.mod h1:PwQAOqBgqbLQRKlj466DuD2qyMjbtcPpfPfj+AqbSBs=
go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg=
go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4=
@@ -2169,15 +2166,16 @@ go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0 h1:Os0ds8fJp2AUa9DNra
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0/go.mod h1:8Lz1GGcrx1kPGE3zqDrK7ZcPzABEfIQqBjq7roQa5ZA=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.30.0 h1:MrUowGDjf4jKGMgjDAIP5Czh6YGdCHc46gfTwlF6eQI=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.30.0/go.mod h1:WulNodDa6sY6ZADi664BgKD6SvXLLQXVZEQ81q5ps9U=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0 h1:giGm8w67Ja7amYNfYMdme7xSp2pIxThWopw8+QP51Yk=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0 h1:Ydage/P0fRrSPpZeCVxzjqGcI6iVmG2xb43+IR8cjqM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE=
go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
go.opentelemetry.io/otel/metric v0.30.0 h1:Hs8eQZ8aQgs0U49diZoaS6Uaxw3+bBE3lcMUKBFIk3c=
go.opentelemetry.io/otel/metric v0.30.0/go.mod h1:/ShZ7+TS4dHzDFmfi1kSXMhMVubNoP0oIaBp70J6UXU=
go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
go.opentelemetry.io/otel/sdk v1.3.0 h1:3278edCoH89MEJ0Ky8WQXVmDQv3FX4ZJ3Pp+9fJreAI=
go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs=
go.opentelemetry.io/otel/sdk v1.7.0 h1:4OmStpcKVOfvDOgCt7UriAPtKolwIhxpnSNI/yK+1B0=
go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU=
@@ -2884,7 +2882,6 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg=
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8=
google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
@@ -2901,7 +2898,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
@@ -0,0 +1,35 @@
package telemetry

import (
"context"
"encoding/json"
"fmt"
"time"

"go.opentelemetry.io/otel/attribute"
)

func MessageEvent(ctx context.Context, msg string) error {
if !IsEnabled() {
return nil
}

trc := tracerProvider.Tracer("telemetry.werf.io")

_, span := trc.Start(ctx, "message")

span.SetAttributes(attribute.Key("ts").Int64(time.Now().UnixMilli()))
span.SetAttributes(attribute.Key("executionID").String(executionID))
span.SetAttributes(attribute.Key("projectID").String(projectID))
span.SetAttributes(attribute.Key("command").String(command))
span.SetAttributes(attribute.Key("eventType").String("message"))

data, err := json.Marshal(map[string]interface{}{"message": msg})
if err != nil {
return fmt.Errorf("unable to marshal message data: %w", err)
}
span.SetAttributes(attribute.Key("data").String(string(data)))
span.End()

return nil
}
@@ -14,28 +14,28 @@ import (
)

var (
ctrl *controller.Controller
exporter *otlpmetric.Exporter
ctrl *controller.Controller
metricExporter *otlpmetric.Exporter
)

func SetupController(ctx context.Context, url string) error {
func SetupMetricExporter(ctx context.Context, url string) error {
{
e, err := NewExporter(url)
e, err := NewMetricExporter(url)
if err != nil {
return fmt.Errorf("unable to create telemetry exporter: %w", err)
return fmt.Errorf("unable to create telemetry metric exporter: %w", err)
}
exporter = e
metricExporter = e
}
ctrl = NewController(exporter)
ctrl = NewController(metricExporter)

if err := exporter.Start(ctx); err != nil {
return fmt.Errorf("error starting telemetry exporter: %w", err)
if err := metricExporter.Start(ctx); err != nil {
return fmt.Errorf("error starting telemetry metric exporter: %w", err)
}

return nil
}

func NewExporter(url string) (*otlpmetric.Exporter, error) {
func NewMetricExporter(url string) (*otlpmetric.Exporter, error) {
urlObj, err := neturl.Parse(url)
if err != nil {
return nil, fmt.Errorf("bad url: %w", err)
@@ -3,18 +3,27 @@ package telemetry
import (
"context"
"fmt"
"os"
"runtime"

"github.com/google/uuid"

"github.com/werf/logboek"
"github.com/werf/werf/pkg/util"
"github.com/werf/werf/pkg/werf"
)

const (
URL = "http://localhost:4318/v1/metrics"
MetricsURL = "http://localhost:4318/v1/metrics"
TracesURL = "http://localhost:4318/v1/traces"
)

var metrics *Metrics
var (
executionID string
projectID string
command string
metrics *Metrics
)

func ChangeMetrics(changeFunc func(metrics *Metrics)) {
if !IsEnabled() {
@@ -28,15 +37,19 @@ func Init(ctx context.Context) error {
return nil
}

executionID = uuid.New().String()
projectID = "26a38bba-51ed-4f93-8104-e6f51a5454ef"
command = fmt.Sprintf("%v", os.Args)

metrics = &Metrics{
Version: werf.Version,
Command: "TODO",
OS: runtime.GOOS,
Arch: runtime.GOARCH,
}

if err := SetupController(ctx, URL); err != nil {
return fmt.Errorf("unable to setup telemetry controller for url %q: %w", URL, err)
if err := SetupTraceExporter(ctx, TracesURL); err != nil {
return fmt.Errorf("unable to setup telemetry traces exporter for url %q: %w", TracesURL, err)
}

return nil
@@ -47,16 +60,12 @@ func Shutdown(ctx context.Context) error {
return nil
}

if err := metrics.WriteOpenTelemetry(ctx, ctrl.Meter("werf")); err != nil {
return fmt.Errorf("unable to write open telemetry data: %w", err)
}

if err := ctrl.Collect(ctx); err != nil {
return fmt.Errorf("otel controller collection failed: %w", err)
if err := traceExporter.Shutdown(ctx); err != nil {
return fmt.Errorf("unable to shutdown trace exporter: %w", err)
}

if err := exporter.Export(ctx, ctrl.Resource(), ctrl); err != nil {
return fmt.Errorf("unable to export telemetry: %w", err)
if err := tracerProvider.Shutdown(ctx); err != nil {
return fmt.Errorf("unable to shutdown trace provider: %w", err)
}

logboek.Context(ctx).Debug().LogF("Telemetry collection done\n")
@@ -0,0 +1,59 @@
package telemetry

import (
"context"
"fmt"
neturl "net/url"
"time"

"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/sdk/trace"
)

var (
tracerProvider *trace.TracerProvider
traceExporter *otlptrace.Exporter
)

func SetupTraceExporter(ctx context.Context, url string) error {
{
e, err := NewTraceExporter(url)
if err != nil {
return fmt.Errorf("unable to create telemetry trace exporter: %w", err)
}
traceExporter = e
}

tracerProvider = trace.NewTracerProvider(trace.WithSyncer(traceExporter))

if err := traceExporter.Start(ctx); err != nil {
return fmt.Errorf("error starting telemetry trace exporter: %w", err)
}

return nil
}

func NewTraceExporter(url string) (*otlptrace.Exporter, error) {
urlObj, err := neturl.Parse(url)
if err != nil {
return nil, fmt.Errorf("bad url: %w", err)
}

var opts []otlptracehttp.Option

if urlObj.Scheme == "http" {
opts = append(opts, otlptracehttp.WithInsecure())
}

opts = append(opts,
otlptracehttp.WithEndpoint(urlObj.Host),
otlptracehttp.WithURLPath(urlObj.Path),
otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}),
otlptracehttp.WithTimeout(5*time.Second),
)

client := otlptracehttp.NewClient(opts...)

return otlptrace.NewUnstarted(client), nil
}
@@ -1,20 +1,29 @@
version: "2"
version: "3"

networks:
otel-clickhouse:

services:

otel-collector:
image: otel/opentelemetry-collector-contrib-dev:latest
command: [ "--config=/etc/otel-collector-config.yaml" ]
image: ghcr.io/werf/telemetrywerfio-collector:dev
# image: otel/opentelemetry-collector-contrib-dev:latest
command:
- "--config=/etc/otel-collector-config.yaml"
- "--set=service.telemetry.logs.level=DEBUG"
volumes:
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
ports:
- "8888:8888" # Prometheus metrics exposed by the collector server itself
- "8889:8889" # Collected metrics for prometheus
- "4318:4318" # OTLP HTTP receiver
- "4318:4318" # OTLP HTTP receiver
depends_on:
- clickhouse
networks:
- otel-clickhouse

prometheus:
container_name: prometheus
image: prom/prometheus:latest
volumes:
- ./prometheus.yaml:/etc/prometheus/prometheus.yml
clickhouse:
image: clickhouse/clickhouse-server:latest
ports:
- "9090:9090"
- "9000:9000"
- "8123:8123"
networks:
- otel-clickhouse

0 comments on commit 2e404a9

Please sign in to comment.