all: add opentelemetry hooks

cmd/clair/httptransport.go

@@ -9,16 +9,13 @@ import (
 
 	"github.com/quay/claircore/libindex"
 	"github.com/quay/claircore/libvuln"
+	"go.opentelemetry.io/otel/plugin/othttp"
 
 	"github.com/quay/clair/v4/config"
 	"github.com/quay/clair/v4/indexer"
 	"github.com/quay/clair/v4/matcher"
 )
 
-const (
-	HealthApiPath = "/healthz"
-)
-
 // httptransport configures an http server according to Clair's operation mode.
 func httptransport(ctx context.Context, conf config.Config) (*http.Server, error) {
 	var srv *http.Server
@@ -71,7 +68,7 @@ func devMode(ctx context.Context, conf config.Config) (*http.Server, error) {
 	matcher.Register(mux)
 	return &http.Server{
 		Addr:    conf.HTTPListenAddr,
-		Handler: Compress(mux),
+		Handler: othttp.NewHandler(Compress(mux), "server"),
 	}, nil
 }
 
@@ -92,7 +89,7 @@ func indexerMode(ctx context.Context, conf config.Config) (*http.Server, error)
 	}
 	return &http.Server{
 		Addr:    conf.Indexer.HTTPListenAddr,
-		Handler: Compress(indexer),
+		Handler: othttp.NewHandler(Compress(indexer), "server"),
 	}, nil
 }
 
@@ -116,7 +113,7 @@ func matcherMode(ctx context.Context, conf config.Config) (*http.Server, error)
 	}
 	return &http.Server{
 		Addr:    conf.Matcher.HTTPListenAddr,
-		Handler: Compress(matcher),
+		Handler: othttp.NewHandler(Compress(matcher), "server"),
 	}, nil
 }
 

cmd/clair/introspection.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/pprof"
+
+	"go.opentelemetry.io/otel/api/global"
+	"go.opentelemetry.io/otel/exporter/metric/prometheus"
+	"go.opentelemetry.io/otel/exporter/trace/stdout"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+
+	"github.com/quay/clair/v4/config"
+)
+
+const (
+	HealthApiPath = "/healthz"
+)
+
+func introspection(ctx context.Context, cfg *config.Config, healthCheck func() bool) (*http.Server, error) {
+	mux := http.NewServeMux()
+	srv := http.Server{
+		Addr:        cfg.IntrospectionAddr,
+		Handler:     mux,
+		BaseContext: func(_ net.Listener) context.Context { return ctx },
+	}
+
+	// TODO Make configurable.
+	pipeline, hf, err := prometheus.InstallNewPipeline(prometheus.Config{})
+	if err != nil {
+		return nil, err
+	}
+	srv.RegisterOnShutdown(pipeline.Stop)
+
+	// TODO Make configurable.
+	exporter, err := stdout.NewExporter(stdout.Options{PrettyPrint: true})
+	if err != nil {
+		return nil, err
+	}
+	tp, err := sdktrace.NewProvider(
+		sdktrace.WithConfig(sdktrace.Config{DefaultSampler: sdktrace.NeverSample()}),
+		sdktrace.WithSyncer(exporter))
+	if err != nil {
+		return nil, err
+	}
+	global.SetTraceProvider(tp)
+
+	mux.HandleFunc("/metrics", hf)
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		if !healthCheck() {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		fmt.Fprint(w, `ok`)
+	})
+	mux.HandleFunc("/debug/pprof/", pprof.Index)
+	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	return &srv, nil
+}

cmd/clair/main.go

@@ -67,6 +67,18 @@ func main() {
 		}
 	}()
 
+	intro, err := introspection(lctx, &conf, func() bool { return true })
+	if err != nil {
+		logger.Error().Err(err).Msg("failed to create introspection server")
+	} else {
+		go func() {
+			if err := intro.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+				logger.Error().Err(err).Msg("unable to start introspection server")
+			}
+			defer intro.Shutdown(lctx)
+		}()
+	}
+
 	// register signal handler
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt)

config/config.go

@@ -20,6 +20,9 @@ type Config struct {
 	Mode string `yaml:"mode"`
 	// indicates the global listen address if running in "Dev"
 	HTTPListenAddr string `yaml:"http_listen_addr"`
+	// IntrospectionAddr is an address to listen on for introspection http
+	// endpoints, e.g. metrics and profiling.
+	IntrospectionAddr string `yaml:"introspection_addr"`
 	// indicates log level for the process
 	LogLevel string `yaml:"log_level"`
 	// indexer mode specific config

go.mod

@@ -8,6 +8,8 @@ require (
 	github.com/mattn/go-sqlite3 v1.11.0 // indirect
 	github.com/quay/claircore v0.0.16
 	github.com/rs/zerolog v1.16.0
+	go.opentelemetry.io/otel v0.2.1
+	go.opentelemetry.io/otel/exporter/metric/prometheus v0.2.1
 	golang.org/x/tools v0.0.0-20191210200704-1bcf67c9cb49 // indirect
 	gopkg.in/square/go-jose.v2 v2.4.1
 	gopkg.in/yaml.v2 v2.2.5