Add TLS config for prometheus-operator

cmd/operator/main.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"flag"
 	"fmt"
@@ -26,6 +27,7 @@ import (
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/coreos/prometheus-operator/pkg/admission"
 	alertmanagercontroller "github.com/coreos/prometheus-operator/pkg/alertmanager"
@@ -36,6 +38,7 @@ import (
 	thanoscontroller "github.com/coreos/prometheus-operator/pkg/thanos"
 	"github.com/coreos/prometheus-operator/pkg/version"
 
+	rbacproxytls "github.com/brancz/kube-rbac-proxy/pkg/tls"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
@@ -59,6 +62,10 @@ const (
 	logFormatJson   = "json"
 )
 
+const (
+	defaultOperatorTLSDir = "/etc/tls/private"
+)
+
 var (
 	ns             = namespaces{}
 	deniedNs       = namespaces{}
@@ -74,9 +81,8 @@ func (n namespaces) Set(value string) error {
 	if n == nil {
 		return errors.New("expected n of type namespaces to be initialized")
 	}
-	ns := strings.Split(value, ",")
-	for i := range ns {
-		n[ns[i]] = struct{}{}
+	for _, ns := range strings.Split(value, ",") {
+		n[ns] = struct{}{}
 	}
 	return nil
 }
@@ -96,16 +102,25 @@ func (n namespaces) asSlice() []string {
 
 func serve(srv *http.Server, listener net.Listener, logger log.Logger) func() error {
 	return func() error {
-		logger.Log("msg", "Staring insecure server on :8080")
+		logger.Log("msg", "Staring insecure server on "+listener.Addr().String())
 		if err := srv.Serve(listener); err != http.ErrServerClosed {
 			return err
 		}
 		return nil
 	}
 }
 
+func serveTLS(srv *http.Server, listener net.Listener, logger log.Logger) func() error {
+	return func() error {
+		logger.Log("msg", "Staring secure server on "+listener.Addr().String())
+		if err := srv.ServeTLS(listener, "", ""); err != http.ErrServerClosed {
+			return err
+		}
+		return nil
+	}
+}
+
 var (
-	cfg                prometheuscontroller.Config
 	availableLogLevels = []string{
 		logLevelAll,
 		logLevelDebug,
@@ -118,12 +133,30 @@ var (
 		logFormatLogfmt,
 		logFormatJson,
 	}
+	cfg = prometheuscontroller.Config{
+		CrdKinds: monitoringv1.DefaultCrdKinds,
+	}
+	rawTLSCipherSuites string
+	serverTLS          bool
+
+	flagset = flag.CommandLine
 )
 
 func init() {
-	cfg.CrdKinds = monitoringv1.DefaultCrdKinds
-	flagset := flag.CommandLine
 	klog.InitFlags(flagset)
+	flagset.StringVar(&cfg.ListenAddress, "web.listen-address", ":8080", "Address on which to expose metrics and web interface.")
+	flagset.BoolVar(&serverTLS, "web.enable-tls", false, "Activate prometheus operator web server TLS.  "+
+		" This is useful for example when using the rule validation webhook.")
+	flagset.StringVar(&cfg.ServerTLSConfig.CertFile, "web.cert-file", defaultOperatorTLSDir+"/tls.crt", "Cert file to be used for operator web server endpoints.")
+	flagset.StringVar(&cfg.ServerTLSConfig.KeyFile, "web.key-file", defaultOperatorTLSDir+"/tls.key", "Private key matching the cert file to be used for operator web server endpoints.")
+	flagset.StringVar(&cfg.ServerTLSConfig.ClientCAFile, "web.client-ca-file", defaultOperatorTLSDir+"/tls-ca.crt", "Client CA certificate file to be used for operator web server endpoints.")
+	flagset.DurationVar(&cfg.ServerTLSConfig.ReloadInterval, "web.tls-reload-interval", time.Minute, "The interval at which to watch for TLS certificate changes, by default set to 1 minute. (default 1m0s).")
+	flag.StringVar(&cfg.ServerTLSConfig.MinVersion, "tls-min-version", "VersionTLS13",
+		"Minimum TLS version supported. Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants.")
+	flag.StringVar(&rawTLSCipherSuites, "tls-cipher-suites", "", "Comma-separated list of cipher suites for the server."+
+		" Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants)."+
+		"If omitted, the default Go cipher suites will be used."+
+		"Note that TLS 1.3 ciphersuites are not configurable.")
 	flagset.StringVar(&cfg.Host, "apiserver", "", "API Server addr, e.g. ' - NOT RECOMMENDED FOR PRODUCTION - http://127.0.0.1:8080'. Omit parameter to run in on-cluster mode and utilize the service account token.")
 	flagset.StringVar(&cfg.TLSConfig.CertFile, "cert-file", "", " - NOT RECOMMENDED FOR PRODUCTION - Path to public TLS certificate file.")
 	flagset.StringVar(&cfg.TLSConfig.KeyFile, "key-file", "", "- NOT RECOMMENDED FOR PRODUCTION - Path to private TLS certificate file.")
@@ -156,32 +189,11 @@ func init() {
 	flagset.StringVar(&cfg.PromSelector, "prometheus-instance-selector", "", "Label selector to filter Prometheus CRDs to manage")
 	flagset.StringVar(&cfg.AlertManagerSelector, "alertmanager-instance-selector", "", "Label selector to filter AlertManager CRDs to manage")
 	flagset.StringVar(&cfg.ThanosRulerSelector, "thanos-ruler-instance-selector", "", "Label selector to filter ThanosRuler CRDs to manage")
-	flagset.Parse(os.Args[1:])
-
-	cfg.Namespaces.AllowList = ns.asSlice()
-	if len(cfg.Namespaces.AllowList) == 0 {
-		cfg.Namespaces.AllowList = append(cfg.Namespaces.AllowList, v1.NamespaceAll)
-	}
-
-	cfg.Namespaces.DenyList = deniedNs.asSlice()
-	cfg.Namespaces.PrometheusAllowList = prometheusNs.asSlice()
-	cfg.Namespaces.AlertmanagerAllowList = alertmanagerNs.asSlice()
-	cfg.Namespaces.ThanosRulerAllowList = thanosRulerNs.asSlice()
-
-	if len(cfg.Namespaces.PrometheusAllowList) == 0 {
-		cfg.Namespaces.PrometheusAllowList = cfg.Namespaces.AllowList
-	}
-
-	if len(cfg.Namespaces.AlertmanagerAllowList) == 0 {
-		cfg.Namespaces.AlertmanagerAllowList = cfg.Namespaces.AllowList
-	}
-
-	if len(cfg.Namespaces.ThanosRulerAllowList) == 0 {
-		cfg.Namespaces.ThanosRulerAllowList = cfg.Namespaces.AllowList
-	}
 }
 
 func Main() int {
+	flagset.Parse(os.Args[1:])
+
 	logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout))
 	if cfg.LogFormat == logFormatJson {
 		logger = log.NewJSONLogger(log.NewSyncWriter(os.Stdout))
@@ -213,6 +225,28 @@ func Main() int {
 		return 1
 	}
 
+	cfg.Namespaces.AllowList = ns
+	if len(cfg.Namespaces.AllowList) == 0 {
+		cfg.Namespaces.AllowList[v1.NamespaceAll] = struct{}{}
+	}
+
+	cfg.Namespaces.DenyList = deniedNs
+	cfg.Namespaces.PrometheusAllowList = prometheusNs
+	cfg.Namespaces.AlertmanagerAllowList = alertmanagerNs
+	cfg.Namespaces.ThanosRulerAllowList = thanosRulerNs
+
+	if len(cfg.Namespaces.PrometheusAllowList) == 0 {
+		cfg.Namespaces.PrometheusAllowList = cfg.Namespaces.AllowList
+	}
+
+	if len(cfg.Namespaces.AlertmanagerAllowList) == 0 {
+		cfg.Namespaces.AlertmanagerAllowList = cfg.Namespaces.AllowList
+	}
+
+	if len(cfg.Namespaces.ThanosRulerAllowList) == 0 {
+		cfg.Namespaces.ThanosRulerAllowList = cfg.Namespaces.AllowList
+	}
+
 	r := prometheus.NewRegistry()
 	po, err := prometheuscontroller.New(cfg, log.With(logger, "component", "prometheusoperator"), r)
 	if err != nil {
@@ -242,12 +276,25 @@ func Main() int {
 
 	web.Register(mux)
 	admit.Register(mux)
-	l, err := net.Listen("tcp", ":8080")
+	l, err := net.Listen("tcp", cfg.ListenAddress)
 	if err != nil {
-		fmt.Fprint(os.Stderr, "listening port 8080 failed", err)
+		fmt.Fprint(os.Stderr, "listening failed", cfg.ListenAddress, err)
 		return 1
 	}
 
+	var tlsConfig *tls.Config = nil
+	if serverTLS {
+		if rawTLSCipherSuites != "" {
+			cfg.ServerTLSConfig.CipherSuites = strings.Split(rawTLSCipherSuites, ",")
+		}
+		tlsConfig, err = operator.NewTLSConfig(logger, cfg.ServerTLSConfig.CertFile, cfg.ServerTLSConfig.KeyFile,
+			cfg.ServerTLSConfig.ClientCAFile, cfg.ServerTLSConfig.MinVersion, cfg.ServerTLSConfig.CipherSuites)
+		if tlsConfig == nil || err != nil {
+			fmt.Fprint(os.Stderr, "invalid TLS config", err)
+			return 1
+		}
+	}
+
 	validationTriggeredCounter := prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "prometheus_operator_rule_validation_triggered_total",
 		Help: "Number of times a prometheusRule object triggered validation",
@@ -284,8 +331,45 @@ func Main() int {
 	wg.Go(func() error { return ao.Run(ctx.Done()) })
 	wg.Go(func() error { return to.Run(ctx.Done()) })
 
-	srv := &http.Server{Handler: mux}
-	wg.Go(serve(srv, l, logger))
+	if tlsConfig != nil {
+		r, err := rbacproxytls.NewCertReloader(
+			cfg.ServerTLSConfig.CertFile,
+			cfg.ServerTLSConfig.KeyFile,
+			cfg.ServerTLSConfig.ReloadInterval,
+		)
+		if err != nil {
+			fmt.Fprint(os.Stderr, "failed to initialize certificate reloader", err)
+			return 1
+		}
+
+		tlsConfig.GetCertificate = r.GetCertificate
+
+		wg.Go(func() error {
+			t := time.NewTicker(cfg.ServerTLSConfig.ReloadInterval)
+			for {
+				select {
+				case <-t.C:
+				case <-ctx.Done():
+					return nil
+				}
+				if err := r.Watch(ctx); err != nil {
+					level.Warn(logger).Log("msg", "error reloading server TLS certificate",
+						"err", err)
+				} else {
+					return nil
+				}
+			}
+		})
+	}
+	srv := &http.Server{
+		Handler:   mux,
+		TLSConfig: tlsConfig,
+	}
+	if srv.TLSConfig == nil {
+		wg.Go(serve(srv, l, logger))
+	} else {
+		wg.Go(serveTLS(srv, l, logger))
+	}
 
 	term := make(chan os.Signal)
 	signal.Notify(term, os.Interrupt, syscall.SIGTERM)

cmd/operator/main_test.go

@@ -0,0 +1,46 @@
+// Copyright 2020 The prometheus-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestNamespacesType(t *testing.T) {
+	var ns namespaces
+	if ns.String() != "" {
+		t.Errorf("incorrect string value for nil namespaces, want: empty string, got %v", ns.String())
+	}
+
+	val := "a,b,c"
+	err := ns.Set(val)
+	if err == nil {
+		t.Error("expected error for nil namespaces")
+	}
+
+	ns = namespaces{}
+	ns.Set(val)
+	if len(ns) != 3 {
+		t.Errorf("incorrect length of namespaces, want: %v, got: %v", 3, len(ns))
+	}
+
+	for _, next := range strings.Split(val, ",") {
+		if _, ok := ns[next]; !ok {
+			t.Errorf("namespace not in map, want: %v, not in map: %v", next, map[string]struct{}(ns))
+		}
+	}
+
+}

go.mod

@@ -5,6 +5,7 @@ go 1.13
 require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d
+	github.com/brancz/kube-rbac-proxy v0.5.0
 	github.com/campoy/embedmd v1.0.0
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
@@ -31,6 +32,7 @@ require (
 	k8s.io/apimachinery v0.17.3
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/code-generator v0.17.3
+	k8s.io/component-base v0.17.3
 	k8s.io/klog v1.0.0
 	sigs.k8s.io/controller-tools v0.2.4
 )

go.sum

@@ -93,6 +93,8 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
 github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d h1:DMb8SuAL9+demT8equqMMzD8C/uxqWmj4cgV7ufrpQo=
 github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d/go.mod h1:IyUJYN1gvWjtLF5ZuygmxbnsAyP3aJS6cHzIuZY50B0=
+github.com/brancz/kube-rbac-proxy v0.5.0 h1:RdMeazKvTXwH66i9DeVtEV/o0XAqyhSaiWkZ8gj54x4=
+github.com/brancz/kube-rbac-proxy v0.5.0/go.mod h1:cL2VjiIFGS90Cjh5ZZ8+It6tMcBt8rwvuw2J6Mamnl0=
 github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY=
 github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
 github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
@@ -256,6 +258,7 @@ github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE=
@@ -896,6 +899,7 @@ k8s.io/api v0.0.0-20190813020757-36bff7324fb7/go.mod h1:3Iy+myeAORNCLgjd/Xu9ebwN
 k8s.io/api v0.0.0-20190918155943-95b840bb6a1f h1:8FRUST8oUkEI45WYKyD8ed7Ad0Kg5v11zHyPkEVb2xo=
 k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48=
 k8s.io/api v0.0.0-20191115095533-47f6de673b26/go.mod h1:iA/8arsvelvo4IDqIhX4IbjTEKBGgvsf2OraTuRtLFU=
+k8s.io/api v0.0.0-20191122220107-b5267f2975e0/go.mod h1:vYpRfxYkMrmPPSesoHEkGNHxNKTk96REAwqm/inQbs0=
 k8s.io/api v0.17.3 h1:XAm3PZp3wnEdzekNkcmj/9Y1zdmQYJ1I4GKSBBZ8aG0=
 k8s.io/api v0.17.3/go.mod h1:YZ0OTkuw7ipbe305fMpIdf3GLXZKRigjtZaV5gzC2J0=
 k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783 h1:V6ndwCPoao1yZ52agqOKaUAl7DYWVGiXjV7ePA2i610=
@@ -906,9 +910,11 @@ k8s.io/apimachinery v0.0.0-20190809020650-423f5d784010/go.mod h1:Waf/xTS2FGRrgXC
 k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655 h1:CS1tBQz3HOXiseWZu6ZicKX361CZLT97UFnnPx0aqBw=
 k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4=
 k8s.io/apimachinery v0.0.0-20191115015347-3c7067801da2/go.mod h1:dXFS2zaQR8fyzuvRdJDHw2Aerij/yVGJSre0bZQSVJA=
+k8s.io/apimachinery v0.0.0-20191121175448-79c2a76c473a/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
 k8s.io/apimachinery v0.17.3 h1:f+uZV6rm4/tHE7xXgLyToprg6xWairaClGVkm2t8omg=
 k8s.io/apimachinery v0.17.3/go.mod h1:gxLnyZcGNdZTCLnq3fgzyg2A5BVCHTNDFrw8AmuJ+0g=
 k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg=
+k8s.io/apiserver v0.0.0-20191122221311-9d521947b1e1/go.mod h1:RbsZY5zzBIWnz4KbctZsTVjwIuOpTp4Z8oCgFHN4kZQ=
 k8s.io/apiserver v0.17.3/go.mod h1:iJtsPpu1ZpEnHaNawpSV0nYTGBhhX2dUlnn7/QS7QiY=
 k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90 h1:mLmhKUm1X+pXu0zXMEzNsOF5E2kKFGe5o6BZBIIqA6A=
 k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90/go.mod h1:J69/JveO6XESwVgG53q3Uz5OSfgsv4uxpScmmyYOOlk=
@@ -917,6 +923,8 @@ k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m
 k8s.io/code-generator v0.17.3 h1:q/hDMk2cvFzSxol7k/VA1qCssR7VSMXHQHhzuX29VJ8=
 k8s.io/code-generator v0.17.3/go.mod h1:l8BLVwASXQZTo2xamW5mQNFCe1XPiAesVq7Y1t7PiQQ=
 k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA=
+k8s.io/component-base v0.0.0-20191122220729-2684fb322cb9/go.mod h1:NFuUusy/X4Tk21m21tcNUihnmp4OI7lXU7/xA+rYXkc=
+k8s.io/component-base v0.17.3 h1:hQzTSshY14aLSR6WGIYvmw+w+u6V4d+iDR2iDGMrlUg=
 k8s.io/component-base v0.17.3/go.mod h1:GeQf4BrgelWm64PXkIXiPh/XS0hnO42d9gx9BtbZRp8=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM=