Add TLS and basic auth

CHANGELOG.md

@@ -19,6 +19,8 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re
 - [#4175](https://github.com/thanos-io/thanos/pull/4175) Added Tag Configuration Support Lightstep Tracing
 - [#4176](https://github.com/thanos-io/thanos/pull/4176) Query API: Adds optional `Stats param` to return stats for query APIs
 - [#4125](https://github.com/thanos-io/thanos/pull/4125) Rule: Add `--alert.relabel-config` / `--alert.relabel-config-file` allowing to specify alert relabel configurations like [Prometheus](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config)
+- [#4211](https://github.com/thanos-io/thanos/pull/4211) Add TLS and basic authentication to Thanos APIs
+
 ### Fixed
 -
 ### Changed

cmd/thanos/compact.go

@@ -177,6 +177,7 @@ func runCompact(
 	srv := httpserver.New(logger, reg, component, httpProbe,
 		httpserver.WithListen(conf.http.bindAddress),
 		httpserver.WithGracePeriod(time.Duration(conf.http.gracePeriod)),
+		httpserver.WithTLSConfig(conf.http.tlsConfig),
 	)
 
 	g.Add(func() error {

cmd/thanos/config.go

@@ -44,6 +44,7 @@ func (gc *grpcConfig) registerFlag(cmd extkingpin.FlagClause) *grpcConfig {
 
 type httpConfig struct {
 	bindAddress string
+	tlsConfig   string
 	gracePeriod model.Duration
 }
 
@@ -54,6 +55,10 @@ func (hc *httpConfig) registerFlag(cmd extkingpin.FlagClause) *httpConfig {
 	cmd.Flag("http-grace-period",
 		"Time to wait after an interrupt received for HTTP Server.").
 		Default("2m").SetValue(&hc.gracePeriod)
+	cmd.Flag(
+		"http.config",
+		"[EXPERIMENTAL] Path to the configuration file that can enable TLS or authentication for all HTTP endpoints.",
+	).Default("").StringVar(&hc.tlsConfig)
 	return hc
 }
 

cmd/thanos/downsample.go

@@ -58,6 +58,7 @@ func RunDownsample(
 	logger log.Logger,
 	reg *prometheus.Registry,
 	httpBindAddr string,
+	httpTLSConfig string,
 	httpGracePeriod time.Duration,
 	dataDir string,
 	objStoreConfig *extflag.PathOrContent,
@@ -136,6 +137,7 @@ func RunDownsample(
 	srv := httpserver.New(logger, reg, comp, httpProbe,
 		httpserver.WithListen(httpBindAddr),
 		httpserver.WithGracePeriod(httpGracePeriod),
+		httpserver.WithTLSConfig(httpTLSConfig),
 	)
 
 	g.Add(func() error {

cmd/thanos/query.go

@@ -57,7 +57,7 @@ func registerQuery(app *extkingpin.App) {
 	comp := component.Query
 	cmd := app.Command(comp.String(), "Query node exposing PromQL enabled Query API with data retrieved from multiple store nodes.")
 
-	httpBindAddr, httpGracePeriod := extkingpin.RegisterHTTPFlags(cmd)
+	httpBindAddr, httpGracePeriod, httpTLSConfig := extkingpin.RegisterHTTPFlags(cmd)
 	grpcBindAddr, grpcGracePeriod, grpcCert, grpcKey, grpcClientCA := extkingpin.RegisterGRPCFlags(cmd)
 
 	secure := cmd.Flag("grpc-client-tls-secure", "Use TLS when talking to the gRPC server").Default("false").Bool()
@@ -227,6 +227,7 @@ func registerQuery(app *extkingpin.App) {
 			*caCert,
 			*serverName,
 			*httpBindAddr,
+			*httpTLSConfig,
 			time.Duration(*httpGracePeriod),
 			*webRoutePrefix,
 			*webExternalPrefix,
@@ -287,6 +288,7 @@ func runQuery(
 	caCert string,
 	serverName string,
 	httpBindAddr string,
+	httpTLSConfig string,
 	httpGracePeriod time.Duration,
 	webRoutePrefix string,
 	webExternalPrefix string,
@@ -593,6 +595,7 @@ func runQuery(
 		srv := httpserver.New(logger, reg, comp, httpProbe,
 			httpserver.WithListen(httpBindAddr),
 			httpserver.WithGracePeriod(httpGracePeriod),
+			httpserver.WithTLSConfig(httpTLSConfig),
 		)
 		srv.Handle("/", router)
 

cmd/thanos/query_frontend.go

@@ -220,6 +220,7 @@ func runQueryFrontend(
 		srv := httpserver.New(logger, reg, comp, httpProbe,
 			httpserver.WithListen(cfg.http.bindAddress),
 			httpserver.WithGracePeriod(time.Duration(cfg.http.gracePeriod)),
+			httpserver.WithTLSConfig(cfg.http.tlsConfig),
 		)
 
 		instr := func(f http.HandlerFunc) http.HandlerFunc {

cmd/thanos/receive.go

@@ -47,9 +47,8 @@ import (
 func registerReceive(app *extkingpin.App) {
 	cmd := app.Command(component.Receive.String(), "Accept Prometheus remote write API requests and write to local tsdb.")
 
-	httpBindAddr, httpGracePeriod := extkingpin.RegisterHTTPFlags(cmd)
+	httpBindAddr, httpGracePeriod, httpTLSConfig := extkingpin.RegisterHTTPFlags(cmd)
 	grpcBindAddr, grpcGracePeriod, grpcCert, grpcKey, grpcClientCA := extkingpin.RegisterGRPCFlags(cmd)
-
 	rwAddress := cmd.Flag("remote-write.address", "Address to listen on for remote write requests.").
 		Default("0.0.0.0:19291").String()
 	rwServerCert := cmd.Flag("remote-write.server-tls-cert", "TLS Certificate for HTTP server, leave blank to disable TLS.").Default("").String()
@@ -155,6 +154,7 @@ func registerReceive(app *extkingpin.App) {
 			*grpcKey,
 			*grpcClientCA,
 			*httpBindAddr,
+			*httpTLSConfig,
 			time.Duration(*httpGracePeriod),
 			*rwAddress,
 			*rwServerCert,
@@ -199,6 +199,7 @@ func runReceive(
 	grpcKey string,
 	grpcClientCA string,
 	httpBindAddr string,
+	httpTLSConfig string,
 	httpGracePeriod time.Duration,
 	rwAddress string,
 	rwServerCert string,
@@ -478,6 +479,7 @@ func runReceive(
 	srv := httpserver.New(logger, reg, comp, httpProbe,
 		httpserver.WithListen(httpBindAddr),
 		httpserver.WithGracePeriod(httpGracePeriod),
+		httpserver.WithTLSConfig(httpTLSConfig),
 	)
 	g.Add(func() error {
 		statusProber.Healthy()

cmd/thanos/rule.go

@@ -573,6 +573,7 @@ func runRule(
 		srv := httpserver.New(logger, reg, comp, httpProbe,
 			httpserver.WithListen(conf.http.bindAddress),
 			httpserver.WithGracePeriod(time.Duration(conf.http.gracePeriod)),
+			httpserver.WithTLSConfig(conf.http.tlsConfig),
 		)
 		srv.Handle("/", router)
 

cmd/thanos/sidecar.go

@@ -118,6 +118,7 @@ func runSidecar(
 	srv := httpserver.New(logger, reg, comp, httpProbe,
 		httpserver.WithListen(conf.http.bindAddress),
 		httpserver.WithGracePeriod(time.Duration(conf.http.gracePeriod)),
+		httpserver.WithTLSConfig(conf.http.tlsConfig),
 	)
 
 	g.Add(func() error {

cmd/thanos/store.go

@@ -221,6 +221,7 @@ func runStore(
 	srv := httpserver.New(logger, reg, conf.component, httpProbe,
 		httpserver.WithListen(conf.httpConfig.bindAddress),
 		httpserver.WithGracePeriod(time.Duration(conf.httpConfig.gracePeriod)),
+		httpserver.WithTLSConfig(conf.httpConfig.tlsConfig),
 	)
 
 	g.Add(func() error {

cmd/thanos/tools_bucket.go

@@ -321,8 +321,7 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat
 // registerBucketWeb exposes a web interface for the state of remote store like `pprof web`.
 func registerBucketWeb(app extkingpin.AppClause, objStoreConfig *extflag.PathOrContent) {
 	cmd := app.Command("web", "Web interface for remote storage bucket.")
-	httpBindAddr, httpGracePeriod := extkingpin.RegisterHTTPFlags(cmd)
-
+	httpBindAddr, httpGracePeriod, httpTLSConfig := extkingpin.RegisterHTTPFlags(cmd)
 	webRoutePrefix := cmd.Flag("web.route-prefix", "Prefix for API and UI endpoints. This allows thanos UI to be served on a sub-path. Defaults to the value of --web.external-prefix. This option is analogous to --web.route-prefix of Prometheus.").Default("").String()
 	webExternalPrefix := cmd.Flag("web.external-prefix", "Static prefix for all HTML links and redirect URLs in the bucket web UI interface. Actual endpoints are still served on / or the web.route-prefix. This allows thanos bucket web UI to be served behind a reverse proxy that strips a URL sub-path.").Default("").String()
 	webPrefixHeaderName := cmd.Flag("web.prefix-header", "Name of HTTP request header used for dynamic prefixing of UI links and redirects. This option is ignored if web.external-prefix argument is set. Security risk: enable this option only if a reverse proxy in front of thanos is resetting the header. The --web.prefix-header=X-Forwarded-Prefix option can be useful, for example, if Thanos UI is served via Traefik reverse proxy with PathPrefixStrip option enabled, which sends the stripped prefix value in X-Forwarded-Prefix header. This allows thanos UI to be served on a sub-path.").Default("").String()
@@ -343,6 +342,7 @@ func registerBucketWeb(app extkingpin.AppClause, objStoreConfig *extflag.PathOrC
 		srv := httpserver.New(logger, reg, comp, httpProbe,
 			httpserver.WithListen(*httpBindAddr),
 			httpserver.WithGracePeriod(time.Duration(*httpGracePeriod)),
+			httpserver.WithTLSConfig(*httpTLSConfig),
 		)
 
 		if *webRoutePrefix == "" {
@@ -462,7 +462,7 @@ func listResLevel() []string {
 
 func registerBucketReplicate(app extkingpin.AppClause, objStoreConfig *extflag.PathOrContent) {
 	cmd := app.Command("replicate", fmt.Sprintf("Replicate data from one object storage to another. NOTE: Currently it works only with Thanos blocks (%v has to have Thanos metadata).", block.MetaFilename))
-	httpBindAddr, httpGracePeriod := extkingpin.RegisterHTTPFlags(cmd)
+	httpBindAddr, httpGracePeriod, httpTLSConfig := extkingpin.RegisterHTTPFlags(cmd)
 	toObjStoreConfig := extkingpin.RegisterCommonObjStoreFlags(cmd, "-to", false, "The object storage which replicate data to.")
 	resolutions := cmd.Flag("resolution", "Only blocks with these resolutions will be replicated. Repeated flag.").Default("0s", "5m", "1h").HintAction(listResLevel).DurationList()
 	compactions := cmd.Flag("compaction", "Only blocks with these compaction levels will be replicated. Repeated flag.").Default("1", "2", "3", "4").Ints()
@@ -500,6 +500,7 @@ func registerBucketReplicate(app extkingpin.AppClause, objStoreConfig *extflag.P
 			reg,
 			tracer,
 			*httpBindAddr,
+			*httpTLSConfig,
 			time.Duration(*httpGracePeriod),
 			matchers,
 			resolutionLevels,
@@ -516,14 +517,14 @@ func registerBucketReplicate(app extkingpin.AppClause, objStoreConfig *extflag.P
 
 func registerBucketDownsample(app extkingpin.AppClause, objStoreConfig *extflag.PathOrContent) {
 	cmd := app.Command(component.Downsample.String(), "Continuously downsamples blocks in an object store bucket.")
-	httpAddr, httpGracePeriod := extkingpin.RegisterHTTPFlags(cmd)
+	httpAddr, httpGracePeriod, httpTLSConfig := extkingpin.RegisterHTTPFlags(cmd)
 	dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings.").
 		Default("./data").String()
 	hashFunc := cmd.Flag("hash-func", "Specify which hash function to use when calculating the hashes of produced files. If no function has been specified, it does not happen. This permits avoiding downloading some files twice albeit at some performance cost. Possible values are: \"\", \"SHA256\".").
 		Default("").Enum("SHA256", "")
 
 	cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error {
-		return RunDownsample(g, logger, reg, *httpAddr, time.Duration(*httpGracePeriod), *dataDir, objStoreConfig, component.Downsample, metadata.HashFunc(*hashFunc))
+		return RunDownsample(g, logger, reg, *httpAddr, *httpTLSConfig, time.Duration(*httpGracePeriod), *dataDir, objStoreConfig, component.Downsample, metadata.HashFunc(*hashFunc))
 	})
 }
 

docs/components/compact.md

@@ -377,6 +377,9 @@ Flags:
                                 Listen host:port for HTTP endpoints.
       --http-grace-period=2m    Time to wait after an interrupt received for
                                 HTTP Server.
+      --http.config=""          [EXPERIMENTAL] Path to the configuration file
+                                that can enable TLS or authentication for all
+                                HTTP endpoints.
       --log.format=logfmt       Log format to use. Possible options: logfmt or
                                 json.
       --log.level=info          Log filtering level.

docs/components/query-frontend.md

@@ -136,6 +136,9 @@ Flags:
                                  Listen host:port for HTTP endpoints.
       --http-grace-period=2m     Time to wait after an interrupt received for
                                  HTTP Server.
+      --http.config=""           [EXPERIMENTAL] Path to the configuration file
+                                 that can enable TLS or authentication for all
+                                 HTTP endpoints.
       --labels.default-time-range=24h
                                  The default metadata time range duration for
                                  retrieving labels through Labels and Series API

docs/components/query.md

@@ -323,6 +323,9 @@ Flags:
                                  Listen host:port for HTTP endpoints.
       --http-grace-period=2m     Time to wait after an interrupt received for
                                  HTTP Server.
+      --http.config=""           [EXPERIMENTAL] Path to the configuration file
+                                 that can enable TLS or authentication for all
+                                 HTTP endpoints.
       --log.format=logfmt        Log format to use. Possible options: logfmt or
                                  json.
       --log.level=info           Log filtering level.

docs/components/receive.md

@@ -101,6 +101,9 @@ Flags:
                                  Listen host:port for HTTP endpoints.
       --http-grace-period=2m     Time to wait after an interrupt received for
                                  HTTP Server.
+      --http.config=""           [EXPERIMENTAL] Path to the configuration file
+                                 that can enable TLS or authentication for all
+                                 HTTP endpoints.
       --label=key="value" ...    External labels to announce. This flag will be
                                  removed in the future when handling multiple
                                  tsdb instances is added.

docs/components/rule.md

@@ -311,6 +311,9 @@ Flags:
                                  Listen host:port for HTTP endpoints.
       --http-grace-period=2m     Time to wait after an interrupt received for
                                  HTTP Server.
+      --http.config=""           [EXPERIMENTAL] Path to the configuration file
+                                 that can enable TLS or authentication for all
+                                 HTTP endpoints.
       --label=<name>="<value>" ...
                                  Labels to be applied to all generated metrics
                                  (repeated). Similar to external labels for

docs/components/sidecar.md

@@ -108,6 +108,9 @@ Flags:
                                  Listen host:port for HTTP endpoints.
       --http-grace-period=2m     Time to wait after an interrupt received for
                                  HTTP Server.
+      --http.config=""           [EXPERIMENTAL] Path to the configuration file
+                                 that can enable TLS or authentication for all
+                                 HTTP endpoints.
       --log.format=logfmt        Log format to use. Possible options: logfmt or
                                  json.
       --log.level=info           Log filtering level.

docs/components/store.md

@@ -75,6 +75,9 @@ Flags:
                                  Listen host:port for HTTP endpoints.
       --http-grace-period=2m     Time to wait after an interrupt received for
                                  HTTP Server.
+      --http.config=""           [EXPERIMENTAL] Path to the configuration file
+                                 that can enable TLS or authentication for all
+                                 HTTP endpoints.
       --ignore-deletion-marks-delay=24h
                                  Duration after which the blocks marked for
                                  deletion will be filtered out while fetching

docs/components/tools.md

@@ -218,6 +218,9 @@ Flags:
                                 Listen host:port for HTTP endpoints.
       --http-grace-period=2m    Time to wait after an interrupt received for
                                 HTTP Server.
+      --http.config=""          [EXPERIMENTAL] Path to the configuration file
+                                that can enable TLS or authentication for all
+                                HTTP endpoints.
       --label=LABEL             Prometheus label to use as timeline title
       --log.format=logfmt       Log format to use. Possible options: logfmt or
                                 json.
@@ -485,6 +488,9 @@ Flags:
                                  Listen host:port for HTTP endpoints.
       --http-grace-period=2m     Time to wait after an interrupt received for
                                  HTTP Server.
+      --http.config=""           [EXPERIMENTAL] Path to the configuration file
+                                 that can enable TLS or authentication for all
+                                 HTTP endpoints.
       --id=ID ...                Block to be replicated to the destination
                                  bucket. IDs will be used to match blocks and
                                  other matchers will be ignored. When specified,
@@ -589,6 +595,9 @@ Flags:
                               Listen host:port for HTTP endpoints.
       --http-grace-period=2m  Time to wait after an interrupt received for HTTP
                               Server.
+      --http.config=""        [EXPERIMENTAL] Path to the configuration file that
+                              can enable TLS or authentication for all HTTP
+                              endpoints.
       --log.format=logfmt     Log format to use. Possible options: logfmt or
                               json.
       --log.level=info        Log filtering level.

docs/operating/https.md

@@ -0,0 +1,92 @@
+---
+title: Running Thanos with HTTPS and basic auth
+type: docs
+menu: operating
+---
+
+# HTTPS and authentication
+
+Thanos supports basic authentication and TLS. This is **experimental** and might change in the future.
+
+To specify which HTTP TLS configuration file to load, use the `--http.config` flag.
+The file is written in [YAML format](https://en.wikipedia.org/wiki/YAML), defined by the scheme described below.
+
+## Scheme
+
+Brackets indicate that a parameter is optional. For non-list parameters the value is set to the specified default.
+The file is read upon every http request, such as any change in the configuration and the certificates is picked up immediately.
+
+Generic placeholders are defined as follows:
+
+- `<boolean>`: a boolean that can take the values `true` or `false`
+- `<filename>`: a valid path in the current working directory
+- `<secret>`: a regular string that is a secret, such as a password
+- `<string>`: a regular string
+
+```yaml
+tls_server_config:
+  # Certificate and key files for server to use to authenticate to client.
+  cert_file: <filename>
+  key_file: <filename>
+
+  # Server policy for client authentication. Maps to ClientAuth Policies.
+  # For more detail on clientAuth options:
+  # https://golang.org/pkg/crypto/tls/#ClientAuthType
+  [ client_auth_type: <string> | default = "NoClientCert" ]
+
+  # CA certificate for client certificate authentication to the server.
+  [ client_ca_file: <filename> ]
+
+  # Minimum TLS version that is acceptable.
+  [ min_version: <string> | default = "TLS12" ]
+
+  # Maximum TLS version that is acceptable.
+  [ max_version: <string> | default = "TLS13" ]
+
+  # List of supported cipher suites for TLS versions up to TLS 1.2. If empty,
+  # Go default cipher suites are used. Available cipher suites are documented
+  # in the go documentation:
+  # https://golang.org/pkg/crypto/tls/#pkg-constants
+  [ cipher_suites:
+    [ - <string> ] ]
+
+  # prefer_server_cipher_suites controls whether the server selects the
+  # client's most preferred ciphersuite, or the server's most preferred
+  # ciphersuite. If true then the server's preference, as expressed in
+  # the order of elements in cipher_suites, is used.
+  [ prefer_server_cipher_suites: <bool> | default = true ]
+
+  # Elliptic curves that will be used in an ECDHE handshake, in preference
+  # order. Available curves are documented in the go documentation:
+  # https://golang.org/pkg/crypto/tls/#CurveID
+  [ curve_preferences:
+    [ - <string> ] ]
+
+http_server_config:
+  # Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS.
+  # This can not be changed on the fly.
+  [ http2: <boolean> | default = true ]
+
+# Usernames and hashed passwords that have full access to the web
+# server via basic authentication. If empty, no basic authentication is
+# required. Passwords are hashed with bcrypt.
+basic_auth_users:
+  [ <string>: <secret> ... ]
+```
+
+## Example
+
+An example configuration file is provided below,
+
+```yaml
+# A certificate and a key file are needed.
+tls_server_config:
+  cert_file: server.crt
+  key_file: server.key
+
+# Usernames and passwords required to connect to Thanos.
+# Passwords are hashed with bcrypt.
+basic_auth_users:
+  alice: $2y$10$mDwo.lAisC94iLAyP81MCesa29IzH37oigHC/42V2pdJlUprsJPze
+  bob: $2y$10$hLqFl9jSjoAAy95Z/zw8Ye8wkdMBM8c5Bn1ptYqP/AXyV0.oy0S8m
+```