Add TLS support to HTTP/GRPC clients

docs/configuration/config-file-reference.md

@@ -646,6 +646,56 @@ The `querier_config` configures the Cortex querier.
 # instances form a ring and addresses are picked from the ring).
 # CLI flag: -experimental.querier.store-gateway-addresses
 [store_gateway_addresses: <string> | default = ""]
+
+store_gateway_client_config:
+  # gRPC client max receive message size (bytes).
+  # CLI flag: -experimental.querier.store-gateway-client.grpc-max-recv-msg-size
+  [max_recv_msg_size: <int> | default = 104857600]
+
+  # gRPC client max send message size (bytes).
+  # CLI flag: -experimental.querier.store-gateway-client.grpc-max-send-msg-size
+  [max_send_msg_size: <int> | default = 16777216]
+
+  # Use compression when sending messages.
+  # CLI flag: -experimental.querier.store-gateway-client.grpc-use-gzip-compression
+  [use_gzip_compression: <boolean> | default = false]
+
+  # Rate limit for gRPC client; 0 means disabled.
+  # CLI flag: -experimental.querier.store-gateway-client.grpc-client-rate-limit
+  [rate_limit: <float> | default = 0]
+
+  # Rate limit burst for gRPC client.
+  # CLI flag: -experimental.querier.store-gateway-client.grpc-client-rate-limit-burst
+  [rate_limit_burst: <int> | default = 0]
+
+  # Enable backoff and retry when we hit ratelimits.
+  # CLI flag: -experimental.querier.store-gateway-client.backoff-on-ratelimits
+  [backoff_on_ratelimits: <boolean> | default = false]
+
+  backoff_config:
+    # Minimum delay when backing off.
+    # CLI flag: -experimental.querier.store-gateway-client.backoff-min-period
+    [min_period: <duration> | default = 100ms]
+
+    # Maximum delay when backing off.
+    # CLI flag: -experimental.querier.store-gateway-client.backoff-max-period
+    [max_period: <duration> | default = 10s]
+
+    # Number of times to backoff and retry before failing.
+    # CLI flag: -experimental.querier.store-gateway-client.backoff-retries
+    [max_retries: <int> | default = 10]
+
+  # TLS cert path for the client
+  # CLI flag: -experimental.querier.store-gateway-client.tls-cert-path
+  [tls_cert_path: <string> | default = ""]
+
+  # TLS key path for the client
+  # CLI flag: -experimental.querier.store-gateway-client.tls-key-path
+  [tls_key_path: <string> | default = ""]
+
+  # TLS CA path for the client
+  # CLI flag: -experimental.querier.store-gateway-client.tls-ca-path
+  [tls_ca_path: <string> | default = ""]
 ```
 
 ### `query_frontend_config`
@@ -757,6 +807,18 @@ The `ruler_config` configures the Cortex ruler.
 # CLI flag: -ruler.external.url
 [external_url: <url> | default = ]
 
+# TLS cert path for the client
+# CLI flag: -ruler.client.tls-cert-path
+[tls_cert_path: <string> | default = ""]
+
+# TLS key path for the client
+# CLI flag: -ruler.client.tls-key-path
+[tls_key_path: <string> | default = ""]
+
+# TLS CA path for the client
+# CLI flag: -ruler.client.tls-ca-path
+[tls_ca_path: <string> | default = ""]
+
 # How frequently to evaluate rules
 # CLI flag: -ruler.evaluation-interval
 [evaluation_interval: <duration> | default = 1m]
@@ -1584,6 +1646,18 @@ bigtable:
       # CLI flag: -bigtable.backoff-retries
       [max_retries: <int> | default = 10]
 
+    # TLS cert path for the client
+    # CLI flag: -bigtable.tls-cert-path
+    [tls_cert_path: <string> | default = ""]
+
+    # TLS key path for the client
+    # CLI flag: -bigtable.tls-key-path
+    [tls_key_path: <string> | default = ""]
+
+    # TLS CA path for the client
+    # CLI flag: -bigtable.tls-ca-path
+    [tls_ca_path: <string> | default = ""]
+
   # If enabled, once a tables info is fetched, it is cached.
   # CLI flag: -bigtable.table-cache.enabled
   [table_cache_enabled: <boolean> | default = true]
@@ -1960,6 +2034,18 @@ grpc_client_config:
     # Number of times to backoff and retry before failing.
     # CLI flag: -ingester.client.backoff-retries
     [max_retries: <int> | default = 10]
+
+  # TLS cert path for the client
+  # CLI flag: -ingester.client.tls-cert-path
+  [tls_cert_path: <string> | default = ""]
+
+  # TLS key path for the client
+  # CLI flag: -ingester.client.tls-key-path
+  [tls_key_path: <string> | default = ""]
+
+  # TLS CA path for the client
+  # CLI flag: -ingester.client.tls-ca-path
+  [tls_ca_path: <string> | default = ""]
 ```
 
 ### `frontend_worker_config`
@@ -2021,6 +2107,18 @@ grpc_client_config:
     # Number of times to backoff and retry before failing.
     # CLI flag: -querier.frontend-client.backoff-retries
     [max_retries: <int> | default = 10]
+
+  # TLS cert path for the client
+  # CLI flag: -querier.frontend-client.tls-cert-path
+  [tls_cert_path: <string> | default = ""]
+
+  # TLS key path for the client
+  # CLI flag: -querier.frontend-client.tls-key-path
+  [tls_key_path: <string> | default = ""]
+
+  # TLS CA path for the client
+  # CLI flag: -querier.frontend-client.tls-ca-path
+  [tls_ca_path: <string> | default = ""]
 ```
 
 ### `etcd_config`
@@ -2526,6 +2624,18 @@ The `configstore_config` configures the config database storing rules and alerts
 # Timeout for requests to Weave Cloud configs service.
 # CLI flag: -<prefix>.configs.client-timeout
 [client_timeout: <duration> | default = 5s]
+
+# TLS cert path for the client
+# CLI flag: -<prefix>.configs.tls-cert-path
+[tls_cert_path: <string> | default = ""]
+
+# TLS key path for the client
+# CLI flag: -<prefix>.configs.tls-key-path
+[tls_key_path: <string> | default = ""]
+
+# TLS CA path for the client
+# CLI flag: -<prefix>.configs.tls-ca-path
+[tls_ca_path: <string> | default = ""]
 ```
 
 ### `tsdb_config`

docs/configuration/single-process-config-blocks-tls.yaml

@@ -0,0 +1,100 @@
+
+# Configuration for running Cortex in single-process mode.
+# This should not be used in production.  It is only for getting started
+# and development.
+
+# Disable the requirement that every request to Cortex has a
+# X-Scope-OrgID header. `fake` will be substituted in instead.
+auth_enabled: false
+
+server:
+  http_listen_port: 9009
+
+  # Configure the server to allow messages up to 100MB.
+  grpc_server_max_recv_msg_size: 104857600
+  grpc_server_max_send_msg_size: 104857600
+  grpc_server_max_concurrent_streams: 1000
+  grpc_tls_config:
+    cert_file: "server.crt"
+    key_file: "server.key"
+    client_auth_type: "RequireAndVerifyClientCert"
+    client_ca_file: "root.crt"
+
+
+distributor:
+  shard_by_all_labels: true
+  pool:
+    health_check_ingesters: true
+
+ingester_client:
+  grpc_client_config:
+    # Configure the client to allow messages up to 100MB.
+    max_recv_msg_size: 104857600
+    max_send_msg_size: 104857600
+    use_gzip_compression: true
+    tls_cert_path: "client.crt"
+    tls_key_path: "client.key"
+    tls_ca_path: "root.crt"
+
+ingester:
+  # Disable blocks transfers on ingesters shutdown or rollout.
+  max_transfer_retries: 0
+
+  lifecycler:
+    # The address to advertise for this ingester.  Will be autodiscovered by
+    # looking up address on eth0 or en0; can be specified if this fails.
+    # address: 127.0.0.1
+
+    # We want to start immediately and flush on shutdown.
+    join_after: 0
+    min_ready_duration: 0s
+    final_sleep: 0s
+    num_tokens: 512
+
+    # Use an in memory ring store, so we don't need to launch a Consul.
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+
+storage:
+  engine: tsdb
+
+tsdb:
+  dir: /tmp/cortex/tsdb
+  bucket_store:
+    sync_dir: /tmp/cortex/tsdb-sync
+
+  # You can choose between local storage and Amazon S3, Google GCS and Azure storage. Each option requires additional configuration
+  # as shown below. All options can be configured via flags as well which might be handy for secret inputs.
+  backend: s3 # s3, gcs, azure or filesystem are valid options
+  s3:
+    bucket_name: cortex
+    endpoint: s3.dualstack.us-east-1.amazonaws.com
+    # Configure your S3 credentials below.
+    # secret_access_key: "TODO"
+    # access_key_id:     "TODO"
+#  gcs:
+#    bucket_name: cortex
+#    service_account: # if empty or omitted Cortex will use your default service account as per Google's fallback logic
+#  azure:
+#    account_name:
+#    account_key:
+#    container_name:
+#    endpoint_suffix:
+#    max_retries: # Number of retries for recoverable errors (defaults to 20)
+# filesystem:
+#   dir: ./data/tsdb
+
+compactor:
+  data_dir: /tmp/cortex/compactor
+  sharding_ring:
+    kvstore:
+      store: inmemory
+
+frontend_worker:
+  match_max_concurrent: true
+  grpc_client_config:
+    tls_cert_path: "client.crt"
+    tls_key_path: "client.key"
+    tls_ca_path: "root.crt"

docs/production/tls.md

@@ -0,0 +1,108 @@
+---
+title: "Securing communication between cortex components with TLS"
+linkTitle: "Securing communication between cortex components with TLS"
+weight: 5
+slug: tls
+---
+
+Cortex is a distributed system with significant traffic between its services.
+To allow for secure communication, Cortex supports TLS between all its
+components. This guide describes the process of setting up TLS.
+
+### Generation of certs to configure TLS
+
+The first step to securing inter-service communication in Cortex with TLS is
+generating certificates. A Certifying Authority (CA) will be used for this
+purpose which should be private to the organization, as any certificates signed
+by this CA will have permissions to communicate with the cluster.
+
+We will use the following script to generate self signed certs for the cluster:
+
+```
+# Refer: https://github.com/joe-elliott/cert-exporter/blob/69d3d7230378325a1de4fa313432d3d6ced4a518/test/files/genCerts.sh
+
+# keys
+openssl genrsa -out root.key
+openssl genrsa -out client.key
+openssl genrsa -out server.key
+
+# root cert / certifying authority
+openssl req -x509 -new -nodes -key root.key -subj "/C=US/ST=KY/O=Org/CN=root" -sha256 -days 100000 -out root.crt
+
+# csrs - certificate signing requests
+openssl req -new -sha256 -key client.key -subj "/C=US/ST=KY/O=Org/CN=client" -out client.csr
+openssl req -new -sha256 -key server.key -subj "/C=US/ST=KY/O=Org/CN=localhost" -out server.csr
+
+# certificates
+openssl x509 -req -in client.csr -CA root.crt -CAkey root.key -CAcreateserial -out client.crt -days 100000 -sha256
+openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -CAcreateserial -out server.crt -days 100000 -sha256
+```
+
+Note that the above script generates certificates that are valid for 100000 days.
+This can be changed by adjusting the `-days` option in the above commands.
+It is recommended that the certs be replaced atleast once every 2 years.
+
+The above script generates keys `client.key, server.key` and certs
+`client.crt, server.crt` for both the client and server. The CA cert is
+generated as `root.crt`.
+
+### Load certs into the HTTP/GRPC server/client
+
+Every HTTP/GRPC link between Cortex components supports TLS configuration
+through the following config parameters:
+
+#### Server flags
+
+```
+    # Path to the TLS Cert for the HTTP Server
+    -server.http-tls-cert-path=/path/to/server.crt
+
+    # Path to the TLS Key for the HTTP Server
+    -server.http-tls-key-path=/path/to/server.key
+
+    # Type of Client Auth for the HTTP Server
+    -server.http-tls-client-auth="RequireAndVerifyClientCert"
+
+    # Path to the Client CA Cert for the HTTP Server
+    -server.http-tls-ca-path="/path/to/root.crt"
+
+    # Path to the TLS Cert for the GRPC Server
+    -server.grpc-tls-cert-path=/path/to/server.crt
+
+    # Path to the TLS Key for the GRPC Server
+    -server.grpc-tls-key-path=/path/to/server.key
+
+    # Type of Client Auth for the GRPC Server
+    -server.grpc-tls-client-auth="RequireAndVerifyClientCert"
+
+    # Path to the Client CA Cert for the GRPC Server
+    -server.grpc-tls-ca-path=/path/to/root.crt
+```
+
+#### Client flags
+
+Client flags are component specific.
+
+For an HTTP client in the Alertmanager:
+```
+    # Path to the TLS Cert for the HTTP Client
+    -alertmanager.configs.tls-cert-path=/path/to/client.crt
+
+    # Path to the TLS Key for the HTTP Client
+    -alertmanager.configs.tls-key-path=/path/to/client.key
+
+    # Path to the TLS CA for the HTTP Client
+    -alertmanager.configs.tls-ca-path=/path/to/root.crt
+```
+
+For a GRPC client in the Querier:
+```
+    # Path to the TLS Cert for the GRPC Client
+    -querier.frontend-client.tls-cert-path=/path/to/client.crt
+
+    # Path to the TLS Key for the GRPC Client
+    -querier.frontend-client.tls-key-path=/path/to/client.key
+
+    # Path to the TLS CA for the GRPC Client
+    -querier.frontend-client.tls-ca-path=/path/to/root.crt
+```