Skip to content

Commit

Permalink
Add etcd s3 config secret implementation
Browse files Browse the repository at this point in the history
Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
  • Loading branch information
brandond committed Jun 11, 2024
1 parent f10cb29 commit f8c0ce0
Show file tree
Hide file tree
Showing 10 changed files with 236 additions and 63 deletions.
12 changes: 10 additions & 2 deletions docs/adrs/etcd-s3-secret.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Support etcd Snapshot Configuration via Kubernetes Secret

Date: 2024-02-06
Revised: 2024-06-10

## Status

Expand Down Expand Up @@ -34,8 +35,8 @@ avoids embedding the credentials directly in the system configuration, chart val
* We will add a `--etcd-s3-proxy` flag that can be used to set the proxy used by the S3 client. This will override the
settings that golang's default HTTP client reads from the `HTTP_PROXY/HTTPS_PROXY/NO_PROXY` environment varibles.
* We will add support for reading etcd snapshot S3 configuration from a Secret. The secret name will be specified via a new
`--etcd-s3-secret` flag, which accepts the name of the Secret in the `kube-system` namespace.
* Presence of the `--etcd-s3-secret` flag does not imply `--etcd-s3`. If S3 is not enabled by use of the `--etcd-s3` flag,
`--etcd-s3-config-secret` flag, which accepts the name of the Secret in the `kube-system` namespace.
* Presence of the `--etcd-s3-config-secret` flag does not imply `--etcd-s3`. If S3 is not enabled by use of the `--etcd-s3` flag,
the Secret will not be used.
* The Secret does not need to exist when K3s starts; it will be checked for every time a snapshot operation is performed.
* Secret and CLI/config values will NOT be merged. The Secret will provide values to be used in absence of other
Expand Down Expand Up @@ -64,6 +65,7 @@ stringData:
etcd-s3-access-key: "AWS_ACCESS_KEY_ID"
etcd-s3-secret-key: "AWS_SECRET_ACCESS_KEY"
etcd-s3-bucket: "bucket"
etcd-s3-folder: "folder"
etcd-s3-region: "us-east-1"
etcd-s3-insecure: "false"
etcd-s3-timeout: "5m"
Expand All @@ -73,3 +75,9 @@ stringData:
## Consequences

This will require additional documentation, tests, and QA work to validate use of secrets for s3 snapshot configuration.

## Revisions

#### 2024-06-10:
* Changed flag to `etcd-s3-config-secret` to avoid confusion with `etcd-s3-secret-key`.
* Added `etcd-s3-folder` to example Secret.
10 changes: 10 additions & 0 deletions pkg/cli/cmds/etcd_snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,16 @@ var EtcdSnapshotFlags = []cli.Flag{
Usage: "(db) S3 folder",
Destination: &ServerConfig.EtcdS3Folder,
},
&cli.StringFlag{
Name: "etcd-s3-proxy",
Usage: "(db) Proxy server to use when connecting to S3, overriding any proxy-releated environment variables",
Destination: &ServerConfig.EtcdS3Proxy,
},
&cli.StringFlag{
Name: "etcd-s3-config-secret",
Usage: "(db) Name of secret in the kube-system namespace used to configure S3, if etcd-s3 is enabled and no other etcd-s3 options are set",
Destination: &ServerConfig.EtcdS3ConfigSecret,
},
&cli.BoolFlag{
Name: "s3-insecure,etcd-s3-insecure",
Usage: "(db) Disables S3 over HTTPS",
Expand Down
12 changes: 12 additions & 0 deletions pkg/cli/cmds/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ type Server struct {
EtcdS3BucketName string
EtcdS3Region string
EtcdS3Folder string
EtcdS3Proxy string
EtcdS3ConfigSecret string
EtcdS3Timeout time.Duration
EtcdS3Insecure bool
ServiceLBNamespace string
Expand Down Expand Up @@ -430,6 +432,16 @@ var ServerFlags = []cli.Flag{
Usage: "(db) S3 folder",
Destination: &ServerConfig.EtcdS3Folder,
},
&cli.StringFlag{
Name: "etcd-s3-proxy",
Usage: "(db) Proxy server to use when connecting to S3, overriding any proxy-releated environment variables",
Destination: &ServerConfig.EtcdS3Proxy,
},
&cli.StringFlag{
Name: "etcd-s3-config-secret",
Usage: "(db) Name of secret in the kube-system namespace used to configure S3, if etcd-s3 is enabled and no other etcd-s3 options are set",
Destination: &ServerConfig.EtcdS3ConfigSecret,
},
&cli.BoolFlag{
Name: "etcd-s3-insecure",
Usage: "(db) Disables S3 over HTTPS",
Expand Down
2 changes: 2 additions & 0 deletions pkg/cli/etcdsnapshot/etcd_snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,12 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c
sr.S3 = &etcd.SnapshotRequestS3{}
sr.S3.AccessKey = cfg.EtcdS3AccessKey
sr.S3.Bucket = cfg.EtcdS3BucketName
sr.S3.ConfigSecret = cfg.EtcdS3ConfigSecret
sr.S3.Endpoint = cfg.EtcdS3Endpoint
sr.S3.EndpointCA = cfg.EtcdS3EndpointCA
sr.S3.Folder = cfg.EtcdS3Folder
sr.S3.Insecure = cfg.EtcdS3Insecure
sr.S3.Proxy = cfg.EtcdS3Proxy
sr.S3.Region = cfg.EtcdS3Region
sr.S3.SecretKey = cfg.EtcdS3SecretKey
sr.S3.SkipSSLVerify = cfg.EtcdS3SkipSSLVerify
Expand Down
12 changes: 7 additions & 5 deletions pkg/cli/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,15 +187,17 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
serverConfig.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir
serverConfig.ControlConfig.EtcdSnapshotRetention = cfg.EtcdSnapshotRetention
serverConfig.ControlConfig.EtcdS3 = cfg.EtcdS3
serverConfig.ControlConfig.EtcdS3Endpoint = cfg.EtcdS3Endpoint
serverConfig.ControlConfig.EtcdS3EndpointCA = cfg.EtcdS3EndpointCA
serverConfig.ControlConfig.EtcdS3SkipSSLVerify = cfg.EtcdS3SkipSSLVerify
serverConfig.ControlConfig.EtcdS3AccessKey = cfg.EtcdS3AccessKey
serverConfig.ControlConfig.EtcdS3SecretKey = cfg.EtcdS3SecretKey
serverConfig.ControlConfig.EtcdS3BucketName = cfg.EtcdS3BucketName
serverConfig.ControlConfig.EtcdS3Region = cfg.EtcdS3Region
serverConfig.ControlConfig.EtcdS3ConfigSecret = cfg.EtcdS3ConfigSecret
serverConfig.ControlConfig.EtcdS3Endpoint = cfg.EtcdS3Endpoint
serverConfig.ControlConfig.EtcdS3EndpointCA = cfg.EtcdS3EndpointCA
serverConfig.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder
serverConfig.ControlConfig.EtcdS3Insecure = cfg.EtcdS3Insecure
serverConfig.ControlConfig.EtcdS3Proxy = cfg.EtcdS3Proxy
serverConfig.ControlConfig.EtcdS3Region = cfg.EtcdS3Region
serverConfig.ControlConfig.EtcdS3SecretKey = cfg.EtcdS3SecretKey
serverConfig.ControlConfig.EtcdS3SkipSSLVerify = cfg.EtcdS3SkipSSLVerify
serverConfig.ControlConfig.EtcdS3Timeout = cfg.EtcdS3Timeout
} else {
logrus.Info("ETCD snapshots are disabled")
Expand Down
14 changes: 8 additions & 6 deletions pkg/daemons/config/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -226,16 +226,18 @@ type Control struct {
EtcdSnapshotCompress bool `json:"-"`
EtcdListFormat string `json:"-"`
EtcdS3 bool `json:"-"`
EtcdS3Endpoint string `json:"-"`
EtcdS3EndpointCA string `json:"-"`
EtcdS3SkipSSLVerify bool `json:"-"`
EtcdS3AccessKey string `json:"-"`
EtcdS3SecretKey string `json:"-"`
EtcdS3BucketName string `json:"-"`
EtcdS3Region string `json:"-"`
EtcdS3ConfigSecret string `json:"-"`
EtcdS3Endpoint string `json:"-"`
EtcdS3EndpointCA string `json:"-"`
EtcdS3Folder string `json:"-"`
EtcdS3Timeout time.Duration `json:"-"`
EtcdS3Insecure bool `json:"-"`
EtcdS3Proxy string `json:"-"`
EtcdS3Region string `json:"-"`
EtcdS3SecretKey string `json:"-"`
EtcdS3SkipSSLVerify bool `json:"-"`
EtcdS3Timeout time.Duration `json:"-"`
ServerNodeName string
VLevel int
VModule string
Expand Down
117 changes: 117 additions & 0 deletions pkg/etcd/config_secret.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package etcd

import (
"encoding/base64"
"strconv"
"strings"
"time"

"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/sirupsen/logrus"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// withS3ConfigSecret returns a modified ETCD struct that is overriden
// with etcd S3 snapshot config from the secret.
// If the secret does not exist, the etcd struct is returned unmodified.
func (e *ETCD) withS3ConfigSecret() *ETCD {
secretName := e.config.EtcdS3ConfigSecret
if secretName == "" {
return e
}

if e.hasS3Config() {
logrus.Infof("Ignoring S3 config from etcd-s3-config-secret %s: S3 configuration is already present", secretName)
return e
}

secret, err := e.config.Runtime.Core.Core().V1().Secret().Get("kube-system", secretName, metav1.GetOptions{})
if err != nil {
if !apierrors.IsNotFound(err) {
logrus.Warnf("Failed to get Secret for etcd-s3-config-secret %s: %v", secretName, err)
}
return e
}

logrus.Infof("Using S3 config from etcd-config-secret %s", secretName)
caBundles := []string{}
re := &ETCD{
client: e.client,
config: &config.Control{
CriticalControlArgs: e.config.CriticalControlArgs,
Runtime: e.config.Runtime,
DataDir: e.config.DataDir,
Datastore: e.config.Datastore,
EtcdSnapshotCompress: e.config.EtcdSnapshotCompress,
EtcdSnapshotDir: e.config.EtcdSnapshotDir,
EtcdSnapshotName: e.config.EtcdSnapshotName,
EtcdSnapshotRetention: e.config.EtcdSnapshotRetention,
},
name: e.name,
address: e.address,
cron: e.cron,
cancel: e.cancel,
snapshotSem: e.snapshotSem,
}
re.config.EtcdS3AccessKey = string(secret.Data["etcd-s3-access-key"])
re.config.EtcdS3BucketName = string(secret.Data["etcd-s3-bucket"])
re.config.EtcdS3Endpoint = string(secret.Data["etcd-s3-endpoint"])
re.config.EtcdS3Folder = string(secret.Data["etcd-s3-folder"])
re.config.EtcdS3Proxy = string(secret.Data["etcd-s3-proxy"])
re.config.EtcdS3Region = string(secret.Data["etcd-s3-region"])
re.config.EtcdS3SecretKey = string(secret.Data["etcd-s3-secret-key"])

re.config.EtcdS3SkipSSLVerify, err = strconv.ParseBool(string(secret.Data["etcd-s3-skip-ssl-verify"]))
if err != nil {
logrus.Warnf("Failed to parse etcd-s3-skip-ssl-verify value from S3 config secret %s: %v", secretName, err)
}

re.config.EtcdS3Insecure, err = strconv.ParseBool(string(secret.Data["etcd-s3-insecure"]))
if err != nil {
logrus.Warnf("Failed to parse etcd-s3-insecure value from S3 config secret %s: %v", secretName, err)
}

re.config.EtcdS3Timeout, err = time.ParseDuration(string(secret.Data["etcd-s3-timeout"]))
if err != nil {
logrus.Warnf("Failed to parse etcd-s3-timeout value from S3 config secret %s: %v", secretName, err)
}

// Add inline CA bundle if set
if len(secret.Data["etcd-s3-endpoint-ca"]) > 0 {
caBundles = append(caBundles, base64.StdEncoding.EncodeToString(secret.Data["etcd-s3-endpoint-ca"]))
}

// Add CA bundles from named configmap if set
if caConfigMapName := string(secret.Data["etcd-s3-endpoint-ca-name"]); caConfigMapName != "" {
configMap, err := e.config.Runtime.Core.Core().V1().ConfigMap().Get("kube-system", caConfigMapName, metav1.GetOptions{})
if err != nil {
logrus.Warnf("Failed to get ConfigMap %s for etcd-s3-endpoint-ca-name value from S3 config secret %s: %v", caConfigMapName, secretName, err)
} else {
for _, v := range configMap.Data {
caBundles = append(caBundles, base64.StdEncoding.EncodeToString([]byte(v)))
}
for _, v := range configMap.BinaryData {
caBundles = append(caBundles, base64.StdEncoding.EncodeToString(v))
}
}
}

// Concatenate all requested CA bundle strings into config var
re.config.EtcdS3EndpointCA = strings.Join(caBundles, " ")

return re
}

func (e *ETCD) hasS3Config() bool {
return e.config.EtcdS3Insecure == true ||
e.config.EtcdS3SkipSSLVerify == true ||
e.config.EtcdS3AccessKey != "" ||
e.config.EtcdS3BucketName != "" ||
e.config.EtcdS3Endpoint != "" ||
e.config.EtcdS3EndpointCA != "" ||
e.config.EtcdS3Folder != "" ||
e.config.EtcdS3Proxy != "" ||
e.config.EtcdS3Region != "" ||
e.config.EtcdS3SecretKey != ""
}
84 changes: 40 additions & 44 deletions pkg/etcd/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ import (
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"fmt"
"io/ioutil"
"net/http"
"net/textproto"
"net/url"
"os"
"path"
"path/filepath"
Expand Down Expand Up @@ -52,19 +52,29 @@ func NewS3(ctx context.Context, config *config.Control) (*S3, error) {
if config.EtcdS3BucketName == "" {
return nil, errors.New("s3 bucket name was not set")
}
tr := http.DefaultTransport
tr := http.DefaultTransport.(*http.Transport).Clone()

switch {
case config.EtcdS3EndpointCA != "":
trCA, err := setTransportCA(tr, config.EtcdS3EndpointCA, config.EtcdS3SkipSSLVerify)
// You can either disable SSL verification or use a custom CA bundle,
// it doesn't make sense to do both - if verification is disabled,
// the CA is not checked!
if config.EtcdS3SkipSSLVerify {
tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
} else if config.EtcdS3EndpointCA != "" {
tlsConfig, err := loadEndpointCAs(config.EtcdS3EndpointCA)
if err != nil {
return nil, err
}
tr = trCA
case config.EtcdS3 && config.EtcdS3SkipSSLVerify:
tr.(*http.Transport).TLSClientConfig = &tls.Config{
InsecureSkipVerify: config.EtcdS3SkipSSLVerify,
tr.TLSClientConfig = tlsConfig
}

// Set a fixed proxy URL, if requested by the user. This replaces the default,
// which calls ProxyFromEnvironment to read proxy settings from the environment.
if config.EtcdS3Proxy != "" {
u, err := url.Parse(config.EtcdS3Proxy)
if err != nil {
return nil, errors.Wrap(err, "failed to parse etcd-s3-proxy value as URL")
}
tr.Proxy = http.ProxyURL(u)
}

var creds *credentials.Credentials
Expand Down Expand Up @@ -445,49 +455,35 @@ func (s *S3) listSnapshots(ctx context.Context) (map[string]snapshotFile, error)
return snapshots, nil
}

func readS3EndpointCA(endpointCA string) ([]byte, error) {
ca, err := base64.StdEncoding.DecodeString(endpointCA)
if err != nil {
return os.ReadFile(endpointCA)
}
return ca, nil
}

func setTransportCA(tr http.RoundTripper, endpointCA string, insecureSkipVerify bool) (http.RoundTripper, error) {
ca, err := readS3EndpointCA(endpointCA)
if err != nil {
return tr, err
}
if !isValidCertificate(ca) {
return tr, errors.New("endpoint-ca is not a valid x509 certificate")
}

func loadEndpointCAs(etcdS3EndpointCA string) (*tls.Config, error) {
var loaded bool
certPool := x509.NewCertPool()
certPool.AppendCertsFromPEM(ca)

tr.(*http.Transport).TLSClientConfig = &tls.Config{
RootCAs: certPool,
InsecureSkipVerify: insecureSkipVerify,
for _, ca := range strings.Split(etcdS3EndpointCA, " ") {
// Try to decode the value as base64-encoded data - yes, a base64 string that itself
// contains multiline, ascii-armored, base64-encoded certificate data - as would be produced
// by `base64 --wrap=0 /path/to/cert.pem`. If this fails, assume the value is the path to a
// file on disk, and try to read that. This is backwards compatible with RKE1.
caData, err := base64.StdEncoding.DecodeString(ca)
if err != nil {
caData, err = os.ReadFile(ca)
}
if err != nil {
return nil, err
}
if certPool.AppendCertsFromPEM(caData) {
loaded = true
}
}

return tr, nil
}

// isValidCertificate checks to see if the given
// byte slice is a valid x509 certificate.
func isValidCertificate(c []byte) bool {
p, _ := pem.Decode(c)
if p == nil {
return false
}
if _, err := x509.ParseCertificates(p.Bytes); err != nil {
return false
if loaded {
return &tls.Config{RootCAs: certPool}, nil
}
return true
return nil, errors.New("no certificates loaded from etcd-s3-endpoint-ca")
}

func bucketLookupType(endpoint string) minio.BucketLookupType {
if strings.Contains(endpoint, "aliyun") { // backwards compt with RKE1
if strings.Contains(endpoint, "aliyun") { // backwards compatible with RKE1
return minio.BucketLookupDNS
}
return minio.BucketLookupAuto
Expand Down
Loading

0 comments on commit f8c0ce0

Please sign in to comment.