From 9a2ddbcf8e0e0100ccad7de7499f68a9cf1a5337 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Thu, 25 Jun 2020 12:03:37 +0000 Subject: [PATCH 1/4] Improve JWT Rotation --- lifecycle.go | 1 + lifecycle_probes.go | 192 ++++++++++ pkg/apis/deployment/v1/hashes.go | 22 +- pkg/apis/deployment/v1/plan.go | 12 + .../deployment/v1/zz_generated.deepcopy.go | 56 ++- pkg/deployment/client/client.go | 55 +++ pkg/deployment/client/encryption.go | 25 +- .../client/jwt.go} | 13 +- pkg/deployment/client/sha.go | 101 ++++++ pkg/deployment/client/tls.go | 3 +- pkg/deployment/client_cache.go | 124 +++++-- pkg/deployment/context_impl.go | 109 +++++- pkg/deployment/deployment.go | 21 +- pkg/deployment/deployment_affinity_test.go | 18 +- pkg/deployment/deployment_core_test.go | 50 +-- pkg/deployment/deployment_encryption_test.go | 8 +- pkg/deployment/deployment_image_test.go | 6 +- pkg/deployment/deployment_inspector.go | 4 +- pkg/deployment/deployment_metrics_test.go | 12 +- pkg/deployment/deployment_pod_probe_test.go | 20 +- .../deployment_pod_resources_test.go | 6 +- pkg/deployment/deployment_pod_sync_test.go | 6 +- pkg/deployment/deployment_pod_tls_sni_test.go | 10 +- pkg/deployment/deployment_pod_volumes_test.go | 6 +- pkg/deployment/deployment_suite_test.go | 105 ++++-- pkg/deployment/images.go | 17 +- pkg/deployment/images_test.go | 2 +- pkg/deployment/pod/encryption.go | 21 +- pkg/deployment/pod/jwt.go | 43 ++- .../reconcile/action_encryption_add.go | 2 +- .../reconcile/action_encryption_refresh.go | 2 +- .../reconcile/action_encryption_remove.go | 2 +- .../action_encryption_status_update.go | 10 +- pkg/deployment/reconcile/action_jwt_add.go | 124 +++++++ pkg/deployment/reconcile/action_jwt_clean.go | 115 ++++++ .../reconcile/action_jwt_propagated.go | 77 ++++ .../reconcile/action_jwt_refresh.go | 78 ++++ .../reconcile/action_jwt_set_active.go | 116 ++++++ .../reconcile/action_jwt_status_update.go | 183 ++++++++++ .../reconcile/action_tls_ca_append.go | 8 +- .../reconcile/action_tls_ca_clean.go | 4 +- .../reconcile/action_tls_keyfile_refresh.go | 2 +- pkg/deployment/reconcile/helper_tls_sni.go | 2 +- pkg/deployment/reconcile/plan_builder.go | 18 +- .../reconcile/plan_builder_encryption.go | 135 ++++--- pkg/deployment/reconcile/plan_builder_jwt.go | 339 ++++++++++++++++++ .../reconcile/plan_builder_restore.go | 2 +- pkg/deployment/reconcile/plan_builder_tls.go | 108 ++++-- pkg/deployment/resources/exporter.go | 8 +- .../resources/pod_creator_probes.go | 146 ++++++-- pkg/deployment/resources/secret_hashes.go | 35 +- pkg/deployment/resources/secrets.go | 64 +++- pkg/util/arangod/client.go | 24 +- pkg/util/arangod/conn/factory.go | 129 +++++++ pkg/util/checksum.go | 4 + pkg/util/k8sutil/lifecycle.go | 8 +- pkg/util/k8sutil/probes.go | 86 ----- pkg/util/k8sutil/probes/probes.go | 142 ++++++++ pkg/util/k8sutil/{ => probes}/probes_test.go | 2 +- 59 files changed, 2573 insertions(+), 470 deletions(-) create mode 100644 lifecycle_probes.go rename pkg/{apis/deployment/v1/key_hashes.go => deployment/client/jwt.go} (77%) create mode 100644 pkg/deployment/client/sha.go create mode 100644 pkg/deployment/reconcile/action_jwt_add.go create mode 100644 pkg/deployment/reconcile/action_jwt_clean.go create mode 100644 pkg/deployment/reconcile/action_jwt_propagated.go create mode 100644 pkg/deployment/reconcile/action_jwt_refresh.go create mode 100644 pkg/deployment/reconcile/action_jwt_set_active.go create mode 100644 pkg/deployment/reconcile/action_jwt_status_update.go create mode 100644 pkg/deployment/reconcile/plan_builder_jwt.go create mode 100644 pkg/util/arangod/conn/factory.go delete mode 100644 pkg/util/k8sutil/probes.go create mode 100644 pkg/util/k8sutil/probes/probes.go rename pkg/util/k8sutil/{ => probes}/probes_test.go (99%) diff --git a/lifecycle.go b/lifecycle.go index e37d4401e..1b02c355c 100644 --- a/lifecycle.go +++ b/lifecycle.go @@ -62,6 +62,7 @@ func init() { cmdMain.AddCommand(cmdLifecycle) cmdLifecycle.AddCommand(cmdLifecyclePreStop) cmdLifecycle.AddCommand(cmdLifecycleCopy) + cmdLifecycle.AddCommand(cmdLifecycleProbe) cmdLifecycleCopy.Flags().StringVar(&lifecycleCopyOptions.TargetDir, "target", "", "Target directory to copy the executable to") } diff --git a/lifecycle_probes.go b/lifecycle_probes.go new file mode 100644 index 000000000..53f973c8b --- /dev/null +++ b/lifecycle_probes.go @@ -0,0 +1,192 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package main + +import ( + "crypto/tls" + "fmt" + "io/ioutil" + "net/http" + "os" + "path" + + "github.com/arangodb/go-driver/jwt" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var ( + cmdLifecycleProbe = &cobra.Command{ + Use: "probe", + Run: cmdLifecycleProbeCheck, + } + + probeInput struct { + SSL bool + Auth bool + Endpoint string + JWTPath string + } +) + +func init() { + f := cmdLifecycleProbe.PersistentFlags() + + f.BoolVarP(&probeInput.SSL, "ssl", "", false, "Determines if SSL is enabled") + f.BoolVarP(&probeInput.Auth, "auth", "", false, "Determines if authentication is enabled") + f.StringVarP(&probeInput.Endpoint, "endpoint", "", "/_api/version", "Determines if SSL is enabled") + f.StringVarP(&probeInput.JWTPath, "jwt", "", k8sutil.ClusterJWTSecretVolumeMountDir, "Path to the JWT tokens") +} + +func probeClient() *http.Client { + tr := &http.Transport{} + + if probeInput.SSL { + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + client := &http.Client{ + Transport: tr, + } + + return client +} + +func probeEndpoint(endpoint string) string { + proto := "http" + if probeInput.SSL { + proto = "https" + } + + return fmt.Sprintf("%s://%s:%d%s", proto, "127.0.0.1", k8sutil.ArangoPort, endpoint) +} + +func readJWTFile(file string) ([]byte, error) { + p := path.Join(probeInput.JWTPath, file) + log.Info().Str("path", p).Msgf("Try to use file") + + f, err := os.Open(p) + if err != nil { + return nil, err + } + + defer f.Close() + data, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + return data, nil +} + +func getJWTToken() ([]byte, error) { + // Try read default one + if token, err := readJWTFile(constants.SecretKeyToken); err == nil { + log.Info().Str("token", constants.SecretKeyToken).Msgf("Using JWT Token") + return token, nil + } + + // Try read active one + if token, err := readJWTFile(pod.ActiveJWTKey); err == nil { + log.Info().Str("token", pod.ActiveJWTKey).Msgf("Using JWT Token") + return token, nil + } + + if files, err := ioutil.ReadDir(probeInput.JWTPath); err == nil { + for _, file := range files { + if token, err := readJWTFile(file.Name()); err == nil { + log.Info().Str("token", file.Name()).Msgf("Using JWT Token") + return token, nil + } + } + } + + return nil, errors.Errorf("Unable to find any token") +} + +func addAuthHeader(req *http.Request) error { + if !probeInput.Auth { + return nil + } + + token, err := getJWTToken() + if err != nil { + return err + } + + header, err := jwt.CreateArangodJwtAuthorizationHeader(string(token), "probe") + if err != nil { + return err + } + + req.Header.Add("Authorization", header) + return nil +} + +func doRequest() (*http.Response, error) { + client := probeClient() + + req, err := http.NewRequest(http.MethodGet, probeEndpoint(probeInput.Endpoint), nil) + if err != nil { + return nil, err + } + + if err := addAuthHeader(req); err != nil { + return nil, err + } + + return client.Do(req) +} + +func cmdLifecycleProbeCheck(cmd *cobra.Command, args []string) { + if err := cmdLifecycleProbeCheckE(); err != nil { + log.Error().Err(err).Msgf("Fatal") + os.Exit(1) + } +} + +func cmdLifecycleProbeCheckE() error { + resp, err := doRequest() + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + if resp.Body != nil { + defer resp.Body.Close() + if data, err := ioutil.ReadAll(resp.Body); err == nil { + return errors.Errorf("Unexpected code: %d - %s", resp.StatusCode, string(data)) + } + } + + return errors.Errorf("Unexpected code: %d", resp.StatusCode) + } + + log.Info().Msgf("Check passed") + + return nil +} diff --git a/pkg/apis/deployment/v1/hashes.go b/pkg/apis/deployment/v1/hashes.go index 852998813..1f0105795 100644 --- a/pkg/apis/deployment/v1/hashes.go +++ b/pkg/apis/deployment/v1/hashes.go @@ -22,12 +22,26 @@ package v1 +import shared "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1" + type DeploymentStatusHashes struct { - Encryption DeploymentStatusHashList `json:"encryption,omitempty"` - TLS DeploymentStatusHashesTLS `json:"tls,omitempty"` + Encryption DeploymentStatusHashesEncryption `json:"rocksDBEncryption,omitempty"` + TLS DeploymentStatusHashesTLS `json:"tls,omitempty"` + JWT DeploymentStatusHashesJWT `json:"jwt,omitempty"` +} + +type DeploymentStatusHashesEncryption struct { + Keys shared.HashList `json:"keys,omitempty"` } type DeploymentStatusHashesTLS struct { - CA *string `json:"ca,omitempty"` - Truststore DeploymentStatusHashList `json:"truststore,omitempty"` + CA *string `json:"ca,omitempty"` + Truststore shared.HashList `json:"truststore,omitempty"` +} + +type DeploymentStatusHashesJWT struct { + Active string `json:"active,omitempty"` + Passive shared.HashList `json:"passive,omitempty"` + + Propagated bool `json:"propagated,omitempty"` } diff --git a/pkg/apis/deployment/v1/plan.go b/pkg/apis/deployment/v1/plan.go index 3b740f5d8..9f8350e5a 100644 --- a/pkg/apis/deployment/v1/plan.go +++ b/pkg/apis/deployment/v1/plan.go @@ -101,6 +101,18 @@ const ( ActionTypeEncryptionKeyRefresh ActionType = "EncryptionKeyRefresh" // ActionTypeEncryptionKeyStatusUpdate update status object with current encryption keys ActionTypeEncryptionKeyStatusUpdate ActionType = "EncryptionKeyStatusUpdate" + // ActionTypeJWTStatusUpdate update status of JWT Secret + ActionTypeJWTStatusUpdate ActionType = "JWTStatusUpdate" + // ActionTypeJWTSetActive change active JWT key + ActionTypeJWTSetActive ActionType = "JWTSetActive" + // ActionTypeJWTAdd add new JWT key + ActionTypeJWTAdd ActionType = "JWTAdd" + // ActionTypeJWTClean Clean old JWT key + ActionTypeJWTClean ActionType = "JWTClean" + // ActionTypeJWTRefresh refresh jwt tokens + ActionTypeJWTRefresh ActionType = "JWTRefresh" + // ActionTypeJWTPropagated change propagated flag + ActionTypeJWTPropagated ActionType = "JWTPropagated" ) const ( diff --git a/pkg/apis/deployment/v1/zz_generated.deepcopy.go b/pkg/apis/deployment/v1/zz_generated.deepcopy.go index aebb49788..85463552e 100644 --- a/pkg/apis/deployment/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1/zz_generated.deepcopy.go @@ -27,6 +27,7 @@ package v1 import ( time "time" + sharedv1 "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" @@ -427,43 +428,62 @@ func (in *DeploymentStatus) DeepCopy() *DeploymentStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in DeploymentStatusHashList) DeepCopyInto(out *DeploymentStatusHashList) { - { - in := &in - *out = make(DeploymentStatusHashList, len(*in)) +func (in *DeploymentStatusHashes) DeepCopyInto(out *DeploymentStatusHashes) { + *out = *in + in.Encryption.DeepCopyInto(&out.Encryption) + in.TLS.DeepCopyInto(&out.TLS) + in.JWT.DeepCopyInto(&out.JWT) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashes. +func (in *DeploymentStatusHashes) DeepCopy() *DeploymentStatusHashes { + if in == nil { + return nil + } + out := new(DeploymentStatusHashes) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentStatusHashesEncryption) DeepCopyInto(out *DeploymentStatusHashesEncryption) { + *out = *in + if in.Keys != nil { + in, out := &in.Keys, &out.Keys + *out = make(sharedv1.HashList, len(*in)) copy(*out, *in) - return } + return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashList. -func (in DeploymentStatusHashList) DeepCopy() DeploymentStatusHashList { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashesEncryption. +func (in *DeploymentStatusHashesEncryption) DeepCopy() *DeploymentStatusHashesEncryption { if in == nil { return nil } - out := new(DeploymentStatusHashList) + out := new(DeploymentStatusHashesEncryption) in.DeepCopyInto(out) - return *out + return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DeploymentStatusHashes) DeepCopyInto(out *DeploymentStatusHashes) { +func (in *DeploymentStatusHashesJWT) DeepCopyInto(out *DeploymentStatusHashesJWT) { *out = *in - if in.Encryption != nil { - in, out := &in.Encryption, &out.Encryption - *out = make(DeploymentStatusHashList, len(*in)) + if in.Passive != nil { + in, out := &in.Passive, &out.Passive + *out = make(sharedv1.HashList, len(*in)) copy(*out, *in) } - in.TLS.DeepCopyInto(&out.TLS) return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashes. -func (in *DeploymentStatusHashes) DeepCopy() *DeploymentStatusHashes { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashesJWT. +func (in *DeploymentStatusHashesJWT) DeepCopy() *DeploymentStatusHashesJWT { if in == nil { return nil } - out := new(DeploymentStatusHashes) + out := new(DeploymentStatusHashesJWT) in.DeepCopyInto(out) return out } @@ -478,7 +498,7 @@ func (in *DeploymentStatusHashesTLS) DeepCopyInto(out *DeploymentStatusHashesTLS } if in.Truststore != nil { in, out := &in.Truststore, &out.Truststore - *out = make(DeploymentStatusHashList, len(*in)) + *out = make(sharedv1.HashList, len(*in)) copy(*out, *in) } return diff --git a/pkg/deployment/client/client.go b/pkg/deployment/client/client.go index e9ab78681..4389fd96e 100644 --- a/pkg/deployment/client/client.go +++ b/pkg/deployment/client/client.go @@ -41,6 +41,9 @@ type Client interface { GetEncryption(ctx context.Context) (EncryptionDetails, error) RefreshEncryption(ctx context.Context) (EncryptionDetails, error) + + GetJWT(ctx context.Context) (JWTDetails, error) + RefreshJWT(ctx context.Context) (JWTDetails, error) } type client struct { @@ -75,6 +78,20 @@ func (c *client) parseEncryptionResponse(response driver.Response) (EncryptionDe return d, nil } +func (c *client) parseJWTResponse(response driver.Response) (JWTDetails, error) { + if err := response.CheckStatus(http.StatusOK); err != nil { + return JWTDetails{}, err + } + + var d JWTDetails + + if err := response.ParseBody("", &d); err != nil { + return JWTDetails{}, err + } + + return d, nil +} + func (c *client) GetTLS(ctx context.Context) (TLSDetails, error) { r, err := c.c.NewRequest(http.MethodGet, "/_admin/server/tls") if err != nil { @@ -150,3 +167,41 @@ func (c *client) RefreshEncryption(ctx context.Context) (EncryptionDetails, erro return d, nil } + +func (c *client) GetJWT(ctx context.Context) (JWTDetails, error) { + r, err := c.c.NewRequest(http.MethodGet, "/_admin/server/jwt") + if err != nil { + return JWTDetails{}, err + } + + response, err := c.c.Do(ctx, r) + if err != nil { + return JWTDetails{}, err + } + + d, err := c.parseJWTResponse(response) + if err != nil { + return JWTDetails{}, err + } + + return d, nil +} + +func (c *client) RefreshJWT(ctx context.Context) (JWTDetails, error) { + r, err := c.c.NewRequest(http.MethodPost, "/_admin/server/jwt") + if err != nil { + return JWTDetails{}, err + } + + response, err := c.c.Do(ctx, r) + if err != nil { + return JWTDetails{}, err + } + + d, err := c.parseJWTResponse(response) + if err != nil { + return JWTDetails{}, err + } + + return d, nil +} diff --git a/pkg/deployment/client/encryption.go b/pkg/deployment/client/encryption.go index f2dc74ed8..1c98d7e40 100644 --- a/pkg/deployment/client/encryption.go +++ b/pkg/deployment/client/encryption.go @@ -22,33 +22,12 @@ package client -type EncryptionKeyEntry struct { - Sha string `json:"sha256,omitempty"` -} - type EncryptionDetailsResult struct { - Keys []EncryptionKeyEntry `json:"encryption-keys,omitempty"` + Keys Entries `json:"encryption-keys,omitempty"` } func (e EncryptionDetailsResult) KeysPresent(m map[string][]byte) bool { - if len(e.Keys) != len(m) { - return false - } - - for key := range m { - ok := false - for _, entry := range e.Keys { - if entry.Sha == key { - ok = true - break - } - } - if !ok { - return false - } - } - - return true + return e.Keys.KeysPresent(m) } type EncryptionDetails struct { diff --git a/pkg/apis/deployment/v1/key_hashes.go b/pkg/deployment/client/jwt.go similarity index 77% rename from pkg/apis/deployment/v1/key_hashes.go rename to pkg/deployment/client/jwt.go index 55df64289..b6a52628b 100644 --- a/pkg/apis/deployment/v1/key_hashes.go +++ b/pkg/deployment/client/jwt.go @@ -20,10 +20,13 @@ // Author Adam Janikowski // -package v1 +package client -import ( - shared "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1" -) +type JWTDetailsResult struct { + Active *Entry `json:"active,omitempty"` + Passive Entries `json:"passive,omitempty"` +} -type DeploymentStatusHashList shared.HashList +type JWTDetails struct { + Result JWTDetailsResult `json:"result,omitempty"` +} diff --git a/pkg/deployment/client/sha.go b/pkg/deployment/client/sha.go new file mode 100644 index 000000000..6b85b628c --- /dev/null +++ b/pkg/deployment/client/sha.go @@ -0,0 +1,101 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package client + +import ( + "strings" +) + +type Sha string + +func (s Sha) String() string { + return string(s) +} + +func (s Sha) Type() string { + z := strings.Split(s.String(), ":") + if len(z) < 2 { + return "sha256" + } + return z[0] +} + +func (s Sha) Checksum() string { + z := strings.Split(s.String(), ":") + if len(z) < 2 { + return z[0] + } + return z[1] +} + +type Entry struct { + Sha256 *Sha `json:"sha256,omitempty"` + Sha256Old *Sha `json:"SHA256,omitempty"` +} + +func (e *Entry) GetSHA() Sha { + if e == nil { + return "" + } + + if e.Sha256 != nil { + return *e.Sha256 + } + if e.Sha256Old != nil { + return *e.Sha256Old + } + + return "" +} + +type Entries []Entry + +func (e Entries) KeysPresent(m map[string][]byte) bool { + if len(e) != len(m) { + return false + } + + for key := range m { + ok := false + for _, entry := range e { + if entry.GetSHA().Checksum() == key { + ok = true + break + } + } + if !ok { + return false + } + } + + return true +} + +func (e Entries) Contains(s string) bool { + for _, entry := range e { + if entry.GetSHA().String() == s { + return true + } + } + return false +} diff --git a/pkg/deployment/client/tls.go b/pkg/deployment/client/tls.go index 69d909d75..4e1243f69 100644 --- a/pkg/deployment/client/tls.go +++ b/pkg/deployment/client/tls.go @@ -23,8 +23,9 @@ package client type TLSKeyFile struct { + *Entry `json:",inline"` + PrivateKeyHash string `json:"privateKeySHA256,omitempty"` - Checksum string `json:"SHA256,omitempty"` Certificates []string `json:"certificates,omitempty"` } diff --git a/pkg/deployment/client_cache.go b/pkg/deployment/client_cache.go index cbc43ae55..adecbabb0 100644 --- a/pkg/deployment/client_cache.go +++ b/pkg/deployment/client_cache.go @@ -25,38 +25,48 @@ package deployment import ( "context" "fmt" + "net" + "strconv" "sync" - "k8s.io/client-go/kubernetes" + "github.com/pkg/errors" + + "github.com/arangodb/go-driver/agency" + "github.com/arangodb/kube-arangodb/pkg/util/arangod/conn" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" - "github.com/arangodb/kube-arangodb/pkg/util/arangod" ) type clientCache struct { - mutex sync.Mutex - clients map[string]driver.Client - kubecli kubernetes.Interface - apiObject *api.ArangoDeployment + mutex sync.Mutex + clients map[string]driver.Client + apiObjectGetter func() *api.ArangoDeployment + databaseClient driver.Client + + factory conn.Factory } -// newClientCache creates a new client cache -func newClientCache(kubecli kubernetes.Interface, apiObject *api.ArangoDeployment) *clientCache { +func newClientCache(apiObjectGetter func() *api.ArangoDeployment, factory conn.Factory) *clientCache { return &clientCache{ - clients: make(map[string]driver.Client), - kubecli: kubecli, - apiObject: apiObject, + clients: make(map[string]driver.Client), + apiObjectGetter: apiObjectGetter, + factory: factory, } } -// Get a cached client for the given ID in the given group, creating one -// if needed. -func (cc *clientCache) Get(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) { - cc.mutex.Lock() - defer cc.mutex.Unlock() +func (cc *clientCache) extendHost(host string) string { + scheme := "http" + if cc.apiObjectGetter().Spec.TLS.IsSecure() { + scheme = "https" + } + + return scheme + "://" + net.JoinHostPort(host, strconv.Itoa(k8sutil.ArangoPort)) +} +func (cc *clientCache) getClient(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) { key := fmt.Sprintf("%d-%s", group, id) c, found := cc.clients[key] if found { @@ -64,7 +74,7 @@ func (cc *clientCache) Get(ctx context.Context, group api.ServerGroup, id string } // Not found, create a new client - c, err := arangod.CreateArangodClient(ctx, cc.kubecli.CoreV1(), cc.apiObject, group, id) + c, err := cc.factory.Client(cc.extendHost(k8sutil.CreatePodDNSName(cc.apiObjectGetter(), group.AsRole(), id))) if err != nil { return nil, maskAny(err) } @@ -72,22 +82,92 @@ func (cc *clientCache) Get(ctx context.Context, group api.ServerGroup, id string return c, nil } -// GetDatabase returns a cached client for the entire database (cluster coordinators or single server), -// creating one if needed. -func (cc *clientCache) GetDatabase(ctx context.Context) (driver.Client, error) { +func (cc *clientCache) get(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) { + client, err := cc.getClient(ctx, group, id) + if err != nil { + return nil, maskAny(err) + } + + if _, err := client.Version(ctx); err == nil { + return client, nil + } else if driver.IsUnauthorized(err) { + delete(cc.clients, fmt.Sprintf("%d-%s", group, id)) + return cc.getClient(ctx, group, id) + } else { + return client, nil + } +} + +// Get a cached client for the given ID in the given group, creating one +// if needed. +func (cc *clientCache) Get(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) { cc.mutex.Lock() defer cc.mutex.Unlock() + return cc.get(ctx, group, id) +} + +func (cc *clientCache) getDatabaseClient() (driver.Client, error) { if c := cc.databaseClient; c != nil { return c, nil } // Not found, create a new client - shortTimeout := false - c, err := arangod.CreateArangodDatabaseClient(ctx, cc.kubecli.CoreV1(), cc.apiObject, shortTimeout) + c, err := cc.factory.Client(cc.extendHost(k8sutil.CreateDatabaseClientServiceDNSName(cc.apiObjectGetter()))) if err != nil { return nil, maskAny(err) } cc.databaseClient = c return c, nil } + +func (cc *clientCache) getDatabase(ctx context.Context) (driver.Client, error) { + client, err := cc.getDatabaseClient() + if err != nil { + return nil, maskAny(err) + } + + if _, err := client.Version(ctx); err == nil { + return client, nil + } else if driver.IsUnauthorized(err) { + cc.databaseClient = nil + return cc.getDatabaseClient() + } else { + return client, nil + } +} + +// GetDatabase returns a cached client for the entire database (cluster coordinators or single server), +// creating one if needed. +func (cc *clientCache) GetDatabase(ctx context.Context) (driver.Client, error) { + cc.mutex.Lock() + defer cc.mutex.Unlock() + + return cc.getDatabase(ctx) +} + +func (cc *clientCache) getAgencyClient() (agency.Agency, error) { + // Not found, create a new client + var dnsNames []string + for _, m := range cc.apiObjectGetter().Status.Members.Agents { + dnsNames = append(dnsNames, cc.extendHost(k8sutil.CreatePodDNSName(cc.apiObjectGetter(), api.ServerGroupAgents.AsRole(), m.ID))) + } + + if len(dnsNames) == 0 { + return nil, errors.Errorf("There is no DNS Name") + } + + c, err := cc.factory.Agency(dnsNames...) + if err != nil { + return nil, maskAny(err) + } + return c, nil +} + +// GetDatabase returns a cached client for the agency +func (cc *clientCache) GetAgency(ctx context.Context) (agency.Agency, error) { + cc.mutex.Lock() + defer cc.mutex.Unlock() + + return cc.getAgencyClient() +} diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go index 2ca3202c6..46b30c6e1 100644 --- a/pkg/deployment/context_impl.go +++ b/pkg/deployment/context_impl.go @@ -24,9 +24,18 @@ package deployment import ( "context" + "crypto/tls" "fmt" "net" + nhttp "net/http" "strconv" + "time" + + "github.com/arangodb/go-driver/http" + "github.com/arangodb/go-driver/jwt" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + goErrors "github.com/pkg/errors" "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" @@ -37,20 +46,19 @@ import ( driver "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/agency" "github.com/rs/zerolog/log" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/resources" - "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" v1 "k8s.io/api/core/v1" ) // GetBackup receives information about a backup resource func (d *Deployment) GetBackup(backup string) (*backupApi.ArangoBackup, error) { - return d.deps.DatabaseCRCli.BackupV1().ArangoBackups(d.Namespace()).Get(backup, metav1.GetOptions{}) + return d.deps.DatabaseCRCli.BackupV1().ArangoBackups(d.Namespace()).Get(backup, meta.GetOptions{}) } // GetAPIObject returns the deployment as k8s object. @@ -198,11 +206,84 @@ func (d *Deployment) GetAgencyClients(ctx context.Context, predicate func(id str // GetAgency returns a connection to the entire agency. func (d *Deployment) GetAgency(ctx context.Context) (agency.Agency, error) { - result, err := arangod.CreateArangodAgencyClient(ctx, d.deps.KubeCli.CoreV1(), d.apiObject) + return d.clientCache.GetAgency(ctx) +} + +func (d *Deployment) getConnConfig() (http.ConnectionConfig, error) { + transport := &nhttp.Transport{ + Proxy: nhttp.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 100 * time.Millisecond, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 100 * time.Millisecond, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + + if d.apiObject.Spec.TLS.IsSecure() { + transport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + } + + connConfig := http.ConnectionConfig{ + Transport: transport, + DontFollowRedirect: true, + } + + return connConfig, nil +} + +func (d *Deployment) getAuth() (driver.Authentication, error) { + if !d.apiObject.Spec.Authentication.IsAuthenticated() { + return nil, nil + } + + secrets := d.GetKubeCli().CoreV1().Secrets(d.apiObject.GetNamespace()) + + var secret string + if i := d.apiObject.Status.CurrentImage; i == nil || i.ArangoDBVersion.CompareTo("3.7.0") < 0 || !i.Enterprise { + s, err := secrets.Get(d.apiObject.Spec.Authentication.GetJWTSecretName(), meta.GetOptions{}) + if err != nil { + return nil, goErrors.Errorf("JWT Secret is missing") + } + + jwt, ok := s.Data[constants.SecretKeyToken] + if !ok { + return nil, goErrors.Errorf("JWT Secret is invalid") + } + + secret = string(jwt) + } else { + s, err := secrets.Get(pod.JWTSecretFolder(d.apiObject.GetName()), meta.GetOptions{}) + if err != nil { + d.deps.Log.Error().Err(err).Msgf("Unable to get secret") + return nil, goErrors.Errorf("JWT Folder Secret is missing") + } + + if len(s.Data) == 0 { + return nil, goErrors.Errorf("JWT Folder Secret is empty") + } + + if q, ok := s.Data[pod.ActiveJWTKey]; ok { + secret = string(q) + } else { + for _, q := range s.Data { + secret = string(q) + break + } + } + } + + jwt, err := jwt.CreateArangodJwtAuthorizationHeader(secret, "kube-arangodb") if err != nil { return nil, maskAny(err) } - return result, nil + + return driver.RawAuthentication(jwt), nil } // GetSyncServerClient returns a cached client for a specific arangosync server. @@ -268,7 +349,7 @@ func (d *Deployment) CreateMember(group api.ServerGroup, id string) (string, err func (d *Deployment) DeletePod(podName string) error { log := d.deps.Log ns := d.apiObject.GetNamespace() - if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(podName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(podName, &meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pod", podName).Msg("Failed to remove pod") return maskAny(err) } @@ -281,8 +362,8 @@ func (d *Deployment) CleanupPod(p *v1.Pod) error { log := d.deps.Log podName := p.GetName() ns := p.GetNamespace() - options := metav1.NewDeleteOptions(0) - options.Preconditions = metav1.NewUIDPreconditions(string(p.GetUID())) + options := meta.NewDeleteOptions(0) + options.Preconditions = meta.NewUIDPreconditions(string(p.GetUID())) if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(podName, options); err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pod", podName).Msg("Failed to cleanup pod") return maskAny(err) @@ -296,7 +377,7 @@ func (d *Deployment) RemovePodFinalizers(podName string) error { log := d.deps.Log ns := d.GetNamespace() kubecli := d.deps.KubeCli - p, err := kubecli.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{}) + p, err := kubecli.CoreV1().Pods(ns).Get(podName, meta.GetOptions{}) if err != nil { if k8sutil.IsNotFound(err) { return nil @@ -314,7 +395,7 @@ func (d *Deployment) RemovePodFinalizers(podName string) error { func (d *Deployment) DeletePvc(pvcName string) error { log := d.deps.Log ns := d.apiObject.GetNamespace() - if err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(ns).Delete(pvcName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + if err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(ns).Delete(pvcName, &meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pvc", pvcName).Msg("Failed to remove pvc") return maskAny(err) } @@ -338,7 +419,7 @@ func (d *Deployment) UpdatePvc(pvc *v1.PersistentVolumeClaim) error { // GetPv returns PV info about PV with given name. func (d *Deployment) GetPv(pvName string) (*v1.PersistentVolume, error) { - pv, err := d.GetKubeCli().CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) + pv, err := d.GetKubeCli().CoreV1().PersistentVolumes().Get(pvName, meta.GetOptions{}) if err == nil { return pv, nil } @@ -366,7 +447,7 @@ func (d *Deployment) GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) { // GetPvc gets a PVC by the given name, in the samespace of the deployment. func (d *Deployment) GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) { - pvc, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.apiObject.GetNamespace()).Get(pvcName, metav1.GetOptions{}) + pvc, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.apiObject.GetNamespace()).Get(pvcName, meta.GetOptions{}) if err != nil { log.Debug().Err(err).Str("pvc-name", pvcName).Msg("Failed to get PVC") return nil, maskAny(err) @@ -392,7 +473,7 @@ func (d *Deployment) GetTLSKeyfile(group api.ServerGroup, member api.MemberStatu func (d *Deployment) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error { secretName := k8sutil.CreateTLSKeyfileSecretName(d.apiObject.GetName(), group.AsRole(), member.ID) ns := d.apiObject.GetNamespace() - if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(secretName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(secretName, &meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { return maskAny(err) } return nil @@ -402,7 +483,7 @@ func (d *Deployment) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberSt // If the secret does not exist, the error is ignored. func (d *Deployment) DeleteSecret(secretName string) error { ns := d.apiObject.GetNamespace() - if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(secretName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(secretName, &meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { return maskAny(err) } return nil diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index d5009bc04..1b4dba4d2 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -29,6 +29,8 @@ import ( "sync/atomic" "time" + "github.com/arangodb/kube-arangodb/pkg/util/arangod/conn" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" "github.com/arangodb/kube-arangodb/pkg/util/arangod" @@ -123,14 +125,17 @@ func New(config Config, deps Dependencies, apiObject *api.ArangoDeployment) (*De if err := apiObject.Spec.Validate(); err != nil { return nil, maskAny(err) } + d := &Deployment{ - apiObject: apiObject, - config: config, - deps: deps, - eventCh: make(chan *deploymentEvent, deploymentEventQueueSize), - stopCh: make(chan struct{}), - clientCache: newClientCache(deps.KubeCli, apiObject), + apiObject: apiObject, + config: config, + deps: deps, + eventCh: make(chan *deploymentEvent, deploymentEventQueueSize), + stopCh: make(chan struct{}), } + + d.clientCache = newClientCache(d.getArangoDeployment, conn.NewFactory(d.getAuth, d.getConnConfig)) + d.status.last = *(apiObject.Status.DeepCopy()) d.reconciler = reconcile.NewReconciler(deps.Log, d) d.resilience = resilience.NewResilience(deps.Log, d) @@ -491,3 +496,7 @@ func (d *Deployment) SetNumberOfServers(ctx context.Context, noCoordinators, noD } return nil } + +func (d *Deployment) getArangoDeployment() *api.ArangoDeployment { + return d.apiObject +} diff --git a/pkg/deployment/deployment_affinity_test.go b/pkg/deployment/deployment_affinity_test.go index 2ca5ff9e9..d59048d8b 100644 --- a/pkg/deployment/deployment_affinity_test.go +++ b/pkg/deployment/deployment_affinity_test.go @@ -98,7 +98,7 @@ func TestEnsurePod_ArangoDB_AntiAffinity(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -160,7 +160,7 @@ func TestEnsurePod_ArangoDB_AntiAffinity(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -225,7 +225,7 @@ func TestEnsurePod_ArangoDB_AntiAffinity(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -295,7 +295,7 @@ func TestEnsurePod_ArangoDB_AntiAffinity(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -374,7 +374,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -439,7 +439,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -507,7 +507,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -580,7 +580,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -661,7 +661,7 @@ func TestEnsurePod_ArangoDB_NodeAffinity(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, diff --git a/pkg/deployment/deployment_core_test.go b/pkg/deployment/deployment_core_test.go index 957a9a722..ae83a7d59 100644 --- a/pkg/deployment/deployment_core_test.go +++ b/pkg/deployment/deployment_core_test.go @@ -76,7 +76,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullAlways, SecurityContext: securityContext.NewSecurityContext(), }, @@ -127,7 +127,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullAlways, SecurityContext: securityContext.NewSecurityContext(), }, @@ -187,7 +187,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -244,7 +244,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -309,7 +309,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -364,7 +364,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -420,7 +420,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -477,7 +477,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -536,7 +536,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -588,7 +588,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -643,7 +643,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -696,7 +696,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.TlsKeyfileVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(true, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, true, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -735,7 +735,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"}) require.NoError(t, err) - testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(false, + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(cmd, false, authorization, k8sutil.ArangoPort) }, ExpectedEvent: "member agent is created", @@ -795,7 +795,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"}) require.NoError(t, err) - testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true, + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(cmd, true, authorization, k8sutil.ArangoPort) }, ExpectedEvent: "member agent is created", @@ -875,7 +875,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.RocksdbEncryptionVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -927,7 +927,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -981,7 +981,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -1043,7 +1043,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -1116,7 +1116,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { }, Resources: emptyResources, Lifecycle: createTestLifecycle(), - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -1188,7 +1188,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { }, Resources: emptyResources, Lifecycle: createTestLifecycle(), - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -1238,7 +1238,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"}) require.NoError(t, err) - testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true, + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(cmd, true, authorization, k8sutil.ArangoPort) }, config: Config{ @@ -1273,7 +1273,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { }, Ports: createTestPorts(), Lifecycle: createTestLifecycle(), - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), VolumeMounts: []core.VolumeMount{ @@ -1327,7 +1327,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { auth, err := createTestToken(deployment, testCase, []string{"/_admin/server/availability"}) require.NoError(t, err) - testCase.ExpectedPod.Spec.Containers[0].ReadinessProbe = createTestReadinessProbe(true, auth) + testCase.ExpectedPod.Spec.Containers[0].ReadinessProbe = createTestReadinessProbe(cmd, true, auth) }, ExpectedEvent: "member coordinator is created", ExpectedPod: core.Pod{ @@ -1391,9 +1391,9 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { authReadiness, err := createTestToken(deployment, testCase, []string{"/_admin/server/availability"}) require.NoError(t, err) - testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true, + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(cmd, true, authLiveness, 0) - testCase.ExpectedPod.Spec.Containers[0].ReadinessProbe = createTestReadinessProbe(true, authReadiness) + testCase.ExpectedPod.Spec.Containers[0].ReadinessProbe = createTestReadinessProbe(cmd, true, authReadiness) }, ExpectedEvent: "member single is created", ExpectedPod: core.Pod{ diff --git a/pkg/deployment/deployment_encryption_test.go b/pkg/deployment/deployment_encryption_test.go index 804916730..74fda1fe4 100644 --- a/pkg/deployment/deployment_encryption_test.go +++ b/pkg/deployment/deployment_encryption_test.go @@ -87,7 +87,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) { k8sutil.RocksdbEncryptionVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -136,7 +136,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) { authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"}) require.NoError(t, err) - testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true, + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(cmd, true, authorization, k8sutil.ArangoPort) }, config: Config{ @@ -171,7 +171,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) { }, Ports: createTestPorts(), Lifecycle: createTestLifecycle(), - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), VolumeMounts: []core.VolumeMount{ @@ -245,7 +245,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) { k8sutil.RocksdbEncryptionReadOnlyVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, diff --git a/pkg/deployment/deployment_image_test.go b/pkg/deployment/deployment_image_test.go index b7fb1199d..4f990b828 100644 --- a/pkg/deployment/deployment_image_test.go +++ b/pkg/deployment/deployment_image_test.go @@ -91,7 +91,7 @@ func TestEnsurePod_ArangoDB_ImagePropagation(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullAlways, SecurityContext: securityContext.NewSecurityContext(), }, @@ -143,7 +143,7 @@ func TestEnsurePod_ArangoDB_ImagePropagation(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullAlways, SecurityContext: securityContext.NewSecurityContext(), }, @@ -195,7 +195,7 @@ func TestEnsurePod_ArangoDB_ImagePropagation(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullAlways, SecurityContext: securityContext.NewSecurityContext(), }, diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 6aec21e01..651f361ad 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -174,9 +174,9 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } // Ensure we have image info - if retrySoon, err := d.ensureImages(d.apiObject); err != nil { + if retrySoon, exists, err := d.ensureImages(d.apiObject); err != nil { return minInspectionInterval, errors.Wrapf(err, "Image detection failed") - } else if retrySoon { + } else if retrySoon || !exists { return minInspectionInterval, nil } diff --git a/pkg/deployment/deployment_metrics_test.go b/pkg/deployment/deployment_metrics_test.go index 72ce29446..d87b8b72d 100644 --- a/pkg/deployment/deployment_metrics_test.go +++ b/pkg/deployment/deployment_metrics_test.go @@ -81,7 +81,7 @@ func TestEnsurePod_Metrics(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -142,7 +142,7 @@ func TestEnsurePod_Metrics(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -212,7 +212,7 @@ func TestEnsurePod_Metrics(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -281,7 +281,7 @@ func TestEnsurePod_Metrics(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -343,7 +343,7 @@ func TestEnsurePod_Metrics(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -410,7 +410,7 @@ func TestEnsurePod_Metrics(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, diff --git a/pkg/deployment/deployment_pod_probe_test.go b/pkg/deployment/deployment_pod_probe_test.go index 93efc1782..a2e3da41d 100644 --- a/pkg/deployment/deployment_pod_probe_test.go +++ b/pkg/deployment/deployment_pod_probe_test.go @@ -69,7 +69,7 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -126,7 +126,7 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: modTestLivenessProbe(false, "", k8sutil.ArangoPort, func(probe *core.Probe) { + LivenessProbe: modTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort, func(probe *core.Probe) { probe.TimeoutSeconds = 50 }), ImagePullPolicy: core.PullIfNotPresent, @@ -184,8 +184,8 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), - ReadinessProbe: createTestReadinessSimpleProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), + ReadinessProbe: createTestReadinessSimpleProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -235,7 +235,7 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -291,8 +291,8 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), - ReadinessProbe: createTestReadinessSimpleProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), + ReadinessProbe: createTestReadinessSimpleProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -342,7 +342,7 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - ReadinessProbe: createTestReadinessProbe(false, ""), + ReadinessProbe: createTestReadinessProbe(cmd, false, ""), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -398,8 +398,8 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) { k8sutil.ArangodVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), - ReadinessProbe: createTestReadinessProbe(false, ""), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), + ReadinessProbe: createTestReadinessProbe(cmd, false, ""), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, diff --git a/pkg/deployment/deployment_pod_resources_test.go b/pkg/deployment/deployment_pod_resources_test.go index 3059a1160..7da45def3 100644 --- a/pkg/deployment/deployment_pod_resources_test.go +++ b/pkg/deployment/deployment_pod_resources_test.go @@ -88,7 +88,7 @@ func TestEnsurePod_ArangoDB_Resources(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -148,7 +148,7 @@ func TestEnsurePod_ArangoDB_Resources(t *testing.T) { Env: []core.EnvVar{ resourceLimitAsEnv(t, resourcesUnfiltered), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -204,7 +204,7 @@ func TestEnsurePod_ArangoDB_Resources(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, diff --git a/pkg/deployment/deployment_pod_sync_test.go b/pkg/deployment/deployment_pod_sync_test.go index b2fb01e6d..04f0678d5 100644 --- a/pkg/deployment/deployment_pod_sync_test.go +++ b/pkg/deployment/deployment_pod_sync_test.go @@ -228,7 +228,7 @@ func TestEnsurePod_Sync_Master(t *testing.T) { require.NoError(t, err) testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( - true, "bearer "+auth, k8sutil.ArangoSyncMasterPort) + "", true, "bearer "+auth, k8sutil.ArangoSyncMasterPort) }, ExpectedEvent: "member syncmaster is created", ExpectedPod: core.Pod{ @@ -307,7 +307,7 @@ func TestEnsurePod_Sync_Master(t *testing.T) { require.NoError(t, err) testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( - true, "bearer "+auth, k8sutil.ArangoSyncMasterPort) + "", true, "bearer "+auth, k8sutil.ArangoSyncMasterPort) }, ExpectedEvent: "member syncmaster is created", ExpectedPod: core.Pod{ @@ -409,7 +409,7 @@ func TestEnsurePod_Sync_Worker(t *testing.T) { require.NoError(t, err) testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( - true, "bearer "+auth, k8sutil.ArangoSyncWorkerPort) + "", true, "bearer "+auth, k8sutil.ArangoSyncWorkerPort) }, ExpectedEvent: "member syncworker is created", ExpectedPod: core.Pod{ diff --git a/pkg/deployment/deployment_pod_tls_sni_test.go b/pkg/deployment/deployment_pod_tls_sni_test.go index b7a61935a..8d789689f 100644 --- a/pkg/deployment/deployment_pod_tls_sni_test.go +++ b/pkg/deployment/deployment_pod_tls_sni_test.go @@ -116,7 +116,7 @@ func TestEnsurePod_ArangoDB_TLS_SNI(t *testing.T) { k8sutil.TlsKeyfileVolumeMount(), }, Resources: emptyResources, - ReadinessProbe: createTestReadinessProbe(true, ""), + ReadinessProbe: createTestReadinessProbe(cmd, true, ""), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -188,7 +188,7 @@ func TestEnsurePod_ArangoDB_TLS_SNI(t *testing.T) { k8sutil.TlsKeyfileVolumeMount(), }, Resources: emptyResources, - ReadinessProbe: createTestReadinessProbe(true, ""), + ReadinessProbe: createTestReadinessProbe(cmd, true, ""), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -260,7 +260,7 @@ func TestEnsurePod_ArangoDB_TLS_SNI(t *testing.T) { k8sutil.TlsKeyfileVolumeMount(), }, Resources: emptyResources, - ReadinessProbe: createTestReadinessProbe(true, ""), + ReadinessProbe: createTestReadinessProbe(cmd, true, ""), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -365,7 +365,7 @@ func TestEnsurePod_ArangoDB_TLS_SNI(t *testing.T) { }, }, Resources: emptyResources, - ReadinessProbe: createTestReadinessProbe(true, ""), + ReadinessProbe: createTestReadinessProbe(cmd, true, ""), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -440,7 +440,7 @@ func TestEnsurePod_ArangoDB_TLS_SNI(t *testing.T) { k8sutil.TlsKeyfileVolumeMount(), }, Resources: emptyResources, - LivenessProbe: createTestLivenessProbe(true, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, true, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, diff --git a/pkg/deployment/deployment_pod_volumes_test.go b/pkg/deployment/deployment_pod_volumes_test.go index adfd7a69b..7a1d030d8 100644 --- a/pkg/deployment/deployment_pod_volumes_test.go +++ b/pkg/deployment/deployment_pod_volumes_test.go @@ -96,7 +96,7 @@ func TestEnsurePod_ArangoDB_Volumes(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -157,7 +157,7 @@ func TestEnsurePod_ArangoDB_Volumes(t *testing.T) { VolumeMounts: []core.VolumeMount{ k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, @@ -221,7 +221,7 @@ func TestEnsurePod_ArangoDB_Volumes(t *testing.T) { k8sutil.ArangodVolumeMount(), createExampleVolumeMount("volume").VolumeMount(), }, - LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort), ImagePullPolicy: core.PullIfNotPresent, SecurityContext: securityContext.NewSecurityContext(), }, diff --git a/pkg/deployment/deployment_suite_test.go b/pkg/deployment/deployment_suite_test.go index 9111c56dd..579a7208f 100644 --- a/pkg/deployment/deployment_suite_test.go +++ b/pkg/deployment/deployment_suite_test.go @@ -28,6 +28,10 @@ import ( "os" "testing" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes" + + "github.com/arangodb/kube-arangodb/pkg/util/arangod/conn" + "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/jwt" @@ -96,16 +100,16 @@ func createTestToken(deployment *Deployment, testCase *testCaseStruct, paths []s return jwt.CreateArangodJwtAuthorizationHeaderAllowedPaths(s, "kube-arangodb", paths) } -func modTestLivenessProbe(secure bool, authorization string, port int, mod func(*core.Probe)) *core.Probe { - probe := createTestLivenessProbe(secure, authorization, port) +func modTestLivenessProbe(mode string, secure bool, authorization string, port int, mod func(*core.Probe)) *core.Probe { + probe := createTestLivenessProbe(mode, secure, authorization, port) mod(probe) return probe } -func createTestReadinessSimpleProbe(secure bool, authorization string, port int) *core.Probe { - probe := createTestLivenessProbe(secure, authorization, port) +func createTestReadinessSimpleProbe(mode string, secure bool, authorization string, port int) *core.Probe { + probe := createTestReadinessProbe(mode, secure, authorization) probe.InitialDelaySeconds = 15 probe.PeriodSeconds = 10 @@ -113,23 +117,74 @@ func createTestReadinessSimpleProbe(secure bool, authorization string, port int) return probe } -func createTestLivenessProbe(secure bool, authorization string, port int) *core.Probe { - return k8sutil.HTTPProbeConfig{ - LocalPath: "/_api/version", +func createTestLivenessProbe(mode string, secure bool, authorization string, port int) *core.Probe { + return getProbeCreator(mode)(secure, authorization, "/_api/version", port).Create() +} + +func createTestReadinessProbe(mode string, secure bool, authorization string) *core.Probe { + p := getProbeCreator(mode)(secure, authorization, "/_admin/server/availability", k8sutil.ArangoPort).Create() + + p.InitialDelaySeconds = 2 + p.PeriodSeconds = 2 + + return p +} + +type probeCreator func(secure bool, authorization, endpoint string, port int) resources.Probe + +const ( + cmd = "cmd" +) + +func getProbeCreator(t string) probeCreator { + switch t { + case cmd: + return getCMDProbeCreator() + default: + return getHTTPProbeCreator() + } +} + +func getHTTPProbeCreator() probeCreator { + return func(secure bool, authorization, endpoint string, port int) resources.Probe { + return createHTTPTestProbe(secure, authorization, endpoint, port) + } +} + +func getCMDProbeCreator() probeCreator { + return func(secure bool, authorization, endpoint string, port int) resources.Probe { + return createCMDTestProbe(secure, authorization != "", endpoint) + } +} + +func createCMDTestProbe(secure, authorization bool, endpoint string) resources.Probe { + args := []string{ + "/lifecycle/tools/___go_test_github_com_arangodb_kube_arangodb_pkg_deployment", + "lifecycle", + "probe", + fmt.Sprintf("--endpoint=%s", endpoint), + } + + if secure { + args = append(args, "--ssl") + } + + if authorization { + args = append(args, "--auth") + } + + return &probes.CMDProbeConfig{ + Command: args, + } +} + +func createHTTPTestProbe(secure bool, authorization string, endpoint string, port int) resources.Probe { + return &probes.HTTPProbeConfig{ + LocalPath: endpoint, Secure: secure, Authorization: authorization, Port: port, - }.Create() -} - -func createTestReadinessProbe(secure bool, authorization string) *core.Probe { - return k8sutil.HTTPProbeConfig{ - LocalPath: "/_admin/server/availability", - Secure: secure, - Authorization: authorization, - InitialDelaySeconds: 2, - PeriodSeconds: 2, - }.Create() + } } func createTestCommandForDBServer(name string, tls, auth, encryptionRocksDB bool) []string { @@ -370,13 +425,13 @@ func createTestDeployment(config Config, arangoDeployment *api.ArangoDeployment) } d := &Deployment{ - apiObject: arangoDeployment, - config: config, - deps: deps, - eventCh: make(chan *deploymentEvent, deploymentEventQueueSize), - stopCh: make(chan struct{}), - clientCache: newClientCache(deps.KubeCli, arangoDeployment), + apiObject: arangoDeployment, + config: config, + deps: deps, + eventCh: make(chan *deploymentEvent, deploymentEventQueueSize), + stopCh: make(chan struct{}), } + d.clientCache = newClientCache(d.getArangoDeployment, conn.NewFactory(d.getAuth, d.getConnConfig)) arangoDeployment.Spec.SetDefaults(arangoDeployment.GetName()) d.resources = resources.NewResources(deps.Log, d) @@ -444,7 +499,7 @@ func createTestExporterCommand(secure bool, port uint16) []string { } func createTestExporterLivenessProbe(secure bool) *core.Probe { - return k8sutil.HTTPProbeConfig{ + return probes.HTTPProbeConfig{ LocalPath: "/", Port: k8sutil.ArangoExporterPort, Secure: secure, diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index ca066c150..11ddd5326 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -71,7 +71,7 @@ type imagesBuilder struct { // ensureImages creates pods needed to detect ImageID for specified images. // Returns: retrySoon, error -func (d *Deployment) ensureImages(apiObject *api.ArangoDeployment) (bool, error) { +func (d *Deployment) ensureImages(apiObject *api.ArangoDeployment) (bool, bool, error) { status, lastVersion := d.GetStatus() ib := imagesBuilder{ APIObject: apiObject, @@ -87,29 +87,28 @@ func (d *Deployment) ensureImages(apiObject *api.ArangoDeployment) (bool, error) }, } ctx := context.Background() - retrySoon, err := ib.Run(ctx) + retrySoon, exists, err := ib.Run(ctx) if err != nil { - return retrySoon, maskAny(err) + return retrySoon, exists, maskAny(err) } - return retrySoon, nil + return retrySoon, exists, nil } // Run creates pods needed to detect ImageID for specified images and puts the found // image ID's into the status.Images list. // Returns: retrySoon, error -func (ib *imagesBuilder) Run(ctx context.Context) (bool, error) { - result := false +func (ib *imagesBuilder) Run(ctx context.Context) (bool, bool, error) { // Check ArangoDB image if _, found := ib.Status.Images.GetByImage(ib.Spec.GetImage()); !found { // We need to find the image ID for the ArangoDB image retrySoon, err := ib.fetchArangoDBImageIDAndVersion(ctx, ib.Spec.GetImage()) if err != nil { - return retrySoon, maskAny(err) + return retrySoon, false, maskAny(err) } - result = result || retrySoon + return retrySoon, false, nil } - return result, nil + return false, true, nil } // fetchArangoDBImageIDAndVersion checks a running pod for fetching the ID of the given image. diff --git a/pkg/deployment/images_test.go b/pkg/deployment/images_test.go index 0738a431b..5ab385294 100644 --- a/pkg/deployment/images_test.go +++ b/pkg/deployment/images_test.go @@ -330,7 +330,7 @@ func TestEnsureImages(t *testing.T) { require.NoError(t, err) // Act - retrySoon, err := d.ensureImages(d.apiObject) + retrySoon, _, err := d.ensureImages(d.apiObject) // Assert assert.EqualValues(t, testCase.RetrySoon, retrySoon) diff --git a/pkg/deployment/pod/encryption.go b/pkg/deployment/pod/encryption.go index 1c1f036b4..ae767953f 100644 --- a/pkg/deployment/pod/encryption.go +++ b/pkg/deployment/pod/encryption.go @@ -95,7 +95,7 @@ func GetEncryptionKeyFromSecret(keyfile *core.Secret) (string, []byte, error) { return sha, d, nil } -func GetKeyfolderSecretName(name string) string { +func GetEncryptionFolderSecretName(name string) string { n := fmt.Sprintf("%s-encryption-folder", name) return n @@ -145,7 +145,7 @@ func (e encryption) Volumes(i Input) ([]core.Volume, []core.VolumeMount) { vol := k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, i.Deployment.RocksDB.Encryption.GetKeySecretName()) return []core.Volume{vol}, []core.VolumeMount{k8sutil.RocksdbEncryptionVolumeMount()} } else { - vol := k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, GetKeyfolderSecretName(i.ApiObject.GetName())) + vol := k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, GetEncryptionFolderSecretName(i.ApiObject.GetName())) return []core.Volume{vol}, []core.VolumeMount{k8sutil.RocksdbEncryptionReadOnlyVolumeMount()} } } @@ -155,17 +155,26 @@ func (e encryption) Verify(i Input, cachedStatus inspector.Inspector) error { return nil } - secret, exists := cachedStatus.Secret(i.Deployment.RocksDB.Encryption.GetKeySecretName()) - if !exists { - return errors.Errorf("Encryption key secret does not exist %s", i.Deployment.RocksDB.Encryption.GetKeySecretName()) + if !GroupEncryptionSupported(i.Deployment.GetMode(), i.Group) { + return nil } if !MultiFileMode(i) { + secret, exists := cachedStatus.Secret(i.Deployment.RocksDB.Encryption.GetKeySecretName()) + if !exists { + return errors.Errorf("Encryption key secret does not exist %s", i.Deployment.RocksDB.Encryption.GetKeySecretName()) + } + if err := k8sutil.ValidateEncryptionKeyFromSecret(secret); err != nil { return errors.Wrapf(err, "RocksDB encryption key secret validation failed") } return nil } else { - return nil + _, exists := cachedStatus.Secret(GetEncryptionFolderSecretName(i.ApiObject.GetName())) + if !exists { + return errors.Errorf("Encryption key folder secret does not exist %s", i.Deployment.RocksDB.Encryption.GetKeySecretName()) + } } + + return nil } diff --git a/pkg/deployment/pod/jwt.go b/pkg/deployment/pod/jwt.go index a7fb7180c..e15adaaac 100644 --- a/pkg/deployment/pod/jwt.go +++ b/pkg/deployment/pod/jwt.go @@ -23,6 +23,7 @@ package pod import ( + "fmt" "path/filepath" "github.com/arangodb/go-driver" @@ -33,6 +34,8 @@ import ( core "k8s.io/api/core/v1" ) +const ActiveJWTKey = "-" + func IsAuthenticated(i Input) bool { return i.Deployment.IsAuthenticated() } @@ -48,8 +51,12 @@ func VersionHasJWTSecretKeyfile(v driver.Version) bool { return false } -func VersionHasJWTSecretKeyfolder(i Input) bool { - return i.Enterprise && i.Version.CompareTo("3.7.0") > 0 +func JWTSecretFolder(name string) string { + return fmt.Sprintf("%s-jwt-folder", name) +} + +func VersionHasJWTSecretKeyfolder(v driver.Version, enterprise bool) bool { + return enterprise && v.CompareTo("3.7.0") > 0 } func JWT() Builder { @@ -81,7 +88,9 @@ func (e jwt) Args(i Input) k8sutil.OptionPairs { options.Add("--server.authentication", "true") - if VersionHasJWTSecretKeyfile(i.Version) { + if VersionHasJWTSecretKeyfolder(i.Version, i.Enterprise) { + options.Add("--server.jwt-secret-folder", k8sutil.ClusterJWTSecretVolumeMountDir) + } else if VersionHasJWTSecretKeyfile(i.Version) { keyPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyToken) options.Add("--server.jwt-secret-keyfile", keyPath) } else { @@ -96,7 +105,12 @@ func (e jwt) Volumes(i Input) ([]core.Volume, []core.VolumeMount) { return nil, nil } - vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, i.Deployment.Authentication.GetJWTSecretName()) + var vol core.Volume + if VersionHasJWTSecretKeyfolder(i.Version, i.Enterprise) { + vol = k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, JWTSecretFolder(i.ApiObject.GetName())) + } else { + vol = k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, i.Deployment.Authentication.GetJWTSecretName()) + } return []core.Volume{vol}, []core.VolumeMount{k8sutil.ClusterJWTVolumeMount()} } @@ -105,13 +119,20 @@ func (e jwt) Verify(i Input, cachedStatus inspector.Inspector) error { return nil } - secret, exists := cachedStatus.Secret(i.Deployment.Authentication.GetJWTSecretName()) - if !exists { - return errors.Errorf("Secret for JWT token is missing %s", i.Deployment.Authentication.GetJWTSecretName()) - } - - if err := k8sutil.ValidateTokenFromSecret(secret); err != nil { - return errors.Wrapf(err, "Cluster JWT secret validation failed") + if VersionHasJWTSecretKeyfolder(i.Version, i.Enterprise) { + _, exists := cachedStatus.Secret(JWTSecretFolder(i.ApiObject.GetName())) + if !exists { + return errors.Errorf("Secret for JWT Folderis missing %s", i.Deployment.Authentication.GetJWTSecretName()) + } + } else { + secret, exists := cachedStatus.Secret(i.Deployment.Authentication.GetJWTSecretName()) + if !exists { + return errors.Errorf("Secret for JWT token is missing %s", i.Deployment.Authentication.GetJWTSecretName()) + } + + if err := k8sutil.ValidateTokenFromSecret(secret); err != nil { + return errors.Wrapf(err, "Cluster JWT secret validation failed") + } } return nil diff --git a/pkg/deployment/reconcile/action_encryption_add.go b/pkg/deployment/reconcile/action_encryption_add.go index 4ef174a17..775f03e16 100644 --- a/pkg/deployment/reconcile/action_encryption_add.go +++ b/pkg/deployment/reconcile/action_encryption_add.go @@ -101,7 +101,7 @@ func (a *encryptionKeyAddAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(pod.GetKeyfolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch) + _, err = a.actionCtx.SecretsInterface().Patch(pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch) if err != nil { return false, err } diff --git a/pkg/deployment/reconcile/action_encryption_refresh.go b/pkg/deployment/reconcile/action_encryption_refresh.go index 5f7ab2ebb..30492a207 100644 --- a/pkg/deployment/reconcile/action_encryption_refresh.go +++ b/pkg/deployment/reconcile/action_encryption_refresh.go @@ -54,7 +54,7 @@ func (a *encryptionKeyRefreshAction) Start(ctx context.Context) (bool, error) { } func (a *encryptionKeyRefreshAction) CheckProgress(ctx context.Context) (bool, bool, error) { - keyfolder, err := a.actionCtx.SecretsInterface().Get(pod.GetKeyfolderSecretName(a.actionCtx.GetName()), meta.GetOptions{}) + keyfolder, err := a.actionCtx.SecretsInterface().Get(pod.GetEncryptionFolderSecretName(a.actionCtx.GetName()), meta.GetOptions{}) if err != nil { a.log.Err(err).Msgf("Unable to fetch encryption folder") return true, false, nil diff --git a/pkg/deployment/reconcile/action_encryption_remove.go b/pkg/deployment/reconcile/action_encryption_remove.go index c6b968115..f68493873 100644 --- a/pkg/deployment/reconcile/action_encryption_remove.go +++ b/pkg/deployment/reconcile/action_encryption_remove.go @@ -77,7 +77,7 @@ func (a *encryptionKeyRemoveAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(pod.GetKeyfolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch) + _, err = a.actionCtx.SecretsInterface().Patch(pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) diff --git a/pkg/deployment/reconcile/action_encryption_status_update.go b/pkg/deployment/reconcile/action_encryption_status_update.go index 4bcff13ef..73e8f748e 100644 --- a/pkg/deployment/reconcile/action_encryption_status_update.go +++ b/pkg/deployment/reconcile/action_encryption_status_update.go @@ -57,7 +57,7 @@ func (a *encryptionKeyStatusUpdateAction) Start(ctx context.Context) (bool, erro return true, nil } - f, err := a.actionCtx.SecretsInterface().Get(pod.GetKeyfolderSecretName(a.actionCtx.GetAPIObject().GetName()), meta.GetOptions{}) + f, err := a.actionCtx.SecretsInterface().Get(pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), meta.GetOptions{}) if err != nil { a.log.Error().Err(err).Msgf("Unable to get folder info") return true, nil @@ -67,16 +67,16 @@ func (a *encryptionKeyStatusUpdateAction) Start(ctx context.Context) (bool, erro if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { if len(keyHashes) == 0 { - if s.Hashes.Encryption != nil { - s.Hashes.Encryption = nil + if s.Hashes.Encryption.Keys != nil { + s.Hashes.Encryption.Keys = nil return true } return false } - if !util.CompareStringArray(keyHashes, s.Hashes.Encryption) { - s.Hashes.Encryption = keyHashes + if !util.CompareStringArray(keyHashes, s.Hashes.Encryption.Keys) { + s.Hashes.Encryption.Keys = keyHashes return true } return false diff --git a/pkg/deployment/reconcile/action_jwt_add.go b/pkg/deployment/reconcile/action_jwt_add.go new file mode 100644 index 000000000..3319daa8c --- /dev/null +++ b/pkg/deployment/reconcile/action_jwt_add.go @@ -0,0 +1,124 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + "encoding/base64" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/patch" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/errors" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/rs/zerolog" + "k8s.io/apimachinery/pkg/types" +) + +func init() { + registerAction(api.ActionTypeJWTAdd, newJWTAdd) +} + +func newJWTAdd(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &jwtAddAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout) + + return a +} + +type jwtAddAction struct { + actionImpl + + actionEmptyCheckProgress +} + +func (a *jwtAddAction) Start(ctx context.Context) (bool, error) { + folder, err := ensureJWTFolderSupportFromAction(a.actionCtx) + if err != nil { + a.log.Error().Err(err).Msgf("Action not supported") + return true, nil + } + + if !folder { + a.log.Error().Msgf("Action not supported") + return true, nil + } + + appendToken, exists := a.action.Params[checksum] + if !exists { + a.log.Warn().Msgf("Key %s is missing in action", checksum) + return true, nil + } + + s, ok := a.actionCtx.GetCachedStatus().Secret(a.actionCtx.GetSpec().Authentication.GetJWTSecretName()) + if !ok { + a.log.Error().Msgf("JWT Secret is missing, no rotation will take place") + return true, nil + } + + jwt, ok := s.Data[constants.SecretKeyToken] + if !ok { + a.log.Error().Msgf("JWT Secret is invalid, no rotation will take place") + return true, nil + } + + jwtSha := util.SHA256(jwt) + + if appendToken != jwtSha { + a.log.Error().Msgf("JWT Secret changed") + return true, nil + } + + f, ok := a.actionCtx.GetCachedStatus().Secret(pod.JWTSecretFolder(a.actionCtx.GetName())) + if !ok { + a.log.Error().Msgf("Unable to get JWT folder info") + return true, nil + } + + if _, ok := f.Data[jwtSha]; ok { + a.log.Info().Msgf("JWT Already exists") + return true, nil + } + + p := patch.NewPatch() + p.ItemAdd(patch.NewPath("data", jwtSha), base64.StdEncoding.EncodeToString(jwt)) + + patch, err := p.Marshal() + if err != nil { + a.log.Error().Err(err).Msgf("Unable to encrypt patch") + return true, nil + } + + _, err = a.actionCtx.SecretsInterface().Patch(pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch) + if err != nil { + if !k8sutil.IsInvalid(err) { + return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) + } + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_jwt_clean.go b/pkg/deployment/reconcile/action_jwt_clean.go new file mode 100644 index 000000000..fc77ca3f3 --- /dev/null +++ b/pkg/deployment/reconcile/action_jwt_clean.go @@ -0,0 +1,115 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/patch" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/errors" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/rs/zerolog" + "k8s.io/apimachinery/pkg/types" +) + +func init() { + registerAction(api.ActionTypeJWTClean, newJWTClean) +} + +func newJWTClean(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &jwtCleanAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout) + + return a +} + +type jwtCleanAction struct { + actionImpl + + actionEmptyCheckProgress +} + +func (a *jwtCleanAction) Start(ctx context.Context) (bool, error) { + folder, err := ensureJWTFolderSupportFromAction(a.actionCtx) + if err != nil { + a.log.Error().Err(err).Msgf("Action not supported") + return true, nil + } + + if !folder { + a.log.Error().Msgf("Action not supported") + return true, nil + } + + cleanToken, exists := a.action.Params[checksum] + if !exists { + a.log.Warn().Msgf("Key %s is missing in action", checksum) + return true, nil + } + + if cleanToken == pod.ActiveJWTKey { + a.log.Error().Msgf("Unable to remove active key") + return true, nil + } + + f, ok := a.actionCtx.GetCachedStatus().Secret(pod.JWTSecretFolder(a.actionCtx.GetName())) + if !ok { + a.log.Error().Msgf("Unable to get JWT folder info") + return true, nil + } + + if key, ok := f.Data[pod.ActiveJWTKey]; !ok { + a.log.Info().Msgf("Active Key is required") + return true, nil + } else if util.SHA256(key) == cleanToken { + a.log.Info().Msgf("Unable to remove active key") + return true, nil + } + + if _, ok := f.Data[cleanToken]; !ok { + a.log.Info().Msgf("KEy to be removed does not exist") + return true, nil + } + + p := patch.NewPatch() + p.ItemRemove(patch.NewPath("data", cleanToken)) + + patch, err := p.Marshal() + if err != nil { + a.log.Error().Err(err).Msgf("Unable to encrypt patch") + return true, nil + } + + _, err = a.actionCtx.SecretsInterface().Patch(pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch) + if err != nil { + if !k8sutil.IsInvalid(err) { + return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) + } + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_jwt_propagated.go b/pkg/deployment/reconcile/action_jwt_propagated.go new file mode 100644 index 000000000..ffc56aa34 --- /dev/null +++ b/pkg/deployment/reconcile/action_jwt_propagated.go @@ -0,0 +1,77 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func init() { + registerAction(api.ActionTypeJWTPropagated, newJWTPropagated) +} + +func newJWTPropagated(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &jwtPropagatedAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout) + + return a +} + +type jwtPropagatedAction struct { + actionImpl + + actionEmptyCheckProgress +} + +func (a *jwtPropagatedAction) Start(ctx context.Context) (bool, error) { + _, err := ensureJWTFolderSupportFromAction(a.actionCtx) + if err != nil { + a.log.Error().Err(err).Msgf("Action not supported") + return true, nil + } + + propagatedFlag, exists := a.action.Params[propagated] + if !exists { + a.log.Error().Err(err).Msgf("Propagated flag is missing") + return true, nil + } + + propagatedFlagBool := propagatedFlag == conditionTrue + + if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if s.Hashes.JWT.Propagated != propagatedFlagBool { + s.Hashes.JWT.Propagated = propagatedFlagBool + return true + } + + return false + }); err != nil { + return false, err + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_jwt_refresh.go b/pkg/deployment/reconcile/action_jwt_refresh.go new file mode 100644 index 000000000..d69533273 --- /dev/null +++ b/pkg/deployment/reconcile/action_jwt_refresh.go @@ -0,0 +1,78 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/client" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/rs/zerolog" +) + +func init() { + registerAction(api.ActionTypeJWTRefresh, newJWTRefresh) +} + +func newJWTRefresh(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &jwtRefreshAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout) + + return a +} + +type jwtRefreshAction struct { + actionImpl +} + +func (a *jwtRefreshAction) CheckProgress(ctx context.Context) (bool, bool, error) { + if folder, err := ensureJWTFolderSupport(a.actionCtx.GetSpec(), a.actionCtx.GetStatus()); err != nil || !folder { + return true, false, nil + } + + folder, ok := a.actionCtx.GetCachedStatus().Secret(pod.JWTSecretFolder(a.actionCtx.GetAPIObject().GetName())) + if !ok { + a.log.Error().Msgf("Unable to get JWT folder info") + return true, false, nil + } + + c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID) + if err != nil { + a.log.Warn().Err(err).Msg("Unable to get client") + return true, false, nil + } + if invalid, err := isMemberJWTTokenInvalid(ctx, client.NewClient(c.Connection()), folder.Data, true); err != nil { + a.log.Warn().Err(err).Msg("Error while getting JWT Status") + return true, false, nil + } else if invalid { + return false, false, nil + } + return true, false, nil +} + +func (a *jwtRefreshAction) Start(ctx context.Context) (bool, error) { + ready, _, err := a.CheckProgress(ctx) + return ready, err +} diff --git a/pkg/deployment/reconcile/action_jwt_set_active.go b/pkg/deployment/reconcile/action_jwt_set_active.go new file mode 100644 index 000000000..2737d95c4 --- /dev/null +++ b/pkg/deployment/reconcile/action_jwt_set_active.go @@ -0,0 +1,116 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + "encoding/base64" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/patch" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/errors" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/rs/zerolog" + "k8s.io/apimachinery/pkg/types" +) + +func init() { + registerAction(api.ActionTypeJWTSetActive, newJWTSetActive) +} + +func newJWTSetActive(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &jwtSetActiveAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout) + + return a +} + +type jwtSetActiveAction struct { + actionImpl + + actionEmptyCheckProgress +} + +func (a *jwtSetActiveAction) Start(ctx context.Context) (bool, error) { + folder, err := ensureJWTFolderSupportFromAction(a.actionCtx) + if err != nil { + a.log.Error().Err(err).Msgf("Action not supported") + return true, nil + } + + if !folder { + a.log.Error().Msgf("Action not supported") + return true, nil + } + + toActiveChecksum, exists := a.action.Params[checksum] + if !exists { + a.log.Warn().Msgf("Key %s is missing in action", checksum) + return true, nil + } + + f, ok := a.actionCtx.GetCachedStatus().Secret(pod.JWTSecretFolder(a.actionCtx.GetName())) + if !ok { + a.log.Error().Msgf("Unable to get JWT folder info") + return true, nil + } + + toActiveData, toActivePresent := f.Data[toActiveChecksum] + if !toActivePresent { + a.log.Error().Msgf("JWT key which is desired to be active is not anymore in secret") + return true, nil + } + + activeKeyData, active := f.Data[pod.ActiveJWTKey] + + if util.SHA256(activeKeyData) == toActiveChecksum { + a.log.Info().Msgf("Desired JWT is already active") + return true, nil + } + + p := patch.NewPatch() + path := patch.NewPath("data", pod.ActiveJWTKey) + if !active { + p.ItemAdd(path, base64.StdEncoding.EncodeToString(toActiveData)) + } else { + p.ItemReplace(path, base64.StdEncoding.EncodeToString(toActiveData)) + } + + patch, err := p.Marshal() + if err != nil { + a.log.Error().Err(err).Msgf("Unable to encrypt patch") + return true, nil + } + + _, err = a.actionCtx.SecretsInterface().Patch(pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch) + if err != nil { + if !k8sutil.IsInvalid(err) { + return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) + } + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_jwt_status_update.go b/pkg/deployment/reconcile/action_jwt_status_update.go new file mode 100644 index 000000000..6a45ab334 --- /dev/null +++ b/pkg/deployment/reconcile/action_jwt_status_update.go @@ -0,0 +1,183 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + "fmt" + "sort" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/pkg/errors" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/rs/zerolog" +) + +const ( + checksum = "checksum" + propagated = "propagated" + + conditionTrue = "True" + conditionFalse = "False" +) + +func ensureJWTFolderSupportFromAction(actionCtx ActionContext) (bool, error) { + return ensureJWTFolderSupport(actionCtx.GetSpec(), actionCtx.GetStatus()) +} + +func ensureJWTFolderSupport(spec api.DeploymentSpec, status api.DeploymentStatus) (bool, error) { + if !spec.IsAuthenticated() { + return false, errors.Errorf("Authentication is disabled") + } + + if image := status.CurrentImage; image == nil { + return false, errors.Errorf("Missing image info") + } else { + if !image.Enterprise { + return false, nil + } + if image.ArangoDBVersion.CompareTo("3.7.0") < 0 { + return false, nil + } + } + return true, nil +} + +func init() { + registerAction(api.ActionTypeJWTStatusUpdate, newJWTStatusUpdate) +} + +func newJWTStatusUpdate(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &jwtStatusUpdateAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout) + + return a +} + +type jwtStatusUpdateAction struct { + actionImpl + + actionEmptyCheckProgress +} + +func (a *jwtStatusUpdateAction) Start(ctx context.Context) (bool, error) { + folder, err := ensureJWTFolderSupportFromAction(a.actionCtx) + if err != nil { + a.log.Error().Err(err).Msgf("Action not supported") + return true, nil + } + + if !folder { + f, ok := a.actionCtx.GetCachedStatus().Secret(a.actionCtx.GetSpec().Authentication.GetJWTSecretName()) + if !ok { + a.log.Error().Msgf("Unable to get JWT secret info") + return true, nil + } + + key, ok := f.Data[constants.SecretKeyToken] + if !ok { + a.log.Error().Msgf("JWT Token is invalid") + return true, nil + } + + keySha := fmt.Sprintf("sha256:%s", util.SHA256(key)) + + if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if s.Hashes.JWT.Passive != nil { + s.Hashes.JWT.Passive = nil + return true + } + + if s.Hashes.JWT.Active != keySha { + s.Hashes.JWT.Active = keySha + return true + } + + return false + }); err != nil { + return false, err + } + + return true, nil + } + + f, ok := a.actionCtx.GetCachedStatus().Secret(pod.JWTSecretFolder(a.actionCtx.GetName())) + if !ok { + a.log.Error().Msgf("Unable to get JWT folder info") + return true, nil + } + + if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) (update bool) { + activeKeyData, active := f.Data[pod.ActiveJWTKey] + activeKeyShort := util.SHA256(activeKeyData) + activeKey := fmt.Sprintf("sha256:%s", activeKeyShort) + if active { + if s.Hashes.JWT.Active != activeKey { + s.Hashes.JWT.Active = activeKey + update = true + } + } + + if len(f.Data) == 0 { + if s.Hashes.JWT.Passive != nil { + s.Hashes.JWT.Passive = nil + update = true + } + } + + var keys []string + + for key := range f.Data { + if key == pod.ActiveJWTKey || key == activeKeyShort { + continue + } + + keys = append(keys, key) + } + + if len(keys) == 0 { + if s.Hashes.JWT.Passive != nil { + s.Hashes.JWT.Passive = nil + update = true + } + } + + sort.Strings(keys) + keys = util.PrefixStringArray(keys, "sha256:") + + if !util.CompareStringArray(keys, s.Hashes.JWT.Passive) { + s.Hashes.JWT.Passive = keys + update = true + } + + return + }); err != nil { + return false, err + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_tls_ca_append.go b/pkg/deployment/reconcile/action_tls_ca_append.go index 278450039..06a307aac 100644 --- a/pkg/deployment/reconcile/action_tls_ca_append.go +++ b/pkg/deployment/reconcile/action_tls_ca_append.go @@ -37,10 +37,6 @@ import ( "github.com/rs/zerolog" ) -const ( - actionTypeAppendTLSCACertificateChecksum = "checksum" -) - func init() { registerAction(api.ActionTypeAppendTLSCACertificate, newAppendTLSCACertificateAction) } @@ -64,9 +60,9 @@ func (a *appendTLSCACertificateAction) Start(ctx context.Context) (bool, error) return true, nil } - certChecksum, exists := a.action.Params[actionTypeAppendTLSCACertificateChecksum] + certChecksum, exists := a.action.Params[checksum] if !exists { - a.log.Warn().Msgf("Key %s is missing in action", actionTypeAppendTLSCACertificateChecksum) + a.log.Warn().Msgf("Key %s is missing in action", checksum) return true, nil } diff --git a/pkg/deployment/reconcile/action_tls_ca_clean.go b/pkg/deployment/reconcile/action_tls_ca_clean.go index 9ee385945..bc172aead 100644 --- a/pkg/deployment/reconcile/action_tls_ca_clean.go +++ b/pkg/deployment/reconcile/action_tls_ca_clean.go @@ -61,9 +61,9 @@ func (a *cleanTLSCACertificateAction) Start(ctx context.Context) (bool, error) { return true, nil } - certChecksum, exists := a.action.Params[actionTypeAppendTLSCACertificateChecksum] + certChecksum, exists := a.action.Params[checksum] if !exists { - a.log.Warn().Msgf("Key %s is missing in action", actionTypeAppendTLSCACertificateChecksum) + a.log.Warn().Msgf("Key %s is missing in action", checksum) return true, nil } diff --git a/pkg/deployment/reconcile/action_tls_keyfile_refresh.go b/pkg/deployment/reconcile/action_tls_keyfile_refresh.go index 003df5b53..a8c0f38af 100644 --- a/pkg/deployment/reconcile/action_tls_keyfile_refresh.go +++ b/pkg/deployment/reconcile/action_tls_keyfile_refresh.go @@ -80,7 +80,7 @@ func (a *refreshTLSKeyfileCertificateAction) CheckProgress(ctx context.Context) return true, false, nil } - if e.Result.KeyFile.Checksum == keyfileSha { + if e.Result.KeyFile.GetSHA().Checksum() == keyfileSha { return true, false, nil } diff --git a/pkg/deployment/reconcile/helper_tls_sni.go b/pkg/deployment/reconcile/helper_tls_sni.go index 117c60ab5..e0f4df45a 100644 --- a/pkg/deployment/reconcile/helper_tls_sni.go +++ b/pkg/deployment/reconcile/helper_tls_sni.go @@ -91,7 +91,7 @@ func compareTLSSNIConfig(ctx context.Context, c driver.Connection, m map[string] return false, errors.Errorf("Unable to fetch TLS SNI state") } - if value.Checksum != currentValue { + if value.GetSHA().Checksum() != currentValue { return false, nil } } diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index 63d97187a..de453a94d 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -90,7 +90,7 @@ func (d *Reconciler) CreatePlan(ctx context.Context, cachedStatus inspector.Insp func fetchAgency(ctx context.Context, log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, - context PlanBuilderContext) (*agency.ArangoPlanDatabases, error) { + cache inspector.Inspector, context PlanBuilderContext) (*agency.ArangoPlanDatabases, error) { if spec.GetMode() != api.DeploymentModeCluster && spec.GetMode() != api.DeploymentModeActiveFailover { return nil, nil } else if status.Members.Agents.MembersReady() > 0 { @@ -99,7 +99,7 @@ func fetchAgency(ctx context.Context, log zerolog.Logger, ret := &agency.ArangoPlanDatabases{} - if err := context.GetAgencyData(agencyCtx, ret, agency.ArangoKey, agency.PlanKey, agency.PlanCollectionsKey); err != nil { + if err := context.GetAgencyData(agencyCtx, cache, agency.ArangoKey, agency.PlanKey, agency.PlanCollectionsKey); err != nil { return nil, err } @@ -123,7 +123,7 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb } // Fetch agency plan - agencyPlan, agencyErr := fetchAgency(ctx, log, spec, status, builderCtx) + agencyPlan, agencyErr := fetchAgency(ctx, log, spec, status, cachedStatus, builderCtx) // Check for various scenario's var plan api.Plan @@ -210,6 +210,10 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb plan = pb.Apply(createTLSStatusUpdate) } + if plan.IsEmpty() { + plan = pb.Apply(createJWTStatusUpdate) + } + // Check for scale up/down if plan.IsEmpty() { plan = pb.Apply(createScaleMemeberPlan) @@ -220,11 +224,15 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb plan = pb.Apply(createRotateOrUpgradePlan) } - // Add encryption keys + // Add keys if plan.IsEmpty() { plan = pb.Apply(createEncryptionKey) } + if plan.IsEmpty() { + plan = pb.Apply(createJWTKeyUpdate) + } + if plan.IsEmpty() { plan = pb.Apply(createCARenewalPlan) } @@ -256,7 +264,7 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb } if plan.IsEmpty() { - plan = pb.Apply(cleanEncryptionKey) + plan = pb.Apply(createEncryptionKeyCleanPlan) } if plan.IsEmpty() { diff --git a/pkg/deployment/reconcile/plan_builder_encryption.go b/pkg/deployment/reconcile/plan_builder_encryption.go index a5ffb61ae..c41ee51fa 100644 --- a/pkg/deployment/reconcile/plan_builder_encryption.go +++ b/pkg/deployment/reconcile/plan_builder_encryption.go @@ -25,6 +25,8 @@ package reconcile import ( "context" + core "k8s.io/api/core/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" @@ -73,7 +75,7 @@ func createEncryptionKey(ctx context.Context, return nil } - keyfolder, exists := cachedStatus.Secret(pod.GetKeyfolderSecretName(context.GetName())) + keyfolder, exists := cachedStatus.Secret(pod.GetEncryptionFolderSecretName(context.GetName())) if !exists { log.Error().Msgf("Encryption key folder does not exist") return nil @@ -88,49 +90,7 @@ func createEncryptionKey(ctx context.Context, return api.Plan{api.NewAction(api.ActionTypeEncryptionKeyAdd, api.ServerGroupUnknown, "")} } - var plan api.Plan - status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { - if !pod.GroupEncryptionSupported(spec.Mode.Get(), group) { - return nil - } - - glog := log.With().Str("group", group.AsRole()) - - for _, m := range members { - if m.Phase != api.MemberPhaseCreated { - // Only make changes when phase is created - continue - } - - if m.ArangoVersion.CompareTo("3.7.0") < 0 { - continue - } - - mlog := glog.Str("member", m.ID).Logger() - - c, err := context.GetServerClient(ctx, group, m.ID) - if err != nil { - mlog.Warn().Err(err).Msg("Unable to get client") - continue - } - - client := client.NewClient(c.Connection()) - - e, err := client.GetEncryption(ctx) - if err != nil { - mlog.Error().Err(err).Msgf("Unable to fetch encryption keys") - continue - } - - if !e.Result.KeysPresent(keyfolder.Data) { - plan = append(plan, api.NewAction(api.ActionTypeEncryptionKeyRefresh, group, m.ID)) - mlog.Info().Msgf("Refresh of encryption keys required") - continue - } - } - - return nil - }) + plan, _ := areEncryptionKeysUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, keyfolder) if !plan.IsEmpty() { return plan @@ -163,7 +123,7 @@ func createEncryptionKeyStatusUpdateRequired(ctx context.Context, return false } - keyfolder, exists := cachedStatus.Secret(pod.GetKeyfolderSecretName(context.GetName())) + keyfolder, exists := cachedStatus.Secret(pod.GetEncryptionFolderSecretName(context.GetName())) if !exists { log.Error().Msgf("Encryption key folder does not exist") return false @@ -171,14 +131,14 @@ func createEncryptionKeyStatusUpdateRequired(ctx context.Context, keyHashes := secretKeysToListWithPrefix("sha256:", keyfolder) - if !util.CompareStringArray(keyHashes, status.Hashes.Encryption) { + if !util.CompareStringArray(keyHashes, status.Hashes.Encryption.Keys) { return true } return false } -func cleanEncryptionKey(ctx context.Context, +func createEncryptionKeyCleanPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { @@ -186,12 +146,24 @@ func cleanEncryptionKey(ctx context.Context, return nil } - keyfolder, exists := cachedStatus.Secret(pod.GetKeyfolderSecretName(context.GetName())) + keyfolder, exists := cachedStatus.Secret(pod.GetEncryptionFolderSecretName(context.GetName())) if !exists { log.Error().Msgf("Encryption key folder does not exist") return nil } + plan, failed := areEncryptionKeysUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, keyfolder) + + if failed { + log.Info().Msgf("Unable to continue with encryption until all servers are ready") + return nil + } + + if len(plan) != 0 { + log.Info().Msgf("Unable to continue with encryption until all servers report state or gonna be upToDate") + return nil + } + if len(keyfolder.Data) <= 1 { return nil } @@ -215,8 +187,6 @@ func cleanEncryptionKey(ctx context.Context, return nil } - var plan api.Plan - for key := range keyfolder.Data { if key != name { plan = append(plan, api.NewAction(api.ActionTypeEncryptionKeyRemove, api.ServerGroupUnknown, "").AddParam("key", key)) @@ -229,3 +199,68 @@ func cleanEncryptionKey(ctx context.Context, return api.Plan{} } + +func areEncryptionKeysUpToDate(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext, + folder *core.Secret) (plan api.Plan, failed bool) { + + status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { + if !pod.GroupEncryptionSupported(spec.Mode.Get(), group) { + return nil + } + + for _, m := range list { + if updateRequired, failedMember := isEncryptionKeyUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, group, m, folder); failedMember { + failed = true + continue + } else if updateRequired { + plan = append(plan, api.NewAction(api.ActionTypeEncryptionKeyRefresh, group, m.ID)) + continue + } + } + + return nil + }) + + return +} + +func isEncryptionKeyUpToDate(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext, + group api.ServerGroup, m api.MemberStatus, + folder *core.Secret) (updateRequired bool, failed bool) { + if m.Phase != api.MemberPhaseCreated { + return false, true + } + + if m.ArangoVersion.CompareTo("3.7.0") < 0 { + return false, false + } + + mlog := log.With().Str("group", group.AsRole()).Str("member", m.ID).Logger() + + c, err := context.GetServerClient(ctx, group, m.ID) + if err != nil { + mlog.Warn().Err(err).Msg("Unable to get client") + return false, true + } + + client := client.NewClient(c.Connection()) + + e, err := client.GetEncryption(ctx) + if err != nil { + mlog.Error().Err(err).Msgf("Unable to fetch encryption keys") + return false, true + } + + if !e.Result.KeysPresent(folder.Data) { + mlog.Info().Msgf("Refresh of encryption keys required") + return true, false + } + + return false, false +} diff --git a/pkg/deployment/reconcile/plan_builder_jwt.go b/pkg/deployment/reconcile/plan_builder_jwt.go new file mode 100644 index 000000000..66f23bc6a --- /dev/null +++ b/pkg/deployment/reconcile/plan_builder_jwt.go @@ -0,0 +1,339 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + "fmt" + "sort" + + "github.com/arangodb/kube-arangodb/pkg/deployment/client" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + core "k8s.io/api/core/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/rs/zerolog" +) + +func createJWTKeyUpdate(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { + if folder, err := ensureJWTFolderSupport(spec, status); err != nil || !folder { + return nil + } + + folder, ok := cachedStatus.Secret(pod.JWTSecretFolder(apiObject.GetName())) + if !ok { + log.Error().Msgf("Unable to get JWT folder info") + return nil + } + + s, ok := cachedStatus.Secret(spec.Authentication.GetJWTSecretName()) + if !ok { + log.Info().Msgf("JWT Secret is missing, no rotation will take place") + return nil + } + + jwt, ok := s.Data[constants.SecretKeyToken] + if !ok { + log.Warn().Msgf("JWT Secret is invalid, no rotation will take place") + return nil + } + + jwtSha := util.SHA256(jwt) + + f, ok := cachedStatus.Secret(pod.JWTSecretFolder(apiObject.GetName())) + if !ok { + log.Info().Msgf("JWT Folder Secret is missing, no rotation will take place") + return nil + } + + if _, ok := f.Data[jwtSha]; !ok { + return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTAdd, api.ServerGroupUnknown, "", "Add JWT key").AddParam(checksum, jwtSha)) + } + + activeKey, ok := f.Data[pod.ActiveJWTKey] + if !ok { + return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTSetActive, api.ServerGroupUnknown, "", "Set active key").AddParam(checksum, jwtSha)) + } + + plan, failed := areJWTTokensUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, folder) + if len(plan) > 0 { + return plan + } + + if failed { + log.Info().Msgf("JWT Failed on one pod, no rotation will take place") + return nil + } + + if util.SHA256(activeKey) != jwtSha { + return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTSetActive, api.ServerGroupUnknown, "", "Set active key").AddParam(checksum, jwtSha)) + } + + for key := range f.Data { + if key == pod.ActiveJWTKey { + continue + } + + if key == jwtSha { + continue + } + + return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTClean, api.ServerGroupUnknown, "", "Remove old key").AddParam(checksum, key)) + } + + return addJWTPropagatedPlanAction(status) +} + +func createJWTStatusUpdate(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { + if _, err := ensureJWTFolderSupport(spec, status); err != nil { + return nil + } + + if createJWTStatusUpdateRequired(ctx, log, apiObject, spec, status, cachedStatus, context) { + return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTStatusUpdate, api.ServerGroupUnknown, "", "Update status")) + } + + return nil +} + +func createJWTStatusUpdateRequired(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) bool { + folder, err := ensureJWTFolderSupport(spec, status) + if err != nil { + log.Error().Err(err).Msgf("Action not supported") + return false + } + + if !folder { + if status.Hashes.JWT.Passive != nil { + return true + } + + f, ok := cachedStatus.Secret(spec.Authentication.GetJWTSecretName()) + if !ok { + log.Error().Msgf("Unable to get JWT secret info") + return false + } + + key, ok := f.Data[constants.SecretKeyToken] + if !ok { + log.Error().Msgf("JWT Token is invalid") + return false + } + + keySha := fmt.Sprintf("sha256:%s", util.SHA256(key)) + + if status.Hashes.JWT.Active != keySha { + log.Error().Msgf("JWT Token is invalid") + return true + } + + return false + } + + f, ok := cachedStatus.Secret(pod.JWTSecretFolder(apiObject.GetName())) + if !ok { + log.Error().Msgf("Unable to get JWT folder info") + return false + } + + activeKeyData, active := f.Data[pod.ActiveJWTKey] + activeKeyShort := util.SHA256(activeKeyData) + activeKey := fmt.Sprintf("sha256:%s", activeKeyShort) + if active { + if status.Hashes.JWT.Active != activeKey { + return true + } + } + + if len(f.Data) == 0 { + if status.Hashes.JWT.Passive != nil { + return true + } + return false + } + + var keys []string + + for key := range f.Data { + if key == pod.ActiveJWTKey || key == activeKeyShort { + continue + } + + keys = append(keys, key) + } + + if len(keys) == 0 { + if status.Hashes.JWT.Passive != nil { + return true + } + return false + } + + sort.Strings(keys) + keys = util.PrefixStringArray(keys, "sha256:") + + if !util.CompareStringArray(keys, status.Hashes.JWT.Passive) { + return true + } + + return false +} + +func areJWTTokensUpToDate(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext, + folder *core.Secret) (plan api.Plan, failed bool) { + + status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { + for _, m := range list { + if updateRequired, failedMember := isJWTTokenUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, group, m, folder); failedMember { + failed = true + continue + } else if updateRequired { + plan = append(plan, api.NewAction(api.ActionTypeJWTRefresh, group, m.ID)) + continue + } + } + + return nil + }) + + return +} + +func isJWTTokenUpToDate(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext, + group api.ServerGroup, m api.MemberStatus, + folder *core.Secret) (updateRequired bool, failed bool) { + if m.Phase != api.MemberPhaseCreated { + return false, true + } + + if m.ArangoVersion.CompareTo("3.7.0") < 0 { + return false, false + } + + mlog := log.With().Str("group", group.AsRole()).Str("member", m.ID).Logger() + + c, err := context.GetServerClient(ctx, group, m.ID) + if err != nil { + mlog.Warn().Err(err).Msg("Unable to get client") + return false, true + } + + if updateRequired, err := isMemberJWTTokenInvalid(ctx, client.NewClient(c.Connection()), folder.Data, false); err != nil { + mlog.Warn().Err(err).Msg("JET UpToDate Check failed") + return false, true + } else if updateRequired { + return true, false + } + + return false, false +} + +func addJWTPropagatedPlanAction(s api.DeploymentStatus, actions ...api.Action) api.Plan { + got := len(actions) != 0 + cond := conditionFalse + if !got { + cond = conditionTrue + } + + if s.Hashes.JWT.Propagated == got { + p := api.Plan{api.NewAction(api.ActionTypeJWTPropagated, api.ServerGroupUnknown, "", "Change propagated flag").AddParam(propagated, cond)} + return append(p, actions...) + } + + return actions +} + +func isMemberJWTTokenInvalid(ctx context.Context, c client.Client, data map[string][]byte, refresh bool) (bool, error) { + cmd := c.GetJWT + if refresh { + cmd = c.RefreshJWT + } + + e, err := cmd(ctx) + if err != nil { + return false, errors.Wrapf(err, "Unable to fetch JWT tokens") + } + + if e.Result.Active == nil { + return false, errors.Wrapf(err, "There is no active JWT Token") + } + + if jwtActive, ok := data[pod.ActiveJWTKey]; !ok { + return false, errors.Errorf("Missing Active JWT Token in folder") + } else if util.SHA256(jwtActive) != e.Result.Active.GetSHA().Checksum() { + log.Info().Str("active", e.Result.Active.GetSHA().Checksum()).Str("expected", util.SHA256(jwtActive)).Msgf("Active key is invalid") + return true, nil + } + + if !compareJWTKeys(e.Result.Passive, data) { + return true, nil + } + + return false, nil +} + +func compareJWTKeys(e client.Entries, keys map[string][]byte) bool { + for k := range keys { + if k == pod.ActiveJWTKey { + continue + } + + if !e.Contains(k) { + log.Info().Msgf("Missing JWT Key") + return false + } + } + + for _, entry := range e { + if entry.GetSHA() == "" { + continue + } + + if _, ok := keys[entry.GetSHA().Checksum()]; !ok { + return false + } + } + + return true +} diff --git a/pkg/deployment/reconcile/plan_builder_restore.go b/pkg/deployment/reconcile/plan_builder_restore.go index f13ffb325..4491d4370 100644 --- a/pkg/deployment/reconcile/plan_builder_restore.go +++ b/pkg/deployment/reconcile/plan_builder_restore.go @@ -83,7 +83,7 @@ func createRestorePlanEncryption(ctx context.Context, log zerolog.Logger, spec a secret := *spec.RestoreEncryptionSecret // Additional logic to do restore with encryption key - keyfolder, err := builderCtx.SecretsInterface().Get(pod.GetKeyfolderSecretName(builderCtx.GetName()), meta.GetOptions{}) + keyfolder, err := builderCtx.SecretsInterface().Get(pod.GetEncryptionFolderSecretName(builderCtx.GetName()), meta.GetOptions{}) if err != nil { log.Err(err).Msgf("Unable to fetch encryption folder") return nil diff --git a/pkg/deployment/reconcile/plan_builder_tls.go b/pkg/deployment/reconcile/plan_builder_tls.go index 5de6f49b3..51c9439fd 100644 --- a/pkg/deployment/reconcile/plan_builder_tls.go +++ b/pkg/deployment/reconcile/plan_builder_tls.go @@ -255,7 +255,7 @@ func createCAAppendPlan(ctx context.Context, if _, exists := trusted.Data[certSha]; !exists { return api.Plan{api.NewAction(api.ActionTypeAppendTLSCACertificate, api.ServerGroupUnknown, "", "Append CA to truststore"). - AddParam(actionTypeAppendTLSCACertificateChecksum, certSha)} + AddParam(checksum, certSha)} } return nil @@ -340,15 +340,14 @@ func createCACleanPlan(ctx context.Context, for sha := range trusted.Data { if certSha != sha { return api.Plan{api.NewAction(api.ActionTypeCleanTLSCACertificate, api.ServerGroupUnknown, "", "Clean CA from truststore"). - AddParam(actionTypeAppendTLSCACertificateChecksum, sha)} + AddParam(checksum, sha)} } } return nil } -// createKeyfileRenewalPlan creates plan to renew server keyfile -func createKeyfileRenewalPlan(ctx context.Context, +func createKeyfileRenewalPlanDefault(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { @@ -367,10 +366,12 @@ func createKeyfileRenewalPlan(ctx context.Context, if !plan.IsEmpty() { return nil } - - if renew, recreate := keyfileRenewalRequired(ctx, log, apiObject, spec, status, cachedStatus, context, group, member); renew { + if renew, recreate := keyfileRenewalRequired(ctx, log, apiObject, spec, status, cachedStatus, context, group, member, api.TLSRotateModeRecreate); renew { log.Info().Msg("Renewal of keyfile required") - plan = append(plan, createKeyfileRotationPlan(log, spec, status, group, member, recreate)...) + if recreate { + plan = append(plan, api.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, group, member.ID, "Remove server keyfile and enforce renewal")) + } + plan = append(plan, createRotateMemberPlan(log, member, group, "Restart server after keyfile removal")...) } } @@ -380,43 +381,84 @@ func createKeyfileRenewalPlan(ctx context.Context, return plan } -func createKeyfileRenewalPlanMode( +func createKeyfileRenewalPlanInPlace(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, - member api.MemberStatus) api.TLSRotateMode { + cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { if !spec.TLS.IsSecure() { - return api.TLSRotateModeRecreate + return nil } - if spec.TLS.Mode.Get() != api.TLSRotateModeInPlace { - return api.TLSRotateModeRecreate - } + var plan api.Plan - if i := status.CurrentImage; i == nil { - return api.TLSRotateModeRecreate - } else { - if !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 || i.ImageID != member.ImageID { - return api.TLSRotateModeRecreate + status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { + if !group.IsArangod() { + return nil } - } - return api.TLSRotateModeInPlace -} + for _, member := range members { + if renew, recreate := keyfileRenewalRequired(ctx, log, apiObject, spec, status, cachedStatus, context, group, member, api.TLSRotateModeInPlace); renew { + log.Info().Msg("Renewal of keyfile required") + if recreate { + plan = append(plan, api.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, group, member.ID, "Remove server keyfile and enforce renewal")) + } + plan = append(plan, api.NewAction(api.ActionTypeRefreshTLSKeyfileCertificate, group, member.ID, "Renew Member Keyfile")) + } + } -func createKeyfileRotationPlan(log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, group api.ServerGroup, member api.MemberStatus, recreate bool) api.Plan { - p := api.Plan{} + return nil + }) + + return plan +} - if recreate { - p = append(p, - api.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, group, member.ID, "Remove server keyfile and enforce renewal")) +func createKeyfileRenewalPlan(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { + if !spec.TLS.IsSecure() { + return nil } - switch createKeyfileRenewalPlanMode(spec, status, member) { + switch createKeyfileRenewalPlanMode(spec, status) { case api.TLSRotateModeInPlace: - p = append(p, api.NewAction(api.ActionTypeRefreshTLSKeyfileCertificate, group, member.ID, "Renew Member Keyfile")) + return createKeyfileRenewalPlanInPlace(ctx, log, apiObject, spec, status, cachedStatus, context) default: - p = append(p, createRotateMemberPlan(log, member, group, "Restart server after keyfile removal")...) + return createKeyfileRenewalPlanDefault(ctx, log, apiObject, spec, status, cachedStatus, context) } - return p +} + +func createKeyfileRenewalPlanMode( + spec api.DeploymentSpec, status api.DeploymentStatus) api.TLSRotateMode { + if !spec.TLS.IsSecure() { + return api.TLSRotateModeRecreate + } + + mode := spec.TLS.Mode.Get() + + status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { + if mode != api.TLSRotateModeInPlace { + return nil + } + + for _, member := range list { + if mode != api.TLSRotateModeInPlace { + return nil + } + + if i := status.CurrentImage; i == nil { + mode = api.TLSRotateModeRecreate + } else { + if !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 || i.ImageID != member.ImageID { + mode = api.TLSRotateModeRecreate + } + } + } + + return nil + }) + + return mode } func checkServerValidCertRequest(ctx context.Context, apiObject k8sutil.APIObject, group api.ServerGroup, member api.MemberStatus, ca Certificates) (*tls.ConnectionState, error) { @@ -440,7 +482,7 @@ func keyfileRenewalRequired(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, cachedStatus inspector.Inspector, context PlanBuilderContext, - group api.ServerGroup, member api.MemberStatus) (bool, bool) { + group api.ServerGroup, member api.MemberStatus, mode api.TLSRotateMode) (bool, bool) { if !spec.TLS.IsSecure() { return false, false } @@ -485,7 +527,7 @@ func keyfileRenewalRequired(ctx context.Context, } // Ensure secret is propagated only on 3.7.0+ enterprise and inplace mode - if createKeyfileRenewalPlanMode(spec, status, member) == api.TLSRotateModeInPlace { + if mode == api.TLSRotateModeInPlace { conn, err := context.GetServerClient(ctx, group, member.ID) if err != nil { log.Warn().Err(err).Msg("Unable to get client") @@ -513,7 +555,7 @@ func keyfileRenewalRequired(ctx context.Context, keyfileSha := util.SHA256(keyfile) - if tls.Result.KeyFile.Checksum != keyfileSha { + if tls.Result.KeyFile.GetSHA().Checksum() != keyfileSha { return true, false } } diff --git a/pkg/deployment/resources/exporter.go b/pkg/deployment/resources/exporter.go index 28ef81f82..b90b22b25 100644 --- a/pkg/deployment/resources/exporter.go +++ b/pkg/deployment/resources/exporter.go @@ -26,6 +26,8 @@ import ( "sort" "strconv" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/constants" @@ -34,7 +36,7 @@ import ( ) // ArangodbExporterContainer creates metrics container -func ArangodbExporterContainer(image string, args []string, livenessProbe *k8sutil.HTTPProbeConfig, +func ArangodbExporterContainer(image string, args []string, livenessProbe *probes.HTTPProbeConfig, resources v1.ResourceRequirements, securityContext *v1.SecurityContext, spec api.DeploymentSpec) v1.Container { @@ -96,8 +98,8 @@ func createExporterArgs(spec api.DeploymentSpec) []string { return args } -func createExporterLivenessProbe(isSecure bool) *k8sutil.HTTPProbeConfig { - probeCfg := &k8sutil.HTTPProbeConfig{ +func createExporterLivenessProbe(isSecure bool) *probes.HTTPProbeConfig { + probeCfg := &probes.HTTPProbeConfig{ LocalPath: "/", Port: k8sutil.ArangoExporterPort, Secure: isSecure, diff --git a/pkg/deployment/resources/pod_creator_probes.go b/pkg/deployment/resources/pod_creator_probes.go index 0bc39ef33..8c5c8aa15 100644 --- a/pkg/deployment/resources/pod_creator_probes.go +++ b/pkg/deployment/resources/pod_creator_probes.go @@ -23,24 +23,37 @@ package resources import ( + "fmt" + "os" + "path/filepath" + "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/jwt" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes" + core "k8s.io/api/core/v1" ) +type Probe interface { + Create() *core.Probe + + SetSpec(spec *api.ServerGroupProbeSpec) +} + type probeCheckBuilder struct { liveness, readiness probeBuilder } -type probeBuilder func(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) +type probeBuilder func(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) -func nilProbeBuilder(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) { +func nilProbeBuilder(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) { return nil, nil } -func (r *Resources) getReadinessProbe(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) { +func (r *Resources) getReadinessProbe(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) { if !r.isReadinessProbeEnabled(spec, group, version) { return nil, nil } @@ -65,16 +78,12 @@ func (r *Resources) getReadinessProbe(spec api.DeploymentSpec, group api.ServerG probeSpec := groupSpec.GetProbesSpec() - config.InitialDelaySeconds = probeSpec.ReadinessProbeSpec.GetInitialDelaySeconds(config.InitialDelaySeconds) - config.PeriodSeconds = probeSpec.ReadinessProbeSpec.GetPeriodSeconds(config.PeriodSeconds) - config.TimeoutSeconds = probeSpec.ReadinessProbeSpec.GetTimeoutSeconds(config.TimeoutSeconds) - config.SuccessThreshold = probeSpec.ReadinessProbeSpec.GetSuccessThreshold(config.SuccessThreshold) - config.FailureThreshold = probeSpec.ReadinessProbeSpec.GetFailureThreshold(config.FailureThreshold) + config.SetSpec(probeSpec.ReadinessProbeSpec) return config, nil } -func (r *Resources) getLivenessProbe(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) { +func (r *Resources) getLivenessProbe(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) { if !r.isLivenessProbeEnabled(spec, group, version) { return nil, nil } @@ -99,11 +108,7 @@ func (r *Resources) getLivenessProbe(spec api.DeploymentSpec, group api.ServerGr probeSpec := groupSpec.GetProbesSpec() - config.InitialDelaySeconds = probeSpec.LivenessProbeSpec.GetInitialDelaySeconds(config.InitialDelaySeconds) - config.PeriodSeconds = probeSpec.LivenessProbeSpec.GetPeriodSeconds(config.PeriodSeconds) - config.TimeoutSeconds = probeSpec.LivenessProbeSpec.GetTimeoutSeconds(config.TimeoutSeconds) - config.SuccessThreshold = probeSpec.LivenessProbeSpec.GetSuccessThreshold(config.SuccessThreshold) - config.FailureThreshold = probeSpec.LivenessProbeSpec.GetFailureThreshold(config.FailureThreshold) + config.SetSpec(probeSpec.LivenessProbeSpec) return config, nil } @@ -139,20 +144,20 @@ func (r *Resources) isLivenessProbeEnabled(spec api.DeploymentSpec, group api.Se func (r *Resources) probeBuilders() map[api.ServerGroup]probeCheckBuilder { return map[api.ServerGroup]probeCheckBuilder{ api.ServerGroupSingle: { - liveness: r.probeBuilderLivenessCore, - readiness: r.probeBuilderReadinessCore, + liveness: r.probeBuilderLivenessCoreOperator, + readiness: r.probeBuilderReadinessCoreOperator, }, api.ServerGroupAgents: { - liveness: r.probeBuilderLivenessCore, - readiness: r.probeBuilderReadinessSimpleCore, + liveness: r.probeBuilderLivenessCoreOperator, + readiness: r.probeBuilderReadinessSimpleCoreOperator, }, api.ServerGroupDBServers: { - liveness: r.probeBuilderLivenessCore, - readiness: r.probeBuilderReadinessSimpleCore, + liveness: r.probeBuilderLivenessCoreOperator, + readiness: r.probeBuilderReadinessSimpleCoreOperator, }, api.ServerGroupCoordinators: { - liveness: r.probeBuilderLivenessCore, - readiness: r.probeBuilderReadinessCore, + liveness: r.probeBuilderLivenessCoreOperator, + readiness: r.probeBuilderReadinessCoreOperator, }, api.ServerGroupSyncMasters: { liveness: r.probeBuilderLivenessSync, @@ -165,7 +170,42 @@ func (r *Resources) probeBuilders() map[api.ServerGroup]probeCheckBuilder { } } -func (r *Resources) probeBuilderLivenessCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) { +func (r *Resources) probeCommand(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version, endpoint string) ([]string, error) { + binaryPath, err := os.Executable() + if err != nil { + return nil, err + } + exePath := filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(binaryPath)) + args := []string{ + exePath, + "lifecycle", + "probe", + fmt.Sprintf("--endpoint=%s", endpoint), + } + + if spec.IsSecure() { + args = append(args, "--ssl") + } + + if spec.IsAuthenticated() { + args = append(args, "--auth") + } + + return args, nil +} + +func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) { + args, err := r.probeCommand(spec, group, version, "/_api/version") + if err != nil { + return nil, err + } + + return &probes.CMDProbeConfig{ + Command: args, + }, nil +} + +func (r *Resources) probeBuilderLivenessCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) { authorization := "" if spec.IsAuthenticated() { secretData, err := r.getJWTSecret(spec) @@ -177,14 +217,32 @@ func (r *Resources) probeBuilderLivenessCore(spec api.DeploymentSpec, group api. return nil, maskAny(err) } } - return &k8sutil.HTTPProbeConfig{ + return &probes.HTTPProbeConfig{ LocalPath: "/_api/version", Secure: spec.IsSecure(), Authorization: authorization, }, nil } -func (r *Resources) probeBuilderReadinessSimpleCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) { +func (r *Resources) probeBuilderReadinessSimpleCoreOperator(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) { + p, err := r.probeBuilderReadinessCoreOperator(spec, group, version) + if err != nil { + return nil, err + } + + if p == nil { + return nil, nil + } + + p.SetSpec(&api.ServerGroupProbeSpec{ + InitialDelaySeconds: util.NewInt32(15), + PeriodSeconds: util.NewInt32(10), + }) + + return p, nil +} + +func (r *Resources) probeBuilderReadinessSimpleCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) { p, err := r.probeBuilderLivenessCore(spec, group, version) if err != nil { return nil, err @@ -194,13 +252,39 @@ func (r *Resources) probeBuilderReadinessSimpleCore(spec api.DeploymentSpec, gro return nil, nil } - p.InitialDelaySeconds = 15 - p.PeriodSeconds = 10 + p.SetSpec(&api.ServerGroupProbeSpec{ + InitialDelaySeconds: util.NewInt32(15), + PeriodSeconds: util.NewInt32(10), + }) return p, nil } -func (r *Resources) probeBuilderReadinessCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) { +func (r *Resources) probeBuilderReadinessCoreOperator(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) { + localPath := "/_api/version" + switch spec.GetMode() { + case api.DeploymentModeActiveFailover: + localPath = "/_admin/echo" + } + + // /_admin/server/availability is the way to go, it is available since 3.3.9 + if version.CompareTo("3.3.9") >= 0 { + localPath = "/_admin/server/availability" + } + + args, err := r.probeCommand(spec, group, version, localPath) + if err != nil { + return nil, err + } + + return &probes.CMDProbeConfig{ + Command: args, + InitialDelaySeconds: 2, + PeriodSeconds: 2, + }, nil +} + +func (r *Resources) probeBuilderReadinessCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) { localPath := "/_api/version" switch spec.GetMode() { case api.DeploymentModeActiveFailover: @@ -223,7 +307,7 @@ func (r *Resources) probeBuilderReadinessCore(spec api.DeploymentSpec, group api return nil, maskAny(err) } } - probeCfg := &k8sutil.HTTPProbeConfig{ + probeCfg := &probes.HTTPProbeConfig{ LocalPath: localPath, Secure: spec.IsSecure(), Authorization: authorization, @@ -234,7 +318,7 @@ func (r *Resources) probeBuilderReadinessCore(spec api.DeploymentSpec, group api return probeCfg, nil } -func (r *Resources) probeBuilderLivenessSync(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) { +func (r *Resources) probeBuilderLivenessSync(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) { authorization := "" port := k8sutil.ArangoSyncMasterPort if group == api.ServerGroupSyncWorkers { @@ -261,7 +345,7 @@ func (r *Resources) probeBuilderLivenessSync(spec api.DeploymentSpec, group api. // Don't have a probe return nil, nil } - return &k8sutil.HTTPProbeConfig{ + return &probes.HTTPProbeConfig{ LocalPath: "/_api/version", Secure: spec.IsSecure(), Authorization: authorization, diff --git a/pkg/deployment/resources/secret_hashes.go b/pkg/deployment/resources/secret_hashes.go index 0f49b73e1..9c97283b7 100644 --- a/pkg/deployment/resources/secret_hashes.go +++ b/pkg/deployment/resources/secret_hashes.go @@ -138,15 +138,30 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspector.Inspector) error } if spec.IsAuthenticated() { - secretName := spec.Authentication.GetJWTSecretName() - getExpectedHash := func() string { return getHashes().AuthJWT } - setExpectedHash := func(h string) error { - return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.AuthJWT = h })) - } - if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil { - return maskAny(err) - } else if !hashOK { - badSecretNames = append(badSecretNames, secretName) + if image == nil || image.ArangoDBVersion.CompareTo("3.7.0") < 0 { + secretName := spec.Authentication.GetJWTSecretName() + getExpectedHash := func() string { return getHashes().AuthJWT } + setExpectedHash := func(h string) error { + return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.AuthJWT = h })) + } + if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil { + return maskAny(err) + } else if !hashOK { + badSecretNames = append(badSecretNames, secretName) + } + } else { + if _, exists := cachedStatus.Secret(pod.JWTSecretFolder(deploymentName)); !exists { + secretName := spec.Authentication.GetJWTSecretName() + getExpectedHash := func() string { return getHashes().AuthJWT } + setExpectedHash := func(h string) error { + return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.AuthJWT = h })) + } + if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil { + return maskAny(err) + } else if !hashOK { + badSecretNames = append(badSecretNames, secretName) + } + } } } if spec.RocksDB.IsEncrypted() { @@ -162,7 +177,7 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspector.Inspector) error badSecretNames = append(badSecretNames, secretName) } } else { - if _, exists := cachedStatus.Secret(pod.GetKeyfolderSecretName(deploymentName)); !exists { + if _, exists := cachedStatus.Secret(pod.GetEncryptionFolderSecretName(deploymentName)); !exists { secretName := spec.RocksDB.Encryption.GetKeySecretName() getExpectedHash := func() string { return getHashes().RocksDBEncryptionKey } setExpectedHash := func(h string) error { diff --git a/pkg/deployment/resources/secrets.go b/pkg/deployment/resources/secrets.go index fa63f4c73..a84f0b6b1 100644 --- a/pkg/deployment/resources/secrets.go +++ b/pkg/deployment/resources/secrets.go @@ -29,6 +29,8 @@ import ( "fmt" "time" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/rs/zerolog" operatorErrors "github.com/arangodb/kube-arangodb/pkg/util/errors" @@ -74,6 +76,8 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspector.Ins status, _ := r.context.GetStatus() apiObject := r.context.GetAPIObject() deploymentName := apiObject.GetName() + image := status.CurrentImage + imageFound := status.CurrentImage != nil defer metrics.SetDuration(inspectSecretsDurationGauges.WithLabelValues(deploymentName), start) counterMetric := inspectedSecretsCounters.WithLabelValues(deploymentName) @@ -83,6 +87,14 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspector.Ins return maskAny(err) } + if imageFound { + if pod.VersionHasJWTSecretKeyfolder(image.ArangoDBVersion, image.Enterprise) { + if err := r.ensureTokenSecretFolder(cachedStatus, secrets, spec.Authentication.GetJWTSecretName(), pod.JWTSecretFolder(deploymentName)); err != nil { + return maskAny(err) + } + } + } + if spec.Metrics.IsEnabled() { if err := r.ensureExporterTokenSecret(cachedStatus, secrets, spec.Metrics.GetJWTTokenSecretName(), spec.Authentication.GetJWTSecretName()); err != nil { return maskAny(err) @@ -131,7 +143,7 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspector.Ins } if spec.RocksDB.IsEncrypted() { if i := status.CurrentImage; i != nil && i.Enterprise && i.ArangoDBVersion.CompareTo("3.7.0") >= 0 { - if err := r.ensureEncryptionKeyfolderSecret(cachedStatus, secrets, spec.RocksDB.Encryption.GetKeySecretName(), pod.GetKeyfolderSecretName(deploymentName)); err != nil { + if err := r.ensureEncryptionKeyfolderSecret(cachedStatus, secrets, spec.RocksDB.Encryption.GetKeySecretName(), pod.GetEncryptionFolderSecretName(deploymentName)); err != nil { return maskAny(err) } } @@ -157,9 +169,28 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspector.Ins return nil } -// ensureTokenSecret checks if a secret with given name exists in the namespace -// of the deployment. If not, it will add such a secret with a random -// token. +func (r *Resources) ensureTokenSecretFolder(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName, folderSecretName string) error { + if _, exists := cachedStatus.Secret(folderSecretName); exists { + return nil + } + + s, exists := cachedStatus.Secret(secretName) + if !exists { + return errors.Errorf("Token secret does not exist") + } + + token, ok := s.Data[constants.SecretKeyToken] + if !ok { + return errors.Errorf("Token secret is invalid") + } + + if err := r.createSecretWithKey(secrets, folderSecretName, util.SHA256(token), token); err != nil { + return err + } + + return nil +} + func (r *Resources) ensureTokenSecret(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName string) error { if _, exists := cachedStatus.Secret(secretName); !exists { return r.createTokenSecret(secrets, secretName) @@ -196,20 +227,28 @@ func (r *Resources) createSecret(secrets k8sutil.SecretInterface, secretName str func (r *Resources) ensureSecretWithEmptyKey(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string) error { if _, exists := cachedStatus.Secret(secretName); !exists { - return r.createSecretWithEmptyKey(secrets, secretName, keyName) + return r.createSecretWithKey(secrets, secretName, keyName, nil) + } + + return nil +} + +func (r *Resources) ensureSecretWithKey(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error { + if _, exists := cachedStatus.Secret(secretName); !exists { + return r.createSecretWithKey(secrets, secretName, keyName, value) } return nil } -func (r *Resources) createSecretWithEmptyKey(secrets k8sutil.SecretInterface, secretName, keyName string) error { +func (r *Resources) createSecretWithKey(secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error { // Create secret secret := &core.Secret{ ObjectMeta: meta.ObjectMeta{ Name: secretName, }, Data: map[string][]byte{ - keyName: {}, + keyName: value, }, } // Attach secret to owner @@ -242,17 +281,28 @@ func (r *Resources) createTokenSecret(secrets k8sutil.SecretInterface, secretNam } func (r *Resources) ensureEncryptionKeyfolderSecret(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, keyfileSecretName, secretName string) error { + _, folderExists := cachedStatus.Secret(secretName) + keyfile, exists := cachedStatus.Secret(keyfileSecretName) if !exists { + if folderExists { + return nil + } return errors.Errorf("Unable to find original secret %s", keyfileSecretName) } if len(keyfile.Data) == 0 { + if folderExists { + return nil + } return errors.Errorf("Missing key in secret") } d, ok := keyfile.Data[constants.SecretEncryptionKey] if !ok { + if folderExists { + return nil + } return errors.Errorf("Missing key in secret") } diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index 6c4552031..1382e8f1c 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -135,6 +135,24 @@ func CreateArangodDatabaseClient(ctx context.Context, cli corev1.CoreV1Interface return c, nil } +func CreateArangodAgencyConnection(ctx context.Context, apiObject *api.ArangoDeployment) (driver.Connection, error) { + var dnsNames []string + for _, m := range apiObject.Status.Members.Agents { + dnsName := k8sutil.CreatePodDNSName(apiObject, api.ServerGroupAgents.AsRole(), m.ID) + dnsNames = append(dnsNames, dnsName) + } + shortTimeout := false + connConfig, err := createArangodHTTPConfigForDNSNames(ctx, apiObject, dnsNames, shortTimeout) + if err != nil { + return nil, maskAny(err) + } + agencyConn, err := agency.NewAgencyConnection(connConfig) + if err != nil { + return nil, maskAny(err) + } + return agencyConn, nil +} + // CreateArangodAgencyClient creates a go-driver client for accessing the agents of the given deployment. func CreateArangodAgencyClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment) (agency.Agency, error) { var dnsNames []string @@ -143,7 +161,7 @@ func CreateArangodAgencyClient(ctx context.Context, cli corev1.CoreV1Interface, dnsNames = append(dnsNames, dnsName) } shortTimeout := false - connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, dnsNames, shortTimeout) + connConfig, err := createArangodHTTPConfigForDNSNames(ctx, apiObject, dnsNames, shortTimeout) if err != nil { return nil, maskAny(err) } @@ -182,7 +200,7 @@ func CreateArangodImageIDClient(ctx context.Context, deployment k8sutil.APIObjec // CreateArangodClientForDNSName creates a go-driver client for a given DNS name. func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string, shortTimeout bool) (driver.Client, error) { - connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}, shortTimeout) + connConfig, err := createArangodHTTPConfigForDNSNames(ctx, apiObject, []string{dnsName}, shortTimeout) if err != nil { return nil, maskAny(err) } @@ -209,7 +227,7 @@ func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interfa } // createArangodHTTPConfigForDNSNames creates a go-driver HTTP connection config for a given DNS names. -func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string, shortTimeout bool) (http.ConnectionConfig, error) { +func createArangodHTTPConfigForDNSNames(ctx context.Context, apiObject *api.ArangoDeployment, dnsNames []string, shortTimeout bool) (http.ConnectionConfig, error) { scheme := "http" transport := sharedHTTPTransport if shortTimeout { diff --git a/pkg/util/arangod/conn/factory.go b/pkg/util/arangod/conn/factory.go new file mode 100644 index 000000000..eb54f2836 --- /dev/null +++ b/pkg/util/arangod/conn/factory.go @@ -0,0 +1,129 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Adam Janikowski +// + +package conn + +import ( + "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/agency" + "github.com/arangodb/go-driver/http" +) + +type Auth func() (driver.Authentication, error) +type Config func() (http.ConnectionConfig, error) + +type Factory interface { + Connection(hosts ...string) (driver.Connection, error) + AgencyConnection(hosts ...string) (driver.Connection, error) + + Client(hosts ...string) (driver.Client, error) + Agency(hosts ...string) (agency.Agency, error) +} + +func NewFactory(auth Auth, config Config) Factory { + return &factory{ + auth: auth, + config: config, + } +} + +type factory struct { + auth Auth + config Config +} + +func (f factory) AgencyConnection(hosts ...string) (driver.Connection, error) { + cfg, err := f.config() + if err != nil { + return nil, err + } + + cfg.Endpoints = hosts + + conn, err := agency.NewAgencyConnection(cfg) + if err != nil { + return nil, err + } + + if f.auth == nil { + return conn, nil + } + auth, err := f.auth() + if err != nil { + return nil, err + } + return conn.SetAuthentication(auth) +} + +func (f factory) Client(hosts ...string) (driver.Client, error) { + conn, err := f.Connection(hosts...) + if err != nil { + return nil, err + } + + config := driver.ClientConfig{ + Connection: conn, + } + + if f.auth != nil { + auth, err := f.auth() + if err != nil { + return nil, err + } + + config.Authentication = auth + } + + return driver.NewClient(config) +} + +func (f factory) Agency(hosts ...string) (agency.Agency, error) { + conn, err := f.AgencyConnection(hosts...) + if err != nil { + return nil, err + } + + return agency.NewAgency(conn) +} + +func (f factory) Connection(hosts ...string) (driver.Connection, error) { + cfg, err := f.config() + if err != nil { + return nil, err + } + + cfg.Endpoints = hosts + + conn, err := http.NewConnection(cfg) + if err != nil { + return nil, err + } + + if f.auth == nil { + return conn, nil + } + auth, err := f.auth() + if err != nil { + return nil, err + } + return conn.SetAuthentication(auth) +} diff --git a/pkg/util/checksum.go b/pkg/util/checksum.go index c16f45ac3..0ff9e50dd 100644 --- a/pkg/util/checksum.go +++ b/pkg/util/checksum.go @@ -27,6 +27,10 @@ import ( "fmt" ) +func SHA256FromString(data string) string { + return SHA256([]byte(data)) +} + func SHA256(data []byte) string { return fmt.Sprintf("%0x", sha256.Sum256(data)) } diff --git a/pkg/util/k8sutil/lifecycle.go b/pkg/util/k8sutil/lifecycle.go index bfebf0793..0856618c5 100644 --- a/pkg/util/k8sutil/lifecycle.go +++ b/pkg/util/k8sutil/lifecycle.go @@ -33,7 +33,7 @@ import ( const ( initLifecycleContainerName = "init-lifecycle" - lifecycleVolumeMountDir = "/lifecycle/tools" + LifecycleVolumeMountDir = "/lifecycle/tools" lifecycleVolumeName = "lifecycle" ) @@ -46,7 +46,7 @@ func InitLifecycleContainer(image string, resources *v1.ResourceRequirements, se c := v1.Container{ Name: initLifecycleContainerName, Image: image, - Command: append([]string{binaryPath}, "lifecycle", "copy", "--target", lifecycleVolumeMountDir), + Command: append([]string{binaryPath}, "lifecycle", "copy", "--target", LifecycleVolumeMountDir), VolumeMounts: []v1.VolumeMount{ LifecycleVolumeMount(), }, @@ -66,7 +66,7 @@ func NewLifecycle() (*v1.Lifecycle, error) { if err != nil { return nil, maskAny(err) } - exePath := filepath.Join(lifecycleVolumeMountDir, filepath.Base(binaryPath)) + exePath := filepath.Join(LifecycleVolumeMountDir, filepath.Base(binaryPath)) lifecycle := &v1.Lifecycle{ PreStop: &v1.Handler{ Exec: &v1.ExecAction{ @@ -91,7 +91,7 @@ func GetLifecycleEnv() []v1.EnvVar { func LifecycleVolumeMount() v1.VolumeMount { return v1.VolumeMount{ Name: lifecycleVolumeName, - MountPath: lifecycleVolumeMountDir, + MountPath: LifecycleVolumeMountDir, } } diff --git a/pkg/util/k8sutil/probes.go b/pkg/util/k8sutil/probes.go deleted file mode 100644 index 47c168b75..000000000 --- a/pkg/util/k8sutil/probes.go +++ /dev/null @@ -1,86 +0,0 @@ -// -// DISCLAIMER -// -// Copyright 2020 ArangoDB GmbH, Cologne, Germany -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Copyright holder is ArangoDB GmbH, Cologne, Germany -// -// Author Ewout Prangsma -// - -package k8sutil - -import ( - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -// HTTPProbeConfig contains settings for creating a liveness/readiness probe. -type HTTPProbeConfig struct { - // Local path to GET - LocalPath string // `e.g. /_api/version` - // Secure connection? - Secure bool - // Value for an Authorization header (can be empty) - Authorization string - // Port to inspect (defaults to ArangoPort) - Port int - // Number of seconds after the container has started before liveness probes are initiated (defaults to 30) - InitialDelaySeconds int32 - // Number of seconds after which the probe times out (defaults to 2). - TimeoutSeconds int32 - // How often (in seconds) to perform the probe (defaults to 10). - PeriodSeconds int32 - // Minimum consecutive successes for the probe to be considered successful after having failed (defaults to 1). - SuccessThreshold int32 - // Minimum consecutive failures for the probe to be considered failed after having succeeded (defaults to 3). - FailureThreshold int32 -} - -// Create creates a probe from given config -func (config HTTPProbeConfig) Create() *v1.Probe { - scheme := v1.URISchemeHTTP - if config.Secure { - scheme = v1.URISchemeHTTPS - } - var headers []v1.HTTPHeader - if config.Authorization != "" { - headers = append(headers, v1.HTTPHeader{ - Name: "Authorization", - Value: config.Authorization, - }) - } - def := func(value, defaultValue int32) int32 { - if value != 0 { - return value - } - return defaultValue - } - return &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: config.LocalPath, - Port: intstr.FromInt(int(def(int32(config.Port), ArangoPort))), - Scheme: scheme, - HTTPHeaders: headers, - }, - }, - InitialDelaySeconds: def(config.InitialDelaySeconds, 15*60), // Wait 15min before first probe - TimeoutSeconds: def(config.TimeoutSeconds, 2), // Timeout of each probe is 2s - PeriodSeconds: def(config.PeriodSeconds, 60), // Interval between probes is 10s - SuccessThreshold: def(config.SuccessThreshold, 1), // Single probe is enough to indicate success - FailureThreshold: def(config.FailureThreshold, 10), // Need 10 failed probes to consider a failed state - } -} diff --git a/pkg/util/k8sutil/probes/probes.go b/pkg/util/k8sutil/probes/probes.go new file mode 100644 index 000000000..cf27e1689 --- /dev/null +++ b/pkg/util/k8sutil/probes/probes.go @@ -0,0 +1,142 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package probes + +import ( + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// HTTPProbeConfig contains settings for creating a liveness/readiness probe. +type HTTPProbeConfig struct { + // Local path to GET + LocalPath string // `e.g. /_api/version` + // Secure connection? + Secure bool + // Value for an Authorization header (can be empty) + Authorization string + // Port to inspect (defaults to ArangoPort) + Port int + // Number of seconds after the container has started before liveness probes are initiated (defaults to 30) + InitialDelaySeconds int32 + // Number of seconds after which the probe times out (defaults to 2). + TimeoutSeconds int32 + // How often (in seconds) to perform the probe (defaults to 10). + PeriodSeconds int32 + // Minimum consecutive successes for the probe to be considered successful after having failed (defaults to 1). + SuccessThreshold int32 + // Minimum consecutive failures for the probe to be considered failed after having succeeded (defaults to 3). + FailureThreshold int32 +} + +func (config *HTTPProbeConfig) SetSpec(spec *api.ServerGroupProbeSpec) { + config.InitialDelaySeconds = spec.GetInitialDelaySeconds(config.InitialDelaySeconds) + config.TimeoutSeconds = spec.GetTimeoutSeconds(config.TimeoutSeconds) + config.PeriodSeconds = spec.GetPeriodSeconds(config.PeriodSeconds) + config.SuccessThreshold = spec.GetSuccessThreshold(config.SuccessThreshold) + config.FailureThreshold = spec.GetFailureThreshold(config.FailureThreshold) +} + +// Create creates a probe from given config +func (config HTTPProbeConfig) Create() *core.Probe { + scheme := core.URISchemeHTTP + if config.Secure { + scheme = core.URISchemeHTTPS + } + var headers []core.HTTPHeader + if config.Authorization != "" { + headers = append(headers, core.HTTPHeader{ + Name: "Authorization", + Value: config.Authorization, + }) + } + def := func(value, defaultValue int32) int32 { + if value != 0 { + return value + } + return defaultValue + } + return &core.Probe{ + Handler: core.Handler{ + HTTPGet: &core.HTTPGetAction{ + Path: config.LocalPath, + Port: intstr.FromInt(int(def(int32(config.Port), k8sutil.ArangoPort))), + Scheme: scheme, + HTTPHeaders: headers, + }, + }, + InitialDelaySeconds: defaultInt32(config.InitialDelaySeconds, 15*60), // Wait 15min before first probe + TimeoutSeconds: defaultInt32(config.TimeoutSeconds, 2), // Timeout of each probe is 2s + PeriodSeconds: defaultInt32(config.PeriodSeconds, 60), // Interval between probes is 10s + SuccessThreshold: defaultInt32(config.SuccessThreshold, 1), // Single probe is enough to indicate success + FailureThreshold: defaultInt32(config.FailureThreshold, 10), // Need 10 failed probes to consider a failed state + } +} + +type CMDProbeConfig struct { + // Command to be executed + Command []string + // Number of seconds after the container has started before liveness probes are initiated (defaults to 30) + InitialDelaySeconds int32 + // Number of seconds after which the probe times out (defaults to 2). + TimeoutSeconds int32 + // How often (in seconds) to perform the probe (defaults to 10). + PeriodSeconds int32 + // Minimum consecutive successes for the probe to be considered successful after having failed (defaults to 1). + SuccessThreshold int32 + // Minimum consecutive failures for the probe to be considered failed after having succeeded (defaults to 3). + FailureThreshold int32 +} + +func (config *CMDProbeConfig) SetSpec(spec *api.ServerGroupProbeSpec) { + config.InitialDelaySeconds = spec.GetInitialDelaySeconds(config.InitialDelaySeconds) + config.TimeoutSeconds = spec.GetTimeoutSeconds(config.TimeoutSeconds) + config.PeriodSeconds = spec.GetPeriodSeconds(config.PeriodSeconds) + config.SuccessThreshold = spec.GetSuccessThreshold(config.SuccessThreshold) + config.FailureThreshold = spec.GetFailureThreshold(config.FailureThreshold) +} + +// Create creates a probe from given config +func (config CMDProbeConfig) Create() *core.Probe { + return &core.Probe{ + Handler: core.Handler{ + Exec: &core.ExecAction{ + Command: config.Command, + }, + }, + InitialDelaySeconds: defaultInt32(config.InitialDelaySeconds, 15*60), // Wait 15min before first probe + TimeoutSeconds: defaultInt32(config.TimeoutSeconds, 2), // Timeout of each probe is 2s + PeriodSeconds: defaultInt32(config.PeriodSeconds, 60), // Interval between probes is 10s + SuccessThreshold: defaultInt32(config.SuccessThreshold, 1), // Single probe is enough to indicate success + FailureThreshold: defaultInt32(config.FailureThreshold, 10), // Need 10 failed probes to consider a failed state + } +} + +func defaultInt32(value, defaultValue int32) int32 { + if value != 0 { + return value + } + return defaultValue +} diff --git a/pkg/util/k8sutil/probes_test.go b/pkg/util/k8sutil/probes/probes_test.go similarity index 99% rename from pkg/util/k8sutil/probes_test.go rename to pkg/util/k8sutil/probes/probes_test.go index 4344f617f..9ca44285c 100644 --- a/pkg/util/k8sutil/probes_test.go +++ b/pkg/util/k8sutil/probes/probes_test.go @@ -20,7 +20,7 @@ // Author Jan Christoph Uhde // -package k8sutil +package probes import ( "testing" From 3d41de637b786fce3641d0835754c997e9955ecd Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Thu, 25 Jun 2020 13:51:19 +0000 Subject: [PATCH 2/4] Fix UT --- pkg/deployment/deployment_suite_test.go | 4 +++- pkg/deployment/pod/encryption.go | 5 ----- pkg/deployment/pod/jwt.go | 7 +------ 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/pkg/deployment/deployment_suite_test.go b/pkg/deployment/deployment_suite_test.go index 579a7208f..3be9a3973 100644 --- a/pkg/deployment/deployment_suite_test.go +++ b/pkg/deployment/deployment_suite_test.go @@ -26,6 +26,7 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "testing" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes" @@ -158,8 +159,9 @@ func getCMDProbeCreator() probeCreator { } func createCMDTestProbe(secure, authorization bool, endpoint string) resources.Probe { + bin, _ := os.Executable() args := []string{ - "/lifecycle/tools/___go_test_github_com_arangodb_kube_arangodb_pkg_deployment", + filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(bin)), "lifecycle", "probe", fmt.Sprintf("--endpoint=%s", endpoint), diff --git a/pkg/deployment/pod/encryption.go b/pkg/deployment/pod/encryption.go index ae767953f..a03c8274b 100644 --- a/pkg/deployment/pod/encryption.go +++ b/pkg/deployment/pod/encryption.go @@ -169,11 +169,6 @@ func (e encryption) Verify(i Input, cachedStatus inspector.Inspector) error { return errors.Wrapf(err, "RocksDB encryption key secret validation failed") } return nil - } else { - _, exists := cachedStatus.Secret(GetEncryptionFolderSecretName(i.ApiObject.GetName())) - if !exists { - return errors.Errorf("Encryption key folder secret does not exist %s", i.Deployment.RocksDB.Encryption.GetKeySecretName()) - } } return nil diff --git a/pkg/deployment/pod/jwt.go b/pkg/deployment/pod/jwt.go index e15adaaac..bcc31e81f 100644 --- a/pkg/deployment/pod/jwt.go +++ b/pkg/deployment/pod/jwt.go @@ -119,12 +119,7 @@ func (e jwt) Verify(i Input, cachedStatus inspector.Inspector) error { return nil } - if VersionHasJWTSecretKeyfolder(i.Version, i.Enterprise) { - _, exists := cachedStatus.Secret(JWTSecretFolder(i.ApiObject.GetName())) - if !exists { - return errors.Errorf("Secret for JWT Folderis missing %s", i.Deployment.Authentication.GetJWTSecretName()) - } - } else { + if !VersionHasJWTSecretKeyfolder(i.Version, i.Enterprise) { secret, exists := cachedStatus.Secret(i.Deployment.Authentication.GetJWTSecretName()) if !exists { return errors.Errorf("Secret for JWT token is missing %s", i.Deployment.Authentication.GetJWTSecretName()) From 3c1cd99383ab7c8dd4d360397043834d6e5cb419 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Thu, 25 Jun 2020 14:35:32 +0000 Subject: [PATCH 3/4] Fix UT --- pkg/deployment/reconcile/plan_builder_jwt.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/pkg/deployment/reconcile/plan_builder_jwt.go b/pkg/deployment/reconcile/plan_builder_jwt.go index 66f23bc6a..fb9961699 100644 --- a/pkg/deployment/reconcile/plan_builder_jwt.go +++ b/pkg/deployment/reconcile/plan_builder_jwt.go @@ -64,22 +64,16 @@ func createJWTKeyUpdate(ctx context.Context, jwt, ok := s.Data[constants.SecretKeyToken] if !ok { log.Warn().Msgf("JWT Secret is invalid, no rotation will take place") - return nil + return addJWTPropagatedPlanAction(status) } jwtSha := util.SHA256(jwt) - f, ok := cachedStatus.Secret(pod.JWTSecretFolder(apiObject.GetName())) - if !ok { - log.Info().Msgf("JWT Folder Secret is missing, no rotation will take place") - return nil - } - - if _, ok := f.Data[jwtSha]; !ok { + if _, ok := folder.Data[jwtSha]; !ok { return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTAdd, api.ServerGroupUnknown, "", "Add JWT key").AddParam(checksum, jwtSha)) } - activeKey, ok := f.Data[pod.ActiveJWTKey] + activeKey, ok := folder.Data[pod.ActiveJWTKey] if !ok { return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTSetActive, api.ServerGroupUnknown, "", "Set active key").AddParam(checksum, jwtSha)) } From 81b1e5d74cfd843dfb8dbdd9c8abc87107d229f9 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Thu, 25 Jun 2020 14:36:57 +0000 Subject: [PATCH 4/4] Changelog --- CHANGELOG.md | 1 + pkg/deployment/reconcile/plan_builder_jwt.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9990e6248..7ff2ab716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Improve Helm 3 support - Allow to customize ID Pod selectors - Add Label and Envs Pod customization +- Improved JWT Rotation ## [1.0.3](https://github.com/arangodb/kube-arangodb/tree/1.0.3) (2020-05-25) - Prevent deletion of not known PVC's diff --git a/pkg/deployment/reconcile/plan_builder_jwt.go b/pkg/deployment/reconcile/plan_builder_jwt.go index fb9961699..82490eb6a 100644 --- a/pkg/deployment/reconcile/plan_builder_jwt.go +++ b/pkg/deployment/reconcile/plan_builder_jwt.go @@ -92,7 +92,7 @@ func createJWTKeyUpdate(ctx context.Context, return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTSetActive, api.ServerGroupUnknown, "", "Set active key").AddParam(checksum, jwtSha)) } - for key := range f.Data { + for key := range folder.Data { if key == pod.ActiveJWTKey { continue }