From 4e63aade830cb0bac637ab66d7f2e7530d6a19e3 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Mon, 6 Dec 2021 09:51:05 +0000 Subject: [PATCH] [Feature] Agency Cache --- CHANGELOG.md | 1 + pkg/deployment/agency.go | 29 +++++ pkg/deployment/agency/cache.go | 113 +++++++++++++++++ pkg/deployment/agency/cache_test.go | 21 ++++ .../agency/{maintenance.go => config.go} | 60 ++++----- pkg/deployment/agency/config_test.go | 77 ++++++++++++ .../{agency.go => current_collections.go} | 26 +--- pkg/deployment/agency/definitions.go | 27 +++- .../{database.go => plan_collections.go} | 34 ++--- pkg/deployment/agency/state.go | 117 ++++++++++++++++++ pkg/deployment/agency/state_test.go | 37 ++++++ pkg/deployment/deployment.go | 68 ++++++---- pkg/deployment/deployment_inspector.go | 8 ++ pkg/deployment/reconcile/action_context.go | 11 +- .../reconcile/action_maintenance_condition.go | 8 +- .../reconcile/action_resign_leadership.go | 10 +- pkg/deployment/reconcile/context.go | 3 +- .../reconcile/plan_builder_common.go | 8 +- .../reconcile/plan_builder_context.go | 3 +- .../reconcile/plan_builder_normal.go | 17 +-- pkg/deployment/reconcile/plan_builder_test.go | 6 + .../reconcile/plan_builder_utils.go | 19 --- pkg/deployment/resources/context.go | 15 ++- 23 files changed, 562 insertions(+), 156 deletions(-) create mode 100644 pkg/deployment/agency.go create mode 100644 pkg/deployment/agency/cache.go create mode 100644 pkg/deployment/agency/cache_test.go rename pkg/deployment/agency/{maintenance.go => config.go} (58%) create mode 100644 pkg/deployment/agency/config_test.go rename pkg/deployment/agency/{agency.go => current_collections.go} (58%) rename pkg/deployment/agency/{database.go => plan_collections.go} (57%) create mode 100644 pkg/deployment/agency/state.go create mode 100644 pkg/deployment/agency/state_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3450493a7..0cecf0ec1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A) - Add ArangoBackup backoff functionality - Allow to abort ArangoBackup uploads by removing spec.upload +- Add Agency Cache internally ## [1.2.5](https://github.com/arangodb/kube-arangodb/tree/1.2.5) (2021-10-25) - Split & Unify Lifecycle management functionality diff --git a/pkg/deployment/agency.go b/pkg/deployment/agency.go new file mode 100644 index 000000000..122eff10a --- /dev/null +++ b/pkg/deployment/agency.go @@ -0,0 +1,29 @@ +// +// DISCLAIMER +// +// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package deployment + +import "github.com/arangodb/kube-arangodb/pkg/metrics" + +var ( + inspectDeploymentAgencyIndex = metrics.MustRegisterGaugeVec(metricsComponent, "inspect_deployment_agency_index", "Index of the agency cache", metrics.DeploymentName) + inspectDeploymentAgencyFetches = metrics.MustRegisterCounterVec(metricsComponent, "inspect_deployment_agency_fetches", "Number of agency fetches", metrics.DeploymentName) + inspectDeploymentAgencyErrors = metrics.MustRegisterCounterVec(metricsComponent, "inspect_deployment_agency_errors", "Number of agency errors", metrics.DeploymentName) +) diff --git a/pkg/deployment/agency/cache.go b/pkg/deployment/agency/cache.go new file mode 100644 index 000000000..bb529a6d6 --- /dev/null +++ b/pkg/deployment/agency/cache.go @@ -0,0 +1,113 @@ +// +// DISCLAIMER +// +// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package agency + +import ( + "context" + "sync" + + "github.com/arangodb/go-driver/agency" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" +) + +type Cache interface { + Reload(ctx context.Context, client agency.Agency) (uint64, error) + Data() (State, bool) + CommitIndex() uint64 +} + +func NewCache(mode *api.DeploymentMode) Cache { + if mode.Get() == api.DeploymentModeSingle { + return NewSingleCache() + } + + return NewAgencyCache() +} + +func NewAgencyCache() Cache { + return &cache{} +} + +func NewSingleCache() Cache { + return &cacheSingle{} +} + +type cacheSingle struct { +} + +func (c cacheSingle) CommitIndex() uint64 { + return 0 +} + +func (c cacheSingle) Reload(ctx context.Context, client agency.Agency) (uint64, error) { + return 0, nil +} + +func (c cacheSingle) Data() (State, bool) { + return State{}, true +} + +type cache struct { + lock sync.Mutex + + valid bool + + commitIndex uint64 + + data State +} + +func (c *cache) CommitIndex() uint64 { + return c.commitIndex +} + +func (c *cache) Data() (State, bool) { + c.lock.Lock() + defer c.lock.Unlock() + + return c.data, c.valid +} + +func (c *cache) Reload(ctx context.Context, client agency.Agency) (uint64, error) { + c.lock.Lock() + defer c.lock.Unlock() + + c.valid = false + + cfg, err := getAgencyConfig(ctx, client) + if err != nil { + return cfg.CommitIndex, err + } + + if cfg.CommitIndex == c.commitIndex { + // We are on same index, nothing to do + return cfg.CommitIndex, err + } + + if data, err := loadState(ctx, client); err != nil { + return cfg.CommitIndex, err + } else { + c.data = data + c.valid = true + c.commitIndex = cfg.CommitIndex + return cfg.CommitIndex, err + } +} diff --git a/pkg/deployment/agency/cache_test.go b/pkg/deployment/agency/cache_test.go new file mode 100644 index 000000000..d56fab58d --- /dev/null +++ b/pkg/deployment/agency/cache_test.go @@ -0,0 +1,21 @@ +// +// DISCLAIMER +// +// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package agency diff --git a/pkg/deployment/agency/maintenance.go b/pkg/deployment/agency/config.go similarity index 58% rename from pkg/deployment/agency/maintenance.go rename to pkg/deployment/agency/config.go index 137718269..8ff9b30e8 100644 --- a/pkg/deployment/agency/maintenance.go +++ b/pkg/deployment/agency/config.go @@ -17,60 +17,52 @@ // // Copyright holder is ArangoDB GmbH, Cologne, Germany // -// Author Adam Janikowski -// package agency import ( "context" + "encoding/json" "net/http" - "github.com/arangodb/go-driver/agency" - "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/agency" ) -func GetMaintenanceMode(ctx context.Context, client agency.Agency) (bool, error) { - var data interface{} - err := client.ReadKey(ctx, []string{"arango", "Supervision", "Maintenance"}, &data) - - if err == nil { - // We got 200 - return true, nil - } +func getAgencyConfig(ctx context.Context, client agency.Agency) (*agencyConfig, error) { + conn := client.Connection() - if agency.IsKeyNotFound(err) { - return false, nil + req, err := client.Connection().NewRequest(http.MethodGet, "/_api/agency/config") + if err != nil { + return nil, err } - return false, err -} - -func SetMaintenanceMode(ctx context.Context, client driver.Client, enabled bool) error { - data := "on" - if !enabled { - data = "off" - } + var data []byte - conn := client.Connection() - r, err := conn.NewRequest(http.MethodPut, "/_admin/cluster/maintenance") + resp, err := conn.Do(driver.WithRawResponse(ctx, &data), req) if err != nil { - return err + return nil, err } - if _, err := r.SetBody(data); err != nil { - return err + if err := resp.CheckStatus(http.StatusOK); err != nil { + return nil, err } - resp, err := conn.Do(ctx, r) - if err != nil { - return err - } + var c agencyConfig - if err := resp.CheckStatus(http.StatusOK); err != nil { - return err + if err := json.Unmarshal(data, &c); err != nil { + return nil, err } - return nil + return &c, nil +} + +type agencyConfig struct { + LeaderId string `json:"leaderId"` + + CommitIndex uint64 `json:"commitIndex"` + + Configuration struct { + ID string `json:"id"` + } `json:"configuration"` } diff --git a/pkg/deployment/agency/config_test.go b/pkg/deployment/agency/config_test.go new file mode 100644 index 000000000..739d06764 --- /dev/null +++ b/pkg/deployment/agency/config_test.go @@ -0,0 +1,77 @@ +// +// DISCLAIMER +// +// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package agency + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_Config_Unmarshal(t *testing.T) { + data := `{ + "term": 0, + "leaderId": "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82", + "commitIndex": 94, + "lastCompactionAt": 0, + "nextCompactionAfter": 500, + "lastAcked": { + "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82": { + "lastAckedTime": 0, + "lastAckedIndex": 94 + } + }, + "configuration": { + "pool": { + "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82": "tcp://[::1]:4001" + }, + "active": [ + "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82" + ], + "id": "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82", + "agency size": 1, + "pool size": 1, + "endpoint": "tcp://[::1]:4001", + "min ping": 1, + "max ping": 5, + "timeoutMult": 1, + "supervision": true, + "supervision frequency": 1, + "compaction step size": 500, + "compaction keep size": 50000, + "supervision grace period": 10, + "supervision ok threshold": 5, + "version": 2, + "startup": "origin" + }, + "engine": "rocksdb", + "version": "3.10.0-devel" +}` + + var cfg agencyConfig + + require.NoError(t, json.Unmarshal([]byte(data), &cfg)) + + require.Equal(t, "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82", cfg.LeaderId) + require.Equal(t, uint64(94), cfg.CommitIndex) + require.Equal(t, "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82", cfg.Configuration.ID) +} diff --git a/pkg/deployment/agency/agency.go b/pkg/deployment/agency/current_collections.go similarity index 58% rename from pkg/deployment/agency/agency.go rename to pkg/deployment/agency/current_collections.go index 7f397d3d2..b02d0923f 100644 --- a/pkg/deployment/agency/agency.go +++ b/pkg/deployment/agency/current_collections.go @@ -17,31 +17,15 @@ // // Copyright holder is ArangoDB GmbH, Cologne, Germany // -// Author Adam Janikowski -// package agency -import ( - "context" - - "github.com/arangodb/kube-arangodb/pkg/util/errors" - - "github.com/arangodb/go-driver/agency" -) +type StateCurrentCollections map[string]StateCurrentDBCollections -type Fetcher func(ctx context.Context, i interface{}, keyParts ...string) error +type StateCurrentDBCollections map[string]StateCurrentDBCollection -func NewFetcher(a agency.Agency) Fetcher { - return func(ctx context.Context, i interface{}, keyParts ...string) error { - if err := a.ReadKey(ctx, []string{ - ArangoKey, - PlanKey, - PlanCollectionsKey, - }, i); err != nil { - return errors.WithStack(err) - } +type StateCurrentDBCollection map[string]StateCurrentDBShard - return nil - } +type StateCurrentDBShard struct { + Servers []string `json:"servers,omitempty"` } diff --git a/pkg/deployment/agency/definitions.go b/pkg/deployment/agency/definitions.go index 2d425931c..4622ce05e 100644 --- a/pkg/deployment/agency/definitions.go +++ b/pkg/deployment/agency/definitions.go @@ -22,8 +22,31 @@ package agency +import ( + "fmt" + "strings" +) + const ( - ArangoKey = "arango" - PlanKey = "Plan" + ArangoKey = "arango" + + PlanKey = "Plan" + CurrentKey = "Current" + PlanCollectionsKey = "Collections" + + SupervisionKey = "Supervision" + SupervisionMaintenanceKey = "Maintenance" ) + +func GetAgencyKey(parts ...string) string { + return fmt.Sprintf("/%s", strings.Join(parts, "/")) +} + +func GetAgencyReadKey(elements ...string) []string { + return elements +} + +func GetAgencyReadRequest(elements ...[]string) [][]string { + return elements +} diff --git a/pkg/deployment/agency/database.go b/pkg/deployment/agency/plan_collections.go similarity index 57% rename from pkg/deployment/agency/database.go rename to pkg/deployment/agency/plan_collections.go index e88c13b0b..12fa6809b 100644 --- a/pkg/deployment/agency/database.go +++ b/pkg/deployment/agency/plan_collections.go @@ -17,30 +17,12 @@ // // Copyright holder is ArangoDB GmbH, Cologne, Germany // -// Author Adam Janikowski -// package agency -import ( - "context" - - "github.com/arangodb/kube-arangodb/pkg/util/errors" -) - -func GetAgencyCollections(ctx context.Context, f Fetcher) (*ArangoPlanDatabases, error) { - ret := &ArangoPlanDatabases{} - - if err := f(ctx, ret, ArangoKey, PlanKey, PlanCollectionsKey); err != nil { - return nil, errors.WithStack(err) - } - - return ret, nil -} - -type ArangoPlanDatabases map[string]ArangoPlanCollections +type StatePlanCollections map[string]StatePlanDBCollections -func (a ArangoPlanDatabases) IsDBServerInDatabases(name string) bool { +func (a StatePlanCollections) IsDBServerInDatabases(name string) bool { for _, collections := range a { if collections.IsDBServerInCollections(name) { return true @@ -49,9 +31,9 @@ func (a ArangoPlanDatabases) IsDBServerInDatabases(name string) bool { return false } -type ArangoPlanCollections map[string]ArangoPlanCollection +type StatePlanDBCollections map[string]StatePlanCollection -func (a ArangoPlanCollections) IsDBServerInCollections(name string) bool { +func (a StatePlanDBCollections) IsDBServerInCollections(name string) bool { for _, collection := range a { if collection.IsDBServerInShards(name) { return true @@ -60,11 +42,11 @@ func (a ArangoPlanCollections) IsDBServerInCollections(name string) bool { return false } -type ArangoPlanCollection struct { - Shards ArangoPlanShard `json:"shards"` +type StatePlanCollection struct { + Shards StatePlanShard `json:"shards"` } -func (a ArangoPlanCollection) IsDBServerInShards(name string) bool { +func (a StatePlanCollection) IsDBServerInShards(name string) bool { for _, dbservers := range a.Shards { for _, dbserver := range dbservers { if dbserver == name { @@ -75,4 +57,4 @@ func (a ArangoPlanCollection) IsDBServerInShards(name string) bool { return false } -type ArangoPlanShard map[string][]string +type StatePlanShard map[string][]string diff --git a/pkg/deployment/agency/state.go b/pkg/deployment/agency/state.go new file mode 100644 index 000000000..16c39103c --- /dev/null +++ b/pkg/deployment/agency/state.go @@ -0,0 +1,117 @@ +// +// DISCLAIMER +// +// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package agency + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/agency" + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +func loadState(ctx context.Context, client agency.Agency) (State, error) { + conn := client.Connection() + + req, err := client.Connection().NewRequest(http.MethodPost, "/_api/agency/read") + if err != nil { + return State{}, err + } + + var data []byte + + req, err = req.SetBody(GetAgencyReadRequest(GetAgencyReadKey(GetAgencyKey(ArangoKey, SupervisionKey, SupervisionMaintenanceKey), GetAgencyKey(ArangoKey, PlanKey, PlanCollectionsKey), GetAgencyKey(ArangoKey, CurrentKey, PlanCollectionsKey)))) + if err != nil { + return State{}, err + } + + resp, err := conn.Do(driver.WithRawResponse(ctx, &data), req) + if err != nil { + return State{}, err + } + + if err := resp.CheckStatus(http.StatusOK); err != nil { + return State{}, err + } + + var c StateRoots + + if err := json.Unmarshal(data, &c); err != nil { + return State{}, err + } + + if len(c) != 1 { + return State{}, errors.Newf("Invalid response size") + } + + state := c[0].Arango + + if _, ok := state.Current.Collections["_system"]; !ok { + return State{}, errors.Newf("Unable to find system database (invalid data)") + } + + if _, ok := state.Plan.Collections["_system"]; !ok { + return State{}, errors.Newf("Unable to find system database (invalid data)") + } + + return state, nil +} + +type StateRoots []StateRoot + +type StateRoot struct { + Arango State `json:"arango"` +} + +type State struct { + Supervision StateSupervision `json:"Supervision"` + Plan StatePlan `json:"Plan"` + Current StateCurrent `json:"Current"` +} + +type StateCurrent struct { + Collections StateCurrentCollections `json:"Collections"` +} + +type StatePlan struct { + Collections StatePlanCollections `json:"Collections"` +} + +type StateSupervision struct { + Maintenance StateExists `json:"Maintenance,omitempty"` +} + +type StateExists bool + +func (d *StateExists) Exists() bool { + if d == nil { + return false + } + + return bool(*d) +} + +func (d *StateExists) UnmarshalJSON(bytes []byte) error { + *d = bytes != nil + return nil +} diff --git a/pkg/deployment/agency/state_test.go b/pkg/deployment/agency/state_test.go new file mode 100644 index 000000000..103c471c3 --- /dev/null +++ b/pkg/deployment/agency/state_test.go @@ -0,0 +1,37 @@ +// +// DISCLAIMER +// +// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package agency + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_Unmarshal_LongData(t *testing.T) { + data := "[{\"arango\":{\"Supervision\":{},\"Current\":{\"Collections\":{\"_system\":{\"10011\":{\"s10022\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010022\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":true,\"fields\":[\"mount\"],\"id\":\"10029\",\"name\":\"idx_1718347303809449984\",\"objectId\":\"2010164\",\"sparse\":true,\"type\":\"hash\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10005\":{\"s10016\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010038\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10027\",\"name\":\"idx_1718347303741292544\",\"objectId\":\"2010144\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10012\":{\"s10023\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010032\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10010\":{\"s10021\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010034\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"queue\",\"status\",\"delayUntil\"],\"id\":\"10030\",\"name\":\"idx_1718347303839858688\",\"objectId\":\"2010174\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"status\",\"queue\",\"delayUntil\"],\"id\":\"10031\",\"name\":\"idx_1718347303866073088\",\"objectId\":\"2010186\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10004\":{\"s10015\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010036\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10026\",\"name\":\"idx_1718347303708786688\",\"objectId\":\"2010134\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10003\":{\"s10014\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010028\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10006\":{\"s10017\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010030\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10028\",\"name\":\"idx_1718347303770652672\",\"objectId\":\"2010154\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10001\":{\"s10002\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010002\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":true,\"fields\":[\"user\"],\"id\":\"10025\",\"name\":\"idx_1718347303681523712\",\"objectId\":\"2010124\",\"sparse\":true,\"type\":\"hash\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10007\":{\"s10018\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010027\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10008\":{\"s10019\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010024\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10009\":{\"s10020\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010040\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10013\":{\"s10024\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010042\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}}}}},\"Plan\":{\"Collections\":{\"_system\":{\"10011\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10022\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_apps\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10011\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"mount\"],\"id\":\"10029\",\"inBackground\":false,\"name\":\"idx_1718347303809449984\",\"sparse\":true,\"type\":\"hash\",\"unique\":true}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10008\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10019\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_aqlfunctions\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10008\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10001\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"shardingStrategy\":\"hash\",\"shards\":{\"s10002\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"waitForSync\":false,\"schema\":null,\"shardKeys\":[\"_key\"],\"isDisjoint\":false,\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"user\"],\"id\":\"10025\",\"inBackground\":false,\"name\":\"idx_1718347303681523712\",\"sparse\":true,\"type\":\"hash\",\"unique\":true}],\"cacheEnabled\":false,\"deleted\":false,\"statusString\":\"loaded\",\"isSmart\":false,\"numberOfShards\":1,\"minReplicationFactor\":1,\"id\":\"10001\",\"name\":\"_users\",\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10007\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10018\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_analyzers\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10007\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10003\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10014\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_graphs\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10003\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10006\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10017\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_statisticsRaw\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10006\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10028\",\"inBackground\":false,\"name\":\"idx_1718347303770652672\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10012\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10023\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_appbundles\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10012\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10010\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10021\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_jobs\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10010\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"queue\",\"status\",\"delayUntil\"],\"id\":\"10030\",\"inBackground\":false,\"name\":\"idx_1718347303839858688\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"status\",\"queue\",\"delayUntil\"],\"id\":\"10031\",\"inBackground\":false,\"name\":\"idx_1718347303866073088\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10004\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10015\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_statistics\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10004\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10026\",\"inBackground\":false,\"name\":\"idx_1718347303708786688\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10005\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10016\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_statistics15\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10005\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10027\",\"inBackground\":false,\"name\":\"idx_1718347303741292544\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10009\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10020\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_queues\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10009\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10013\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10024\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_frontend\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10013\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}}}}}}}]" + var s StateRoots + + require.NoError(t, json.Unmarshal([]byte(data), &s)) + + t.Logf("%+v", s) +} diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 5b7f85817..edc9487ad 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -25,6 +25,7 @@ package deployment import ( "context" + "net/http" "sync" "sync/atomic" "time" @@ -128,6 +129,7 @@ type Deployment struct { updateDeploymentTrigger trigger.Trigger clientCache deploymentClient.Cache currentState inspectorInterface.Inspector + agencyCache agency.Cache recentInspectionErrors int clusterScalingIntegration *clusterScalingIntegration reconciler *reconcile.Reconciler @@ -138,24 +140,23 @@ type Deployment struct { haveServiceMonitorCRD bool } -func (d *Deployment) GetAgencyMaintenanceMode(ctx context.Context) (bool, error) { - if !d.Mode().HasAgents() { - return false, nil - } +func (d *Deployment) GetAgencyCache() (agency.State, bool) { + return d.agencyCache.Data() +} - ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) - defer cancel() +func (d *Deployment) RefreshAgencyCache(ctx context.Context) (uint64, error) { + lCtx, c := context.WithTimeout(ctx, time.Second) + defer c() - client, err := d.GetAgency(ctxChild) - if err != nil { - return false, err + if d.apiObject.Spec.Mode.Get() == api.DeploymentModeSingle { + return 0, nil } - if enabled, err := agency.GetMaintenanceMode(ctxChild, client); err != nil { - return false, err - } else { - return enabled, nil + a, err := d.GetAgency(lCtx) + if err != nil { + return 0, err } + return d.agencyCache.Reload(lCtx, a) } func (d *Deployment) SetAgencyMaintenanceMode(ctx context.Context, enabled bool) error { @@ -170,7 +171,31 @@ func (d *Deployment) SetAgencyMaintenanceMode(ctx context.Context, enabled bool) return err } - return agency.SetMaintenanceMode(ctxChild, client, enabled) + data := "on" + if !enabled { + data = "off" + } + + conn := client.Connection() + r, err := conn.NewRequest(http.MethodPut, "/_admin/cluster/maintenance") + if err != nil { + return err + } + + if _, err := r.SetBody(data); err != nil { + return err + } + + resp, err := conn.Do(ctx, r) + if err != nil { + return err + } + + if err := resp.CheckStatus(http.StatusOK); err != nil { + return err + } + + return nil } // New creates a new Deployment from the given API object. @@ -180,13 +205,14 @@ func New(config Config, deps Dependencies, apiObject *api.ArangoDeployment) (*De } d := &Deployment{ - apiObject: apiObject, - name: apiObject.GetName(), - namespace: apiObject.GetNamespace(), - config: config, - deps: deps, - eventCh: make(chan *deploymentEvent, deploymentEventQueueSize), - stopCh: make(chan struct{}), + apiObject: apiObject, + name: apiObject.GetName(), + namespace: apiObject.GetNamespace(), + config: config, + deps: deps, + eventCh: make(chan *deploymentEvent, deploymentEventQueueSize), + stopCh: make(chan struct{}), + agencyCache: agency.NewCache(apiObject.Spec.Mode), } d.clientCache = deploymentClient.NewClientCache(d.getArangoDeployment, conn.NewFactory(d.getAuth, d.getConnConfig)) diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 851d433f1..150ae5f9e 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -261,6 +261,14 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva nextInterval = interval } + inspectDeploymentAgencyFetches.WithLabelValues(d.GetName()).Inc() + if offset, err := d.RefreshAgencyCache(ctx); err != nil { + inspectDeploymentAgencyErrors.WithLabelValues(d.GetName()).Inc() + d.deps.Log.Err(err).Msgf("Unable to refresh agency") + } else { + inspectDeploymentAgencyIndex.WithLabelValues(d.GetName()).Set(float64(offset)) + } + // Refresh maintenance lock d.refreshMaintenanceTTL(ctx) diff --git a/pkg/deployment/reconcile/action_context.go b/pkg/deployment/reconcile/action_context.go index 59c10f8bc..b56ea51e9 100644 --- a/pkg/deployment/reconcile/action_context.go +++ b/pkg/deployment/reconcile/action_context.go @@ -26,6 +26,8 @@ package reconcile import ( "context" + agencyCache "github.com/arangodb/kube-arangodb/pkg/deployment/agency" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/arangomember" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/persistentvolumeclaim" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/pod" @@ -64,6 +66,7 @@ type ActionContext interface { resources.DeploymentPodRenderer resources.DeploymentModInterfaces resources.DeploymentCachedStatus + resources.ArangoAgencyGet // GetAPIObject returns the deployment as k8s object. GetAPIObject() k8sutil.APIObject @@ -171,6 +174,10 @@ type actionContext struct { cachedStatus inspectorInterface.Inspector } +func (ac *actionContext) GetAgencyCache() (agencyCache.State, bool) { + return ac.context.GetAgencyCache() +} + func (ac *actionContext) RenderPodForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.Pod, error) { return ac.context.RenderPodForMemberFromCurrent(ctx, cachedStatus, memberID) } @@ -179,10 +186,6 @@ func (ac *actionContext) RenderPodTemplateForMemberFromCurrent(ctx context.Conte return ac.context.RenderPodTemplateForMemberFromCurrent(ctx, cachedStatus, memberID) } -func (ac *actionContext) GetAgencyMaintenanceMode(ctx context.Context) (bool, error) { - return ac.context.GetAgencyMaintenanceMode(ctx) -} - func (ac *actionContext) SetAgencyMaintenanceMode(ctx context.Context, enabled bool) error { return ac.context.SetAgencyMaintenanceMode(ctx, enabled) } diff --git a/pkg/deployment/reconcile/action_maintenance_condition.go b/pkg/deployment/reconcile/action_maintenance_condition.go index 642884d95..28fabe7d3 100644 --- a/pkg/deployment/reconcile/action_maintenance_condition.go +++ b/pkg/deployment/reconcile/action_maintenance_condition.go @@ -56,13 +56,13 @@ func (a *actionSetMaintenanceCondition) Start(ctx context.Context) (bool, error) return true, nil } - if maintenance, err := a.actionCtx.GetAgencyMaintenanceMode(ctx); err != nil { - a.log.Error().Err(err).Msgf("Unable to set maintenance condition") - return true, nil + agencyState, agencyOK := a.actionCtx.GetAgencyCache() + if !agencyOK { + a.log.Error().Msgf("Unable to determine maintenance condition") } else { if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { - if maintenance { + if agencyState.Supervision.Maintenance { return s.Conditions.Update(api.ConditionTypeMaintenanceMode, true, "Maintenance", "Maintenance enabled") } else { return s.Conditions.Remove(api.ConditionTypeMaintenanceMode) diff --git a/pkg/deployment/reconcile/action_resign_leadership.go b/pkg/deployment/reconcile/action_resign_leadership.go index 504c1fd6f..03a9acc9e 100644 --- a/pkg/deployment/reconcile/action_resign_leadership.go +++ b/pkg/deployment/reconcile/action_resign_leadership.go @@ -78,10 +78,10 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) { switch group { case api.ServerGroupDBServers: - if enabled, err := a.actionCtx.GetAgencyMaintenanceMode(ctx); err != nil { + if agencyState, agencyOK := a.actionCtx.GetAgencyCache(); !agencyOK { log.Warn().Err(err).Msgf("Maintenance is enabled, skipping action") return true, errors.WithStack(err) - } else if enabled { + } else if agencyState.Supervision.Maintenance { // We are done, action cannot be handled on maintenance mode log.Warn().Msgf("Maintenance is enabled, skipping action") return true, nil @@ -127,10 +127,10 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool, return true, false, nil } - if enabled, err := a.actionCtx.GetAgencyMaintenanceMode(ctx); err != nil { - log.Error().Err(err).Msgf("Unable to get maintenance mode") + if agencyState, agencyOK := a.actionCtx.GetAgencyCache(); !agencyOK { + log.Error().Msgf("Unable to get maintenance mode") return false, false, nil - } else if enabled { + } else if agencyState.Supervision.Maintenance { log.Warn().Msgf("Maintenance is enabled, skipping action") // We are done, action cannot be handled on maintenance mode m.CleanoutJobID = "" diff --git a/pkg/deployment/reconcile/context.go b/pkg/deployment/reconcile/context.go index 13679f41b..010c23fce 100644 --- a/pkg/deployment/reconcile/context.go +++ b/pkg/deployment/reconcile/context.go @@ -50,6 +50,7 @@ type Context interface { resources.DeploymentImageManager resources.DeploymentModInterfaces resources.DeploymentCachedStatus + resources.ArangoAgencyGet // GetAPIObject returns the deployment as k8s object. GetAPIObject() k8sutil.APIObject @@ -116,8 +117,6 @@ type Context interface { DisableScalingCluster(ctx context.Context) error // EnableScalingCluster enables scaling DBservers and coordinators EnableScalingCluster(ctx context.Context) error - // GetAgencyData object for key path - GetAgencyData(ctx context.Context, i interface{}, keyParts ...string) error // GetBackup receives information about a backup resource GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) // GetName receives deployment name diff --git a/pkg/deployment/reconcile/plan_builder_common.go b/pkg/deployment/reconcile/plan_builder_common.go index e2a17f1ba..a4c625b42 100644 --- a/pkg/deployment/reconcile/plan_builder_common.go +++ b/pkg/deployment/reconcile/plan_builder_common.go @@ -46,12 +46,14 @@ func createMaintenanceManagementPlan(ctx context.Context, return nil } - enabled, err := planCtx.GetAgencyMaintenanceMode(ctx) - if err != nil { - log.Error().Err(err).Msgf("Unable to get agency mode") + agencyState, agencyOK := planCtx.GetAgencyCache() + if !agencyOK { + log.Error().Msgf("Unable to get agency mode") return nil } + enabled := agencyState.Supervision.Maintenance.Exists() + if !enabled && spec.Database.GetMaintenance() { log.Info().Msgf("Enabling maintenance mode") return api.Plan{api.NewAction(api.ActionTypeEnableMaintenance, api.ServerGroupUnknown, ""), api.NewAction(api.ActionTypeSetMaintenanceCondition, api.ServerGroupUnknown, "")} diff --git a/pkg/deployment/reconcile/plan_builder_context.go b/pkg/deployment/reconcile/plan_builder_context.go index 40c7512a9..4f85f73ba 100644 --- a/pkg/deployment/reconcile/plan_builder_context.go +++ b/pkg/deployment/reconcile/plan_builder_context.go @@ -48,6 +48,7 @@ type PlanBuilderContext interface { resources.DeploymentImageManager resources.DeploymentModInterfaces resources.DeploymentCachedStatus + resources.ArangoAgencyGet // GetTLSKeyfile returns the keyfile encoded TLS certificate+key for // the given member. @@ -65,8 +66,6 @@ type PlanBuilderContext interface { GetStatus() (api.DeploymentStatus, int32) // GetStatus returns the current spec of the deployment GetSpec() api.DeploymentSpec - // GetAgencyData object for key path - GetAgencyData(ctx context.Context, i interface{}, keyParts ...string) error // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), // creating one if needed. GetDatabaseClient(ctx context.Context) (driver.Client, error) diff --git a/pkg/deployment/reconcile/plan_builder_normal.go b/pkg/deployment/reconcile/plan_builder_normal.go index ffe0fe365..8485fa7d4 100644 --- a/pkg/deployment/reconcile/plan_builder_normal.go +++ b/pkg/deployment/reconcile/plan_builder_normal.go @@ -135,7 +135,7 @@ func createMemberFailedRestorePlan(ctx context.Context, var plan api.Plan // Fetch agency plan - agencyPlan, agencyErr := fetchAgency(ctx, spec, status, context) + agencyState, agencyOK := context.GetAgencyCache() // Check for members in failed state status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { @@ -148,17 +148,12 @@ func createMemberFailedRestorePlan(ctx context.Context, if group == api.ServerGroupDBServers && spec.GetMode() == api.DeploymentModeCluster { // Do pre check for DBServers. If agency is down DBServers should not be touch - if agencyErr != nil { - memberLog.Msg("Error in agency") + if !agencyOK { + memberLog.Msg("Agency state is not present") continue } - if agencyPlan == nil { - memberLog.Msg("AgencyPlan is nil") - continue - } - - if agencyPlan.IsDBServerInDatabases(m.ID) { + if agencyState.Plan.Collections.IsDBServerInDatabases(m.ID) { // DBServer still exists in agency plan! Will not be removed, but needs to be recreated memberLog.Msg("Recreating DBServer - it cannot be removed gracefully") plan = append(plan, @@ -199,8 +194,8 @@ func createMemberFailedRestorePlan(ctx context.Context, }) // Ensure that we were able to get agency info - if len(plan) == 0 && agencyErr != nil { - log.Err(agencyErr).Msg("unable to build further plan without access to agency") + if len(plan) == 0 && !agencyOK { + log.Warn().Msgf("unable to build further plan without access to agency") plan = append(plan, api.NewAction(api.ActionTypeIdle, api.ServerGroupUnknown, "")) } diff --git a/pkg/deployment/reconcile/plan_builder_test.go b/pkg/deployment/reconcile/plan_builder_test.go index 52d876085..3fd10f2da 100644 --- a/pkg/deployment/reconcile/plan_builder_test.go +++ b/pkg/deployment/reconcile/plan_builder_test.go @@ -29,6 +29,8 @@ import ( "io/ioutil" "testing" + agencyCache "github.com/arangodb/kube-arangodb/pkg/deployment/agency" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/arangomember" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/persistentvolumeclaim" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/pod" @@ -89,6 +91,10 @@ type testContext struct { RecordedEvent *k8sutil.Event } +func (c *testContext) GetAgencyCache() (agencyCache.State, bool) { + return agencyCache.State{}, true +} + func (c *testContext) SecretsModInterface() secret.ModInterface { panic("implement me") } diff --git a/pkg/deployment/reconcile/plan_builder_utils.go b/pkg/deployment/reconcile/plan_builder_utils.go index 009d838e9..3a69470fd 100644 --- a/pkg/deployment/reconcile/plan_builder_utils.go +++ b/pkg/deployment/reconcile/plan_builder_utils.go @@ -23,12 +23,7 @@ package reconcile import ( - "context" - "time" - api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" - "github.com/arangodb/kube-arangodb/pkg/deployment/agency" - "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/rs/zerolog" ) @@ -51,17 +46,3 @@ func createRotateMemberPlan(log zerolog.Logger, member api.MemberStatus, } return plan } - -func fetchAgency(ctx context.Context, spec api.DeploymentSpec, status api.DeploymentStatus, - pctx PlanBuilderContext) (*agency.ArangoPlanDatabases, error) { - if spec.GetMode() != api.DeploymentModeCluster && spec.GetMode() != api.DeploymentModeActiveFailover { - return nil, nil - } else if status.Members.Agents.MembersReady() > 0 { - agencyCtx, agencyCancel := context.WithTimeout(ctx, time.Minute) - defer agencyCancel() - - return agency.GetAgencyCollections(agencyCtx, pctx.GetAgencyData) - } else { - return nil, errors.Newf("not able to read from agency when agency is down") - } -} diff --git a/pkg/deployment/resources/context.go b/pkg/deployment/resources/context.go index 106ac5ea8..d71cc587a 100644 --- a/pkg/deployment/resources/context.go +++ b/pkg/deployment/resources/context.go @@ -26,6 +26,8 @@ package resources import ( "context" + agencyCache "github.com/arangodb/kube-arangodb/pkg/deployment/agency" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/arangomember" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/persistentvolumeclaim" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/pod" @@ -68,8 +70,6 @@ type DeploymentStatusUpdate interface { } type DeploymentAgencyMaintenance interface { - // GetAgencyMaintenanceMode returns info if maintenance mode is enabled - GetAgencyMaintenanceMode(ctx context.Context) (bool, error) // SetAgencyMaintenanceMode set maintenance mode info SetAgencyMaintenanceMode(ctx context.Context, enabled bool) error } @@ -128,6 +128,16 @@ type ArangoMemberContext interface { WithArangoMemberStatusUpdate(ctx context.Context, namespace, name string, action ArangoMemberStatusUpdateFunc) error } +type ArangoAgencyGet interface { + GetAgencyCache() (agencyCache.State, bool) +} + +type ArangoAgency interface { + ArangoAgencyGet + + RefreshAgencyCache(ctx context.Context) (uint64, error) +} + // Context provides all functions needed by the Resources service // to perform its service. type Context interface { @@ -137,6 +147,7 @@ type Context interface { DeploymentImageManager DeploymentModInterfaces DeploymentCachedStatus + ArangoAgency // GetAPIObject returns the deployment as k8s object. GetAPIObject() k8sutil.APIObject