From 1d8b2da746314d364f089bc7fea06929de7a9cf7 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Tue, 28 Jun 2022 21:29:34 +0000 Subject: [PATCH 1/2] [Bugfix] Add DistributeShardsLike support --- CHANGELOG.md | 1 + docs/generated/metrics/README.md | 29 +- .../arangodb_operator_rebalancer_enabled.md | 12 + ...ngodb_operator_rebalancer_moves_current.md | 12 + ...angodb_operator_rebalancer_moves_failed.md | 12 + ...odb_operator_rebalancer_moves_generated.md | 12 + ...odb_operator_rebalancer_moves_succeeded.md | 12 + go.mod | 1 - internal/metrics.yaml | 46 + pkg/apis/deployment/v1/plan.go | 1 + pkg/apis/deployment/v2alpha1/plan.go | 1 + pkg/deployment/agency/cache.go | 4 +- pkg/deployment/agency/config_test.go | 45 +- pkg/deployment/agency/plan_collections.go | 1 + pkg/deployment/agency/rf.go | 4 + pkg/deployment/agency/state_test.go | 37 +- pkg/deployment/agency/testdata/config.json | 38 + pkg/deployment/agency/testdata/longdata.json | 1120 +++++++++++++++++ pkg/deployment/reconcile/action_context.go | 10 +- pkg/deployment/reconcile/metrics.go | 93 ++ pkg/deployment/reconcile/plan_executor.go | 95 +- pkg/deployment/reconcile/reconciler.go | 2 + .../arangodb_operator_rebalancer_enabled.go} | 16 +- ...ngodb_operator_rebalancer_moves_current.go | 35 + ...angodb_operator_rebalancer_moves_failed.go | 35 + ...odb_operator_rebalancer_moves_generated.go | 35 + ...odb_operator_rebalancer_moves_succeeded.go | 35 + 27 files changed, 1657 insertions(+), 87 deletions(-) create mode 100644 docs/generated/metrics/arangodb_operator_rebalancer_enabled.md create mode 100644 docs/generated/metrics/arangodb_operator_rebalancer_moves_current.md create mode 100644 docs/generated/metrics/arangodb_operator_rebalancer_moves_failed.md create mode 100644 docs/generated/metrics/arangodb_operator_rebalancer_moves_generated.md create mode 100644 docs/generated/metrics/arangodb_operator_rebalancer_moves_succeeded.md create mode 100644 pkg/deployment/agency/testdata/config.json create mode 100644 pkg/deployment/agency/testdata/longdata.json create mode 100644 pkg/deployment/reconcile/metrics.go rename pkg/{deployment/agency/cache_test.go => generated/metric_descriptions/arangodb_operator_rebalancer_enabled.go} (60%) create mode 100644 pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_current.go create mode 100644 pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_failed.go create mode 100644 pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_generated.go create mode 100644 pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_succeeded.go diff --git a/CHANGELOG.md b/CHANGELOG.md index aacbf6c99..8bb660f4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - (Documentation) Refactor metrics (Part 1) - (Bugfix) Extend Agency HealthCheck for replace - (Bugfix) Allow to remove resources (CPU & Memory) on the managed pods +- (Bugfix) Add DistributeShardsLike support ## [1.2.13](https://github.com/arangodb/kube-arangodb/tree/1.2.13) (2022-06-07) - (Bugfix) Fix arangosync members state inspection diff --git a/docs/generated/metrics/README.md b/docs/generated/metrics/README.md index 7e03e65d4..5d8065e18 100644 --- a/docs/generated/metrics/README.md +++ b/docs/generated/metrics/README.md @@ -2,15 +2,20 @@ ## List -| Name | Namespace | Group | Type | Description | -|:---------------------------------------------------------------------------------------------------------------:|:-----------------:|:------------:|:-----:|:---------------------------------------------------| -| [arangodb_operator_agency_errors](./arangodb_operator_agency_errors.md) | arangodb_operator | agency | Count | Current count of agency cache fetch errors | -| [arangodb_operator_agency_fetches](./arangodb_operator_agency_fetches.md) | arangodb_operator | agency | Count | Current count of agency cache fetches | -| [arangodb_operator_agency_index](./arangodb_operator_agency_index.md) | arangodb_operator | agency | Gauge | Current index of the agency cache | -| [arangodb_operator_agency_cache_health_present](./arangodb_operator_agency_cache_health_present.md) | arangodb_operator | agency_cache | Gauge | Determines if local agency cache health is present | -| [arangodb_operator_agency_cache_healthy](./arangodb_operator_agency_cache_healthy.md) | arangodb_operator | agency_cache | Gauge | Determines if agency is healthy | -| [arangodb_operator_agency_cache_leaders](./arangodb_operator_agency_cache_leaders.md) | arangodb_operator | agency_cache | Gauge | Determines agency leader vote count | -| [arangodb_operator_agency_cache_member_commit_offset](./arangodb_operator_agency_cache_member_commit_offset.md) | arangodb_operator | agency_cache | Gauge | Determines agency member commit offset | -| [arangodb_operator_agency_cache_member_serving](./arangodb_operator_agency_cache_member_serving.md) | arangodb_operator | agency_cache | Gauge | Determines if agency member is reachable | -| [arangodb_operator_agency_cache_present](./arangodb_operator_agency_cache_present.md) | arangodb_operator | agency_cache | Gauge | Determines if local agency cache is present | -| [arangodb_operator_agency_cache_serving](./arangodb_operator_agency_cache_serving.md) | arangodb_operator | agency_cache | Gauge | Determines if agency is serving | +| Name | Namespace | Group | Type | Description | +|:---------------------------------------------------------------------------------------------------------------:|:-----------------:|:------------:|:-------:|:---------------------------------------------------| +| [arangodb_operator_agency_errors](./arangodb_operator_agency_errors.md) | arangodb_operator | agency | Count | Current count of agency cache fetch errors | +| [arangodb_operator_agency_fetches](./arangodb_operator_agency_fetches.md) | arangodb_operator | agency | Count | Current count of agency cache fetches | +| [arangodb_operator_agency_index](./arangodb_operator_agency_index.md) | arangodb_operator | agency | Gauge | Current index of the agency cache | +| [arangodb_operator_agency_cache_health_present](./arangodb_operator_agency_cache_health_present.md) | arangodb_operator | agency_cache | Gauge | Determines if local agency cache health is present | +| [arangodb_operator_agency_cache_healthy](./arangodb_operator_agency_cache_healthy.md) | arangodb_operator | agency_cache | Gauge | Determines if agency is healthy | +| [arangodb_operator_agency_cache_leaders](./arangodb_operator_agency_cache_leaders.md) | arangodb_operator | agency_cache | Gauge | Determines agency leader vote count | +| [arangodb_operator_agency_cache_member_commit_offset](./arangodb_operator_agency_cache_member_commit_offset.md) | arangodb_operator | agency_cache | Gauge | Determines agency member commit offset | +| [arangodb_operator_agency_cache_member_serving](./arangodb_operator_agency_cache_member_serving.md) | arangodb_operator | agency_cache | Gauge | Determines if agency member is reachable | +| [arangodb_operator_agency_cache_present](./arangodb_operator_agency_cache_present.md) | arangodb_operator | agency_cache | Gauge | Determines if local agency cache is present | +| [arangodb_operator_agency_cache_serving](./arangodb_operator_agency_cache_serving.md) | arangodb_operator | agency_cache | Gauge | Determines if agency is serving | +| [arangodb_operator_rebalancer_enabled](./arangodb_operator_rebalancer_enabled.md) | arangodb_operator | rebalancer | Gauge | Determines if rebalancer is enabled | +| [arangodb_operator_rebalancer_moves_current](./arangodb_operator_rebalancer_moves_current.md) | arangodb_operator | rebalancer | Gauge | Define how many moves are currently in progress | +| [arangodb_operator_rebalancer_moves_failed](./arangodb_operator_rebalancer_moves_failed.md) | arangodb_operator | rebalancer | Counter | Define how many moves failed | +| [arangodb_operator_rebalancer_moves_generated](./arangodb_operator_rebalancer_moves_generated.md) | arangodb_operator | rebalancer | Counter | Define how many moves were generated | +| [arangodb_operator_rebalancer_moves_succeeded](./arangodb_operator_rebalancer_moves_succeeded.md) | arangodb_operator | rebalancer | Counter | Define how many moves succeeded | diff --git a/docs/generated/metrics/arangodb_operator_rebalancer_enabled.md b/docs/generated/metrics/arangodb_operator_rebalancer_enabled.md new file mode 100644 index 000000000..fdc513a7a --- /dev/null +++ b/docs/generated/metrics/arangodb_operator_rebalancer_enabled.md @@ -0,0 +1,12 @@ +# arangodb_operator_rebalancer_enabled (Gauge) + +## Description + +Determines if rebalancer is enabled + +## Labels + +| Label | Description | +|:---------:|:---------------------| +| namespace | Deployment Namespace | +| name | Deployment Name | diff --git a/docs/generated/metrics/arangodb_operator_rebalancer_moves_current.md b/docs/generated/metrics/arangodb_operator_rebalancer_moves_current.md new file mode 100644 index 000000000..ab1c67038 --- /dev/null +++ b/docs/generated/metrics/arangodb_operator_rebalancer_moves_current.md @@ -0,0 +1,12 @@ +# arangodb_operator_rebalancer_moves_current (Gauge) + +## Description + +Define how many moves are currently in progress + +## Labels + +| Label | Description | +|:---------:|:---------------------| +| namespace | Deployment Namespace | +| name | Deployment Name | diff --git a/docs/generated/metrics/arangodb_operator_rebalancer_moves_failed.md b/docs/generated/metrics/arangodb_operator_rebalancer_moves_failed.md new file mode 100644 index 000000000..22ace44e2 --- /dev/null +++ b/docs/generated/metrics/arangodb_operator_rebalancer_moves_failed.md @@ -0,0 +1,12 @@ +# arangodb_operator_rebalancer_moves_failed (Counter) + +## Description + +Define how many moves failed + +## Labels + +| Label | Description | +|:---------:|:---------------------| +| namespace | Deployment Namespace | +| name | Deployment Name | diff --git a/docs/generated/metrics/arangodb_operator_rebalancer_moves_generated.md b/docs/generated/metrics/arangodb_operator_rebalancer_moves_generated.md new file mode 100644 index 000000000..9ae74acd6 --- /dev/null +++ b/docs/generated/metrics/arangodb_operator_rebalancer_moves_generated.md @@ -0,0 +1,12 @@ +# arangodb_operator_rebalancer_moves_generated (Counter) + +## Description + +Define how many moves were generated + +## Labels + +| Label | Description | +|:---------:|:---------------------| +| namespace | Deployment Namespace | +| name | Deployment Name | diff --git a/docs/generated/metrics/arangodb_operator_rebalancer_moves_succeeded.md b/docs/generated/metrics/arangodb_operator_rebalancer_moves_succeeded.md new file mode 100644 index 000000000..c4ee3c5fc --- /dev/null +++ b/docs/generated/metrics/arangodb_operator_rebalancer_moves_succeeded.md @@ -0,0 +1,12 @@ +# arangodb_operator_rebalancer_moves_succeeded (Counter) + +## Description + +Define how many moves succeeded + +## Labels + +| Label | Description | +|:---------:|:---------------------| +| namespace | Deployment Namespace | +| name | Deployment Name | diff --git a/go.mod b/go.mod index e29b4d2c3..cb70a7245 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( github.com/arangodb/go-driver v1.2.1 github.com/arangodb/go-driver/v2 v2.0.0-20211021031401-d92dcd5a4c83 github.com/arangodb/go-upgrade-rules v0.0.0-20180809110947-031b4774ff21 - github.com/arangodb/rebalancer v0.1.1 github.com/cenkalti/backoff v2.2.1+incompatible github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 github.com/gin-gonic/gin v1.7.2 diff --git a/internal/metrics.yaml b/internal/metrics.yaml index c0a985ad1..e66c62c77 100644 --- a/internal/metrics.yaml +++ b/internal/metrics.yaml @@ -98,6 +98,52 @@ namespaces: shortDescription: "Current count of agency cache fetch errors" description: "Current count of agency cache fetch errors" type: "Count" + labels: + - key: namespace + description: "Deployment Namespace" + - key: name + description: "Deployment Name" + rebalancer: + enabled: + shortDescription: "Determines if rebalancer is enabled" + description: "Determines if rebalancer is enabled" + type: "Gauge" + labels: + - key: namespace + description: "Deployment Namespace" + - key: name + description: "Deployment Name" + moves_generated: + shortDescription: "Define how many moves were generated" + description: "Define how many moves were generated" + type: "Counter" + labels: + - key: namespace + description: "Deployment Namespace" + - key: name + description: "Deployment Name" + moves_succeeded: + shortDescription: "Define how many moves succeeded" + description: "Define how many moves succeeded" + type: "Counter" + labels: + - key: namespace + description: "Deployment Namespace" + - key: name + description: "Deployment Name" + moves_failed: + shortDescription: "Define how many moves failed" + description: "Define how many moves failed" + type: "Counter" + labels: + - key: namespace + description: "Deployment Namespace" + - key: name + description: "Deployment Name" + moves_current: + shortDescription: "Define how many moves are currently in progress" + description: "Define how many moves are currently in progress" + type: "Gauge" labels: - key: namespace description: "Deployment Namespace" diff --git a/pkg/apis/deployment/v1/plan.go b/pkg/apis/deployment/v1/plan.go index 1d09b37b9..446f0d3db 100644 --- a/pkg/apis/deployment/v1/plan.go +++ b/pkg/apis/deployment/v1/plan.go @@ -192,6 +192,7 @@ const ( // Rebalancer ActionTypeRebalancerGenerate ActionType = "RebalancerGenerate" ActionTypeRebalancerCheck ActionType = "RebalancerCheck" + ActionTypeRebalancerClean ActionType = "RebalancerClean" // Resources ActionTypeResourceSync ActionType = "ResourceSync" diff --git a/pkg/apis/deployment/v2alpha1/plan.go b/pkg/apis/deployment/v2alpha1/plan.go index 527bd8aac..1b2ae03ab 100644 --- a/pkg/apis/deployment/v2alpha1/plan.go +++ b/pkg/apis/deployment/v2alpha1/plan.go @@ -192,6 +192,7 @@ const ( // Rebalancer ActionTypeRebalancerGenerate ActionType = "RebalancerGenerate" ActionTypeRebalancerCheck ActionType = "RebalancerCheck" + ActionTypeRebalancerClean ActionType = "RebalancerClean" // Resources ActionTypeResourceSync ActionType = "ResourceSync" diff --git a/pkg/deployment/agency/cache.go b/pkg/deployment/agency/cache.go index 029ec4581..bc317f8e5 100644 --- a/pkg/deployment/agency/cache.go +++ b/pkg/deployment/agency/cache.go @@ -333,7 +333,9 @@ func getLeader(ctx context.Context, size int, clients map[string]agency.Agency) for id := range names { if h.leaderID == h.names[id] { h.leader = clients[names[id]].Connection() - return clients[names[id]], configs[id], h, nil + if cfg := configs[id]; cfg != nil { + return clients[names[id]], cfg, h, nil + } } } diff --git a/pkg/deployment/agency/config_test.go b/pkg/deployment/agency/config_test.go index f8c8be23d..f8d29bb91 100644 --- a/pkg/deployment/agency/config_test.go +++ b/pkg/deployment/agency/config_test.go @@ -21,55 +21,20 @@ package agency import ( + _ "embed" "encoding/json" "testing" "github.com/stretchr/testify/require" ) -func Test_Config_Unmarshal(t *testing.T) { - data := `{ - "term": 0, - "leaderId": "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82", - "commitIndex": 94, - "lastCompactionAt": 0, - "nextCompactionAfter": 500, - "lastAcked": { - "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82": { - "lastAckedTime": 0, - "lastAckedIndex": 94 - } - }, - "configuration": { - "pool": { - "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82": "tcp://[::1]:4001" - }, - "active": [ - "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82" - ], - "id": "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82", - "agency size": 1, - "pool size": 1, - "endpoint": "tcp://[::1]:4001", - "min ping": 1, - "max ping": 5, - "timeoutMult": 1, - "supervision": true, - "supervision frequency": 1, - "compaction step size": 500, - "compaction keep size": 50000, - "supervision grace period": 10, - "supervision ok threshold": 5, - "version": 2, - "startup": "origin" - }, - "engine": "rocksdb", - "version": "3.10.0-devel" -}` +//go:embed testdata/config.json +var config []byte +func Test_Config_Unmarshal(t *testing.T) { var cfg Config - require.NoError(t, json.Unmarshal([]byte(data), &cfg)) + require.NoError(t, json.Unmarshal(config, &cfg)) require.Equal(t, "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82", cfg.LeaderId) require.Equal(t, uint64(94), cfg.CommitIndex) diff --git a/pkg/deployment/agency/plan_collections.go b/pkg/deployment/agency/plan_collections.go index a044e29c3..c7e63e0ac 100644 --- a/pkg/deployment/agency/plan_collections.go +++ b/pkg/deployment/agency/plan_collections.go @@ -60,6 +60,7 @@ type StatePlanCollection struct { MinReplicationFactor *int `json:"minReplicationFactor,omitempty"` WriteConcern *int `json:"writeConcern,omitempty"` ReplicationFactor *ReplicationFactor `json:"replicationFactor,omitempty"` + DistributeShardsLike *string `json:"distributeShardsLike,omitempty"` } func (a *StatePlanCollection) GetReplicationFactor(shard string) ReplicationFactor { diff --git a/pkg/deployment/agency/rf.go b/pkg/deployment/agency/rf.go index a0492fde4..97bb346e8 100644 --- a/pkg/deployment/agency/rf.go +++ b/pkg/deployment/agency/rf.go @@ -34,6 +34,10 @@ const ( type ReplicationFactor int +func (r *ReplicationFactor) IsNil() bool { + return r == nil +} + func (r *ReplicationFactor) IsUnknown() bool { if r == nil { return false diff --git a/pkg/deployment/agency/state_test.go b/pkg/deployment/agency/state_test.go index 23b77c0e5..2a103f56c 100644 --- a/pkg/deployment/agency/state_test.go +++ b/pkg/deployment/agency/state_test.go @@ -49,6 +49,9 @@ var agencyDump39HotBackup []byte //go:embed testdata/agency_dump.3.9.jobs.json var agencyDump39Jobs []byte +//go:embed testdata/longdata.json +var longData []byte + var ( data = map[string][]byte{ "3.6": agencyDump36, @@ -65,6 +68,37 @@ func Test_Unmarshal_MultiVersion(t *testing.T) { t.Run(v, func(t *testing.T) { var s DumpState require.NoError(t, json.Unmarshal(data, &s)) + + t.Run("Ensure Names", func(t *testing.T) { + for _, collections := range s.Agency.Arango.Plan.Collections { + for _, collection := range collections { + require.NotNil(t, collection.Name) + } + } + }) + + t.Run("Ensure distributeShardsLike", func(t *testing.T) { + collections, ok := s.Agency.Arango.Plan.Collections["_system"] + require.True(t, ok) + + for _, collection := range collections { + name := *collection.Name + t.Run(name, func(t *testing.T) { + if name == "_users" { + require.Nil(t, collection.DistributeShardsLike) + return + } + require.NotNil(t, collection.DistributeShardsLike) + + n, ok := collections[*collection.DistributeShardsLike] + require.True(t, ok) + + require.NotNil(t, n.Name) + require.Nil(t, n.DistributeShardsLike) + require.Equal(t, "_users", *n.Name) + }) + } + }) }) } } @@ -108,10 +142,9 @@ func Test_Unmarshal_Jobs(t *testing.T) { } func Test_Unmarshal_LongData(t *testing.T) { - data := "[{\"arango\":{\"Supervision\":{},\"Current\":{\"Collections\":{\"_system\":{\"10011\":{\"s10022\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010022\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":true,\"fields\":[\"mount\"],\"id\":\"10029\",\"name\":\"idx_1718347303809449984\",\"objectId\":\"2010164\",\"sparse\":true,\"type\":\"hash\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10005\":{\"s10016\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010038\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10027\",\"name\":\"idx_1718347303741292544\",\"objectId\":\"2010144\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10012\":{\"s10023\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010032\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10010\":{\"s10021\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010034\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"queue\",\"status\",\"delayUntil\"],\"id\":\"10030\",\"name\":\"idx_1718347303839858688\",\"objectId\":\"2010174\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"status\",\"queue\",\"delayUntil\"],\"id\":\"10031\",\"name\":\"idx_1718347303866073088\",\"objectId\":\"2010186\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10004\":{\"s10015\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010036\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10026\",\"name\":\"idx_1718347303708786688\",\"objectId\":\"2010134\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10003\":{\"s10014\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010028\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10006\":{\"s10017\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010030\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10028\",\"name\":\"idx_1718347303770652672\",\"objectId\":\"2010154\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10001\":{\"s10002\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010002\",\"sparse\":false,\"type\":\"primary\",\"unique\":true},{\"deduplicate\":true,\"estimates\":true,\"fields\":[\"user\"],\"id\":\"10025\",\"name\":\"idx_1718347303681523712\",\"objectId\":\"2010124\",\"sparse\":true,\"type\":\"hash\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10007\":{\"s10018\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010027\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10008\":{\"s10019\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010024\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10009\":{\"s10020\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010040\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}},\"10013\":{\"s10024\":{\"failoverCandidates\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"],\"errorNum\":0,\"errorMessage\":\"\",\"error\":false,\"indexes\":[{\"fields\":[\"_key\"],\"id\":\"0\",\"name\":\"primary\",\"objectId\":\"2010042\",\"sparse\":false,\"type\":\"primary\",\"unique\":true}],\"servers\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]}}}}},\"Plan\":{\"Collections\":{\"_system\":{\"10011\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10022\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_apps\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10011\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"mount\"],\"id\":\"10029\",\"inBackground\":false,\"name\":\"idx_1718347303809449984\",\"sparse\":true,\"type\":\"hash\",\"unique\":true}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10008\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10019\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_aqlfunctions\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10008\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10001\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"shardingStrategy\":\"hash\",\"shards\":{\"s10002\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"waitForSync\":false,\"schema\":null,\"shardKeys\":[\"_key\"],\"isDisjoint\":false,\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"user\"],\"id\":\"10025\",\"inBackground\":false,\"name\":\"idx_1718347303681523712\",\"sparse\":true,\"type\":\"hash\",\"unique\":true}],\"cacheEnabled\":false,\"deleted\":false,\"statusString\":\"loaded\",\"isSmart\":false,\"numberOfShards\":1,\"minReplicationFactor\":1,\"id\":\"10001\",\"name\":\"_users\",\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10007\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10018\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_analyzers\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10007\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10003\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10014\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_graphs\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10003\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10006\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10017\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_statisticsRaw\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10006\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10028\",\"inBackground\":false,\"name\":\"idx_1718347303770652672\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10012\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10023\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_appbundles\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10012\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10010\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10021\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_jobs\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10010\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"queue\",\"status\",\"delayUntil\"],\"id\":\"10030\",\"inBackground\":false,\"name\":\"idx_1718347303839858688\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"status\",\"queue\",\"delayUntil\"],\"id\":\"10031\",\"inBackground\":false,\"name\":\"idx_1718347303866073088\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10004\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10015\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_statistics\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10004\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10026\",\"inBackground\":false,\"name\":\"idx_1718347303708786688\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10005\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10016\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_statistics15\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10005\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false},{\"deduplicate\":true,\"estimates\":false,\"fields\":[\"time\"],\"id\":\"10027\",\"inBackground\":false,\"name\":\"idx_1718347303741292544\",\"sparse\":false,\"type\":\"skiplist\",\"unique\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10009\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10020\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_queues\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10009\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}},\"10013\":{\"usesRevisionsAsDocumentIds\":true,\"syncByRevision\":true,\"isSmartChild\":false,\"distributeShardsLike\":\"10001\",\"shardingStrategy\":\"hash\",\"shards\":{\"s10024\":[\"PRMR-igofehwp\",\"PRMR-lamgjtvh\"]},\"type\":2,\"status\":3,\"replicationFactor\":2,\"writeConcern\":1,\"name\":\"_frontend\",\"statusString\":\"loaded\",\"isSmart\":false,\"schema\":null,\"cacheEnabled\":false,\"numberOfShards\":1,\"id\":\"10013\",\"minReplicationFactor\":1,\"deleted\":false,\"shardKeys\":[\"_key\"],\"indexes\":[{\"id\":\"0\",\"type\":\"primary\",\"name\":\"primary\",\"fields\":[\"_key\"],\"unique\":true,\"sparse\":false}],\"isDisjoint\":false,\"waitForSync\":false,\"isSystem\":true,\"keyOptions\":{\"allowUserKeys\":true,\"type\":\"traditional\"}}}}}}}]" var s StateRoots - require.NoError(t, json.Unmarshal([]byte(data), &s)) + require.NoError(t, json.Unmarshal(longData, &s)) t.Logf("%+v", s) } diff --git a/pkg/deployment/agency/testdata/config.json b/pkg/deployment/agency/testdata/config.json new file mode 100644 index 000000000..250ed2e89 --- /dev/null +++ b/pkg/deployment/agency/testdata/config.json @@ -0,0 +1,38 @@ +{ + "term": 0, + "leaderId": "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82", + "commitIndex": 94, + "lastCompactionAt": 0, + "nextCompactionAfter": 500, + "lastAcked": { + "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82": { + "lastAckedTime": 0, + "lastAckedIndex": 94 + } + }, + "configuration": { + "pool": { + "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82": "tcp://[::1]:4001" + }, + "active": [ + "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82" + ], + "id": "AGNT-fd0f4fc7-b60b-44bb-9f5e-5fc91f708f82", + "agency size": 1, + "pool size": 1, + "endpoint": "tcp://[::1]:4001", + "min ping": 1, + "max ping": 5, + "timeoutMult": 1, + "supervision": true, + "supervision frequency": 1, + "compaction step size": 500, + "compaction keep size": 50000, + "supervision grace period": 10, + "supervision ok threshold": 5, + "version": 2, + "startup": "origin" + }, + "engine": "rocksdb", + "version": "3.10.0-devel" +} \ No newline at end of file diff --git a/pkg/deployment/agency/testdata/longdata.json b/pkg/deployment/agency/testdata/longdata.json new file mode 100644 index 000000000..8e53bb63b --- /dev/null +++ b/pkg/deployment/agency/testdata/longdata.json @@ -0,0 +1,1120 @@ +[ + { + "arango": { + "Supervision": {}, + "Current": { + "Collections": { + "_system": { + "10011": { + "s10022": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010022", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "estimates": true, + "fields": [ + "mount" + ], + "id": "10029", + "name": "idx_1718347303809449984", + "objectId": "2010164", + "sparse": true, + "type": "hash", + "unique": true + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + }, + "10005": { + "s10016": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010038", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "time" + ], + "id": "10027", + "name": "idx_1718347303741292544", + "objectId": "2010144", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + }, + "10012": { + "s10023": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010032", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + }, + "10010": { + "s10021": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010034", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "queue", + "status", + "delayUntil" + ], + "id": "10030", + "name": "idx_1718347303839858688", + "objectId": "2010174", + "sparse": false, + "type": "skiplist", + "unique": false + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "status", + "queue", + "delayUntil" + ], + "id": "10031", + "name": "idx_1718347303866073088", + "objectId": "2010186", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + }, + "10004": { + "s10015": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010036", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "time" + ], + "id": "10026", + "name": "idx_1718347303708786688", + "objectId": "2010134", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + }, + "10003": { + "s10014": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010028", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + }, + "10006": { + "s10017": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010030", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "time" + ], + "id": "10028", + "name": "idx_1718347303770652672", + "objectId": "2010154", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + }, + "10001": { + "s10002": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010002", + "sparse": false, + "type": "primary", + "unique": true + }, + { + "deduplicate": true, + "estimates": true, + "fields": [ + "user" + ], + "id": "10025", + "name": "idx_1718347303681523712", + "objectId": "2010124", + "sparse": true, + "type": "hash", + "unique": true + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + }, + "10007": { + "s10018": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010027", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + }, + "10008": { + "s10019": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010024", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + }, + "10009": { + "s10020": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010040", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + }, + "10013": { + "s10024": { + "failoverCandidates": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ], + "errorNum": 0, + "errorMessage": "", + "error": false, + "indexes": [ + { + "fields": [ + "_key" + ], + "id": "0", + "name": "primary", + "objectId": "2010042", + "sparse": false, + "type": "primary", + "unique": true + } + ], + "servers": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + } + } + } + } + }, + "Plan": { + "Collections": { + "_system": { + "10011": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "distributeShardsLike": "10001", + "shardingStrategy": "hash", + "shards": { + "s10022": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "name": "_apps", + "statusString": "loaded", + "isSmart": false, + "schema": null, + "cacheEnabled": false, + "numberOfShards": 1, + "id": "10011", + "minReplicationFactor": 1, + "deleted": false, + "shardKeys": [ + "_key" + ], + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "mount" + ], + "id": "10029", + "inBackground": false, + "name": "idx_1718347303809449984", + "sparse": true, + "type": "hash", + "unique": true + } + ], + "isDisjoint": false, + "waitForSync": false, + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + }, + "10008": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "distributeShardsLike": "10001", + "shardingStrategy": "hash", + "shards": { + "s10019": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "name": "_aqlfunctions", + "statusString": "loaded", + "isSmart": false, + "schema": null, + "cacheEnabled": false, + "numberOfShards": 1, + "id": "10008", + "minReplicationFactor": 1, + "deleted": false, + "shardKeys": [ + "_key" + ], + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + } + ], + "isDisjoint": false, + "waitForSync": false, + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + }, + "10001": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "shardingStrategy": "hash", + "shards": { + "s10002": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "waitForSync": false, + "schema": null, + "shardKeys": [ + "_key" + ], + "isDisjoint": false, + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "user" + ], + "id": "10025", + "inBackground": false, + "name": "idx_1718347303681523712", + "sparse": true, + "type": "hash", + "unique": true + } + ], + "cacheEnabled": false, + "deleted": false, + "statusString": "loaded", + "isSmart": false, + "numberOfShards": 1, + "minReplicationFactor": 1, + "id": "10001", + "name": "_users", + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + }, + "10007": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "distributeShardsLike": "10001", + "shardingStrategy": "hash", + "shards": { + "s10018": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "name": "_analyzers", + "statusString": "loaded", + "isSmart": false, + "schema": null, + "cacheEnabled": false, + "numberOfShards": 1, + "id": "10007", + "minReplicationFactor": 1, + "deleted": false, + "shardKeys": [ + "_key" + ], + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + } + ], + "isDisjoint": false, + "waitForSync": false, + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + }, + "10003": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "distributeShardsLike": "10001", + "shardingStrategy": "hash", + "shards": { + "s10014": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "name": "_graphs", + "statusString": "loaded", + "isSmart": false, + "schema": null, + "cacheEnabled": false, + "numberOfShards": 1, + "id": "10003", + "minReplicationFactor": 1, + "deleted": false, + "shardKeys": [ + "_key" + ], + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + } + ], + "isDisjoint": false, + "waitForSync": false, + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + }, + "10006": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "distributeShardsLike": "10001", + "shardingStrategy": "hash", + "shards": { + "s10017": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "name": "_statisticsRaw", + "statusString": "loaded", + "isSmart": false, + "schema": null, + "cacheEnabled": false, + "numberOfShards": 1, + "id": "10006", + "minReplicationFactor": 1, + "deleted": false, + "shardKeys": [ + "_key" + ], + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "time" + ], + "id": "10028", + "inBackground": false, + "name": "idx_1718347303770652672", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "isDisjoint": false, + "waitForSync": false, + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + }, + "10012": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "distributeShardsLike": "10001", + "shardingStrategy": "hash", + "shards": { + "s10023": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "name": "_appbundles", + "statusString": "loaded", + "isSmart": false, + "schema": null, + "cacheEnabled": false, + "numberOfShards": 1, + "id": "10012", + "minReplicationFactor": 1, + "deleted": false, + "shardKeys": [ + "_key" + ], + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + } + ], + "isDisjoint": false, + "waitForSync": false, + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + }, + "10010": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "distributeShardsLike": "10001", + "shardingStrategy": "hash", + "shards": { + "s10021": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "name": "_jobs", + "statusString": "loaded", + "isSmart": false, + "schema": null, + "cacheEnabled": false, + "numberOfShards": 1, + "id": "10010", + "minReplicationFactor": 1, + "deleted": false, + "shardKeys": [ + "_key" + ], + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "queue", + "status", + "delayUntil" + ], + "id": "10030", + "inBackground": false, + "name": "idx_1718347303839858688", + "sparse": false, + "type": "skiplist", + "unique": false + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "status", + "queue", + "delayUntil" + ], + "id": "10031", + "inBackground": false, + "name": "idx_1718347303866073088", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "isDisjoint": false, + "waitForSync": false, + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + }, + "10004": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "distributeShardsLike": "10001", + "shardingStrategy": "hash", + "shards": { + "s10015": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "name": "_statistics", + "statusString": "loaded", + "isSmart": false, + "schema": null, + "cacheEnabled": false, + "numberOfShards": 1, + "id": "10004", + "minReplicationFactor": 1, + "deleted": false, + "shardKeys": [ + "_key" + ], + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "time" + ], + "id": "10026", + "inBackground": false, + "name": "idx_1718347303708786688", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "isDisjoint": false, + "waitForSync": false, + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + }, + "10005": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "distributeShardsLike": "10001", + "shardingStrategy": "hash", + "shards": { + "s10016": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "name": "_statistics15", + "statusString": "loaded", + "isSmart": false, + "schema": null, + "cacheEnabled": false, + "numberOfShards": 1, + "id": "10005", + "minReplicationFactor": 1, + "deleted": false, + "shardKeys": [ + "_key" + ], + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + }, + { + "deduplicate": true, + "estimates": false, + "fields": [ + "time" + ], + "id": "10027", + "inBackground": false, + "name": "idx_1718347303741292544", + "sparse": false, + "type": "skiplist", + "unique": false + } + ], + "isDisjoint": false, + "waitForSync": false, + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + }, + "10009": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "distributeShardsLike": "10001", + "shardingStrategy": "hash", + "shards": { + "s10020": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "name": "_queues", + "statusString": "loaded", + "isSmart": false, + "schema": null, + "cacheEnabled": false, + "numberOfShards": 1, + "id": "10009", + "minReplicationFactor": 1, + "deleted": false, + "shardKeys": [ + "_key" + ], + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + } + ], + "isDisjoint": false, + "waitForSync": false, + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + }, + "10013": { + "usesRevisionsAsDocumentIds": true, + "syncByRevision": true, + "isSmartChild": false, + "distributeShardsLike": "10001", + "shardingStrategy": "hash", + "shards": { + "s10024": [ + "PRMR-igofehwp", + "PRMR-lamgjtvh" + ] + }, + "type": 2, + "status": 3, + "replicationFactor": 2, + "writeConcern": 1, + "name": "_frontend", + "statusString": "loaded", + "isSmart": false, + "schema": null, + "cacheEnabled": false, + "numberOfShards": 1, + "id": "10013", + "minReplicationFactor": 1, + "deleted": false, + "shardKeys": [ + "_key" + ], + "indexes": [ + { + "id": "0", + "type": "primary", + "name": "primary", + "fields": [ + "_key" + ], + "unique": true, + "sparse": false + } + ], + "isDisjoint": false, + "waitForSync": false, + "isSystem": true, + "keyOptions": { + "allowUserKeys": true, + "type": "traditional" + } + } + } + } + } + } + } +] diff --git a/pkg/deployment/reconcile/action_context.go b/pkg/deployment/reconcile/action_context.go index c5f4e4971..814119fa2 100644 --- a/pkg/deployment/reconcile/action_context.go +++ b/pkg/deployment/reconcile/action_context.go @@ -56,6 +56,8 @@ type ActionContext interface { sutil.ACSGetter + Metrics() *Metrics + ActionLocalsContext // GetMemberStatusByID returns the current member status @@ -106,10 +108,11 @@ type ActionLocalsContext interface { } // newActionContext creates a new ActionContext implementation. -func newActionContext(log logging.Logger, context Context) ActionContext { +func newActionContext(log logging.Logger, context Context, metrics *Metrics) ActionContext { return &actionContext{ log: log, context: context, + metrics: metrics, } } @@ -119,6 +122,11 @@ type actionContext struct { log logging.Logger cachedStatus inspectorInterface.Inspector locals api.PlanLocals + metrics *Metrics +} + +func (ac *actionContext) Metrics() *Metrics { + return ac.metrics } func (ac *actionContext) ACS() sutil.ACS { diff --git a/pkg/deployment/reconcile/metrics.go b/pkg/deployment/reconcile/metrics.go new file mode 100644 index 000000000..246a67ee4 --- /dev/null +++ b/pkg/deployment/reconcile/metrics.go @@ -0,0 +1,93 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package reconcile + +import ( + "github.com/arangodb/kube-arangodb/pkg/generated/metric_descriptions" + "github.com/arangodb/kube-arangodb/pkg/util/metrics" +) + +type Metrics struct { + Rebalancer MetricsRebalancer +} + +func (m *Metrics) GetRebalancer() *MetricsRebalancer { + if m == nil { + return nil + } + + return &m.Rebalancer +} + +type MetricsRebalancer struct { + enabled bool + moves int + current int + + succeeded, failed int +} + +func (m *MetricsRebalancer) SetEnabled(enabled bool) { + if m == nil { + return + } + m.enabled = enabled +} + +func (m *MetricsRebalancer) AddMoves(moves int) { + if m == nil { + return + } + m.moves += moves +} + +func (m *MetricsRebalancer) SetCurrent(current int) { + if m == nil { + return + } + m.current = current +} + +func (m *MetricsRebalancer) AddFailures(i int) { + if m == nil { + return + } + m.failed += i +} + +func (m *MetricsRebalancer) AddSuccesses(i int) { + if m == nil { + return + } + m.succeeded += i +} + +func (r *Reconciler) CollectMetrics(m metrics.PushMetric) { + if r.metrics.Rebalancer.enabled { + m.Push(metric_descriptions.ArangodbOperatorRebalancerEnabled().Gauge(1, r.namespace, r.name)) + m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesCurrent().Gauge(float64(r.metrics.Rebalancer.current), r.namespace, r.name)) + m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesGenerated().Gauge(float64(r.metrics.Rebalancer.moves), r.namespace, r.name)) + m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesSucceeded().Gauge(float64(r.metrics.Rebalancer.succeeded), r.namespace, r.name)) + m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesFailed().Gauge(float64(r.metrics.Rebalancer.failed), r.namespace, r.name)) + } else { + m.Push(metric_descriptions.ArangodbOperatorRebalancerEnabled().Gauge(0, r.namespace, r.name)) + } +} diff --git a/pkg/deployment/reconcile/plan_executor.go b/pkg/deployment/reconcile/plan_executor.go index d0602928e..de46b57d9 100644 --- a/pkg/deployment/reconcile/plan_executor.go +++ b/pkg/deployment/reconcile/plan_executor.go @@ -116,40 +116,76 @@ func (p plannerResources) Set(deployment *api.DeploymentStatus, plan api.Plan) b // Returns true when it has to be called again soon. // False otherwise. func (d *Reconciler) ExecutePlan(ctx context.Context) (bool, error) { + execution := 0 + + var retrySoon bool + + for { + if execution >= 32 { + return retrySoon, nil + } + + execution++ + + if retrySoonCall, recall, err := d.executePlanInLoop(ctx); err != nil { + return false, err + } else { + retrySoon = retrySoon || retrySoonCall + if recall { + continue + } + } + + break + } + + return retrySoon, nil +} + +// ExecutePlan tries to execute the plan as far as possible. +// Returns true when it has to be called again soon. +// False otherwise. +func (d *Reconciler) executePlanInLoop(ctx context.Context) (bool, bool, error) { var callAgain bool + var callInLoop bool - if again, err := d.executePlanStatus(ctx, plannerHigh{}); err != nil { - return false, errors.WithStack(err) - } else if again { - callAgain = true + if again, inLoop, err := d.executePlanStatus(ctx, plannerHigh{}); err != nil { + d.planLogger.Err(err).Error("Execution of plan failed") + return false, false, errors.WithStack(err) + } else { + callAgain = callAgain || again + callInLoop = callInLoop || inLoop } - if again, err := d.executePlanStatus(ctx, plannerResources{}); err != nil { + if again, inLoop, err := d.executePlanStatus(ctx, plannerResources{}); err != nil { d.planLogger.Err(err).Error("Execution of plan failed") - return false, nil - } else if again { - callAgain = true + return false, false, nil + } else { + callAgain = callAgain || again + callInLoop = callInLoop || inLoop } - if again, err := d.executePlanStatus(ctx, plannerNormal{}); err != nil { - return false, errors.WithStack(err) - } else if again { - callAgain = true + if again, inLoop, err := d.executePlanStatus(ctx, plannerNormal{}); err != nil { + d.planLogger.Err(err).Error("Execution of plan failed") + return false, false, errors.WithStack(err) + } else { + callAgain = callAgain || again + callInLoop = callInLoop || inLoop } - return callAgain, nil + return callAgain, callInLoop, nil } -func (d *Reconciler) executePlanStatus(ctx context.Context, pg planner) (bool, error) { +func (d *Reconciler) executePlanStatus(ctx context.Context, pg planner) (bool, bool, error) { loopStatus, _ := d.context.GetStatus() plan := pg.Get(&loopStatus) if len(plan) == 0 { - return false, nil + return false, false, nil } - newPlan, callAgain, err := d.executePlan(ctx, plan, pg) + newPlan, callAgain, callInLoop, err := d.executePlan(ctx, plan, pg) // Refresh current status loopStatus, lastVersion := d.context.GetStatus() @@ -158,23 +194,23 @@ func (d *Reconciler) executePlanStatus(ctx context.Context, pg planner) (bool, e d.planLogger.Info("Updating plan") if err := d.context.UpdateStatus(ctx, loopStatus, lastVersion, true); err != nil { d.planLogger.Err(err).Debug("Failed to update CR status") - return false, errors.WithStack(err) + return false, false, errors.WithStack(err) } } if err != nil { - return false, err + return false, false, err } - return callAgain, nil + return callAgain, callInLoop, nil } -func (d *Reconciler) executePlan(ctx context.Context, statusPlan api.Plan, pg planner) (newPlan api.Plan, callAgain bool, err error) { +func (d *Reconciler) executePlan(ctx context.Context, statusPlan api.Plan, pg planner) (newPlan api.Plan, callAgain, callInLoop bool, err error) { plan := statusPlan.DeepCopy() for { if len(plan) == 0 { - return nil, false, nil + return nil, false, false, nil } // Take first action @@ -185,14 +221,14 @@ func (d *Reconciler) executePlan(ctx context.Context, statusPlan api.Plan, pg pl done, abort, recall, retry, err := d.executeAction(ctx, planAction, action) if err != nil { if retry { - return plan, true, nil + return plan, true, false, nil } // The Plan will be cleaned up, so no actions will be in the queue. actionsCurrentPlan.WithLabelValues(d.context.GetName(), planAction.Group.AsRole(), planAction.MemberID, planAction.Type.String(), pg.Type()).Set(0.0) actionsFailedMetrics.WithLabelValues(d.context.GetName(), planAction.Type.String(), pg.Type()).Inc() - return nil, false, errors.WithStack(err) + return nil, false, false, errors.WithStack(err) } if abort { @@ -201,7 +237,7 @@ func (d *Reconciler) executePlan(ctx context.Context, statusPlan api.Plan, pg pl planAction.Type.String(), pg.Type()).Set(0.0) actionsFailedMetrics.WithLabelValues(d.context.GetName(), planAction.Type.String(), pg.Type()).Inc() - return nil, true, nil + return nil, true, false, nil } if done { @@ -232,19 +268,19 @@ func (d *Reconciler) executePlan(ctx context.Context, statusPlan api.Plan, pg pl d.planLogger.Info("Reloading cached status") if err := c.Refresh(ctx); err != nil { d.planLogger.Err(err).Warn("Unable to reload cached status") - return plan, recall, nil + return plan, recall, false, nil } } } if newPlan, changed := getActionPlanAppender(action, plan); changed { // Our actions have been added to the end of plan - return newPlan, true, nil + return newPlan, false, true, nil } if err := getActionPost(action, ctx); err != nil { d.planLogger.Err(err).Error("Post action failed") - return nil, false, errors.WithStack(err) + return nil, false, false, errors.WithStack(err) } } else { if !plan[0].IsStarted() { @@ -258,13 +294,14 @@ func (d *Reconciler) executePlan(ctx context.Context, statusPlan api.Plan, pg pl plan[0].Locals.Merge(actionContext.CurrentLocals()) - return plan, recall, nil + return plan, recall, false, nil } } } func (d *Reconciler) executeAction(ctx context.Context, planAction api.Action, action Action) (done, abort, callAgain, retry bool, err error) { log := d.planLogger.Str("action", string(planAction.Type)).Str("member", planAction.MemberID) + log.Info("Executing action") if !planAction.IsStarted() { // Not started yet @@ -327,7 +364,7 @@ func (d *Reconciler) executeAction(ctx context.Context, planAction api.Action, a // createAction create action object based on action type func (d *Reconciler) createAction(action api.Action) (Action, ActionContext) { - actionCtx := newActionContext(d.log, d.context) + actionCtx := newActionContext(d.log, d.context, &d.metrics) f, ok := getActionFactory(action.Type) if !ok { diff --git a/pkg/deployment/reconcile/reconciler.go b/pkg/deployment/reconcile/reconciler.go index be3a902e6..252935c27 100644 --- a/pkg/deployment/reconcile/reconciler.go +++ b/pkg/deployment/reconcile/reconciler.go @@ -39,6 +39,8 @@ type Reconciler struct { log logging.Logger planLogger logging.Logger context Context + + metrics Metrics } // NewReconciler creates a new reconciler with given context. diff --git a/pkg/deployment/agency/cache_test.go b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_enabled.go similarity index 60% rename from pkg/deployment/agency/cache_test.go rename to pkg/generated/metric_descriptions/arangodb_operator_rebalancer_enabled.go index 04e04bf3b..080c95759 100644 --- a/pkg/deployment/agency/cache_test.go +++ b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_enabled.go @@ -18,4 +18,18 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // -package agency +package metric_descriptions + +import "github.com/arangodb/kube-arangodb/pkg/util/metrics" + +var ( + arangodbOperatorRebalancerEnabled = metrics.NewDescription("arangodb_operator_rebalancer_enabled", "Determines if rebalancer is enabled", []string{`namespace`, `name`}, nil) +) + +func init() { + registerDescription(arangodbOperatorRebalancerEnabled) +} + +func ArangodbOperatorRebalancerEnabled() metrics.Description { + return arangodbOperatorRebalancerEnabled +} diff --git a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_current.go b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_current.go new file mode 100644 index 000000000..8d72decd5 --- /dev/null +++ b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_current.go @@ -0,0 +1,35 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package metric_descriptions + +import "github.com/arangodb/kube-arangodb/pkg/util/metrics" + +var ( + arangodbOperatorRebalancerMovesCurrent = metrics.NewDescription("arangodb_operator_rebalancer_moves_current", "Define how many moves are currently in progress", []string{`namespace`, `name`}, nil) +) + +func init() { + registerDescription(arangodbOperatorRebalancerMovesCurrent) +} + +func ArangodbOperatorRebalancerMovesCurrent() metrics.Description { + return arangodbOperatorRebalancerMovesCurrent +} diff --git a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_failed.go b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_failed.go new file mode 100644 index 000000000..c4006f054 --- /dev/null +++ b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_failed.go @@ -0,0 +1,35 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package metric_descriptions + +import "github.com/arangodb/kube-arangodb/pkg/util/metrics" + +var ( + arangodbOperatorRebalancerMovesFailed = metrics.NewDescription("arangodb_operator_rebalancer_moves_failed", "Define how many moves failed", []string{`namespace`, `name`}, nil) +) + +func init() { + registerDescription(arangodbOperatorRebalancerMovesFailed) +} + +func ArangodbOperatorRebalancerMovesFailed() metrics.Description { + return arangodbOperatorRebalancerMovesFailed +} diff --git a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_generated.go b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_generated.go new file mode 100644 index 000000000..8c187e9a2 --- /dev/null +++ b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_generated.go @@ -0,0 +1,35 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package metric_descriptions + +import "github.com/arangodb/kube-arangodb/pkg/util/metrics" + +var ( + arangodbOperatorRebalancerMovesGenerated = metrics.NewDescription("arangodb_operator_rebalancer_moves_generated", "Define how many moves were generated", []string{`namespace`, `name`}, nil) +) + +func init() { + registerDescription(arangodbOperatorRebalancerMovesGenerated) +} + +func ArangodbOperatorRebalancerMovesGenerated() metrics.Description { + return arangodbOperatorRebalancerMovesGenerated +} diff --git a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_succeeded.go b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_succeeded.go new file mode 100644 index 000000000..d4983f3ee --- /dev/null +++ b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_succeeded.go @@ -0,0 +1,35 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package metric_descriptions + +import "github.com/arangodb/kube-arangodb/pkg/util/metrics" + +var ( + arangodbOperatorRebalancerMovesSucceeded = metrics.NewDescription("arangodb_operator_rebalancer_moves_succeeded", "Define how many moves succeeded", []string{`namespace`, `name`}, nil) +) + +func init() { + registerDescription(arangodbOperatorRebalancerMovesSucceeded) +} + +func ArangodbOperatorRebalancerMovesSucceeded() metrics.Description { + return arangodbOperatorRebalancerMovesSucceeded +} From b3c37fa8a5dd74bcf251f00c51a17af12098f902 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Thu, 30 Jun 2022 20:06:15 +0000 Subject: [PATCH 2/2] s --- docs/generated/metrics/README.md | 4 ++-- .../metrics/arangodb_operator_agency_errors.md | 2 +- .../metrics/arangodb_operator_agency_fetches.md | 2 +- internal/metrics.yaml | 4 ++-- pkg/deployment/metrics.go | 8 ++++++-- pkg/deployment/reconcile/metrics.go | 12 ++++++------ .../arangodb_operator_agency_errors.go | 2 +- .../arangodb_operator_agency_fetches.go | 2 +- .../arangodb_operator_rebalancer_enabled.go | 4 ++++ .../arangodb_operator_rebalancer_moves_current.go | 4 ++++ .../arangodb_operator_rebalancer_moves_failed.go | 4 ++++ .../arangodb_operator_rebalancer_moves_generated.go | 4 ++++ .../arangodb_operator_rebalancer_moves_succeeded.go | 4 ++++ 13 files changed, 40 insertions(+), 16 deletions(-) diff --git a/docs/generated/metrics/README.md b/docs/generated/metrics/README.md index 5d8065e18..e4c150f53 100644 --- a/docs/generated/metrics/README.md +++ b/docs/generated/metrics/README.md @@ -4,8 +4,8 @@ | Name | Namespace | Group | Type | Description | |:---------------------------------------------------------------------------------------------------------------:|:-----------------:|:------------:|:-------:|:---------------------------------------------------| -| [arangodb_operator_agency_errors](./arangodb_operator_agency_errors.md) | arangodb_operator | agency | Count | Current count of agency cache fetch errors | -| [arangodb_operator_agency_fetches](./arangodb_operator_agency_fetches.md) | arangodb_operator | agency | Count | Current count of agency cache fetches | +| [arangodb_operator_agency_errors](./arangodb_operator_agency_errors.md) | arangodb_operator | agency | Counter | Current count of agency cache fetch errors | +| [arangodb_operator_agency_fetches](./arangodb_operator_agency_fetches.md) | arangodb_operator | agency | Counter | Current count of agency cache fetches | | [arangodb_operator_agency_index](./arangodb_operator_agency_index.md) | arangodb_operator | agency | Gauge | Current index of the agency cache | | [arangodb_operator_agency_cache_health_present](./arangodb_operator_agency_cache_health_present.md) | arangodb_operator | agency_cache | Gauge | Determines if local agency cache health is present | | [arangodb_operator_agency_cache_healthy](./arangodb_operator_agency_cache_healthy.md) | arangodb_operator | agency_cache | Gauge | Determines if agency is healthy | diff --git a/docs/generated/metrics/arangodb_operator_agency_errors.md b/docs/generated/metrics/arangodb_operator_agency_errors.md index 87f9fc39e..c6746ac64 100644 --- a/docs/generated/metrics/arangodb_operator_agency_errors.md +++ b/docs/generated/metrics/arangodb_operator_agency_errors.md @@ -1,4 +1,4 @@ -# arangodb_operator_agency_errors (Count) +# arangodb_operator_agency_errors (Counter) ## Description diff --git a/docs/generated/metrics/arangodb_operator_agency_fetches.md b/docs/generated/metrics/arangodb_operator_agency_fetches.md index 7a3c265ec..c7139fea9 100644 --- a/docs/generated/metrics/arangodb_operator_agency_fetches.md +++ b/docs/generated/metrics/arangodb_operator_agency_fetches.md @@ -1,4 +1,4 @@ -# arangodb_operator_agency_fetches (Count) +# arangodb_operator_agency_fetches (Counter) ## Description diff --git a/internal/metrics.yaml b/internal/metrics.yaml index e66c62c77..583c9c77b 100644 --- a/internal/metrics.yaml +++ b/internal/metrics.yaml @@ -88,7 +88,7 @@ namespaces: fetches: shortDescription: "Current count of agency cache fetches" description: "Current count of agency cache fetches" - type: "Count" + type: "Counter" labels: - key: namespace description: "Deployment Namespace" @@ -97,7 +97,7 @@ namespaces: errors: shortDescription: "Current count of agency cache fetch errors" description: "Current count of agency cache fetch errors" - type: "Count" + type: "Counter" labels: - key: namespace description: "Deployment Namespace" diff --git a/pkg/deployment/metrics.go b/pkg/deployment/metrics.go index fcbb4d4d4..f6a345d97 100644 --- a/pkg/deployment/metrics.go +++ b/pkg/deployment/metrics.go @@ -157,8 +157,8 @@ func (i *inventory) Add(d *Deployment) { } func (d *Deployment) CollectMetrics(m metrics.PushMetric) { - m.Push(metric_descriptions.ArangodbOperatorAgencyErrorsCount(float64(d.metrics.agency.errors), d.namespace, d.name)) - m.Push(metric_descriptions.ArangodbOperatorAgencyFetchesCount(float64(d.metrics.agency.fetches), d.namespace, d.name)) + m.Push(metric_descriptions.ArangodbOperatorAgencyErrorsCounter(float64(d.metrics.agency.errors), d.namespace, d.name)) + m.Push(metric_descriptions.ArangodbOperatorAgencyFetchesCounter(float64(d.metrics.agency.fetches), d.namespace, d.name)) m.Push(metric_descriptions.ArangodbOperatorAgencyIndexGauge(float64(d.metrics.agency.index), d.namespace, d.name)) if c := d.agencyCache; c != nil { @@ -173,4 +173,8 @@ func (d *Deployment) CollectMetrics(m metrics.PushMetric) { } else { m.Push(metric_descriptions.ArangodbOperatorAgencyCachePresentGauge(0, d.namespace, d.name)) } + + if c := d.reconciler; c != nil { + c.CollectMetrics(m) + } } diff --git a/pkg/deployment/reconcile/metrics.go b/pkg/deployment/reconcile/metrics.go index 246a67ee4..fe2766fcc 100644 --- a/pkg/deployment/reconcile/metrics.go +++ b/pkg/deployment/reconcile/metrics.go @@ -82,12 +82,12 @@ func (m *MetricsRebalancer) AddSuccesses(i int) { func (r *Reconciler) CollectMetrics(m metrics.PushMetric) { if r.metrics.Rebalancer.enabled { - m.Push(metric_descriptions.ArangodbOperatorRebalancerEnabled().Gauge(1, r.namespace, r.name)) - m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesCurrent().Gauge(float64(r.metrics.Rebalancer.current), r.namespace, r.name)) - m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesGenerated().Gauge(float64(r.metrics.Rebalancer.moves), r.namespace, r.name)) - m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesSucceeded().Gauge(float64(r.metrics.Rebalancer.succeeded), r.namespace, r.name)) - m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesFailed().Gauge(float64(r.metrics.Rebalancer.failed), r.namespace, r.name)) + m.Push(metric_descriptions.ArangodbOperatorRebalancerEnabledGauge(1, r.namespace, r.name)) + m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesCurrentGauge(float64(r.metrics.Rebalancer.current), r.namespace, r.name)) + m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesGeneratedCounter(float64(r.metrics.Rebalancer.moves), r.namespace, r.name)) + m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesSucceededCounter(float64(r.metrics.Rebalancer.succeeded), r.namespace, r.name)) + m.Push(metric_descriptions.ArangodbOperatorRebalancerMovesFailedCounter(float64(r.metrics.Rebalancer.failed), r.namespace, r.name)) } else { - m.Push(metric_descriptions.ArangodbOperatorRebalancerEnabled().Gauge(0, r.namespace, r.name)) + m.Push(metric_descriptions.ArangodbOperatorRebalancerEnabledGauge(0, r.namespace, r.name)) } } diff --git a/pkg/generated/metric_descriptions/arangodb_operator_agency_errors.go b/pkg/generated/metric_descriptions/arangodb_operator_agency_errors.go index ff73e10cd..96c660a38 100644 --- a/pkg/generated/metric_descriptions/arangodb_operator_agency_errors.go +++ b/pkg/generated/metric_descriptions/arangodb_operator_agency_errors.go @@ -34,6 +34,6 @@ func ArangodbOperatorAgencyErrors() metrics.Description { return arangodbOperatorAgencyErrors } -func ArangodbOperatorAgencyErrorsCount(value float64, namespace string, name string) metrics.Metric { +func ArangodbOperatorAgencyErrorsCounter(value float64, namespace string, name string) metrics.Metric { return ArangodbOperatorAgencyErrors().Gauge(value, namespace, name) } diff --git a/pkg/generated/metric_descriptions/arangodb_operator_agency_fetches.go b/pkg/generated/metric_descriptions/arangodb_operator_agency_fetches.go index c8fbe9979..0d881f97b 100644 --- a/pkg/generated/metric_descriptions/arangodb_operator_agency_fetches.go +++ b/pkg/generated/metric_descriptions/arangodb_operator_agency_fetches.go @@ -34,6 +34,6 @@ func ArangodbOperatorAgencyFetches() metrics.Description { return arangodbOperatorAgencyFetches } -func ArangodbOperatorAgencyFetchesCount(value float64, namespace string, name string) metrics.Metric { +func ArangodbOperatorAgencyFetchesCounter(value float64, namespace string, name string) metrics.Metric { return ArangodbOperatorAgencyFetches().Gauge(value, namespace, name) } diff --git a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_enabled.go b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_enabled.go index 080c95759..aed700af3 100644 --- a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_enabled.go +++ b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_enabled.go @@ -33,3 +33,7 @@ func init() { func ArangodbOperatorRebalancerEnabled() metrics.Description { return arangodbOperatorRebalancerEnabled } + +func ArangodbOperatorRebalancerEnabledGauge(value float64, namespace string, name string) metrics.Metric { + return ArangodbOperatorRebalancerEnabled().Gauge(value, namespace, name) +} diff --git a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_current.go b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_current.go index 8d72decd5..8b010522e 100644 --- a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_current.go +++ b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_current.go @@ -33,3 +33,7 @@ func init() { func ArangodbOperatorRebalancerMovesCurrent() metrics.Description { return arangodbOperatorRebalancerMovesCurrent } + +func ArangodbOperatorRebalancerMovesCurrentGauge(value float64, namespace string, name string) metrics.Metric { + return ArangodbOperatorRebalancerMovesCurrent().Gauge(value, namespace, name) +} diff --git a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_failed.go b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_failed.go index c4006f054..35198590b 100644 --- a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_failed.go +++ b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_failed.go @@ -33,3 +33,7 @@ func init() { func ArangodbOperatorRebalancerMovesFailed() metrics.Description { return arangodbOperatorRebalancerMovesFailed } + +func ArangodbOperatorRebalancerMovesFailedCounter(value float64, namespace string, name string) metrics.Metric { + return ArangodbOperatorRebalancerMovesFailed().Gauge(value, namespace, name) +} diff --git a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_generated.go b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_generated.go index 8c187e9a2..ef97cf1c9 100644 --- a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_generated.go +++ b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_generated.go @@ -33,3 +33,7 @@ func init() { func ArangodbOperatorRebalancerMovesGenerated() metrics.Description { return arangodbOperatorRebalancerMovesGenerated } + +func ArangodbOperatorRebalancerMovesGeneratedCounter(value float64, namespace string, name string) metrics.Metric { + return ArangodbOperatorRebalancerMovesGenerated().Gauge(value, namespace, name) +} diff --git a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_succeeded.go b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_succeeded.go index d4983f3ee..8c0051f5d 100644 --- a/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_succeeded.go +++ b/pkg/generated/metric_descriptions/arangodb_operator_rebalancer_moves_succeeded.go @@ -33,3 +33,7 @@ func init() { func ArangodbOperatorRebalancerMovesSucceeded() metrics.Description { return arangodbOperatorRebalancerMovesSucceeded } + +func ArangodbOperatorRebalancerMovesSucceededCounter(value float64, namespace string, name string) metrics.Metric { + return ArangodbOperatorRebalancerMovesSucceeded().Gauge(value, namespace, name) +}