From d2e392e5bdbdc90fbdd0b2a9e5de5b1df1839ff6 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Tue, 29 Mar 2022 13:35:14 +0300 Subject: [PATCH] fix: more responsive ConfigMap changes Huge refactoring to speedup reactions to ConfigMap changes. Fixes in enabled state calculations. KubeConfigManager is now responsible to signal if config is changed or invalid. - fix: Update metric if ConfigMap is invalid (malformed module name or module values YAML) - use runtime config to enable debug - add locking ModuleManager is now the only component that runs enabled scripts. It is responsible for calculating ModulesReload/ReloadAll event. It lists Helm releases once at start to detect initial enabled modules and modules to purge. - fix: Fresh config values for enabled scripts (#184, #16) AddonOperator uses task ConvergeModules to handle changes in ConfigMap or global values and reload all modules or only changed. - fix: ModulePurge tasks moved between GlobalSynchronization and first ConvergeModules (#233) - fix: Clear queues from tasks for disabled module or errored hooks on ConfigMap changes (#43) - ref: ConvergeModules is a new multi-phase task. Other: - mocks for HelmResourcesManager and KubeConfigManager - Helm struct instead of global variables helm.NewClient and helm.HealthzHandler - by dependabot: update gomega --- cmd/addon-operator/main.go | 17 +- go.mod | 8 +- go.sum | 14 +- pkg/addon-operator/bootstrap.go | 130 ++ pkg/addon-operator/converge.go | 91 + pkg/addon-operator/debug_server.go | 130 ++ pkg/addon-operator/http_server.go | 71 + pkg/addon-operator/kube_client.go | 39 +- pkg/addon-operator/metrics.go | 26 + pkg/addon-operator/operator.go | 1826 ++++++++--------- pkg/addon-operator/operator_test.go | 375 +++- pkg/addon-operator/queue.go | 138 ++ pkg/addon-operator/queue_test.go | 371 ++++ .../config_map.yaml | 3 +- .../global-hooks/hook01_startup_20_kube.sh | 15 + .../global-hooks/hook02_startup_1_schedule.sh | 13 + .../hook03_startup_10_kube_schedule.sh | 17 + .../hooks/hook01_startup_20_kube.sh} | 5 +- .../hooks/hook02_startup_1_schedule.sh | 13 + .../hooks/hook01_startup_20_kube.sh} | 6 +- .../hooks/hook02_after_delete_helm.sh | 13 + .../global-hooks/hook01_startup_20_kube.sh | 14 + .../global-hooks/hook02_startup_1_schedule.sh | 13 + .../hook03_startup_10_kube_schedule.sh | 16 + pkg/app/app.go | 18 +- pkg/helm/helm.go | 39 +- pkg/helm/{helm_mock.go => mock.go} | 14 + pkg/helm_resources_manager/mock.go | 60 + pkg/kube_config_manager/access_config_map.go | 75 + pkg/kube_config_manager/checksums.go | 64 + pkg/kube_config_manager/checksums_test.go | 30 + pkg/kube_config_manager/config.go | 42 + pkg/kube_config_manager/config_test.go | 97 + .../kube_config_manager.go | 601 +++--- .../kube_config_manager_test.go | 259 ++- pkg/kube_config_manager/mock.go | 17 + pkg/kube_config_manager/module_kube_config.go | 25 +- pkg/module_manager/global_hook.go | 3 +- pkg/module_manager/hook.go | 43 +- pkg/module_manager/hook_executor.go | 16 +- pkg/module_manager/hook_executor_test.go | 2 +- pkg/module_manager/module.go | 29 +- pkg/module_manager/module_hook.go | 3 +- pkg/module_manager/module_manager.go | 851 ++++---- pkg/module_manager/module_manager_test.go | 1306 ++++++------ pkg/module_manager/synchronization_state.go | 15 +- .../config_map.yaml | 6 - .../global-hooks/000-all-bindings/all | 69 - .../global-hooks/000-all-bindings/hook | 56 + .../sub/sub/{nested-before-all => hook} | 0 .../modules/000-all-bindings/hooks/all | 70 - .../000-all-bindings/hooks/all-bindings | 58 + .../hooks/sub/sub/nested-before-helm | 9 +- .../config_map.yaml | 2 +- .../config_map.yaml | 2 +- .../global-hooks/openapi/config-values.yaml | 2 +- .../modules/values.yaml | 2 +- .../config_map.yaml | 2 +- .../modules/001-module-one}/.gitkeep | 0 .../modules/003-module-three}/.gitkeep | 0 .../modules/values.yaml | 2 + .../modules/000-module-c}/.gitkeep | 0 .../modules/000-module-c/values.yaml | 0 .../modules/100-module-a}/.gitkeep | 0 .../modules/200-module-b}/.gitkeep | 0 .../modules/200-module-b/values.yaml | 0 .../modules/300-module-disabled/values.yaml | 0 .../modules/001-module-1}/.gitkeep | 0 .../modules/003-module-3}/.gitkeep | 0 .../modules/004-module-4}/.gitkeep | 0 .../modules/007-module-7}/.gitkeep | 0 .../modules/008-module-8/.gitkeep | 0 .../modules/009-module-9/.gitkeep | 0 .../modules/values.yaml | 0 .../modules/001-alpha/enabled | 0 .../modules/002-beta/enabled | 0 .../modules/003-gamma/enabled | 0 .../modules/004-delta/enabled | 0 .../modules/005-epsilon/enabled | 0 .../modules/006-zeta/enabled | 0 .../modules/007-eta/enabled | 0 .../modules/values.yaml | 0 .../modules/000-module/hooks/hook-1 | 11 +- .../modules/000-module/hooks/hook-2 | 12 +- .../modules/000-module/values.yaml | 7 +- .../merge_and_patch_values | 9 +- .../modules/000-module/hooks/hook-1 | 11 +- .../modules/000-module/hooks/hook-2 | 11 +- .../modules/000-module/hooks/hook-3 | 11 +- .../modules/000-module/hooks/hook-4 | 11 +- .../modules/000-module/values.yaml | 10 +- .../hooks/merge_and_patch_values | 9 +- .../hooks/merge_and_patch_values | 9 +- pkg/task/hook_metadata.go | 7 +- pkg/task/task.go | 26 +- pkg/task/test/task_metadata_test.go | 4 +- pkg/utils/module_config.go | 3 + pkg/utils/module_list.go | 34 + pkg/utils/values.go | 4 + 99 files changed, 4476 insertions(+), 2966 deletions(-) create mode 100644 pkg/addon-operator/bootstrap.go create mode 100644 pkg/addon-operator/converge.go create mode 100644 pkg/addon-operator/debug_server.go create mode 100644 pkg/addon-operator/http_server.go create mode 100644 pkg/addon-operator/queue.go create mode 100644 pkg/addon-operator/queue_test.go rename pkg/{module_manager/testdata/discover_modules_state__with_enabled_scripts => addon-operator/testdata/converge__main_queue_only}/config_map.yaml (54%) create mode 100755 pkg/addon-operator/testdata/converge__main_queue_only/global-hooks/hook01_startup_20_kube.sh create mode 100755 pkg/addon-operator/testdata/converge__main_queue_only/global-hooks/hook02_startup_1_schedule.sh create mode 100755 pkg/addon-operator/testdata/converge__main_queue_only/global-hooks/hook03_startup_10_kube_schedule.sh rename pkg/addon-operator/testdata/{global_hooks/hook_one.sh => converge__main_queue_only/modules/000-module-alpha/hooks/hook01_startup_20_kube.sh} (67%) create mode 100755 pkg/addon-operator/testdata/converge__main_queue_only/modules/000-module-alpha/hooks/hook02_startup_1_schedule.sh rename pkg/addon-operator/testdata/{global_hooks/hook_two.sh => converge__main_queue_only/modules/001-module-beta/hooks/hook01_startup_20_kube.sh} (67%) create mode 100755 pkg/addon-operator/testdata/converge__main_queue_only/modules/001-module-beta/hooks/hook02_after_delete_helm.sh create mode 100755 pkg/addon-operator/testdata/startup_tasks/global-hooks/hook01_startup_20_kube.sh create mode 100755 pkg/addon-operator/testdata/startup_tasks/global-hooks/hook02_startup_1_schedule.sh create mode 100755 pkg/addon-operator/testdata/startup_tasks/global-hooks/hook03_startup_10_kube_schedule.sh rename pkg/helm/{helm_mock.go => mock.go} (83%) create mode 100644 pkg/helm_resources_manager/mock.go create mode 100644 pkg/kube_config_manager/access_config_map.go create mode 100644 pkg/kube_config_manager/checksums.go create mode 100644 pkg/kube_config_manager/checksums_test.go create mode 100644 pkg/kube_config_manager/config.go create mode 100644 pkg/kube_config_manager/config_test.go create mode 100644 pkg/kube_config_manager/mock.go delete mode 100644 pkg/module_manager/testdata/discover_modules_state__module_names_order/config_map.yaml delete mode 100755 pkg/module_manager/testdata/get__global_hook/global-hooks/000-all-bindings/all create mode 100755 pkg/module_manager/testdata/get__global_hook/global-hooks/000-all-bindings/hook rename pkg/module_manager/testdata/get__global_hook/global-hooks/100-nested-hook/sub/sub/{nested-before-all => hook} (100%) delete mode 100755 pkg/module_manager/testdata/get__module_hook/modules/000-all-bindings/hooks/all create mode 100755 pkg/module_manager/testdata/get__module_hook/modules/000-all-bindings/hooks/all-bindings rename pkg/module_manager/testdata/{discover_modules_state__simple => modules_state__detect_cm_changes}/config_map.yaml (70%) rename pkg/module_manager/testdata/{discover_modules_state__module_names_order/modules/000-module-c => modules_state__detect_cm_changes/modules/001-module-one}/.gitkeep (100%) rename pkg/module_manager/testdata/{discover_modules_state__module_names_order/modules/100-module-a => modules_state__detect_cm_changes/modules/003-module-three}/.gitkeep (100%) create mode 100644 pkg/module_manager/testdata/modules_state__detect_cm_changes/modules/values.yaml rename pkg/module_manager/testdata/{discover_modules_state__module_names_order/modules/200-module-b => modules_state__no_cm__module_names_order/modules/000-module-c}/.gitkeep (100%) rename pkg/module_manager/testdata/{discover_modules_state__module_names_order => modules_state__no_cm__module_names_order}/modules/000-module-c/values.yaml (100%) rename pkg/module_manager/testdata/{discover_modules_state__simple/modules/001-module-1 => modules_state__no_cm__module_names_order/modules/100-module-a}/.gitkeep (100%) rename pkg/module_manager/testdata/{discover_modules_state__simple/modules/003-module-3 => modules_state__no_cm__module_names_order/modules/200-module-b}/.gitkeep (100%) rename pkg/module_manager/testdata/{discover_modules_state__module_names_order => modules_state__no_cm__module_names_order}/modules/200-module-b/values.yaml (100%) rename pkg/module_manager/testdata/{discover_modules_state__module_names_order => modules_state__no_cm__module_names_order}/modules/300-module-disabled/values.yaml (100%) rename pkg/module_manager/testdata/{discover_modules_state__simple/modules/004-module-4 => modules_state__no_cm__simple/modules/001-module-1}/.gitkeep (100%) rename pkg/module_manager/testdata/{discover_modules_state__simple/modules/007-module-7 => modules_state__no_cm__simple/modules/003-module-3}/.gitkeep (100%) rename pkg/module_manager/testdata/{discover_modules_state__simple/modules/008-module-8 => modules_state__no_cm__simple/modules/004-module-4}/.gitkeep (100%) rename pkg/module_manager/testdata/{discover_modules_state__simple/modules/009-module-9 => modules_state__no_cm__simple/modules/007-module-7}/.gitkeep (100%) create mode 100644 pkg/module_manager/testdata/modules_state__no_cm__simple/modules/008-module-8/.gitkeep create mode 100644 pkg/module_manager/testdata/modules_state__no_cm__simple/modules/009-module-9/.gitkeep rename pkg/module_manager/testdata/{discover_modules_state__simple => modules_state__no_cm__simple}/modules/values.yaml (100%) rename pkg/module_manager/testdata/{discover_modules_state__with_enabled_scripts => modules_state__no_cm__with_enabled_scripts}/modules/001-alpha/enabled (100%) rename pkg/module_manager/testdata/{discover_modules_state__with_enabled_scripts => modules_state__no_cm__with_enabled_scripts}/modules/002-beta/enabled (100%) rename pkg/module_manager/testdata/{discover_modules_state__with_enabled_scripts => modules_state__no_cm__with_enabled_scripts}/modules/003-gamma/enabled (100%) rename pkg/module_manager/testdata/{discover_modules_state__with_enabled_scripts => modules_state__no_cm__with_enabled_scripts}/modules/004-delta/enabled (100%) rename pkg/module_manager/testdata/{discover_modules_state__with_enabled_scripts => modules_state__no_cm__with_enabled_scripts}/modules/005-epsilon/enabled (100%) rename pkg/module_manager/testdata/{discover_modules_state__with_enabled_scripts => modules_state__no_cm__with_enabled_scripts}/modules/006-zeta/enabled (100%) rename pkg/module_manager/testdata/{discover_modules_state__with_enabled_scripts => modules_state__no_cm__with_enabled_scripts}/modules/007-eta/enabled (100%) rename pkg/module_manager/testdata/{discover_modules_state__with_enabled_scripts => modules_state__no_cm__with_enabled_scripts}/modules/values.yaml (100%) diff --git a/cmd/addon-operator/main.go b/cmd/addon-operator/main.go index 46abe771..9cce4d83 100644 --- a/cmd/addon-operator/main.go +++ b/cmd/addon-operator/main.go @@ -2,14 +2,14 @@ package main import ( "fmt" + "math/rand" "os" + "time" "github.com/flant/kube-client/klogtologrus" - log "github.com/sirupsen/logrus" "gopkg.in/alecthomas/kingpin.v2" sh_app "github.com/flant/shell-operator/pkg/app" - "github.com/flant/shell-operator/pkg/config" "github.com/flant/shell-operator/pkg/debug" utils_signal "github.com/flant/shell-operator/pkg/utils/signal" @@ -40,17 +40,16 @@ func main() { startCmd := kpApp.Command("start", "Start events processing."). Default(). Action(func(c *kingpin.ParseContext) error { - runtimeConfig := config.NewConfig() - // Init logging subsystem. - sh_app.SetupLogging(runtimeConfig) - log.Infof("%s %s, shell-operator %s", app.AppName, app.Version, sh_app.Version) + sh_app.AppStartMessage = fmt.Sprintf("%s %s, shell-operator %s", app.AppName, app.Version, sh_app.Version) - operator := addon_operator.DefaultOperator() - operator.WithRuntimeConfig(runtimeConfig) - err := addon_operator.InitAndStart(operator) + // Init rand generator. + rand.Seed(time.Now().UnixNano()) + + operator, err := addon_operator.Init() if err != nil { os.Exit(1) } + operator.Start() // Block action by waiting signals from OS. utils_signal.WaitForProcessInterruption(func() { diff --git a/go.mod b/go.mod index 0ed8e69b..10f2bfc8 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,12 @@ module github.com/flant/addon-operator -go 1.15 +go 1.16 require ( github.com/davecgh/go-spew v1.1.1 github.com/evanphx/json-patch v4.11.0+incompatible github.com/flant/kube-client v0.0.6 - github.com/flant/shell-operator v1.0.10-0.20220324171037-a48626e8b125 + github.com/flant/shell-operator v1.0.10-0.20220411072300-19eb91114325 github.com/go-chi/chi v4.0.3+incompatible github.com/go-openapi/spec v0.19.8 github.com/go-openapi/strfmt v0.19.5 @@ -14,7 +14,7 @@ require ( github.com/go-openapi/validate v0.19.12 github.com/hashicorp/go-multierror v1.1.1 github.com/kennygrant/sanitize v1.2.4 - github.com/onsi/gomega v1.17.0 + github.com/onsi/gomega v1.18.1 github.com/peterbourgon/mergemap v0.0.0-20130613134717-e21c03b7a721 github.com/prometheus/client_golang v1.11.0 github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734 @@ -42,3 +42,5 @@ replace github.com/go-openapi/validate => github.com/flant/go-openapi-validate v replace k8s.io/client-go => k8s.io/client-go v0.19.11 replace k8s.io/api => k8s.io/api v0.19.11 + +replace github.com/flant/shell-operator => ../shell-operator diff --git a/go.sum b/go.sum index e6fd8d95..62966cfa 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,8 @@ github.com/flant/kube-client v0.0.6 h1:cHFMf7xGtJOgg+KBuPcA+7q+M7IJRSgf2pHVStv2a github.com/flant/kube-client v0.0.6/go.mod h1:pVKIewJQ5oaBiE6AlTaWAUkd0548DEiyvkqkLaby3Zg= github.com/flant/libjq-go v1.6.2-0.20200616114952-907039e8a02a h1:PlStPekqPtTSWDDKFlwgETsT1OiXD1gZtRHcNxAs1lc= github.com/flant/libjq-go v1.6.2-0.20200616114952-907039e8a02a/go.mod h1:+SYqi5wsNjtQVlkPg0Ep5IOuN+ydg79Jo/gk4/PuS8c= -github.com/flant/shell-operator v1.0.10-0.20220324171037-a48626e8b125 h1:XaPqZE2PtFC0DQFHMX2w84SP+bAD3tYpRbFmOZVYMQk= -github.com/flant/shell-operator v1.0.10-0.20220324171037-a48626e8b125/go.mod h1:bHcTpRq0k0c/kaVQl6sODi/Nz8mqmtmMk9ff9dxrpN4= +github.com/flant/shell-operator v1.0.10-0.20220411072300-19eb91114325 h1:3E2JrvRsJvr6nLc8EIJuE29kTIqQdcAvX/neZKwN5oI= +github.com/flant/shell-operator v1.0.10-0.20220411072300-19eb91114325/go.mod h1:bHcTpRq0k0c/kaVQl6sODi/Nz8mqmtmMk9ff9dxrpN4= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -391,6 +391,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -454,6 +455,7 @@ github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= @@ -606,6 +608,8 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -614,8 +618,9 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -983,8 +988,9 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pkg/addon-operator/bootstrap.go b/pkg/addon-operator/bootstrap.go new file mode 100644 index 00000000..b2d03878 --- /dev/null +++ b/pkg/addon-operator/bootstrap.go @@ -0,0 +1,130 @@ +package addon_operator + +import ( + "context" + "fmt" + + sh_app "github.com/flant/shell-operator/pkg/app" + "github.com/flant/shell-operator/pkg/config" + "github.com/flant/shell-operator/pkg/debug" + shell_operator "github.com/flant/shell-operator/pkg/shell-operator" + log "github.com/sirupsen/logrus" + + "github.com/flant/addon-operator/pkg/app" + "github.com/flant/addon-operator/pkg/helm" + "github.com/flant/addon-operator/pkg/kube_config_manager" + "github.com/flant/addon-operator/pkg/module_manager" +) + +func Init() (*AddonOperator, error) { + runtimeConfig := config.NewConfig() + // Init logging subsystem. + sh_app.SetupLogging(runtimeConfig) + log.Infof(sh_app.AppStartMessage) + + modulesDir, err := shell_operator.RequireExistingDirectory(app.ModulesDir) + if err != nil { + log.Errorf("Fatal: modules directory: %s", err) + return nil, err + } + log.Infof("Modules directory: %s", modulesDir) + + globalHooksDir, err := shell_operator.RequireExistingDirectory(app.GlobalHooksDir) + if err != nil { + log.Errorf("Fatal: global hooks directory: %s", err) + return nil, err + } + log.Infof("Global hooks directory: %s", globalHooksDir) + + tempDir, err := shell_operator.EnsureTempDirectory(sh_app.TempDir) + if err != nil { + log.Errorf("Fatal: temp directory: %s", err) + return nil, err + } + + log.Infof("Addon-operator namespace: %s", app.Namespace) + + op := NewAddonOperator() + op.WithContext(context.Background()) + + // Debug server. + debugServer, err := shell_operator.InitDefaultDebugServer() + if err != nil { + log.Errorf("Fatal: start Debug server: %s", err) + return nil, err + } + + err = shell_operator.AssembleCommonOperator(op.ShellOperator) + if err != nil { + log.Errorf("Fatal: %s", err) + return nil, err + } + + err = AssembleAddonOperator(op, modulesDir, globalHooksDir, tempDir, debugServer, runtimeConfig) + if err != nil { + log.Errorf("Fatal: %s", err) + return nil, err + } + + return op, nil +} + +func AssembleAddonOperator(op *AddonOperator, modulesDir string, globalHooksDir string, tempDir string, debugServer *debug.Server, runtimeConfig *config.Config) (err error) { + RegisterDefaultRoutes(op) + RegisterAddonOperatorMetrics(op.MetricStorage) + StartLiveTicksUpdater(op.MetricStorage) + StartTasksQueueLengthUpdater(op.MetricStorage, op.TaskQueues) + + // Register routes in debug server. + shell_operator.RegisterDebugQueueRoutes(debugServer, op.ShellOperator) + shell_operator.RegisterDebugConfigRoutes(debugServer, runtimeConfig) + RegisterDebugGlobalRoutes(debugServer, op) + RegisterDebugModuleRoutes(debugServer, op) + + // Helm client factory. + op.Helm = helm.New() + op.Helm.WithKubeClient(op.KubeClient) + err = op.Helm.Init() + if err != nil { + return fmt.Errorf("initialize Helm: %s", err) + } + + // Helm resources monitor. + // It uses a separate client-go instance. (Metrics are registered when 'main' client is initialized). + op.HelmResourcesManager, err = InitDefaultHelmResourcesManager(op.ctx, op.MetricStorage) + if err != nil { + return fmt.Errorf("initialize Helm resources manager: %s", err) + } + + SetupModuleManager(op, modulesDir, globalHooksDir, tempDir, runtimeConfig) + + err = op.InitModuleManager() + if err != nil { + return err + } + + return nil +} + +func SetupModuleManager(op *AddonOperator, modulesDir string, globalHooksDir string, tempDir string, runtimeConfig *config.Config) { + // Create manager to check values in ConfigMap. + op.KubeConfigManager = kube_config_manager.NewKubeConfigManager() + op.KubeConfigManager.WithKubeClient(op.KubeClient) + op.KubeConfigManager.WithContext(op.ctx) + op.KubeConfigManager.WithNamespace(app.Namespace) + op.KubeConfigManager.WithConfigMapName(app.ConfigMapName) + op.KubeConfigManager.WithRuntimeConfig(runtimeConfig) + + // Create manager that runs modules and hooks. + op.ModuleManager = module_manager.NewModuleManager() + op.ModuleManager.WithContext(op.ctx) + op.ModuleManager.WithDirectories(modulesDir, globalHooksDir, tempDir) + op.ModuleManager.WithKubeConfigManager(op.KubeConfigManager) + op.ModuleManager.WithHelm(op.Helm) + op.ModuleManager.WithScheduleManager(op.ScheduleManager) + op.ModuleManager.WithKubeEventManager(op.KubeEventsManager) + op.ModuleManager.WithKubeObjectPatcher(op.ObjectPatcher) + op.ModuleManager.WithMetricStorage(op.MetricStorage) + op.ModuleManager.WithHookMetricStorage(op.HookMetricStorage) + op.ModuleManager.WithHelmResourcesManager(op.HelmResourcesManager) +} diff --git a/pkg/addon-operator/converge.go b/pkg/addon-operator/converge.go new file mode 100644 index 00000000..19d5d41c --- /dev/null +++ b/pkg/addon-operator/converge.go @@ -0,0 +1,91 @@ +package addon_operator + +import ( + "time" + + sh_task "github.com/flant/shell-operator/pkg/task" + + . "github.com/flant/addon-operator/pkg/hook/types" + "github.com/flant/addon-operator/pkg/task" +) + +type ConvergeState struct { + Phase ConvergePhase + FirstStarted bool + FirstDone bool + StartedAt int64 + Activation string +} + +type ConvergePhase string + +const ( + StandBy ConvergePhase = "StandBy" + RunBeforeAll ConvergePhase = "RunBeforeAll" + WaitBeforeAll ConvergePhase = "WaitBeforeAll" + WaitDeleteAndRunModules ConvergePhase = "WaitDeleteAndRunModules" + WaitAfterAll ConvergePhase = "WaitAfterAll" +) + +func NewConvergeState() *ConvergeState { + return &ConvergeState{ + Phase: StandBy, + } +} + +const ConvergeEventProp = "converge.event" + +type ConvergeEvent string + +const ( + // OperatorStartup is a first converge during startup. + OperatorStartup ConvergeEvent = "OperatorStartup" + // GlobalValuesChanged is a converge initiated by changing values in the global hook. + GlobalValuesChanged ConvergeEvent = "GlobalValuesChanged" + // KubeConfigChanged is a converge started after changing ConfigMap. + KubeConfigChanged ConvergeEvent = "KubeConfigChanged" + // ReloadAllModules is a converge queued to the + ReloadAllModules ConvergeEvent = "ReloadAllModules" +) + +func IsConvergeTask(t sh_task.Task) bool { + taskType := t.GetType() + switch taskType { + case task.ModuleDelete, task.ModuleRun, task.ConvergeModules: + return true + } + hm := task.HookMetadataAccessor(t) + switch taskType { + case task.GlobalHookRun: + switch hm.BindingType { + case BeforeAll, AfterAll: + return true + } + case task.ModuleHookRun: + if hm.IsSynchronization() { + return true + } + } + return false +} + +func IsFirstConvergeTask(t sh_task.Task) bool { + taskType := t.GetType() + switch taskType { + case task.ModulePurge, task.DiscoverHelmReleases, task.GlobalHookEnableKubernetesBindings, task.GlobalHookEnableScheduleBindings: + return true + } + return false +} + +func NewConvergeModulesTask(description string, convergeEvent ConvergeEvent, logLabels map[string]string) sh_task.Task { + convergeTask := sh_task.NewTask(task.ConvergeModules). + WithLogLabels(logLabels). + WithQueueName("main"). + WithMetadata(task.HookMetadata{ + EventDescription: description, + }). + WithQueuedAt(time.Now()) + convergeTask.SetProp(ConvergeEventProp, convergeEvent) + return convergeTask +} diff --git a/pkg/addon-operator/debug_server.go b/pkg/addon-operator/debug_server.go new file mode 100644 index 00000000..fd2bd89c --- /dev/null +++ b/pkg/addon-operator/debug_server.go @@ -0,0 +1,130 @@ +package addon_operator + +import ( + "fmt" + "net/http" + "os" + + "github.com/flant/shell-operator/pkg/debug" + "github.com/flant/shell-operator/pkg/hook/types" + "github.com/go-chi/chi" + + "github.com/flant/addon-operator/pkg/app" +) + +func RegisterDebugGlobalRoutes(dbgSrv *debug.Server, op *AddonOperator) { + dbgSrv.Route("/global/list.{format:(json|yaml)}", func(_ *http.Request) (interface{}, error) { + return map[string]interface{}{ + "globalHooks": op.ModuleManager.GetGlobalHooksNames(), + }, nil + }) + + dbgSrv.Route("/global/values.{format:(json|yaml)}", func(_ *http.Request) (interface{}, error) { + return op.ModuleManager.GlobalValues() + }) + + dbgSrv.Route("/global/config.{format:(json|yaml)}", func(_ *http.Request) (interface{}, error) { + return op.ModuleManager.GlobalConfigValues(), nil + }) + + dbgSrv.Route("/global/patches.{format:(json|yaml)}", func(_ *http.Request) (interface{}, error) { + return op.ModuleManager.GlobalValuesPatches(), nil + }) + + dbgSrv.Route("/global/snapshots.{format:(json|yaml)}", func(r *http.Request) (interface{}, error) { + kubeHookNames := op.ModuleManager.GetGlobalHooksInOrder(types.OnKubernetesEvent) + snapshots := make(map[string]interface{}) + for _, hName := range kubeHookNames { + h := op.ModuleManager.GetGlobalHook(hName) + snapshots[hName] = h.HookController.SnapshotsDump() + } + + return snapshots, nil + }) +} + +func RegisterDebugModuleRoutes(dbgSrv *debug.Server, op *AddonOperator) { + dbgSrv.Route("/module/list.{format:(json|yaml|text)}", func(_ *http.Request) (interface{}, error) { + return map[string][]string{"enabledModules": op.ModuleManager.GetEnabledModuleNames()}, nil + }) + + dbgSrv.Route("/module/{name}/{type:(config|values)}.{format:(json|yaml)}", func(r *http.Request) (interface{}, error) { + modName := chi.URLParam(r, "name") + valType := chi.URLParam(r, "type") + + m := op.ModuleManager.GetModule(modName) + if m == nil { + return nil, fmt.Errorf("Module not found") + } + + switch valType { + case "config": + return m.ConfigValues(), nil + case "values": + return m.Values() + } + return "no values", nil + }) + + dbgSrv.Route("/module/{name}/render", func(r *http.Request) (interface{}, error) { + modName := chi.URLParam(r, "name") + + m := op.ModuleManager.GetModule(modName) + if m == nil { + return nil, fmt.Errorf("Module not found") + } + + valuesPath, err := m.PrepareValuesYamlFile() + if err != nil { + return nil, err + } + defer os.Remove(valuesPath) + + return op.Helm.NewClient().Render(m.Name, m.Path, []string{valuesPath}, nil, app.Namespace) + }) + + dbgSrv.Route("/module/{name}/patches.json", func(r *http.Request) (interface{}, error) { + modName := chi.URLParam(r, "name") + + m := op.ModuleManager.GetModule(modName) + if m == nil { + return nil, fmt.Errorf("Module not found") + } + + return m.ValuesPatches(), nil + }) + + dbgSrv.Route("/module/resource-monitor.{format:(json|yaml)}", func(_ *http.Request) (interface{}, error) { + dump := map[string]interface{}{} + + for _, moduleName := range op.ModuleManager.GetEnabledModuleNames() { + if !op.HelmResourcesManager.HasMonitor(moduleName) { + dump[moduleName] = "No monitor" + continue + } + + ids := op.HelmResourcesManager.GetMonitor(moduleName).ResourceIds() + dump[moduleName] = ids + } + + return dump, nil + }) + + dbgSrv.Route("/module/{name}/snapshots.{format:(json|yaml)}", func(r *http.Request) (interface{}, error) { + modName := chi.URLParam(r, "name") + + m := op.ModuleManager.GetModule(modName) + if m == nil { + return nil, fmt.Errorf("Module not found") + } + + mHookNames := op.ModuleManager.GetModuleHookNames(m.Name) + snapshots := make(map[string]interface{}) + for _, hName := range mHookNames { + h := op.ModuleManager.GetModuleHook(hName) + snapshots[hName] = h.HookController.SnapshotsDump() + } + + return snapshots, nil + }) +} diff --git a/pkg/addon-operator/http_server.go b/pkg/addon-operator/http_server.go new file mode 100644 index 00000000..2541ec7f --- /dev/null +++ b/pkg/addon-operator/http_server.go @@ -0,0 +1,71 @@ +package addon_operator + +import ( + "fmt" + "net/http" + "strings" + + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func RegisterDefaultRoutes(op *AddonOperator) { + http.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) { + _, _ = writer.Write([]byte(` + Addon-operator + +

Addon-operator

+ 
go tool pprof goprofex http://ADDON_OPERATOR_IP:9115/debug/pprof/profile
+

+ prometheus metrics + health url +

+ + `)) + }) + http.Handle("/metrics", promhttp.Handler()) + + http.HandleFunc("/healthz", func(writer http.ResponseWriter, request *http.Request) { + helmHealthHandler := op.Helm.HealthzHandler() + if helmHealthHandler == nil { + writer.WriteHeader(http.StatusOK) + return + } + helmHealthHandler(writer, request) + }) + + http.HandleFunc("/ready", func(w http.ResponseWriter, request *http.Request) { + if op.IsStartupConvergeDone() { + w.WriteHeader(200) + _, _ = w.Write([]byte("Startup converge done.\n")) + } else { + w.WriteHeader(500) + _, _ = w.Write([]byte("Startup converge in progress\n")) + } + }) + + http.HandleFunc("/status/converge", func(writer http.ResponseWriter, request *http.Request) { + convergeTasks := ConvergeTasksInQueue(op.TaskQueues.GetMain()) + + statusLines := make([]string, 0) + if op.IsStartupConvergeDone() { + statusLines = append(statusLines, "STARTUP_CONVERGE_DONE") + if convergeTasks > 0 { + statusLines = append(statusLines, fmt.Sprintf("CONVERGE_IN_PROGRESS: %d tasks", convergeTasks)) + } else { + statusLines = append(statusLines, "CONVERGE_WAIT_TASK") + } + } else { + if op.ConvergeState.FirstStarted { + if convergeTasks > 0 { + statusLines = append(statusLines, fmt.Sprintf("STARTUP_CONVERGE_IN_PROGRESS: %d tasks", convergeTasks)) + } else { + statusLines = append(statusLines, "STARTUP_CONVERGE_DONE") + } + } else { + statusLines = append(statusLines, "STARTUP_CONVERGE_WAIT_TASKS") + } + } + + _, _ = writer.Write([]byte(strings.Join(statusLines, "\n") + "\n")) + }) +} diff --git a/pkg/addon-operator/kube_client.go b/pkg/addon-operator/kube_client.go index 4e0be083..12b5d152 100644 --- a/pkg/addon-operator/kube_client.go +++ b/pkg/addon-operator/kube_client.go @@ -1,31 +1,40 @@ package addon_operator import ( - "github.com/flant/addon-operator/pkg/app" + "context" + "fmt" + klient "github.com/flant/kube-client/client" sh_app "github.com/flant/shell-operator/pkg/app" + "github.com/flant/shell-operator/pkg/metric_storage" + shell_operator "github.com/flant/shell-operator/pkg/shell-operator" + + "github.com/flant/addon-operator/pkg/app" + "github.com/flant/addon-operator/pkg/helm_resources_manager" ) // Important! These labels should be consistent with similar labels in ShellOperator! var DefaultHelmMonitorKubeClientMetricLabels = map[string]string{"component": "helm_monitor"} -func (op *AddonOperator) GetHelmMonitorKubeClientMetricLabels() map[string]string { - if op.HelmMonitorKubeClientMetricLabels == nil { - return DefaultHelmMonitorKubeClientMetricLabels - } - return op.HelmMonitorKubeClientMetricLabels -} - -// InitHelmMonitorKubeClient initializes a Kubernetes client for helm monitor. -func (op *AddonOperator) InitHelmMonitorKubeClient() (klient.Client, error) { +// DefaultHelmMonitorKubeClient initializes a Kubernetes client for helm monitor. +func DefaultHelmMonitorKubeClient(metricStorage *metric_storage.MetricStorage, metricLabels map[string]string) klient.Client { client := klient.New() client.WithContextName(sh_app.KubeContext) client.WithConfigPath(sh_app.KubeConfig) client.WithRateLimiterSettings(app.HelmMonitorKubeClientQps, app.HelmMonitorKubeClientBurst) - client.WithMetricStorage(op.MetricStorage) - client.WithMetricLabels(op.GetHelmMonitorKubeClientMetricLabels()) - if err := client.Init(); err != nil { - return nil, err + client.WithMetricStorage(metricStorage) + client.WithMetricLabels(shell_operator.DefaultIfEmpty(metricLabels, DefaultHelmMonitorKubeClientMetricLabels)) + return client +} + +func InitDefaultHelmResourcesManager(ctx context.Context, metricStorage *metric_storage.MetricStorage) (helm_resources_manager.HelmResourcesManager, error) { + kubeClient := DefaultHelmMonitorKubeClient(metricStorage, DefaultHelmMonitorKubeClientMetricLabels) + if err := kubeClient.Init(); err != nil { + return nil, fmt.Errorf("initialize Kubernetes client for Helm resources manager: %s\n", err) } - return client, nil + mgr := helm_resources_manager.NewHelmResourcesManager() + mgr.WithContext(ctx) + mgr.WithKubeClient(kubeClient) + mgr.WithDefaultNamespace(app.Namespace) + return mgr, nil } diff --git a/pkg/addon-operator/metrics.go b/pkg/addon-operator/metrics.go index 7835f0b8..279f6856 100644 --- a/pkg/addon-operator/metrics.go +++ b/pkg/addon-operator/metrics.go @@ -1,8 +1,11 @@ package addon_operator import ( + "time" + "github.com/flant/shell-operator/pkg/metric_storage" sh_op "github.com/flant/shell-operator/pkg/shell-operator" + "github.com/flant/shell-operator/pkg/task/queue" ) func RegisterAddonOperatorMetrics(metricStorage *metric_storage.MetricStorage) { @@ -133,3 +136,26 @@ func RegisterHookMetrics(metricStorage *metric_storage.MetricStorage) { "queue": "", }) } + +func StartLiveTicksUpdater(metricStorage *metric_storage.MetricStorage) { + // Addon-operator live ticks. + go func() { + for { + metricStorage.CounterAdd("{PREFIX}live_ticks", 1.0, map[string]string{}) + time.Sleep(10 * time.Second) + } + }() +} + +func StartTasksQueueLengthUpdater(metricStorage *metric_storage.MetricStorage, tqs *queue.TaskQueueSet) { + go func() { + for { + // Gather task queues lengths. + tqs.Iterate(func(queue *queue.TaskQueue) { + queueLen := float64(queue.Length()) + metricStorage.GaugeSet("{PREFIX}tasks_queue_length", queueLen, map[string]string{"queue": queue.Name}) + }) + time.Sleep(5 * time.Second) + } + }() +} diff --git a/pkg/addon-operator/operator.go b/pkg/addon-operator/operator.go index b173ff7d..6a6231ef 100644 --- a/pkg/addon-operator/operator.go +++ b/pkg/addon-operator/operator.go @@ -3,31 +3,23 @@ package addon_operator import ( "context" "fmt" - "net/http" _ "net/http/pprof" - "os" "path" "runtime/trace" "strings" "time" - klient "github.com/flant/kube-client/client" - sh_app "github.com/flant/shell-operator/pkg/app" . "github.com/flant/shell-operator/pkg/hook/binding_context" "github.com/flant/shell-operator/pkg/hook/controller" . "github.com/flant/shell-operator/pkg/hook/types" "github.com/flant/shell-operator/pkg/kube_events_manager/types" - "github.com/flant/shell-operator/pkg/metric_storage" shell_operator "github.com/flant/shell-operator/pkg/shell-operator" sh_task "github.com/flant/shell-operator/pkg/task" "github.com/flant/shell-operator/pkg/task/queue" . "github.com/flant/shell-operator/pkg/utils/measure" - "github.com/go-chi/chi" - "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" uuid "gopkg.in/satori/go.uuid.v1" - "github.com/flant/addon-operator/pkg/app" "github.com/flant/addon-operator/pkg/helm" "github.com/flant/addon-operator/pkg/helm_resources_manager" . "github.com/flant/addon-operator/pkg/hook/types" @@ -44,40 +36,29 @@ type AddonOperator struct { ctx context.Context cancel context.CancelFunc - ModulesDir string - GlobalHooksDir string - + // KubeConfigManager monitors changes in ConfigMap. KubeConfigManager kube_config_manager.KubeConfigManager // ModuleManager is the module manager object, which monitors configuration // and variable changes. ModuleManager module_manager.ModuleManager + Helm *helm.Helm + + // HelmResourcesManager monitors absent resources created for modules. HelmResourcesManager helm_resources_manager.HelmResourcesManager // converge state - StartupConvergeStarted bool - StartupConvergeDone bool - ConvergeStarted int64 - ConvergeActivation string - - HelmMonitorKubeClientMetricLabels map[string]string + ConvergeState *ConvergeState } func NewAddonOperator() *AddonOperator { return &AddonOperator{ ShellOperator: &shell_operator.ShellOperator{}, + ConvergeState: NewConvergeState(), } } -func (op *AddonOperator) WithModulesDir(dir string) { - op.ModulesDir = dir -} - -func (op *AddonOperator) WithGlobalHooksDir(dir string) { - op.GlobalHooksDir = dir -} - func (op *AddonOperator) WithContext(ctx context.Context) *AddonOperator { op.ctx, op.cancel = context.WithCancel(ctx) op.ShellOperator.WithContext(op.ctx) @@ -91,145 +72,55 @@ func (op *AddonOperator) Stop() { } func (op *AddonOperator) IsStartupConvergeDone() bool { - return op.StartupConvergeDone -} - -func (op *AddonOperator) SetStartupConvergeDone() { - op.StartupConvergeDone = true -} - -// InitMetricStorage creates new MetricStorage instance in AddonOperator -// if it is not initialized yet. Run it before Init() to override default -// MetricStorage instance in shell-operator. -func (op *AddonOperator) InitMetricStorage() { - if op.MetricStorage != nil { - return - } - // Metric storage. - metricStorage := metric_storage.NewMetricStorage() - metricStorage.WithContext(op.ctx) - metricStorage.WithPrefix(sh_app.PrometheusMetricsPrefix) - metricStorage.Start() - RegisterAddonOperatorMetrics(metricStorage) - op.MetricStorage = metricStorage + return op.ConvergeState.FirstDone } -// InitModuleManager initialize objects for addon-operator. -// This method should run after Init(). -// -// Addon-operator settings: -// -// - directory with modules -// - directory with global hooks -// - dump file path -// -// Objects: -// - helm client -// - kube config manager -// - module manager -// -// Also set handlers for task types and handlers to emit tasks. +// InitModuleManager initialize KubeConfigManager and ModuleManager, +// reads values from ConfigMap for the first time and sets handlers +// for kubernetes and schedule events. func (op *AddonOperator) InitModuleManager() error { - logLabels := map[string]string{ - "operator.component": "Init", - } - logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) - var err error - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("get working directory of process: %s", err) - } - - // TODO: check if directories are existed - op.ModulesDir = os.Getenv("MODULES_DIR") - if op.ModulesDir == "" { - op.ModulesDir = path.Join(cwd, app.ModulesDir) - } - op.GlobalHooksDir = os.Getenv("GLOBAL_HOOKS_DIR") - if op.GlobalHooksDir == "" { - op.GlobalHooksDir = path.Join(cwd, app.GlobalHooksDir) - } - logEntry.Infof("Global hooks directory: %s", op.GlobalHooksDir) - logEntry.Infof("Modules directory: %s", op.ModulesDir) - - logEntry.Infof("Addon-operator namespace: %s", app.Namespace) - - // Initialize helm client, choose helm3 or helm2+tiller - err = helm.Init(op.KubeClient) - if err != nil { - return err - } - // Initializing ConfigMap storage for values - op.KubeConfigManager = kube_config_manager.NewKubeConfigManager() - op.KubeConfigManager.WithKubeClient(op.KubeClient) - op.KubeConfigManager.WithContext(op.ctx) - op.KubeConfigManager.WithNamespace(app.Namespace) - op.KubeConfigManager.WithConfigMapName(app.ConfigMapName) - err = op.KubeConfigManager.Init() if err != nil { return fmt.Errorf("init kube config manager: %s", err) } - op.ModuleManager = module_manager.NewMainModuleManager() - op.ModuleManager.WithContext(op.ctx) - op.ModuleManager.WithDirectories(op.ModulesDir, op.GlobalHooksDir, op.TempDir) - op.ModuleManager.WithKubeConfigManager(op.KubeConfigManager) - op.ModuleManager.WithScheduleManager(op.ScheduleManager) - op.ModuleManager.WithKubeEventManager(op.KubeEventsManager) - op.ModuleManager.WithKubeObjectPatcher(op.ObjectPatcher) - op.ModuleManager.WithMetricStorage(op.MetricStorage) - op.ModuleManager.WithHookMetricStorage(op.HookMetricStorage) err = op.ModuleManager.Init() if err != nil { return fmt.Errorf("init module manager: %s", err) } - op.DefineEventHandlers() - - // Helm resources monitor. - // Use separate client-go instance. (Metrics are registered when 'main' client is initialized). - helmMonitorKubeClient, err := op.InitHelmMonitorKubeClient() + // Load existing config values from ConfigMap. + op.KubeConfigManager.SafeReadConfig(func(config *kube_config_manager.KubeConfig) { + _, err = op.ModuleManager.HandleNewKubeConfig(config) + }) if err != nil { - log.Errorf("MAIN Fatal: initialize kube client for helm: %s\n", err) - return err + return fmt.Errorf("init module manager with kube config: %s", err) } - // Init helm resources manager. - op.HelmResourcesManager = helm_resources_manager.NewHelmResourcesManager() - op.HelmResourcesManager.WithContext(op.ctx) - op.HelmResourcesManager.WithKubeClient(helmMonitorKubeClient) - op.HelmResourcesManager.WithDefaultNamespace(app.Namespace) - op.ModuleManager.WithHelmResourcesManager(op.HelmResourcesManager) + // Initialize 'valid kube config' flag. + op.ModuleManager.SetKubeConfigValid(true) + + // ManagerEventsHandlers created, register handlers to create tasks from events. + op.RegisterManagerEventsHandlers() return nil } -func (op *AddonOperator) NeedAddCrontabTask(hook *module_manager.CommonHook) bool { - if op.IsStartupConvergeDone() { +// AllowHandleScheduleEvent returns false if the Schedule event can be ignored. +func (op *AddonOperator) AllowHandleScheduleEvent(hook *module_manager.CommonHook) bool { + // Always allow if first converge is done. + if op.ConvergeState.FirstDone { return true } - // converge not done into next lines - - // shell hooks will be scheduled after converge done - // TODO maybe need to add parameter to ShellOperator same to go hooks - if hook.GoHook == nil { - return false - } - - s := hook.GoHook.Config().Settings - - if s != nil && s.EnableSchedulesOnStartup { - return true - } - - return false + // Allow when first converge is still in progress, + // but hook explicitly enable schedules after OnStartup. + return hook.ShouldEnableSchedulesOnStartup() } -func (op *AddonOperator) DefineEventHandlers() { +func (op *AddonOperator) RegisterManagerEventsHandlers() { op.ManagerEventsHandler.WithScheduleEventHandler(func(crontab string) []sh_task.Task { logLabels := map[string]string{ "event.id": uuid.NewV4().String(), @@ -241,7 +132,7 @@ func (op *AddonOperator) DefineEventHandlers() { var tasks []sh_task.Task err := op.ModuleManager.HandleScheduleEvent(crontab, func(globalHook *module_manager.GlobalHook, info controller.BindingExecutionInfo) { - if !op.NeedAddCrontabTask(globalHook.CommonHook) { + if !op.AllowHandleScheduleEvent(globalHook.CommonHook) { return } @@ -269,7 +160,7 @@ func (op *AddonOperator) DefineEventHandlers() { tasks = append(tasks, newTask) }, func(module *module_manager.Module, moduleHook *module_manager.ModuleHook, info controller.BindingExecutionInfo) { - if !op.NeedAddCrontabTask(moduleHook.CommonHook) { + if !op.AllowHandleScheduleEvent(moduleHook.CommonHook) { return } @@ -303,6 +194,7 @@ func (op *AddonOperator) DefineEventHandlers() { return []sh_task.Task{} } + op.logTaskAdd(logEntry, "Schedule event received, append", tasks...) return tasks }) @@ -370,64 +262,77 @@ func (op *AddonOperator) DefineEventHandlers() { tasks = append(tasks, newTask) }) + op.logTaskAdd(logEntry, "Kubernetes event received, append", tasks...) return tasks }) } -// Run runs all managers, event and queue handlers. -// -// The main process is blocked by the 'for-select' in the queue handler. +// Start runs all managers, event and queue handlers. func (op *AddonOperator) Start() { log.Info("start addon-operator") // Loading the onStartup hooks into the queue and running all modules. // Turning tracking changes on only after startup ends. - // Start emit "live" metrics - op.RunAddonOperatorMetrics() - - // Prepopulate main queue with onStartup tasks and enable kubernetes bindings tasks. - op.PrepopulateMainQueue(op.TaskQueues) + // Bootstrap main queue with tasks to run Startup process. + op.BootstrapMainQueue(op.TaskQueues) // Start main task queue handler op.TaskQueues.StartMain() - op.InitAndStartHookQueues() + // Global hooks are registered, initialize their queues. + op.CreateAndStartQueuesForGlobalHooks() - // Managers are generating events. This go-routine handles all events and converts them into queued tasks. + // ManagerEventsHandler handle events for kubernetes and schedule bindings. // Start it before start all informers to catch all kubernetes events (#42) op.ManagerEventsHandler.Start() - // add schedules to schedule manager - //op.HookManager.EnableScheduleBindings() + // Enable events from schedule manager. op.ScheduleManager.Start() + op.KubeConfigManager.Start() op.ModuleManager.Start() op.StartModuleManagerEventHandler() } -// PrepopulateMainQueue adds tasks to run hooks with OnStartup bindings -// and tasks to enable kubernetes bindings. -func (op *AddonOperator) PrepopulateMainQueue(tqs *queue.TaskQueueSet) { - onStartupLabels := map[string]string{} - onStartupLabels["event.type"] = "OperatorStartup" - +// BootstrapMainQueue adds tasks to initiate Startup sequence: +// +// - Run onStartup hooks. +// - Enable global schedule bindings. +// - Enable kubernetes bindings: run Synchronization tasks. +// - Purge unknown Helm releases. +// - Start reload all modules. +func (op *AddonOperator) BootstrapMainQueue(tqs *queue.TaskQueueSet) { + logLabels := map[string]string{ + "event.type": "OperatorStartup", + } // create onStartup for global hooks - logEntry := log.WithFields(utils.LabelsToLogFields(onStartupLabels)) + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) // Prepopulate main queue with 'onStartup' and 'enable kubernetes bindings' tasks for // global hooks and add a task to discover modules state. tqs.WithMainName("main") tqs.NewNamedQueue("main", op.TaskHandler) - onStartupHooks := op.ModuleManager.GetGlobalHooksInOrder(OnStartup) + tasks := op.CreateBootstrapTasks(logLabels) + op.logTaskAdd(logEntry, "append", tasks...) + for _, tsk := range tasks { + op.TaskQueues.GetMain().AddLast(tsk) + } +} +func (op *AddonOperator) CreateBootstrapTasks(logLabels map[string]string) []sh_task.Task { + const eventDescription = "BootstrapMainQueue" + var tasks = make([]sh_task.Task, 0) + var queuedAt = time.Now() + + // 'OnStartup' global hooks. + onStartupHooks := op.ModuleManager.GetGlobalHooksInOrder(OnStartup) for _, hookName := range onStartupHooks { - hookLogLabels := utils.MergeLabels(onStartupLabels, map[string]string{ + hookLogLabels := utils.MergeLabels(logLabels, map[string]string{ "hook": hookName, "hook.type": "global", "queue": "main", "binding": string(OnStartup), }) - //delete(hookLogLabels, "task.id") onStartupBindingContext := BindingContext{Binding: string(OnStartup)} onStartupBindingContext.Metadata.BindingType = OnStartup @@ -436,21 +341,19 @@ func (op *AddonOperator) PrepopulateMainQueue(tqs *queue.TaskQueueSet) { WithLogLabels(hookLogLabels). WithQueueName("main"). WithMetadata(task.HookMetadata{ - EventDescription: "PrepopulateMainQueue", + EventDescription: eventDescription, HookName: hookName, BindingType: OnStartup, BindingContext: []BindingContext{onStartupBindingContext}, ReloadAllOnValuesChanges: false, }) - op.TaskQueues.GetMain().AddLast(newTask.WithQueuedAt(time.Now())) - - logEntry.WithFields(utils.LabelsToLogFields(newTask.LogLabels)). - Infof("queue task %s", newTask.GetDescription()) + tasks = append(tasks, newTask.WithQueuedAt(queuedAt)) } + // 'Schedule' global hooks. schedHooks := op.ModuleManager.GetGlobalHooksInOrder(Schedule) for _, hookName := range schedHooks { - hookLogLabels := utils.MergeLabels(onStartupLabels, map[string]string{ + hookLogLabels := utils.MergeLabels(logLabels, map[string]string{ "hook": hookName, "hook.type": "global", "queue": "main", @@ -461,41 +364,34 @@ func (op *AddonOperator) PrepopulateMainQueue(tqs *queue.TaskQueueSet) { WithLogLabels(hookLogLabels). WithQueueName("main"). WithMetadata(task.HookMetadata{ - EventDescription: "PrepopulateMainQueue", + EventDescription: eventDescription, HookName: hookName, }) - op.TaskQueues.GetMain().AddLast(newTask.WithQueuedAt(time.Now())) - - logEntry.WithFields(utils.LabelsToLogFields(newTask.LogLabels)). - Infof("queue task %s", newTask.GetDescription()) + tasks = append(tasks, newTask.WithQueuedAt(queuedAt)) } - // create tasks to enable kubernetes events for all global hooks with kubernetes bindings + // Tasks to enable kubernetes events for all global hooks with kubernetes bindings. kubeHooks := op.ModuleManager.GetGlobalHooksInOrder(OnKubernetesEvent) for _, hookName := range kubeHooks { - hookLogLabels := utils.MergeLabels(onStartupLabels, map[string]string{ + hookLogLabels := utils.MergeLabels(logLabels, map[string]string{ "hook": hookName, "hook.type": "global", "queue": "main", "binding": string(task.GlobalHookEnableKubernetesBindings), }) - //delete(hookLogLabels, "task.id") newTask := sh_task.NewTask(task.GlobalHookEnableKubernetesBindings). WithLogLabels(hookLogLabels). WithQueueName("main"). WithMetadata(task.HookMetadata{ - EventDescription: "PrepopulateMainQueue", + EventDescription: eventDescription, HookName: hookName, }) - op.TaskQueues.GetMain().AddLast(newTask.WithQueuedAt(time.Now())) - - logEntry.WithFields(utils.LabelsToLogFields(newTask.LogLabels)). - Infof("queue task %s", newTask.GetDescription()) + tasks = append(tasks, newTask.WithQueuedAt(queuedAt)) } - // wait for kubernetes.Synchronization - waitLogLabels := utils.MergeLabels(onStartupLabels, map[string]string{ + // Task to wait for kubernetes.Synchronization. + waitLogLabels := utils.MergeLabels(logLabels, map[string]string{ "queue": "main", "binding": string(task.GlobalHookWaitKubernetesSynchronization), }) @@ -503,34 +399,212 @@ func (op *AddonOperator) PrepopulateMainQueue(tqs *queue.TaskQueueSet) { WithLogLabels(waitLogLabels). WithQueueName("main"). WithMetadata(task.HookMetadata{ - EventDescription: "PrepopulateMainQueue", + EventDescription: eventDescription, }) - op.TaskQueues.GetMain().AddLast(waitTask.WithQueuedAt(time.Now())) + tasks = append(tasks, waitTask.WithQueuedAt(queuedAt)) - logEntry.WithFields(utils.LabelsToLogFields(waitTask.LogLabels)). - Infof("queue task %s", waitTask.GetDescription()) - - // Create "ReloadAllModules" task with onStartup flag turned on to discover modules state for the first time. - logLabels := utils.MergeLabels(onStartupLabels, map[string]string{ + // Add "DiscoverHelmReleases" task to detect unknown releases and purge them. + discoverLabels := utils.MergeLabels(logLabels, map[string]string{ "queue": "main", - "binding": string(task.ReloadAllModules), + "binding": string(task.DiscoverHelmReleases), }) - reloadAllModulesTask := sh_task.NewTask(task.ReloadAllModules). - WithLogLabels(logLabels). + discoverTask := sh_task.NewTask(task.DiscoverHelmReleases). + WithLogLabels(discoverLabels). WithQueueName("main"). WithMetadata(task.HookMetadata{ - EventDescription: "PrepopulateMainQueue", - OnStartupHooks: true, + EventDescription: eventDescription, }) - op.TaskQueues.GetMain().AddLast(reloadAllModulesTask.WithQueuedAt(time.Now())) + tasks = append(tasks, discoverTask.WithQueuedAt(queuedAt)) + + // Add "ConvergeModules" task to run modules converge sequence for the first time. + convergeLabels := utils.MergeLabels(logLabels, map[string]string{ + "queue": "main", + "binding": string(task.ConvergeModules), + }) + convergeTask := NewConvergeModulesTask(eventDescription, OperatorStartup, convergeLabels) + tasks = append(tasks, convergeTask.WithQueuedAt(queuedAt)) + + return tasks +} + +// CreatePurgeTasks returns ModulePurge tasks for each unknown Helm release. +func (op *AddonOperator) CreatePurgeTasks(modulesToPurge []string, t sh_task.Task) []sh_task.Task { + var newTasks []sh_task.Task + var queuedAt = time.Now() + + // Add ModulePurge tasks to purge unknown helm releases at start. + for _, moduleName := range modulesToPurge { + newLogLabels := utils.MergeLabels(t.GetLogLabels()) + newLogLabels["module"] = moduleName + delete(newLogLabels, "task.id") + + newTask := sh_task.NewTask(task.ModulePurge). + WithLogLabels(newLogLabels). + WithQueueName("main"). + WithMetadata(task.HookMetadata{ + EventDescription: t.GetDescription(), + ModuleName: moduleName, + }) + newTasks = append(newTasks, newTask.WithQueuedAt(queuedAt)) + } + + return newTasks } -// CreateReloadAllTasks -func (op *AddonOperator) CreateReloadAllTasks(onStartup bool, logLabels map[string]string, eventDescription string) []sh_task.Task { +// HandleConvergeModules is a multi-phase task. +func (op *AddonOperator) HandleConvergeModules(t sh_task.Task, logLabels map[string]string) (res queue.TaskResult) { + defer trace.StartRegion(context.Background(), "ConvergeModules").End() logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) + + taskEvent, ok := t.GetProp(ConvergeEventProp).(ConvergeEvent) + if !ok { + logEntry.Errorf("Possible bug! Wrong prop type in ConvergeModules: got %T(%#[1]v) instead string.", t.GetProp("event")) + res.Status = queue.Fail + return res + } + + var handleErr error + + if taskEvent == KubeConfigChanged { + logEntry.Infof("ConvergeModules: handle KubeConfigChanged") + var state *module_manager.ModulesState + op.KubeConfigManager.SafeReadConfig(func(config *kube_config_manager.KubeConfig) { + state, handleErr = op.ModuleManager.HandleNewKubeConfig(config) + }) + if handleErr == nil { + // KubeConfigChanged task should be removed. + res.Status = queue.Success + + if state == nil { + logEntry.Infof("ConvergeModules: no changes in config values after kube config modification") + return + } + + if len(state.ModulesToReload) > 0 { + // Append ModuleRun tasks if ModuleRun is not queued already. + reloadTasks := op.CreateReloadModulesTasks(state.ModulesToReload, t.GetLogLabels(), "KubeConfig-Changed-Modules") + if len(reloadTasks) > 0 { + logEntry.Infof("ConvergeModules: append %d tasks to reload modules %+v after kube config modification", len(reloadTasks), state.ModulesToReload) + } + res.TailTasks = reloadTasks + op.logTaskAdd(logEntry, "tail", res.TailTasks...) + return + } + + // No modules to reload -> full reload is required. + // If Converge is in progress, drain converge tasks after current task + // and put ConvergeModules at start. + // Else put ConvergeModules to the end of the main queue. + convergeDrained := RemoveCurrentConvergeTasks(op.TaskQueues.GetMain(), t.GetId()) + if convergeDrained { + logEntry.Infof("Adjacent ConvergeModules tasks was removed, will reset converge state from '%s'", op.ConvergeState.Phase) + res.AfterTasks = []sh_task.Task{ + NewConvergeModulesTask("InPlace-ReloadAll-After-KubeConfigChange", ReloadAllModules, t.GetLogLabels()), + } + op.logTaskAdd(logEntry, "after", res.AfterTasks...) + } else { + res.TailTasks = []sh_task.Task{ + NewConvergeModulesTask("Delayed-ReloadAll-After-KubeConfigChange", ReloadAllModules, t.GetLogLabels()), + } + op.logTaskAdd(logEntry, "tail", res.TailTasks...) + } + // ConvergeModules may be in progress Reset converge state. + op.ConvergeState.Phase = StandBy + return + } + } else { + // Handle other converge events: OperatorStartup, GlobalValuesChanged, ReloadAllModules. + if op.ConvergeState.Phase == StandBy { + logEntry.Infof("ConvergeModules: start") + + // Deduplicate tasks: remove ConvergeModules tasks right after the current task. + RemoveAdjacentConvergeModules(op.TaskQueues.GetByName(t.GetQueueName()), t.GetId()) + + op.ConvergeState.Phase = RunBeforeAll + } + + if op.ConvergeState.Phase == RunBeforeAll { + // Put BeforeAll tasks before current task. + tasks := op.CreateBeforeAllTasks(t.GetLogLabels(), t.GetDescription()) + op.ConvergeState.Phase = WaitBeforeAll + if len(tasks) > 0 { + res.HeadTasks = tasks + res.Status = queue.Keep + op.logTaskAdd(logEntry, "head", res.HeadTasks...) + return + } + } + + if op.ConvergeState.Phase == WaitBeforeAll { + logEntry.Infof("ConvergeModules: beforeAll hooks done, queue modules tasks") + var state *module_manager.ModulesState + state, handleErr = op.ModuleManager.RefreshEnabledState(t.GetLogLabels()) + if handleErr == nil { + // TODO disable hooks before was done in DiscoverModulesStateRefresh. Should we stick to this solution or disable events later during the handling each ModuleDelete task? + // Disable events for disabled modules. + for _, moduleName := range state.ModulesToDisable { + op.ModuleManager.DisableModuleHooks(moduleName) + //op.DrainModuleQueues(moduleName) + } + // Set ModulesToEnable list to run onStartup hooks for first converge. + if !op.ConvergeState.FirstDone { + state.ModulesToEnable = state.AllEnabledModules + } + tasks := op.CreateConvergeModulesTasks(state, t.GetLogLabels(), string(taskEvent)) + op.ConvergeState.Phase = WaitDeleteAndRunModules + if len(tasks) > 0 { + res.HeadTasks = tasks + res.Status = queue.Keep + op.logTaskAdd(logEntry, "head", res.HeadTasks...) + return + } + } + } + + if op.ConvergeState.Phase == WaitDeleteAndRunModules { + logEntry.Infof("ConvergeModules: modules tasks done, queue afterAll tasks") + // Put AfterAll tasks before current task. + tasks, handleErr := op.CreateAfterAllTasks(t.GetLogLabels(), t.GetDescription()) + if handleErr == nil { + op.ConvergeState.Phase = WaitAfterAll + if len(tasks) > 0 { + res.HeadTasks = tasks + res.Status = queue.Keep + op.logTaskAdd(logEntry, "head", res.HeadTasks...) + return + } + } + } + + // It is the last phase of ConvergeModules task, reset operator's Converge phase. + if op.ConvergeState.Phase == WaitAfterAll { + op.ConvergeState.Phase = StandBy + logEntry.Infof("ConvergeModules task done") + res.Status = queue.Success + return res + } + } + + if handleErr != nil { + res.Status = queue.Fail + logEntry.Errorf("ConvergeModules failed in phase '%s', requeue task to retry after delay. Failed count is %d. Error: %s", op.ConvergeState.Phase, t.GetFailureCount()+1, handleErr) + op.MetricStorage.CounterAdd("{PREFIX}modules_discover_errors_total", 1.0, map[string]string{}) + t.UpdateFailureMessage(handleErr.Error()) + t.WithQueuedAt(time.Now()) + return res + } + + logEntry.Infof("ConvergeModules success") + res.Status = queue.Success + return res +} + +// CreateBeforeAllTasks returns tasks to run BeforeAll global hooks. +func (op *AddonOperator) CreateBeforeAllTasks(logLabels map[string]string, eventDescription string) []sh_task.Task { var tasks = make([]sh_task.Task, 0) + var queuedAt = time.Now() - // Queue beforeAll global hooks. + // Get 'beforeAll' global hooks. beforeAllHooks := op.ModuleManager.GetGlobalHooksInOrder(BeforeAll) for _, hookName := range beforeAllHooks { @@ -562,191 +636,173 @@ func (op *AddonOperator) CreateReloadAllTasks(onStartup bool, logLabels map[stri BindingContext: []BindingContext{beforeAllBc}, ReloadAllOnValuesChanges: false, }) - tasks = append(tasks, newTask) - - logEntry.WithFields(utils.LabelsToLogFields(newTask.LogLabels)). - Infof("queue task %s", newTask.GetDescription()) + tasks = append(tasks, newTask.WithQueuedAt(queuedAt)) } + return tasks +} - discoverLogLabels := utils.MergeLabels(logLabels, map[string]string{ - "queue": "main", - }) - // remove task.id — it is set by NewTask - delete(discoverLogLabels, "task.id") - discoverTask := sh_task.NewTask(task.DiscoverModulesState). - WithLogLabels(logLabels). - WithQueueName("main"). - WithMetadata(task.HookMetadata{ - EventDescription: eventDescription, - OnStartupHooks: onStartup, +// CreateAfterAllTasks returns tasks to run AfterAll global hooks. +func (op *AddonOperator) CreateAfterAllTasks(logLabels map[string]string, eventDescription string) ([]sh_task.Task, error) { + var tasks = make([]sh_task.Task, 0) + var queuedAt = time.Now() + + // Get 'afterAll' global hooks. + afterAllHooks := op.ModuleManager.GetGlobalHooksInOrder(AfterAll) + + for i, hookName := range afterAllHooks { + hookLogLabels := utils.MergeLabels(logLabels, map[string]string{ + "hook": hookName, + "hook.type": "global", + "queue": "main", + "binding": string(AfterAll), }) - tasks = append(tasks, discoverTask) - logEntry.WithFields(utils.LabelsToLogFields(discoverTask.LogLabels)). - Infof("queue task %s", discoverTask.GetDescription()) - return tasks + delete(hookLogLabels, "task.id") + + afterAllBc := BindingContext{ + Binding: string(AfterAll), + } + afterAllBc.Metadata.BindingType = AfterAll + afterAllBc.Metadata.IncludeAllSnapshots = true + + taskMetadata := task.HookMetadata{ + EventDescription: eventDescription, + HookName: hookName, + BindingType: AfterAll, + BindingContext: []BindingContext{afterAllBc}, + } + if i == len(afterAllHooks)-1 { + taskMetadata.LastAfterAllHook = true + globalValues, err := op.ModuleManager.GlobalValues() + if err != nil { + return nil, err + } + taskMetadata.ValuesChecksum, err = globalValues.Checksum() + taskMetadata.DynamicEnabledChecksum = op.ModuleManager.DynamicEnabledChecksum() + if err != nil { + return nil, err + } + } + + newTask := sh_task.NewTask(task.GlobalHookRun). + WithLogLabels(hookLogLabels). + WithQueueName("main"). + WithMetadata(taskMetadata) + tasks = append(tasks, newTask.WithQueuedAt(queuedAt)) + } + + return tasks, nil +} + +// CreateAndStartQueue creates a named queue and starts it. +// It returns false is queue is already created +func (op *AddonOperator) CreateAndStartQueue(queueName string) bool { + if op.TaskQueues.GetByName(queueName) != nil { + return false + } + op.TaskQueues.NewNamedQueue(queueName, op.TaskHandler) + op.TaskQueues.GetByName(queueName).Start() + return true } -// CreateQueues create all queues defined in hooks -func (op *AddonOperator) InitAndStartHookQueues() { - schHooks := op.ModuleManager.GetGlobalHooksInOrder(Schedule) - for _, hookName := range schHooks { +// CreateAndStartQueuesForGlobalHooks creates queues for all registered global hooks. +// It is safe to run this method multiple times, as it checks +// for existing queues. +func (op *AddonOperator) CreateAndStartQueuesForGlobalHooks() { + for _, hookName := range op.ModuleManager.GetGlobalHooksNames() { h := op.ModuleManager.GetGlobalHook(hookName) for _, hookBinding := range h.Config.Schedules { - if op.TaskQueues.GetByName(hookBinding.Queue) == nil { - op.TaskQueues.NewNamedQueue(hookBinding.Queue, op.TaskHandler) - op.TaskQueues.GetByName(hookBinding.Queue).Start() + if op.CreateAndStartQueue(hookBinding.Queue) { log.Infof("Queue '%s' started for global 'schedule' hook %s", hookBinding.Queue, hookName) } } - } - - kubeHooks := op.ModuleManager.GetGlobalHooksInOrder(OnKubernetesEvent) - for _, hookName := range kubeHooks { - h := op.ModuleManager.GetGlobalHook(hookName) for _, hookBinding := range h.Config.OnKubernetesEvents { - if op.TaskQueues.GetByName(hookBinding.Queue) == nil { - op.TaskQueues.NewNamedQueue(hookBinding.Queue, op.TaskHandler) - op.TaskQueues.GetByName(hookBinding.Queue).Start() + if op.CreateAndStartQueue(hookBinding.Queue) { log.Infof("Queue '%s' started for global 'kubernetes' hook %s", hookBinding.Queue, hookName) } } } +} - // module hooks - modules := op.ModuleManager.GetModuleNamesInOrder() - for _, modName := range modules { - schHooks := op.ModuleManager.GetModuleHooksInOrder(modName, Schedule) - for _, hookName := range schHooks { - h := op.ModuleManager.GetModuleHook(hookName) - for _, hookBinding := range h.Config.Schedules { - if op.TaskQueues.GetByName(hookBinding.Queue) == nil { - op.TaskQueues.NewNamedQueue(hookBinding.Queue, op.TaskHandler) - op.TaskQueues.GetByName(hookBinding.Queue).Start() - log.Infof("Queue '%s' started for module 'schedule' hook %s", hookBinding.Queue, hookName) - } +// CreateAndStartQueuesForModuleHooks creates queues for registered module hooks. +// It is safe to run this method multiple times, as it checks +// for existing queues. +func (op *AddonOperator) CreateAndStartQueuesForModuleHooks(moduleName string) { + for _, hookName := range op.ModuleManager.GetModuleHookNames(moduleName) { + h := op.ModuleManager.GetModuleHook(hookName) + for _, hookBinding := range h.Config.Schedules { + if op.CreateAndStartQueue(hookBinding.Queue) { + log.Infof("Queue '%s' started for module 'schedule' hook %s", hookBinding.Queue, hookName) } } - - kubeHooks := op.ModuleManager.GetModuleHooksInOrder(modName, OnKubernetesEvent) - for _, hookName := range kubeHooks { - h := op.ModuleManager.GetModuleHook(hookName) - for _, hookBinding := range h.Config.OnKubernetesEvents { - if op.TaskQueues.GetByName(hookBinding.Queue) == nil { - op.TaskQueues.NewNamedQueue(hookBinding.Queue, op.TaskHandler) - op.TaskQueues.GetByName(hookBinding.Queue).Start() - log.Infof("Queue '%s' started for module 'kubernetes' hook %s", hookBinding.Queue, hookName) - } + for _, hookBinding := range h.Config.OnKubernetesEvents { + if op.CreateAndStartQueue(hookBinding.Queue) { + log.Infof("Queue '%s' started for module 'kubernetes' hook %s", hookBinding.Queue, hookName) } } } } -func (op *AddonOperator) DrainModuleQueues(modName string) { - drainQueue := func(queueName string) { - if queueName == "main" { - return - } - q := op.TaskQueues.GetByName(queueName) - if q == nil { - return - } - - // Remove all tasks. - q.Filter(func(_ sh_task.Task) bool { - return false - }) +// CreateAndStartQueuesForAllModuleHooks creates queues for all registered modules. +// It is safe to run this method multiple times, as it checks +// for existing queues. +func (op *AddonOperator) CreateAndStartQueuesForAllModuleHooks() { + for _, moduleName := range op.ModuleManager.GetEnabledModuleNames() { + op.CreateAndStartQueuesForModuleHooks(moduleName) } +} - schHooks := op.ModuleManager.GetModuleHooksInOrder(modName, Schedule) - for _, hookName := range schHooks { +func (op *AddonOperator) DrainModuleQueues(modName string) { + for _, hookName := range op.ModuleManager.GetModuleHookNames(modName) { h := op.ModuleManager.GetModuleHook(hookName) for _, hookBinding := range h.Config.Schedules { - drainQueue(hookBinding.Queue) + DrainNonMainQueue(op.TaskQueues.GetByName(hookBinding.Queue)) } - } - - kubeHooks := op.ModuleManager.GetModuleHooksInOrder(modName, OnKubernetesEvent) - for _, hookName := range kubeHooks { - h := op.ModuleManager.GetModuleHook(hookName) for _, hookBinding := range h.Config.OnKubernetesEvents { - drainQueue(hookBinding.Queue) + DrainNonMainQueue(op.TaskQueues.GetByName(hookBinding.Queue)) } } } func (op *AddonOperator) StartModuleManagerEventHandler() { go func() { + logEntry := log.WithField("operator.component", "handleManagerEvents") for { select { - // Event from module manager (module restart or full restart). - case moduleEvent := <-op.ModuleManager.Ch(): + case kubeConfigEvent := <-op.KubeConfigManager.KubeConfigEventCh(): logLabels := map[string]string{ "event.id": uuid.NewV4().String(), } - eventLogEntry := log.WithField("operator.component", "handleManagerEvents"). - WithFields(utils.LabelsToLogFields(logLabels)) - // Event from module manager can come if modules list have changed, - // so event hooks need to be re-register with: - // RegisterScheduledHooks() - // RegisterKubeEventHooks() - switch moduleEvent.Type { - // Some modules have changed. - case module_manager.ModulesChanged: - logLabels["event.type"] = "ModulesChanged" - - logEntry := eventLogEntry.WithFields(utils.LabelsToLogFields(logLabels)) - for _, moduleChange := range moduleEvent.ModulesChanges { - // Do not add ModuleRun task if it is already queued. - hasTask := QueueHasPendingModuleRunTask(op.TaskQueues.GetMain(), moduleChange.Name) - if !hasTask { - logEntry.WithField("module", moduleChange.Name).Infof("module values are changed, queue ModuleRun task") - newLabels := utils.MergeLabels(logLabels) - newLabels["module"] = moduleChange.Name - newTask := sh_task.NewTask(task.ModuleRun). - WithLogLabels(newLabels). - WithQueueName("main"). - WithMetadata(task.HookMetadata{ - EventDescription: "ModuleValuesChanged", - ModuleName: moduleChange.Name, - }) - op.TaskQueues.GetMain().AddLast(newTask.WithQueuedAt(time.Now())) - logEntry.WithFields(utils.LabelsToLogFields(newTask.LogLabels)). - Infof("queue task %s", newTask.GetDescription()) - } else { - logEntry.WithField("module", moduleChange.Name).Infof("module values are changed, ModuleRun task already exists") - } + eventLogEntry := logEntry.WithFields(utils.LabelsToLogFields(logLabels)) + + if kubeConfigEvent == kube_config_manager.KubeConfigInvalid { + op.ModuleManager.SetKubeConfigValid(false) + eventLogEntry.Infof("KubeConfig become invalid") + } + + if kubeConfigEvent == kube_config_manager.KubeConfigChanged { + if !op.ModuleManager.GetKubeConfigValid() { + eventLogEntry.Infof("KubeConfig become valid") } - // As module list may have changed, hook schedule index must be re-created. - // TODO SNAPSHOT: Check this - //ScheduleHooksController.UpdateScheduleHooks() - case module_manager.GlobalChanged: - // Global values are changed, all modules must be restarted. - logLabels["event.type"] = "GlobalChanged" - logEntry := eventLogEntry.WithFields(utils.LabelsToLogFields(logLabels)) - logEntry.Infof("queue tasks for ReloadAll: global config values are changed") - - // Stop and remove all resource monitors before run modules discovery. - op.HelmResourcesManager.StopMonitors() - - // Create "ReloadAllModules" task with onStartup flag turned off. - reloadAllModulesTask := sh_task.NewTask(task.ReloadAllModules). - WithLogLabels(logLabels). - WithQueueName("main"). - WithMetadata(task.HookMetadata{ - EventDescription: "GlobalConfigValuesChanged", - OnStartupHooks: false, - }) - op.TaskQueues.GetMain().AddLast(reloadAllModulesTask.WithQueuedAt(time.Now())) + // Config is valid now, add task to update ModuleManager state. + op.ModuleManager.SetKubeConfigValid(true) + // Run ConvergeModules task asap. + convergeTask := NewConvergeModulesTask( + fmt.Sprintf("ConfigMap-%s", kubeConfigEvent), + KubeConfigChanged, + logLabels, + ) + op.TaskQueues.GetMain().AddFirst(convergeTask) + // Cancel delay if the head task is stuck in the error loop. + op.TaskQueues.GetMain().CancelTaskDelay() + op.logTaskAdd(eventLogEntry, "KubeConfig is changed, put first", convergeTask) } + case absentResourcesEvent := <-op.HelmResourcesManager.Ch(): logLabels := map[string]string{ "event.id": uuid.NewV4().String(), "module": absentResourcesEvent.ModuleName, } - eventLogEntry := log.WithField("operator.component", "handleManagerEvents"). - WithFields(utils.LabelsToLogFields(logLabels)) - - //eventLogEntry.Debugf("Got %d absent resources from module", len(absentResourcesEvent.Absent)) + eventLogEntry := logEntry.WithFields(utils.LabelsToLogFields(logLabels)) // Do not add ModuleRun task if it is already queued. hasTask := QueueHasPendingModuleRunTask(op.TaskQueues.GetMain(), absentResourcesEvent.ModuleName) @@ -759,17 +815,17 @@ func (op *AddonOperator) StartModuleManagerEventHandler() { ModuleName: absentResourcesEvent.ModuleName, }) op.TaskQueues.GetMain().AddLast(newTask.WithQueuedAt(time.Now())) - eventLogEntry.WithFields(utils.LabelsToLogFields(newTask.LogLabels)). - Infof("queue task %s - got %d absent module resources", newTask.GetDescription(), len(absentResourcesEvent.Absent)) + taskAddDescription := fmt.Sprintf("got %d absent module resources, append", len(absentResourcesEvent.Absent)) + op.logTaskAdd(logEntry, taskAddDescription, newTask) } else { - eventLogEntry.Infof("Got %d absent module resources, ModuleRun task already queued", len(absentResourcesEvent.Absent)) + eventLogEntry.WithField("task.flow", "noop").Infof("Got %d absent module resources, ModuleRun task already queued", len(absentResourcesEvent.Absent)) } } } }() } -// TasksRunner handle tasks in queue. +// TaskHandler handles tasks in queue. func (op *AddonOperator) TaskHandler(t sh_task.Task) queue.TaskResult { var taskLogLabels = utils.MergeLabels(map[string]string{ "operator.component": "taskRunner", @@ -777,6 +833,8 @@ func (op *AddonOperator) TaskHandler(t sh_task.Task) queue.TaskResult { var taskLogEntry = log.WithFields(utils.LabelsToLogFields(taskLogLabels)) var res queue.TaskResult + op.logTaskStart(taskLogEntry, t) + op.UpdateWaitInQueueMetric(t) switch t.GetType() { @@ -788,207 +846,42 @@ func (op *AddonOperator) TaskHandler(t sh_task.Task) queue.TaskResult { hm := task.HookMetadataAccessor(t) globalHook := op.ModuleManager.GetGlobalHook(hm.HookName) globalHook.HookController.EnableScheduleBindings() - res.Status = "Success" + res.Status = queue.Success case task.GlobalHookEnableKubernetesBindings: - taskLogEntry.Infof("Global hook enable kubernetes bindings") - hm := task.HookMetadataAccessor(t) - globalHook := op.ModuleManager.GetGlobalHook(hm.HookName) - - var mainSyncTasks = make([]sh_task.Task, 0) - var parallelSyncTasks = make([]sh_task.Task, 0) - var parallelSyncTasksToWait = make([]sh_task.Task, 0) - - eventDescription := hm.EventDescription - if !strings.Contains(eventDescription, "HandleGlobalEnableKubernetesBindings") { - eventDescription += ".HandleGlobalEnableKubernetesBindings" - } - - newLogLabels := utils.MergeLabels(t.GetLogLabels()) - delete(newLogLabels, "task.id") - - err := op.ModuleManager.HandleGlobalEnableKubernetesBindings(hm.HookName, func(hook *module_manager.GlobalHook, info controller.BindingExecutionInfo) { - hookLogLabels := utils.MergeLabels(t.GetLogLabels(), map[string]string{ - "hook": hook.GetName(), - "hook.type": "global", - "queue": info.QueueName, - "binding": string(OnKubernetesEvent), - }) - delete(hookLogLabels, "task.id") - - kubernetesBindingID := uuid.NewV4().String() - newTask := sh_task.NewTask(task.GlobalHookRun). - WithLogLabels(hookLogLabels). - WithQueueName(info.QueueName). - WithMetadata(task.HookMetadata{ - EventDescription: eventDescription, - HookName: hook.GetName(), - BindingType: OnKubernetesEvent, - BindingContext: info.BindingContext, - AllowFailure: info.AllowFailure, - ReloadAllOnValuesChanges: false, // Ignore global values changes in global Synchronization tasks. - KubernetesBindingId: kubernetesBindingID, - WaitForSynchronization: info.KubernetesBinding.WaitForSynchronization, - MonitorIDs: []string{info.KubernetesBinding.Monitor.Metadata.MonitorId}, - ExecuteOnSynchronization: info.KubernetesBinding.ExecuteHookOnSynchronization, - }) - - if info.QueueName == t.GetQueueName() { - // Ignore "waitForSynchronization: false" for hooks in the main queue. - // There is no way to not wait for these hooks. - mainSyncTasks = append(mainSyncTasks, newTask) - } else { - // Do not wait for parallel hooks on "waitForSynchronization: false". - if info.KubernetesBinding.WaitForSynchronization { - parallelSyncTasksToWait = append(parallelSyncTasksToWait, newTask) - } else { - parallelSyncTasks = append(parallelSyncTasks, newTask) - } - } - }) - - if err != nil { - hookLabel := path.Base(globalHook.Path) - // TODO use separate metric, as in shell-operator? - op.MetricStorage.CounterAdd("{PREFIX}global_hook_errors_total", 1.0, map[string]string{ - "hook": hookLabel, - "binding": "GlobalEnableKubernetesBindings", - "queue": t.GetQueueName(), - "activation": "", - }) - taskLogEntry.Errorf("Global hook enable kubernetes bindings failed, requeue task to retry after delay. Failed count is %d. Error: %s", t.GetFailureCount()+1, err) - t.UpdateFailureMessage(err.Error()) - t.WithQueuedAt(time.Now()) - res.Status = "Fail" - } else { - // Substitute current task with Synchronization tasks for the main queue. - // Other Synchronization tasks are queued into specified queues. - // Informers can be started now — their events will be added to the queue tail. - taskLogEntry.Infof("Global hook enable kubernetes bindings success") - - // "Wait" tasks are queued first - for _, tsk := range parallelSyncTasksToWait { - q := op.TaskQueues.GetByName(tsk.GetQueueName()) - if q == nil { - log.Errorf("Queue %s is not created while run GlobalHookEnableKubernetesBindings task!", tsk.GetQueueName()) - } else { - // Skip state creation if WaitForSynchronization is disabled. - thm := task.HookMetadataAccessor(tsk) - taskLogEntry.Infof("queue task %s - Synchronization after onStartup, id=%s", tsk.GetDescription(), thm.KubernetesBindingId) - q.AddLast(tsk.WithQueuedAt(time.Now())) - op.ModuleManager.GlobalSynchronizationState().QueuedForBinding(thm) - } - } - - for _, tsk := range parallelSyncTasks { - q := op.TaskQueues.GetByName(tsk.GetQueueName()) - if q == nil { - log.Errorf("Queue %s is not created while run GlobalHookEnableKubernetesBindings task!", tsk.GetQueueName()) - } else { - q.AddLast(tsk.WithQueuedAt(time.Now())) - } - } - - res.Status = "Success" - // Note: No need to add "main" Synchronization tasks to the GlobalSynchronizationState. - for _, tsk := range mainSyncTasks { - tsk.WithQueuedAt(time.Now()) - } - res.HeadTasks = mainSyncTasks - } + res = op.HandleGlobalHookEnableKubernetesBindings(t, taskLogLabels) case task.GlobalHookWaitKubernetesSynchronization: - res.Status = "Success" + res.Status = queue.Success if op.ModuleManager.GlobalSynchronizationNeeded() && !op.ModuleManager.GlobalSynchronizationState().IsComplete() { // dump state op.ModuleManager.GlobalSynchronizationState().DebugDumpState(taskLogEntry) t.WithQueuedAt(time.Now()) - res.Status = "Repeat" + res.Status = queue.Repeat } else { taskLogEntry.Infof("Global 'Synchronization' is done") } - case task.ReloadAllModules: - taskLogEntry.Info("queue beforeAll and discoverModulesState tasks") - hm := task.HookMetadataAccessor(t) + case task.DiscoverHelmReleases: + res = op.HandleDiscoverHelmReleases(t, taskLogLabels) - // Remove adjacent ReloadAllModules tasks - stopFilter := false - op.TaskQueues.GetByName(t.GetQueueName()).Filter(func(tsk sh_task.Task) bool { - // Ignore current task - if tsk.GetId() == t.GetId() { - return true - } - if tsk.GetType() != task.ReloadAllModules { - stopFilter = true - } - return stopFilter - }) - - res.Status = "Success" - reloadAllTasks := op.CreateReloadAllTasks(hm.OnStartupHooks, t.GetLogLabels(), hm.EventDescription) - for _, tsk := range reloadAllTasks { - tsk.WithQueuedAt(time.Now()) - } - res.AfterTasks = reloadAllTasks - - case task.DiscoverModulesState: - taskLogEntry.Info("Discover modules start") - tasks, err := op.RunDiscoverModulesState(t, t.GetLogLabels()) - if err != nil { - op.MetricStorage.CounterAdd("{PREFIX}modules_discover_errors_total", 1.0, map[string]string{}) - taskLogEntry.Errorf("Discover modules failed, requeue task to retry after delay. Failed count is %d. Error: %s", t.GetFailureCount()+1, err) - t.UpdateFailureMessage(err.Error()) - t.WithQueuedAt(time.Now()) - res.Status = "Fail" - } else { - taskLogEntry.Infof("Discover modules success") - res.Status = "Success" - for _, tsk := range tasks { - tsk.WithQueuedAt(time.Now()) - } - res.AfterTasks = tasks - } + case task.ConvergeModules: + res = op.HandleConvergeModules(t, taskLogLabels) case task.ModuleRun: res = op.HandleModuleRun(t, taskLogLabels) case task.ModuleDelete: - // TODO wait while module's tasks in other queues are done. - hm := task.HookMetadataAccessor(t) - taskLogEntry.Infof("Module delete '%s'", hm.ModuleName) - // Remove all hooks from parallel queues. - op.DrainModuleQueues(hm.ModuleName) - err := op.ModuleManager.DeleteModule(hm.ModuleName, t.GetLogLabels()) - if err != nil { - op.MetricStorage.CounterAdd("{PREFIX}module_delete_errors_total", 1.0, map[string]string{"module": hm.ModuleName}) - taskLogEntry.Errorf("Module delete failed, requeue task to retry after delay. Failed count is %d. Error: %s", t.GetFailureCount()+1, err) - t.UpdateFailureMessage(err.Error()) - t.WithQueuedAt(time.Now()) - res.Status = "Fail" - } else { - taskLogEntry.Infof("Module delete success '%s'", hm.ModuleName) - res.Status = "Success" - } + res.Status = op.HandleModuleDelete(t, taskLogLabels) case task.ModuleHookRun: res = op.HandleModuleHookRun(t, taskLogLabels) case task.ModulePurge: - // Purge is for unknown modules, so error is just ignored. - taskLogEntry.Infof("Module purge start") - hm := task.HookMetadataAccessor(t) - - err := helm.NewClient(t.GetLogLabels()).DeleteRelease(hm.ModuleName) - if err != nil { - taskLogEntry.Warnf("Module purge failed, no retry. Error: %s", err) - } else { - taskLogEntry.Infof("Module purge success") - } - res.Status = "Success" + res.Status = op.HandleModulePurge(t, taskLogLabels) } - if res.Status == "Success" { + if res.Status == queue.Success { origAfterHandle := res.AfterHandle res.AfterHandle = func() { op.CheckConvergeStatus(t) @@ -998,9 +891,12 @@ func (op *AddonOperator) TaskHandler(t sh_task.Task) queue.TaskResult { } } + op.logTaskEnd(taskLogEntry, t, res) + return res } +// UpdateWaitInQueueMetric increases task_wait_in_queue_seconds_total counter for the task type. // TODO pass queue name from handler, not from task func (op *AddonOperator) UpdateWaitInQueueMetric(t sh_task.Task) { metricLabels := map[string]string{ @@ -1025,8 +921,8 @@ func (op *AddonOperator) UpdateWaitInQueueMetric(t sh_task.Task) { task.ModulePurge: metricLabels["module"] = hm.ModuleName - case task.ReloadAllModules, - task.DiscoverModulesState: + case task.ConvergeModules, + task.DiscoverHelmReleases: // no action required } @@ -1040,8 +936,205 @@ func (op *AddonOperator) UpdateWaitInQueueMetric(t sh_task.Task) { metricLabels["binding"] = hm.Binding } - taskWaitTime := time.Since(t.GetQueuedAt()).Seconds() - op.MetricStorage.CounterAdd("{PREFIX}task_wait_in_queue_seconds_total", taskWaitTime, metricLabels) + taskWaitTime := time.Since(t.GetQueuedAt()).Seconds() + op.MetricStorage.CounterAdd("{PREFIX}task_wait_in_queue_seconds_total", taskWaitTime, metricLabels) +} + +// HandleGlobalHookEnableKubernetesBindings add Synchronization tasks. +func (op *AddonOperator) HandleGlobalHookEnableKubernetesBindings(t sh_task.Task, labels map[string]string) (res queue.TaskResult) { + defer trace.StartRegion(context.Background(), "DiscoverHelmReleases").End() + + logEntry := log.WithFields(utils.LabelsToLogFields(labels)) + logEntry.Infof("Global hook enable kubernetes bindings") + + hm := task.HookMetadataAccessor(t) + globalHook := op.ModuleManager.GetGlobalHook(hm.HookName) + + var mainSyncTasks = make([]sh_task.Task, 0) + var parallelSyncTasks = make([]sh_task.Task, 0) + var parallelSyncTasksToWait = make([]sh_task.Task, 0) + var queuedAt = time.Now() + + eventDescription := hm.EventDescription + if !strings.Contains(eventDescription, "HandleGlobalEnableKubernetesBindings") { + eventDescription += ".HandleGlobalEnableKubernetesBindings" + } + + newLogLabels := utils.MergeLabels(t.GetLogLabels()) + delete(newLogLabels, "task.id") + + err := op.ModuleManager.HandleGlobalEnableKubernetesBindings(hm.HookName, func(hook *module_manager.GlobalHook, info controller.BindingExecutionInfo) { + hookLogLabels := utils.MergeLabels(t.GetLogLabels(), map[string]string{ + "hook": hook.GetName(), + "hook.type": "global", + "queue": info.QueueName, + "binding": string(OnKubernetesEvent), + }) + delete(hookLogLabels, "task.id") + + kubernetesBindingID := uuid.NewV4().String() + newTask := sh_task.NewTask(task.GlobalHookRun). + WithLogLabels(hookLogLabels). + WithQueueName(info.QueueName). + WithMetadata(task.HookMetadata{ + EventDescription: eventDescription, + HookName: hook.GetName(), + BindingType: OnKubernetesEvent, + BindingContext: info.BindingContext, + AllowFailure: info.AllowFailure, + ReloadAllOnValuesChanges: false, // Ignore global values changes in global Synchronization tasks. + KubernetesBindingId: kubernetesBindingID, + WaitForSynchronization: info.KubernetesBinding.WaitForSynchronization, + MonitorIDs: []string{info.KubernetesBinding.Monitor.Metadata.MonitorId}, + ExecuteOnSynchronization: info.KubernetesBinding.ExecuteHookOnSynchronization, + }) + newTask.WithQueuedAt(queuedAt) + + if info.QueueName == t.GetQueueName() { + // Ignore "waitForSynchronization: false" for hooks in the main queue. + // There is no way to not wait for these hooks. + mainSyncTasks = append(mainSyncTasks, newTask) + } else { + // Do not wait for parallel hooks on "waitForSynchronization: false". + if info.KubernetesBinding.WaitForSynchronization { + parallelSyncTasksToWait = append(parallelSyncTasksToWait, newTask) + } else { + parallelSyncTasks = append(parallelSyncTasks, newTask) + } + } + logEntry.WithField("task.flow", "add").Infof("queue task %s", newTask.GetDescription()) + }) + + if err != nil { + hookLabel := path.Base(globalHook.Path) + // TODO use separate metric, as in shell-operator? + op.MetricStorage.CounterAdd("{PREFIX}global_hook_errors_total", 1.0, map[string]string{ + "hook": hookLabel, + "binding": "GlobalEnableKubernetesBindings", + "queue": t.GetQueueName(), + "activation": "", + }) + logEntry.Errorf("Global hook enable kubernetes bindings failed, requeue task to retry after delay. Failed count is %d. Error: %s", t.GetFailureCount()+1, err) + t.UpdateFailureMessage(err.Error()) + t.WithQueuedAt(queuedAt) + res.Status = queue.Fail + return + } + // Substitute current task with Synchronization tasks for the main queue. + // Other Synchronization tasks are queued into specified queues. + // Informers can be started now — their events will be added to the queue tail. + logEntry.Infof("Global hook enable kubernetes bindings success") + + // "Wait" tasks are queued first + for _, tsk := range parallelSyncTasksToWait { + q := op.TaskQueues.GetByName(tsk.GetQueueName()) + if q == nil { + log.Errorf("Queue %s is not created while run GlobalHookEnableKubernetesBindings task!", tsk.GetQueueName()) + } else { + // Skip state creation if WaitForSynchronization is disabled. + thm := task.HookMetadataAccessor(tsk) + logEntry.Infof("queue task %s - Synchronization after onStartup, id=%s", tsk.GetDescription(), thm.KubernetesBindingId) + q.AddLast(tsk) + op.ModuleManager.GlobalSynchronizationState().QueuedForBinding(thm) + } + } + op.logTaskAdd(logEntry, "append", parallelSyncTasksToWait...) + + for _, tsk := range parallelSyncTasks { + q := op.TaskQueues.GetByName(tsk.GetQueueName()) + if q == nil { + log.Errorf("Queue %s is not created while run GlobalHookEnableKubernetesBindings task!", tsk.GetQueueName()) + } else { + q.AddLast(tsk) + } + } + op.logTaskAdd(logEntry, "append", parallelSyncTasks...) + + // Note: No need to add "main" Synchronization tasks to the GlobalSynchronizationState. + res.HeadTasks = mainSyncTasks + op.logTaskAdd(logEntry, "head", res.HeadTasks...) + + res.Status = queue.Success + + return +} + +// HandleDiscoverHelmReleases runs RefreshStateFromHelmReleases to detect modules state at start. +func (op *AddonOperator) HandleDiscoverHelmReleases(t sh_task.Task, labels map[string]string) (res queue.TaskResult) { + defer trace.StartRegion(context.Background(), "DiscoverHelmReleases").End() + + logEntry := log.WithFields(utils.LabelsToLogFields(labels)) + logEntry.Infof("Discover Helm releases state") + + state, err := op.ModuleManager.RefreshStateFromHelmReleases(t.GetLogLabels()) + if err != nil { + res.Status = queue.Fail + logEntry.Errorf("Discover helm releases failed, requeue task to retry after delay. Failed count is %d. Error: %s", t.GetFailureCount()+1, err) + t.UpdateFailureMessage(err.Error()) + t.WithQueuedAt(time.Now()) + } else { + res.Status = queue.Success + tasks := op.CreatePurgeTasks(state.ModulesToPurge, t) + res.AfterTasks = tasks + op.logTaskAdd(logEntry, "after", res.AfterTasks...) + } + return +} + +// HandleModulePurge run helm purge for unknown module. +func (op *AddonOperator) HandleModulePurge(t sh_task.Task, labels map[string]string) (status queue.TaskStatus) { + defer trace.StartRegion(context.Background(), "ModulePurge").End() + + logEntry := log.WithFields(utils.LabelsToLogFields(labels)) + logEntry.Infof("Module purge start") + + hm := task.HookMetadataAccessor(t) + err := op.Helm.NewClient(t.GetLogLabels()).DeleteRelease(hm.ModuleName) + if err != nil { + // Purge is for unknown modules, just print warning. + logEntry.Warnf("Module purge failed, no retry. Error: %s", err) + } else { + logEntry.Infof("Module purge success") + } + status = queue.Success + return +} + +// HandleModuleDelete deletes helm release for known module. +func (op *AddonOperator) HandleModuleDelete(t sh_task.Task, labels map[string]string) (status queue.TaskStatus) { + defer trace.StartRegion(context.Background(), "ModuleDelete").End() + + hm := task.HookMetadataAccessor(t) + module := op.ModuleManager.GetModule(hm.ModuleName) + + logEntry := log.WithFields(utils.LabelsToLogFields(labels)) + logEntry.Infof("Module delete '%s'", hm.ModuleName) + + // Register module hooks to run afterHelmDelete hooks on startup. + // It's a noop if registration is done before. + err := op.ModuleManager.RegisterModuleHooks(module, t.GetLogLabels()) + + // TODO disable events and drain queues here or earlier during ConvergeModules.RunBeforeAll phase? + if err == nil { + // Disable events + //op.ModuleManager.DisableModuleHooks(hm.ModuleName) + // Remove all hooks from parallel queues. + op.DrainModuleQueues(hm.ModuleName) + err = op.ModuleManager.DeleteModule(hm.ModuleName, t.GetLogLabels()) + } + + if err != nil { + op.MetricStorage.CounterAdd("{PREFIX}module_delete_errors_total", 1.0, map[string]string{"module": hm.ModuleName}) + logEntry.Errorf("Module delete failed, requeue task to retry after delay. Failed count is %d. Error: %s", t.GetFailureCount()+1, err) + t.UpdateFailureMessage(err.Error()) + t.WithQueuedAt(time.Now()) + status = queue.Fail + } else { + logEntry.Infof("Module delete success '%s'", hm.ModuleName) + status = queue.Success + } + + return } // HandleModuleRun starts a module by executing module hooks and installing a Helm chart. @@ -1065,6 +1158,12 @@ func (op *AddonOperator) HandleModuleRun(t sh_task.Task, labels map[string]strin hm := task.HookMetadataAccessor(t) module := op.ModuleManager.GetModule(hm.ModuleName) + // Break error loop when module becomes disabled. + if !op.ModuleManager.IsModuleEnabled(module.Name) { + res.Status = queue.Success + return + } + metricLabels := map[string]string{ "module": hm.ModuleName, "activation": labels["event.type"], @@ -1074,28 +1173,31 @@ func (op *AddonOperator) HandleModuleRun(t sh_task.Task, labels map[string]strin op.MetricStorage.HistogramObserve("{PREFIX}module_run_seconds", d.Seconds(), metricLabels, nil) })() - //var syncQueueName = fmt.Sprintf("main-subqueue-kubernetes-Synchronization-module-%s", hm.ModuleName) var moduleRunErr error var valuesChanged = false // First module run on operator startup or when module is enabled. if module.State.Phase == module_manager.Startup { - if hm.OnStartupHooks { - logEntry.Debugf("ModuleRun '%s' phase", module.State.Phase) + // Register module hooks on every enable. + moduleRunErr = op.ModuleManager.RegisterModuleHooks(module, labels) + if moduleRunErr == nil { + if hm.OnStartupHooks { + logEntry.Debugf("ModuleRun '%s' phase", module.State.Phase) - treg := trace.StartRegion(context.Background(), "ModuleRun-OnStartup") + treg := trace.StartRegion(context.Background(), "ModuleRun-OnStartup") - // DiscoverModules registered all module hooks, so queues can be started now. - op.InitAndStartHookQueues() + // Start queues for module hooks. + op.CreateAndStartQueuesForModuleHooks(module.Name) - // run onStartup hooks - moduleRunErr = module.RunOnStartup(t.GetLogLabels()) - if moduleRunErr == nil { + // Run onStartup hooks. + moduleRunErr = module.RunOnStartup(t.GetLogLabels()) + if moduleRunErr == nil { + module.State.Phase = module_manager.OnStartupDone + } + treg.End() + } else { module.State.Phase = module_manager.OnStartupDone } - treg.End() - } else { - module.State.Phase = module_manager.OnStartupDone } } @@ -1160,6 +1262,7 @@ func (op *AddonOperator) HandleModuleRun(t sh_task.Task, labels map[string]strin WithLogLabels(hookLogLabels). WithQueueName(queueName). WithMetadata(taskMeta) + newTask.WithQueuedAt(time.Now()) if info.QueueName == t.GetQueueName() { // Ignore "waitForSynchronization: false" for hooks in the main queue. @@ -1186,10 +1289,11 @@ func (op *AddonOperator) HandleModuleRun(t sh_task.Task, labels map[string]strin } else { thm := task.HookMetadataAccessor(tsk) logEntry.Infof("queue 'wait Synchronization' task %s, id=%s", tsk.GetDescription(), thm.KubernetesBindingId) - q.AddLast(tsk.WithQueuedAt(time.Now())) + q.AddLast(tsk) module.State.Synchronization().QueuedForBinding(thm) } } + op.logTaskAdd(logEntry, "append", parallelSyncTasksToWait...) // Queue regular parallel tasks. for _, tsk := range parallelSyncTasks { @@ -1197,26 +1301,10 @@ func (op *AddonOperator) HandleModuleRun(t sh_task.Task, labels map[string]strin if q == nil { logEntry.Errorf("queue %s is not found while EnableKubernetesBindings task", tsk.GetQueueName()) } else { - q.AddLast(tsk.WithQueuedAt(time.Now())) + q.AddLast(tsk) } } - - //// Queue Synchronization tasks for the "main" queue in the subqueue to wait for them. - //if len(mainSyncTasks) > 0 { - // // EnableKubernetesBindings and StartInformers for all kubernetes bindings. - // op.TaskQueues.NewNamedQueue(syncQueueName, op.TaskHandler) - // syncSubQueue := op.TaskQueues.GetByName(syncQueueName) - // - // for _, tsk := range mainSyncTasks { - // thm := task.HookMetadataAccessor(tsk) - // logEntry.WithFields(utils.LabelsToLogFields(tsk.GetLogLabels())). - // Infof("queue 'wait Synchronization' task %s, id=%s", tsk.GetDescription(), thm.KubernetesBindingId) - // syncSubQueue.AddLast(tsk.WithQueuedAt(time.Now())) - // module.SynchronizationState().QueuedForBinding(thm) - // } - // logEntry.Infof("Queue '%s' started for module 'kubernetes.Synchronization' hooks", syncQueueName) - // syncSubQueue.Start() - //} + op.logTaskAdd(logEntry, "append", parallelSyncTasks...) if len(parallelSyncTasksToWait) == 0 { // Skip waiting tasks in parallel queues, proceed to schedule bindings. @@ -1230,18 +1318,9 @@ func (op *AddonOperator) HandleModuleRun(t sh_task.Task, labels map[string]strin // Put Synchronization tasks for kubernetes hooks before ModuleRun task. if len(mainSyncTasks) > 0 { - // TODO Add "Keep" status for the queue handler in the shell-operator. - // Copy current task to keep it in the main queue. Current task will be deleted by its ID. - moduleRunTask := sh_task.NewTask(t.GetType()). - WithQueueName(t.GetQueueName()). - WithMetadata(t.GetMetadata()). - WithLogLabels(t.GetLogLabels()) - for _, tsk := range mainSyncTasks { - tsk.WithQueuedAt(time.Now()) - } - mainSyncTasks = append(mainSyncTasks, moduleRunTask) res.HeadTasks = mainSyncTasks - res.Status = "Success" + res.Status = queue.Keep + op.logTaskAdd(logEntry, "head", res.HeadTasks...) return } } @@ -1262,7 +1341,7 @@ func (op *AddonOperator) HandleModuleRun(t sh_task.Task, labels map[string]strin } logEntry.Debugf("Synchronization not complete, keep ModuleRun task in repeat mode") t.WithQueuedAt(time.Now()) - res.Status = "Repeat" + res.Status = queue.Repeat return } } @@ -1283,13 +1362,13 @@ func (op *AddonOperator) HandleModuleRun(t sh_task.Task, labels map[string]strin } if moduleRunErr != nil { - res.Status = "Fail" + res.Status = queue.Fail logEntry.Errorf("ModuleRun failed in phase '%s'. Requeue task to retry after delay. Failed count is %d. Error: %s", module.State.Phase, t.GetFailureCount()+1, moduleRunErr) op.MetricStorage.CounterAdd("{PREFIX}module_run_errors_total", 1.0, map[string]string{"module": hm.ModuleName}) t.UpdateFailureMessage(moduleRunErr.Error()) t.WithQueuedAt(time.Now()) } else { - res.Status = "Success" + res.Status = queue.Success if valuesChanged { logEntry.Infof("ModuleRun success, values changed, restart module") // One of afterHelm hooks changes values, run ModuleRun again: copy task, but disable startup hooks. @@ -1305,7 +1384,9 @@ func (op *AddonOperator) HandleModuleRun(t sh_task.Task, labels map[string]strin WithLogLabels(newLabels). WithQueueName(t.GetQueueName()). WithMetadata(hm) + res.AfterTasks = []sh_task.Task{newTask.WithQueuedAt(time.Now())} + op.logTaskAdd(logEntry, "after", res.AfterTasks...) } else { logEntry.Infof("ModuleRun success, module is ready") } @@ -1322,8 +1403,8 @@ func (op *AddonOperator) HandleModuleHookRun(t sh_task.Task, labels map[string]s taskHook := op.ModuleManager.GetModuleHook(hm.HookName) // Prevent hook running in parallel queue if module is disabled in "main" queue. - if !taskHook.Module.State.Enabled { - res.Status = "Success" + if !op.ModuleManager.IsModuleEnabled(taskHook.Module.Name) { + res.Status = queue.Success return } @@ -1332,9 +1413,8 @@ func (op *AddonOperator) HandleModuleHookRun(t sh_task.Task, labels map[string]s // This could happen when the Context is // canceled, or the expected wait time exceeds the Context's Deadline. // The best we can do without proper context usage is to repeat the task. - return queue.TaskResult{ - Status: "Repeat", - } + res.Status = queue.Repeat + return } metricLabels := map[string]string{ @@ -1356,12 +1436,12 @@ func (op *AddonOperator) HandleModuleHookRun(t sh_task.Task, labels map[string]s // Synchronization is not a part of v0 contract, skip hook execution. if taskHook.Config.Version == "v0" { shouldRunHook = false - res.Status = "Success" + res.Status = queue.Success } // Check for "executeOnSynchronization: false". if !hm.ExecuteOnSynchronization { shouldRunHook = false - res.Status = "Success" + res.Status = queue.Success } } @@ -1393,7 +1473,7 @@ func (op *AddonOperator) HandleModuleHookRun(t sh_task.Task, labels map[string]s if len(combineResult.MonitorIDs) > 0 { hm.MonitorIDs = append(hm.MonitorIDs, combineResult.MonitorIDs...) } - logEntry.Infof("Got monitorIDs: %+v", hm.MonitorIDs) + logEntry.Debugf("Got monitorIDs: %+v", hm.MonitorIDs) t.UpdateMetadata(hm) } } @@ -1411,23 +1491,80 @@ func (op *AddonOperator) HandleModuleHookRun(t sh_task.Task, labels map[string]s success := 0.0 allowed := 0.0 - err = op.ModuleManager.RunModuleHook(hm.HookName, hm.BindingType, hm.BindingContext, t.GetLogLabels()) + beforeChecksum, afterChecksum, err := op.ModuleManager.RunModuleHook(hm.HookName, hm.BindingType, hm.BindingContext, t.GetLogLabels()) if err != nil { if hm.AllowFailure { allowed = 1.0 logEntry.Infof("Module hook failed, but allowed to fail. Error: %v", err) - res.Status = "Success" + res.Status = queue.Success } else { errors = 1.0 logEntry.Errorf("Module hook failed, requeue task to retry after delay. Failed count is %d. Error: %s", t.GetFailureCount()+1, err) t.UpdateFailureMessage(err.Error()) t.WithQueuedAt(time.Now()) - res.Status = "Fail" + res.Status = queue.Fail } } else { success = 1.0 logEntry.Infof("Module hook success '%s'", hm.HookName) - res.Status = "Success" + res.Status = queue.Success + + // Handle module values change. + reloadModule := false + eventDescription := "" + switch hm.BindingType { + case Schedule: + if beforeChecksum != afterChecksum { + reloadModule = true + eventDescription = fmt.Sprintf("Schedule-Change-ModuleValues(%s)", hm.GetHookName()) + } + case OnKubernetesEvent: + // Do not reload module on changes during Synchronization. + if !hm.IsSynchronization() && beforeChecksum != afterChecksum { + reloadModule = true + eventDescription = fmt.Sprintf("Kubernetes-Change-ModuleValues(%s)", hm.GetHookName()) + } + } + if reloadModule { + // relabel + logLabels := t.GetLogLabels() + if hookLabel, ok := logLabels["hook"]; ok { + logLabels["event.triggered-by.hook"] = hookLabel + delete(logLabels, "hook") + delete(logLabels, "hook.type") + } + if label, ok := logLabels["binding"]; ok { + logLabels["event.triggered-by.binding"] = label + delete(logLabels, "binding") + } + if label, ok := logLabels["binding.name"]; ok { + logLabels["event.triggered-by.binding.name"] = label + delete(logLabels, "binding.name") + } + if label, ok := logLabels["watchEvent"]; ok { + logLabels["event.triggered-by.watchEvent"] = label + delete(logLabels, "watchEvent") + } + + // TODO Remove event.type label in log and activation label in metrics. + logLabels["event.type"] = "ModulesChanged" + + // Do not add ModuleRun task if it is already queued. + hasTask := QueueHasPendingModuleRunTask(op.TaskQueues.GetMain(), hm.ModuleName) + if !hasTask { + newTask := sh_task.NewTask(task.ModuleRun). + WithLogLabels(logLabels). + WithQueueName("main"). + WithMetadata(task.HookMetadata{ + EventDescription: eventDescription, + ModuleName: hm.ModuleName, + }) + op.TaskQueues.GetMain().AddLast(newTask.WithQueuedAt(time.Now())) + op.logTaskAdd(logEntry, "module values are changed, append", newTask) + } else { + logEntry.WithField("task.flow", "noop").Infof("module values are changed, ModuleRun task already queued") + } + } } op.MetricStorage.CounterAdd("{PREFIX}module_hook_allowed_errors_total", allowed, metricLabels) @@ -1435,10 +1572,10 @@ func (op *AddonOperator) HandleModuleHookRun(t sh_task.Task, labels map[string]s op.MetricStorage.CounterAdd("{PREFIX}module_hook_success_total", success, metricLabels) } - if isSynchronization && res.Status == "Success" { + if isSynchronization && res.Status == queue.Success { taskHook.Module.State.Synchronization().DoneForBinding(hm.KubernetesBindingId) // Unlock Kubernetes events for all monitors when Synchronization task is done. - logEntry.Info("Unlock kubernetes.Event tasks") + logEntry.Info("Synchronization done, unlock Kubernetes events") for _, monitorID := range hm.MonitorIDs { taskHook.HookController.UnlockKubernetesEventsFor(monitorID) } @@ -1482,12 +1619,12 @@ func (op *AddonOperator) HandleGlobalHookRun(t sh_task.Task, labels map[string]s // Synchronization is not a part of v0 contract, skip hook execution. if taskHook.Config.Version == "v0" { shouldRunHook = false - res.Status = "Success" + res.Status = queue.Success } // Check for "executeOnSynchronization: false". if !hm.ExecuteOnSynchronization { shouldRunHook = false - res.Status = "Success" + res.Status = queue.Success } } @@ -1542,13 +1679,13 @@ func (op *AddonOperator) HandleGlobalHookRun(t sh_task.Task, labels map[string]s if hm.AllowFailure { allowed = 1.0 logEntry.Infof("Global hook failed, but allowed to fail. Error: %v", err) - res.Status = "Success" + res.Status = queue.Success } else { errors = 1.0 logEntry.Errorf("Global hook failed, requeue task to retry after delay. Failed count is %d. Error: %s", t.GetFailureCount()+1, err) t.UpdateFailureMessage(err.Error()) t.WithQueuedAt(time.Now()) - res.Status = "Fail" + res.Status = queue.Fail } } else { // Calculate new checksum of *Enabled values. @@ -1556,7 +1693,7 @@ func (op *AddonOperator) HandleGlobalHookRun(t sh_task.Task, labels map[string]s success = 1.0 logEntry.Infof("Global hook success '%s'", taskHook.Name) logEntry.Debugf("GlobalHookRun checksums: before=%s after=%s saved=%s", beforeChecksum, afterChecksum, hm.ValuesChecksum) - res.Status = "Success" + res.Status = queue.Success reloadAll := false eventDescription := "" @@ -1628,17 +1765,10 @@ func (op *AddonOperator) HandleGlobalHookRun(t sh_task.Task, labels map[string]s logLabels["event.triggered-by.watchEvent"] = label delete(logLabels, "watchEvent") } - // Put "ReloadAllModules" task with onStartup flag turned off to the end of the 'main' queue. - reloadAllModulesTask := sh_task.NewTask(task.ReloadAllModules). - WithLogLabels(t.GetLogLabels()). - WithQueueName("main"). - WithMetadata(task.HookMetadata{ - EventDescription: eventDescription, - OnStartupHooks: false, - }). - WithQueuedAt(time.Now()) - op.TaskQueues.GetMain().AddLast(reloadAllModulesTask.WithQueuedAt(time.Now())) - logEntry.Infof("ReloadAllModules queued by hook '%s' (task: %s)", taskHook.Name, hm.GetDescription()) + // Reload all using "ConvergeModules" task. + newTask := NewConvergeModulesTask(eventDescription, GlobalValuesChanged, t.GetLogLabels()) + op.TaskQueues.GetMain().AddLast(newTask) + op.logTaskAdd(logEntry, "global values are changed, append", newTask) } // TODO rethink helm monitors pause-resume. It is not working well with parallel hooks without locks. But locks will destroy parallelization. //else { @@ -1651,11 +1781,10 @@ func (op *AddonOperator) HandleGlobalHookRun(t sh_task.Task, labels map[string]s op.MetricStorage.CounterAdd("{PREFIX}global_hook_success_total", success, metricLabels) } - if isSynchronization && res.Status == "Success" { - logEntry.Debugf("Synchronization task for %s/%s is successful, mark it as Done: id=%s", hm.HookName, hm.Binding, hm.KubernetesBindingId) + if isSynchronization && res.Status == queue.Success { op.ModuleManager.GlobalSynchronizationState().DoneForBinding(hm.KubernetesBindingId) // Unlock Kubernetes events for all monitors when Synchronization task is done. - logEntry.Info("Unlock kubernetes.Event tasks") + logEntry.Info("Synchronization done, unlock Kubernetes events") for _, monitorID := range hm.MonitorIDs { taskHook.HookController.UnlockKubernetesEventsFor(monitorID) } @@ -1664,67 +1793,45 @@ func (op *AddonOperator) HandleGlobalHookRun(t sh_task.Task, labels map[string]s return res } -func (op *AddonOperator) RunDiscoverModulesState(discoverTask sh_task.Task, logLabels map[string]string) ([]sh_task.Task, error) { - logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) - modulesState, err := op.ModuleManager.DiscoverModulesState(logLabels) - if err != nil { - return nil, err - } - +func (op *AddonOperator) CreateReloadModulesTasks(moduleNames []string, logLabels map[string]string, eventDescription string) []sh_task.Task { var newTasks []sh_task.Task + var queuedAt = time.Now() - hm := task.HookMetadataAccessor(discoverTask) + queuedModuleNames := ModulesWithPendingModuleRun(op.TaskQueues.GetMain()) - eventDescription := hm.EventDescription - if !strings.Contains(eventDescription, "DiscoverModulesState") { - eventDescription += ".DiscoverModulesState" - } + // Add ModuleRun tasks to reload only specific modules. + for _, moduleName := range moduleNames { + // No task is required if ModuleRun is already in queue. + if _, has := queuedModuleNames[moduleName]; has { + continue + } - // queue ModuleRun tasks for enabled modules - for _, moduleName := range modulesState.EnabledModules { newLogLabels := utils.MergeLabels(logLabels) newLogLabels["module"] = moduleName delete(newLogLabels, "task.id") - // Run OnStartup hooks on application startup or if module become enabled - runOnStartupHooks := hm.OnStartupHooks - if !runOnStartupHooks { - for _, name := range modulesState.NewlyEnabledModules { - if name == moduleName { - runOnStartupHooks = true - break - } - } - } - newTask := sh_task.NewTask(task.ModuleRun). WithLogLabels(newLogLabels). WithQueueName("main"). WithMetadata(task.HookMetadata{ EventDescription: eventDescription, ModuleName: moduleName, - OnStartupHooks: runOnStartupHooks, }) - newTasks = append(newTasks, newTask) - - logEntry.WithFields(utils.LabelsToLogFields(newTask.LogLabels)). - Infof("queue task %s", newTask.GetDescription()) + newTasks = append(newTasks, newTask.WithQueuedAt(queuedAt)) } + return newTasks +} + +// CreateConvergeModulesTasks creates ModuleRun/ModuleDelete tasks based on moduleManager state. +func (op *AddonOperator) CreateConvergeModulesTasks(state *module_manager.ModulesState, logLabels map[string]string, eventDescription string) []sh_task.Task { + var newTasks []sh_task.Task + var queuedAt = time.Now() - // queue ModuleDelete tasks for disabled modules - for _, moduleName := range modulesState.ModulesToDisable { + // Add ModuleDelete tasks to delete helm releases of disabled modules. + for _, moduleName := range state.ModulesToDisable { newLogLabels := utils.MergeLabels(logLabels) newLogLabels["module"] = moduleName delete(newLogLabels, "task.id") - // Register module hooks for deleted modules to able to - // run afterHelmDelete hooks on Addon-operator startup. - if hm.OnStartupHooks { - // error can be ignored, DiscoverModulesState should return existed modules - disabledModule := op.ModuleManager.GetModule(moduleName) - if err = op.ModuleManager.RegisterModuleHooks(disabledModule, newLogLabels); err != nil { - return nil, err - } - } newTask := sh_task.NewTask(task.ModuleDelete). WithLogLabels(newLogLabels). @@ -1733,338 +1840,83 @@ func (op *AddonOperator) RunDiscoverModulesState(discoverTask sh_task.Task, logL EventDescription: eventDescription, ModuleName: moduleName, }) - newTasks = append(newTasks, newTask) - - logEntry.WithFields(utils.LabelsToLogFields(newTask.LogLabels)). - Infof("queue task %s", newTask.GetDescription()) + newTasks = append(newTasks, newTask.WithQueuedAt(queuedAt)) } - // queue ModulePurge tasks for unknown modules - for _, moduleName := range modulesState.ReleasedUnknownModules { + // Add ModuleRun tasks to install or reload enabled modules. + newlyEnabled := utils.ListToMapStringStruct(state.ModulesToEnable) + for _, moduleName := range state.AllEnabledModules { newLogLabels := utils.MergeLabels(logLabels) newLogLabels["module"] = moduleName delete(newLogLabels, "task.id") - newTask := sh_task.NewTask(task.ModulePurge). + // Run OnStartup hooks on application startup or if module become enabled. + runOnStartupHooks := false + if _, has := newlyEnabled[moduleName]; has { + runOnStartupHooks = true + } + + newTask := sh_task.NewTask(task.ModuleRun). WithLogLabels(newLogLabels). WithQueueName("main"). WithMetadata(task.HookMetadata{ EventDescription: eventDescription, ModuleName: moduleName, + OnStartupHooks: runOnStartupHooks, }) - newTasks = append(newTasks, newTask) - - logEntry.WithFields(utils.LabelsToLogFields(newTask.LogLabels)). - Infof("queue task %s", newTask.GetDescription()) + newTasks = append(newTasks, newTask.WithQueuedAt(queuedAt)) } - // Queue afterAll global hooks - afterAllHooks := op.ModuleManager.GetGlobalHooksInOrder(AfterAll) - for i, hookName := range afterAllHooks { - hookLogLabels := utils.MergeLabels(logLabels, map[string]string{ - "hook": hookName, - "hook.type": "global", - "queue": "main", - "binding": string(AfterAll), - }) - delete(hookLogLabels, "task.id") - - afterAllBc := BindingContext{ - Binding: string(AfterAll), - } - afterAllBc.Metadata.BindingType = AfterAll - afterAllBc.Metadata.IncludeAllSnapshots = true - - taskMetadata := task.HookMetadata{ - EventDescription: eventDescription, - HookName: hookName, - BindingType: AfterAll, - BindingContext: []BindingContext{afterAllBc}, - } - if i == len(afterAllHooks)-1 { - taskMetadata.LastAfterAllHook = true - globalValues, err := op.ModuleManager.GlobalValues() - if err != nil { - return nil, err - } - taskMetadata.ValuesChecksum, err = globalValues.Checksum() - taskMetadata.DynamicEnabledChecksum = op.ModuleManager.DynamicEnabledChecksum() - if err != nil { - return nil, err - } - } + return newTasks +} - newTask := sh_task.NewTask(task.GlobalHookRun). - WithLogLabels(hookLogLabels). - WithQueueName("main"). - WithMetadata(taskMetadata) - newTasks = append(newTasks, newTask) +// CheckConvergeStatus detects if converge process is started and +// updates ConvergeState. It updates metrics on converge finish. +func (op *AddonOperator) CheckConvergeStatus(t sh_task.Task) { + convergeTasks := ConvergeTasksInQueue(op.TaskQueues.GetMain()) - logEntry.WithFields(utils.LabelsToLogFields(newTask.LogLabels)). - Infof("queue task %s", newTask.GetDescription()) + // Converge state is 'Started'. Update StartedAt and + // Activation if the converge process is just started. + if convergeTasks > 0 && op.ConvergeState.StartedAt == 0 { + op.ConvergeState.StartedAt = time.Now().UnixNano() + op.ConvergeState.Activation = t.GetLogLabels()["event.type"] } - // TODO queues should be cleaned from hook run tasks of deleted module! - // Disable kubernetes informers and schedule - for _, moduleName := range modulesState.ModulesToDisable { - op.ModuleManager.DisableModuleHooks(moduleName) + // Converge state is 'Done'. Update convergence_* metrics, + // reset StartedAt and Activation if the converge process is just stopped. + if convergeTasks == 0 && op.ConvergeState.StartedAt != 0 { + convergeSeconds := time.Duration(time.Now().UnixNano() - op.ConvergeState.StartedAt).Seconds() + op.MetricStorage.CounterAdd("{PREFIX}convergence_seconds", convergeSeconds, map[string]string{"activation": op.ConvergeState.Activation}) + op.MetricStorage.CounterAdd("{PREFIX}convergence_total", 1.0, map[string]string{"activation": op.ConvergeState.Activation}) + op.ConvergeState.StartedAt = 0 + op.ConvergeState.Activation = "" } - return newTasks, nil -} - -func (op *AddonOperator) RunAddonOperatorMetrics() { - // Addon-operator live ticks. - go func() { - for { - op.MetricStorage.CounterAdd("{PREFIX}live_ticks", 1.0, map[string]string{}) - time.Sleep(10 * time.Second) - } - }() - - go func() { - for { - // task queue length - op.TaskQueues.Iterate(func(queue *queue.TaskQueue) { - queueLen := float64(queue.Length()) - op.MetricStorage.GaugeSet("{PREFIX}tasks_queue_length", queueLen, map[string]string{"queue": queue.Name}) - }) - time.Sleep(5 * time.Second) - } - }() -} - -func (op *AddonOperator) SetupDebugServerHandles() { - op.DebugServer.Route("/global/list.{format:(json|yaml)}", func(_ *http.Request) (interface{}, error) { - return map[string]interface{}{ - "globalHooks": op.ModuleManager.GetGlobalHooksNames(), - }, nil - }) - - op.DebugServer.Route("/global/values.{format:(json|yaml)}", func(_ *http.Request) (interface{}, error) { - return op.ModuleManager.GlobalValues() - }) - - op.DebugServer.Route("/global/config.{format:(json|yaml)}", func(_ *http.Request) (interface{}, error) { - return op.ModuleManager.GlobalConfigValues(), nil - }) - - op.DebugServer.Route("/global/patches.{format:(json|yaml)}", func(_ *http.Request) (interface{}, error) { - return op.ModuleManager.GlobalValuesPatches(), nil - }) - - op.DebugServer.Route("/global/snapshots.{format:(json|yaml)}", func(r *http.Request) (interface{}, error) { - kubeHookNames := op.ModuleManager.GetGlobalHooksInOrder(OnKubernetesEvent) - snapshots := make(map[string]interface{}) - for _, hName := range kubeHookNames { - h := op.ModuleManager.GetGlobalHook(hName) - snapshots[hName] = h.HookController.SnapshotsDump() - } - - return snapshots, nil - }) - - op.DebugServer.Route("/module/list.{format:(json|yaml|text)}", func(_ *http.Request) (interface{}, error) { - return map[string][]string{"enabledModules": op.ModuleManager.GetModuleNamesInOrder()}, nil - }) - - op.DebugServer.Route("/module/{name}/{type:(config|values)}.{format:(json|yaml)}", func(r *http.Request) (interface{}, error) { - modName := chi.URLParam(r, "name") - valType := chi.URLParam(r, "type") - - m := op.ModuleManager.GetModule(modName) - if m == nil { - return nil, fmt.Errorf("Module not found") - } - - switch valType { - case "config": - return m.ConfigValues(), nil - case "values": - return m.Values() - } - return "no values", nil - }) - - op.DebugServer.Route("/module/{name}/render", func(r *http.Request) (interface{}, error) { - modName := chi.URLParam(r, "name") - - m := op.ModuleManager.GetModule(modName) - if m == nil { - return nil, fmt.Errorf("Module not found") - } - - valuesPath, err := m.PrepareValuesYamlFile() - if err != nil { - return nil, err - } - defer os.Remove(valuesPath) - - helmCl := helm.NewClient() - return helmCl.Render(m.Name, m.Path, []string{valuesPath}, nil, app.Namespace) - }) - - op.DebugServer.Route("/module/{name}/patches.json", func(r *http.Request) (interface{}, error) { - modName := chi.URLParam(r, "name") - - m := op.ModuleManager.GetModule(modName) - if m == nil { - return nil, fmt.Errorf("Module not found") - } - - return m.ValuesPatches(), nil - }) - - op.DebugServer.Route("/module/resource-monitor.{format:(json|yaml)}", func(_ *http.Request) (interface{}, error) { - dump := map[string]interface{}{} - - for _, moduleName := range op.ModuleManager.GetModuleNamesInOrder() { - if !op.HelmResourcesManager.HasMonitor(moduleName) { - dump[moduleName] = "No monitor" - continue - } - - ids := op.HelmResourcesManager.GetMonitor(moduleName).ResourceIds() - dump[moduleName] = ids - } - - return dump, nil - }) - - op.DebugServer.Route("/module/{name}/snapshots.{format:(json|yaml)}", func(r *http.Request) (interface{}, error) { - modName := chi.URLParam(r, "name") - - m := op.ModuleManager.GetModule(modName) - if m == nil { - return nil, fmt.Errorf("Module not found") - } - - mHookNames := op.ModuleManager.GetModuleHookNames(m.Name) - snapshots := make(map[string]interface{}) - for _, hName := range mHookNames { - h := op.ModuleManager.GetModuleHook(hName) - snapshots[hName] = h.HookController.SnapshotsDump() - } - - return snapshots, nil - }) - -} - -func (op *AddonOperator) SetupHttpServerHandles() { - http.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) { - _, _ = writer.Write([]byte(` - Addon-operator - -

Addon-operator

- 
go tool pprof goprofex http://ADDON_OPERATOR_IP:9115/debug/pprof/profile
-

- prometheus metrics - health url -

- - `)) - }) - http.Handle("/metrics", promhttp.Handler()) - - http.HandleFunc("/healthz", func(writer http.ResponseWriter, request *http.Request) { - if helm.HealthzHandler == nil { - writer.WriteHeader(http.StatusOK) - return - } - helm.HealthzHandler(writer, request) - }) - - http.HandleFunc("/ready", func(w http.ResponseWriter, request *http.Request) { - if op.IsStartupConvergeDone() { - w.WriteHeader(200) - _, _ = w.Write([]byte("Startup converge done.\n")) - } else { - w.WriteHeader(500) - _, _ = w.Write([]byte("Startup converge in progress\n")) - } - }) - - http.HandleFunc("/status/converge", func(writer http.ResponseWriter, request *http.Request) { - convergeTasks := op.MainQueueHasConvergeTasks() - - statusLines := make([]string, 0) - if op.IsStartupConvergeDone() { - statusLines = append(statusLines, "STARTUP_CONVERGE_DONE") - if convergeTasks > 0 { - statusLines = append(statusLines, fmt.Sprintf("CONVERGE_IN_PROGRESS: %d tasks", convergeTasks)) - } else { - statusLines = append(statusLines, "CONVERGE_WAIT_TASK") - } - } else { - if op.StartupConvergeStarted { - if convergeTasks > 0 { - statusLines = append(statusLines, fmt.Sprintf("STARTUP_CONVERGE_IN_PROGRESS: %d tasks", convergeTasks)) - } else { - statusLines = append(statusLines, "STARTUP_CONVERGE_DONE") - } - } else { - statusLines = append(statusLines, "STARTUP_CONVERGE_WAIT_TASKS") - } - } - - _, _ = writer.Write([]byte(strings.Join(statusLines, "\n") + "\n")) - }) -} - -func (op *AddonOperator) MainQueueHasConvergeTasks() int { - convergeTasks := 0 - op.TaskQueues.GetMain().Iterate(func(t sh_task.Task) { - ttype := t.GetType() - switch ttype { - case task.ModuleRun, task.DiscoverModulesState, task.ModuleDelete, task.ModulePurge, task.ReloadAllModules, task.GlobalHookEnableKubernetesBindings, task.GlobalHookEnableScheduleBindings: - convergeTasks++ - return - } - - hm := task.HookMetadataAccessor(t) - if ttype == task.GlobalHookRun { - switch hm.BindingType { - case BeforeAll, AfterAll: - convergeTasks++ - return - } - } - }) - - return convergeTasks + // Update field for the first converge. + op.UpdateFirstConvergeStatus(t, convergeTasks) } -func (op *AddonOperator) CheckConvergeStatus(t sh_task.Task) { - convergeTasks := op.MainQueueHasConvergeTasks() +// UpdateFirstConvergeStatus checks first converge status and prints log messages if first converge +// is in progress. +func (op *AddonOperator) UpdateFirstConvergeStatus(t sh_task.Task, convergeTasks int) { + if op.ConvergeState.FirstDone { + return + } logEntry := log.WithFields(utils.LabelsToLogFields(t.GetLogLabels())) - logEntry.Infof("Queue 'main' contains %d converge tasks after handle '%s'", convergeTasks, string(t.GetType())) - // Trigger Started. if convergeTasks > 0 { - if !op.StartupConvergeStarted { + if !op.ConvergeState.FirstStarted { logEntry.Infof("First converge is started.") - op.StartupConvergeStarted = true - } - if op.ConvergeStarted == 0 { - op.ConvergeStarted = time.Now().UnixNano() - op.ConvergeActivation = t.GetLogLabels()["event.type"] + op.ConvergeState.FirstStarted = true } + logEntry.Infof("Queue 'main' contains %d converge tasks after handle '%s'", convergeTasks, string(t.GetType())) } - // Trigger Done. - if convergeTasks == 0 { - if !op.IsStartupConvergeDone() && op.StartupConvergeStarted { - logEntry.Infof("First converge is finished. Operator is ready now.") - op.SetStartupConvergeDone() - } - if op.ConvergeStarted != 0 { - convergeSeconds := time.Duration(time.Now().UnixNano() - op.ConvergeStarted).Seconds() - op.MetricStorage.CounterAdd("{PREFIX}convergence_seconds", convergeSeconds, map[string]string{"activation": op.ConvergeActivation}) - op.MetricStorage.CounterAdd("{PREFIX}convergence_total", 1.0, map[string]string{"activation": op.ConvergeActivation}) - op.ConvergeStarted = 0 - } + // First converge is done if started and no converge tasks left in the main queue. + if convergeTasks == 0 && op.ConvergeState.FirstStarted { + logEntry.Infof("First converge is finished. Operator is ready now.") + op.ConvergeState.FirstDone = true } } @@ -2073,78 +1925,42 @@ func (op *AddonOperator) Shutdown() { op.ShellOperator.Shutdown() } -func DefaultOperator() *AddonOperator { - operator := NewAddonOperator() - operator.WithContext(context.Background()) - return operator +func (op *AddonOperator) logTaskAdd(logEntry *log.Entry, description string, tasks ...sh_task.Task) { + logger := logEntry.WithField("task.flow", "add") + for _, tsk := range tasks { + logger.Infof("%s task %s", description, tsk.GetDescription()) + } } -func InitAndStart(operator *AddonOperator) error { - err := operator.StartHttpServer(sh_app.ListenAddress, sh_app.ListenPort, http.DefaultServeMux) - if err != nil { - log.Errorf("HTTP SERVER start failed: %v", err) - return err +func (op *AddonOperator) logTaskStart(logEntry *log.Entry, tsk sh_task.Task) { + logger := logEntry.WithField("task.flow", "start") + phase := op.taskPhase(tsk) + if phase != "" { + phase = fmt.Sprintf(" in phase '%s'", phase) } - // Override shell-operator's metricStorage and register metrics specific for addon-operator. - operator.InitMetricStorage() - operator.SetupHttpServerHandles() + logger.Infof("%s starts%s: %s", tsk.GetType(), phase, tsk.GetDescription()) +} - err = operator.SetupHookMetricStorageAndServer() - if err != nil { - log.Errorf("HTTP SERVER for hook metrics start failed: %v", err) - return err - } - - // Create a default 'main' Kubernetes client if not initialized externally. - // Register metrics for kubernetes client with the default custom label "component". - if operator.KubeClient == nil { - // Register metrics for client-go. - //nolint:staticcheck - klient.RegisterKubernetesClientMetrics(operator.MetricStorage, operator.GetMainKubeClientMetricLabels()) - // Initialize 'main' Kubernetes client. - operator.KubeClient, err = operator.InitMainKubeClient() - if err != nil { - log.Errorf("MAIN Fatal: initialize kube client for hooks: %s\n", err) - return err - } - } +func (op *AddonOperator) logTaskEnd(logEntry *log.Entry, tsk sh_task.Task, result queue.TaskResult) { + logger := logEntry.WithField("task.flow", "end") - err = operator.Init() - if err != nil { - log.Errorf("INIT failed: %v", err) - return err + phase := op.taskPhase(tsk) + if phase != "" { + phase = fmt.Sprintf(", phase '%s'", phase) } - operator.ShellOperator.SetupDebugServerHandles() - operator.SetupDebugServerHandles() - - err = operator.InitModuleManager() - if err != nil { - log.Errorf("INIT ModuleManager failed: %s", err) - return err - } + logger.Infof("%s ends with result '%s'%s (%s)", tsk.GetType(), result.Status, phase, tsk.GetDescription()) - operator.Start() - return nil } -// QueueHasPendingModuleRunTask returns true if queue has pending tasks -// with the type "ModuleRun" related to the module "moduleName". -func QueueHasPendingModuleRunTask(q *queue.TaskQueue, moduleName string) bool { - hasTask := false - firstTask := true - - q.Iterate(func(t sh_task.Task) { - // Skip the first task in the queue as it can be executed already, i.e. "not pending". - if firstTask { - firstTask = false - return - } - - if t.GetType() == task.ModuleRun && task.HookMetadataAccessor(t).ModuleName == moduleName { - hasTask = true - } - }) - - return hasTask +func (op *AddonOperator) taskPhase(tsk sh_task.Task) string { + switch tsk.GetType() { + case task.ConvergeModules: + return string(op.ConvergeState.Phase) + case task.ModuleRun: + hm := task.HookMetadataAccessor(tsk) + module := op.ModuleManager.GetModule(hm.ModuleName) + return string(module.State.Phase) + } + return "" } diff --git a/pkg/addon-operator/operator_test.go b/pkg/addon-operator/operator_test.go index 3aed5d89..09da3ac4 100644 --- a/pkg/addon-operator/operator_test.go +++ b/pkg/addon-operator/operator_test.go @@ -2,116 +2,327 @@ package addon_operator import ( "context" + "io/ioutil" + "path/filepath" + "strings" "testing" + "time" + "github.com/flant/kube-client/fake" . "github.com/onsi/gomega" - - klient "github.com/flant/kube-client/client" - sh_app "github.com/flant/shell-operator/pkg/app" + log "github.com/sirupsen/logrus" + //. "github.com/flant/shell-operator/pkg/hook/binding_context" . "github.com/flant/shell-operator/pkg/hook/types" + shell_operator "github.com/flant/shell-operator/pkg/shell-operator" sh_task "github.com/flant/shell-operator/pkg/task" "github.com/flant/shell-operator/pkg/task/queue" + file_utils "github.com/flant/shell-operator/pkg/utils/file" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" + "github.com/flant/addon-operator/pkg/helm" + "github.com/flant/addon-operator/pkg/helm_resources_manager" + . "github.com/flant/addon-operator/pkg/hook/types" + "github.com/flant/addon-operator/pkg/kube_config_manager" + "github.com/flant/addon-operator/pkg/module_manager" "github.com/flant/addon-operator/pkg/task" ) -// CreateOnStartupTasks fills a working queue with onStartup hooks. -// TaskRunner should run all hooks and clean the queue. -func Test_Operator_Startup(t *testing.T) { - // TODO tiller and helm should be mocked - t.SkipNow() +type assembleResult struct { + operator *AddonOperator + helmClient *helm.MockHelmClient + helmResourcesManager *helm_resources_manager.MockHelmResourcesManager + cmName string + cmNamespace string +} + +func assembleTestAddonOperator(t *testing.T, configPath string) (*AddonOperator, *assembleResult) { g := NewWithT(t) - kubeClient := klient.NewFake(nil) + const defaultNamespace = "default" + const defaultName = "addon-operator" + + result := new(assembleResult) + + // Check content in configPath. + rootDir := filepath.Join("testdata", configPath) + g.Expect(rootDir).Should(BeADirectory()) + + modulesDir := filepath.Join(rootDir, "modules") + if exists, _ := file_utils.DirExists(modulesDir); !exists { + modulesDir = "" + } + globalHooksDir := filepath.Join(rootDir, "global-hooks") + if exists, _ := file_utils.DirExists(globalHooksDir); !exists { + globalHooksDir = "" + } + if globalHooksDir == "" { + globalHooksDir = filepath.Join(rootDir, "global") + if exists, _ := file_utils.DirExists(globalHooksDir); !exists { + globalHooksDir = "" + } + } + + // Load config values from config_map.yaml. + cmFilePath := filepath.Join(rootDir, "config_map.yaml") + cmExists, _ := file_utils.FileExists(cmFilePath) + + var cmObj *v1.ConfigMap + if cmExists { + cmDataBytes, err := ioutil.ReadFile(cmFilePath) + g.Expect(err).ShouldNot(HaveOccurred(), "Should read config map file '%s'", cmFilePath) - sh_app.DebugUnixSocket = "testdata/debug.socket" + cmObj = new(v1.ConfigMap) + err = yaml.Unmarshal(cmDataBytes, &cmObj) + g.Expect(err).ShouldNot(HaveOccurred(), "Should parse YAML in %s", cmFilePath) + if cmObj.Namespace == "" { + cmObj.SetNamespace(defaultNamespace) + } + } else { + cmObj = &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + Namespace: defaultNamespace, + }, + Data: nil, + } + } + result.cmName = cmObj.Name + result.cmNamespace = cmObj.Namespace + + // Create ConfigMap. + kubeClient := fake.NewFakeCluster(fake.ClusterVersionV119).Client + _, err := kubeClient.CoreV1().ConfigMaps(result.cmNamespace).Create(context.TODO(), cmObj, metav1.CreateOptions{}) + g.Expect(err).ShouldNot(HaveOccurred(), "Should create ConfigMap/%s", result.cmName) + + // Assemble AddonOperator. op := NewAddonOperator() op.WithContext(context.Background()) - op.WithKubernetesClient(kubeClient) - op.WithGlobalHooksDir("testdata/global_hooks") + op.KubeClient = kubeClient + // Mock helm client for ModuleManager + result.helmClient = &helm.MockHelmClient{} + op.Helm = helm.MockHelm(result.helmClient) + // Mock helm resources manager to execute module actions: run, delete. + op.HelmResourcesManager = &helm_resources_manager.MockHelmResourcesManager{} + + shell_operator.SetupEventManagers(op.ShellOperator) - var err = op.Init() - g.Expect(err).ShouldNot(HaveOccurred()) + op.KubeConfigManager = kube_config_manager.NewKubeConfigManager() + op.KubeConfigManager.WithKubeClient(op.KubeClient) + op.KubeConfigManager.WithContext(op.ctx) + op.KubeConfigManager.WithNamespace(result.cmNamespace) + op.KubeConfigManager.WithConfigMapName(result.cmName) + + op.ModuleManager = module_manager.NewModuleManager() + op.ModuleManager.WithContext(op.ctx) + op.ModuleManager.WithDirectories(modulesDir, globalHooksDir, t.TempDir()) + op.ModuleManager.WithKubeConfigManager(op.KubeConfigManager) + op.ModuleManager.WithHelm(op.Helm) + op.ModuleManager.WithScheduleManager(op.ScheduleManager) + op.ModuleManager.WithKubeEventManager(op.KubeEventsManager) + op.ModuleManager.WithHelmResourcesManager(op.HelmResourcesManager) err = op.InitModuleManager() - g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(err).ShouldNot(HaveOccurred(), "Should init ModuleManager") - op.PrepopulateMainQueue(op.TaskQueues) + return op, result +} - var head sh_task.Task - var hm task.HookMetadata +// CreateOnStartupTasks fills a working queue with onStartup hooks. +// TaskRunner should run all hooks and clean the queue. +func Test_Operator_startup_tasks(t *testing.T) { + g := NewWithT(t) + + op, _ := assembleTestAddonOperator(t, "startup_tasks") - head = op.TaskQueues.GetMain().RemoveFirst() - hm = task.HookMetadataAccessor(head) - g.Expect(hm.BindingType).To(Equal(OnStartup)) + op.BootstrapMainQueue(op.TaskQueues) + + expectTasks := []struct { + taskType sh_task.TaskType + bindingType BindingType + hookPrefix string + }{ + // OnStartup in specified order. + // onStartup: 1 + {task.GlobalHookRun, OnStartup, "hook02"}, + // onStartup: 10 + {task.GlobalHookRun, OnStartup, "hook03"}, + // onStartup: 20 + {task.GlobalHookRun, OnStartup, "hook01"}, + // EnableSchedule in alphabet order. + {task.GlobalHookEnableScheduleBindings, "", "hook02"}, + {task.GlobalHookEnableScheduleBindings, "", "hook03"}, + // Synchronization for kubernetes bindings in alphabet order. + {task.GlobalHookEnableKubernetesBindings, "", "hook01"}, + {task.GlobalHookEnableKubernetesBindings, "", "hook03"}, + {task.GlobalHookWaitKubernetesSynchronization, "", ""}, + } - head = op.TaskQueues.GetMain().RemoveFirst() - hm = task.HookMetadataAccessor(head) - g.Expect(hm.BindingType).To(Equal(OnStartup)) + i := 0 + op.TaskQueues.GetMain().Iterate(func(tsk sh_task.Task) { + // Stop checking if no expects left. + if i >= len(expectTasks) { + return + } + + expect := expectTasks[i] + hm := task.HookMetadataAccessor(tsk) + g.Expect(tsk.GetType()).To(Equal(expect.taskType), "task type should match for task %d, got %+v %+v", i, tsk, hm) + g.Expect(hm.BindingType).To(Equal(expect.bindingType), "binding should match for task %d, got %+v %+v", i, tsk, hm) + g.Expect(hm.HookName).To(HavePrefix(expect.hookPrefix), "hook name should match for task %d, got %+v %+v", i, tsk, hm) + i++ + }) } -func Test_Operator_QueueHasPendingModuleRunTask(t *testing.T) { +// Load global hooks and modules, setup wrapper for TaskHandler, and check +// tasks sequence until converge is done. +func Test_Operator_ConvergeModules_main_queue_only(t *testing.T) { g := NewWithT(t) + // Mute messages about registration and tasks queueing. + log.SetLevel(log.ErrorLevel) - tests := []struct { - name string - result bool - queue func() *queue.TaskQueue - }{ - { - name: "Normal", - result: true, - queue: func() *queue.TaskQueue { - q := queue.NewTasksQueue() - - Task := &sh_task.BaseTask{Type: task.ModuleRun, Id: "unknown"} - q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) - - Task = &sh_task.BaseTask{Type: task.ModuleRun, Id: "unknown"} - q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) - - Task = &sh_task.BaseTask{Type: task.ModuleRun, Id: "test"} - q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "test"})) - return q - }}, - { - name: "First task", - result: false, - queue: func() *queue.TaskQueue { - q := queue.NewTasksQueue() - - Task := &sh_task.BaseTask{Type: task.ModuleRun, Id: "test"} - q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "test"})) - - Task = &sh_task.BaseTask{Type: task.GlobalHookRun, Id: "unknown"} - q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) - - Task = &sh_task.BaseTask{Type: task.ModuleRun, Id: "unknown"} - q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) - return q - }}, - { - name: "No module run", - result: false, - queue: func() *queue.TaskQueue { - q := queue.NewTasksQueue() - - Task := &sh_task.BaseTask{Type: task.ModuleRun, Id: "unknown"} - q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) - - Task = &sh_task.BaseTask{Type: task.ModuleHookRun, Id: "test"} - q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "test"})) - - Task = &sh_task.BaseTask{Type: task.ModuleRun, Id: "unknown"} - q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) - return q - }}, + op, res := assembleTestAddonOperator(t, "converge__main_queue_only") + op.BootstrapMainQueue(op.TaskQueues) + + // Fill mocked helm with two releases: one to purge and one to disable during converge process. + moduleToPurge := "moduleToPurge" + moduleToDelete := "module-beta" + + res.helmClient.ReleaseNames = []string{moduleToPurge, moduleToDelete} + + type taskInfo struct { + taskType sh_task.TaskType + bindingType BindingType + moduleName string + hookName string + spawnerTaskPhase string } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := QueueHasPendingModuleRunTask(tt.queue(), "test") - g.Expect(result).To(Equal(tt.result)) + taskHandleHistory := make([]taskInfo, 0) + op.TaskQueues.GetMain().WithHandler(func(tsk sh_task.Task) queue.TaskResult { + // Put task info to history. + hm := task.HookMetadataAccessor(tsk) + phase := "" + switch tsk.GetType() { + case task.ConvergeModules: + phase = string(op.ConvergeState.Phase) + case task.ModuleRun: + phase = string(op.ModuleManager.GetModule(hm.ModuleName).State.Phase) + } + taskHandleHistory = append(taskHandleHistory, taskInfo{ + taskType: tsk.GetType(), + bindingType: hm.BindingType, + moduleName: hm.ModuleName, + hookName: hm.HookName, + spawnerTaskPhase: phase, }) + + // Handle it. + return op.TaskHandler(tsk) + }) + + op.TaskQueues.StartMain() + + // Wait until converge is done. + stopTimer := time.NewTimer(30 * time.Second) + checkTicker := time.NewTicker(200 * time.Millisecond) +waitConverge: + for { + select { + case <-stopTimer.C: + t.Fatal("Operator not ready after timeout.") + case <-checkTicker.C: + if op.IsStartupConvergeDone() { + break waitConverge + } + mainQueue := op.TaskQueues.GetMain() + if !mainQueue.IsEmpty() && mainQueue.GetFirst().GetFailureCount() >= 2 { + t.Fatal("Error loop detected.") + } + } + } + + // Match history with expected tasks. + historyExpects := []struct { + taskType sh_task.TaskType + bindingType BindingType + namePrefix string + convergePhase string + }{ + // OnStartup in specified order. + // onStartup: 1 + {task.GlobalHookRun, OnStartup, "hook02", ""}, + // onStartup: 10 + {task.GlobalHookRun, OnStartup, "hook03", ""}, + // onStartup: 20 + {task.GlobalHookRun, OnStartup, "hook01", ""}, + // EnableSchedule in alphabet order. + {task.GlobalHookEnableScheduleBindings, "", "hook02", ""}, + {task.GlobalHookEnableScheduleBindings, "", "hook03", ""}, + // Synchronization for kubernetes bindings in alphabet order. + {task.GlobalHookEnableKubernetesBindings, "", "hook01", ""}, + {task.GlobalHookRun, OnKubernetesEvent, "hook01", ""}, + + // hook03 has executeForSynchronization false, but GlobalHookRun is present to properly enable events. + {task.GlobalHookEnableKubernetesBindings, "", "hook03", ""}, + {task.GlobalHookRun, OnKubernetesEvent, "hook03", ""}, + + // There are no parallel queues, so wait task runs without repeating. + {task.GlobalHookWaitKubernetesSynchronization, "", "", ""}, + + // TODO DiscoverHelmReleases can add ModulePurge tasks. + {task.DiscoverHelmReleases, "", "", ""}, + {task.ModulePurge, "", moduleToPurge, ""}, + + // ConvergeModules runs after global Synchronization and emerges BeforeAll tasks. + {task.ConvergeModules, "", "", string(StandBy)}, + {task.GlobalHookRun, BeforeAll, "hook02", ""}, + {task.GlobalHookRun, BeforeAll, "hook01", ""}, + + {task.ConvergeModules, "", "", string(WaitBeforeAll)}, + + // ConvergeModules adds ModuleDelete and ModuleRun tasks. + {task.ModuleDelete, "", "module-beta", ""}, + + {task.ModuleRun, "", "module-alpha", string(module_manager.Startup)}, + + // Only one hook with kubernetes binding. + {task.ModuleHookRun, OnKubernetesEvent, "module-alpha/hook01", ""}, + //{task.ModuleHookRun, OnKubernetesEvent, "module-alpha/hook02", ""}, + + // Skip waiting tasks in parallel queues, proceed to schedule bindings. + {task.ModuleRun, "", "module-alpha", string(module_manager.EnableScheduleBindings)}, + + // ConvergeModules emerges afterAll tasks + {task.ConvergeModules, "", "", string(WaitDeleteAndRunModules)}, + {task.GlobalHookRun, AfterAll, "hook03", ""}, + + {task.ConvergeModules, "", "", string(WaitAfterAll)}, + } + + for i, historyInfo := range taskHandleHistory { + if i >= len(historyExpects) { + break + } + expect := historyExpects[i] + g.Expect(historyInfo.taskType).To(Equal(expect.taskType), "task type should match for history entry %d, got %+v %+v", i, historyInfo) + g.Expect(historyInfo.bindingType).To(Equal(expect.bindingType), "binding should match for history entry %d, got %+v %+v", i, historyInfo) + g.Expect(historyInfo.spawnerTaskPhase).To(Equal(expect.convergePhase), "converge phase should match for history entry %d, got %+v %+v", i, historyInfo) + + switch historyInfo.taskType { + case task.ModuleRun, task.ModulePurge, task.ModuleDelete: + g.Expect(historyInfo.moduleName).To(ContainSubstring(expect.namePrefix), "module name should match for history entry %d, got %+v %+v", i, historyInfo) + case task.GlobalHookRun, task.GlobalHookEnableScheduleBindings, task.GlobalHookEnableKubernetesBindings: + g.Expect(historyInfo.hookName).To(HavePrefix(expect.namePrefix), "hook name should match for history entry %d, got %+v %+v", i, historyInfo) + case task.ModuleHookRun: + parts := strings.Split(expect.namePrefix, "/") + g.Expect(historyInfo.moduleName).To(ContainSubstring(parts[0]), "module name should match for history entry %d, got %+v %+v", i, historyInfo) + g.Expect(historyInfo.hookName).To(ContainSubstring("/"+parts[1]), "hook name should match for history entry %d, got %+v %+v", i, historyInfo) + } } } diff --git a/pkg/addon-operator/queue.go b/pkg/addon-operator/queue.go new file mode 100644 index 00000000..ec929287 --- /dev/null +++ b/pkg/addon-operator/queue.go @@ -0,0 +1,138 @@ +package addon_operator + +import ( + "github.com/flant/addon-operator/pkg/task" + sh_task "github.com/flant/shell-operator/pkg/task" + "github.com/flant/shell-operator/pkg/task/queue" +) + +// QueueHasPendingModuleRunTask returns true if queue has pending tasks +// with the type "ModuleRun" related to the module "moduleName". +func QueueHasPendingModuleRunTask(q *queue.TaskQueue, moduleName string) bool { + if q == nil { + return false + } + modules := ModulesWithPendingModuleRun(q) + _, has := modules[moduleName] + return has +} + +// ModulesWithPendingModuleRun returns names of all modules in pending +// ModuleRun tasks. First task in queue considered not pending and is ignored. +func ModulesWithPendingModuleRun(q *queue.TaskQueue) map[string]struct{} { + if q == nil { + return nil + } + + modules := make(map[string]struct{}) + + skipFirstTask := true + + q.Iterate(func(t sh_task.Task) { + // Skip the first task in the queue as it can be executed already, i.e. "not pending". + if skipFirstTask { + skipFirstTask = false + return + } + + if t.GetType() == task.ModuleRun { + hm := task.HookMetadataAccessor(t) + modules[hm.ModuleName] = struct{}{} + } + }) + + return modules +} + +func ConvergeTasksInQueue(q *queue.TaskQueue) int { + if q == nil { + return 0 + } + + convergeTasks := 0 + q.Iterate(func(t sh_task.Task) { + if IsConvergeTask(t) || IsFirstConvergeTask(t) { + convergeTasks++ + } + }) + + return convergeTasks +} + +// RemoveCurrentConvergeTasks detects if converge tasks present in the main +// queue after task if id equal to 'afterID'. These tasks are drained +// and the method returns true. +func RemoveCurrentConvergeTasks(q *queue.TaskQueue, afterId string) bool { + if q == nil { + return false + } + + IDFound := false + convergeDrained := false + stop := false + q.Filter(func(t sh_task.Task) bool { + if stop { + return true + } + // Keep tasks until specified task. + if !IDFound { + // Also keep specified task. + if t.GetId() == afterId { + IDFound = true + } + return true + } + + // Return false to remove converge task right after the specified task. + if IsConvergeTask(t) { + convergeDrained = true + return false + } + // Stop filtering when there is non-converge task after specified task. + stop = true + return true + }) + + return convergeDrained +} + +// RemoveAdjacentConvergeModules removes ConvergeModules tasks right +// after the task with the specified ID. +func RemoveAdjacentConvergeModules(q *queue.TaskQueue, afterId string) { + if q == nil { + return + } + + IDFound := false + stop := false + q.Filter(func(t sh_task.Task) bool { + if stop { + return true + } + if !IDFound { + if t.GetId() == afterId { + IDFound = true + } + return true + } + + // Remove ConvergeModules after current. + if t.GetType() == task.ConvergeModules { + return false + } + + stop = true + return true + }) +} + +func DrainNonMainQueue(q *queue.TaskQueue) { + if q == nil || q.Name == "main" { + return + } + + // Remove all tasks. + q.Filter(func(_ sh_task.Task) bool { + return false + }) +} diff --git a/pkg/addon-operator/queue_test.go b/pkg/addon-operator/queue_test.go new file mode 100644 index 00000000..7dec0bb1 --- /dev/null +++ b/pkg/addon-operator/queue_test.go @@ -0,0 +1,371 @@ +package addon_operator + +import ( + "testing" + + "github.com/flant/addon-operator/pkg/task" + sh_task "github.com/flant/shell-operator/pkg/task" + "github.com/flant/shell-operator/pkg/task/queue" + "github.com/stretchr/testify/require" +) + +func Test_QueueHasPendingModuleRunTask(t *testing.T) { + tests := []struct { + name string + result bool + queue func() *queue.TaskQueue + }{ + { + name: "Normal", + result: true, + queue: func() *queue.TaskQueue { + q := queue.NewTasksQueue() + + Task := &sh_task.BaseTask{Type: task.ModuleRun, Id: "unknown"} + q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) + + Task = &sh_task.BaseTask{Type: task.ModuleRun, Id: "unknown"} + q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) + + Task = &sh_task.BaseTask{Type: task.ModuleRun, Id: "test"} + q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "test"})) + return q + }}, + { + name: "First task", + result: false, + queue: func() *queue.TaskQueue { + q := queue.NewTasksQueue() + + Task := &sh_task.BaseTask{Type: task.ModuleRun, Id: "test"} + q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "test"})) + + Task = &sh_task.BaseTask{Type: task.GlobalHookRun, Id: "unknown"} + q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) + + Task = &sh_task.BaseTask{Type: task.ModuleRun, Id: "unknown"} + q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) + return q + }}, + { + name: "No module run", + result: false, + queue: func() *queue.TaskQueue { + q := queue.NewTasksQueue() + + Task := &sh_task.BaseTask{Type: task.ModuleRun, Id: "unknown"} + q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) + + Task = &sh_task.BaseTask{Type: task.ModuleHookRun, Id: "test"} + q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "test"})) + + Task = &sh_task.BaseTask{Type: task.ModuleRun, Id: "unknown"} + q.AddLast(Task.WithMetadata(task.HookMetadata{ModuleName: "unknown"})) + return q + }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := QueueHasPendingModuleRunTask(tt.queue(), "test") + require.Equal(t, tt.result, result, "QueueHasPendingModuleRunTask should run correctly") + }) + } +} + +func Test_RemoveAdjacentConvergeModules(t *testing.T) { + tests := []struct { + name string + afterID string + in []sh_task.BaseTask + expect []sh_task.BaseTask + }{ + { + name: "No adjacent ConvergeModules", + afterID: "1", + in: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleRun, Id: "2"}, + {Type: task.GlobalHookRun, Id: "3"}, + }, + expect: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleRun, Id: "2"}, + {Type: task.GlobalHookRun, Id: "3"}, + }, + }, + { + name: "No adjacent ConvergeModules, preceding tasks present", + afterID: "1", + in: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "-1"}, + {Type: task.ConvergeModules, Id: "0"}, + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleRun, Id: "2"}, + {Type: task.GlobalHookRun, Id: "3"}, + }, + expect: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "-1"}, + {Type: task.ConvergeModules, Id: "0"}, + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleRun, Id: "2"}, + {Type: task.GlobalHookRun, Id: "3"}, + }, + }, + { + name: "Single adjacent ConvergeModules task", + afterID: "1", + in: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ConvergeModules, Id: "2"}, + {Type: task.ModuleRun, Id: "3"}, + {Type: task.GlobalHookRun, Id: "4"}, + }, + expect: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleRun, Id: "3"}, + {Type: task.GlobalHookRun, Id: "4"}, + }, + }, + { + name: "Multiple adjacent ConvergeModules tasks", + afterID: "1", + in: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ConvergeModules, Id: "2"}, + {Type: task.ConvergeModules, Id: "3"}, + {Type: task.ConvergeModules, Id: "4"}, + {Type: task.ModuleRun, Id: "5"}, + {Type: task.GlobalHookRun, Id: "6"}, + }, + expect: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleRun, Id: "5"}, + {Type: task.GlobalHookRun, Id: "6"}, + }, + }, + { + name: "Interleaving ConvergeModules tasks", + afterID: "1", + in: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ConvergeModules, Id: "2"}, + {Type: task.ModuleRun, Id: "3"}, + {Type: task.ConvergeModules, Id: "4"}, + {Type: task.ConvergeModules, Id: "5"}, + {Type: task.GlobalHookRun, Id: "6"}, + }, + expect: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleRun, Id: "3"}, + {Type: task.ConvergeModules, Id: "4"}, + {Type: task.ConvergeModules, Id: "5"}, + {Type: task.GlobalHookRun, Id: "6"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q := queue.NewTasksQueue() + for _, tsk := range tt.in { + tmpTsk := tsk + q.AddLast(&tmpTsk) + } + require.Equal(t, len(tt.in), q.Length(), "Should add all tasks to the queue.") + + RemoveAdjacentConvergeModules(q, tt.afterID) + + // Check tasks after remove. + require.Equal(t, len(tt.expect), q.Length(), "queue length should match length of expected tasks") + i := 0 + q.Iterate(func(tsk sh_task.Task) { + require.Equal(t, tt.expect[i].Id, tsk.GetId(), "ID should match for task %d %+v", i, tsk) + require.Equal(t, tt.expect[i].Type, tsk.GetType(), "Type should match for task %d %+v", i, tsk) + i++ + }) + }) + } +} +func Test_ModulesWithPendingModuleRun(t *testing.T) { + moduleRunTask := func(id string, moduleName string) *sh_task.BaseTask { + tsk := &sh_task.BaseTask{Type: task.ModuleRun, Id: id} + return tsk.WithMetadata(task.HookMetadata{ + ModuleName: moduleName, + }) + } + + tests := []struct { + name string + afterID string + in []*sh_task.BaseTask + expect map[string]struct{} + }{ + { + name: "No ModuleRun tasks", + afterID: "1", + in: []*sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleHookRun, Id: "2"}, + {Type: task.GlobalHookRun, Id: "3"}, + }, + expect: map[string]struct{}{}, + }, + { + name: "First task is ModuleRun", + afterID: "1", + in: []*sh_task.BaseTask{ + moduleRunTask("1", "module_1"), + {Type: task.ModuleHookRun, Id: "2"}, + {Type: task.GlobalHookRun, Id: "3"}, + }, + expect: map[string]struct{}{}, + }, + { + name: "One pending ModuleRun", + afterID: "1", + in: []*sh_task.BaseTask{ + {Type: task.GlobalHookRun, Id: "1"}, + moduleRunTask("2", "module_1"), + {Type: task.GlobalHookRun, Id: "3"}, + }, + expect: map[string]struct{}{ + "module_1": {}, + }, + }, + { + name: "Multiple ModuleRun tasks", + afterID: "1", + in: []*sh_task.BaseTask{ + {Type: task.GlobalHookRun, Id: "1"}, + moduleRunTask("2", "module_1"), + {Type: task.ConvergeModules, Id: "3"}, + moduleRunTask("4", "module_2"), + {Type: task.GlobalHookRun, Id: "5"}, + }, + expect: map[string]struct{}{ + "module_1": {}, + "module_2": {}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q := queue.NewTasksQueue() + for _, tsk := range tt.in { + q.AddLast(tsk) + } + require.Equal(t, len(tt.in), q.Length(), "Should add all tasks to the queue.") + + actual := ModulesWithPendingModuleRun(q) + + // Check tasks after remove. + require.Equal(t, len(tt.expect), len(actual), "Should match length of expected modules") + require.Equal(t, tt.expect, actual, "Should match expected modules") + }) + } +} + +func Test_RemoveCurrentConvergeTasks(t *testing.T) { + tests := []struct { + name string + afterID string + in []sh_task.BaseTask + expect []sh_task.BaseTask + }{ + { + name: "No Converge tasks", + afterID: "1", + in: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleHookRun, Id: "2"}, + {Type: task.GlobalHookRun, Id: "3"}, + }, + expect: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleHookRun, Id: "2"}, + {Type: task.GlobalHookRun, Id: "3"}, + }, + }, + { + name: "No Converge tasks, preceding tasks present", + afterID: "1", + in: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "-1"}, + {Type: task.ConvergeModules, Id: "0"}, + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleHookRun, Id: "2"}, + {Type: task.GlobalHookRun, Id: "3"}, + }, + expect: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "-1"}, + {Type: task.ConvergeModules, Id: "0"}, + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleHookRun, Id: "2"}, + {Type: task.GlobalHookRun, Id: "3"}, + }, + }, + { + name: "Single adjacent ConvergeModules task with more Converge tasks", + afterID: "1", + in: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ConvergeModules, Id: "2"}, + {Type: task.ModuleRun, Id: "3"}, + {Type: task.ModuleDelete, Id: "4"}, + }, + expect: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleRun, Id: "3"}, + {Type: task.ModuleDelete, Id: "4"}, + }, + }, + { + name: "Converge in progress", + afterID: "1", + in: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ModuleDelete, Id: "2"}, + {Type: task.ModuleDelete, Id: "3"}, + {Type: task.ModuleRun, Id: "4"}, + {Type: task.ModuleRun, Id: "5"}, + {Type: task.ModuleRun, Id: "6"}, + {Type: task.ConvergeModules, Id: "7"}, + {Type: task.ConvergeModules, Id: "8"}, + {Type: task.ConvergeModules, Id: "9"}, + {Type: task.ModuleRun, Id: "11"}, + {Type: task.GlobalHookRun, Id: "10"}, + }, + expect: []sh_task.BaseTask{ + {Type: task.ConvergeModules, Id: "1"}, + {Type: task.ConvergeModules, Id: "8"}, + {Type: task.ConvergeModules, Id: "9"}, + {Type: task.ModuleRun, Id: "11"}, + {Type: task.GlobalHookRun, Id: "10"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q := queue.NewTasksQueue() + for _, tsk := range tt.in { + tmpTsk := tsk + q.AddLast(&tmpTsk) + } + require.Equal(t, len(tt.in), q.Length(), "Should add all tasks to the queue.") + + RemoveAdjacentConvergeModules(q, tt.afterID) + + // Check tasks after remove. + require.Equal(t, len(tt.expect), q.Length(), "queue length should match length of expected tasks") + i := 0 + q.Iterate(func(tsk sh_task.Task) { + require.Equal(t, tt.expect[i].Id, tsk.GetId(), "ID should match for task %d %+v", i, tsk) + require.Equal(t, tt.expect[i].Type, tsk.GetType(), "Type should match for task %d %+v", i, tsk) + i++ + }) + }) + } +} diff --git a/pkg/module_manager/testdata/discover_modules_state__with_enabled_scripts/config_map.yaml b/pkg/addon-operator/testdata/converge__main_queue_only/config_map.yaml similarity index 54% rename from pkg/module_manager/testdata/discover_modules_state__with_enabled_scripts/config_map.yaml rename to pkg/addon-operator/testdata/converge__main_queue_only/config_map.yaml index 5ae6cc28..d2d6c188 100644 --- a/pkg/module_manager/testdata/discover_modules_state__with_enabled_scripts/config_map.yaml +++ b/pkg/addon-operator/testdata/converge__main_queue_only/config_map.yaml @@ -3,4 +3,5 @@ kind: ConfigMap metadata: name: addon-operator data: - global: {} \ No newline at end of file + moduleAlphaEnabled: "true" + moduleBetaEnabled: "false" diff --git a/pkg/addon-operator/testdata/converge__main_queue_only/global-hooks/hook01_startup_20_kube.sh b/pkg/addon-operator/testdata/converge__main_queue_only/global-hooks/hook01_startup_20_kube.sh new file mode 100755 index 00000000..73a1a2c0 --- /dev/null +++ b/pkg/addon-operator/testdata/converge__main_queue_only/global-hooks/hook01_startup_20_kube.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +if [[ $1 == "--config" ]] ; then +cat < 0 +} + +// HasEqualChecksum returns true if there is a checksum for name equal to the input checksum. +func (c *Checksums) HasEqualChecksum(name string, checksum string) bool { + for chk := range c.sums[name] { + if chk == checksum { + return true + } + } + return false +} + +func (c *Checksums) Names() map[string]struct{} { + names := make(map[string]struct{}) + for name := range c.sums { + names[name] = struct{}{} + } + return names +} diff --git a/pkg/kube_config_manager/checksums_test.go b/pkg/kube_config_manager/checksums_test.go new file mode 100644 index 00000000..214ac97a --- /dev/null +++ b/pkg/kube_config_manager/checksums_test.go @@ -0,0 +1,30 @@ +package kube_config_manager + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_Checksums(t *testing.T) { + c := NewChecksums() + + // Adding + c.Add("global", "qwe") + + assert.True(t, c.HasChecksum("global")) + assert.True(t, c.HasEqualChecksum("global", "qwe")) + assert.False(t, c.HasChecksum("non-global"), "Should be false for non added name") + + // Names + expectedNames := map[string]struct{}{ + "global": {}, + } + assert.Equal(t, expectedNames, c.Names()) + + c.Remove("global", "unknown-checksum") + assert.True(t, c.HasEqualChecksum("global", "qwe")) + + c.Remove("global", "qwe") + assert.False(t, c.HasEqualChecksum("global", "qwe")) + +} diff --git a/pkg/kube_config_manager/config.go b/pkg/kube_config_manager/config.go new file mode 100644 index 00000000..bfe792d0 --- /dev/null +++ b/pkg/kube_config_manager/config.go @@ -0,0 +1,42 @@ +package kube_config_manager + +type KubeConfig struct { + Global *GlobalKubeConfig + Modules map[string]*ModuleKubeConfig +} + +func NewConfig() *KubeConfig { + return &KubeConfig{ + Modules: make(map[string]*ModuleKubeConfig), + } +} + +type KubeConfigEvent string + +const ( + KubeConfigChanged KubeConfigEvent = "Changed" + KubeConfigInvalid KubeConfigEvent = "Invalid" +) + +func ParseConfigMapData(data map[string]string) (cfg *KubeConfig, err error) { + cfg = NewConfig() + // Parse values in global section. + cfg.Global, err = GetGlobalKubeConfigFromConfigData(data) + if err != nil { + return nil, err + } + + moduleNames, err := GetModulesNamesFromConfigData(data) + if err != nil { + return nil, err + } + + for moduleName := range moduleNames { + cfg.Modules[moduleName], err = ExtractModuleKubeConfig(moduleName, data) + if err != nil { + return nil, err + } + } + + return cfg, nil +} diff --git a/pkg/kube_config_manager/config_test.go b/pkg/kube_config_manager/config_test.go new file mode 100644 index 00000000..5c4b1003 --- /dev/null +++ b/pkg/kube_config_manager/config_test.go @@ -0,0 +1,97 @@ +package kube_config_manager + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_ParseCM_Nil(t *testing.T) { + cfg, err := ParseConfigMapData(nil) + assert.NoError(t, err, "Should parse nil data correctly") + assert.NotNil(t, cfg, "Config should not be nil for nil data") + assert.Nil(t, cfg.Global, "No global config should present for nil data") + assert.NotNil(t, cfg.Modules, "Modules should not be nil for nil data") + assert.Len(t, cfg.Modules, 0, "No module configs should present for nil data") +} + +func Test_ParseCM_Empty(t *testing.T) { + cfg, err := ParseConfigMapData(map[string]string{}) + assert.NoError(t, err, "Should parse empty data correctly") + assert.NotNil(t, cfg, "Config should not be nil for empty data") + assert.Nil(t, cfg.Global, "No global config should present for empty data") + assert.NotNil(t, cfg.Modules, "Modules should not be nil for empty data") + assert.Len(t, cfg.Modules, 0, "No module configs should present for empty data") +} + +func Test_ParseCM_only_Global(t *testing.T) { + cfg, err := ParseConfigMapData(map[string]string{ + "global": ` +param1: val1 +param2: val2 +`, + }) + assert.NoError(t, err, "Should parse global only data correctly") + assert.NotNil(t, cfg, "Config should not be nil for global only data") + assert.NotNil(t, cfg.Global, "Global config should present for global only data") + assert.True(t, cfg.Global.Values.HasGlobal(), "Config should have global values for global only data") + assert.NotNil(t, cfg.Modules, "Modules should not be nil for global only data") + assert.Len(t, cfg.Modules, 0, "No module configs should present for global only data") + +} + +func Test_ParseCM_only_Modules(t *testing.T) { + cfg, err := ParseConfigMapData(map[string]string{ + "modOne": ` +param1: val1 +param2: val2 +`, + "modOneEnabled": `false`, + "modTwo": ` +param1: val1 +param2: val2 +`, + "modThreeEnabled": `true`, + }) + assert.NoError(t, err, "Should parse modules only data correctly") + assert.NotNil(t, cfg, "Config should not be nil for modules only data") + assert.Nil(t, cfg.Global, "Global config should not present for modules only data") + assert.NotNil(t, cfg.Modules, "Modules should not be nil for modules only data") + assert.Len(t, cfg.Modules, 3, "Module configs should present for modules only data") + + assert.Containsf(t, cfg.Modules, "mod-one", "Config for modOne should present for modules only data") + mod := cfg.Modules["mod-one"] + assert.Equal(t, "modOneEnabled", mod.ModuleEnabledKey) + assert.Equal(t, "modOne", mod.ModuleConfigKey) + assert.Equal(t, "false", mod.GetEnabled()) + assert.Containsf(t, cfg.Modules, "mod-two", "Config for modOne should present for modules only data") + mod = cfg.Modules["mod-two"] + assert.Equal(t, "modTwoEnabled", mod.ModuleEnabledKey) + assert.Equal(t, "modTwo", mod.ModuleConfigKey) + assert.Equal(t, "n/d", mod.GetEnabled()) + assert.Containsf(t, cfg.Modules, "mod-three", "Config for modOne should present for modules only data") + mod = cfg.Modules["mod-three"] + assert.Equal(t, "modThreeEnabled", mod.ModuleEnabledKey) + assert.Equal(t, "modThree", mod.ModuleConfigKey) + assert.Equal(t, "true", mod.GetEnabled()) +} + +func Test_ParseCM_Malformed_Data(t *testing.T) { + var err error + + _, err = ParseConfigMapData(map[string]string{ + "Malformed-section-name": ` +param1: val1 +param2: val2 +`, + }) + assert.Error(t, err, "Should parse malformed module name with error") + + _, err = ParseConfigMapData(map[string]string{ + "invalidYAML": ` +param1: val1 + param2: val2 +`, + }) + assert.Error(t, err, "Should parse bad module values with error") +} diff --git a/pkg/kube_config_manager/kube_config_manager.go b/pkg/kube_config_manager/kube_config_manager.go index 86e5c76e..6c99508d 100644 --- a/pkg/kube_config_manager/kube_config_manager.go +++ b/pkg/kube_config_manager/kube_config_manager.go @@ -3,10 +3,12 @@ package kube_config_manager import ( "context" "fmt" - "os" + "strconv" + "sync" "time" klient "github.com/flant/kube-client/client" + "github.com/flant/shell-operator/pkg/config" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" v1 "k8s.io/api/core/v1" @@ -18,18 +20,27 @@ import ( "github.com/flant/addon-operator/pkg/utils" ) +/** + * KubeConfigManager watches for changes in ConfigMap/addon-operator and provides + * methods to change its content. + * + * It stores values parsed from ConfigMap data. OpenAPI validation of these config values + * is not a responsibility of this component. + */ + type KubeConfigManager interface { WithContext(ctx context.Context) WithKubeClient(client klient.Client) WithNamespace(namespace string) WithConfigMapName(configMap string) - SetKubeGlobalValues(values utils.Values) error - SetKubeModuleValues(moduleName string, values utils.Values) error + WithRuntimeConfig(config *config.Config) + SaveGlobalConfigValues(values utils.Values) error + SaveModuleConfigValues(moduleName string, values utils.Values) error Init() error Start() Stop() - InitialConfig() *Config - CurrentConfig() *Config + KubeConfigEventCh() chan KubeConfigEvent + SafeReadConfig(handler func(config *KubeConfig)) } type kubeConfigManager struct { @@ -40,136 +51,41 @@ type kubeConfigManager struct { Namespace string ConfigMapName string - initialConfig *Config - currentConfig *Config - - GlobalValuesChecksum string - ModulesValuesChecksum map[string]string -} - -// kubeConfigManager should implement KubeConfigManager -var _ KubeConfigManager = &kubeConfigManager{} + m sync.Mutex + currentConfig *KubeConfig -type ModuleConfigs map[string]utils.ModuleConfig + // Checksums to ignore self-initiated updates. + knownChecksums *Checksums -func (m ModuleConfigs) Names() []string { - names := make([]string, 0) - for _, newModuleConfig := range m { - names = append(names, fmt.Sprintf("'%s'", newModuleConfig.ModuleName)) - } - return names -} - -type Config struct { - Values utils.Values - ModuleConfigs ModuleConfigs -} + // Channel to emit events. + configEventCh chan KubeConfigEvent -func NewConfig() *Config { - return &Config{ - Values: make(utils.Values), - ModuleConfigs: make(map[string]utils.ModuleConfig), - } + // Runtime config to enable logging all events from the ConfigMap at runtime. + runtimeConfig *config.Config + logConfigMapEvents bool + logEntry *log.Entry } -var ( - VerboseDebug bool - // ConfigUpdated chan receives a new Config when global values are changed - ConfigUpdated chan Config - // ModuleConfigsUpdated chan receives a list of all ModuleConfig in configData. Updated items marked as IsUpdated. - ModuleConfigsUpdated chan ModuleConfigs -) +// kubeConfigManager should implement KubeConfigManager +var _ KubeConfigManager = &kubeConfigManager{} -func simpleMergeConfigMapData(data map[string]string, newData map[string]string) map[string]string { - for k, v := range newData { - data[k] = v +func NewKubeConfigManager() KubeConfigManager { + return &kubeConfigManager{ + currentConfig: NewConfig(), + knownChecksums: NewChecksums(), + configEventCh: make(chan KubeConfigEvent, 1), + logEntry: log.WithField("component", "KubeConfigManager"), } - return data } func (kcm *kubeConfigManager) WithContext(ctx context.Context) { kcm.ctx, kcm.cancel = context.WithCancel(ctx) } -func (kcm *kubeConfigManager) Stop() { - if kcm.cancel != nil { - kcm.cancel() - } -} - func (kcm *kubeConfigManager) WithKubeClient(client klient.Client) { kcm.KubeClient = client } -func (kcm *kubeConfigManager) saveGlobalKubeConfig(globalKubeConfig GlobalKubeConfig) error { - err := kcm.changeOrCreateKubeConfig(func(obj *v1.ConfigMap) error { - obj.Data = simpleMergeConfigMapData(obj.Data, globalKubeConfig.ConfigData) - return nil - }) - if err != nil { - return err - } - // If ConfigMap is updated, save checksum for global section. - kcm.GlobalValuesChecksum = globalKubeConfig.Checksum - return nil -} - -func (kcm *kubeConfigManager) saveModuleKubeConfig(moduleKubeConfig ModuleKubeConfig) error { - err := kcm.changeOrCreateKubeConfig(func(obj *v1.ConfigMap) error { - obj.Data = simpleMergeConfigMapData(obj.Data, moduleKubeConfig.ConfigData) - return nil - }) - if err != nil { - return err - } - // TODO add a mutex for this map? Config patch from hook can run in parallel with ConfigMap editing... - kcm.ModulesValuesChecksum[moduleKubeConfig.ModuleName] = moduleKubeConfig.Checksum - return nil -} - -func (kcm *kubeConfigManager) changeOrCreateKubeConfig(configChangeFunc func(*v1.ConfigMap) error) error { - var err error - - obj, err := kcm.getConfigMap() - if err != nil { - return nil - } - - if obj != nil { - if obj.Data == nil { - obj.Data = make(map[string]string) - } - - err = configChangeFunc(obj) - if err != nil { - return err - } - - _, err := kcm.KubeClient.CoreV1().ConfigMaps(kcm.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) - if err != nil { - return err - } - - return nil - } else { - obj := &v1.ConfigMap{} - obj.Name = kcm.ConfigMapName - obj.Data = make(map[string]string) - - err = configChangeFunc(obj) - if err != nil { - return err - } - - _, err := kcm.KubeClient.CoreV1().ConfigMaps(kcm.Namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) - if err != nil { - return err - } - - return nil - } -} - func (kcm *kubeConfigManager) WithNamespace(namespace string) { kcm.Namespace = namespace } @@ -178,143 +94,124 @@ func (kcm *kubeConfigManager) WithConfigMapName(configMap string) { kcm.ConfigMapName = configMap } -func (kcm *kubeConfigManager) SetKubeGlobalValues(values utils.Values) error { +func (kcm *kubeConfigManager) WithRuntimeConfig(config *config.Config) { + kcm.runtimeConfig = config +} + +func (kcm *kubeConfigManager) SaveGlobalConfigValues(values utils.Values) error { globalKubeConfig, err := GetGlobalKubeConfigFromValues(values) if err != nil { return err } + if globalKubeConfig == nil { + return nil + } - if globalKubeConfig != nil { - log.Debugf("Kube config manager: set kube global values:\n%s", values.DebugString()) - - err := kcm.saveGlobalKubeConfig(*globalKubeConfig) - if err != nil { - return err - } + if kcm.logConfigMapEvents { + kcm.logEntry.Infof("Save global values to ConfigMap/%s:\n%s", kcm.ConfigMapName, values.DebugString()) + } else { + kcm.logEntry.Infof("Save global values to ConfigMap/%s", kcm.ConfigMapName) } - return nil -} + // Put checksum to known to ignore self-update. + kcm.withLock(func() { + kcm.knownChecksums.Add(utils.GlobalValuesKey, globalKubeConfig.Checksum) + }) -func (kcm *kubeConfigManager) SetKubeModuleValues(moduleName string, values utils.Values) error { - moduleKubeConfig, err := GetModuleKubeConfigFromValues(moduleName, values) + err = ConfigMapMergeValues(kcm.KubeClient, kcm.Namespace, kcm.ConfigMapName, globalKubeConfig.Values) if err != nil { + // Remove known checksum on error. + kcm.withLock(func() { + kcm.knownChecksums.Remove(utils.GlobalValuesKey, globalKubeConfig.Checksum) + }) return err } - if moduleKubeConfig != nil { - log.Debugf("Kube config manager: set kube module values:\n%s", moduleKubeConfig.ModuleConfig.String()) - - err := kcm.saveModuleKubeConfig(*moduleKubeConfig) - if err != nil { - return err - } - } - return nil } -func (kcm *kubeConfigManager) getConfigMap() (*v1.ConfigMap, error) { - list, err := kcm.KubeClient.CoreV1(). - ConfigMaps(kcm.Namespace). - List(context.TODO(), metav1.ListOptions{}) +// SaveModuleConfigValues updates module section in ConfigMap. +// It uses knownChecksums to prevent KubeConfigChanged event on self-update. +func (kcm *kubeConfigManager) SaveModuleConfigValues(moduleName string, values utils.Values) error { + moduleKubeConfig, err := GetModuleKubeConfigFromValues(moduleName, values) if err != nil { - return nil, err + return err } - objExists := false - for _, obj := range list.Items { - if obj.ObjectMeta.Name == kcm.ConfigMapName { - objExists = true - break - } + if moduleKubeConfig == nil { + return nil } - if objExists { - obj, err := kcm.KubeClient.CoreV1(). - ConfigMaps(kcm.Namespace). - Get(context.TODO(), kcm.ConfigMapName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - log.Debugf("KUBE_CONFIG_MANAGER: Will use ConfigMap/%s for persistent values", kcm.ConfigMapName) - return obj, nil + if kcm.logConfigMapEvents { + kcm.logEntry.Infof("Save module '%s' values to ConfigMap/%s:\n%s", moduleName, kcm.ConfigMapName, values.DebugString()) } else { - log.Debugf("KUBE_CONFIG_MANAGER: ConfigMap/%s is not created", kcm.ConfigMapName) - return nil, nil + kcm.logEntry.Infof("Save module '%s' values to ConfigMap/%s", moduleName, kcm.ConfigMapName) } -} -func (kcm *kubeConfigManager) InitialConfig() *Config { - return kcm.initialConfig -} + // Put checksum to known to ignore self-update. + kcm.withLock(func() { + kcm.knownChecksums.Add(moduleName, moduleKubeConfig.Checksum) + }) -func (kcm *kubeConfigManager) CurrentConfig() *Config { - return kcm.currentConfig + err = ConfigMapMergeValues(kcm.KubeClient, kcm.Namespace, kcm.ConfigMapName, moduleKubeConfig.Values) + if err != nil { + kcm.withLock(func() { + kcm.knownChecksums.Remove(moduleName, moduleKubeConfig.Checksum) + }) + return err + } + + return nil } -func NewKubeConfigManager() KubeConfigManager { - return &kubeConfigManager{ - initialConfig: NewConfig(), - currentConfig: NewConfig(), - ModulesValuesChecksum: map[string]string{}, - } +// KubeConfigEventCh return a channel that emits new KubeConfig on ConfigMap changes in global section or enabled modules. +func (kcm *kubeConfigManager) KubeConfigEventCh() chan KubeConfigEvent { + return kcm.configEventCh } -func (kcm *kubeConfigManager) initConfig() error { - obj, err := kcm.getConfigMap() +// loadConfig gets config from ConfigMap before starting informer. +// Set checksums for global section and modules. +func (kcm *kubeConfigManager) loadConfig() error { + obj, err := ConfigMapGet(kcm.KubeClient, kcm.Namespace, kcm.ConfigMapName) if err != nil { return err } if obj == nil { - log.Infof("Init config from ConfigMap: cm/%s is not found", kcm.ConfigMapName) + kcm.logEntry.Infof("Initial config from ConfigMap/%s: resource is not found", kcm.ConfigMapName) return nil } - initialConfig := NewConfig() - globalValuesChecksum := "" - modulesValuesChecksum := make(map[string]string) - - globalKubeConfig, err := GetGlobalKubeConfigFromConfigData(obj.Data) + newConfig, err := ParseConfigMapData(obj.Data) if err != nil { return err } - if globalKubeConfig != nil { - initialConfig.Values = globalKubeConfig.Values - globalValuesChecksum = globalKubeConfig.Checksum - } - - for moduleName := range GetModulesNamesFromConfigData(obj.Data) { - // all GetModulesNamesFromConfigData must exist - moduleKubeConfig, err := ExtractModuleKubeConfig(moduleName, obj.Data) - if err != nil { - return err - } - - initialConfig.ModuleConfigs[moduleKubeConfig.ModuleName] = moduleKubeConfig.ModuleConfig - modulesValuesChecksum[moduleKubeConfig.ModuleName] = moduleKubeConfig.Checksum - } - - kcm.initialConfig = initialConfig - kcm.currentConfig = initialConfig - kcm.GlobalValuesChecksum = globalValuesChecksum - kcm.ModulesValuesChecksum = modulesValuesChecksum + kcm.currentConfig = newConfig return nil } func (kcm *kubeConfigManager) Init() error { - log.Debug("INIT: KUBE_CONFIG") - - VerboseDebug = false - if os.Getenv("KUBE_CONFIG_MANAGER_DEBUG") != "" { - VerboseDebug = true + kcm.logEntry.Debug("INIT: KUBE_CONFIG") + + if kcm.runtimeConfig != nil { + kcm.runtimeConfig.Register( + "log.configmap.events", + fmt.Sprintf("Set to true to log all operations with ConfigMap/%s", kcm.ConfigMapName), + "false", + func(oldValue string, newValue string) error { + val, err := strconv.ParseBool(newValue) + if err != nil { + return err + } + kcm.logConfigMapEvents = val + return nil + }, + nil, + ) } - ConfigUpdated = make(chan Config, 1) - ModuleConfigsUpdated = make(chan ModuleConfigs, 1) - - err := kcm.initConfig() + // Load config and calculate checksums at start. No locking required. + err := kcm.loadConfig() if err != nil { return err } @@ -322,175 +219,109 @@ func (kcm *kubeConfigManager) Init() error { return nil } -// handleNewCm determine changes in kube config. -// -// New Config is send over ConfigUpdate channel if global section is changed. -// -// Array of actual ModuleConfig is send over ModuleConfigsUpdated channel -// if module sections are changed or deleted. -func (kcm *kubeConfigManager) handleNewCm(obj *v1.ConfigMap) error { - globalKubeConfig, err := GetGlobalKubeConfigFromConfigData(obj.Data) - if err != nil { - return err +// currentModuleNames gather modules names from the checksums map and from the currentConfig struct. +func (kcm *kubeConfigManager) currentModuleNames() map[string]struct{} { + names := make(map[string]struct{}) + for name := range kcm.currentConfig.Modules { + names[name] = struct{}{} } + return names +} - // if global values are changed or deleted then new config should be sent over ConfigUpdated channel - isGlobalUpdated := globalKubeConfig != nil && - globalKubeConfig.Checksum != kcm.GlobalValuesChecksum - isGlobalDeleted := globalKubeConfig == nil && kcm.GlobalValuesChecksum != "" - - if isGlobalUpdated || isGlobalDeleted { - log.Infof("Kube config manager: detect changes in global section") - newConfig := NewConfig() - - // calculate new checksum of a global section - newGlobalValuesChecksum := "" - if globalKubeConfig != nil { - newConfig.Values = globalKubeConfig.Values - newGlobalValuesChecksum = globalKubeConfig.Checksum - } - kcm.GlobalValuesChecksum = newGlobalValuesChecksum +// handleNewCm determine changes in kube config. It sends KubeConfigChanged event if something +// changed or KubeConfigInvalid event if ConfigMap is incorrect. +func (kcm *kubeConfigManager) handleCmEvent(obj *v1.ConfigMap) error { + // ConfigMap is deleted, reset cached config and fire event. + if obj == nil { + kcm.m.Lock() + kcm.currentConfig = NewConfig() + kcm.m.Unlock() + kcm.configEventCh <- KubeConfigChanged + return nil + } - // calculate new checksums of a module sections - newModulesValuesChecksum := make(map[string]string) - for moduleName := range GetModulesNamesFromConfigData(obj.Data) { - // all GetModulesNamesFromConfigData must exist - moduleKubeConfig, err := ExtractModuleKubeConfig(moduleName, obj.Data) - if err != nil { - return err - } + newConfig, err := ParseConfigMapData(obj.Data) + if err != nil { + // Do not update caches to detect changes on next update. + kcm.configEventCh <- KubeConfigInvalid + kcm.logEntry.Errorf("ConfigMap/%s invalid: %v", kcm.ConfigMapName, err) + return err + } - newConfig.ModuleConfigs[moduleKubeConfig.ModuleName] = moduleKubeConfig.ModuleConfig - newModulesValuesChecksum[moduleKubeConfig.ModuleName] = moduleKubeConfig.Checksum - } - kcm.ModulesValuesChecksum = newModulesValuesChecksum + // Lock to read known checksums and update config. + kcm.m.Lock() - log.Debugf("Kube config manager: global section new values:\n%s", - newConfig.Values.DebugString()) - for _, moduleConfig := range newConfig.ModuleConfigs { - log.Debugf("%s", moduleConfig.String()) + globalChanged := false + if newConfig.Global == nil { + // Global section is deleted if ConfigMap has no global section but global config is cached. + // Note: no checksum checking, save operations can't delete global section. + if kcm.currentConfig.Global != nil { + globalChanged = true + kcm.logEntry.Infof("Global section deleted") } - - ConfigUpdated <- *newConfig - - kcm.currentConfig = newConfig } else { - actualModulesNames := GetModulesNamesFromConfigData(obj.Data) - - moduleConfigsActual := make(ModuleConfigs) - updatedCount := 0 - removedCount := 0 - - // create ModuleConfig for each module in configData - // IsUpdated flag set for updated configs - for moduleName := range actualModulesNames { - // all GetModulesNamesFromConfigData must exist - moduleKubeConfig, err := ExtractModuleKubeConfig(moduleName, obj.Data) - if err != nil { - return err - } - - if moduleKubeConfig.Checksum != kcm.ModulesValuesChecksum[moduleName] { - kcm.ModulesValuesChecksum[moduleName] = moduleKubeConfig.Checksum - moduleKubeConfig.ModuleConfig.IsUpdated = true - updatedCount++ - } else { - moduleKubeConfig.ModuleConfig.IsUpdated = false - } - moduleConfigsActual[moduleName] = moduleKubeConfig.ModuleConfig - } - - // delete checksums for removed module sections - for module := range kcm.ModulesValuesChecksum { - if _, isActual := actualModulesNames[module]; isActual { - continue - } - delete(kcm.ModulesValuesChecksum, module) - removedCount++ - } - - if updatedCount > 0 || removedCount > 0 { - log.Infof("KUBE_CONFIG Detect module sections changes: %d updated, %d removed", updatedCount, removedCount) - for _, moduleConfig := range moduleConfigsActual { - log.Debugf("%s", moduleConfig.String()) + newChecksum := newConfig.Global.Checksum + // Global section is updated if a new checksum not equal to the saved one and not in knownChecksum. + if kcm.knownChecksums.HasEqualChecksum(utils.GlobalValuesKey, newChecksum) { + // Remove known checksum, do not fire event on self-update. + kcm.knownChecksums.Remove(utils.GlobalValuesKey, newChecksum) + } else if kcm.currentConfig.Global != nil { + if kcm.currentConfig.Global.Checksum != newChecksum { + globalChanged = true + kcm.logEntry.Infof("Global section updated") } - ModuleConfigsUpdated <- moduleConfigsActual - kcm.currentConfig.ModuleConfigs = moduleConfigsActual } } - return nil -} - -func (kcm *kubeConfigManager) handleCmAdd(obj *v1.ConfigMap) error { - if VerboseDebug { - objYaml, err := yaml.Marshal(obj) - if err != nil { - return err - } - log.Debugf("Kube config manager: informer: handle ConfigMap '%s' add:\n%s", obj.Name, objYaml) - } - - return kcm.handleNewCm(obj) -} - -func (kcm *kubeConfigManager) handleCmUpdate(_ *v1.ConfigMap, obj *v1.ConfigMap) error { - if VerboseDebug { - objYaml, err := yaml.Marshal(obj) - if err != nil { - return err + // Parse values in module sections, create new ModuleConfigs and checksums map. + currentModuleNames := kcm.currentModuleNames() + modulesChanged := false + + for moduleName, moduleCfg := range newConfig.Modules { + // Remove module name from current names to detect deleted sections. + delete(currentModuleNames, moduleName) + + // Module section is changed if new checksum not equal to saved one and not in known checksums. + if kcm.knownChecksums.HasEqualChecksum(moduleName, moduleCfg.Checksum) { + // Remove known checksum, do not fire event on self-update. + kcm.knownChecksums.Remove(moduleName, moduleCfg.Checksum) + } else { + if currModuleCfg, has := kcm.currentConfig.Modules[moduleName]; has { + if currModuleCfg.Checksum != moduleCfg.Checksum { + modulesChanged = true + kcm.logEntry.Infof("Module section '%s' changed. Enabled flag transition: %s--%s", + moduleName, + kcm.currentConfig.Modules[moduleName].GetEnabled(), + moduleCfg.GetEnabled(), + ) + } + } else { + modulesChanged = true + kcm.logEntry.Infof("Module section '%s' added. Enabled flag: %s", moduleName, moduleCfg.GetEnabled()) + } } - log.Debugf("Kube config manager: informer: handle ConfigMap '%s' update:\n%s", obj.Name, objYaml) } - return kcm.handleNewCm(obj) -} - -func (kcm *kubeConfigManager) handleCmDelete(obj *v1.ConfigMap) error { - if VerboseDebug { - objYaml, err := yaml.Marshal(obj) - if err != nil { - return err - } - log.Debugf("Kube config manager: handle ConfigMap '%s' delete:\n%s", obj.Name, objYaml) + // currentModuleNames now contains deleted module sections. + if len(currentModuleNames) > 0 { + modulesChanged = true + kcm.logEntry.Infof("Module sections deleted: %+v", currentModuleNames) } - if kcm.GlobalValuesChecksum != "" { - kcm.GlobalValuesChecksum = "" - kcm.ModulesValuesChecksum = make(map[string]string) - - ConfigUpdated <- Config{ - Values: make(utils.Values), - ModuleConfigs: make(map[string]utils.ModuleConfig), - } - } else { - // Global values is already known to be empty. - // So check each module values change separately, - // and generate signals per-module. - // Note: Only ModuleName field is needed in ModuleConfig. - - moduleConfigsUpdate := make(ModuleConfigs) - - updateModulesNames := make([]string, 0) - for module := range kcm.ModulesValuesChecksum { - updateModulesNames = append(updateModulesNames, module) - } - for _, module := range updateModulesNames { - delete(kcm.ModulesValuesChecksum, module) - moduleConfigsUpdate[module] = utils.ModuleConfig{ - ModuleName: module, - Values: make(utils.Values), - } - } + // Update state after successful parsing. + kcm.currentConfig = newConfig + kcm.m.Unlock() - ModuleConfigsUpdated <- moduleConfigsUpdate + // Fire event if ConfigMap has changes. + if globalChanged || modulesChanged { + kcm.configEventCh <- KubeConfigChanged } return nil } func (kcm *kubeConfigManager) Start() { - log.Debugf("Run kube config manager") + kcm.logEntry.Debugf("Start kube config manager") // define resyncPeriod for informer resyncPeriod := time.Duration(5) * time.Minute @@ -506,24 +337,64 @@ func (kcm *kubeConfigManager) Start() { cmInformer := corev1.NewFilteredConfigMapInformer(kcm.KubeClient, kcm.Namespace, resyncPeriod, indexers, tweakListOptions) cmInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - err := kcm.handleCmAdd(obj.(*v1.ConfigMap)) + kcm.logConfigMapEvent(obj, "add") + err := kcm.handleCmEvent(obj.(*v1.ConfigMap)) if err != nil { - log.Errorf("Kube config manager: cannot handle ConfigMap add: %s", err) + kcm.logEntry.Errorf("Handle ConfigMap/%s 'add' error: %s", kcm.ConfigMapName, err) } }, UpdateFunc: func(prevObj interface{}, obj interface{}) { - err := kcm.handleCmUpdate(prevObj.(*v1.ConfigMap), obj.(*v1.ConfigMap)) + kcm.logConfigMapEvent(obj, "update") + err := kcm.handleCmEvent(obj.(*v1.ConfigMap)) if err != nil { - log.Errorf("Kube config manager: cannot handle ConfigMap update: %s", err) + kcm.logEntry.Errorf("Handle ConfigMap/%s 'update' error: %s", kcm.ConfigMapName, err) } }, DeleteFunc: func(obj interface{}) { - err := kcm.handleCmDelete(obj.(*v1.ConfigMap)) - if err != nil { - log.Errorf("Kube config manager: cannot handle ConfigMap delete: %s", err) - } + kcm.logConfigMapEvent(obj, "delete") + _ = kcm.handleCmEvent(nil) }, }) - cmInformer.Run(kcm.ctx.Done()) + go func() { + cmInformer.Run(kcm.ctx.Done()) + }() +} + +func (kcm *kubeConfigManager) Stop() { + if kcm.cancel != nil { + kcm.cancel() + } +} + +func (kcm *kubeConfigManager) logConfigMapEvent(obj interface{}, eventName string) { + if !kcm.logConfigMapEvents { + return + } + + objYaml, err := yaml.Marshal(obj) + if err != nil { + kcm.logEntry.Infof("Dump ConfigMap/%s '%s' error: %s", kcm.ConfigMapName, eventName, err) + return + } + kcm.logEntry.Infof("Dump ConfigMap/%s '%s':\n%s", kcm.ConfigMapName, eventName, objYaml) +} + +// SafeReadConfig locks currentConfig to safely read from it in external services. +func (kcm *kubeConfigManager) SafeReadConfig(handler func(config *KubeConfig)) { + if handler == nil { + return + } + kcm.withLock(func() { + handler(kcm.currentConfig) + }) +} + +func (kcm *kubeConfigManager) withLock(fn func()) { + if fn == nil { + return + } + kcm.m.Lock() + fn() + kcm.m.Unlock() } diff --git a/pkg/kube_config_manager/kube_config_manager_test.go b/pkg/kube_config_manager/kube_config_manager_test.go index 8716ca91..3d165d08 100644 --- a/pkg/kube_config_manager/kube_config_manager_test.go +++ b/pkg/kube_config_manager/kube_config_manager_test.go @@ -2,22 +2,54 @@ package kube_config_manager import ( "context" - klient "github.com/flant/kube-client/client" - "sync" "testing" - "github.com/flant/addon-operator/pkg/app" + klient "github.com/flant/kube-client/client" . "github.com/onsi/gomega" - "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "github.com/flant/addon-operator/pkg/utils" ) -func Test_LoadValues_On_Init(t *testing.T) { +const testConfigMapName = "test-addon-operator" + +func initKubeConfigManager(t *testing.T, kubeClient klient.Client, cmData map[string]string, cmContent string) KubeConfigManager { + g := NewWithT(t) + + cm := &v1.ConfigMap{} + cm.SetNamespace("default") + cm.SetName(testConfigMapName) + + if cmData != nil { + cm.Data = cmData + } else { + cmData := map[string]string{} + _ = yaml.Unmarshal([]byte(cmContent), cmData) + cm.Data = cmData + } + + _, err := kubeClient.CoreV1().ConfigMaps("default").Create(context.TODO(), cm, metav1.CreateOptions{}) + g.Expect(err).ShouldNot(HaveOccurred(), "ConfigMap should be created") + + kcm := NewKubeConfigManager() + kcm.WithContext(context.Background()) + kcm.WithKubeClient(kubeClient) + kcm.WithNamespace("default") + kcm.WithConfigMapName(testConfigMapName) + + err = kcm.Init() + g.Expect(err).ShouldNot(HaveOccurred(), "KubeConfigManager should init correctly") + + go kcm.Start() + + return kcm +} + +func Test_KubeConfigManager_loadConfig(t *testing.T) { cmDataText := ` global: | project: tfprod @@ -40,25 +72,12 @@ prometheus: | userPassword: qwerty kubeLegoEnabled: "false" ` - cmData := map[string]string{} - _ = yaml.Unmarshal([]byte(cmDataText), cmData) kubeClient := klient.NewFake(nil) - _, _ = kubeClient.CoreV1().ConfigMaps("default").Create(context.TODO(), &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "addon-operator"}, - Data: cmData, - }, metav1.CreateOptions{}) - kcm := NewKubeConfigManager() - kcm.WithKubeClient(kubeClient) - kcm.WithNamespace("default") - kcm.WithConfigMapName("addon-operator") + kcm := initKubeConfigManager(t, kubeClient, nil, cmDataText) - err := kcm.Init() - if err != nil { - t.Errorf("kube_config_manager initialization error: %s", err) - } - config := kcm.InitialConfig() + defer kcm.Stop() tests := map[string]struct { isEnabled *bool @@ -110,28 +129,30 @@ kubeLegoEnabled: "false" }, } + // No need to use lock in KubeConfigManager. for name, expect := range tests { t.Run(name, func(t *testing.T) { if name == "global" { - assert.Equal(t, expect.values, config.Values) + kcm.SafeReadConfig(func(config *KubeConfig) { + assert.Equal(t, expect.values, config.Global.Values) + }) } else { - // module - moduleConfig, hasConfig := config.ModuleConfigs[name] - assert.True(t, hasConfig) - assert.Equal(t, expect.isEnabled, moduleConfig.IsEnabled) - assert.Equal(t, expect.values, moduleConfig.Values) + kcm.SafeReadConfig(func(config *KubeConfig) { + // module + moduleConfig, hasConfig := config.Modules[name] + assert.True(t, hasConfig) + assert.Equal(t, expect.isEnabled, moduleConfig.IsEnabled) + assert.Equal(t, expect.values, moduleConfig.Values) + }) } }) } } -func Test_SaveValuesToConfigMap(t *testing.T) { +func Test_KubeConfigManager_SaveValuesToConfigMap(t *testing.T) { kubeClient := klient.NewFake(nil) - kcm := NewKubeConfigManager() - kcm.WithKubeClient(kubeClient) - kcm.WithNamespace("default") - kcm.WithConfigMapName("addon-operator") + kcm := initKubeConfigManager(t, kubeClient, nil, "") var err error var cm *v1.ConfigMap @@ -141,7 +162,7 @@ func Test_SaveValuesToConfigMap(t *testing.T) { globalValues *utils.Values moduleValues *utils.Values moduleName string - testFn func(global *utils.Values, module *utils.Values) + testFn func(t *testing.T, global *utils.Values, module *utils.Values) }{ { "scenario 1: first save with non existent ConfigMap", @@ -155,7 +176,7 @@ func Test_SaveValuesToConfigMap(t *testing.T) { }, nil, "", - func(global *utils.Values, module *utils.Values) { + func(t *testing.T, global *utils.Values, module *utils.Values) { // Check values in a 'global' key assert.Contains(t, cm.Data, "global", "ConfigMap should contain a 'global' key") savedGlobalValues, err := utils.NewGlobalValues(cm.Data["global"]) @@ -179,7 +200,7 @@ func Test_SaveValuesToConfigMap(t *testing.T) { }, }, nil, "", - func(global *utils.Values, module *utils.Values) { + func(t *testing.T, global *utils.Values, module *utils.Values) { // Check values in a 'global' key assert.Contains(t, cm.Data, "global", "ConfigMap should contain a 'global' key") savedGlobalValues, err := utils.NewGlobalValues(cm.Data["global"]) @@ -198,7 +219,7 @@ func Test_SaveValuesToConfigMap(t *testing.T) { }, }, "mymodule", - func(global *utils.Values, module *utils.Values) { + func(t *testing.T, global *utils.Values, module *utils.Values) { // Check values in a 'global' key assert.Contains(t, cm.Data, "global", "ConfigMap should contain a 'global' key") @@ -234,26 +255,26 @@ func Test_SaveValuesToConfigMap(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { if test.globalValues != nil { - err = kcm.SetKubeGlobalValues(*test.globalValues) + err = kcm.SaveGlobalConfigValues(*test.globalValues) if !assert.NoError(t, err, "Global Values should be saved") { t.FailNow() } } else if test.moduleValues != nil { - err = kcm.SetKubeModuleValues(test.moduleName, *test.moduleValues) + err = kcm.SaveModuleConfigValues(test.moduleName, *test.moduleValues) if !assert.NoError(t, err, "Module Values should be saved") { t.FailNow() } } // Check that ConfigMap is created or exists - cm, err = kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), "addon-operator", metav1.GetOptions{}) - if assert.NoError(t, err, "ConfigMap should exist after SetKubeGlobalValues") { + cm, err = kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), testConfigMapName, metav1.GetOptions{}) + if assert.NoError(t, err, "ConfigMap should exist after SaveGlobalConfigValues") { assert.NotNil(t, cm, "ConfigMap should not be nil") } else { t.FailNow() } - test.testFn(test.globalValues, test.moduleValues) + test.testFn(t, test.globalValues, test.moduleValues) }) } @@ -261,88 +282,58 @@ func Test_SaveValuesToConfigMap(t *testing.T) { // Receive message over ModuleConfigsUpdate when ConfigMap is // externally modified. -func TestKubeConfigManager_ModuleConfigsUpdated_chan(t *testing.T) { +func Test_KubeConfigManager_event_after_adding_module_section(t *testing.T) { g := NewWithT(t) kubeClient := klient.NewFake(nil) - cm := &v1.ConfigMap{} - cm.SetNamespace("default") - cm.SetName(app.ConfigMapName) - cm.Data = map[string]string{ + kcm := initKubeConfigManager(t, kubeClient, map[string]string{ "global": ` param1: val1 param2: val2 `, - } - - _, err := kubeClient.CoreV1().ConfigMaps("default").Create(context.TODO(), cm, metav1.CreateOptions{}) - g.Expect(err).ShouldNot(HaveOccurred(), "ConfigMap should be created") + }, "") - kcm := NewKubeConfigManager() - kcm.WithContext(context.Background()) - kcm.WithKubeClient(kubeClient) - kcm.WithNamespace("default") - kcm.WithConfigMapName(app.ConfigMapName) - - err = kcm.Init() - g.Expect(err).ShouldNot(HaveOccurred(), "KubeConfigManager should init correctly") - - go kcm.Start() defer kcm.Stop() - var newModuleConfigs ModuleConfigs - - wg := sync.WaitGroup{} - wg.Add(1) - - go func() { - newModuleConfigs = <-ModuleConfigsUpdated - wg.Done() - }() - - // update cm - cm.Data["module2"] = ` -modParam1: val1 -modParam2: val2 -` - _, err = kubeClient.CoreV1().ConfigMaps("default").Update(context.TODO(), cm, metav1.UpdateOptions{}) - g.Expect(err).ShouldNot(HaveOccurred(), "ConfigMap should be created") - - wg.Wait() - - g.Expect(newModuleConfigs).To(HaveLen(1)) + // Check initial modules configs. + kcm.SafeReadConfig(func(config *KubeConfig) { + g.Expect(config.Modules).To(HaveLen(0), "No modules section should be after Init()") + }) + + // Update ConfigMap with new module section. + sectionPatch := `[{"op": "add", +"path": "/data/module2", +"value": "modParam1: val1\nmodParam2: val2"}]` + + _, err := kubeClient.CoreV1().ConfigMaps("default").Patch(context.TODO(), + testConfigMapName, + types.JSONPatchType, + []byte(sectionPatch), + metav1.PatchOptions{}, + ) + g.Expect(err).ShouldNot(HaveOccurred(), "ConfigMap should be patched") + + // Wait for event + <-kcm.KubeConfigEventCh() + + kcm.SafeReadConfig(func(config *KubeConfig) { + g.Expect(config.Modules).To(HaveLen(1), "Module section should appear after ConfigMap update") + }) } -// SetKubeModuleValues should update ConfigMap's data -func TestKubeConfigManager_SetKubeModuleValues(t *testing.T) { +// SaveModuleConfigValues should update ConfigMap's data +func Test_KubeConfigManager_SaveModuleConfigValues(t *testing.T) { g := NewWithT(t) - kubeClient := klient.NewFake(nil) - cm := &v1.ConfigMap{} - cm.SetNamespace("default") - cm.SetName(app.ConfigMapName) - cm.Data = map[string]string{ + kcm := initKubeConfigManager(t, kubeClient, map[string]string{ "global": ` param1: val1 param2: val2 `, - } + }, "") - _, err := kubeClient.CoreV1().ConfigMaps("default").Create(context.TODO(), cm, metav1.CreateOptions{}) - g.Expect(err).ShouldNot(HaveOccurred(), "ConfigMap should be created") - - kcm := NewKubeConfigManager() - kcm.WithContext(context.Background()) - kcm.WithKubeClient(kubeClient) - kcm.WithNamespace("default") - kcm.WithConfigMapName(app.ConfigMapName) - - err = kcm.Init() - g.Expect(err).ShouldNot(HaveOccurred(), "KubeConfigManager should init correctly") - - go kcm.Start() defer kcm.Stop() // Set modules values @@ -354,14 +345,76 @@ moduleLongName: g.Expect(err).ShouldNot(HaveOccurred(), "values should load from bytes") g.Expect(modVals).To(HaveKey("moduleLongName")) - err = kcm.SetKubeModuleValues("module-long-name", modVals) + err = kcm.SaveModuleConfigValues("module-long-name", modVals) g.Expect(err).ShouldNot(HaveOccurred()) // Check that values are updated in ConfigMap - cm, err = kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), app.ConfigMapName, metav1.GetOptions{}) + cm, err := kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), testConfigMapName, metav1.GetOptions{}) g.Expect(err).ShouldNot(HaveOccurred(), "ConfigMap get") g.Expect(cm.Data).Should(HaveLen(2)) g.Expect(cm.Data).To(HaveKey("global")) g.Expect(cm.Data).To(HaveKey("moduleLongName")) } + +// Check error if ConfigMap on init +func Test_KubeConfigManager_error_on_Init(t *testing.T) { + g := NewWithT(t) + + kubeClient := klient.NewFake(nil) + kcm := initKubeConfigManager(t, kubeClient, nil, "global: ''") + defer kcm.Stop() + + // Update ConfigMap with new module section with bad name. + badNameSectionPatch := `[{"op": "add", +"path": "/data/InvalidName-module", +"value": "modParam1: val1\nmodParam2: val2"}]` + + _, err := kubeClient.CoreV1().ConfigMaps("default").Patch(context.TODO(), + testConfigMapName, + types.JSONPatchType, + []byte(badNameSectionPatch), + metav1.PatchOptions{}, + ) + g.Expect(err).ShouldNot(HaveOccurred(), "ConfigMap should be patched") + + // Wait for event + ev := <-kcm.KubeConfigEventCh() + + g.Expect(ev).To(Equal(KubeConfigInvalid), "Invalid name in module section should generate 'invalid' event") + + //kcm.SafeReadConfig(func(config *KubeConfig) { + // g.Expect(config.IsInvalid).To(Equal(true), "Current config should be invalid") + //}) + + // Update ConfigMap with new module section with bad name. + validSectionPatch := `[{"op": "add", +"path": "/data/validModuleName", +"value": "modParam1: val1\nmodParam2: val2"}, +{"op": "remove", +"path": "/data/InvalidName-module"}]` + + _, err = kubeClient.CoreV1().ConfigMaps("default").Patch(context.TODO(), + testConfigMapName, + types.JSONPatchType, + []byte(validSectionPatch), + metav1.PatchOptions{}, + ) + g.Expect(err).ShouldNot(HaveOccurred(), "ConfigMap should be patched") + + // Wait for event + ev = <-kcm.KubeConfigEventCh() + + g.Expect(ev).To(Equal(KubeConfigChanged), "Valid section patch should generate 'changed' event") + + kcm.SafeReadConfig(func(config *KubeConfig) { + //g.Expect(config.IsInvalid).To(Equal(false), "Current config should be valid") + g.Expect(config.Modules).To(HaveLen(1), "Current config should have module sections") + g.Expect(config.Modules).To(HaveKey("valid-module-name"), "Current config should have module section for 'valid-module-name'") + modValues := config.Modules["valid-module-name"].Values + g.Expect(modValues.HasKey("validModuleName")).To(BeTrue()) + m := modValues["validModuleName"] + vals := m.(map[string]interface{}) + g.Expect(vals).To(HaveKey("modParam2"), "Module config values should contain modParam2 key") + }) +} diff --git a/pkg/kube_config_manager/mock.go b/pkg/kube_config_manager/mock.go new file mode 100644 index 00000000..cbbdf813 --- /dev/null +++ b/pkg/kube_config_manager/mock.go @@ -0,0 +1,17 @@ +// +build !release + +package kube_config_manager + +import "github.com/flant/addon-operator/pkg/utils" + +type MockKubeConfigManager struct { + KubeConfigManager +} + +func (kcm MockKubeConfigManager) SaveGlobalConfigValues(values utils.Values) error { + return nil +} + +func (kcm MockKubeConfigManager) SaveModuleConfigValues(moduleName string, values utils.Values) error { + return nil +} diff --git a/pkg/kube_config_manager/module_kube_config.go b/pkg/kube_config_manager/module_kube_config.go index ac413ae7..8dc3bb60 100644 --- a/pkg/kube_config_manager/module_kube_config.go +++ b/pkg/kube_config_manager/module_kube_config.go @@ -5,33 +5,31 @@ import ( "strings" "github.com/flant/addon-operator/pkg/utils" - log "github.com/sirupsen/logrus" ) -// TODO make a method of KubeConfig -// TODO LOG: multierror? // GetModulesNamesFromConfigData returns all keys in kube config except global // modNameEnabled keys are also handled -func GetModulesNamesFromConfigData(configData map[string]string) map[string]bool { +func GetModulesNamesFromConfigData(configData map[string]string) (map[string]bool, error) { res := make(map[string]bool) for key := range configData { + // Ignore global section. if key == utils.GlobalValuesKey { continue } + // Treat Enabled flags as module section. key = strings.TrimSuffix(key, "Enabled") modName := utils.ModuleNameFromValuesKey(key) if utils.ModuleNameToValuesKey(modName) != key { - log.Errorf("Bad module name '%s': should be camelCased module name: ignoring data", key) - continue + return nil, fmt.Errorf("bad module name '%s': should be camelCased", key) } res[modName] = true } - return res + return res, nil } type ModuleKubeConfig struct { @@ -40,7 +38,13 @@ type ModuleKubeConfig struct { ConfigData map[string]string } -// TODO make a method of KubeConfig +func (m *ModuleKubeConfig) GetEnabled() string { + if m == nil { + return "" + } + return m.ModuleConfig.GetEnabled() +} + func GetModuleKubeConfigFromValues(moduleName string, values utils.Values) (*ModuleKubeConfig, error) { valuesKey := utils.ModuleNameToValuesKey(moduleName) if !values.HasKey(valuesKey) { @@ -69,16 +73,15 @@ func GetModuleKubeConfigFromValues(moduleName string, values utils.Values) (*Mod }, nil } -// TODO make a method of KubeConfig // ExtractModuleKubeConfig returns ModuleKubeConfig with values loaded from ConfigMap func ExtractModuleKubeConfig(moduleName string, configData map[string]string) (*ModuleKubeConfig, error) { moduleConfig, err := utils.NewModuleConfig(moduleName).FromConfigMapData(configData) if err != nil { - return nil, fmt.Errorf("ConfigMap: bad yaml at key '%s': %s", utils.ModuleNameToValuesKey(moduleName), err) + return nil, fmt.Errorf("bad yaml at key '%s': %s", utils.ModuleNameToValuesKey(moduleName), err) } // NOTE this should never happen because of GetModulesNamesFromConfigData if moduleConfig == nil { - return nil, fmt.Errorf("possible bug!!! Kube config for module '%s' is not found in ConfigMap.data", moduleName) + return nil, fmt.Errorf("possible bug!!! No section '%s' for module '%s'", utils.ModuleNameToValuesKey(moduleName), moduleName) } return &ModuleKubeConfig{ diff --git a/pkg/module_manager/global_hook.go b/pkg/module_manager/global_hook.go index f7a9d9e0..4c0e77ec 100644 --- a/pkg/module_manager/global_hook.go +++ b/pkg/module_manager/global_hook.go @@ -157,6 +157,7 @@ func (h *GlobalHook) Run(bindingType BindingType, bindingContext []BindingContex globalHookExecutor := NewHookExecutor(h, bindingContext, h.Config.Version, h.moduleManager.KubeObjectPatcher) globalHookExecutor.WithLogLabels(logLabels) + globalHookExecutor.WithHelm(h.moduleManager.helm) hookResult, err := globalHookExecutor.Run() if hookResult != nil && hookResult.Usage != nil { metricLabels := map[string]string{ @@ -214,7 +215,7 @@ func (h *GlobalHook) Run(bindingType BindingType, bindingContext []BindingContex ) } - err := h.moduleManager.kubeConfigManager.SetKubeGlobalValues(configValuesPatchResult.Values) + err := h.moduleManager.kubeConfigManager.SaveGlobalConfigValues(configValuesPatchResult.Values) if err != nil { log.Debugf("Global hook '%s' kube config global values stay unchanged:\n%s", h.Name, h.moduleManager.kubeGlobalConfigValues.DebugString()) return fmt.Errorf("global hook '%s': set kube config failed: %s", h.Name, err) diff --git a/pkg/module_manager/hook.go b/pkg/module_manager/hook.go index 20912a3a..ee505926 100644 --- a/pkg/module_manager/hook.go +++ b/pkg/module_manager/hook.go @@ -31,10 +31,6 @@ type Hook interface { Order(binding sh_op_types.BindingType) float64 } -func (k *KubernetesBindingSynchronizationState) String() string { - return fmt.Sprintf("queue=%v done=%v", k.Queued, k.Done) -} - type CommonHook struct { hook.Hook @@ -73,8 +69,29 @@ func (h *CommonHook) SynchronizationNeeded() bool { return false } +// ShouldEnableSchedulesOnStartup returns true for Go hooks if EnableSchedulesOnStartup is set. +// This flag for schedule hooks that start after onStartup hooks. +func (h *CommonHook) ShouldEnableSchedulesOnStartup() bool { + if h.GoHook == nil { + return false + } + + s := h.GoHook.Config().Settings + + if s != nil && s.EnableSchedulesOnStartup { + return true + } + + return false +} + // SearchGlobalHooks recursively find all executables in hooksDir. Absent hooksDir is not an error. func SearchGlobalHooks(hooksDir string) (hooks []*GlobalHook, err error) { + if hooksDir == "" { + log.Warnf("Global hooks directory path is empty! No global hooks to load.") + return nil, nil + } + hooks = make([]*GlobalHook, 0) shellHooks, err := SearchGlobalShellHooks(hooksDir) if err != nil { @@ -241,9 +258,13 @@ func (mm *moduleManager) RegisterGlobalHooks() error { if err != nil { return err } - log.Debugf("Found %d global hooks:", len(hooks)) - for _, h := range hooks { - log.Debugf(" GlobalHook: Name=%s, Path=%s", h.Name, h.Path) + if len(hooks) > 0 { + log.Debugf("Found %d global hooks:", len(hooks)) + for _, h := range hooks { + log.Debugf(" GlobalHook: Name=%s, Path=%s", h.Name, h.Path) + } + } else { + log.Debugf("Found no global hooks in %s", mm.GlobalHooksDir) } for _, globalHook := range hooks { @@ -256,7 +277,9 @@ func (mm *moduleManager) RegisterGlobalHooks() error { if globalHook.GoHook != nil { goConfig = globalHook.GoHook.Config() } else { - yamlConfigBytes, err = NewHookExecutor(globalHook, nil, "", nil).Config() + hookExecutor := NewHookExecutor(globalHook, nil, "", nil) + hookExecutor.WithHelm(mm.helm) + yamlConfigBytes, err = hookExecutor.Config() if err != nil { logEntry.Errorf("Run --config: %s", err) return fmt.Errorf("global hook --config run problem") @@ -364,7 +387,9 @@ func (mm *moduleManager) RegisterModuleHooks(module *Module, logLabels map[strin if moduleHook.GoHook != nil { goConfig = moduleHook.GoHook.Config() } else { - yamlConfigBytes, err = NewHookExecutor(moduleHook, nil, "", nil).Config() + hookExecutor := NewHookExecutor(moduleHook, nil, "", nil) + hookExecutor.WithHelm(mm.helm) + yamlConfigBytes, err = hookExecutor.Config() if err != nil { hookLogEntry.Errorf("Run --config: %s", err) return fmt.Errorf("module hook --config run problem") diff --git a/pkg/module_manager/hook_executor.go b/pkg/module_manager/hook_executor.go index 62989ff4..1ae9bacf 100644 --- a/pkg/module_manager/hook_executor.go +++ b/pkg/module_manager/hook_executor.go @@ -33,6 +33,7 @@ type HookExecutor struct { ObjectPatcher *object_patch.ObjectPatcher KubernetesPatchPath string LogLabels map[string]string + Helm *helm.Helm } func NewHookExecutor(h Hook, context []BindingContext, configVersion string, objectPatcher *object_patch.ObjectPatcher) *HookExecutor { @@ -49,6 +50,10 @@ func (e *HookExecutor) WithLogLabels(logLabels map[string]string) { e.LogLabels = logLabels } +func (e *HookExecutor) WithHelm(helm *helm.Helm) { + e.Helm = helm +} + type HookResult struct { Usage *executor.CmdUsage Patches map[utils.ValuesPatchType]*utils.ValuesPatch @@ -99,7 +104,7 @@ func (e *HookExecutor) Run() (result *HookResult, err error) { for envName, filePath := range tmpFiles { envs = append(envs, fmt.Sprintf("%s=%s", envName, filePath)) } - envs = append(envs, helm.NewClient().CommandEnv()...) + envs = append(envs, e.helmEnv()...) cmd := executor.MakeCommand("", e.Hook.GetPath(), []string{}, envs) @@ -211,7 +216,7 @@ func (e *HookExecutor) Config() (configOutput []byte, err error) { envs := make([]string, 0) envs = append(envs, os.Environ()...) - envs = append(envs, helm.NewClient().CommandEnv()...) + envs = append(envs, e.helmEnv()...) cmd := executor.MakeCommand("", e.Hook.GetPath(), []string{"--config"}, envs) @@ -228,3 +233,10 @@ func (e *HookExecutor) Config() (configOutput []byte, err error) { return output, nil } + +func (e *HookExecutor) helmEnv() []string { + if e.Helm == nil { + return []string{} + } + return e.Helm.NewClient().CommandEnv() +} diff --git a/pkg/module_manager/hook_executor_test.go b/pkg/module_manager/hook_executor_test.go index ba8b8be1..ceda1700 100644 --- a/pkg/module_manager/hook_executor_test.go +++ b/pkg/module_manager/hook_executor_test.go @@ -15,7 +15,7 @@ import ( func Test_Config_GoHook(t *testing.T) { g := NewWithT(t) - moduleManager := NewMainModuleManager() + moduleManager := NewModuleManager() expectedGoHookName := "simple.go" expectedGoHookPath := "/global-hooks/simple.go" diff --git a/pkg/module_manager/module.go b/pkg/module_manager/module.go index 939fd973..46709a07 100644 --- a/pkg/module_manager/module.go +++ b/pkg/module_manager/module.go @@ -45,6 +45,7 @@ type Module struct { moduleManager *moduleManager metricStorage *metric_storage.MetricStorage + helm *helm.Helm } func NewModule(name, path string) *Module { @@ -63,6 +64,10 @@ func (m *Module) WithMetricStorage(mstor *metric_storage.MetricStorage) { m.metricStorage = mstor } +func (m *Module) WithHelm(helm *helm.Helm) { + m.helm = helm +} + func (m *Module) SafeName() string { return sanitize.BaseName(m.Name) } @@ -172,7 +177,7 @@ func (m *Module) Delete(logLabels map[string]string) error { // Module has chart and release -> execute helm delete. chartExists, _ := m.checkHelmChart() if chartExists { - releaseExists, err := helm.NewClient(deleteLogLabels).IsReleaseExists(m.generateHelmReleaseName()) + releaseExists, err := m.helm.NewClient(deleteLogLabels).IsReleaseExists(m.generateHelmReleaseName()) if !releaseExists { if err != nil { logEntry.Warnf("Cannot find helm release '%s' for module '%s'. Helm error: %s", m.generateHelmReleaseName(), m.Name, err) @@ -181,7 +186,7 @@ func (m *Module) Delete(logLabels map[string]string) error { } } else { // Chart and release are existed, so run helm delete command - err := helm.NewClient(deleteLogLabels).DeleteRelease(m.generateHelmReleaseName()) + err := m.helm.NewClient(deleteLogLabels).DeleteRelease(m.generateHelmReleaseName()) if err != nil { return err } @@ -211,11 +216,11 @@ func (m *Module) cleanup() error { "module": m.Name, } - if err := helm.NewClient(helmLogLabels).DeleteSingleFailedRevision(m.generateHelmReleaseName()); err != nil { + if err := m.helm.NewClient(helmLogLabels).DeleteSingleFailedRevision(m.generateHelmReleaseName()); err != nil { return err } - if err := helm.NewClient(helmLogLabels).DeleteOldFailedRevisions(m.generateHelmReleaseName()); err != nil { + if err := m.helm.NewClient(helmLogLabels).DeleteOldFailedRevisions(m.generateHelmReleaseName()); err != nil { return err } @@ -254,7 +259,7 @@ func (m *Module) runHelmInstall(logLabels map[string]string) error { } defer os.Remove(valuesPath) - helmClient := helm.NewClient(logLabels) + helmClient := m.helm.NewClient(logLabels) // Render templates to prevent excess helm runs. var renderedManifests string @@ -746,7 +751,7 @@ func (m *Module) Values() (utils.Values, error) { res = MergeLayers( res, utils.Values{"global": map[string]interface{}{ - "enabledModules": m.moduleManager.enabledModulesInOrder, + "enabledModules": m.moduleManager.enabledModules, }}, ) @@ -804,7 +809,11 @@ func (m *Module) readModuleEnabledResult(filePath string) (bool, error) { return false, fmt.Errorf("expected 'true' or 'false', got '%s'", value) } -func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string, logLabels map[string]string) (bool, error) { +func (m *Module) runEnabledScript(precedingEnabledModules []string, logLabels map[string]string) (bool, error) { + // Copy labels and set 'module' label. + logLabels = utils.MergeLabels(logLabels) + logLabels["module"] = m.Name + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) enabledScriptPath := filepath.Join(m.Path, "enabled") @@ -916,6 +925,11 @@ func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string, logLab var ValidModuleNameRe = regexp.MustCompile(`^[0-9][0-9][0-9]-(.*)$`) func SearchModules(modulesDir string) (modules []*Module, err error) { + if modulesDir == "" { + log.Warnf("Modules directory path is empty! No modules to load.") + return nil, nil + } + files, err := ioutil.ReadDir(modulesDir) // returns a list of modules sorted by filename if err != nil { return nil, fmt.Errorf("list modules directory '%s': %s", modulesDir, err) @@ -967,6 +981,7 @@ func (mm *moduleManager) RegisterModules() error { module.WithModuleManager(mm) module.WithMetricStorage(mm.metricStorage) + module.WithHelm(mm.helm) // load static config from values.yaml err := module.loadStaticValues() diff --git a/pkg/module_manager/module_hook.go b/pkg/module_manager/module_hook.go index 0cc20e25..f8017684 100644 --- a/pkg/module_manager/module_hook.go +++ b/pkg/module_manager/module_hook.go @@ -155,6 +155,7 @@ func (h *ModuleHook) Run(bindingType BindingType, context []BindingContext, logL moduleHookExecutor := NewHookExecutor(h, context, h.Config.Version, h.moduleManager.KubeObjectPatcher) moduleHookExecutor.WithLogLabels(logLabels) + moduleHookExecutor.WithHelm(h.moduleManager.helm) hookResult, err := moduleHookExecutor.Run() if hookResult != nil && hookResult.Usage != nil { // usage metrics @@ -214,7 +215,7 @@ func (h *ModuleHook) Run(bindingType BindingType, context []BindingContext, logL ) } - err := h.moduleManager.kubeConfigManager.SetKubeModuleValues(moduleName, configValuesPatchResult.Values) + err := h.moduleManager.kubeConfigManager.SaveModuleConfigValues(moduleName, configValuesPatchResult.Values) if err != nil { log.Debugf("Module hook '%s' kube module config values stay unchanged:\n%s", h.Name, h.moduleManager.kubeModulesConfigValues[moduleName].DebugString()) return fmt.Errorf("module hook '%s': set kube module config failed: %s", h.Name, err) diff --git a/pkg/module_manager/module_manager.go b/pkg/module_manager/module_manager.go index 40629d06..f27deeb9 100644 --- a/pkg/module_manager/module_manager.go +++ b/pkg/module_manager/module_manager.go @@ -4,13 +4,11 @@ import ( "context" "encoding/json" "fmt" - "reflect" "sort" "strings" "sync" "time" - klient "github.com/flant/kube-client/client" "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" @@ -42,7 +40,6 @@ import ( type ModuleManager interface { Init() error Start() - Ch() chan Event // Dependencies WithContext(ctx context.Context) @@ -50,7 +47,8 @@ type ModuleManager interface { WithKubeEventManager(kube_events_manager.KubeEventsManager) WithKubeObjectPatcher(*object_patch.ObjectPatcher) WithScheduleManager(schedule_manager.ScheduleManager) - WithKubeConfigManager(kubeConfigManager kube_config_manager.KubeConfigManager) ModuleManager + WithKubeConfigManager(kubeConfigManager kube_config_manager.KubeConfigManager) + WithHelm(*helm.Helm) WithHelmResourcesManager(manager helm_resources_manager.HelmResourcesManager) WithMetricStorage(storage *metric_storage.MetricStorage) WithHookMetricStorage(storage *metric_storage.MetricStorage) @@ -59,7 +57,8 @@ type ModuleManager interface { GetGlobalHooksNames() []string GetGlobalHook(name string) *GlobalHook - GetModuleNamesInOrder() []string + GetEnabledModuleNames() []string + IsModuleEnabled(moduleName string) bool GetModule(name string) *Module GetModuleHookNames(moduleName string) []string GetModuleHook(name string) *ModuleHook @@ -75,12 +74,19 @@ type ModuleManager interface { UpdateModuleConfigValues(moduleName string, configValues utils.Values) UpdateModuleDynamicValuesPatches(moduleName string, valuesPatch utils.ValuesPatch) - // Actions for tasks - DiscoverModulesState(logLabels map[string]string) (*ModulesState, error) + GetKubeConfigValid() bool + SetKubeConfigValid(valid bool) + + // Methods to change module manager's state. + RefreshStateFromHelmReleases(logLabels map[string]string) (*ModulesState, error) + HandleNewKubeConfig(kubeConfig *kube_config_manager.KubeConfig) (*ModulesState, error) + RefreshEnabledState(logLabels map[string]string) (*ModulesState, error) + + // Actions for tasks. DeleteModule(moduleName string, logLabels map[string]string) error RunModule(moduleName string, onStartup bool, logLabels map[string]string, afterStartupCb func() error) (bool, error) RunGlobalHook(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) (beforeChecksum string, afterChecksum string, err error) - RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) error + RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) (beforeChecksum string, afterChecksum string, err error) RegisterModuleHooks(module *Module, logLabels map[string]string) error @@ -98,34 +104,37 @@ type ModuleManager interface { GlobalSynchronizationState() *SynchronizationState } -// ModulesState is a result of Discovery process, that determines which -// modules should be enabled, disabled or purged. +// ModulesState determines which modules should be enabled, disabled or reloaded. type ModulesState struct { - // modules that should be run - EnabledModules []string - // modules that should be deleted + // All enabled modules. + AllEnabledModules []string + // Modules that should be deleted. ModulesToDisable []string - // modules that should be purged - ReleasedUnknownModules []string - // modules that was disabled and now are enabled - NewlyEnabledModules []string + // Modules that was disabled and now are enabled. + ModulesToEnable []string + // Modules changed after ConfigMap changes + ModulesToReload []string + // Helm releases without module directory (unknown modules). + ModulesToPurge []string } type moduleManager struct { ctx context.Context cancel context.CancelFunc - // Directories + logEntry *log.Entry + + // Directories. ModulesDir string GlobalHooksDir string TempDir string - EventCh chan Event - - KubeClient klient.Client + // Dependencies. KubeObjectPatcher *object_patch.ObjectPatcher kubeEventsManager kube_events_manager.KubeEventsManager scheduleManager schedule_manager.ScheduleManager + kubeConfigManager kube_config_manager.KubeConfigManager + helm *helm.Helm HelmResourcesManager helm_resources_manager.HelmResourcesManager metricStorage *metric_storage.MetricStorage hookMetricStorage *metric_storage.MetricStorage @@ -149,13 +158,10 @@ type moduleManager struct { // List of modules enabled by values.yaml or by kube config. // This list is changed on ConfigMap updates. - enabledModulesByConfig []string + enabledModulesByConfig map[string]struct{} - // TODO calculate from enabledValues - // Effective list of enabled modules after enabled script running. - // List is sorted by module name. - // This list is changed on ConfigMap changes. - enabledModulesInOrder []string + // List of effectively enabled modules after running enabled scripts. + enabledModules []string // Index of all global hooks. Key is global hook name globalHooksByName map[string]*GlobalHook @@ -180,8 +186,11 @@ type moduleManager struct { kubeGlobalConfigValues utils.Values // module values from ConfigMap, only for enabled modules kubeModulesConfigValues map[string]utils.Values - // marks addon-operator config is valid or not - kubeConfigIsValid bool + + // addon-operator config is valid. + kubeConfigValid bool + // Static and config values are valid using OpenAPI schemas. + kubeConfigValuesValid bool // Invariant: do not store patches that cannot be applied. // Give user error for patches early, after patch receive. @@ -190,64 +199,23 @@ type moduleManager struct { globalDynamicValuesPatches []utils.ValuesPatch // Pathces for dynamic module values modulesDynamicValuesPatches map[string][]utils.ValuesPatch - - // Internal event: module values are changed. - // This event leads to module run action. - moduleValuesChanged chan string - // Internal event: global values are changed. - // This event leads to module discovery action. - globalValuesChanged chan bool - - kubeConfigManager kube_config_manager.KubeConfigManager } var _ ModuleManager = &moduleManager{} -// EventType are events for the main loop. -type EventType string - -const ( - // There are modules with changed values. - ModulesChanged EventType = "MODULES_CHANGED" - // Global section is changed. - GlobalChanged EventType = "GLOBAL_CHANGED" - // Something wrong with module manager. - AmbiguousState EventType = "AMBIGUOUS_STATE" -) - -// ChangeType are types of module changes. -type ChangeType string - -const ( - // All other types are deprecated. This const can be removed in future versions. - // Module values are changed - Changed ChangeType = "MODULE_CHANGED" -) - -// ModuleChange contains module name and type of module changes. -type ModuleChange struct { - Name string - ChangeType ChangeType -} - -// Event is used to send module events to the main loop. -type Event struct { - ModulesChanges []ModuleChange - Type EventType -} - -// NewMainModuleManager returns new MainModuleManager -func NewMainModuleManager() *moduleManager { +// NewModuleManager returns new MainModuleManager +func NewModuleManager() *moduleManager { return &moduleManager{ - EventCh: make(chan Event), + logEntry: log.WithField("operator.component", "ModuleManager"), + valuesLayersLock: sync.Mutex{}, ValuesValidator: validation.NewValuesValidator(), allModulesByName: make(map[string]*Module), allModulesNamesInOrder: make([]string, 0), - enabledModulesByConfig: make([]string, 0), - enabledModulesInOrder: make([]string, 0), + enabledModulesByConfig: make(map[string]struct{}), + enabledModules: make([]string, 0), dynamicEnabled: make(map[string]*bool), globalHooksByName: make(map[string]*GlobalHook), globalHooksOrder: make(map[BindingType][]*GlobalHook), @@ -258,11 +226,6 @@ func NewMainModuleManager() *moduleManager { globalDynamicValuesPatches: make([]utils.ValuesPatch, 0), modulesDynamicValuesPatches: make(map[string][]utils.ValuesPatch), - moduleValuesChanged: make(chan string, 1), - globalValuesChanged: make(chan bool, 1), - - kubeConfigManager: nil, - globalSynchronizationState: NewSynchronizationState(), } } @@ -274,9 +237,12 @@ func (mm *moduleManager) WithDirectories(modulesDir string, globalHooksDir strin return mm } -func (mm *moduleManager) WithKubeConfigManager(kubeConfigManager kube_config_manager.KubeConfigManager) ModuleManager { +func (mm *moduleManager) WithKubeConfigManager(kubeConfigManager kube_config_manager.KubeConfigManager) { mm.kubeConfigManager = kubeConfigManager - return mm +} + +func (mm *moduleManager) WithHelm(helm *helm.Helm) { + mm.helm = helm } func (mm *moduleManager) WithKubeEventManager(mgr kube_events_manager.KubeEventsManager) { @@ -313,226 +279,235 @@ func (mm *moduleManager) Stop() { } } -// RunModulesEnabledScript runs enable script for each module that is enabled by config. -// Enable script receives a list of previously enabled modules. -func (mm *moduleManager) RunModulesEnabledScript(enabledByConfig []string, logLabels map[string]string) ([]string, error) { - enabledModules := make([]string, 0) +// runModulesEnabledScript runs enable script for each module from the list. +// Each 'enabled' script receives a list of previously enabled modules. +func (mm *moduleManager) runModulesEnabledScript(modules []string, logLabels map[string]string) ([]string, error) { + enabled := make([]string, 0) - for _, name := range utils.SortByReference(enabledByConfig, mm.allModulesNamesInOrder) { - moduleLogLabels := utils.MergeLabels(logLabels) - moduleLogLabels["module"] = name - module := mm.allModulesByName[name] - moduleIsEnabled, err := module.checkIsEnabledByScript(enabledModules, moduleLogLabels) + for _, moduleName := range modules { + module := mm.GetModule(moduleName) + isEnabled, err := module.runEnabledScript(enabled, logLabels) if err != nil { return nil, err } - if moduleIsEnabled { - enabledModules = append(enabledModules, name) + if isEnabled { + enabled = append(enabled, moduleName) } } - return enabledModules, nil + return enabled, nil } -// kubeUpdate -type kubeUpdate struct { - EnabledModulesByConfig []string - KubeGlobalConfigValues utils.Values - KubeModulesConfigValues map[string]utils.Values - Events []Event -} +// HandleNewKubeConfig validates new config values with config schemas, +// checks which parts changed and returns state with AllEnabledModules and +// ModulesToReload list if only module sections are changed. +// It returns a nil state if new KubeConfig not changing +// config values or 'enabled by config' state. +// +// This method updates 'config values' caches: +// - mm.enabledModulesByConfig +// - mm.kubeGlobalConfigValues +// - mm.kubeModulesConfigValues +func (mm *moduleManager) HandleNewKubeConfig(kubeConfig *kube_config_manager.KubeConfig) (*ModulesState, error) { + var err error -func (mm *moduleManager) applyKubeUpdate(kubeUpdate *kubeUpdate) error { - log.Debugf("Apply kubeupdate %+v", kubeUpdate) - mm.kubeGlobalConfigValues = kubeUpdate.KubeGlobalConfigValues - mm.kubeModulesConfigValues = kubeUpdate.KubeModulesConfigValues - mm.enabledModulesByConfig = kubeUpdate.EnabledModulesByConfig + mm.warnAboutUnknownModules(kubeConfig) - for _, event := range kubeUpdate.Events { - mm.EventCh <- event - } + // Get map of enabled modules after ConfigMap changes. + newEnabledByConfig := mm.calculateEnabledModulesByConfig(kubeConfig) - return nil -} + // Check if values in new KubeConfig are valid. Return error to prevent poisoning caches with invalid values. + err = mm.validateKubeConfig(kubeConfig, newEnabledByConfig) + if err != nil { + return nil, fmt.Errorf("config not valid: %v", err) + } -func (mm *moduleManager) handleNewKubeConfig(newConfig kube_config_manager.Config) (*kubeUpdate, error) { - logEntry := log.WithField("operator.component", "ModuleManager"). - WithField("operator.action", "handleNewKubeConfig") - logEntry.Debugf("new kube config received") + // Detect changes in global section. + hasGlobalChange := false + newGlobalValues := mm.kubeGlobalConfigValues + if (kubeConfig == nil || kubeConfig.Global == nil) && mm.kubeGlobalConfigValues.HasGlobal() { + hasGlobalChange = true + newGlobalValues = make(utils.Values) + } + if kubeConfig != nil && kubeConfig.Global != nil { + globalChecksum, err := mm.kubeGlobalConfigValues.Checksum() + if err != nil { + return nil, err + } - res := &kubeUpdate{ - KubeGlobalConfigValues: newConfig.Values, - Events: []Event{{Type: GlobalChanged}}, + if kubeConfig.Global.Checksum != globalChecksum { + hasGlobalChange = true + } + newGlobalValues = kubeConfig.Global.Values } - var unknown []utils.ModuleConfig - res.EnabledModulesByConfig, res.KubeModulesConfigValues, unknown = mm.calculateEnabledModulesByConfig(newConfig.ModuleConfigs) + // Full reload if enabled flags are changed. + isEnabledChanged := false + for moduleName := range mm.allModulesByName { + // Current module state. + _, wasEnabled := mm.enabledModulesByConfig[moduleName] + _, isEnabled := newEnabledByConfig[moduleName] - for _, moduleConfig := range unknown { - logEntry.Warnf("Ignore ConfigMap section '%s' for absent module : \n%s", - moduleConfig.ModuleName, - moduleConfig.String(), - ) + if wasEnabled != isEnabled { + isEnabledChanged = true + break + } } - return res, nil -} + // Detect changed module sections for enabled modules. + modulesChanged := make([]string, 0) + if !isEnabledChanged { + // enabledModules is a subset of enabledModulesByConfig. + // Module can be enabled by config, but disabled with enabled script. + // So check only sections for effectively enabled modules. + for _, moduleName := range mm.enabledModules { + modValues, hasConfigValues := mm.kubeModulesConfigValues[moduleName] + // New module state from ConfigMap. + hasNewKubeConfig := false + var newModConfig *kube_config_manager.ModuleKubeConfig + if kubeConfig != nil { + newModConfig, hasNewKubeConfig = kubeConfig.Modules[moduleName] + } -func (mm *moduleManager) handleNewKubeModuleConfigs(moduleConfigs kube_config_manager.ModuleConfigs) (*kubeUpdate, error) { - logLabels := map[string]string{ - "operator.component": "HandleConfigMap", - } - logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) + // Section added or disappeared from ConfigMap, values changed. + if hasConfigValues != hasNewKubeConfig { + modulesChanged = append(modulesChanged, moduleName) + continue + } - logEntry.Debugf("handle changes in module sections") + // Compare checksums for new and saved values. + if hasConfigValues && hasNewKubeConfig { + modValuesChecksum, err := modValues.Checksum() + if err != nil { + return nil, err + } + newModValuesChecksum, err := newModConfig.Values.Checksum() + if err != nil { + return nil, err + } + if modValuesChecksum != newModValuesChecksum { + modulesChanged = append(modulesChanged, moduleName) + } + } + } + } - res := &kubeUpdate{ - Events: make([]Event, 0), - KubeGlobalConfigValues: mm.kubeGlobalConfigValues, + // Create new map with config values. + newKubeModuleConfigValues := make(map[string]utils.Values) + if kubeConfig != nil { + for moduleName, moduleConfig := range kubeConfig.Modules { + newKubeModuleConfigValues[moduleName] = moduleConfig.Values + } } - // NOTE: values for non changed modules were copied from mm.kubeModulesConfigValues[moduleName]. - // Now calculateEnabledModulesByConfig got values for modules from moduleConfigs — as they are in ConfigMap now. - // TODO this should not be a problem because of a checksum matching in kube_config_manager - var unknown []utils.ModuleConfig - res.EnabledModulesByConfig, res.KubeModulesConfigValues, unknown = mm.calculateEnabledModulesByConfig(moduleConfigs) + // Update caches from ConfigMap content. + mm.valuesLayersLock.Lock() + mm.enabledModulesByConfig = newEnabledByConfig + mm.kubeGlobalConfigValues = newGlobalValues + mm.kubeModulesConfigValues = newKubeModuleConfigValues + mm.valuesLayersLock.Unlock() - for _, moduleConfig := range unknown { - logEntry.Warnf("ignore module section for unknown module '%s':\n%s", - moduleConfig.ModuleName, moduleConfig.String()) + // Return empty state on global change. + if hasGlobalChange || isEnabledChanged { + return &ModulesState{}, nil } - // Detect removed module sections for statically enabled modules. - // This removal should be handled like kube config update. - updateOnSectionRemove := make(map[string]bool) - for moduleName, module := range mm.allModulesByName { - _, hasKubeConfig := moduleConfigs[moduleName] - if !hasKubeConfig { - isEnabled := mergeEnabled( - module.CommonStaticConfig.IsEnabled, - module.StaticConfig.IsEnabled, - mm.dynamicEnabled[moduleName]) - _, hasValues := mm.kubeModulesConfigValues[moduleName] - if isEnabled && hasValues { - updateOnSectionRemove[moduleName] = true - } - } + // Return list of changed modules when only values are changed. + if len(modulesChanged) > 0 { + return &ModulesState{ + AllEnabledModules: mm.enabledModules, + ModulesToReload: modulesChanged, + }, nil } - // New version of mm.enabledModulesByConfig - res.EnabledModulesByConfig = utils.SortByReference(res.EnabledModulesByConfig, mm.allModulesNamesInOrder) + // Return nil if cached state is not changed by ConfigMap. + return nil, nil +} - // Run enable scripts - logEntry.Debugf("Run enabled script for %+v", res.EnabledModulesByConfig) - enabledModules, err := mm.RunModulesEnabledScript(res.EnabledModulesByConfig, logLabels) - if err != nil { - return nil, err +// warnAboutUnknownModules prints to log all unknown module section names. +func (mm *moduleManager) warnAboutUnknownModules(kubeConfig *kube_config_manager.KubeConfig) { + // Ignore empty kube config. + if kubeConfig == nil { + return } - logEntry.Infof("Modules enabled by script: %+v", enabledModules) - // Configure events - if !reflect.DeepEqual(mm.enabledModulesInOrder, enabledModules) { - // Enabled modules set is changed — return GlobalChanged event, that will - // create a Discover task, run enabled scripts again, init new module hooks, - // update mm.enabledModulesInOrder - logEntry.Debugf("enabled modules set changed from %v to %v: generate GlobalChanged event", mm.enabledModulesInOrder, res.EnabledModulesByConfig) - res.Events = append(res.Events, Event{Type: GlobalChanged}) - } else { - // Enabled modules set is not changed, only values in configmap are changed. - logEntry.Debugf("generate ModulesChanged events...") - - moduleChanges := make([]ModuleChange, 0) - - // make Changed event for each enabled module with updated config - for _, name := range enabledModules { - // Module has updated kube config - isUpdated := false - moduleConfig, hasKubeConfig := moduleConfigs[name] - - if hasKubeConfig { - isUpdated = moduleConfig.IsUpdated - // skip not updated module configs - if !isUpdated { - logEntry.Debugf("ignore module '%s': kube config is not updated", name) - continue - } - } + unknownNames := make([]string, 0) + for moduleName := range kubeConfig.Modules { + if _, isKnown := mm.allModulesByName[moduleName]; !isKnown { + unknownNames = append(unknownNames, moduleName) + } + } + if len(unknownNames) > 0 { + mm.logEntry.Warnf("ConfigMap/%s has values for unknown modules: %+v", app.ConfigMapName, unknownNames) + } +} - // Update module if kube config is removed - _, shouldUpdateAfterRemoval := updateOnSectionRemove[name] +// calculateEnabledModulesByConfig determine enable state for all modules +// by checking *Enabled fields in values.yaml, ConfigMap and dynamicEnable map. +// Method returns list of enabled modules. +// +// Module is enabled by config if module section in ConfigMap is a map or an array +// or ConfigMap has no module section and module has a map or an array in values.yaml +func (mm *moduleManager) calculateEnabledModulesByConfig(config *kube_config_manager.KubeConfig) map[string]struct{} { + enabledByConfig := make(map[string]struct{}) - if (hasKubeConfig && isUpdated) || shouldUpdateAfterRemoval { - moduleChanges = append(moduleChanges, ModuleChange{Name: name, ChangeType: Changed}) + for moduleName, module := range mm.allModulesByName { + var kubeConfigEnabled *bool + var kubeConfigEnabledStr string + if config != nil { + if kubeConfig, hasKubeConfig := config.Modules[moduleName]; hasKubeConfig { + kubeConfigEnabled = kubeConfig.IsEnabled + kubeConfigEnabledStr = kubeConfig.GetEnabled() } } - if len(moduleChanges) > 0 { - logEntry.Infof("fire ModulesChanged event for %d modules", len(moduleChanges)) - logEntry.Debugf("event changes: %v", moduleChanges) - res.Events = append(res.Events, Event{Type: ModulesChanged, ModulesChanges: moduleChanges}) + isEnabled := mergeEnabled( + module.CommonStaticConfig.IsEnabled, + module.StaticConfig.IsEnabled, + kubeConfigEnabled, + ) + + if isEnabled { + enabledByConfig[moduleName] = struct{}{} } + + log.Debugf("enabledByConfig: module '%s' enabled flags: common '%v', static '%v', kubeConfig '%v', result: '%v'", + module.Name, + module.CommonStaticConfig.GetEnabled(), + module.StaticConfig.GetEnabled(), + kubeConfigEnabledStr, + isEnabled) } - return res, nil + return enabledByConfig + //enabled = utils.SortByReference(enabled, mm.allModulesNamesInOrder) } -// calculateEnabledModulesByConfig determine enable state for all modules -// by values.yaml, ConfigMap and dynamicEnable map. -// Method returns list of enabled modules and their values. Also the map of disabled modules and a list of unknown -// keys in a ConfigMap. +// calculateEnabledModulesWithDynamic determine enable state for all modules +// by checking *Enabled fields in values.yaml, ConfigMap and dynamicEnable map. +// Method returns list of enabled modules. // // Module is enabled by config if module section in ConfigMap is a map or an array // or ConfigMap has no module section and module has a map or an array in values.yaml -func (mm *moduleManager) calculateEnabledModulesByConfig(moduleConfigs kube_config_manager.ModuleConfigs) (enabled []string, values map[string]utils.Values, unknown []utils.ModuleConfig) { - values = make(map[string]utils.Values) - +func (mm *moduleManager) calculateEnabledModulesWithDynamic(enabledByConfig map[string]struct{}) []string { log.Debugf("calculateEnabled: dynamicEnabled is %s", mm.DumpDynamicEnabled()) - for moduleName, module := range mm.allModulesByName { - kubeConfig, hasKubeConfig := moduleConfigs[moduleName] - if hasKubeConfig { - isEnabled := mergeEnabled( - module.CommonStaticConfig.IsEnabled, - module.StaticConfig.IsEnabled, - kubeConfig.IsEnabled, - mm.dynamicEnabled[moduleName]) - - if isEnabled { - enabled = append(enabled, moduleName) - values[moduleName] = kubeConfig.Values - } - log.Debugf("calculateEnabled: module '%s': static enabled %v, kubeConfig: enabled %v, updated %v, dynamic enabled: %v", - module.Name, - module.StaticConfig.GetEnabled(), - kubeConfig.IsEnabled, - kubeConfig.IsUpdated, - mm.dynamicEnabled[moduleName]) - } else { - isEnabled := mergeEnabled( - module.CommonStaticConfig.IsEnabled, - module.StaticConfig.IsEnabled, - mm.dynamicEnabled[moduleName]) + enabled := make([]string, 0) + for _, moduleName := range mm.allModulesNamesInOrder { + _, isEnabledByConfig := enabledByConfig[moduleName] - if isEnabled { - enabled = append(enabled, moduleName) - } - log.Debugf("calculateEnabled: module '%s': static enabled %v, no kubeConfig, dynamic enabled: %v", - module.Name, - module.StaticConfig.GetEnabled(), - mm.dynamicEnabled[moduleName]) - } - } + isEnabled := mergeEnabled( + &isEnabledByConfig, + mm.dynamicEnabled[moduleName], + ) - for _, kubeConfig := range moduleConfigs { - if _, hasKey := mm.allModulesByName[kubeConfig.ModuleName]; !hasKey { - unknown = append(unknown, kubeConfig) + if isEnabled { + enabled = append(enabled, moduleName) } } - enabled = utils.SortByReference(enabled, mm.allModulesNamesInOrder) - - return + return enabled } // Init — initialize module manager @@ -547,46 +522,36 @@ func (mm *moduleManager) Init() error { return err } - kubeConfig := mm.kubeConfigManager.InitialConfig() - mm.kubeGlobalConfigValues = kubeConfig.Values - - var unknown []utils.ModuleConfig - mm.enabledModulesByConfig, mm.kubeModulesConfigValues, unknown = mm.calculateEnabledModulesByConfig(kubeConfig.ModuleConfigs) - - unknownNames := make([]string, 0) - for _, config := range unknown { - unknownNames = append(unknownNames, config.ModuleName) - } - if len(unknownNames) > 0 { - log.Warnf("ConfigMap/%s has values for absent modules: %+v", app.ConfigMapName, unknownNames) - } - - // Initialize kubeConfigIsValid flag and start checking it in go routine. - err := mm.validateKubeConfig(mm.kubeConfigManager.CurrentConfig()) - - go mm.checkConfig() - - return err + return nil } -func (mm *moduleManager) validateKubeConfig(kubeConfig *kube_config_manager.Config) error { - // Validate global and module sections in ConfigMap merged with static values. +// validateKubeConfig checks validity of all sections in ConfigMap with OpenAPI schemas. +func (mm *moduleManager) validateKubeConfig(kubeConfig *kube_config_manager.KubeConfig, enabledModules map[string]struct{}) error { + // Ignore empty kube config. + if kubeConfig == nil { + mm.SetKubeConfigValuesValid(true) + return nil + } + // Validate values in global section merged with static values. var validationErr error - globalErr := mm.ValuesValidator.ValidateGlobalConfigValues(mm.GlobalStaticAndNewValues(kubeConfig.Values)) - if globalErr != nil { - validationErr = multierror.Append( - validationErr, - fmt.Errorf("'global' section in ConfigMap/%s is not valid", app.ConfigMapName), - globalErr, - ) + if kubeConfig.Global != nil { + err := mm.ValuesValidator.ValidateGlobalConfigValues(mm.GlobalStaticAndNewValues(kubeConfig.Global.Values)) + if err != nil { + validationErr = multierror.Append( + validationErr, + fmt.Errorf("'global' section in ConfigMap/%s is not valid", app.ConfigMapName), + err, + ) + } } - for _, moduleName := range mm.enabledModulesByConfig { - mod := mm.allModulesByName[moduleName] - modCfg, has := kubeConfig.ModuleConfigs[moduleName] + // Validate config values for enabled modules. + for moduleName := range enabledModules { + modCfg, has := kubeConfig.Modules[moduleName] if !has { continue } + mod := mm.GetModule(moduleName) moduleErr := mm.ValuesValidator.ValidateModuleConfigValues(mod.ValuesKey(), mod.StaticAndNewValues(modCfg.Values)) if moduleErr != nil { validationErr = multierror.Append( @@ -597,202 +562,156 @@ func (mm *moduleManager) validateKubeConfig(kubeConfig *kube_config_manager.Conf } } - if validationErr != nil { - mm.kubeConfigIsValid = false - } else { - mm.kubeConfigIsValid = true - } + // Set valid flag to false if there is validation error + mm.SetKubeConfigValuesValid(validationErr == nil) return validationErr } +func (mm *moduleManager) GetKubeConfigValid() bool { + return mm.kubeConfigValid +} + +func (mm *moduleManager) SetKubeConfigValid(valid bool) { + mm.kubeConfigValid = valid +} + +func (mm *moduleManager) SetKubeConfigValuesValid(valid bool) { + mm.kubeConfigValuesValid = valid +} + // checkConfig increases config_values_errors_total metric when kubeConfig becomes invalid. func (mm *moduleManager) checkConfig() { for { if mm.ctx.Err() != nil { return } - if !mm.kubeConfigIsValid { + if !mm.kubeConfigValid || !mm.kubeConfigValuesValid { mm.metricStorage.CounterAdd("{PREFIX}config_values_errors_total", 1.0, map[string]string{}) } time.Sleep(5 * time.Second) } } -// Module manager loop +// Start runs service go routine. func (mm *moduleManager) Start() { - go mm.kubeConfigManager.Start() - - go func() { - for { - select { - case <-mm.globalValuesChanged: - log.Debugf("MODULE_MANAGER_RUN global values changed") - mm.EventCh <- Event{Type: GlobalChanged} - - case moduleName := <-mm.moduleValuesChanged: - log.Debugf("MODULE_MANAGER_RUN module '%s' values changed", moduleName) - - // Перезапускать enabled-скрипт не нужно, т.к. - // изменение values модуля не может вызвать - // изменение состояния включенности модуля - mm.EventCh <- Event{ - Type: ModulesChanged, - ModulesChanges: []ModuleChange{ - {Name: moduleName, ChangeType: Changed}, - }, - } - - case newKubeConfig := <-kube_config_manager.ConfigUpdated: - // For simplicity, check the whole config. - err := mm.validateKubeConfig(mm.kubeConfigManager.CurrentConfig()) - if err != nil { - log.Errorf("MODULE_MANAGER_RUN ConfigMap changed and is not valid, no ReloadAllModules: %v", err) - break - } - - handleRes, err := mm.handleNewKubeConfig(newKubeConfig) - if err != nil { - log.Errorf("MODULE_MANAGER_RUN unable to handle kube config update: %s", err) - } - if handleRes != nil { - err = mm.applyKubeUpdate(handleRes) - if err != nil { - log.Errorf("MODULE_MANAGER_RUN cannot apply kube config update: %s", err) - } - } - - case newModuleConfigs := <-kube_config_manager.ModuleConfigsUpdated: - // For simplicity, check the whole config. - err := mm.validateKubeConfig(mm.kubeConfigManager.CurrentConfig()) - if err != nil { - log.Errorf("MODULE_MANAGER_RUN ConfigMap changed and is not valid, no module restart: %v", err) - break - } - - moduleUpdates, err := mm.handleNewKubeModuleConfigs(newModuleConfigs) - if err != nil { - log.Errorf("Unable to handle update of ConfigMap for modules [%s]: %s", strings.Join(newModuleConfigs.Names(), ", "), err) - } - if moduleUpdates != nil { - err = mm.applyKubeUpdate(moduleUpdates) - if err != nil { - log.Errorf("ConfigMap update cannot be applied to values for modules %s: %s", strings.Join(newModuleConfigs.Names(), ", "), err) - } - } - } - } - }() -} - -func (mm *moduleManager) Ch() chan Event { - return mm.EventCh + // Start checking kubeConfigIsValid flag in go routine. + go mm.checkConfig() } -// DiscoverModulesState handles DiscoverModulesState event: it calculates new arrays of enabled modules, -// modules that should be disabled and modules that should be purged. -// -// This method updates module state indices and values -// - mm.enabledModulesByConfig -// - mm.enabledModulesInOrder -// - mm.kubeModulesConfigValues are updated. -func (mm *moduleManager) DiscoverModulesState(logLabels map[string]string) (state *ModulesState, err error) { - discoverLogLabels := utils.MergeLabels(logLabels, map[string]string{ - "operator.component": "moduleManager.discoverModulesState", - }) - logEntry := log.WithFields(utils.LabelsToLogFields(discoverLogLabels)) - - logEntry.Debugf("DISCOVER state current:\n"+ - " mm.enabledModulesByConfig: %v\n"+ - " mm.enabledModulesInOrder: %v\n", - mm.enabledModulesByConfig, - mm.enabledModulesInOrder) +// RefreshStateFromHelmReleases retrieves all Helm releases. It treats releases for known modules as +// an initial list of enabled modules. +// Run this method once at startup. +func (mm *moduleManager) RefreshStateFromHelmReleases(logLabels map[string]string) (*ModulesState, error) { + if mm.helm == nil { + return &ModulesState{}, nil + } + releasedModules, err := mm.helm.NewClient(logLabels).ListReleasesNames(nil) + if err != nil { + return nil, err + } - currentEnabledModules := mm.enabledModulesInOrder + state := mm.stateFromHelmReleases(releasedModules) - updateEnabledModules, updateModuleValues, _ := mm.calculateEnabledModulesByConfig(mm.kubeConfigManager.CurrentConfig().ModuleConfigs) - updateEnabledModules = utils.SortByReference(updateEnabledModules, mm.allModulesNamesInOrder) + // Initiate enabled modules list. + mm.enabledModules = state.AllEnabledModules - mm.enabledModulesByConfig = updateEnabledModules - mm.kubeModulesConfigValues = updateModuleValues + return state, nil +} - logEntry.Debugf("DISCOVER state updated:\n"+ - " mm.enabledModulesByConfig: %v\n"+ - " mm.enabledModulesInOrder: %v\n", - mm.enabledModulesByConfig, - mm.enabledModulesInOrder) +// stateFromHelmReleases calculates enabled modules and modules to purge from Helm releases. +func (mm *moduleManager) stateFromHelmReleases(releases []string) *ModulesState { + releasesMap := utils.ListToMapStringStruct(releases) - state = &ModulesState{ - EnabledModules: []string{}, - ModulesToDisable: []string{}, - ReleasedUnknownModules: []string{}, - NewlyEnabledModules: []string{}, + // Filter out known modules. + enabledModules := make([]string, 0) + for _, modName := range mm.allModulesNamesInOrder { + // Remove known module to detect unknown ones. + if _, has := releasesMap[modName]; has { + // Treat known module as enabled module. + enabledModules = append(enabledModules, modName) + } + delete(releasesMap, modName) } - releasedModules, err := helm.NewClient(discoverLogLabels).ListReleasesNames(nil) - if err != nil { - return nil, err + // Filter out dynamically enabled modules (a way to save unknown releases). + for modName, dynEnable := range mm.dynamicEnabled { + if dynEnable != nil && *dynEnable { + delete(releasesMap, modName) + } } - // calculate unknown released modules to purge them in reverse order - state.ReleasedUnknownModules = utils.ListSubtract(releasedModules, mm.allModulesNamesInOrder) - // purge unknown modules in reverse order - state.ReleasedUnknownModules = utils.SortReverse(state.ReleasedUnknownModules) - if len(state.ReleasedUnknownModules) > 0 { - logEntry.Infof("found modules with releases: %s", state.ReleasedUnknownModules) + purge := utils.MapStringStructKeys(releasesMap) + purge = utils.SortReverse(purge) + return &ModulesState{ + AllEnabledModules: enabledModules, + ModulesToPurge: purge, } +} - // ignore unknown released modules for next operations - releasedModules = utils.ListIntersection(releasedModules, mm.allModulesNamesInOrder) +// RefreshEnabledState runs enabled hooks for all 'enabled by config' modules and +// calculates new arrays of enabled modules. It returns ModulesState with +// lists of modules to disable and enable. +// +// This method is called after beforeAll hooks to take into account +// possible changes to 'dynamic enabled'. +// +// This method updates caches: +// - mm.enabledModules +func (mm *moduleManager) RefreshEnabledState(logLabels map[string]string) (*ModulesState, error) { + refreshLogLabels := utils.MergeLabels(logLabels, map[string]string{ + "operator.component": "ModuleManager.RefreshEnabledState", + }) + logEntry := log.WithFields(utils.LabelsToLogFields(refreshLogLabels)) - // modules finally enabled with enable script - // no need to refresh mm.enabledModulesByConfig because - // it is updated before in Init or in applyKubeUpdate - logEntry.Debugf("Run enabled script for %+v", mm.enabledModulesByConfig) - enabledModules, err := mm.RunModulesEnabledScript(mm.enabledModulesByConfig, logLabels) - logEntry.Infof("Modules enabled by script: %+v", enabledModules) + logEntry.Debugf("Refresh state current:\n"+ + " mm.enabledModulesByConfig: %v\n"+ + " mm.enabledModules: %v\n", + mm.enabledModulesByConfig, + mm.enabledModules) + // Correct enabled modules list with dynamic enabled. + enabledByDynamic := mm.calculateEnabledModulesWithDynamic(mm.enabledModulesByConfig) + // Calculate final enabled modules list by running 'enabled' scripts. + enabledModules, err := mm.runModulesEnabledScript(enabledByDynamic, logLabels) if err != nil { return nil, err } + logEntry.Infof("Modules enabled by script: %+v", enabledModules) - for _, moduleName := range enabledModules { - if err = mm.RegisterModuleHooks(mm.allModulesByName[moduleName], logLabels); err != nil { - return nil, err - } - } - - state.EnabledModules = enabledModules - - state.NewlyEnabledModules = utils.ListSubtract(enabledModules, mm.enabledModulesInOrder) - // save enabled modules for future usages - mm.enabledModulesInOrder = enabledModules + // Difference between the list of currently enabled modules and the list + // of enabled modules after running enabled scripts. + // Newly enabled modules are that present in the list after running enabled scripts + // but not present in the list of currently enabled modules. + newlyEnabledModules := utils.ListSubtract(enabledModules, mm.enabledModules) - // Calculate disabled known modules that has helm release and/or was enabled. - // Sort them in reverse order for proper deletion. - state.ModulesToDisable = utils.ListSubtract(mm.allModulesNamesInOrder, enabledModules) - enabledAndReleased := utils.ListUnion(currentEnabledModules, releasedModules) - state.ModulesToDisable = utils.ListIntersection(state.ModulesToDisable, enabledAndReleased) - // disable modules in reverse order - state.ModulesToDisable = utils.SortReverseByReference(state.ModulesToDisable, mm.allModulesNamesInOrder) + // Disabled modules are that present in the list of currently enabled modules + // but not present in the list after running enabled scripts + disabledModules := utils.ListSubtract(mm.enabledModules, enabledModules) + disabledModules = utils.SortReverseByReference(disabledModules, mm.allModulesNamesInOrder) - logEntry.Debugf("DISCOVER state results:\n"+ + logEntry.Debugf("Refresh state results:\n"+ " mm.enabledModulesByConfig: %v\n"+ - " mm.enabledModulesInOrder: %v\n"+ - " releasedModules: %v\n"+ - " ReleasedUnknownModules: %v\n"+ + " mm.enabledModules: %v\n"+ " ModulesToDisable: %v\n"+ - " NewlyEnabled: %v\n", + " ModulesToEnable: %v\n", mm.enabledModulesByConfig, - mm.enabledModulesInOrder, - releasedModules, - state.ReleasedUnknownModules, - state.ModulesToDisable, - state.NewlyEnabledModules) - return + mm.enabledModules, + disabledModules, + newlyEnabledModules) + + // Update state + mm.enabledModules = enabledModules + //mm.enabledModulesIdx := utils.MapStringStructKeys(enabledModules) + + // Return lists for ConvergeModules task. + return &ModulesState{ + AllEnabledModules: mm.enabledModules, + ModulesToDisable: disabledModules, + ModulesToEnable: newlyEnabledModules, + }, nil } -// TODO replace with Module and ModuleShouldExists func (mm *moduleManager) GetModule(name string) *Module { module, exist := mm.allModulesByName[name] if exist { @@ -803,8 +722,17 @@ func (mm *moduleManager) GetModule(name string) *Module { } } -func (mm *moduleManager) GetModuleNamesInOrder() []string { - return mm.enabledModulesInOrder +func (mm *moduleManager) GetEnabledModuleNames() []string { + return mm.enabledModules +} + +func (mm *moduleManager) IsModuleEnabled(moduleName string) bool { + for _, modName := range mm.enabledModules { + if modName == moduleName { + return true + } + } + return false } func (mm *moduleManager) GetGlobalHook(name string) *GlobalHook { @@ -891,19 +819,14 @@ func (mm *moduleManager) GetModuleHookNames(moduleName string) []string { return []string{} } - moduleHookNamesMap := map[string]bool{} + moduleHookNamesMap := make(map[string]struct{}) for _, moduleHooks := range moduleHooksByBinding { for _, moduleHook := range moduleHooks { - moduleHookNamesMap[moduleHook.Name] = true + moduleHookNamesMap[moduleHook.Name] = struct{}{} } } - var moduleHookNames []string - for name := range moduleHookNamesMap { - moduleHookNames = append(moduleHookNames, name) - } - - return moduleHookNames + return utils.MapStringStructKeys(moduleHookNamesMap) } // TODO: moduleManager.GetModule(modName).Delete() @@ -917,7 +840,7 @@ func (mm *moduleManager) DeleteModule(moduleName string, logLabels map[string]st return err } - // remove module hooks from indexes + // Unregister module hooks. delete(mm.modulesHooksOrderByName, moduleName) return nil @@ -977,16 +900,16 @@ func (mm *moduleManager) RunGlobalHook(hookName string, binding BindingType, bin return beforeChecksum, afterChecksum, nil } -func (mm *moduleManager) RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) error { +func (mm *moduleManager) RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) (beforeChecksum string, afterChecksum string, err error) { moduleHook := mm.GetModuleHook(hookName) values, err := moduleHook.Module.Values() if err != nil { - return err + return "", "", err } valuesChecksum, err := values.Checksum() if err != nil { - return err + return "", "", err } // Update kubernetes snapshots just before execute a hook @@ -1004,26 +927,19 @@ func (mm *moduleManager) RunModuleHook(hookName string, binding BindingType, bin } if err := moduleHook.Run(binding, bindingContext, logLabels, metricLabels); err != nil { - return err + return "", "", err } newValues, err := moduleHook.Module.Values() if err != nil { - return err + return "", "", err } newValuesChecksum, err := newValues.Checksum() if err != nil { - return err - } - - if newValuesChecksum != valuesChecksum { - switch binding { - case Schedule, OnKubernetesEvent: - mm.moduleValuesChanged <- moduleHook.Module.Name - } + return "", "", err } - return nil + return valuesChecksum, newValuesChecksum, nil } // GlobalConfigValues return raw global values only from a ConfigMap. @@ -1213,14 +1129,6 @@ func (mm *moduleManager) DisableModuleHooks(moduleName string) { } } -//func (mm *moduleManager) EnableModuleScheduleBindings(moduleName) { -// -//} - -//func (mm *moduleManager) EnableGlobalScheduleBindings(moduleName) { -// -//} - func (mm *moduleManager) HandleScheduleEvent(crontab string, createGlobalTaskFn func(*GlobalHook, controller.BindingExecutionInfo), createModuleTaskFn func(*Module, *ModuleHook, controller.BindingExecutionInfo)) error { mm.LoopByBinding(Schedule, func(gh *GlobalHook, m *Module, mh *ModuleHook) { if gh != nil { @@ -1253,20 +1161,17 @@ func (mm *moduleManager) LoopByBinding(binding BindingType, fn func(gh *GlobalHo fn(gh, nil, nil) } - modules := mm.enabledModulesInOrder - - for _, moduleName := range modules { + for _, moduleName := range mm.enabledModules { m := mm.GetModule(moduleName) moduleHooks := mm.GetModuleHooksInOrder(moduleName, binding) for _, hookName := range moduleHooks { mh := mm.GetModuleHook(hookName) - fn(nil, m, mh) } - } } +// ApplyEnabledPatch changes "dynamicEnabled" map with patches. func (mm *moduleManager) ApplyEnabledPatch(enabledPatch utils.ValuesPatch) error { newDynamicEnabled := map[string]*bool{} for k, v := range mm.dynamicEnabled { diff --git a/pkg/module_manager/module_manager_test.go b/pkg/module_manager/module_manager_test.go index dfbf8bb9..4be7fb91 100644 --- a/pkg/module_manager/module_manager_test.go +++ b/pkg/module_manager/module_manager_test.go @@ -10,15 +10,18 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8types "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/yaml" "github.com/flant/addon-operator/pkg/helm" - "github.com/flant/addon-operator/pkg/helm/client" + "github.com/flant/addon-operator/pkg/helm_resources_manager" "github.com/flant/addon-operator/pkg/kube_config_manager" "github.com/flant/addon-operator/pkg/utils" klient "github.com/flant/kube-client/client" + "github.com/flant/shell-operator/pkg/kube_events_manager/types" utils_file "github.com/flant/shell-operator/pkg/utils/file" . "github.com/flant/addon-operator/pkg/hook/types" @@ -29,64 +32,102 @@ import ( _ "github.com/flant/addon-operator/pkg/module_manager/test/go_hooks/global-hooks" ) -// initModuleManager is a test version of an Init method -func initModuleManager(t *testing.T, mm *moduleManager, configPath string) { - rootDir := filepath.Join("testdata", configPath) - - var err error - tempDir, err := ioutil.TempDir("", "addon-operator-") - t.Logf("TEMP DIR %s", tempDir) - if err != nil { - t.Fatal(err) - } +type initModuleManagerResult struct { + moduleManager *moduleManager + kubeConfigManager kube_config_manager.KubeConfigManager + helmClient *helm.MockHelmClient + helmResourcesManager *helm_resources_manager.MockHelmResourcesManager + kubeClient klient.Client + initialState *ModulesState + initialStateErr error + cmName string + cmNamespace string +} - mm.WithDirectories(filepath.Join(rootDir, "modules"), filepath.Join(rootDir, "global-hooks"), tempDir) +// initModuleManager creates a ready-to-use ModuleManager instance and some dependencies. +func initModuleManager(t *testing.T, configPath string) (ModuleManager, *initModuleManagerResult) { + const defaultNamespace = "default" + const defaultName = "addon-operator" - if err := mm.RegisterModules(); err != nil { - t.Fatal(err) - } + result := new(initModuleManagerResult) - if err := mm.RegisterGlobalHooks(); err != nil { - t.Fatal(err) - } + // Mock helm client for module.go, hook_executor.go + result.helmClient = &helm.MockHelmClient{} - cmFilePath := filepath.Join(rootDir, "config_map.yaml") - exists, _ := utils_file.FileExists(cmFilePath) - if exists { - cmDataBytes, err := ioutil.ReadFile(cmFilePath) - if err != nil { - t.Fatalf("congig map file '%s': %s", cmFilePath, err) - } + // Mock helm resources manager to execute module actions: run, delete. + result.helmResourcesManager = &helm_resources_manager.MockHelmResourcesManager{} - var cmObj = new(v1.ConfigMap) - _ = yaml.Unmarshal(cmDataBytes, &cmObj) + // Init directories + rootDir := filepath.Join("testdata", configPath) - kubeClient := klient.NewFake(nil) - _, _ = kubeClient.CoreV1().ConfigMaps("default").Create(context.TODO(), cmObj, metav1.CreateOptions{}) + var err error - KubeConfigManager := kube_config_manager.NewKubeConfigManager() - KubeConfigManager.WithKubeClient(kubeClient) - KubeConfigManager.WithContext(context.Background()) - KubeConfigManager.WithNamespace("default") - KubeConfigManager.WithConfigMapName("addon-operator") + // Create and init moduleManager instance + // Note: skip KubeEventManager, ScheduleManager, KubeObjectPatcher, MetricStorage, HookMetricStorage + result.moduleManager = NewModuleManager() + result.moduleManager.WithContext(context.Background()) + result.moduleManager.WithDirectories(filepath.Join(rootDir, "modules"), filepath.Join(rootDir, "global-hooks"), t.TempDir()) + result.moduleManager.WithHelm(helm.MockHelm(result.helmClient)) - err = KubeConfigManager.Init() - if err != nil { - t.Fatalf("KubeConfigManager.Init(): %v", err) - } - mm.WithKubeConfigManager(KubeConfigManager) + err = result.moduleManager.Init() + require.NoError(t, err, "Should register global hooks and all modules") - kubeConfig := KubeConfigManager.InitialConfig() - mm.kubeGlobalConfigValues = kubeConfig.Values + result.moduleManager.WithHelmResourcesManager(result.helmResourcesManager) - mm.enabledModulesByConfig, mm.kubeModulesConfigValues, _ = mm.calculateEnabledModulesByConfig(kubeConfig.ModuleConfigs) + // Load config values from config_map.yaml. + cmFilePath := filepath.Join(rootDir, "config_map.yaml") + cmExists, _ := utils_file.FileExists(cmFilePath) + var cmObj *v1.ConfigMap + if cmExists { + cmDataBytes, err := ioutil.ReadFile(cmFilePath) + require.NoError(t, err, "Should read config map file '%s'", cmFilePath) + cmObj = new(v1.ConfigMap) + err = yaml.Unmarshal(cmDataBytes, &cmObj) + require.NoError(t, err, "Should parse YAML in %s", cmFilePath) + if cmObj.Namespace == "" { + cmObj.SetNamespace(defaultNamespace) + } } else { - mm.enabledModulesByConfig, mm.kubeModulesConfigValues, _ = mm.calculateEnabledModulesByConfig(kube_config_manager.ModuleConfigs{}) + cmObj = &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + Namespace: defaultNamespace, + }, + Data: nil, + } } + result.cmName = cmObj.Name + result.cmNamespace = cmObj.Namespace + result.kubeClient = klient.NewFake(nil) + _, err = result.kubeClient.CoreV1().ConfigMaps(result.cmNamespace).Create(context.TODO(), cmObj, metav1.CreateOptions{}) + require.NoError(t, err, "Should create ConfigMap/%s", result.cmName) + + result.kubeConfigManager = kube_config_manager.NewKubeConfigManager() + result.kubeConfigManager.WithKubeClient(result.kubeClient) + result.kubeConfigManager.WithContext(context.Background()) + result.kubeConfigManager.WithNamespace(result.cmNamespace) + result.kubeConfigManager.WithConfigMapName(result.cmName) + + err = result.kubeConfigManager.Init() + require.NoError(t, err, "KubeConfigManager.Init should not fail") + result.moduleManager.WithKubeConfigManager(result.kubeConfigManager) + + result.kubeConfigManager.SafeReadConfig(func(config *kube_config_manager.KubeConfig) { + result.initialState, result.initialStateErr = result.moduleManager.HandleNewKubeConfig(config) + }) + + // Start KubeConfigManager to be able to change config values via patching ConfigMap. + result.kubeConfigManager.Start() + + return result.moduleManager, result } -func Test_MainModuleManager_LoadValuesInInit(t *testing.T) { +func Test_ModuleManager_LoadValuesInInit(t *testing.T) { var mm *moduleManager tests := []struct { @@ -194,8 +235,8 @@ func Test_MainModuleManager_LoadValuesInInit(t *testing.T) { fmt.Printf("kubeModulesConfigValues: %#v\n", mm.kubeModulesConfigValues) - // with-values-2 has kube config but disabled - assert.NotContains(t, mm.kubeModulesConfigValues, "with-values-2") + // with-values-2 module is disabled but config values should be loaded. + assert.Contains(t, mm.kubeModulesConfigValues, "with-values-2") // with-kube-values has kube config and is enabled assert.Contains(t, mm.kubeModulesConfigValues, "with-kube-values") @@ -213,26 +254,24 @@ func Test_MainModuleManager_LoadValuesInInit(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - mm = NewMainModuleManager() - initModuleManager(t, mm, test.configPath) + _, res := initModuleManager(t, test.configPath) + mm = res.moduleManager test.testFn() }) } } -func Test_MainModuleManager_LoadValues_ApplyDefaults(t *testing.T) { - mm := NewMainModuleManager() - - initModuleManager(t, mm, "load_values__module_apply_defaults") +func Test_ModuleManager_LoadValues_ApplyDefaults(t *testing.T) { + _, res := initModuleManager(t, "load_values__module_apply_defaults") //assert.Len(t, mm.commonStaticValues, 1) //assert.Len(t, mm.commonStaticValues.Global(), 1) - assert.Len(t, mm.allModulesByName, 1) + assert.Len(t, res.moduleManager.allModulesByName, 1) - assert.Contains(t, mm.allModulesByName, "module-one") + assert.Contains(t, res.moduleManager.allModulesByName, "module-one") - modOne := mm.allModulesByName["module-one"] + modOne := res.moduleManager.allModulesByName["module-one"] assert.NotNil(t, modOne.CommonStaticConfig) assert.NotNil(t, modOne.StaticConfig) assert.Equal(t, "module-one", modOne.CommonStaticConfig.ModuleName) @@ -244,7 +283,7 @@ func Test_MainModuleManager_LoadValues_ApplyDefaults(t *testing.T) { assert.Nil(t, modOne.CommonStaticConfig.IsEnabled) assert.Nil(t, modOne.StaticConfig.IsEnabled) - assert.Contains(t, mm.kubeModulesConfigValues, "module-one") + assert.Contains(t, res.moduleManager.kubeModulesConfigValues, "module-one") vals, err := modOne.Values() assert.Nil(t, err) @@ -267,70 +306,52 @@ func Test_MainModuleManager_LoadValues_ApplyDefaults(t *testing.T) { // 'azaza' field from modules/values.yaml. assert.Contains(t, globVals, "init") initVals := globVals["init"].(map[string]interface{}) - assert.Contains(t, initVals, "azaza") + assert.Contains(t, initVals, "param1") // 'discovery' field default from values.yaml schema. assert.Contains(t, globVals, "discovery") } -func Test_MainModuleManager_Get_Module(t *testing.T) { - mm := NewMainModuleManager() - - initModuleManager(t, mm, "get__module") +func Test_ModuleManager_Get_Module(t *testing.T) { + mm, res := initModuleManager(t, "get__module") programmaticModule := &Module{Name: "programmatic-module"} - mm.allModulesByName["programmatic-module"] = programmaticModule + res.moduleManager.allModulesByName["programmatic-module"] = programmaticModule var module *Module tests := []struct { name string moduleName string - testFn func() + testFn func(t *testing.T) }{ { "module_loaded_from_files", "module", - func() { - expectedModule := &Module{ - Name: "module", - Path: filepath.Join(mm.ModulesDir, "000-module"), - CommonStaticConfig: &utils.ModuleConfig{ - ModuleName: "module", - Values: utils.Values{}, - IsEnabled: nil, - IsUpdated: false, - ModuleConfigKey: "module", - ModuleEnabledKey: "moduleEnabled", - RawConfig: []string{}, - }, - StaticConfig: &utils.ModuleConfig{ - ModuleName: "module", - Values: utils.Values{}, - IsEnabled: nil, - IsUpdated: false, - ModuleConfigKey: "module", - ModuleEnabledKey: "moduleEnabled", - RawConfig: []string{}, - }, - State: NewModuleState(), - moduleManager: mm, - } - assert.Equal(t, expectedModule, module) + func(t *testing.T) { + require.NotNil(t, module) + require.Equal(t, "module", module.Name) + require.Equal(t, filepath.Join(res.moduleManager.ModulesDir, "000-module"), module.Path) + require.NotNil(t, module.CommonStaticConfig) + require.Nil(t, module.CommonStaticConfig.IsEnabled) + require.NotNil(t, module.StaticConfig) + require.Nil(t, module.StaticConfig.IsEnabled) + require.NotNil(t, module.State) + require.NotNil(t, module.moduleManager) }, }, { "direct_add_module_to_index", "programmatic-module", - func() { - assert.Equal(t, programmaticModule, module) + func(t *testing.T) { + require.Equal(t, programmaticModule, module) }, }, { "error-on-non-existent-module", "non-existent", - func() { - assert.Nil(t, module) + func(t *testing.T) { + require.Nil(t, module) }, }, } @@ -339,200 +360,98 @@ func Test_MainModuleManager_Get_Module(t *testing.T) { t.Run(test.name, func(t *testing.T) { module = nil module = mm.GetModule(test.moduleName) - test.testFn() + test.testFn(t) }) } } -//func Test_MainModuleManager_Get_ModuleHook(t *testing.T) { -// t.SkipNow() -// mm := NewMainModuleManager() -// -// initModuleManager(t, mm, "get__module_hook") -// -// var moduleHook *ModuleHook -// var err error -// -// tests := []struct{ -// name string -// hookName string -// testFn func() -// } { -// { -// "module-hook-all-bindings", -// "000-all-bindings/hooks/all", -// func() { -// expectedHook := &ModuleHook{ -// &CommonHook{ -// "000-all-bindings/hooks/all", -// filepath.Join(mm.ModulesDir, "000-all-bindings/hooks/all"), -// []BindingType{BeforeHelm, AfterHelm, AfterDeleteHelm, OnStartup, Schedule, KubeEvents}, -// map[BindingType]float64 { -// BeforeHelm: 1.0, -// AfterHelm: 1.0, -// AfterDeleteHelm: 1.0, -// OnStartup: 1.0, -// }, -// mm, -// }, -// &Module{}, -// &ModuleHookConfig{ -// HookConfig{ -// 1.0, -// []schedule_manager.ScheduleConfig{ -// { -// Crontab: "* * * * *", -// AllowFailure: true, -// }, -// }, -// []kube_events_manager.OnKubernetesEventConfig{ -// { -// EventTypes: []kube_events_manager.OnKubernetesEventType{kube_events_manager.KubernetesEventOnAdd}, -// Kind: "configmap", -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "component": "component1", -// }, -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// { -// Key: "tier", -// Operator: "In", -// Values: []string{"cache"}, -// }, -// }, -// }, -// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ -// MatchNames: []string{"namespace1"}, -// Any: false, -// }, -// JqFilter: ".items[] | del(.metadata, .field1)", -// AllowFailure: true, -// }, -// { -// EventTypes: []kube_events_manager.OnKubernetesEventType{ -// kube_events_manager.KubernetesEventOnAdd, -// kube_events_manager.KubernetesEventOnUpdate, -// kube_events_manager.KubernetesEventOnDelete, -// }, -// Kind: "namespace", -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "component": "component2", -// }, -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// { -// Key: "tier", -// Operator: "In", -// Values: []string{"cache"}, -// }, -// }, -// }, -// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ -// MatchNames: []string{"namespace2"}, -// Any: false, -// }, -// JqFilter: ".items[] | del(.metadata, .field2)", -// AllowFailure: true, -// }, -// { -// EventTypes: []kube_events_manager.OnKubernetesEventType{ -// kube_events_manager.KubernetesEventOnAdd, -// kube_events_manager.KubernetesEventOnUpdate, -// kube_events_manager.KubernetesEventOnDelete, -// }, -// Kind: "pod", -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "component": "component3", -// }, -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// { -// Key: "tier", -// Operator: "In", -// Values: []string{"cache"}, -// }, -// }, -// }, -// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ -// MatchNames: nil, -// Any: true, -// }, -// JqFilter: ".items[] | del(.metadata, .field3)", -// AllowFailure: true, -// }, -// }, -// }, -// 1.0, -// 1.0, -// 1.0, -// }, -// } -// if assert.NoError(t, err) { -// moduleHook.Module = &Module{} -// assert.Equal(t, expectedHook, moduleHook) -// } -// }, -// }, -// { -// "nested-module-hook", -// "100-nested-hooks/hooks/sub/sub/nested-before-helm", -// func() { -// expectedHook := &ModuleHook{ -// &CommonHook{ -// "100-nested-hooks/hooks/sub/sub/nested-before-helm", -// filepath.Join(mm.ModulesDir, "100-nested-hooks/hooks/sub/sub/nested-before-helm"), -// []BindingType{BeforeHelm}, -// map[BindingType]float64 { -// BeforeHelm: 1.0, -// }, -// mm, -// }, -// &Module{}, -// &ModuleHookConfig{ -// HookConfig{ -// OnStartup: nil, -// Schedule: nil, -// OnKubernetesEvent: nil, -// }, -// 1.0, -// nil, -// nil, -// }, -// } -// if assert.NoError(t, err) { -// moduleHook.Module = &Module{} -// assert.Equal(t, expectedHook, moduleHook) -// } -// }, -// }, -// { -// "error-on-non-existent-module-hook", -// "non-existent", -// func() { -// assert.Error(t, err) -// }, -// }, -// } -// -// for _, test := range tests { -// t.Run(test.name, func(t *testing.T) { -// moduleHook = nil -// err = nil -// moduleHook, err = mm.GetModuleHook(test.hookName) -// test.testFn() -// }) -// } -//} - -func Test_MainModuleManager_Get_ModuleHooksInOrder(t *testing.T) { - helm.NewClient = func(logLabels ...map[string]string) client.HelmClient { - return &helm.MockHelmClient{} +// Modules in get__module_hook path: +// - 000-all-bindings module with all-bindings hook with all possible bindings +// - 100-nested-hooks with hook in deep subdirectory. +func Test_ModuleManager_Get_ModuleHook(t *testing.T) { + mm, _ := initModuleManager(t, "get__module_hook") + + // Register modules hooks. + for _, modName := range []string{"all-bindings", "nested-hooks"} { + err := mm.RegisterModuleHooks(mm.GetModule(modName), map[string]string{}) + require.NoError(t, err, "Should register hooks for module '%s'", modName) } - mm := NewMainModuleManager() - initModuleManager(t, mm, "get__module_hooks_in_order") + var moduleHook *ModuleHook - _, _ = mm.DiscoverModulesState(map[string]string{}) + tests := []struct { + name string + hookName string + testFn func(t *testing.T) + }{ + { + "module-hook-all-bindings", + "000-all-bindings/hooks/all-bindings", + func(t *testing.T) { + require.NotNil(t, moduleHook, "Module hook 'all-bindings' should be registered") + + assert.Equal(t, "000-all-bindings/hooks/all-bindings", moduleHook.Name) + assert.NotNil(t, moduleHook.Config) + assert.Equal(t, []BindingType{OnStartup, Schedule, OnKubernetesEvent, BeforeHelm, AfterHelm, AfterDeleteHelm}, moduleHook.Config.Bindings()) + assert.Len(t, moduleHook.Config.OnKubernetesEvents, 3, "Should register 3 kubernetes bindings") + assert.Len(t, moduleHook.Config.Schedules, 1, "Should register 1 schedule binding") + + // Schedule binding is 'every minute' with allow failure. + schBinding := moduleHook.Config.Schedules[0] + assert.Equal(t, "* * * * *", schBinding.ScheduleEntry.Crontab) + assert.True(t, schBinding.AllowFailure) + + kBinding := moduleHook.Config.OnKubernetesEvents[0] + assert.True(t, kBinding.AllowFailure) + assert.NotNil(t, kBinding.Monitor) + assert.NotNil(t, kBinding.Monitor.NamespaceSelector) + assert.NotNil(t, kBinding.Monitor.LabelSelector) + assert.Nil(t, kBinding.Monitor.NameSelector) + assert.Nil(t, kBinding.Monitor.FieldSelector) + assert.Equal(t, "configmap", kBinding.Monitor.Kind) + assert.Equal(t, []types.WatchEventType{types.WatchEventAdded}, kBinding.Monitor.EventTypes) + + // Binding without executeHookOnEvent should have all events + kBinding = moduleHook.Config.OnKubernetesEvents[1] + assert.True(t, kBinding.AllowFailure) + assert.Equal(t, []types.WatchEventType{types.WatchEventAdded, types.WatchEventModified, types.WatchEventDeleted}, kBinding.Monitor.EventTypes) + + // Binding without allowFailure + kBinding = moduleHook.Config.OnKubernetesEvents[2] + assert.False(t, kBinding.AllowFailure) + }, + }, + { + "nested-module-hook", + "100-nested-hooks/hooks/sub/sub/nested-before-helm", + func(t *testing.T) { + require.NotNil(t, moduleHook, "Module hook 'nested-before-helm' should be registered") + assert.Equal(t, "100-nested-hooks/hooks/sub/sub/nested-before-helm", moduleHook.Name) + assert.NotNil(t, moduleHook.Config) + assert.Equal(t, []BindingType{BeforeHelm}, moduleHook.Config.Bindings()) + }, + }, + { + "nil-on-non-existent-module-hook", + "non-existent-hook-name", + func(t *testing.T) { + assert.Nil(t, moduleHook) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + moduleHook = mm.GetModuleHook(test.hookName) + test.testFn(t) + }) + } +} + +func Test_ModuleManager_Get_ModuleHooksInOrder(t *testing.T) { + mm, res := initModuleManager(t, "get__module_hooks_in_order") + + _, _ = mm.RefreshEnabledState(map[string]string{}) var moduleHooks []string @@ -547,8 +466,8 @@ func Test_MainModuleManager_Get_ModuleHooksInOrder(t *testing.T) { "after-helm-binding-hooks", AfterHelm, func() { - assert.Len(t, mm.allModulesByName, 1) - assert.Len(t, mm.modulesHooksOrderByName, 1) + assert.Len(t, res.moduleManager.allModulesByName, 1) + assert.Len(t, res.moduleManager.modulesHooksOrderByName, 1) expectedOrder := []string{ "107-after-helm-binding-hooks/hooks/b", @@ -568,7 +487,7 @@ func Test_MainModuleManager_Get_ModuleHooksInOrder(t *testing.T) { }, }, { - "error-on-non-existent-module", + "no-hooks-for-existent-module", "after-helm-binding-hookssss", BeforeHelm, func() { @@ -586,129 +505,134 @@ func Test_MainModuleManager_Get_ModuleHooksInOrder(t *testing.T) { } } -type MockKubeConfigManager struct { - kube_config_manager.KubeConfigManager -} - -func (kcm MockKubeConfigManager) SetKubeGlobalValues(values utils.Values) error { - return nil -} - -func (kcm MockKubeConfigManager) SetKubeModuleValues(moduleName string, values utils.Values) error { - return nil -} - -func Test_MainModuleManager_RunModule(t *testing.T) { - // TODO something wrong here with patches from afterHelm and beforeHelm hooks - t.SkipNow() - hc := &helm.MockHelmClient{} +// Path test_run_module contains only one module with beforeHelm and afterHelm hooks. +// This test runs module and checks resulting values. +func Test_ModuleManager_RunModule(t *testing.T) { + const ModuleName = "module" + var err error - helm.NewClient = func(logLabels ...map[string]string) client.HelmClient { - return hc - } + mm, res := initModuleManager(t, "test_run_module") - mm := NewMainModuleManager() + module := mm.GetModule(ModuleName) + require.NotNil(t, module, "Should get module %s", ModuleName) - mm.WithKubeConfigManager(MockKubeConfigManager{}) + // Register module hooks. + err = mm.RegisterModuleHooks(module, map[string]string{}) + require.NoError(t, err, "Should register module hooks") - initModuleManager(t, mm, "test_run_module") - - moduleName := "module" - expectedModuleValues := utils.Values{ - "global": map[string]interface{}{ - "enabledModules": []string{}, - }, - "module": map[string]interface{}{ - "afterHelm": "override-value", - "beforeHelm": "override-value", - "replicaCount": 1.0, - "image": map[string]interface{}{ - "repository": "nginx", - "tag": "stable", - "pullPolicy": "IfNotPresent", - }, - }, - } - - _, err := mm.RunModule(moduleName, false, map[string]string{}, nil) - if err != nil { - t.Fatal(err) - } - - module := mm.GetModule(moduleName) + valuesChanged, err := mm.RunModule(ModuleName, false, map[string]string{}, nil) + require.NoError(t, err, "Module %s should run successfully", ModuleName) + require.True(t, valuesChanged, "Module hooks should change values") + // Check values after running hooks: + // global: + // enabledModules: [] + // module: + // afterHelm: "value-from-after-helm" + // beforeHelm: "value-from-before-helm-20" + // imageName: "nginx:stable" values, err := module.Values() - if !assert.NoError(t, err) { - t.FailNow() - } - - if !reflect.DeepEqual(expectedModuleValues, values) { - t.Errorf("\n[EXPECTED]: %#v\n[GOT]: %#v", expectedModuleValues, values) + require.NoError(t, err, "Should collect effective module values") + { + require.True(t, values.HasGlobal(), "Global values should present") + + // Check values contains section for "module" module. + require.Contains(t, values, "module", "Should has module key in values") + modVals, ok := values["module"].(map[string]interface{}) + require.True(t, ok, "value on module key should be map") + { + // Check value from hook-4. + require.Contains(t, modVals, "afterHelm", "Should has afterHelm field") + afterHelm, ok := modVals["afterHelm"].(string) + require.True(t, ok, "afterHelm value should be string") + require.Equal(t, afterHelm, "value-from-after-helm") + + // Check final value from hook-1, hook-2, and hook-3. + require.Contains(t, modVals, "beforeHelm", "Should has beforeHelm field") + beforeHelm, ok := modVals["beforeHelm"].(string) + require.True(t, ok, "beforeHelm value should be string") + require.Equal(t, beforeHelm, "value-from-before-helm-20", "beforeHelm should be from hook-1") + + // Check values from values.yaml. + require.Contains(t, modVals, "imageName", "Should has imageName field from values.yaml") + imageName, ok := modVals["imageName"].(string) + require.True(t, ok, "imageName value should be string") + require.Equal(t, "nginx:stable", imageName, "should have imageName value from values.yaml") + } } - assert.Equal(t, hc.DeleteSingleFailedRevisionExecuted, true, "helm.DeleteSingleFailedRevision must be executed!") - assert.Equal(t, hc.UpgradeReleaseExecuted, true, "helm.UpgradeReleaseExecuted must be executed!") + // Check for helm client methods calls. + assert.Equal(t, res.helmClient.DeleteSingleFailedRevisionExecuted, true, "helm.DeleteSingleFailedRevision must be executed!") + assert.Equal(t, res.helmClient.UpgradeReleaseExecuted, true, "helm.UpgradeReleaseExecuted must be executed!") } -func Test_MainModuleManager_DeleteModule(t *testing.T) { - // TODO check afterHelmDelete patch - t.SkipNow() - hc := &helm.MockHelmClient{} - - helm.NewClient = func(logLabels ...map[string]string) client.HelmClient { - return hc - } - - mm := NewMainModuleManager() - mm.WithKubeConfigManager(MockKubeConfigManager{}) +// Path test_delete_module contains only one module with afterDeleteHelm hooks. +// This test runs module and checks resulting values. +func Test_ModuleManager_DeleteModule(t *testing.T) { + const ModuleName = "module" + var err error - initModuleManager(t, mm, "test_delete_module") + mm, res := initModuleManager(t, "test_delete_module") - moduleName := "module" - expectedModuleValues := utils.Values{ - "global": map[string]interface{}{ - "enabledModules": []string{}, - }, - "module": map[string]interface{}{ - "afterDeleteHelm": "override-value", - "replicaCount": 1.0, - "image": map[string]interface{}{ - "repository": "nginx", - "tag": "stable", - "pullPolicy": "IfNotPresent", - }, - }, - } + module := mm.GetModule(ModuleName) + require.NotNil(t, module, "Should get module %s", ModuleName) - err := mm.DeleteModule(moduleName, map[string]string{}) - if err != nil { - t.Fatal(err) - } + // Register module hooks. + err = mm.RegisterModuleHooks(module, map[string]string{}) + require.NoError(t, err, "Should register module hooks") - module := mm.GetModule(moduleName) + err = mm.DeleteModule(ModuleName, map[string]string{}) + require.NoError(t, err, "Should delete module") + //if !reflect.DeepEqual(expectedModuleValues, values) { + // t.Errorf("\n[EXPECTED]: %#v\n[GOT]: %#v", expectedModuleValues, values) + //} + // Check values after running hooks: + // global: + // enabledModules: [] + // module: + // afterDeleteHelm: "value-from-after-delete-helm-20" + // imageName: "nginx:stable" values, err := module.Values() - if !assert.NoError(t, err) { - t.FailNow() - } - - if !reflect.DeepEqual(expectedModuleValues, values) { - t.Errorf("\n[EXPECTED]: %#v\n[GOT]: %#v", expectedModuleValues, values) + require.NoError(t, err, "Should collect effective module values") + { + require.True(t, values.HasGlobal(), "Global values should present") + + // Check values contains section for "module" module. + require.Contains(t, values, "module", "Should has module key in values") + modVals, ok := values["module"].(map[string]interface{}) + require.True(t, ok, "value on module key should be map") + { + // Check value from hook-1. + require.Contains(t, modVals, "afterDeleteHelm", "Should has afterDeleteHelm field") + afterHelm, ok := modVals["afterDeleteHelm"].(string) + require.True(t, ok, "afterDeleteHelm value should be string") + require.Equal(t, afterHelm, "value-from-after-delete-helm-20") + + // Check values from values.yaml. + require.Contains(t, modVals, "imageName", "Should has imageName field from values.yaml") + imageName, ok := modVals["imageName"].(string) + require.True(t, ok, "imageName value should be string") + require.Equal(t, "nginx:stable", imageName, "should have imageName value from values.yaml") + } } - assert.Equal(t, hc.DeleteReleaseExecuted, true, "helm.DeleteRelease must be executed!") + assert.Equal(t, res.helmClient.DeleteReleaseExecuted, true, "helm.DeleteRelease must be executed!") } -func Test_MainModuleManager_RunModuleHook(t *testing.T) { - // TODO hooks not found - t.SkipNow() - helm.NewClient = func(logLabels ...map[string]string) client.HelmClient { - return &helm.MockHelmClient{} +// Modules in test_run_module_hook path: +// - 000-update-kube-module-config with hook that add and remove some config values. +// - 000-update-module-dynamic with hook that add some dynamic values. +// +// Test runs these hooks and checks resulting values. +func Test_ModuleManager_RunModuleHook(t *testing.T) { + mm, res := initModuleManager(t, "test_run_module_hook") + + // Register modules hooks. + for _, modName := range []string{"update-kube-module-config", "update-module-dynamic"} { + err := mm.RegisterModuleHooks(mm.GetModule(modName), map[string]string{}) + require.NoError(t, err, "Should register hooks for module '%s'", modName) } - mm := NewMainModuleManager() - mm.WithKubeConfigManager(MockKubeConfigManager{}) - - initModuleManager(t, mm, "test_run_module_hook") expectations := []struct { testName string @@ -814,15 +738,14 @@ func Test_MainModuleManager_RunModuleHook(t *testing.T) { }, } - mm.kubeModulesConfigValues = make(map[string]utils.Values) + res.moduleManager.kubeModulesConfigValues = make(map[string]utils.Values) for _, expectation := range expectations { t.Run(expectation.testName, func(t *testing.T) { - mm.kubeModulesConfigValues[expectation.moduleName] = expectation.kubeModuleConfigValues - mm.modulesDynamicValuesPatches[expectation.moduleName] = expectation.moduleDynamicValuesPatches + res.moduleManager.kubeModulesConfigValues[expectation.moduleName] = expectation.kubeModuleConfigValues + res.moduleManager.modulesDynamicValuesPatches[expectation.moduleName] = expectation.moduleDynamicValuesPatches - if err := mm.RunModuleHook(expectation.hookName, BeforeHelm, nil, map[string]string{}); err != nil { - t.Fatal(err) - } + _, _, err := mm.RunModuleHook(expectation.hookName, BeforeHelm, nil, map[string]string{}) + require.NoError(t, err, "Hook %s should not fail", expectation.hookName) module := mm.GetModule(expectation.moduleName) @@ -831,9 +754,7 @@ func Test_MainModuleManager_RunModuleHook(t *testing.T) { } values, err := module.Values() - if !assert.NoError(t, err) { - t.FailNow() - } + require.NoError(t, err, "Should collect effective values for module") if !reflect.DeepEqual(expectation.expectedModuleValues, values) { t.Errorf("\n[EXPECTED]: %#v\n[GOT]: %#v", expectation.expectedModuleValues, values) @@ -842,185 +763,88 @@ func Test_MainModuleManager_RunModuleHook(t *testing.T) { } } -//func Test_MainModuleManager_Get_GlobalHook(t *testing.T) { -// mm := NewMainModuleManager() -// -// initModuleManager(t, mm, "get__global_hook") -// -// var globalHook *GlobalHook -// var err error -// -// tests := []struct { -// name string -// hookName string -// testFn func() -// }{ -// { -// "global-hook-with-all-bindings", -// "000-all-bindings/all", -// func() { -// expectedHook := &GlobalHook{ -// &CommonHook{ -// "000-all-bindings/all", -// filepath.Join(mm.GlobalHooksDir, "000-all-bindings/all"), -// []BindingType{BeforeAll, AfterAll, OnStartup, Schedule, KubeEvents}, -// map[BindingType]float64{ -// BeforeAll: 1.0, -// AfterAll: 1.0, -// OnStartup: 1.0, -// }, -// mm, -// }, -// &GlobalHookConfig{ -// HookConfig{ -// 1.0, -// []schedule_manager.ScheduleConfig{ -// { -// Crontab: "* * * * *", -// AllowFailure: true, -// }, -// }, -// []kube_events_manager.OnKubernetesEventConfig{ -// { -// EventTypes: []kube_events_manager.OnKubernetesEventType{kube_events_manager.KubernetesEventOnAdd}, -// Kind: "configmap", -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "component": "component1", -// }, -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// { -// Key: "tier", -// Operator: "In", -// Values: []string{"cache"}, -// }, -// }, -// }, -// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ -// MatchNames: []string{"namespace1"}, -// Any: false, -// }, -// JqFilter: ".items[] | del(.metadata, .field1)", -// AllowFailure: true, -// }, -// { -// EventTypes: []kube_events_manager.OnKubernetesEventType{ -// kube_events_manager.KubernetesEventOnAdd, -// kube_events_manager.KubernetesEventOnUpdate, -// kube_events_manager.KubernetesEventOnDelete, -// }, -// Kind: "namespace", -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "component": "component2", -// }, -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// { -// Key: "tier", -// Operator: "In", -// Values: []string{"cache"}, -// }, -// }, -// }, -// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ -// MatchNames: []string{"namespace2"}, -// Any: false, -// }, -// JqFilter: ".items[] | del(.metadata, .field2)", -// AllowFailure: true, -// }, -// { -// EventTypes: []kube_events_manager.OnKubernetesEventType{ -// kube_events_manager.KubernetesEventOnAdd, -// kube_events_manager.KubernetesEventOnUpdate, -// kube_events_manager.KubernetesEventOnDelete, -// }, -// Kind: "pod", -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "component": "component3", -// }, -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// { -// Key: "tier", -// Operator: "In", -// Values: []string{"cache"}, -// }, -// }, -// }, -// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ -// MatchNames: nil, -// Any: true, -// }, -// JqFilter: ".items[] | del(.metadata, .field3)", -// AllowFailure: true, -// }, -// }, -// }, -// 1.0, -// 1.0, -// }, -// } -// -// if assert.NoError(t, err) { -// assert.Equal(t, expectedHook, globalHook) -// } -// }, -// }, -// { -// "global-hook-nested", -// "100-nested-hook/sub/sub/nested-before-all", -// func() { -// expectedHook := &GlobalHook{ -// &CommonHook { -// "100-nested-hook/sub/sub/nested-before-all", -// filepath.Join(mm.GlobalHooksDir, "100-nested-hook/sub/sub/nested-before-all"), -// []BindingType{BeforeAll}, -// map[BindingType]float64{ -// BeforeAll: 1.0, -// }, -// mm, -// }, -// &GlobalHookConfig{ -// HookConfig{ -// nil, -// nil, -// nil, -// }, -// 1.0, -// nil, -// }, -// } -// if assert.NoError(t, err) { -// assert.Equal(t, expectedHook, globalHook) -// } -// }, -// }, -// { -// "error-if-hook-not-registered", -// "non-existent", -// func(){ -// assert.Error(t, err) -// assert.Nil(t, globalHook) -// }, -// }, -// } +// Global hooks in test_run_global_hook path: +// - 000-update-kube-config hook that add and remove some config values. +// - 100-update-dynamic hook that add some dynamic values. // -// for _, test := range tests { -// t.Run(test.name, func(t *testing.T) { -// globalHook, err = mm.GetGlobalHook(test.hookName) -// test.testFn() -// }) -// } -//} - -func Test_MainModuleManager_Get_GlobalHooksInOrder(t *testing.T) { - helm.NewClient = func(logLabels ...map[string]string) client.HelmClient { - return &helm.MockHelmClient{} +// Test checks hooks registration. +func Test_MainModuleManager_Get_GlobalHook(t *testing.T) { + mm, _ := initModuleManager(t, "get__global_hook") + + var globalHook *GlobalHook + + tests := []struct { + name string + hookName string + testFn func() + }{ + { + "global-hook-with-all-bindings", + "000-all-bindings/hook", + func() { + require.NotNil(t, globalHook, "Global hook '000-all-bindings/hook' should be registered") + + assert.Equal(t, "000-all-bindings/hook", globalHook.Name) + assert.NotNil(t, globalHook.Config) + assert.Equal(t, []BindingType{OnStartup, Schedule, OnKubernetesEvent, BeforeAll, AfterAll}, globalHook.Config.Bindings()) + assert.Len(t, globalHook.Config.OnKubernetesEvents, 3, "Should register 3 kubernetes bindings") + assert.Len(t, globalHook.Config.Schedules, 1, "Should register 1 schedule binding") + + // Schedule binding is 'every minute' with allow failure. + schBinding := globalHook.Config.Schedules[0] + assert.Equal(t, "* * * * *", schBinding.ScheduleEntry.Crontab) + assert.True(t, schBinding.AllowFailure) + + kBinding := globalHook.Config.OnKubernetesEvents[0] + assert.True(t, kBinding.AllowFailure) + assert.NotNil(t, kBinding.Monitor) + assert.NotNil(t, kBinding.Monitor.NamespaceSelector) + assert.NotNil(t, kBinding.Monitor.LabelSelector) + assert.Nil(t, kBinding.Monitor.NameSelector) + assert.Nil(t, kBinding.Monitor.FieldSelector) + assert.Equal(t, "configmap", kBinding.Monitor.Kind) + assert.Equal(t, []types.WatchEventType{types.WatchEventAdded}, kBinding.Monitor.EventTypes) + + // Binding without executeHookOnEvent should have all events + kBinding = globalHook.Config.OnKubernetesEvents[1] + assert.True(t, kBinding.AllowFailure) + assert.Equal(t, []types.WatchEventType{types.WatchEventAdded, types.WatchEventModified, types.WatchEventDeleted}, kBinding.Monitor.EventTypes) + + // Binding without allowFailure + kBinding = globalHook.Config.OnKubernetesEvents[2] + assert.False(t, kBinding.AllowFailure) + }, + }, + { + "global-hook-nested", + "100-nested-hook/sub/sub/hook", + func() { + require.NotNil(t, globalHook, "Global hook '100-nested-hook/sub/sub/hook' should be registered") + assert.Equal(t, "100-nested-hook/sub/sub/hook", globalHook.Name) + assert.NotNil(t, globalHook.Config) + assert.Equal(t, []BindingType{BeforeAll}, globalHook.Config.Bindings()) + }, + }, + { + "nil-if-global-hook-not-registered", + "non-existent-hook-name", + func() { + assert.Nil(t, globalHook) + }, + }, } - mm := NewMainModuleManager() - initModuleManager(t, mm, "get__global_hooks_in_order") + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + globalHook = mm.GetGlobalHook(test.hookName) + test.testFn() + }) + } +} + +// This test checks sorting 'beforeAll' hooks in specified order. +func Test_ModuleManager_Get_GlobalHooksInOrder(t *testing.T) { + mm, _ := initModuleManager(t, "get__global_hooks_in_order") var expectations = []struct { testName string @@ -1052,14 +876,13 @@ func Test_MainModuleManager_Get_GlobalHooksInOrder(t *testing.T) { } } -func Test_MainModuleManager_Run_GlobalHook(t *testing.T) { - helm.NewClient = func(logLabels ...map[string]string) client.HelmClient { - return &helm.MockHelmClient{} - } - mm := NewMainModuleManager() - mm.WithKubeConfigManager(MockKubeConfigManager{}) - - initModuleManager(t, mm, "test_run_global_hook") +// Global hooks in test_run_global_hook path: +// - 000-update-kube-config hook that add and remove some config values. +// - 100-update-dynamic hook that add some dynamic values. +// +// Test runs global hooks and checks resulting values. +func Test_ModuleManager_Run_GlobalHook(t *testing.T) { + mm, res := initModuleManager(t, "test_run_global_hook") expectations := []struct { testName string @@ -1146,13 +969,11 @@ func Test_MainModuleManager_Run_GlobalHook(t *testing.T) { for _, expectation := range expectations { t.Run(expectation.testName, func(t *testing.T) { - mm.kubeGlobalConfigValues = expectation.kubeGlobalConfigValues - mm.globalDynamicValuesPatches = expectation.globalDynamicValuesPatches + res.moduleManager.kubeGlobalConfigValues = expectation.kubeGlobalConfigValues + res.moduleManager.globalDynamicValuesPatches = expectation.globalDynamicValuesPatches _, _, err := mm.RunGlobalHook(expectation.hookName, BeforeAll, []BindingContext{}, map[string]string{}) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err, "Hook %s should not fail", expectation.hookName) var configValues = mm.GlobalConfigValues() if !reflect.DeepEqual(expectation.expectedConfigValues, configValues) { @@ -1160,9 +981,7 @@ func Test_MainModuleManager_Run_GlobalHook(t *testing.T) { } values, err := mm.GlobalValues() - if !assert.NoError(t, err) { - t.FailNow() - } + require.NoError(t, err, "Should collect effective global values") if !reflect.DeepEqual(expectation.expectedValues, values) { t.Errorf("\n[EXPECTED]: %#v\n[GOT]: %#v", spew.Sdump(expectation.expectedValues), spew.Sdump(values)) } @@ -1170,7 +989,11 @@ func Test_MainModuleManager_Run_GlobalHook(t *testing.T) { } } -func Test_MainModuleManager_DiscoverModulesState(t *testing.T) { +// Test modules in discover_modules_state_* paths: +// __simple - 6 modules , 2 modules should be disabled on startup. +// __with_enabled_scripts - check running enabled scripts. +// __module_names_order - check modules order. +func Test_ModuleManager_ModulesState_no_ConfigMap(t *testing.T) { var mm *moduleManager var modulesState *ModulesState var err error @@ -1179,47 +1002,130 @@ func Test_MainModuleManager_DiscoverModulesState(t *testing.T) { name string configPath string helmReleases []string - testFn func() + testFn func(t *testing.T) }{ { "static_config_and_helm_releases", - "discover_modules_state__simple", + "modules_state__no_cm__simple", []string{"module-1", "module-2", "module-3", "module-5", "module-6", "module-9"}, - func() { - if assert.NoError(t, err) { - assert.Equal(t, []string{"module-1", "module-4", "module-8"}, mm.enabledModulesByConfig) - assert.Equal(t, []string{"module-6", "module-5", "module-2"}, modulesState.ReleasedUnknownModules) - assert.Equal(t, []string{"module-9", "module-3"}, modulesState.ModulesToDisable) - } + func(t *testing.T) { + // At start: + // - module-1, module-4 and module-8 are enabled by values.yaml - they should be in 'enabledByConfig' list. + // After RefreshFromHelmReleases: + // - module-1, module-3 and module-9 are in releases, they should be in AllEnabled list. + // - module-2, module-5, and module-6 are unknown, they should be in 'purge' list. + // After RefreshEnabledState: + // - module-1, module-4 and module-8 should be in AllEnabled list. + // - module-4 and module-8 should be in 'enabled' list. (module-1 is already enabled as helm release) + // - module-9 and module-3 should be in 'disabled' list. (releases are present, but modules are disabled) + + expectEnabledByConfig := map[string]struct{}{"module-1": {}, "module-4": {}, "module-8": {}} + require.Equal(t, expectEnabledByConfig, mm.enabledModulesByConfig) + + expectAllEnabled := []string{"module-1", "module-3", "module-9"} + // Note: purge in reversed order + expectToPurge := []string{"module-6", "module-5", "module-2"} + require.Equal(t, expectAllEnabled, modulesState.AllEnabledModules) + require.Equal(t, expectToPurge, modulesState.ModulesToPurge) + require.Len(t, modulesState.ModulesToEnable, 0) + require.Len(t, modulesState.ModulesToDisable, 0) + require.Len(t, modulesState.ModulesToReload, 0) + + modulesState, err := mm.RefreshEnabledState(map[string]string{}) + expectAllEnabled = []string{"module-1", "module-4", "module-8"} + expectToEnable := []string{"module-4", "module-8"} + // Note: 'disable' list should be in reverse order. + expectToDisable := []string{"module-9", "module-3"} + require.NoError(t, err, "Should refresh enabled state") + require.Equal(t, expectAllEnabled, modulesState.AllEnabledModules) + require.Equal(t, expectToEnable, modulesState.ModulesToEnable) + require.Equal(t, expectToDisable, modulesState.ModulesToDisable) }, }, { "enabled_script", - "discover_modules_state__with_enabled_scripts", + "modules_state__no_cm__with_enabled_scripts", []string{}, - func() { - // If all modules are enabled by default, then beta should be disabled by script. - assert.Equal(t, []string{"alpha", "gamma", "delta", "epsilon", "zeta", "eta"}, modulesState.EnabledModules) + func(t *testing.T) { + // At start: + // - alpha, beta, gamma, delta, epsilon, zeta, eta are enabled by values.yaml - they should be in 'enabledByConfig' list. + // After RefreshFromHelmReleases: + // - No helm releases -> empty lists in modulesState. + // After RefreshEnabledState: + // - alpha enabled, + // - beta disabled + // - gamma requires alpha -> enabled + // - delta requires alpha -> enabled + // - epsilon enabled + // - zeta requires delta and gamma -> enabled + // - eta enabled + // No beta in AllEnabled list. + // After disable alpha with dynamicEnabled and RefreshEnabledState: + // - all modules that depend on alpha should be disabled, so: + // - epsilon and eta should be in AllEnabled list. + + expectEnabledByConfig := map[string]struct{}{ + "alpha": {}, + "beta": {}, + "gamma": {}, + "delta": {}, + "epsilon": {}, + "zeta": {}, + "eta": {}, + } + + require.Equal(t, expectEnabledByConfig, mm.enabledModulesByConfig, "All modules should be enabled by config at start") + + // No helm releases, modulesState is empty after RefreshFromHelmReleases. + require.Len(t, modulesState.AllEnabledModules, 0) + require.Len(t, modulesState.ModulesToPurge, 0) - // Turn off alpha so gamma, delta and zeta should become disabled - // with the next call to DiscoverModulesState. + modulesState, err := mm.RefreshEnabledState(map[string]string{}) + require.NoError(t, err, "Should refresh enabled state") + + require.NotContains(t, modulesState.AllEnabledModules, "beta", "Should not return disabled 'beta' module") + expectAllEnabled := []string{"alpha", "gamma", "delta", "epsilon", "zeta", "eta"} + require.Equal(t, expectAllEnabled, modulesState.AllEnabledModules) + + // Turn off module 'alpha'. mm.dynamicEnabled["alpha"] = &utils.ModuleDisabled - modulesState, err = mm.DiscoverModulesState(map[string]string{}) - assert.Equal(t, []string{"epsilon", "eta"}, modulesState.EnabledModules) + modulesState, err = mm.RefreshEnabledState(map[string]string{}) + require.NoError(t, err, "Should refresh enabled state") + expectAllEnabled = []string{"epsilon", "eta"} + require.Equal(t, expectAllEnabled, modulesState.AllEnabledModules) + // Note: 'disable' list should be in reverse order. + expectToDisable := []string{"zeta", "delta", "gamma", "alpha"} + require.Equal(t, expectToDisable, modulesState.ModulesToDisable) + require.Len(t, modulesState.ModulesToEnable, 0) }, }, { "module_names_in_order", - "discover_modules_state__module_names_order", + "modules_state__no_cm__module_names_order", []string{}, - func() { - expectedModules := []string{ + func(t *testing.T) { + // At start: + // - module-c, module-b are enabled by config. + // After RefreshEnabledState: + // Enabled modules are not changed. + expectEnabledByConfig := map[string]struct{}{ + "module-c": {}, + "module-b": {}, + } + require.Equal(t, expectEnabledByConfig, mm.enabledModulesByConfig, "All modules should be enabled by config at start") + + // No helm releases, modulesState is empty after RefreshFromHelmReleases. + require.Len(t, modulesState.AllEnabledModules, 0) + require.Len(t, modulesState.ModulesToPurge, 0) + + modulesState, err := mm.RefreshEnabledState(map[string]string{}) + require.NoError(t, err, "Should refresh enabled state") + + expectAllEnabled := []string{ "module-c", "module-b", } - // if all modules are enabled by default, then beta should be disabled by script - assert.Equal(t, expectedModules, modulesState.EnabledModules) - + require.Equal(t, expectAllEnabled, modulesState.AllEnabledModules) }, }, } @@ -1229,18 +1135,182 @@ func Test_MainModuleManager_DiscoverModulesState(t *testing.T) { modulesState = nil err = nil - helm.NewClient = func(logLabels ...map[string]string) client.HelmClient { - return &helm.MockHelmClient{ - ReleaseNames: test.helmReleases, - } - } - mm = NewMainModuleManager() - initModuleManager(t, mm, test.configPath) + _, res := initModuleManager(t, test.configPath) + require.NoError(t, res.initialStateErr, "Should load ConfigMap state") + mm = res.moduleManager - modulesState, err = mm.DiscoverModulesState(map[string]string{}) + mm.WithHelm(helm.MockHelm(&helm.MockHelmClient{ + ReleaseNames: test.helmReleases, + })) - test.testFn() + modulesState, err = mm.RefreshStateFromHelmReleases(map[string]string{}) + require.NoError(t, err, "Should refresh from helm releases") + require.NotNil(t, modulesState, "Should have state after refresh from helm releases") + test.testFn(t) }) } } + +// Modules in modules_state__purge: +// - 'module-one' is enabled by common config. +// Initial state: +// - 'module-one', 'module-two', and 'module-three' are present as helm releases. +// - 'module-three' is disabled in ConfigMap. +// This test should detect 'module-two' as a helm release to purge and +// 'module-three' as a module to delete. +// 'module-one' should present in enabledModulesByConfig and enabledModules caches. +// 'module-three' should present in enabledModulesByConfig. +func Test_ModuleManager_ModulesState_detect_ConfigMap_changes(t *testing.T) { + var state *ModulesState + var err error + + mm, res := initModuleManager(t, "modules_state__detect_cm_changes") + require.NoError(t, res.initialStateErr, "Should load initial config from ConfigMap") + + // enabledModulesByConfig is filled from values.yaml and ConfigMap data. + require.Len(t, res.moduleManager.enabledModulesByConfig, 1) + require.Contains(t, res.moduleManager.enabledModulesByConfig, "module-one") + + // RefreshStateFromHelmReleases should detect all modules from helm releases as enabled. + mm.WithHelm(helm.MockHelm(&helm.MockHelmClient{ + ReleaseNames: []string{"module-one", "module-two", "module-three"}, + })) + state, err = mm.RefreshStateFromHelmReleases(map[string]string{}) + require.NoError(t, err, "RefreshStateFromHelmReleases should not fail") + require.NotNil(t, state) + require.Len(t, state.AllEnabledModules, 2) + require.Contains(t, state.AllEnabledModules, "module-one") + require.Contains(t, state.AllEnabledModules, "module-three") + require.Equal(t, []string{"module-two"}, state.ModulesToPurge) + + // No modules to enable, no modules to disable. + state, err = mm.RefreshEnabledState(map[string]string{}) + require.NoError(t, err, "RefreshStateFromHelmReleases should not fail") + require.NotNil(t, state) + require.Equal(t, []string{"module-one"}, state.AllEnabledModules) + require.Len(t, state.ModulesToEnable, 0) + require.Len(t, state.ModulesToReload, 0) + require.Len(t, state.ModulesToPurge, 0) + require.Equal(t, []string{"module-three"}, state.ModulesToDisable) + + // Change ConfigMap: patch moduleThreeEnabled field, detect enabled module. + { + moduleThreeEnabledPatch := ` +[{ +"op": "replace", +"path": "/data/moduleThreeEnabled", +"value": "true"}]` + + _, err := res.kubeClient.CoreV1().ConfigMaps(res.cmNamespace).Patch(context.TODO(), + res.cmName, + k8types.JSONPatchType, + []byte(moduleThreeEnabledPatch), + metav1.PatchOptions{}, + ) + require.NoError(t, err, "ConfigMap should be patched") + + // Emulate ConvergeModules task: Wait for event, handle new ConfigMap, refresh enabled state. + <-res.kubeConfigManager.KubeConfigEventCh() + + var state *ModulesState + res.kubeConfigManager.SafeReadConfig(func(config *kube_config_manager.KubeConfig) { + state, err = mm.HandleNewKubeConfig(config) + }) + require.Len(t, state.ModulesToReload, 0, "Enabled flag change should lead to reload all modules") + + // module-one and module-three should be enabled. + state, err = mm.RefreshEnabledState(map[string]string{}) + require.NoError(t, err, "Should refresh state") + require.NotNil(t, state, "Should return state") + require.Len(t, state.AllEnabledModules, 2) + require.Contains(t, state.AllEnabledModules, "module-one") + require.Contains(t, state.AllEnabledModules, "module-three") + // module-three is a newly enabled module. + require.Len(t, state.ModulesToEnable, 1) + require.Contains(t, state.ModulesToEnable, "module-three") + } + + // Change ConfigMap: patch moduleOne and moduleThree section, detect reload modules. + { + moduleValuesChangePatch := ` +[{ +"op": "replace", +"path": "/data/moduleOne", +"value": "param: newValue"},{ +"op": "replace", +"path": "/data/moduleThree", +"value": "param: newValue"}]` + + _, err := res.kubeClient.CoreV1().ConfigMaps(res.cmNamespace).Patch(context.TODO(), + res.cmName, + k8types.JSONPatchType, + []byte(moduleValuesChangePatch), + metav1.PatchOptions{}, + ) + require.NoError(t, err, "ConfigMap should be patched") + + // Emulate ConvergeModules task: Wait for event, handle new ConfigMap, refresh enabled state. + <-res.kubeConfigManager.KubeConfigEventCh() + + var state *ModulesState + res.kubeConfigManager.SafeReadConfig(func(config *kube_config_manager.KubeConfig) { + state, err = mm.HandleNewKubeConfig(config) + }) + require.Len(t, state.ModulesToReload, 2, "Enabled flag change should lead to reload all modules") + require.Contains(t, state.ModulesToReload, "module-one") + require.Contains(t, state.ModulesToReload, "module-three") + + // module-one and module-three should be enabled. + state, err = mm.RefreshEnabledState(map[string]string{}) + require.NoError(t, err, "Should refresh state") + require.NotNil(t, state, "Should return state") + require.Len(t, state.AllEnabledModules, 2) + require.Contains(t, state.AllEnabledModules, "module-one") + require.Contains(t, state.AllEnabledModules, "module-three") + // Should be no changes in modules state. + require.Len(t, state.ModulesToEnable, 0) + require.Len(t, state.ModulesToDisable, 0) + require.Len(t, state.ModulesToPurge, 0) + require.Len(t, state.ModulesToReload, 0) + } + + // Change ConfigMap: remove moduleThreeEnabled field, detect no changes, as module-three is enabled by values.yaml. + { + moduleThreeEnabledPatch := ` +[{ +"op": "remove", +"path": "/data/moduleThreeEnabled"}]` + + _, err := res.kubeClient.CoreV1().ConfigMaps(res.cmNamespace).Patch(context.TODO(), + res.cmName, + k8types.JSONPatchType, + []byte(moduleThreeEnabledPatch), + metav1.PatchOptions{}, + ) + require.NoError(t, err, "ConfigMap should be patched") + + // Emulate ConvergeModules task: Wait for event, handle new ConfigMap, refresh enabled state. + <-res.kubeConfigManager.KubeConfigEventCh() + + var state *ModulesState + res.kubeConfigManager.SafeReadConfig(func(config *kube_config_manager.KubeConfig) { + state, err = mm.HandleNewKubeConfig(config) + }) + require.NoError(t, err, "Should handle new ConfigMap") + require.Nil(t, state, "Should be no changes in state from new ConfigMap") + + // module-one and module-three should still be enabled. + state, err = mm.RefreshEnabledState(map[string]string{}) + require.NoError(t, err, "Should refresh state") + require.NotNil(t, state, "Should return state") + require.Len(t, state.AllEnabledModules, 2) + require.Contains(t, state.AllEnabledModules, "module-one") + require.Contains(t, state.AllEnabledModules, "module-three") + // Should be no changes in modules state. + require.Len(t, state.ModulesToEnable, 0) + require.Len(t, state.ModulesToDisable, 0) + require.Len(t, state.ModulesToPurge, 0) + require.Len(t, state.ModulesToReload, 0) + } +} diff --git a/pkg/module_manager/synchronization_state.go b/pkg/module_manager/synchronization_state.go index 46601007..d997199c 100644 --- a/pkg/module_manager/synchronization_state.go +++ b/pkg/module_manager/synchronization_state.go @@ -1,13 +1,16 @@ package module_manager import ( - log "github.com/sirupsen/logrus" + "fmt" "sync" + log "github.com/sirupsen/logrus" + "github.com/flant/addon-operator/pkg/task" ) -// KubernetesBindingSynchronizationState is a state of the single Synchronization task. +// KubernetesBindingSynchronizationState is a state of the single Synchronization task +// for one kubernetes binding. type KubernetesBindingSynchronizationState struct { HookName string BindingName string @@ -15,7 +18,13 @@ type KubernetesBindingSynchronizationState struct { Done bool } -// SynchronizationState can be used to track synchronization tasks for global or for module. +func (k *KubernetesBindingSynchronizationState) String() string { + return fmt.Sprintf("queue=%v done=%v", k.Queued, k.Done) +} + +// SynchronizationState stores state to track synchronization +// tasks for kubernetes bindings either for all global hooks +// or for module's hooks. type SynchronizationState struct { state map[string]*KubernetesBindingSynchronizationState m sync.RWMutex diff --git a/pkg/module_manager/testdata/discover_modules_state__module_names_order/config_map.yaml b/pkg/module_manager/testdata/discover_modules_state__module_names_order/config_map.yaml deleted file mode 100644 index 5ae6cc28..00000000 --- a/pkg/module_manager/testdata/discover_modules_state__module_names_order/config_map.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: addon-operator -data: - global: {} \ No newline at end of file diff --git a/pkg/module_manager/testdata/get__global_hook/global-hooks/000-all-bindings/all b/pkg/module_manager/testdata/get__global_hook/global-hooks/000-all-bindings/all deleted file mode 100755 index a83a0dc7..00000000 --- a/pkg/module_manager/testdata/get__global_hook/global-hooks/000-all-bindings/all +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -e - -if [[ "$1" == "--config" ]]; then - cat << EndOfJSON -{ - "afterAll": 1, - "beforeAll": 1, - "onStartup": 1, - "schedule": [{ - "crontab": "* * * * *", - "allowFailure": true - }], - "onKubernetesEvent": [{ - "event": ["add"], - "kind": "configmap", - "selector": { - "matchLabels": { - "component": "component1" - }, - "matchExpressions": [{ - "key": "tier", - "operator": "In", - "values": ["cache"] - }] - }, - "namespaceSelector": { - "matchNames": ["namespace1"], - "any": false - }, - "jqFilter": ".items[] | del(.metadata, .field1)", - "allowFailure": true - }, - { - "kind": "namespace", - "selector": { - "matchLabels": { - "component": "component2" - }, - "matchExpressions": [{ - "key": "tier", - "operator": "In", - "values": ["cache"] - }] - }, - "namespaceSelector": { - "matchNames": ["namespace2"], - "any": false - }, - "jqFilter": ".items[] | del(.metadata, .field2)", - "allowFailure": true - }, - { - "kind": "pod", - "selector": { - "matchLabels": { - "component": "component3" - }, - "matchExpressions": [{ - "key": "tier", - "operator": "In", - "values": ["cache"] - }] - }, - "jqFilter": ".items[] | del(.metadata, .field3)", - "allowFailure": true - }] -} -EndOfJSON -fi diff --git a/pkg/module_manager/testdata/get__global_hook/global-hooks/000-all-bindings/hook b/pkg/module_manager/testdata/get__global_hook/global-hooks/000-all-bindings/hook new file mode 100755 index 00000000..adb4a3c5 --- /dev/null +++ b/pkg/module_manager/testdata/get__global_hook/global-hooks/000-all-bindings/hook @@ -0,0 +1,56 @@ +#!/bin/bash -e + +if [[ "$1" == "--config" ]]; then + cat << EOF +configVersion: v1 +onStartup: 1 +afterAll: 1 +beforeAll: 1 +schedule: +- crontab: "* * * * *" + allowFailure: true + +kubernetes: +- executeHookOnEvent: ["Added"] + kind: "configmap" + labelSelector: + matchLabels: + component: component1 + matchExpressions: + - key: "tier" + operator: "In" + values: ["cache"] + namespace: + nameSelector: + matchNames: ["namespace1"] + jqFilter: ".items[] | del(.metadata, .field1)" + allowFailure: true + +- kind: "namespace" + labelSelector: + matchLabels: + component: component2 + matchExpressions: + - key: "tier" + operator: "In" + values: ["cache"] + namespace: + nameSelector: + matchNames: ["namespace2"] + jqFilter: ".items[] | del(.metadata, .field2)" + allowFailure: true + +- kind: "pod" + labelSelector: + matchLabels: + component: component3 + matchExpressions: + - key: "tier" + operator: "In" + values: ["cache"] + namespace: + nameSelector: + matchNames: ["namespace2"] + jqFilter: ".items[] | del(.metadata, .field3)" +EOF +fi diff --git a/pkg/module_manager/testdata/get__global_hook/global-hooks/100-nested-hook/sub/sub/nested-before-all b/pkg/module_manager/testdata/get__global_hook/global-hooks/100-nested-hook/sub/sub/hook similarity index 100% rename from pkg/module_manager/testdata/get__global_hook/global-hooks/100-nested-hook/sub/sub/nested-before-all rename to pkg/module_manager/testdata/get__global_hook/global-hooks/100-nested-hook/sub/sub/hook diff --git a/pkg/module_manager/testdata/get__module_hook/modules/000-all-bindings/hooks/all b/pkg/module_manager/testdata/get__module_hook/modules/000-all-bindings/hooks/all deleted file mode 100755 index 1a9f8064..00000000 --- a/pkg/module_manager/testdata/get__module_hook/modules/000-all-bindings/hooks/all +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -e - -if [[ "$1" == "--config" ]]; then - cat << EndOfJSON -{ - "afterHelm": 1, - "beforeHelm": 1, - "afterDeleteHelm": 1, - "onStartup": 1, - "schedule": [{ - "crontab": "* * * * *", - "allowFailure": true - }], - "onKubernetesEvent": [{ - "event": ["add"], - "kind": "configmap", - "selector": { - "matchLabels": { - "component": "component1" - }, - "matchExpressions": [{ - "key": "tier", - "operator": "In", - "values": ["cache"] - }] - }, - "namespaceSelector": { - "matchNames": ["namespace1"], - "any": false - }, - "jqFilter": ".items[] | del(.metadata, .field1)", - "allowFailure": true - }, - { - "kind": "namespace", - "selector": { - "matchLabels": { - "component": "component2" - }, - "matchExpressions": [{ - "key": "tier", - "operator": "In", - "values": ["cache"] - }] - }, - "namespaceSelector": { - "matchNames": ["namespace2"], - "any": false - }, - "jqFilter": ".items[] | del(.metadata, .field2)", - "allowFailure": true - }, - { - "kind": "pod", - "selector": { - "matchLabels": { - "component": "component3" - }, - "matchExpressions": [{ - "key": "tier", - "operator": "In", - "values": ["cache"] - }] - }, - "jqFilter": ".items[] | del(.metadata, .field3)", - "allowFailure": true - }] -} -EndOfJSON -fi diff --git a/pkg/module_manager/testdata/get__module_hook/modules/000-all-bindings/hooks/all-bindings b/pkg/module_manager/testdata/get__module_hook/modules/000-all-bindings/hooks/all-bindings new file mode 100755 index 00000000..5b297ab2 --- /dev/null +++ b/pkg/module_manager/testdata/get__module_hook/modules/000-all-bindings/hooks/all-bindings @@ -0,0 +1,58 @@ +#!/bin/bash -e + +if [[ "$1" == "--config" ]]; then + cat << EOF +configVersion: v1 +onStartup: 1 +afterHelm: 1 +beforeHelm: 1 +afterDeleteHelm: 1 + +schedule: +- crontab: "* * * * *" + allowFailure: true + +kubernetes: +- executeHookOnEvent: ["Added"] + kind: "configmap" + labelSelector: + matchLabels: + component: component1 + matchExpressions: + - key: "tier" + operator: "In" + values: ["cache"] + namespace: + nameSelector: + matchNames: ["namespace1"] + jqFilter: ".items[] | del(.metadata, .field1)" + allowFailure: true + +- kind: "namespace" + labelSelector: + matchLabels: + component: component2 + matchExpressions: + - key: "tier" + operator: "In" + values: ["cache"] + namespace: + nameSelector: + matchNames: ["namespace2"] + jqFilter: ".items[] | del(.metadata, .field2)" + allowFailure: true + +- kind: "pod" + labelSelector: + matchLabels: + component: component3 + matchExpressions: + - key: "tier" + operator: "In" + values: ["cache"] + namespace: + nameSelector: + matchNames: ["namespace2"] + jqFilter: ".items[] | del(.metadata, .field3)" +EOF +fi diff --git a/pkg/module_manager/testdata/get__module_hook/modules/100-nested-hooks/hooks/sub/sub/nested-before-helm b/pkg/module_manager/testdata/get__module_hook/modules/100-nested-hooks/hooks/sub/sub/nested-before-helm index 257eb668..5cc5433c 100755 --- a/pkg/module_manager/testdata/get__module_hook/modules/100-nested-hooks/hooks/sub/sub/nested-before-helm +++ b/pkg/module_manager/testdata/get__module_hook/modules/100-nested-hooks/hooks/sub/sub/nested-before-helm @@ -1,9 +1,8 @@ #!/bin/bash -e if [[ "$1" == "--config" ]]; then - echo " -{ - \"beforeHelm\": 1 -} -" + cat < "$VALUES_JSON_PATCH_PATH" [ - { "op": "add", "path": "/module/afterDeleteHelm", "value": "override-value" } + { "op": "add", "path": "/module/afterDeleteHelm", "value": "value-from-after-delete-helm-20" } ] EOF fi diff --git a/pkg/module_manager/testdata/test_delete_module/modules/000-module/hooks/hook-2 b/pkg/module_manager/testdata/test_delete_module/modules/000-module/hooks/hook-2 index 01cbe92d..9c03bfa4 100755 --- a/pkg/module_manager/testdata/test_delete_module/modules/000-module/hooks/hook-2 +++ b/pkg/module_manager/testdata/test_delete_module/modules/000-module/hooks/hook-2 @@ -1,15 +1,15 @@ #!/bin/bash -e if [[ "$1" == "--config" ]]; then - echo " -{ - \"afterDeleteHelm\": 1 -} -" + cat < "$VALUES_JSON_PATCH_PATH" [ - { "op": "add", "path": "/module/afterDeleteHelm", "value": "value" } + { "op": "add", "path": "/module/afterDeleteHelm", "value": "value-from-after-delete-helm-10" } ] EOF fi diff --git a/pkg/module_manager/testdata/test_delete_module/modules/000-module/values.yaml b/pkg/module_manager/testdata/test_delete_module/modules/000-module/values.yaml index 5edc6c28..538ffcbe 100644 --- a/pkg/module_manager/testdata/test_delete_module/modules/000-module/values.yaml +++ b/pkg/module_manager/testdata/test_delete_module/modules/000-module/values.yaml @@ -3,9 +3,4 @@ # Declare variables to be passed into your templates. module: - replicaCount: 1 - - image: - repository: nginx - tag: stable - pullPolicy: IfNotPresent + imageName: "nginx:stable" diff --git a/pkg/module_manager/testdata/test_run_global_hook/global-hooks/000-update-kube-config/merge_and_patch_values b/pkg/module_manager/testdata/test_run_global_hook/global-hooks/000-update-kube-config/merge_and_patch_values index 3585f29e..f30fce50 100755 --- a/pkg/module_manager/testdata/test_run_global_hook/global-hooks/000-update-kube-config/merge_and_patch_values +++ b/pkg/module_manager/testdata/test_run_global_hook/global-hooks/000-update-kube-config/merge_and_patch_values @@ -1,11 +1,10 @@ #!/bin/bash -e if [[ "$1" == "--config" ]]; then - echo " -{ - \"beforeAll\": 1 -} -" + cat < "$CONFIG_VALUES_JSON_PATCH_PATH" [ diff --git a/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-1 b/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-1 index 2f768623..f4c03461 100755 --- a/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-1 +++ b/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-1 @@ -1,15 +1,14 @@ #!/bin/bash -e if [[ "$1" == "--config" ]]; then - echo " -{ - \"beforeHelm\": 2 -} -" + cat < "$VALUES_JSON_PATCH_PATH" [ - { "op": "add", "path": "/module/beforeHelm", "value": "override-value" } + { "op": "add", "path": "/module/beforeHelm", "value": "value-from-before-helm-20" } ] EOF fi diff --git a/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-2 b/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-2 index 421413b6..0515335e 100755 --- a/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-2 +++ b/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-2 @@ -1,15 +1,14 @@ #!/bin/bash -e if [[ "$1" == "--config" ]]; then - echo " -{ - \"afterHelm\": 2 -} -" + cat < "$VALUES_JSON_PATCH_PATH" [ - { "op": "add", "path": "/module/afterHelm", "value": "override-value" } + { "op": "add", "path": "/module/afterHelm", "value": "value-from-before-helm-2" } ] EOF fi diff --git a/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-3 b/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-3 index 17f602b2..5d8cbd6a 100755 --- a/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-3 +++ b/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-3 @@ -1,15 +1,14 @@ #!/bin/bash -e if [[ "$1" == "--config" ]]; then - echo " -{ - \"beforeHelm\": 1 -} -" + cat < "$VALUES_JSON_PATCH_PATH" [ - { "op": "add", "path": "/module/beforeHelm", "value": "value" } + { "op": "add", "path": "/module/beforeHelm", "value": "value-from-before-helm-1" } ] EOF fi diff --git a/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-4 b/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-4 index f77329ed..99dbd35d 100755 --- a/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-4 +++ b/pkg/module_manager/testdata/test_run_module/modules/000-module/hooks/hook-4 @@ -1,15 +1,14 @@ #!/bin/bash -e if [[ "$1" == "--config" ]]; then - echo " -{ - \"afterHelm\": 1 -} -" + cat < "$VALUES_JSON_PATCH_PATH" [ - { "op": "add", "path": "/module/afterHelm", "value": "value" } + { "op": "add", "path": "/module/afterHelm", "value": "value-from-after-helm" } ] EOF fi diff --git a/pkg/module_manager/testdata/test_run_module/modules/000-module/values.yaml b/pkg/module_manager/testdata/test_run_module/modules/000-module/values.yaml index 5edc6c28..9ef24b92 100644 --- a/pkg/module_manager/testdata/test_run_module/modules/000-module/values.yaml +++ b/pkg/module_manager/testdata/test_run_module/modules/000-module/values.yaml @@ -1,11 +1,3 @@ # Default values for 000-module. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - module: - replicaCount: 1 - - image: - repository: nginx - tag: stable - pullPolicy: IfNotPresent + imageName: "nginx:stable" diff --git a/pkg/module_manager/testdata/test_run_module_hook/modules/000-update-kube-module-config/hooks/merge_and_patch_values b/pkg/module_manager/testdata/test_run_module_hook/modules/000-update-kube-module-config/hooks/merge_and_patch_values index 9ef47062..29003178 100755 --- a/pkg/module_manager/testdata/test_run_module_hook/modules/000-update-kube-module-config/hooks/merge_and_patch_values +++ b/pkg/module_manager/testdata/test_run_module_hook/modules/000-update-kube-module-config/hooks/merge_and_patch_values @@ -1,11 +1,10 @@ #!/bin/bash -e if [[ "$1" == "--config" ]]; then - echo " -{ - \"beforeHelm\": 1 -} -" + cat < "$CONFIG_VALUES_JSON_PATCH_PATH" [ diff --git a/pkg/module_manager/testdata/test_run_module_hook/modules/100-update-module-dynamic/hooks/merge_and_patch_values b/pkg/module_manager/testdata/test_run_module_hook/modules/100-update-module-dynamic/hooks/merge_and_patch_values index db8e0da5..7e3e509d 100755 --- a/pkg/module_manager/testdata/test_run_module_hook/modules/100-update-module-dynamic/hooks/merge_and_patch_values +++ b/pkg/module_manager/testdata/test_run_module_hook/modules/100-update-module-dynamic/hooks/merge_and_patch_values @@ -1,11 +1,10 @@ #!/bin/bash -e if [[ "$1" == "--config" ]]; then - echo " -{ - \"beforeHelm\": 1 -} -" + cat < "$VALUES_JSON_PATCH_PATH" [ diff --git a/pkg/task/hook_metadata.go b/pkg/task/hook_metadata.go index 604c3306..596eb507 100644 --- a/pkg/task/hook_metadata.go +++ b/pkg/task/hook_metadata.go @@ -22,12 +22,13 @@ type HookMetadata struct { BindingContext []BindingContext AllowFailure bool //Task considered as 'ok' if hook failed. False by default. Can be true for some schedule hooks. + OnStartup bool // Execute onStartup and kubernetes@Synchronization hooks for module OnStartupHooks bool // Execute onStartup and kubernetes@Synchronization hooks for module ValuesChecksum string // checksum of global values before first afterAll hook execution DynamicEnabledChecksum string // checksum of dynamicEnabled before first afterAll hook execution LastAfterAllHook bool // true if task is a last afterAll hook in sequence - ReloadAllOnValuesChanges bool // whether or not run DiscoverModules process if hook change global values + ReloadAllOnValuesChanges bool // whether to run DiscoverModules process if hook change global values KubernetesBindingId string // Unique id for kubernetes bindings WaitForSynchronization bool // kubernetes.Synchronization task should be waited @@ -80,8 +81,8 @@ func (hm HookMetadata) GetDescription() string { if hm.HookName == "" { // module run osh := "" - if hm.OnStartupHooks { - osh = ":onStartupHooks" + if hm.OnStartup { + osh = ":onStartup" } return fmt.Sprintf("%s%s%s:%s", hm.ModuleName, osh, bindingNames, hm.EventDescription) } else { diff --git a/pkg/task/task.go b/pkg/task/task.go index 6023ff59..4df9f620 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -6,18 +6,24 @@ import ( // Addon-operator specific task types const ( - ModuleDelete task.TaskType = "ModuleDelete" - ModuleRun task.TaskType = "ModuleRun" - ModuleHookRun task.TaskType = "ModuleHookRun" - GlobalHookRun task.TaskType = "GlobalHookRun" - ReloadAllModules task.TaskType = "ReloadAllModules" - DiscoverModulesState task.TaskType = "DiscoverModulesState" + // GlobalHookRun runs a global hook. + GlobalHookRun task.TaskType = "GlobalHookRun" + // ModuleHookRun runs schedule or kubernetes hook. + ModuleHookRun task.TaskType = "ModuleHookRun" + // ModuleDelete runs helm delete/afterHelmDelete sequence. + ModuleDelete task.TaskType = "ModuleDelete" + // ModuleRun runs beforeHelm/helm upgrade/afterHelm sequence. + ModuleRun task.TaskType = "ModuleRun" + // ModulePurge - delete unknown helm release (no module in ModulesDir) + ModulePurge task.TaskType = "ModulePurge" + + // DiscoverHelmReleases lists helm releases to detect unknown modules and initiate enabled modules list. + DiscoverHelmReleases task.TaskType = "DiscoverHelmReleases" + + // ConvergeModules runs beforeAll/run modules/afterAll sequence for all enabled modules. + ConvergeModules task.TaskType = "ConvergeModules" GlobalHookEnableKubernetesBindings task.TaskType = "GlobalHookEnableKubernetesBindings" GlobalHookWaitKubernetesSynchronization task.TaskType = "GlobalHookWaitKubernetesSynchronization" GlobalHookEnableScheduleBindings task.TaskType = "GlobalHookEnableScheduleBindings" - //ModuleHookEnableKubernetesBindings task.TaskType = "ModuleHookEnableKubernetesBindings" - - // Delete unknown helm release when no module in ModulesDir - ModulePurge task.TaskType = "ModulePurge" ) diff --git a/pkg/task/test/task_metadata_test.go b/pkg/task/test/task_metadata_test.go index 7a005df8..369e7b5c 100644 --- a/pkg/task/test/task_metadata_test.go +++ b/pkg/task/test/task_metadata_test.go @@ -51,9 +51,9 @@ func Test_TaskDescription(t *testing.T) { "module run", HookMetadata{ ModuleName: "module-name", - EventDescription: "PrepopulateMainQueue", + EventDescription: "BootstrapMainQueue", }, - "module-name:PrepopulateMainQueue", + "module-name:BootstrapMainQueue", }, { "module run with onStartup", diff --git a/pkg/utils/module_config.go b/pkg/utils/module_config.go index 61da05e3..3e010798 100644 --- a/pkg/utils/module_config.go +++ b/pkg/utils/module_config.go @@ -30,6 +30,9 @@ func (mc ModuleConfig) String() string { // GetEnabled returns string description of enabled status. func (mc *ModuleConfig) GetEnabled() string { + if mc == nil { + return "" + } switch { case mc.IsEnabled == nil: return "n/d" diff --git a/pkg/utils/module_list.go b/pkg/utils/module_list.go index 5a301f50..3b60f722 100644 --- a/pkg/utils/module_list.go +++ b/pkg/utils/module_list.go @@ -49,6 +49,20 @@ func SortByReference(in []string, ref []string) []string { return res } +// KeysSortedByReference returns keys from map sorted by the order of 'ref' array. +// Note: keys not in ref are ignored. +func KeysSortedByReference(m map[string]struct{}, ref []string) []string { + res := make([]string, 0) + + for _, v := range ref { + if _, hasKey := m[v]; hasKey { + res = append(res, v) + } + } + + return res +} + // ListSubtract creates a new array from 'src' array with items that are // not present in 'ignored' arrays. func ListSubtract(src []string, ignored ...[]string) (result []string) { @@ -133,3 +147,23 @@ func ListFullyIn(arr []string, ref []string) bool { return res } + +func MapStringStructKeys(m map[string]struct{}) []string { + res := make([]string, 0, len(m)) + + for k := range m { + res = append(res, k) + } + + return res +} + +func ListToMapStringStruct(items []string) map[string]struct{} { + res := make(map[string]struct{}) + + for _, item := range items { + res[item] = struct{}{} + } + + return res +} diff --git a/pkg/utils/values.go b/pkg/utils/values.go index 58a31b17..a0d4b035 100644 --- a/pkg/utils/values.go +++ b/pkg/utils/values.go @@ -209,3 +209,7 @@ func (v Values) YamlString() (string, error) { func (v Values) YamlBytes() ([]byte, error) { return v.AsBytes("yaml") } + +func (v Values) IsEmpty() bool { + return len(v) == 0 +}