From 1db84d660380d0aee5819ccac28e880f95140b64 Mon Sep 17 00:00:00 2001 From: Benjamin Pineau Date: Mon, 16 Apr 2018 21:39:45 +0200 Subject: [PATCH] Implement dump-only mode And by the way, waiting for the informer's caches to be fully dumped before stopping controllers is much saner. --- README.md | 23 +++++++++++++++-------- cmd/execute.go | 5 +++++ config/config.go | 3 +++ pkg/controller/controller.go | 19 +++++++++++++++++-- pkg/run/run.go | 4 +++- 5 files changed, 43 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 4a12683..58bf2c2 100644 --- a/README.md +++ b/README.md @@ -10,21 +10,27 @@ This provides real time, continuous backups, and keeps detailled changes history ## Usage +Just dump the cluster's content and exit: ```bash -katafygio --git-url https://user:token@github.com/myorg/myrepos.git --local-dir /tmp/kfdump +katafygio --dump-only --local-dir /tmp/clusterdump/ +``` + +Create a local git repository in /tmp/kfdump and continuously save the cluster content: +```bash +katafygio --local-dir /tmp/kfdump ``` -The remote git url is optional: if not specified, katafygio will create a local -repository in the target dump directory. katafygio may be used that way to -quickly dump a cluster's content. +Same, but also continously push the local repository to a remote: +```bash +katafygio --git-url https://user:token@github.com/myorg/myrepos.git --local-dir /tmp/kfdump +``` -Filtering out irrelevant objects (esp. ReplicaSets and Pods) with `-x` and `-y` -will help to keep memory usage low. Same for low value and frequently changing -objects, that may bloat the git history. +Filtering out irrelevant objects (esp. ReplicaSets and Pods) with `-x` or `-y` +will help to keep resources usage low and git history concise. ```bash -# Filtering out replicasets and pods since they are managed by Deployments +# Filtering out replicasets and pods since they are generated by Deployments # (already archived), endpoints (managed by Services), secrets (to keep them # confidential), events and node (irrelevant), and the leader-elector # configmap that has low value and changes a lot, causing commits churn. @@ -55,6 +61,7 @@ Flags: -s, --api-server string Kubernetes api-server url -c, --config string Configuration file (default "/etc/katafygio/katafygio.yaml") -d, --dry-run Dry-run mode: don't store anything. + -m, --dump-only Dump mode: dump everything and exit -x, --exclude-kind stringSlice Ressource kind to exclude. Eg. 'deployment' -y, --exclude-object stringSlice Object to exclude. Eg. 'configmap:kube-system/kube-dns' -l, --filter string Label filter. Select only objects matching the label. diff --git a/cmd/execute.go b/cmd/execute.go index b271062..f2322af 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -25,6 +25,7 @@ var ( apiServer string kubeConf string dryRun bool + dumpMode bool logLevel string logOutput string logServer string @@ -54,6 +55,7 @@ var ( RunE: func(cmd *cobra.Command, args []string) error { conf := &config.KfConfig{ DryRun: viper.GetBool("dry-run"), + DumpMode: viper.GetBool("dump-only"), Logger: klog.New(viper.GetString("log.level"), viper.GetString("log.server"), viper.GetString("log.output")), LocalDir: viper.GetString("local-dir"), GitURL: viper.GetString("git-url"), @@ -105,6 +107,9 @@ func init() { RootCmd.PersistentFlags().BoolVarP(&dryRun, "dry-run", "d", false, "Dry-run mode: don't store anything.") bindPFlag("dry-run", "dry-run") + RootCmd.PersistentFlags().BoolVarP(&dumpMode, "dump-only", "m", false, "Dump mode: dump everything and exit") + bindPFlag("dump-only", "dump-only") + RootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "v", "info", "Log level") bindPFlag("log.level", "log-level") diff --git a/config/config.go b/config/config.go index 99474a3..45ced6d 100644 --- a/config/config.go +++ b/config/config.go @@ -15,6 +15,9 @@ type KfConfig struct { // When DryRun is true, we don't write to disk and we don't commit/push DryRun bool + // When DumpMode is true, we just dump everything once and exit + DumpMode bool + // Logger should be used to send all logs Logger *logrus.Logger diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 8650f52..48969e2 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -25,7 +25,10 @@ import ( "github.com/ghodss/yaml" ) -const maxProcessRetry = 6 +var ( + maxProcessRetry = 6 + canaryKey = "$katafygio-canary" +) // Interface describe a standard kubernetes controller type Interface interface { @@ -41,6 +44,7 @@ type Controller struct { name string stopCh chan struct{} doneCh chan struct{} + syncCh chan struct{} notifier event.Notifier config *config.KfConfig queue workqueue.RateLimitingInterface @@ -93,6 +97,7 @@ func New(client cache.ListerWatcher, notifier event.Notifier, name string, confi return &Controller{ stopCh: make(chan struct{}), doneCh: make(chan struct{}), + syncCh: make(chan struct{}, 1), notifier: notifier, name: name, config: config, @@ -113,15 +118,18 @@ func (c *Controller) Start() { return } + c.queue.Add(canaryKey) + go wait.Until(c.runWorker, time.Second, c.stopCh) } // Stop halts the controller func (c *Controller) Stop() { + c.config.Logger.Infof("Stopping %s controller", c.name) + <-c.syncCh close(c.stopCh) c.queue.ShutDown() <-c.doneCh - c.config.Logger.Infof("Stopping %s controller", c.name) } func (c *Controller) runWorker() { @@ -138,6 +146,13 @@ func (c *Controller) processNextItem() bool { } defer c.queue.Done(key) + if strings.Compare(key.(string), canaryKey) == 0 { + c.config.Logger.Infof("Initial sync completed for %s controller", c.name) + c.syncCh <- struct{}{} + c.queue.Forget(key) + return true + } + err := c.processItem(key.(string)) if err == nil { diff --git a/pkg/run/run.go b/pkg/run/run.go index d7de7db..ac0b57e 100644 --- a/pkg/run/run.go +++ b/pkg/run/run.go @@ -31,7 +31,9 @@ func Run(config *config.KfConfig) { sigterm := make(chan os.Signal, 1) signal.Notify(sigterm, syscall.SIGTERM) signal.Notify(sigterm, syscall.SIGINT) - <-sigterm + if !config.DumpMode { + <-sigterm + } obsv.Stop() repo.Stop()