Exclude namespaces by regex

README.md

@@ -59,26 +59,30 @@ Available Commands:
   version     Print the version number
 
 Flags:
-  -s, --api-server string        Kubernetes api-server url
-  -c, --config string            Configuration file (default "/etc/katafygio/katafygio.yaml")
-  -q, --context string           Kubernetes configuration context
-  -d, --dry-run                  Dry-run mode: don't store anything
-  -m, --dump-only                Dump mode: dump everything once and exit
-  -x, --exclude-kind strings     Ressource kind to exclude. Eg. 'deployment'
-  -y, --exclude-object strings   Object to exclude. Eg. 'configmap:kube-system/kube-dns'
-  -l, --filter string            Label filter. Select only objects matching the label
-  -t, --git-timeout duration     Git operations timeout (default 5m0s)
-  -g, --git-url string           Git repository URL
-  -p, --healthcheck-port int     Port for answering healthchecks on /health url
-  -h, --help                     help for katafygio
-  -k, --kube-config string       Kubernetes configuration path
-  -e, --local-dir string         Where to dump yaml files (default "./kubernetes-backup")
-  -v, --log-level string         Log level (default "info")
-  -o, --log-output string        Log output (default "stderr")
-  -r, --log-server string        Log server (if using syslog)
-  -a, --namespace string         Only dump objects from this namespace
-  -n, --no-git                   Don't version with git
-  -i, --resync-interval int      Full resync interval in seconds (0 to disable) (default 900)
+  -s, --api-server string            Kubernetes api-server url
+  -j, --check-interval int           Check interval in seconds (default 10)
+  -c, --config string                Configuration file (default "/etc/katafygio/katafygio.yaml")
+  -q, --context string               Kubernetes configuration context
+  -d, --dry-run                      Dry-run mode: don't store anything
+  -m, --dump-only                    Dump mode: dump everything once and exit
+  -x, --exclude-kind strings         Ressource kind to exclude. Eg. 'deployment'
+  -z, --exclude-namespaces strings   Namespaces to exclude. Eg. 'temp.*' as regexes. This collects all namespaces and then filters them. Don't use it with the namespace flag.
+  -y, --exclude-object strings       Object to exclude. Eg. 'configmap:kube-system/kube-dns'
+  -l, --filter string                Label filter. Select only objects matching the label
+  -b, --git-author string            Author for git commits (default "Katafygio")
+  -f, --git-email string             Email address for git commits (default "katafygio@localhost")
+  -t, --git-timeout duration         Git operations timeout (default 5m0s)
+  -g, --git-url string               Git repository URL
+  -p, --healthcheck-port int         Port for answering healthchecks on /health url
+  -h, --help                         help for katafygio
+  -k, --kube-config string           Kubernetes configuration path
+  -e, --local-dir string             Where to dump yaml files (default "./kubernetes-backup")
+  -v, --log-level string             Log level (default "info")
+  -o, --log-output string            Log output (default "stderr")
+  -r, --log-server string            Log server (if using syslog)
+  -a, --namespace string             Only dump objects from this namespace
+  -n, --no-git                       Don't version with git
+  -i, --resync-interval int          Full resync interval in seconds (0 to disable) (default 900)
 ```
 
 ## Configuration file and env variables

cmd/execute.go

@@ -64,14 +64,14 @@ func runE(cmd *cobra.Command, args []string) (err error) {
 
 	var repo *git.Store
 	if !noGit {
-		repo, err = git.New(logger, dryRun, localDir, gitURL, gitAuthor, gitEmail, gitTimeout, time.Duration(checkInt) * time.Second).Start()
+		repo, err = git.New(logger, dryRun, localDir, gitURL, gitAuthor, gitEmail, gitTimeout, time.Duration(checkInt)*time.Second).Start()
 	}
 	if err != nil {
 		return fmt.Errorf("failed to start git repo handler: %v", err)
 	}
 
 	evts := event.New()
-	fact := controller.NewFactory(logger, filter, resyncInt, exclobj)
+	fact := controller.NewFactory(logger, filter, resyncInt, exclobj, exclnamespaces)
 	reco := recorder.New(logger, evts, localDir, resyncInt*2, dryRun).Start()
 	obsv := observer.New(logger, restcfg, evts, fact, exclkind, namespace).Start()
 

cmd/flags.go

@@ -9,28 +9,29 @@ import (
 )
 
 var (
-	cfgFile    string
-	apiServer  string
-	context    string
-	namespace  string
-	kubeConf   string
-	dryRun     bool
-	dumpMode   bool
-	logLevel   string
-	logOutput  string
-	logServer  string
-	filter     string
-	localDir   string
-	gitURL     string
-	gitTimeout time.Duration
-	healthP    int
-	resyncInt  int
-	checkInt   int
-	exclkind   []string
-	exclobj    []string
-	noGit      bool
-	gitAuthor  string
-	gitEmail   string
+	cfgFile        string
+	apiServer      string
+	context        string
+	namespace      string
+	exclnamespaces []string
+	kubeConf       string
+	dryRun         bool
+	dumpMode       bool
+	logLevel       string
+	logOutput      string
+	logServer      string
+	filter         string
+	localDir       string
+	gitURL         string
+	gitTimeout     time.Duration
+	healthP        int
+	resyncInt      int
+	checkInt       int
+	exclkind       []string
+	exclobj        []string
+	noGit          bool
+	gitAuthor      string
+	gitEmail       string
 )
 
 func bindPFlag(key string, cmd string) {
@@ -55,6 +56,9 @@ func init() {
 	RootCmd.PersistentFlags().StringVarP(&namespace, "namespace", "a", "", "Only dump objects from this namespace")
 	bindPFlag("namespace", "namespace")
 
+	RootCmd.PersistentFlags().StringSliceVarP(&exclnamespaces, "exclude-namespaces", "z", nil, "Namespaces to exclude. Eg. 'temp.*' as regexes. This collects all namespaces and then filters them. Don't use it with the namespace flag.")
+	bindPFlag("exclude-namespaces", "exclude-namespaces")
+
 	RootCmd.PersistentFlags().StringVarP(&kubeConf, "kube-config", "k", "", "Kubernetes configuration path")
 	bindPFlag("kube-config", "kube-config")
 
@@ -115,6 +119,7 @@ func bindConf(cmd *cobra.Command, args []string) {
 	apiServer = viper.GetString("api-server")
 	context = viper.GetString("context")
 	namespace = viper.GetString("namespace")
+	exclnamespaces = viper.GetStringSlice("exclude-namespaces")
 	kubeConf = viper.GetString("kube-config")
 	dryRun = viper.GetBool("dry-run")
 	dumpMode = viper.GetBool("dump-only")

pkg/controller/controller.go

@@ -7,6 +7,7 @@ package controller
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 	"time"
 
@@ -43,24 +44,26 @@ type logger interface {
 
 // Factory generate controllers
 type Factory struct {
-	logger     logger
-	filter     string
-	resyncIntv time.Duration
-	excluded   []string
+	logger      logger
+	filter      string
+	resyncIntv  time.Duration
+	excludedobj []string
+	excludedns  []string
 }
 
 // Controller is a generic kubernetes controller
 type Controller struct {
-	name       string
-	stopCh     chan struct{}
-	doneCh     chan struct{}
-	syncCh     chan struct{}
-	notifier   event.Notifier
-	queue      workqueue.RateLimitingInterface
-	informer   cache.SharedIndexInformer
-	logger     logger
-	resyncIntv time.Duration
-	excluded   []string
+	name        string
+	stopCh      chan struct{}
+	doneCh      chan struct{}
+	syncCh      chan struct{}
+	notifier    event.Notifier
+	queue       workqueue.RateLimitingInterface
+	informer    cache.SharedIndexInformer
+	logger      logger
+	resyncIntv  time.Duration
+	excludedobj []string
+	excludedns  []*regexp.Regexp
 }
 
 // New return a kubernetes controller using the provided client
@@ -70,7 +73,8 @@ func New(client cache.ListerWatcher,
 	name string,
 	filter string,
 	resync time.Duration,
-	excluded []string,
+	excludedobj []string,
+	excludednamespace []string,
 ) *Controller {
 
 	selector := metav1.ListOptions{LabelSelector: filter, ResourceVersion: "0"}
@@ -113,17 +117,23 @@ func New(client cache.ListerWatcher,
 		},
 	})
 
+	exclnsre := make([]*regexp.Regexp, 0)
+	for _, ns := range excludednamespace {
+		exclnsre = append(exclnsre, regexp.MustCompile(ns))
+	}
+
 	return &Controller{
-		stopCh:     make(chan struct{}),
-		doneCh:     make(chan struct{}),
-		syncCh:     make(chan struct{}, 1),
-		notifier:   notifier,
-		name:       name,
-		queue:      queue,
-		informer:   informer,
-		logger:     log,
-		resyncIntv: resync,
-		excluded:   excluded,
+		stopCh:      make(chan struct{}),
+		doneCh:      make(chan struct{}),
+		syncCh:      make(chan struct{}, 1),
+		notifier:    notifier,
+		name:        name,
+		queue:       queue,
+		informer:    informer,
+		logger:      log,
+		resyncIntv:  resync,
+		excludedobj: excludedobj,
+		excludedns:  exclnsre,
 	}
 }
 
@@ -198,7 +208,7 @@ func (c *Controller) processItem(key string) error {
 		return fmt.Errorf("error fetching %s from store: %v", key, err)
 	}
 
-	for _, obj := range c.excluded {
+	for _, obj := range c.excludedobj {
 		if strings.Compare(strings.ToLower(obj), strings.ToLower(c.name+":"+key)) == 0 {
 			return nil
 		}
@@ -220,6 +230,14 @@ func (c *Controller) processItem(key string) error {
 		delete(md, attr)
 	}
 
+	if namespace, ok := md["namespace"].(string); ok {
+		for _, nsre := range c.excludedns {
+			if nsre.MatchString(namespace) {
+				return nil
+			}
+		}
+	}
+
 	yml, err := yaml.Marshal(obj)
 	if err != nil {
 		return fmt.Errorf("failed to marshal %s: %v", key, err)
@@ -234,16 +252,17 @@ func (c *Controller) enqueue(notif *event.Notification) {
 }
 
 // NewFactory create a controller factory
-func NewFactory(logger logger, filter string, resync int, excluded []string) *Factory {
+func NewFactory(logger logger, filter string, resync int, excludedobj []string, excludedns []string) *Factory {
 	return &Factory{
-		logger:     logger,
-		filter:     filter,
-		resyncIntv: time.Duration(resync) * time.Second,
-		excluded:   excluded,
+		logger:      logger,
+		filter:      filter,
+		resyncIntv:  time.Duration(resync) * time.Second,
+		excludedobj: excludedobj,
+		excludedns:  excludedns,
 	}
 }
 
 // NewController create a controller.Controller
 func (f *Factory) NewController(client cache.ListerWatcher, notifier event.Notifier, name string) Interface {
-	return New(client, notifier, f.logger, name, f.filter, f.resyncIntv, f.excluded)
+	return New(client, notifier, f.logger, name, f.filter, f.resyncIntv, f.excludedobj, f.excludedns)
 }

pkg/controller/controller_test.go

@@ -121,7 +121,7 @@ func TestController(t *testing.T) {
 
 	evt := new(mockNotifier)
 	log := new(mockLog)
-	f := NewFactory(log, "label1=something", 60, []string{"pod:ns3/Bar3"})
+	f := NewFactory(log, "label1=something", 60, []string{"pod:ns3/Bar3"}, []string{})
 	ctrl := f.NewController(client, evt, "pod")
 
 	// this will trigger a deletion event

pkg/store/git/git.go

@@ -25,37 +25,37 @@ type logger interface {
 
 // Store will maintain a git repository off dumped kube objects
 type Store struct {
-	Logger   logger
-	LocalDir string
-	URL      string
-	Timeout  time.Duration
+	Logger        logger
+	LocalDir      string
+	URL           string
+	Timeout       time.Duration
 	CheckInterval time.Duration
-	Author   string
-	Email    string
-	Msg      string
-	DryRun   bool
-	stopch   chan struct{}
-	donech   chan struct{}
+	Author        string
+	Email         string
+	Msg           string
+	DryRun        bool
+	stopch        chan struct{}
+	donech        chan struct{}
 }
 
 // New instantiate a new git Store. url is optional.
 func New(log logger, dryRun bool, dir, url string, author string, email string, timeout time.Duration, checkInterval time.Duration) *Store {
 	return &Store{
-		Logger:   log,
-		LocalDir: dir,
-		URL:      url,
-		Timeout:  timeout,
+		Logger:        log,
+		LocalDir:      dir,
+		URL:           url,
+		Timeout:       timeout,
 		CheckInterval: checkInterval,
-		Author:   author,
-		Email:    email,
-		Msg:      GitMsg,
-		DryRun:   dryRun,
+		Author:        author,
+		Email:         email,
+		Msg:           GitMsg,
+		DryRun:        dryRun,
 	}
 }
 
 // Start maintains a directory content committed
 func (s *Store) Start() (*Store, error) {
-	s.Logger.Infof(fmt.Sprintf("Starting git repository synchronizer every %s",s.CheckInterval.String()))
+	s.Logger.Infof(fmt.Sprintf("Starting git repository synchronizer every %s", s.CheckInterval.String()))
 	s.stopch = make(chan struct{})
 	s.donech = make(chan struct{})
 

pkg/store/git/git_test.go

@@ -35,7 +35,7 @@ func TestGitDryRun(t *testing.T) {
 
 	appFs = afero.NewMemMapFs()
 
-	repo, err := New(new(mockLog), true, "/tmp/ktest", "", timeout).Start()
+	repo, err := New(new(mockLog), true, "/tmp/ktest", "", "test", "test@test", timeout, 10*time.Second).Start()
 	if err != nil {
 		t.Errorf("failed to start git: %v", err)
 	}
@@ -62,7 +62,7 @@ func TestGit(t *testing.T) {
 
 	defer os.RemoveAll(dir)
 
-	repo, err := New(new(mockLog), false, dir, "", timeout).Start()
+	repo, err := New(new(mockLog), false, dir, "", "test", "test@test", timeout, 10*time.Second).Start()
 	if err != nil {
 		t.Errorf("failed to start git: %v", err)
 	}