Configurable git commands deadline

* Start logging early, so we can see wether the application started
* Expose the git commands timeout as a command line flags (and sets generous defaults)
* Don't display --help/usage info with hard failures error messages
* Don't wait for git clone to succeed before hooking health probes listener: it was configured to fail probes early, when the initial clone was failing; but an app logging an error message before exiting makes diagnostics easier than kubelet probes failures.

README.md

@@ -65,6 +65,7 @@ Flags:
   -x, --exclude-kind strings     Ressource kind to exclude. Eg. 'deployment'
   -y, --exclude-object strings   Object to exclude. Eg. 'configmap:kube-system/kube-dns'
   -l, --filter string            Label filter. Select only objects matching the label
+  -t, --git-timeout duration     Git operations timeout (default 5m0s)
   -g, --git-url string           Git repository URL
   -p, --healthcheck-port int     Port for answering healthchecks on /health url
   -h, --help                     help for katafygio

assets/helm-chart/katafygio/templates/deployment.yaml

@@ -26,6 +26,9 @@ spec:
           args:
             - --local-dir={{ .Values.localDir }}
             - --healthcheck-port={{ .Values.healthcheckPort }}
+          {{- if .Values.gitTimeout }}
+            - --git-timeout={{ .Values.gitTimeout }}
+          {{- end }}
           {{- if .Values.gitUrl }}
             - --git-url={{ .Values.gitUrl }}
           {{- end }}

assets/helm-chart/katafygio/values.yaml

@@ -7,6 +7,10 @@
 # pod-local git repository, which can be on a persistent volume (see above).
 #gitUrl: https://user:token@github.com/myorg/myrepos.git
 
+# gitTimeout (optional) defines the deadline for git commands
+# (available with Katafygio v0.7.4 and up).
+#gitTimeout: 300s
+
 # noGit disable git versioning when true (will only keep an unversioned local dump up-to-date).
 noGit: false
 

assets/katafygio.yaml

@@ -14,6 +14,9 @@ local-dir: /var/cache/katafygio
 # Remote url is optional. If provided, Katafygio will push there.
 #git-url: https://user:token@github.com/myorg/myrepos.git
 
+# Git timeout is the deadline for git commands
+#git-timeout: 300s
+
 # Port to listen for http health check probes. 0 to disable.
 healthcheck-port: 0
 

cmd/execute.go

@@ -32,8 +32,10 @@ var (
 		Short: "Backup Kubernetes cluster as yaml files",
 		Long: "Backup Kubernetes cluster as yaml files in a git repository.\n" +
 			"--exclude-kind (-x) and --exclude-object (-y) may be specified several times.",
-		PreRun: bindConf,
-		RunE:   runE,
+		SilenceUsage:  true,
+		SilenceErrors: true,
+		PreRun:        bindConf,
+		RunE:          runE,
 	}
 )
 
@@ -42,6 +44,7 @@ func runE(cmd *cobra.Command, args []string) (err error) {
 	if err != nil {
 		return fmt.Errorf("failed to create a logger: %v", err)
 	}
+	logger.Info(appName, " starting")
 
 	if restcfg == nil {
 		restcfg, err = client.New(apiServer, kubeConf)
@@ -55,9 +58,11 @@ func runE(cmd *cobra.Command, args []string) (err error) {
 		return fmt.Errorf("Can't create directory %s: %v", localDir, err)
 	}
 
+	http := health.New(logger, healthP).Start()
+
 	var repo *git.Store
 	if !noGit {
-		repo, err = git.New(logger, dryRun, localDir, gitURL).Start()
+		repo, err = git.New(logger, dryRun, localDir, gitURL, gitTimeout).Start()
 	}
 	if err != nil {
 		return fmt.Errorf("failed to start git repo handler: %v", err)
@@ -67,21 +72,23 @@ func runE(cmd *cobra.Command, args []string) (err error) {
 	fact := controller.NewFactory(logger, filter, resyncInt, exclobj)
 	reco := recorder.New(logger, evts, localDir, resyncInt*2, dryRun).Start()
 	obsv := observer.New(logger, restcfg, evts, fact, exclkind).Start()
-	http := health.New(logger, healthP).Start()
 
+	logger.Info(appName, " started")
 	sigterm := make(chan os.Signal, 1)
 	signal.Notify(sigterm, syscall.SIGTERM)
 	signal.Notify(sigterm, syscall.SIGINT)
 	if !dumpMode {
 		<-sigterm
 	}
 
+	logger.Info(appName, " stopping")
 	obsv.Stop()
 	reco.Stop()
 	http.Stop()
 	if !noGit {
 		repo.Stop()
 	}
+	logger.Info(appName, " stopped")
 
 	return nil
 }

cmd/flags.go

@@ -2,28 +2,30 @@ package cmd
 
 import (
 	"log"
+	"time"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 var (
-	cfgFile   string
-	apiServer string
-	kubeConf  string
-	dryRun    bool
-	dumpMode  bool
-	logLevel  string
-	logOutput string
-	logServer string
-	filter    string
-	localDir  string
-	gitURL    string
-	healthP   int
-	resyncInt int
-	exclkind  []string
-	exclobj   []string
-	noGit     bool
+	cfgFile    string
+	apiServer  string
+	kubeConf   string
+	dryRun     bool
+	dumpMode   bool
+	logLevel   string
+	logOutput  string
+	logServer  string
+	filter     string
+	localDir   string
+	gitURL     string
+	gitTimeout time.Duration
+	healthP    int
+	resyncInt  int
+	exclkind   []string
+	exclobj    []string
+	noGit      bool
 )
 
 func bindPFlag(key string, cmd string) {
@@ -69,6 +71,9 @@ func init() {
 	RootCmd.PersistentFlags().StringVarP(&gitURL, "git-url", "g", "", "Git repository URL")
 	bindPFlag("git-url", "git-url")
 
+	RootCmd.PersistentFlags().DurationVarP(&gitTimeout, "git-timeout", "t", 300*time.Second, "Git operations timeout")
+	bindPFlag("git-timeout", "git-timeout")
+
 	RootCmd.PersistentFlags().StringSliceVarP(&exclkind, "exclude-kind", "x", nil, "Ressource kind to exclude. Eg. 'deployment'")
 	bindPFlag("exclude-kind", "exclude-kind")
 
@@ -100,6 +105,7 @@ func bindConf(cmd *cobra.Command, args []string) {
 	filter = viper.GetString("filter")
 	localDir = viper.GetString("local-dir")
 	gitURL = viper.GetString("git-url")
+	gitTimeout = viper.GetDuration("git-timeout")
 	healthP = viper.GetInt("healthcheck-port")
 	resyncInt = viper.GetInt("resync-interval")
 	exclkind = viper.GetStringSlice("exclude-kind")

main.go

@@ -17,7 +17,7 @@ func ExitWrapper(exit int) {
 func main() {
 	err := cmd.Execute()
 	if err != nil {
-		fmt.Printf("%+v", err)
+		fmt.Printf("%+v\n", err)
 		ExitWrapper(1)
 	}
 }

pkg/store/git/git.go

@@ -12,9 +12,6 @@ import (
 )
 
 var (
-	// TimeoutCommands defines the max execution time for git commands
-	TimeoutCommands = 60 * time.Second
-
 	// CheckInterval defines the interval between local directory checks
 	CheckInterval = 10 * time.Second
 
@@ -40,6 +37,7 @@ type Store struct {
 	Logger   logger
 	LocalDir string
 	URL      string
+	Timeout  time.Duration
 	Author   string
 	Email    string
 	Msg      string
@@ -49,11 +47,12 @@ type Store struct {
 }
 
 // New instantiate a new git Store. url is optional.
-func New(log logger, dryRun bool, dir, url string) *Store {
+func New(log logger, dryRun bool, dir, url string, timeout time.Duration) *Store {
 	return &Store{
 		Logger:   log,
 		LocalDir: dir,
 		URL:      url,
+		Timeout:  timeout,
 		Author:   GitAuthor,
 		Email:    GitEmail,
 		Msg:      GitMsg,
@@ -103,7 +102,7 @@ func (s *Store) Git(args ...string) error {
 		return nil
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), TimeoutCommands)
+	ctx, cancel := context.WithTimeout(context.Background(), s.Timeout)
 	defer cancel()
 
 	cmd := exec.CommandContext(ctx, "git", args...) // #nosec
@@ -112,6 +111,9 @@ func (s *Store) Git(args ...string) error {
 
 	out, err := cmd.CombinedOutput()
 	if err != nil {
+		if ctx.Err() == context.DeadlineExceeded {
+			return fmt.Errorf("git %s timed out (%v, %s)", args[0], err, out)
+		}
 		return fmt.Errorf("git %s failed with code %v: %s", args[0], err, out)
 	}
 
@@ -124,7 +126,7 @@ func (s *Store) Status() (changed bool, err error) {
 		return false, nil
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), TimeoutCommands)
+	ctx, cancel := context.WithTimeout(context.Background(), s.Timeout)
 	defer cancel()
 
 	cmd := exec.CommandContext(ctx, "git", "status", "--porcelain") // #nosec
@@ -133,6 +135,9 @@ func (s *Store) Status() (changed bool, err error) {
 
 	out, err := cmd.CombinedOutput()
 	if err != nil {
+		if ctx.Err() == context.DeadlineExceeded {
+			return false, fmt.Errorf("git status timed out (%v, %s)", err, out)
+		}
 		return false, fmt.Errorf("git status failed with code %v: %s", err, out)
 	}
 

pkg/store/git/git_test.go

@@ -5,11 +5,15 @@ import (
 	"os"
 	"os/exec"
 	"testing"
+	"time"
 
 	"github.com/spf13/afero"
 )
 
-var testHasGit bool
+var (
+	testHasGit bool
+	timeout    = 5 * time.Second
+)
 
 func init() {
 	// Thanks to Mitchell Hashimoto!
@@ -31,7 +35,7 @@ func TestGitDryRun(t *testing.T) {
 
 	appFs = afero.NewMemMapFs()
 
-	repo, err := New(new(mockLog), true, "/tmp/ktest", "").Start()
+	repo, err := New(new(mockLog), true, "/tmp/ktest", "", timeout).Start()
 	if err != nil {
 		t.Errorf("failed to start git: %v", err)
 	}
@@ -58,7 +62,7 @@ func TestGit(t *testing.T) {
 
 	defer os.RemoveAll(dir)
 
-	repo, err := New(new(mockLog), false, dir, "").Start()
+	repo, err := New(new(mockLog), false, dir, "", timeout).Start()
 	if err != nil {
 		t.Errorf("failed to start git: %v", err)
 	}