Feature: Add --term-delay option to CLI

CHANGELOG.md

@@ -83,6 +83,12 @@
   [#1766](https://github.com/Kong/kubernetes-ingress-controller/issues/1766)
 - Added support for `TLSRoute` resources.
   [#2476](https://github.com/Kong/kubernetes-ingress-controller/issues/2476)
+- Added `--term-delay` flag to support setting a time delay before processing
+  `SIGTERM` and `SIGINT` signals. This was added to specifically help in
+  situations where the Kong Gateway has a load-balancer in front of it to help
+  stagger and stabilize the shutdown procedure when the load-balancer is
+  draining or otherwise needs to remove the Gateway from it's rotation.
+  [#2494](https://github.com/Kong/kubernetes-ingress-controller/pull/2494)
 
 #### Fixed
 

internal/cmd/fips/main.go

@@ -6,13 +6,11 @@ package main
 import (
 	_ "crypto/tls/fipsonly"
 
-	ctrl "sigs.k8s.io/controller-runtime"
-
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/cmd/rootcmd"
 )
 
 //go:generate go run github.com/kong/kubernetes-ingress-controller/v2/hack/generators/controllers/networking
 
 func main() {
-	rootcmd.Execute(ctrl.SetupSignalHandler())
+	rootcmd.Execute()
 }

internal/cmd/main.go

@@ -1,13 +1,11 @@
 package main
 
 import (
-	ctrl "sigs.k8s.io/controller-runtime"
-
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/cmd/rootcmd"
 )
 
 //go:generate go run github.com/kong/kubernetes-ingress-controller/v2/hack/generators/controllers/networking
 
 func main() {
-	rootcmd.Execute(ctrl.SetupSignalHandler())
+	rootcmd.Execute()
 }

internal/cmd/rootcmd/rootcmd.go

@@ -2,8 +2,6 @@
 package rootcmd
 
 import (
-	"context"
-
 	"github.com/spf13/cobra"
 
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/manager"
@@ -18,12 +16,12 @@ func init() {
 var rootCmd = &cobra.Command{
 	PersistentPreRunE: bindEnvVars,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return Run(cmd.Context(), &cfg)
+		return Run(&cfg)
 	},
 	SilenceUsage: true,
 }
 
 // Execute is the entry point to the controller manager.
-func Execute(ctx context.Context) {
-	cobra.CheckErr(rootCmd.ExecuteContext(ctx))
+func Execute() {
+	cobra.CheckErr(rootCmd.Execute())
 }

internal/cmd/rootcmd/run.go

@@ -1,13 +1,17 @@
 package rootcmd
 
 import (
-	"context"
 	"fmt"
 
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/manager"
 )
 
-func Run(ctx context.Context, c *manager.Config) error {
+func Run(c *manager.Config) error {
+	ctx, err := SetupSignalHandler(c)
+	if err != nil {
+		return fmt.Errorf("failed to setup signal handler: %w", err)
+	}
+
 	diag, err := StartDiagnosticsServer(ctx, manager.DiagnosticsPort, c)
 	if err != nil {
 		return fmt.Errorf("failed to start diagnostics server: %w", err)

internal/cmd/rootcmd/signal.go

@@ -0,0 +1,61 @@
+package rootcmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/bombsimon/logrusr/v2"
+
+	"github.com/kong/kubernetes-ingress-controller/v2/internal/manager"
+	"github.com/kong/kubernetes-ingress-controller/v2/internal/util"
+)
+
+var mutex sync.Mutex
+var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM}
+
+// SetupSignalHandler registers for SIGTERM and SIGINT. A context is returned
+// which is canceled on one of these signals. If a second signal is not caught, the program
+// will delay for the configured period of time before terminating. If a second signal is caught,
+// the program is terminated with exit code 1.
+func SetupSignalHandler(cfg *manager.Config) (context.Context, error) {
+
+	// This will prevent multiple signal handlers from being created
+	if ok := mutex.TryLock(); !ok {
+		return nil, errors.New("signal handler can only be setup once")
+	}
+
+	deprecatedLogger, err := util.MakeLogger(cfg.LogLevel, cfg.LogFormat)
+	if err != nil {
+		return nil, err
+	}
+	logger := logrusr.New(deprecatedLogger)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, shutdownSignals...)
+	go func() {
+		<-c
+		logger.Info("Signal received, shutting down", "timeout", fmt.Sprint(cfg.TermDelay))
+
+		select {
+		case <-time.After(cfg.TermDelay):
+			cancel()
+		case <-c:
+			logger.Info("Signal received during termination delay, exiting immediately")
+			os.Exit(1) // second signal. Exit directly.
+		}
+
+		<-c
+		logger.Info("Signal received during graceful shutdown, exiting immediately")
+		os.Exit(1) // second signal. Exit directly.
+	}()
+
+	return ctx, nil
+}

internal/manager/config.go

@@ -92,6 +92,12 @@ type Config struct {
 
 	// Feature Gates
 	FeatureGates map[string]bool
+
+	// TermDelay is the time.Duration which the controller manager will wait
+	// after receiving SIGTERM or SIGINT before shutting down. This can be
+	// helpful for advanced cases with load-balancers so that the ingress
+	// controller can be gracefully removed/drained from their rotation.
+	TermDelay time.Duration
 }
 
 // -----------------------------------------------------------------------------
@@ -203,6 +209,9 @@ func (c *Config) FlagSet() *pflag.FlagSet {
 	flagSet.Var(cliflag.NewMapStringBool(&c.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/beta/experimental features. "+
 		fmt.Sprintf("See the Feature Gates documentation for information and available options: %s", featureGatesDocsURL))
 
+	// SIGTERM or SIGINT signal delay
+	flagSet.DurationVar(&c.TermDelay, "term-delay", time.Second*0, "The time delay to sleep before SIGTERM or SIGINT will shut down the Ingress Controller")
+
 	// Deprecated (to be removed in future releases)
 	flagSet.Float32Var(&c.ProxySyncSeconds, "sync-rate-limit", dataplane.DefaultSyncSeconds,
 		"Define the rate (in seconds) in which configuration updates will be applied to the Kong Admin API (DEPRECATED, use --proxy-sync-seconds instead)",

internal/util/test/controller_manager.go

@@ -79,7 +79,7 @@ func DeployControllerManagerForCluster(ctx context.Context, cluster clusters.Clu
 	go func() {
 		defer os.Remove(kubeconfig.Name())
 		fmt.Fprintf(os.Stderr, "INFO: Starting Controller Manager for Cluster %s with Configuration: %+v\n", cluster.Name(), config)
-		if err := rootcmd.Run(ctx, &config); err != nil {
+		if err := rootcmd.Run(&config); err != nil {
 			panic(err)
 		}
 	}()