Merge pull request #4 from incident-io/upgrade-controller-runtime

Update controller-runtime to v0.16.3

VERSION

@@ -1 +1 @@
-4.3.1
+5.0.0

apis/vault/v1alpha1/secretsinjector_webhook.go

@@ -14,6 +14,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -31,11 +32,12 @@ type SecretsInjector struct {
 	opts    SecretsInjectorOptions
 }
 
-func NewSecretsInjector(c client.Client, logger logr.Logger, opts SecretsInjectorOptions) *SecretsInjector {
+func NewSecretsInjector(c client.Client, logger logr.Logger, opts SecretsInjectorOptions, scheme *runtime.Scheme) *SecretsInjector {
 	return &SecretsInjector{
-		client: c,
-		logger: logger,
-		opts:   opts,
+		client:  c,
+		logger:  logger,
+		opts:    opts,
+		decoder: admission.NewDecoder(scheme),
 	}
 }
 

apis/workloads/v1alpha1/console_attach_webhook.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -25,21 +26,17 @@ type ConsoleAttachObserverWebhook struct {
 	requestTimeout    time.Duration
 }
 
-func NewConsoleAttachObserverWebhook(c client.Client, recorder record.EventRecorder, lifecycleRecorder LifecycleEventRecorder, logger logr.Logger, requestTimeout time.Duration) *ConsoleAttachObserverWebhook {
+func NewConsoleAttachObserverWebhook(c client.Client, recorder record.EventRecorder, lifecycleRecorder LifecycleEventRecorder, logger logr.Logger, requestTimeout time.Duration, scheme *runtime.Scheme) *ConsoleAttachObserverWebhook {
 	return &ConsoleAttachObserverWebhook{
 		client:            c,
 		recorder:          recorder,
 		lifecycleRecorder: lifecycleRecorder,
 		logger:            logger,
 		requestTimeout:    requestTimeout,
+		decoder:           admission.NewDecoder(scheme),
 	}
 }
 
-func (c *ConsoleAttachObserverWebhook) InjectDecoder(d *admission.Decoder) error {
-	c.decoder = d
-	return nil
-}
-
 func (c *ConsoleAttachObserverWebhook) Handle(ctx context.Context, req admission.Request) admission.Response {
 	logger := c.logger.WithValues(
 		"uuid", string(req.UID),

apis/workloads/v1alpha1/console_authenticator_webhook.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
+	runtime "k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 )
 
@@ -18,18 +19,14 @@ type ConsoleAuthenticatorWebhook struct {
 	decoder           *admission.Decoder
 }
 
-func NewConsoleAuthenticatorWebhook(lifecycleRecorder LifecycleEventRecorder, logger logr.Logger) *ConsoleAuthenticatorWebhook {
+func NewConsoleAuthenticatorWebhook(lifecycleRecorder LifecycleEventRecorder, logger logr.Logger, scheme *runtime.Scheme) *ConsoleAuthenticatorWebhook {
 	return &ConsoleAuthenticatorWebhook{
 		lifecycleRecorder: lifecycleRecorder,
 		logger:            logger,
+		decoder:           admission.NewDecoder(scheme),
 	}
 }
 
-func (c *ConsoleAuthenticatorWebhook) InjectDecoder(d *admission.Decoder) error {
-	c.decoder = d
-	return nil
-}
-
 func (c *ConsoleAuthenticatorWebhook) Handle(ctx context.Context, req admission.Request) admission.Response {
 	logger := c.logger.WithValues("uuid", string(req.UID))
 	logger.Info("starting request", "event", "request.start")

apis/workloads/v1alpha1/console_authorisation_webhook.go

@@ -10,6 +10,7 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
+	runtime "k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
@@ -25,19 +26,15 @@ type ConsoleAuthorisationWebhook struct {
 	decoder           *admission.Decoder
 }
 
-func NewConsoleAuthorisationWebhook(c client.Client, lifecycleRecorder LifecycleEventRecorder, logger logr.Logger) *ConsoleAuthorisationWebhook {
+func NewConsoleAuthorisationWebhook(c client.Client, lifecycleRecorder LifecycleEventRecorder, logger logr.Logger, scheme *runtime.Scheme) *ConsoleAuthorisationWebhook {
 	return &ConsoleAuthorisationWebhook{
 		client:            c,
 		lifecycleRecorder: lifecycleRecorder,
 		logger:            logger,
+		decoder:           admission.NewDecoder(scheme),
 	}
 }
 
-func (c *ConsoleAuthorisationWebhook) InjectDecoder(d *admission.Decoder) error {
-	c.decoder = d
-	return nil
-}
-
 func (c *ConsoleAuthorisationWebhook) Handle(ctx context.Context, req admission.Request) admission.Response {
 	logger := c.logger.WithValues("uuid", string(req.UID))
 	logger.Info("starting request", "event", "request.start")

apis/workloads/v1alpha1/console_template_validation_webhook.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
+	runtime "k8s.io/apimachinery/pkg/runtime"
 
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 )
@@ -17,17 +18,13 @@ type ConsoleTemplateValidationWebhook struct {
 	decoder *admission.Decoder
 }
 
-func NewConsoleTemplateValidationWebhook(logger logr.Logger) *ConsoleTemplateValidationWebhook {
+func NewConsoleTemplateValidationWebhook(logger logr.Logger, scheme *runtime.Scheme) *ConsoleTemplateValidationWebhook {
 	return &ConsoleTemplateValidationWebhook{
-		logger: logger,
+		logger:  logger,
+		decoder: admission.NewDecoder(scheme),
 	}
 }
 
-func (c *ConsoleTemplateValidationWebhook) InjectDecoder(d *admission.Decoder) error {
-	c.decoder = d
-	return nil
-}
-
 func (c *ConsoleTemplateValidationWebhook) Handle(ctx context.Context, req admission.Request) admission.Response {
 	logger := c.logger.WithValues("uuid", string(req.UID))
 	logger.Info("starting request", "event", "request.start")

cmd/acceptance/main.go

@@ -38,7 +38,7 @@ var (
 	prepareImage         = prepare.Flag("image", "Docker image tag used for exchanging test images").Default("theatre:latest").String()
 	prepareConfigFile    = prepare.Flag("config-file", "Path to Kind config file").Default("kind-e2e.yaml").ExistingFile()
 	prepareDockerfile    = prepare.Flag("dockerfile", "Path to acceptance dockerfile").Default("Dockerfile").ExistingFile()
-	prepareKindNodeImage = prepare.Flag("kind-node-image", "Kind Node Image").Default("kindest/node:v1.24.13").String()
+	prepareKindNodeImage = prepare.Flag("kind-node-image", "Kind Node Image").Default("kindest/node:v1.27.3").String()
 	prepareVerbose       = prepare.Flag("verbose", "Use a higher log level when creating the cluster").Short('v').Bool()
 
 	destroy = app.Command("destroy", "Destroys the test Kubernetes cluster and other resources")
@@ -138,8 +138,21 @@ func main() {
 		contextTimeout := 3 * time.Minute
 		ctx, deadline := context.WithTimeout(ctx, contextTimeout)
 		defer deadline()
-		waitCmd := exec.CommandContext(ctx, "kubectl", "--context", fmt.Sprintf("kind-%s", *clusterName), "wait", "--all-namespaces", "--for", "condition=Ready", "pods", "--all", "--timeout", "2m")
 
+		// Wait for Deployments
+		// We do this to guard against a race condition where, if you only have the "wait for
+		// pods" check below, but the controller hasn't yet actually *spawned* any pods for
+		// deployments, then you can proceed with the preparation when the cluster isn't in a
+		// good state.
+		// The most notable issue is cert-manager; if the pods aren't up, and therefore
+		// serving webhooks, then subsequently the installation of any controllers which have
+		// webhooks, and therefore require a certificate, will fail.
+		waitCmd := exec.CommandContext(ctx, "kubectl", "--context", fmt.Sprintf("kind-%s", *clusterName), "wait", "--all-namespaces", "--for", "condition=Available", "deployments", "--all", "--timeout", "2m")
+		if err := pipeOutput(waitCmd).Run(); err != nil {
+			app.Fatalf("not all setup resources are running: %v", err)
+		}
+		// Pods - covers those created by Statefulsets
+		waitCmd = exec.CommandContext(ctx, "kubectl", "--context", fmt.Sprintf("kind-%s", *clusterName), "wait", "--all-namespaces", "--for", "condition=Ready", "pods", "--all", "--timeout", "2m")
 		if err := pipeOutput(waitCmd).Run(); err != nil {
 			app.Fatalf("not all setup resources are running: %v", err)
 		}

cmd/rbac-manager/main.go

@@ -15,6 +15,7 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // this is required to auth against GCP
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	rbacv1alpha1 "github.com/gocardless/theatre/v4/apis/rbac/v1alpha1"
 	"github.com/gocardless/theatre/v4/cmd"
@@ -72,11 +73,10 @@ func main() {
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:             scheme,
-		MetricsBindAddress: fmt.Sprintf("%s:%d", commonOpts.MetricAddress, commonOpts.MetricPort),
-		Port:               9443,
-		LeaderElection:     commonOpts.ManagerLeaderElection,
-		LeaderElectionID:   "rbac.crds.gocardless.com",
+		Scheme:           scheme,
+		Metrics:          metricsserver.Options{BindAddress: fmt.Sprintf("%s:%d", commonOpts.MetricAddress, commonOpts.MetricPort)},
+		LeaderElection:   commonOpts.ManagerLeaderElection,
+		LeaderElectionID: "rbac.crds.gocardless.com",
 	})
 	if err != nil {
 		app.Fatalf("failed to create manager: %v", err)

cmd/vault-manager/main.go

@@ -9,6 +9,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	vaultv1alpha1 "github.com/gocardless/theatre/v4/apis/vault/v1alpha1"
@@ -57,10 +59,12 @@ func main() {
 	defer cancel()
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		MetricsBindAddress: fmt.Sprintf("%s:%d", commonOpts.MetricAddress, commonOpts.MetricPort),
-		Port:               443,
-		LeaderElection:     commonOpts.ManagerLeaderElection,
-		LeaderElectionID:   "vault.crds.gocardless.com",
+		Metrics:          metricsserver.Options{BindAddress: fmt.Sprintf("%s:%d", commonOpts.MetricAddress, commonOpts.MetricPort)},
+		LeaderElection:   commonOpts.ManagerLeaderElection,
+		LeaderElectionID: "vault.crds.gocardless.com",
+		WebhookServer: webhook.NewServer(webhook.Options{
+			Port: 443,
+		}),
 	})
 	if err != nil {
 		app.Fatalf("failed to create manager: %v", err)
@@ -86,6 +90,7 @@ func main() {
 			mgr.GetClient(),
 			logger.WithName("webhooks").WithName("secrets-injector"),
 			injectorOpts,
+			mgr.GetScheme(),
 		),
 	})
 

cmd/workloads-manager/acceptance/acceptance.go

@@ -17,6 +17,8 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	rbacv1alpha1 "github.com/gocardless/theatre/v4/apis/rbac/v1alpha1"
 	workloadsv1alpha1 "github.com/gocardless/theatre/v4/apis/workloads/v1alpha1"
@@ -45,6 +47,10 @@ func init() {
 }
 
 func newClient(config *rest.Config) client.Client {
+	// Prevent the following warning from being printed to logs:
+	// [controller-runtime] log.SetLogger(...) was never called; logs will not be displayed.
+	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+
 	kubeClient, err := client.New(config, client.Options{Scheme: scheme})
 	Expect(err).NotTo(HaveOccurred(), "could not connect to kubernetes cluster")
 

cmd/workloads-manager/main.go

@@ -11,6 +11,8 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // this is required to auth against GCP
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	rbacv1alpha1 "github.com/gocardless/theatre/v4/apis/rbac/v1alpha1"
@@ -82,11 +84,13 @@ func main() {
 	lifecycleRecorder := workloadsv1alpha1.NewLifecycleEventRecorder(*contextName, logger, publisher, idBuilder)
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		MetricsBindAddress: fmt.Sprintf("%s:%d", commonOpts.MetricAddress, commonOpts.MetricPort),
-		Port:               443,
-		LeaderElection:     commonOpts.ManagerLeaderElection,
-		LeaderElectionID:   "workloads.crds.gocardless.com",
-		Scheme:             scheme,
+		Metrics:          metricsserver.Options{BindAddress: fmt.Sprintf("%s:%d", commonOpts.MetricAddress, commonOpts.MetricPort)},
+		LeaderElection:   commonOpts.ManagerLeaderElection,
+		LeaderElectionID: "workloads.crds.gocardless.com",
+		Scheme:           scheme,
+		WebhookServer: webhook.NewServer(webhook.Options{
+			Port: 443,
+		}),
 	})
 	if err != nil {
 		app.Fatalf("failed to create manager: %v", err)
@@ -108,11 +112,16 @@ func main() {
 		app.Fatalf("failed to create controller: %v", err)
 	}
 
+	// NOTE: We may want to simplify the implementation of webhooks, like this:
+	// https://book.kubebuilder.io/cronjob-tutorial/webhook-implementation
+	// Currently there's a lot of boilerplate/wiring up, which isn't really necessary.
+
 	// console authenticator webhook
 	mgr.GetWebhookServer().Register("/mutate-consoles", &admission.Webhook{
 		Handler: workloadsv1alpha1.NewConsoleAuthenticatorWebhook(
 			lifecycleRecorder,
 			logger.WithName("webhooks").WithName("console-authenticator"),
+			mgr.GetScheme(),
 		),
 	})
 
@@ -122,13 +131,15 @@ func main() {
 			mgr.GetClient(),
 			lifecycleRecorder,
 			logger.WithName("webhooks").WithName("console-authorisation"),
+			mgr.GetScheme(),
 		),
 	})
 
 	// console template webhook
 	mgr.GetWebhookServer().Register("/validate-consoletemplates", &admission.Webhook{
 		Handler: workloadsv1alpha1.NewConsoleTemplateValidationWebhook(
 			logger.WithName("webhooks").WithName("console-template"),
+			mgr.GetScheme(),
 		),
 	})
 
@@ -140,6 +151,7 @@ func main() {
 			lifecycleRecorder,
 			logger.WithName("webhooks").WithName("console-attach-observer"),
 			10*time.Second,
+			mgr.GetScheme(),
 		),
 	})