main.go

package main

import (
	"context"
	"errors"
	"flag"
	"os"
	"path/filepath"
	"strings"
	"time"

	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
	// to ensure that exec-entrypoint and run can make use of them.
	"k8s.io/client-go/kubernetes"
	_ "k8s.io/client-go/plugin/pkg/client/auth"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/klog/v2"

	"github.com/ondat/operator-toolkit/declarative/loader"
	"github.com/ondat/operator-toolkit/telemetry/export"
	"github.com/ondat/operator-toolkit/webhook/cert"
	"go.uber.org/zap/zapcore"
	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/healthz"
	"sigs.k8s.io/controller-runtime/pkg/log/zap"

	metricsv1 "github.com/ondat/metrics-exporter/api/config.storageos.com/v1"
	configstorageoscomv1 "github.com/storageos/operator/api/config.storageos.com/v1"
	storageoscomv1 "github.com/storageos/operator/api/v1"
	"github.com/storageos/operator/controllers"
	whctrlr "github.com/storageos/operator/controllers/webhook"
	"github.com/storageos/operator/internal/distro"
	"github.com/storageos/operator/internal/version"
	// +kubebuilder:scaffold:imports
)

const (
	// podNamespace is the operator's pod namespace environment variable.
	podNamespace = "POD_NAMESPACE"

	// disableConfigWatch is the operator's disable config watch environment variable.
	disableConfigWatch = "DISABLE_CONFIG_WATCH"
)

var (
	shutDownperiod = 5 * time.Second
	scheme         = runtime.NewScheme()
	setupLog       = ctrl.Log.WithName("setup")
)

var SupportedMinKubeVersion string

func init() {
	utilruntime.Must(clientgoscheme.AddToScheme(scheme))

	utilruntime.Must(storageoscomv1.AddToScheme(scheme))
	utilruntime.Must(configstorageoscomv1.AddToScheme(scheme))
	utilruntime.Must(metricsv1.AddToScheme(scheme))
	// +kubebuilder:scaffold:scheme
}

// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations;validatingwebhookconfigurations,verbs=*
// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups="",resources=configmaps/status,verbs=get;update;patch
// +kubebuilder:rbac:groups="coordination.k8s.io",resources=leases,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups="extensions",resources=podsecuritypolicies,verbs=use

func main() {
	ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
	defer func() {
		cancel()

		time.Sleep(shutDownperiod)

		os.Exit(1)
	}()

	var configFile string
	flag.StringVar(&configFile, "config", "",
		"The controller will load its initial configuration from this file. "+
			"Omit this flag to use the default configuration values. "+
			"Command-line flags override configuration from this file.")

	var leaderRenewSeconds uint
	flag.UintVar(&leaderRenewSeconds, "leader-renew-seconds", 10, "Leader renewal frequency")

	var opts zap.Options
	opts.BindFlags(flag.CommandLine)
	flag.Parse()

	// Configure logger.
	f := func(ec *zapcore.EncoderConfig) {
		ec.TimeKey = "timestamp"
		ec.EncodeTime = zapcore.RFC3339NanoTimeEncoder
	}
	encoderOpts := func(o *zap.Options) {
		o.EncoderConfigOptions = append(o.EncoderConfigOptions, f)
	}

	zapLogger := zap.New(zap.UseFlagOptions(&opts), zap.StacktraceLevel(zapcore.PanicLevel), encoderOpts)
	ctrl.SetLogger(zapLogger)
	klog.SetLogger(zapLogger)

	// Setup telemetry.
	telemetryShutdown, err := export.InstallJaegerExporter("storageos-operator")
	if err != nil {
		setupLog.Error(err, "unable to setup telemetry exporter")
		os.Exit(1)
	}
	defer telemetryShutdown()

	currentNS := os.Getenv(podNamespace)
	if len(currentNS) == 0 {
		setupLog.Error(errors.New("current namespace not found"), "failed to get current namespace")
		os.Exit(1)
	}

	renewDeadline := time.Duration(leaderRenewSeconds) * time.Second
	leaseDuration := time.Duration(int(1.2*float64(leaderRenewSeconds))) * time.Second
	leaderRetryDuration := renewDeadline / 2

	// Load controller manager configuration and create manager options from
	// it.
	serialExecutionStrategy := false
	ctrlConfig := configstorageoscomv1.OperatorConfig{}
	options := ctrl.Options{
		Scheme:                     scheme,
		LeaderElection:             true,
		LeaderElectionID:           "storageos-operator-leader-imewimw",
		LeaderElectionNamespace:    currentNS,
		LeaderElectionResourceLock: resourcelock.LeasesResourceLock,
		RenewDeadline:              &renewDeadline,
		LeaseDuration:              &leaseDuration,
		RetryPeriod:                &leaderRetryDuration,
	}
	if configFile != "" {
		var err error
		cfg := ctrl.ConfigFile()
		options, err = options.AndFrom(cfg.AtPath(configFile).OfKind(&ctrlConfig))
		if err != nil {
			setupLog.Error(err, "unable to load the config file")
			os.Exit(1)
		}
		// get the execution strategy from the config file
		serialExecutionStrategy = cfg.DeepCopyObject().(*configstorageoscomv1.OperatorConfig).SerialExecutionStrategy
	}

	defaultRestConfig := ctrl.GetConfigOrDie()
	defaultKubeClient := kubernetes.NewForConfigOrDie(defaultRestConfig)

	restConfig := ctrl.GetConfigOrDie()
	restConfig.Timeout = time.Minute
	timeoutKubeClient := kubernetes.NewForConfigOrDie(restConfig)

	// Get Kubernetes version.
	kubeVersion, err := timeoutKubeClient.Discovery().ServerVersion()
	if err != nil {
		setupLog.Error(err, "unable to get Kubernetes version")
		os.Exit(1)
	}

	// Validate Kubernetes version
	if SupportedMinKubeVersion != "" && !version.IsSupported(kubeVersion.String(), SupportedMinKubeVersion) {
		setupLog.Error(errors.New("unsupported Kubernetes version"), "current version of Kubernetes is lower than required minimum version", "supported", SupportedMinKubeVersion, "current", kubeVersion.String())
		os.Exit(1)
	}

	// Get Kubernetes distro.
	kubeDistro := distro.DetermineDistribution(kubeVersion.String())

	mgr, err := ctrl.NewManager(defaultRestConfig, options)
	if err != nil {
		setupLog.Error(err, "unable to start manager")
		os.Exit(1)
	}

	// Create an uncached client to be used in the certificate manager
	// and ConfigMap watcher.
	// NOTE: Cached client from manager can't be used here because the cache is
	// uninitialized at this point.
	cli, err := client.New(mgr.GetConfig(), client.Options{Scheme: mgr.GetScheme()})
	if err != nil {
		setupLog.Error(err, "failed to create raw client")
		os.Exit(1)
	}

	// Configure the certificate manager.
	certOpts := cert.Options{
		CertRefreshInterval: ctrlConfig.WebhookCertRefreshInterval.Duration,
		Service: &admissionregistrationv1.ServiceReference{
			Name:      ctrlConfig.WebhookServiceName,
			Namespace: currentNS,
		},
		Client:                      cli,
		SecretRef:                   &types.NamespacedName{Name: ctrlConfig.WebhookSecretRef, Namespace: currentNS},
		ValidatingWebhookConfigRefs: []types.NamespacedName{{Name: ctrlConfig.ValidatingWebhookConfigRef}},
	}
	// Create certificate manager without manager to start the provisioning
	// immediately.
	// NOTE: Certificate Manager implements nonLeaderElectionRunnable interface
	// but since the webhook server is also a nonLeaderElectionRunnable, they
	// start at the same time, resulting in a race condition where sometimes
	// the certificates aren't available when the webhook server starts. By
	// passing nil instead of the manager, the certificate manager is not
	// managed by the controller manager. It starts immediately, in a blocking
	// fashion, ensuring that the cert is created before the webhook server
	// starts.
	if err := cert.NewManager(nil, certOpts); err != nil {
		setupLog.Error(err, "unable to provision certificate")
		os.Exit(1)
	}
	// Set channel based on K8s version. If that channel does not exist, use default channelDir 'stable'.
	cleanKubeVersion := version.CleanupVersion(kubeVersion.String())
	channel := version.MajorMinor(cleanKubeVersion)
	_, err = os.Stat(filepath.Join(loader.DefaultChannelDir, channel))
	if err != nil {
		channel = loader.DefaultChannelName
	}

	dcw := os.Getenv(disableConfigWatch)
	if dcw == "" || strings.ToLower(dcw) == "false" {
		if err = controllers.NewConfigReconciler(mgr).
			SetupWithManager(mgr, currentNS); err != nil {
			setupLog.Error(err, "unable to create controller",
				"controller", "ConfigReconciler")
			os.Exit(1)
		}
	}

	if err = controllers.NewStorageOSClusterReconciler(mgr).
		SetupWithManager(mgr, defaultKubeClient.AppsV1(), defaultKubeClient.PolicyV1(), cleanKubeVersion, kubeDistro, channel, serialExecutionStrategy); err != nil {
		setupLog.Error(err, "unable to create controller",
			"controller", "StorageOSCluster")
		os.Exit(1)
	}

	// Create and set up admission webhook controller.
	clusterWh, err := whctrlr.NewStorageOSClusterWebhook(mgr.GetClient(), mgr.GetScheme())
	if err != nil {
		setupLog.Error(err, "unable to create admission webhook controller",
			"controller", clusterWh.CtrlName)
		os.Exit(1)
	}
	if err := clusterWh.SetupWithManager(mgr); err != nil {
		setupLog.Error(err, "unable to setup webhook controller with manager",
			"controller", clusterWh.CtrlName)
		os.Exit(1)
	}

	// +kubebuilder:scaffold:builder

	if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
		setupLog.Error(err, "unable to set up health check")
		os.Exit(1)
	}
	if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil {
		setupLog.Error(err, "unable to set up ready check")
		os.Exit(1)
	}

	setupLog.Info("starting manager")
	if err := mgr.Start(ctx); err != nil {
		setupLog.Error(err, "problem running manager")
		os.Exit(1)
	}
}