KUDO init --upgrade

.gitignore

@@ -9,6 +9,7 @@ config/manager_image_patch.yaml-e
 # created by /hack/generate_krew.sh during release
 kudo.yaml
 kudo-e2e-test.yaml
+kudo-upgrade-test.yaml
 kind-logs-*
 operators
 

Makefile

@@ -45,6 +45,10 @@ integration-test: cli-fast manager-fast
 operator-test: cli-fast manager-fast
 	./hack/run-operator-tests.sh
 
+.PHONY: upgrade-test
+upgrade-test: cli-fast manager-fast
+	./hack/run-upgrade-tests.sh
+
 .PHONY: test-clean
 # Clean test reports
 test-clean:

hack/run-upgrade-tests.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+INTEGRATION_OUTPUT_JUNIT=${INTEGRATION_OUTPUT_JUNIT:-false}
+KUDO_VERSION=${KUDO_VERSION:-test}
+
+docker build . \
+    --build-arg ldflags_arg="" \
+    -t "kudobuilder/controller:$KUDO_VERSION"
+
+sed "s/%version%/$KUDO_VERSION/" kudo-upgrade-test.yaml.tmpl > kudo-upgrade-test.yaml
+
+if [ "$INTEGRATION_OUTPUT_JUNIT" == true ]
+then
+    echo "Running Upgrade tests with junit output"
+    mkdir -p reports/
+    go get github.com/jstemmer/go-junit-report
+
+    ./bin/kubectl-kudo test --config kudo-upgrade-test.yaml 2>&1 \
+        | tee /dev/fd/2 \
+        | go-junit-report -set-exit-code \
+        > reports/kudo_upgrade_test_report.xml
+else
+    echo "Running Upgrade tests without junit output"
+
+    ./bin/kubectl-kudo test --config kudo-upgrade-test.yaml
+fi

kudo-upgrade-test.yaml.tmpl

@@ -0,0 +1,8 @@
+apiVersion: kudo.dev/v1beta1
+kind: TestSuite
+testDirs:
+- ./test/upgrade
+startKIND: true
+kindContainers:
+- kudobuilder/controller:%version%
+timeout: 300

pkg/kudoctl/cmd/init.go

@@ -6,8 +6,6 @@ import (
 	"io"
 	"strings"
 
-	"github.com/kudobuilder/kudo/pkg/kudoctl/verifier"
-
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	flag "github.com/spf13/pflag"
@@ -18,6 +16,7 @@ import (
 	"github.com/kudobuilder/kudo/pkg/kudoctl/kudoinit"
 	"github.com/kudobuilder/kudo/pkg/kudoctl/kudoinit/setup"
 	"github.com/kudobuilder/kudo/pkg/kudoctl/util/repo"
+	"github.com/kudobuilder/kudo/pkg/kudoctl/verifier"
 )
 
 const (
@@ -55,6 +54,10 @@ and finishes with success if KUDO is already installed.
   kubectl kudo init --service-account testaccount
   # install kudo using self-signed CA bundle for the webhooks (for testing and development)
   kubectl kudo init --unsafe-self-signed-webhook-ca
+  # upgrade an existing KUDO installation
+  kubectl kudo init --upgrade
+  # verify the current KUDO installation
+  kubectl kudo init --verify
 `
 )
 
@@ -73,6 +76,8 @@ type initCmd struct {
 	timeout             int64
 	clientOnly          bool
 	crdOnly             bool
+	upgrade             bool
+	verify              bool
 	home                kudohome.Home
 	client              *kube.Client
 	selfSignedWebhookCA bool
@@ -103,10 +108,12 @@ func newInitCmd(fs afero.Fs, out io.Writer, errOut io.Writer, client *kube.Clien
 	f := cmd.Flags()
 	f.BoolVarP(&i.clientOnly, "client-only", "c", false, "If set does not install KUDO on the server")
 	f.StringVarP(&i.image, "kudo-image", "i", "", "Override KUDO controller image and/or version")
-	f.StringVarP(&i.imagePullPolicy, "kudo-image-pull-policy", "", "", "Override KUDO controller image pull policy")
+	f.StringVarP(&i.imagePullPolicy, "kudo-image-pull-policy", "", "Always", "Override KUDO controller image pull policy")
 	f.StringVarP(&i.version, "version", "", "", "Override KUDO controller version of the KUDO image")
 	f.StringVarP(&i.output, "output", "o", "", "Output format")
 	f.BoolVar(&i.dryRun, "dry-run", false, "Do not install local or remote")
+	f.BoolVar(&i.upgrade, "upgrade", false, "Upgrade an existing KUDO installation")
+	f.BoolVar(&i.verify, "verify", false, "Verify an existing KUDO installation")
 	f.BoolVar(&i.crdOnly, "crd-only", false, "Add only KUDO CRDs to your cluster")
 	f.BoolVarP(&i.wait, "wait", "w", false, "Block until KUDO manager is running and ready to receive requests")
 	f.Int64Var(&i.timeout, "wait-timeout", 300, "Wait timeout to be used")
@@ -132,13 +139,22 @@ func (initCmd *initCmd) validate(flags *flag.FlagSet) error {
 	if flags.Changed("wait-timeout") && !initCmd.wait {
 		return errors.New("wait-timeout is only useful when using the flag '--wait'")
 	}
+	if initCmd.upgrade && initCmd.verify {
+		return errors.New("'--upgrade' and '--verify' can not be used at the same time")
+	}
+	if initCmd.verify && initCmd.dryRun {
+		return errors.New("'--dry-run' and '--verify' can not be used at the same time")
+	}
+	if initCmd.crdOnly && initCmd.upgrade {
+		return errors.New("'--upgrade' and '--crd-only' can not be used at the same time")
+	}
 
 	return nil
 }
 
 // run initializes local config and installs KUDO manager to Kubernetes cluster.
 func (initCmd *initCmd) run() error {
-	opts := kudoinit.NewOptions(initCmd.version, initCmd.ns, initCmd.serviceAccount, initCmd.selfSignedWebhookCA)
+	opts := kudoinit.NewOptions(initCmd.version, initCmd.ns, initCmd.serviceAccount, initCmd.upgrade, initCmd.selfSignedWebhookCA)
 	// if image provided switch to it.
 	if initCmd.image != "" {
 		opts.Image = initCmd.image
@@ -157,9 +173,9 @@ func (initCmd *initCmd) run() error {
 
 	installer := setup.NewInstaller(opts, initCmd.crdOnly)
 
-	// initialize client
+	// ensureClient client
 	if !initCmd.dryRun {
-		if err := initCmd.initialize(); err != nil {
+		if err := initCmd.ensureClient(); err != nil {
 			return clog.Errorf("error initializing: %s", err)
 		}
 	}
@@ -172,15 +188,41 @@ func (initCmd *initCmd) run() error {
 		return nil
 	}
 
-	// initialize server
-	clog.V(4).Printf("initializing server")
+	// ensureClient server
+	clog.V(4).Printf("create client")
 	if initCmd.client == nil {
 		client, err := kube.GetKubeClient(Settings.KubeConfig)
 		if err != nil {
 			return clog.Errorf("could not get Kubernetes client: %s", err)
 		}
 		initCmd.client = client
 	}
+	if initCmd.verify {
+		return initCmd.verifyExistingInstallation(installer)
+	}
+
+	if initCmd.upgrade {
+		if err := initCmd.runUpgrade(installer); err != nil {
+			return err
+		}
+	} else {
+		if err := initCmd.runInstall(installer); err != nil {
+			return err
+		}
+	}
+
+	if initCmd.wait {
+		clog.Printf("⌛Waiting for KUDO controller to be ready in your cluster...")
+		err := setup.WatchKUDOUntilReady(initCmd.client.KubeClient, opts, initCmd.timeout)
+		if err != nil {
+			return errors.New("watch timed out, readiness uncertain")
+		}
+	}
+
+	return nil
+}
+
+func (initCmd *initCmd) runInstall(installer *setup.Installer) error {
 	ok, err := initCmd.preInstallVerify(installer)
 	if err != nil {
 		return err
@@ -189,6 +231,40 @@ func (initCmd *initCmd) run() error {
 		return fmt.Errorf("failed to verify installation requirements")
 	}
 
+	if err = initCmd.runYamlOutput(installer); err != nil {
+		return err
+	}
+
+	if !initCmd.dryRun {
+		if err := installer.Install(initCmd.client); err != nil {
+			return clog.Errorf("error installing: %s", err)
+		}
+	}
+	return nil
+}
+
+func (initCmd *initCmd) runUpgrade(installer *setup.Installer) error {
+	ok, err := initCmd.preUpgradeVerify(installer)
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return fmt.Errorf("failed to verify upgrade requirements")
+	}
+
+	if err = initCmd.runYamlOutput(installer); err != nil {
+		return err
+	}
+
+	if !initCmd.dryRun {
+		if err := installer.Upgrade(initCmd.client); err != nil {
+			return clog.Errorf("error upgrading: %s", err)
+		}
+	}
+	return nil
+}
+
+func (initCmd *initCmd) runYamlOutput(installer *setup.Installer) error {
 	//TODO: implement output=yaml|json (define a type for output to constrain)
 	//define an Encoder to replace YAMLWriter
 	if strings.ToLower(initCmd.output) == "yaml" {
@@ -200,26 +276,26 @@ func (initCmd *initCmd) run() error {
 			return err
 		}
 	}
+	return nil
+}
 
-	if !initCmd.dryRun {
-		if err := installer.Install(initCmd.client); err != nil {
-			return clog.Errorf("error installing: %s", err)
-		}
-
-		if initCmd.wait {
-			clog.Printf("⌛Waiting for KUDO controller to be ready in your cluster...")
-			err := setup.WatchKUDOUntilReady(initCmd.client.KubeClient, opts, initCmd.timeout)
-			if err != nil {
-				return errors.New("watch timed out, readiness uncertain")
-			}
-		}
+// verifyExistingInstallation checks if the current installation is valid and as expected
+func (initCmd *initCmd) verifyExistingInstallation(v kudoinit.InstallVerifier) error {
+	clog.V(4).Printf("verify existing installation")
+	result := verifier.NewResult()
+	if err := v.VerifyInstallation(initCmd.client, &result); err != nil {
+		return err
+	}
+	result.PrintWarnings(initCmd.out)
+	if !result.IsValid() {
+		result.PrintErrors(initCmd.out)
 	}
-
 	return nil
 }
 
 // preInstallVerify runs the pre-installation verification and returns true if the installation can continue
 func (initCmd *initCmd) preInstallVerify(v kudoinit.InstallVerifier) (bool, error) {
+	clog.V(4).Printf("Run pre-install verify")
 	result := verifier.NewResult()
 	if err := v.PreInstallVerify(initCmd.client, &result); err != nil {
 		return false, err
@@ -232,6 +308,21 @@ func (initCmd *initCmd) preInstallVerify(v kudoinit.InstallVerifier) (bool, erro
 	return true, nil
 }
 
+// preUpgradeVerify runs the pre-upgrade verification and returns true if the upgrade can continue
+func (initCmd *initCmd) preUpgradeVerify(v kudoinit.InstallVerifier) (bool, error) {
+	clog.V(4).Printf("Run pre-upgrade verify")
+	result := verifier.NewResult()
+	if err := v.PreUpgradeVerify(initCmd.client, &result); err != nil {
+		return false, err
+	}
+	result.PrintWarnings(initCmd.out)
+	if !result.IsValid() {
+		result.PrintErrors(initCmd.out)
+		return false, nil
+	}
+	return true, nil
+}
+
 // YAMLWriter writes yaml to writer.   Looked into using https://godoc.org/gopkg.in/yaml.v2#NewEncoder which
 // looks like a better way, however the omitted JSON elements are encoded which results in a very verbose output.
 //TODO: Write a Encoder util which uses the "sigs.k8s.io/yaml" library for marshalling
@@ -251,8 +342,7 @@ func (initCmd *initCmd) YAMLWriter(w io.Writer, manifests []string) error {
 	return err
 }
 
-//func initialize(fs afero.Fs, settings env.Settings, out io.Writer) error {
-func (initCmd *initCmd) initialize() error {
+func (initCmd *initCmd) ensureClient() error {
 
 	if err := ensureDirectories(initCmd.fs, initCmd.home); err != nil {
 		return err

pkg/kudoctl/cmd/init_integration_test.go

@@ -142,7 +142,8 @@ func TestIntegInitWithNameSpace(t *testing.T) {
 
 	instance := testutils.NewResource("kudo.dev/v1beta1", "Instance", "zk", "ns")
 	// Verify that we cannot create the instance, because the test environment is empty.
-	assert.IsType(t, &meta.NoKindMatchError{}, testClient.Create(context.TODO(), instance))
+	err = testClient.Create(context.TODO(), instance)
+	assert.Error(t, err, "Expected an Error but got none")
 
 	// Install all of the CRDs.
 	crds := crd.NewInitializer().Resources()
@@ -173,7 +174,7 @@ func TestIntegInitWithNameSpace(t *testing.T) {
 
 	// On second attempt run should succeed.
 	err = cmd.run()
-	assert.NoError(t, err)
+	assert.NoError(t, err, buf.String())
 	defer func() {
 		assert.NoError(t, deleteInitPrereqs(cmd, testClient))
 	}()
@@ -342,7 +343,7 @@ func TestInitWithServiceAccount(t *testing.T) {
 	}
 }
 
-func TestNoErrorOnReInit(t *testing.T) {
+func TestReInitFails(t *testing.T) {
 	//	 if the CRD exists and we init again there should be no error
 	testClient, err := testutils.NewRetryClient(testenv.Config, client.Options{
 		Scheme: testutils.Scheme(),
@@ -380,8 +381,9 @@ func TestNoErrorOnReInit(t *testing.T) {
 
 	// second run will have an output that it already exists
 	err = cmd.run()
-	assert.NoError(t, err)
-	assert.True(t, strings.Contains(buf.String(), "crd operators.kudo.dev already exists"))
+
+	assert.Equal(t, "failed to verify installation requirements", err.Error())
+	assertStringContains(t, "CRD operators.kudo.dev is already installed. Did you mean to use --upgrade?", errBuf.String())
 }
 
 func deleteObjects(objs []runtime.Object, client *testutils.RetryClient) error {
@@ -395,7 +397,7 @@ func deleteObjects(objs []runtime.Object, client *testutils.RetryClient) error {
 }
 
 func deleteInitPrereqs(cmd *initCmd, client *testutils.RetryClient) error {
-	opts := kudoinit.NewOptions(cmd.version, cmd.ns, cmd.serviceAccount, cmd.selfSignedWebhookCA)
+	opts := kudoinit.NewOptions(cmd.version, cmd.ns, cmd.serviceAccount, cmd.upgrade, cmd.selfSignedWebhookCA)
 
 	objs := append([]runtime.Object{}, prereq.NewWebHookInitializer(opts).Resources()...)
 	objs = append(objs, prereq.NewServiceAccountInitializer(opts).Resources()...)