Upgrade with migration for 0.10 -> 0.11 k0s (#93)

* Initial upgrade with server->controller migration

Signed-off-by: Jussi Nummelin <jnummelin@mirantis.com>

* Add lint target to Makefile

Signed-off-by: Jussi Nummelin <jnummelin@mirantis.com>

* Handle errors properly; cleanup un-used code

Signed-off-by: Jussi Nummelin <jnummelin@mirantis.com>

* Ad 0.11.0-beta.1 to smokes

Signed-off-by: Jussi Nummelin <jnummelin@mirantis.com>

* use 0.11.0-beta.2 for smokes

Signed-off-by: Jussi Nummelin <jnummelin@mirantis.com>

* Upgrade smoke test

Signed-off-by: Jussi Nummelin <jnummelin@mirantis.com>

* Wait for kube api to be available after controller upgrade

Signed-off-by: Jussi Nummelin <jnummelin@mirantis.com>

* Add iptables install in host prep phase

Signed-off-by: Jussi Nummelin <jnummelin@mirantis.com>

* Add no-drain flag and wait for workers ready after upgrade

Signed-off-by: Jussi Nummelin <jnummelin@mirantis.com>

* Cleanup execs

Co-authored-by: Kimmo Lehto <kimmo.lehto@gmail.com>

* Cleanup upgrade version comparison

Co-authored-by: Kimmo Lehto <kimmo.lehto@gmail.com>

* Restart controller service on re-configure phase only when no upgrade is happening

Signed-off-by: Jussi Nummelin <jnummelin@mirantis.com>

* Better log message for service migration

Co-authored-by: Kimmo Lehto <kimmo.lehto@gmail.com>

* Add prerelease '-0' to migration version constraint check

Signed-off-by: Jussi Nummelin <jnummelin@mirantis.com>

* Add TODO note to refactor package install to use list of packages instead of individual installs

Co-authored-by: Kimmo Lehto <kimmo.lehto@gmail.com>

Co-authored-by: Kimmo Lehto <kimmo.lehto@gmail.com>

.github/workflows/go.yml

@@ -82,3 +82,37 @@ jobs:
         env:
           LINUX_IMAGE: ${{ matrix.image }}
         run: make smoke-basic
+
+  smoke-upgrade:
+    strategy:
+      matrix:
+        image:
+          - quay.io/footloose/ubuntu18.04
+          - quay.io/footloose/centos7
+          #- quay.io/footloose/amazonlinux2
+          #- quay.io/footloose/debian10
+          #- quay.io/footloose/fedora29
+    name: Upgrade 0.10 --> 0.11
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+
+      - name: Restore compiled binary for smoke testing
+        uses: actions/cache@v2
+        id: restore-compiled-binary
+        with:
+          path: |
+            k0sctl
+          key: build-${{ github.run_id }}
+
+      - name: Run smoke tests
+        env:
+          LINUX_IMAGE: ${{ matrix.image }}
+        run: make smoke-upgrade

Makefile

@@ -56,7 +56,19 @@ upload-%: bin/% $(github_release)
 .PHONY: upload
 upload: $(addprefix upload-,$(bins) $(checksums))
 
-smoketests := smoke-basic
+smoketests := smoke-basic smoke-upgrade
 .PHONY: $(smoketests)
 $(smoketests): k0sctl
-	$(MAKE) -C smoke-test $@
+	$(MAKE) -C smoke-test $@
+
+golint := $(shell which golangci-lint)
+ifeq ($(golint),)
+golint := $(shell go env GOPATH)/bin/golangci-lint
+endif
+
+$(golint):
+	go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.31.0
+
+.PHONY: lint
+lint: $(golint)
+	$(golint) run ./...

cmd/apply.go

@@ -23,6 +23,10 @@ var applyCommand = &cli.Command{
 			Name:  "no-wait",
 			Usage: "Do not wait for worker nodes to join",
 		},
+		&cli.BoolFlag{
+			Name:  "no-drain",
+			Usage: "Do not drain worker nodes when upgrading",
+		},
 		debugFlag,
 		traceFlag,
 		analyticsFlag,
@@ -46,6 +50,7 @@ var applyCommand = &cli.Command{
 		}
 
 		phase.NoWait = ctx.Bool("no-wait")
+
 		manager := phase.Manager{Config: &c}
 
 		manager.AddPhase(
@@ -62,6 +67,10 @@ var applyCommand = &cli.Command{
 			&phase.InitializeK0s{},
 			&phase.InstallControllers{},
 			&phase.InstallWorkers{},
+			&phase.UpgradeControllers{},
+			&phase.UpgradeWorkers{
+				NoDrain: ctx.Bool("no-drain"),
+			},
 			&phase.Disconnect{},
 		)
 

config/cluster/host.go

@@ -3,6 +3,7 @@ package cluster
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/avast/retry-go"
@@ -74,6 +75,7 @@ type HostMetadata struct {
 	IsK0sLeader       bool
 	Hostname          string
 	Ready             bool
+	NeedsUpgrade      bool
 }
 
 // UnmarshalYAML sets in some sane defaults when unmarshaling the data from yaml
@@ -188,6 +190,33 @@ func (h *Host) K0sServiceName() string {
 	return "k0s" + h.Role
 }
 
+// UpdateK0sBinary updates the binary on the host either by downloading or uploading, based on the config
+func (h *Host) UpdateK0sBinary(version string) error {
+	if h.K0sBinaryPath != "" {
+		if err := h.Upload(h.K0sBinaryPath, h.Configurer.K0sBinaryPath()); err != nil {
+			return err
+		}
+		if err := h.Configurer.Chmod(h, h.Configurer.K0sBinaryPath(), "0700"); err != nil {
+			return err
+		}
+	} else {
+		if err := h.Configurer.DownloadK0s(h, version, h.Metadata.Arch); err != nil {
+			return err
+		}
+
+		output, err := h.ExecOutput(h.Configurer.K0sCmdf("version"))
+		if err != nil {
+			return fmt.Errorf("downloaded k0s binary is invalid: %s", err.Error())
+		}
+		output = strings.TrimPrefix(output, "v")
+		if output != version {
+			return fmt.Errorf("downloaded k0s binary version is %s not %s", output, version)
+		}
+	}
+	h.Metadata.K0sBinaryVersion = version
+	return nil
+}
+
 type kubeNodeStatus struct {
 	Items []struct {
 		Status struct {
@@ -240,6 +269,16 @@ func (h *Host) WaitKubeNodeReady(node *Host) error {
 	)
 }
 
+// DrainNode drains the given node
+func (h *Host) DrainNode(node *Host) error {
+	return h.Exec(h.Configurer.KubectlCmdf("drain --grace-period=120 --force --timeout=5m --ignore-daemonsets --delete-local-data %s", node.Metadata.Hostname))
+}
+
+// UncordonNode marks the node schedulable again
+func (h *Host) UncordonNode(node *Host) error {
+	return h.Exec(h.Configurer.KubectlCmdf("uncordon %s", node.Metadata.Hostname))
+}
+
 // CheckHTTPStatus will perform a web request to the url and return an error if the http status is not the expected
 func (h *Host) CheckHTTPStatus(url string, expected int) error {
 	status, err := h.Configurer.HTTPStatus(h, url)
@@ -303,6 +342,21 @@ func (h *Host) NeedCurl() bool {
 	return false
 }
 
+// NeedIPTables returns true when the iptables package is needed on the host
+func (h *Host) NeedIPTables() bool {
+	// Windows does not need iptables 
+	if h.Configurer.Kind() == "windows" {
+		return false
+	}
+
+	// Controllers do not need iptables
+	if h.IsController() {
+		return false
+	}
+
+	return !h.Configurer.CommandExist(h, "iptables")
+}
+
 // WaitKubeAPIReady blocks until the local kube api responds to /version
 func (h *Host) WaitKubeAPIReady() error {
 	return h.WaitHTTPStatus("https://localhost:6443/version", 200)

config/cluster/k0s.go

@@ -13,7 +13,7 @@ import (
 )
 
 // K0sMinVersion is the minimum k0s version supported
-const K0sMinVersion = "0.11.0-beta1"
+const K0sMinVersion = "0.11.0-beta.2"
 
 // K0s holds configuration for bootstraping a k0s cluster
 type K0s struct {

go.mod

@@ -5,10 +5,12 @@ go 1.15
 replace github.com/segmentio/analytics-go v3.1.0+incompatible => github.com/kke/analytics-go v1.2.1-0.20210209122110-10364370169e
 
 require (
+	github.com/Masterminds/semver v1.5.0
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
 	github.com/creasty/defaults v1.5.1
 	github.com/denisbrodbeck/machineid v1.0.1
+	github.com/gammazero/workerpool v1.1.1
 	github.com/go-playground/validator/v10 v10.4.1
 	github.com/hashicorp/go-version v1.2.1
 	github.com/k0sproject/dig v0.1.0
@@ -20,6 +22,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli/v2 v2.3.0
 	golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4
+	google.golang.org/appengine v1.6.5
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/client-go v0.19.3
 )

go.sum

@@ -28,6 +28,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022 h1:y8Gs8CzNfDF5AZvjr+5UyGQvQEBL7pwo+v+wX6q9JI8=
 github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
@@ -63,6 +65,10 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gammazero/deque v0.0.0-20200721202602-07291166fe33 h1:UG4wNrJX9xSKnm/Gck5yTbxnOhpNleuE4MQRdmcGySo=
+github.com/gammazero/deque v0.0.0-20200721202602-07291166fe33/go.mod h1:D90+MBHVc9Sk1lJAbEVgws0eYEurY4mv2TDso3Nxh3w=
+github.com/gammazero/workerpool v1.1.1 h1:MN29GcZtZZAgzTU+Zk54Y+J9XkE54MoXON/NCZvNulo=
+github.com/gammazero/workerpool v1.1.1/go.mod h1:5BN0IJVRjSFAypo9QTJCaWdijjNz9Jjl6VFS1PRjCeg=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=

phase/configure_k0s.go

@@ -51,7 +51,7 @@ func (p *ConfigureK0s) Run() error {
 
 func (p *ConfigureK0s) validateConfig(h *cluster.Host) error {
 	log.Infof("%s: validating configuration", h)
-	output, err := h.ExecOutput(h.Configurer.K0sCmdf(`validate config -c "%s"`, h.K0sConfigPath()))
+	output, err := h.ExecOutput(h.Configurer.K0sCmdf(`validate config --config "%s"`, h.K0sConfigPath()))
 	if err != nil {
 		return fmt.Errorf("spec.k0s.config fails validation:\n%s", output)
 	}
@@ -96,7 +96,7 @@ func (p *ConfigureK0s) configureK0s(h *cluster.Host) error {
 		log.Debugf("%s: configuration did not change", h)
 	} else {
 		log.Infof("%s: configuration was changed", h)
-		if h.Metadata.K0sRunningVersion != "" {
+		if h.Metadata.K0sRunningVersion != "" && !h.Metadata.NeedsUpgrade {
 			log.Infof("%s: restarting the k0s service", h)
 			if err := h.Configurer.RestartService(h, h.K0sServiceName()); err != nil {
 				return err

phase/download_k0s.go

@@ -24,7 +24,7 @@ func (p *DownloadK0s) Title() string {
 func (p *DownloadK0s) Prepare(config *config.Cluster) error {
 	p.Config = config
 	p.hosts = p.Config.Spec.Hosts.Filter(func(h *cluster.Host) bool {
-		return h.Metadata.K0sBinaryVersion != p.Config.Spec.K0s.Version
+		return h.Metadata.K0sBinaryVersion != p.Config.Spec.K0s.Version && !h.Metadata.NeedsUpgrade
 	})
 	return nil
 }

phase/gather_k0s_facts.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/Masterminds/semver"
 	"github.com/k0sproject/k0sctl/config/cluster"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
@@ -95,11 +96,12 @@ func (p *GatherK0sFacts) investigateK0s(h *cluster.Host) error {
 	}
 
 	h.Metadata.K0sRunningVersion = strings.TrimPrefix(status.Version, "v")
-	if p.Config.Spec.K0s.Version != h.Metadata.K0sRunningVersion {
-		return fmt.Errorf("%s: is running k0s %s version %s but target is %s - upgrade is not yet supported", h, h.Role, h.Metadata.K0sRunningVersion, p.Config.Spec.K0s.Version)
-	}
+	h.Metadata.NeedsUpgrade = p.needsUpgrade(h)
 
 	log.Infof("%s: is running k0s %s version %s", h, h.Role, h.Metadata.K0sRunningVersion)
+	if h.Metadata.NeedsUpgrade {
+		log.Warnf("%s: k0s will be upgraded", h)
+	}
 
 	if !h.IsController() {
 		log.Infof("%s: checking if worker %s has joined", p.leader, h.Metadata.Hostname)
@@ -112,3 +114,18 @@ func (p *GatherK0sFacts) investigateK0s(h *cluster.Host) error {
 
 	return nil
 }
+
+func (p *GatherK0sFacts) needsUpgrade(h *cluster.Host) bool {
+	target, err := semver.NewVersion(p.Config.Spec.K0s.Version)
+	if err != nil {
+		log.Warnf("%s: failed to parse target version: %s", h, err.Error())
+		return false
+	}
+	current, err := semver.NewVersion(h.Metadata.K0sRunningVersion)
+	if err != nil {
+		log.Warnf("%s: failed to parse running version: %s", h, err.Error())
+		return false
+	}
+
+	return target.GreaterThan(current)
+}

phase/prepare_hosts.go

@@ -35,6 +35,13 @@ func (p *PrepareHosts) prepareHost(h *cluster.Host) error {
 		}
 	}
 
+	// TODO: combine with above using a slice of packages as each  InstallPackage call triggers an apt/zypper/etc update.
+	if h.NeedIPTables() {
+		if err := h.Configurer.InstallPackage(h, "iptables"); err != nil {
+			return err
+		}
+	}
+
 	if h.IsController() && !h.Configurer.CommandExist(h, "kubectl") {
 		log.Infof("%s: installing kubectl", h)
 		if err := h.Configurer.InstallKubectl(h); err != nil {