feat: adding support for kots application host preflights

.github/workflows/e2e.yaml

@@ -19,6 +19,7 @@ jobs:
         - TestMultiNodeInteractiveInstallation
         - TestInstallWithDisabledAddons
         - TestEmbedAddonsOnly
+        - TestHostPreflight
     steps:
     - name: Move Docker aside
       run: |

Makefile

@@ -7,6 +7,7 @@ K0SCTL_VERSION = v0.15.5
 TERRAFORM_VERSION = 1.5.4
 OPENEBS_VERSION = 3.7.0
 K0S_VERSION = v1.27.5+k0s.0
+PREFLIGHT_VERSION = v0.71.1
 LD_FLAGS = -X github.com/replicatedhq/helmvm/pkg/defaults.K0sVersion=$(K0S_VERSION) -X main.Version=$(VERSION)
 
 default: helmvm-linux-amd64
@@ -94,9 +95,18 @@ pkg/goods/bins/helmvm/k0sctl-darwin-arm64:
 	curl -L -o pkg/goods/bins/helmvm/k0sctl-darwin-arm64 "https://github.com/k0sproject/k0sctl/releases/download/$(K0SCTL_VERSION)/k0sctl-darwin-arm64"
 	chmod +x pkg/goods/bins/helmvm/k0sctl-darwin-arm64
 
+pkg/goods/bins/helmvm/preflight:
+	mkdir -p pkg/goods/bins/helmvm
+	mkdir -p output/tmp/preflight
+	curl -L -o output/tmp/preflight/preflight.tar.gz https://github.com/replicatedhq/troubleshoot/releases/download/$(PREFLIGHT_VERSION)/preflight_linux_amd64.tar.gz
+	tar -xzf output/tmp/preflight/preflight.tar.gz -C output/tmp/preflight
+	mv output/tmp/preflight/preflight pkg/goods/bins/helmvm/preflight
+
 .PHONY: static
 static: pkg/addons/adminconsole/charts/adminconsole-$(ADMIN_CONSOLE_CHART_VERSION).tgz \
-	output/bin/yq pkg/goods/bins/k0sctl/k0s-$(K0S_VERSION) \
+	output/bin/yq \
+	pkg/goods/bins/helmvm/preflight \
+	pkg/goods/bins/k0sctl/k0s-$(K0S_VERSION) \
 	pkg/goods/images/list.txt
 
 .PHONY: static-darwin-arm64

cmd/helmvm/install.go

@@ -24,6 +24,7 @@ import (
 	"github.com/replicatedhq/helmvm/pkg/defaults"
 	"github.com/replicatedhq/helmvm/pkg/goods"
 	"github.com/replicatedhq/helmvm/pkg/infra"
+	"github.com/replicatedhq/helmvm/pkg/preflights"
 	pb "github.com/replicatedhq/helmvm/pkg/progressbar"
 	"github.com/replicatedhq/helmvm/pkg/prompts"
 )
@@ -53,13 +54,54 @@ func runPostApply(ctx context.Context) error {
 	return nil
 }
 
+// runHostPreflights run the host preflights we found embedded in the binary
+// on all configured hosts. We attempt to read HostPreflights from all the
+// embedded Helm Charts and from the Kots Application Release files.
+func runHostPreflights(c *cli.Context) error {
+	logrus.Infof("Running host preflights on nodes")
+	cfg, err := config.ReadConfigFile(defaults.PathToConfig("k0sctl.yaml"))
+	if err != nil {
+		return fmt.Errorf("unable to read cluster config: %w", err)
+	}
+	hpf, err := addons.NewApplier().HostPreflights()
+	if err != nil {
+		return fmt.Errorf("unable to read host preflights: %w", err)
+	}
+	if len(hpf.Collectors) == 0 && len(hpf.Analyzers) == 0 {
+		logrus.Info("No host preflights found")
+		return nil
+	}
+	outputs := preflights.NewOutputs()
+	for _, host := range cfg.Spec.Hosts {
+		addr := host.Address()
+		out, err := preflights.Run(c.Context, host, hpf)
+		if err != nil {
+			return fmt.Errorf("preflight failed on %s: %w", addr, err)
+		}
+		outputs[addr] = out
+	}
+	outputs.PrintTable()
+	if outputs.HaveFails() {
+		return fmt.Errorf("preflights haven't passed on one or more hosts")
+	}
+	if !outputs.HaveWarns() || c.Bool("no-prompt") {
+		return nil
+	}
+	logrus.Warn("Host preflights have warnings on one or more hosts")
+	if !prompts.New().Confirm("Do you want to continue ?", false) {
+		return fmt.Errorf("user aborted")
+	}
+	return nil
+}
+
 // runPostApply runs the post-apply script on a host. XXX I don't think this
 // belongs here and needs to be refactored in a more generic way. It's here
 // because I have other things to do and this is a prototype.
 func runPostApplyOnHost(ctx context.Context, host *cluster.Host) error {
 	if err := host.Connect(); err != nil {
 		return fmt.Errorf("failed to connect to host: %w", err)
 	}
+	defer host.Disconnect()
 	src := "/etc/systemd/system/k0scontroller.service"
 	if host.Role == "worker" {
 		src = "/etc/systemd/system/k0sworker.service"
@@ -276,6 +318,9 @@ func applyK0sctl(c *cli.Context, useprompt bool, nodes []infra.Node) error {
 	if err := ensureK0sctlConfig(c, nodes, useprompt); err != nil {
 		return fmt.Errorf("unable to create config file: %w", err)
 	}
+	if err := runHostPreflights(c); err != nil {
+		return fmt.Errorf("unable to finish preflight checks: %w", err)
+	}
 	logrus.Infof("Applying cluster configuration")
 	if err := runK0sctlApply(c.Context); err != nil {
 		logrus.Errorf("Installation or upgrade failed.")

cmd/helmvm/join.go

@@ -33,6 +33,9 @@ var joinCommand = &cli.Command{
 		if err := goods.Materialize(); err != nil {
 			return fmt.Errorf("unable to materialize binaries: %w", err)
 		}
+		if err := runHostPreflightsLocally(c); err != nil {
+			return fmt.Errorf("unable to run host preflights locally: %w", err)
+		}
 		logrus.Infof("Saving token to disk")
 		if err := saveTokenToDisk(c.Args().First()); err != nil {
 			return fmt.Errorf("unable to save token to disk: %w", err)

cmd/helmvm/upgrade.go

@@ -13,6 +13,8 @@ import (
 	"github.com/replicatedhq/helmvm/pkg/addons"
 	"github.com/replicatedhq/helmvm/pkg/defaults"
 	"github.com/replicatedhq/helmvm/pkg/goods"
+	"github.com/replicatedhq/helmvm/pkg/preflights"
+	"github.com/replicatedhq/helmvm/pkg/prompts"
 )
 
 func stopHelmVM() error {
@@ -56,6 +58,36 @@ func canRunUpgrade(c *cli.Context) error {
 	return fmt.Errorf("command not available")
 }
 
+// runHostPreflightsLocally runs the embedded host preflights in the local node prior to
+// node upgrade.
+func runHostPreflightsLocally(c *cli.Context) error {
+	logrus.Infof("Running host preflights locally")
+	hpf, err := addons.NewApplier().HostPreflights()
+	if err != nil {
+		return fmt.Errorf("unable to read host preflights: %w", err)
+	}
+	if len(hpf.Collectors) == 0 && len(hpf.Analyzers) == 0 {
+		logrus.Info("No host preflights found")
+		return nil
+	}
+	out, err := preflights.RunLocal(c.Context, hpf)
+	if err != nil {
+		return fmt.Errorf("preflight failed: %w", err)
+	}
+	out.PrintTable()
+	if out.HasFail() {
+		return fmt.Errorf("preflights haven't passed on one or more hosts")
+	}
+	if !out.HasWarn() || c.Bool("no-prompt") {
+		return nil
+	}
+	logrus.Warn("Host preflights have warnings on one or more hosts")
+	if !prompts.New().Confirm("Do you want to continue ?", false) {
+		return fmt.Errorf("user aborted")
+	}
+	return nil
+}
+
 var upgradeCommand = &cli.Command{
 	Name:  "upgrade",
 	Usage: "Upgrade the local node",
@@ -78,6 +110,9 @@ var upgradeCommand = &cli.Command{
 		if err := goods.Materialize(); err != nil {
 			return fmt.Errorf("unable to materialize binaries: %w", err)
 		}
+		if err := runHostPreflightsLocally(c); err != nil {
+			return fmt.Errorf("unable to run host preflights locally: %w", err)
+		}
 		logrus.Infof("Stopping %s", defaults.BinaryName())
 		if err := stopHelmVM(); err != nil {
 			return fmt.Errorf("unable to stop: %w", err)

e2e/install_test.go

@@ -252,3 +252,30 @@ func TestInstallWithDisabledAddons(t *testing.T) {
 		t.Fatalf("fail to install embedded ssh in node 0: %v", err)
 	}
 }
+
+func TestHostPreflight(t *testing.T) {
+	t.Parallel()
+	tc := cluster.NewTestCluster(&cluster.Input{
+		T:             t,
+		Nodes:         1,
+		Image:         "centos/8-Stream",
+		SSHPublicKey:  "../output/tmp/id_rsa.pub",
+		SSHPrivateKey: "../output/tmp/id_rsa",
+		HelmVMPath:    "../output/bin/helmvm",
+	})
+	defer tc.Destroy()
+	t.Log("installing ssh and binutils on node 0")
+	commands := [][]string{
+		{"dnf", "install", "-y", "openssh-server", "binutils", "tar"},
+		{"systemctl", "enable", "sshd"},
+		{"systemctl", "start", "sshd"},
+	}
+	if err := RunCommandsOnNode(t, tc, 0, commands); err != nil {
+		t.Fatalf("fail to install ssh on node %s: %v", tc.Nodes[0], err)
+	}
+	t.Log("installing helmvm on node 0")
+	line := []string{"embedded-preflight.sh"}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)
+	}
+}

e2e/scripts/embedded-preflight.sh

@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+preflight_with_failure="
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+spec:
+  collectors:
+    - tcpPortStatus:
+        collectorName: Port 24
+        port: 24
+    - tcpPortStatus:
+        collectorName: Port 22
+        port: 22
+  analyzers:
+    - tcpPortStatus:
+        checkName: Port 24
+        collectorName: Port 24
+        outcomes:
+          - fail:
+              when: connection-refused
+              message: Connection to port 24 was refused.
+          - warn:
+              when: address-in-use
+              message: Another process was already listening on port 24.
+          - fail:
+              when: connection-timeout
+              message: Timed out connecting to port 24.
+          - fail:
+              when: error
+              message: Unexpected port status
+          - pass:
+              when: connected
+              message: Port 24 is available
+          - warn:
+              message: Unexpected port status
+    - tcpPortStatus:
+        checkName: Port 22
+        collectorName: Port 22
+        outcomes:
+          - fail:
+              when: connection-refused
+              message: Connection to port 22 was refused.
+          - fail:
+              when: address-in-use
+              message: Another process was already listening on port 22.
+          - fail:
+              when: connection-timeout
+              message: Timed out connecting to port 22.
+          - fail:
+              when: error
+              message: Unexpected port status
+          - pass:
+              when: connected
+              message: Port 22 is available
+          - warn:
+              message: Unexpected port status
+"
+
+preflight_with_warning="
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+spec:
+  collectors:
+    - tcpPortStatus:
+        collectorName: Port 24
+        port: 24
+    - tcpPortStatus:
+        collectorName: Port 22
+        port: 22
+  analyzers:
+    - tcpPortStatus:
+        checkName: Port 24
+        collectorName: Port 24
+        outcomes:
+          - fail:
+              when: connection-refused
+              message: Connection to port 24 was refused.
+          - warn:
+              when: address-in-use
+              message: Another process was already listening on port 24.
+          - fail:
+              when: connection-timeout
+              message: Timed out connecting to port 24.
+          - fail:
+              when: error
+              message: Unexpected port status
+          - pass:
+              when: connected
+              message: Port 24 is available
+          - warn:
+              message: Unexpected port status
+    - tcpPortStatus:
+        checkName: Port 22
+        collectorName: Port 22
+        outcomes:
+          - fail:
+              when: connection-refused
+              message: Connection to port 22 was refused.
+          - warn:
+              when: address-in-use
+              message: Another process was already listening on port 22.
+          - fail:
+              when: connection-timeout
+              message: Timed out connecting to port 22.
+          - fail:
+              when: error
+              message: Unexpected port status
+          - pass:
+              when: connected
+              message: Port 22 is available
+          - warn:
+              message: Unexpected port status
+"
+
+embed_preflight() {
+    content="$1"
+    rm -rf /root/preflight*
+    echo "$content" > /root/preflight.yaml
+    tar -czvf /root/preflight.tar.gz /root/preflight.yaml
+    objcopy --input-target binary --output-target binary --rename-section .data=sec_bundle /root/preflight.tar.gz /root/preflight.o
+    rm -rf /usr/local/bin/helmvm
+    cp -Rfp /usr/local/bin/helmvm-copy /usr/local/bin/helmvm
+    objcopy --add-section sec_bundle=/root/preflight.o /usr/local/bin/helmvm
+}
+
+has_applied_host_preflight() {
+    if ! grep -q "Port 24 is available" /tmp/log ; then
+        return 1
+    fi
+    if ! grep -q "Another process was already listening on port 22" /tmp/log ; then
+        return 1
+    fi
+}
+
+wait_for_healthy_node() {
+    ready=$(kubectl get nodes | grep -v NotReady | grep -c Ready || true)
+    counter=0
+    while [ "$ready" -lt "1" ]; do
+        if [ "$counter" -gt 36 ]; then
+            return 1
+        fi
+        sleep 5
+        counter=$((counter+1))
+        echo "Waiting for node to be ready"
+        ready=$(kubectl get nodes | grep -v NotReady | grep -c Ready || true)
+        kubectl get nodes || true
+    done
+    return 0
+}
+
+main() {
+    cp -Rfp /usr/local/bin/helmvm /usr/local/bin/helmvm-copy
+    embed_preflight "$preflight_with_failure"
+    if helmvm install --no-prompt 2>&1 | tee /tmp/log ; then
+        cat /tmp/log
+        echo "Expected installation to fail"
+        exit 1
+    fi
+    if ! has_applied_host_preflight; then
+        echo "Install hasn't applied host preflight"
+        cat /tmp/log
+        exit 1
+    fi
+    mv /tmp/log /tmp/log-failure
+    embed_preflight "$preflight_with_warning"
+    if ! helmvm install --no-prompt 2>&1 | tee /tmp/log ; then
+        cat /etc/os-release
+        echo "Failed to install helmvm"
+        exit 1
+    fi
+    if ! grep -q "You can now access your cluster" /tmp/log; then
+        echo "Failed to install helmvm"
+        exit 1
+    fi
+    if ! has_applied_host_preflight; then
+        echo "Install hasn't applied host preflight"
+        cat /tmp/log
+        exit 1
+    fi
+    if ! wait_for_healthy_node; then
+        echo "Failed to install helmvm"
+        exit 1
+    fi
+}
+
+export KUBECONFIG=/root/.helmvm/etc/kubeconfig
+export PATH=$PATH:/root/.helmvm/bin
+main