feat: restart mirror on binary change (#461)

if the binary has changed the mirror should stop so systemd can restart
it using the new binary.

Author: ricardomaraschini

cmd/embedded-cluster/install.go

@@ -34,11 +34,6 @@ var ErrNothingElseToAdd = fmt.Errorf("")
 // service is responsible for serving on localhost, through http, all files that are used
 // during a cluster upgrade.
 func installAndEnableLocalArtifactMirror() error {
-	ourbin := defaults.PathToEmbeddedClusterBinary("local-artifact-mirror")
-	hstbin := defaults.LocalArtifactMirrorPath()
-	if err := helpers.MoveFile(ourbin, hstbin); err != nil {
-		return fmt.Errorf("unable to move local artifact mirror binary: %w", err)
-	}
 	if err := goods.MaterializeLocalArtifactMirrorUnitFile(); err != nil {
 		return fmt.Errorf("failed to materialize artifact mirror unit: %w", err)
 	}

cmd/embedded-cluster/uninstall.go

@@ -402,9 +402,6 @@ var resetCommand = &cli.Command{
 			if err := os.RemoveAll(lamPath); err != nil {
 				return err
 			}
-			if err := os.RemoveAll(defaults.LocalArtifactMirrorPath()); err != nil {
-				return err
-			}
 		}
 
 		if _, err := os.Stat(defaults.EmbeddedClusterHomeDirectory()); err == nil {

cmd/local-artifact-mirror/serve.go

@@ -37,6 +37,10 @@ var serveCommand = &cli.Command{
 		stop := make(chan os.Signal, 1)
 		signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
 
+		if err := startBinaryWatcher(stop); err != nil {
+			panic(err)
+		}
+
 		server := &http.Server{Addr: "127.0.0.1:50000"}
 		go func() {
 			fmt.Println("Starting server on 127.0.0.1:50000")
@@ -60,6 +64,34 @@ var serveCommand = &cli.Command{
 	},
 }
 
+// startBinaryWatcher starts a loop that observes the binary until its modification
+// time changes. When the modification time changes a SIGTERM is send in the provided
+// channel.
+func startBinaryWatcher(stop chan os.Signal) error {
+	fpath := defaults.PathToEmbeddedClusterBinary("local-artifact-mirror")
+	stat, err := os.Stat(fpath)
+	if err != nil {
+		return fmt.Errorf("unable to stat %s: %s", fpath, err)
+	}
+	lastmod := stat.ModTime()
+	go func() {
+		fmt.Println("Watching for changes in the binary")
+		ticker := time.NewTicker(5 * time.Second)
+		for range ticker.C {
+			if stat, err = os.Stat(fpath); err != nil {
+				fmt.Println("Unable to stat binary:", err)
+				continue
+			}
+			if stat.ModTime().Equal(lastmod) {
+				continue
+			}
+			fmt.Println("Binary changed, sending signal to stop")
+			stop <- syscall.SIGTERM
+		}
+	}()
+	return nil
+}
+
 // logAndFilterRequest is a middleware that logs the HTTP request details. Returns 404
 // if attempting to read the log files as those are not served by this server.
 func logAndFilterRequest(handler http.Handler) http.Handler {

e2e/local-artifact-mirror_test.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"strings"
 	"testing"
 	"time"
 
@@ -54,5 +55,31 @@ func TestLocalArtifactMirror(t *testing.T) {
 		t.Fatalf("we should not be able to fetch paths with ../")
 	}
 
+	t.Logf("testing local artifact mirror restart after materialize")
+	command = []string{"embedded-cluster", "materialize"}
+	if _, _, err := RunCommandOnNode(t, tc, 0, command); err != nil {
+		t.Fatalf("fail materialize embedded cluster binaries: %v", err)
+	}
+
+	t.Logf("waiting to verify if local artifact mirror has restarted")
+	time.Sleep(20 * time.Second)
+
+	command = []string{"journalctl", "-u", "local-artifact-mirror"}
+	stdout, _, err := RunCommandOnNode(t, tc, 0, command)
+	if err != nil {
+		t.Fatalf("fail to get journalctl logs: %v", err)
+	}
+
+	expected := []string{
+		"Binary changed, sending signal to stop",
+		"Scheduled restart job, restart counter is at",
+	}
+	for _, str := range expected {
+		if !strings.Contains(stdout, str) {
+			t.Fatalf("expected %q in journalctl logs, got %q", str, stdout)
+		}
+		t.Logf("found %q in journalctl logs", str)
+	}
+
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }

pkg/defaults/defaults.go

@@ -64,11 +64,6 @@ func K0sBinaryPath() string {
 	return def().K0sBinaryPath()
 }
 
-// LocalArtifactMirrorPath calls LocalArtifactMirrorPath on the default provider.
-func LocalArtifactMirrorPath() string {
-	return def().LocalArtifactMirrorPath()
-}
-
 // PathToEmbeddedClusterBinary calls PathToEmbeddedClusterBinary on the default provider.
 func PathToEmbeddedClusterBinary(name string) string {
 	return def().PathToEmbeddedClusterBinary(name)

pkg/defaults/provider.go

@@ -105,12 +105,6 @@ func (d *Provider) K0sBinaryPath() string {
 	return "/usr/local/bin/k0s"
 }
 
-// LocalArtifactMirrorPath returns the path where we install the local artifact mirror
-// binary.
-func (d *Provider) LocalArtifactMirrorPath() string {
-	return "/usr/local/bin/local-artifact-mirror"
-}
-
 // PathToEmbeddedClusterBinary is an utility function that returns the full path to a
 // materialized binary that belongs to embedded-cluster. This function does not check
 // if the file exists.

pkg/goods/systemd/local-artifact-mirror.service

@@ -2,7 +2,7 @@
 Description=Embedded Cluster Local Artifact Mirror
 
 [Service]
-ExecStart=/usr/local/bin/local-artifact-mirror serve
+ExecStart=/var/lib/embedded-cluster/bin/local-artifact-mirror serve
 Restart=always
 RestartSec=5s