Handle containerExec hooks when using containerd

kubernetes · Jun 24, 2021 · 1918153 · 1918153
1 parent cb179b3
commit 1918153
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 9 deletions.
diff --git a/nodeup/pkg/model/convenience.go b/nodeup/pkg/model/convenience.go
@@ -36,11 +36,11 @@ func b(v bool) *bool {
 	return fi.Bool(v)
 }
 
-// buildDockerEnvironmentVars just converts a series of keypairs to docker environment variables switches
-func buildDockerEnvironmentVars(env map[string]string) []string {
+// buildContainerRuntimeEnvironmentVars just converts a series of keypairs to docker environment variables switches
+func buildContainerRuntimeEnvironmentVars(env map[string]string) []string {
 	var list []string
 	for k, v := range env {
-		list = append(list, []string{"-e", fmt.Sprintf("%s=%s", k, v)}...)
+		list = append(list, []string{"--env", fmt.Sprintf("%s=%s", k, v)}...)
 	}
 
 	return list

diff --git a/nodeup/pkg/model/hooks.go b/nodeup/pkg/model/hooks.go
@@ -124,8 +124,17 @@ func (h *HookBuilder) buildSystemdService(name string, hook *kops.HookSpec) (*no
 		case nil:
 			unit.SetSection("Service", hook.Manifest)
 		default:
-			if err := h.buildDockerService(unit, hook); err != nil {
-				return nil, err
+			switch h.Cluster.Spec.ContainerRuntime {
+			case "containerd":
+				if err := h.buildContainerdService(unit, hook, name); err != nil {
+					return nil, err
+				}
+			case "docker":
+				if err := h.buildDockerService(unit, hook); err != nil {
+					return nil, err
+				}
+			default:
+				return nil, fmt.Errorf("unknown container runtime %q", h.Cluster.Spec.ContainerRuntime)
 			}
 		}
 		definition = s(unit.Render())
@@ -141,6 +150,41 @@ func (h *HookBuilder) buildSystemdService(name string, hook *kops.HookSpec) (*no
 	return service, nil
 }
 
+// buildContainerdService is responsible for generating a containerd exec unit file
+func (h *HookBuilder) buildContainerdService(unit *systemd.Manifest, hook *kops.HookSpec, name string) error {
+	containerdImage := hook.ExecContainer.Image
+	if !strings.Contains(containerdImage, "/") {
+		containerdImage = "docker.io/library/" + containerdImage
+	}
+	if !strings.Contains(containerdImage, ":") {
+		containerdImage = containerdImage + ":latest"
+	}
+
+	containerdArgs := []string{
+		"/usr/bin/ctr", "--namespace", "k8s.io", "run",
+		"--mount", "type=bind,src=/,dst=/rootfs,options=rbind:rslave",
+		"--mount", "type=bind,src=/var/run/dbus,dst=/var/run/dbus,options=rbind:rprivate",
+		"--mount", "type=bind,src=/run/systemd,dst=/run/systemd,options=rbind:rprivate",
+		"--net-host",
+		"--privileged",
+	}
+	containerdArgs = append(containerdArgs, buildContainerRuntimeEnvironmentVars(hook.ExecContainer.Environment)...)
+	containerdArgs = append(containerdArgs, containerdImage)
+	containerdArgs = append(containerdArgs, name)
+	containerdArgs = append(containerdArgs, hook.ExecContainer.Command...)
+
+	containerdRunCommand := systemd.EscapeCommand(containerdArgs)
+	containerdPullCommand := systemd.EscapeCommand([]string{"/usr/bin/ctr", "--namespace", "k8s.io", "image", "pull", containerdImage})
+
+	unit.Set("Unit", "Requires", "containerd.service")
+	unit.Set("Service", "ExecStartPre", containerdPullCommand)
+	unit.Set("Service", "ExecStart", containerdRunCommand)
+	unit.Set("Service", "Type", "oneshot")
+	unit.Set("Install", "WantedBy", "multi-user.target")
+
+	return nil
+}
+
 // buildDockerService is responsible for generating a docker exec unit file
 func (h *HookBuilder) buildDockerService(unit *systemd.Manifest, hook *kops.HookSpec) error {
 	dockerArgs := []string{
@@ -151,7 +195,7 @@ func (h *HookBuilder) buildDockerService(unit *systemd.Manifest, hook *kops.Hook
 		"--net=host",
 		"--privileged",
 	}
-	dockerArgs = append(dockerArgs, buildDockerEnvironmentVars(hook.ExecContainer.Environment)...)
+	dockerArgs = append(dockerArgs, buildContainerRuntimeEnvironmentVars(hook.ExecContainer.Environment)...)
 	dockerArgs = append(dockerArgs, hook.ExecContainer.Image)
 	dockerArgs = append(dockerArgs, hook.ExecContainer.Command...)
 

diff --git a/nodeup/pkg/model/tests/golden/hooks-containerd-exec/tasks-hooks.yaml b/nodeup/pkg/model/tests/golden/hooks-containerd-exec/tasks-hooks.yaml
@@ -2,11 +2,11 @@ Name: kops-hook-0.service
 definition: |
   [Unit]
   Description=Kops Hook kops-hook-0
-  Requires=docker.service
+  Requires=containerd.service
 
   [Service]
-  ExecStartPre=/usr/bin/docker pull busybox
-  ExecStart=/usr/bin/docker run -v /:/rootfs/ -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd --net=host --privileged busybox sh -c "chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common"
+  ExecStartPre=/usr/bin/ctr --namespace k8s.io image pull docker.io/library/busybox:latest
+  ExecStart=/usr/bin/ctr --namespace k8s.io run --mount type=bind,src=/,dst=/rootfs,options=rbind:rslave --mount type=bind,src=/var/run/dbus,dst=/var/run/dbus,options=rbind:rprivate --mount type=bind,src=/run/systemd,dst=/run/systemd,options=rbind:rprivate --net-host --privileged docker.io/library/busybox:latest kops-hook-0 sh -c "chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common"
   Type=oneshot
 
   [Install]