Implement Nix compatibility

Signed-off-by: main <magic_rb@redalder.org>

containerd/containerd.go

@@ -35,7 +35,7 @@ import (
 )
 
 type ContainerConfig struct {
-	Image                 containerd.Image
+	RootFSPath            string
 	ContainerName         string
 	ContainerSnapshotName string
 	NetworkNamespacePath  string
@@ -115,16 +115,17 @@ func (d *Driver) pullImage(imageName, imagePullTimeout string, auth *RegistryAut
 	return d.client.Pull(ctxWithTimeout, named.String(), pullOpts...)
 }
 
-func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskConfig) (containerd.Container, error) {
+func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskConfig, mainStorePath string) (containerd.Container, error) {
 	if config.Command != "" && config.Entrypoint != nil {
 		return nil, fmt.Errorf("Both command and entrypoint are set. Only one of them needs to be set.")
 	}
 
 	// Entrypoint or Command set by the user, to override entrypoint or cmd defined in the image.
 	var args []string
 	if config.Command != "" {
-		args = append(args, config.Command)
+		args = append(args, mainStorePath + "/" + config.Command)
 	} else if config.Entrypoint != nil && config.Entrypoint[0] != "" {
+		config.Entrypoint[0] = mainStorePath + "/" + config.Entrypoint[0]
 		args = append(args, config.Entrypoint...)
 	}
 
@@ -136,13 +137,13 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC
 	var opts []oci.SpecOpts
 
 	if config.Entrypoint != nil {
-		opts = append(opts, oci.WithImageConfig(containerConfig.Image))
+		// opts = append(opts, oci.WithImageConfig(containerConfig.Image))
 		// WithProcessArgs replaces the args on the generated spec.
 		opts = append(opts, oci.WithProcessArgs(args...))
 	} else {
 		// WithImageConfigArgs configures the spec to from the configuration of an Image
 		// with additional args that replaces the CMD of the image.
-		opts = append(opts, oci.WithImageConfigArgs(containerConfig.Image, args))
+		// opts = append(opts, oci.WithImageConfigArgs(containerConfig.Image, args))
 	}
 
 	if !d.config.AllowPrivileged && config.Privileged {
@@ -336,11 +337,13 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC
 	ctxWithTimeout, cancel := context.WithTimeout(d.ctxContainerd, 30*time.Second)
 	defer cancel()
 
+	opts = append(opts, oci.WithRootFSPath(containerConfig.RootFSPath))
+
 	return d.client.NewContainer(
 		ctxWithTimeout,
 		containerConfig.ContainerName,
 		containerd.WithRuntime(d.config.ContainerdRuntime, nil),
-		containerd.WithNewSnapshot(containerConfig.ContainerSnapshotName, containerConfig.Image),
+		//containerd.WithNewSnapshot(containerConfig.ContainerSnapshotName, containerConfig.Image),
 		containerd.WithNewSpec(opts...),
 	)
 }

containerd/driver.go

@@ -23,6 +23,11 @@ import (
 	"syscall"
 	"time"
 
+	"os/exec"
+	"os"
+	"strings"
+	"encoding/json"
+
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/hashicorp/consul-template/signals"
@@ -89,14 +94,30 @@ var (
 			"username": hclspec.NewAttr("username", "string", true),
 			"password": hclspec.NewAttr("password", "string", true),
 		})),
+
+		"nix_executable": hclspec.NewDefault(
+			hclspec.NewAttr("nix_executable", "string", false),
+			hclspec.NewLiteral(`"nix"`),
+		),
+		"gc_roots_root": hclspec.NewDefault(
+			hclspec.NewAttr("gc_roots_root", "string", false),
+			hclspec.NewLiteral(`"/nix/var/nix/gcroots/containerd-nix"`),
+		),
+		"rootfs_root": hclspec.NewDefault(
+			hclspec.NewAttr("rootfs_root", "string", false),
+			hclspec.NewLiteral(`"/var/containerd-nix/rootfs"`),
+		),
 	})
 
 	// taskConfigSpec is the specification of the plugin's configuration for
 	// a task
 	// this is used to validate the configuration specified for the plugin
 	// when a job is submitted.
 	taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{
-		"image":      hclspec.NewAttr("image", "string", true),
+		// "image":      hclspec.NewAttr("image", "string", true),
+		"flake_ref":  hclspec.NewAttr("flake_ref", "string", true),
+		"flake_sha":  hclspec.NewAttr("flake_sha", "string", false),
+
 		"command":    hclspec.NewAttr("command", "string", false),
 		"args":       hclspec.NewAttr("args", "list(string)", false),
 		"cap_add":    hclspec.NewAttr("cap_add", "list(string)", false),
@@ -156,6 +177,10 @@ type Config struct {
 	StatsInterval     string       `codec:"stats_interval"`
 	AllowPrivileged   bool         `codec:"allow_privileged"`
 	Auth              RegistryAuth `codec:"auth"`
+
+	NixExecutable     string       `codec:"nix_executable"`
+	GCRootsRoot       string       `codec:"gc_roots_root"`
+	RootFSRoot        string       `codec:"rootfs_root"`
 }
 
 // Volume, bind, and tmpfs type mounts are supported.
@@ -176,7 +201,9 @@ type RegistryAuth struct {
 // TaskConfig contains configuration information for a task that runs with
 // this plugin
 type TaskConfig struct {
-	Image            string             `codec:"image"`
+	FlakeRef         string             `codec:"flake_ref"`
+	FlakeSha         string             `codec:"flake_sha"`
+
 	Command          string             `codec:"command"`
 	Args             []string           `codec:"args"`
 	CapAdd           []string           `codec:"cap_add"`
@@ -410,6 +437,162 @@ func (d *Driver) buildFingerprint() *drivers.Fingerprint {
 	return fp
 }
 
+func (d *Driver) NixGetDeps(executable string, flakeRef string) ([]string, error) {
+	nixDepsCmd := &exec.Cmd {
+		Path: executable,
+		Args: []string{
+			executable,
+			"path-info",
+			"-r",
+			flakeRef,
+		},
+	}
+	res, err := nixDepsCmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get dependencies of built flake-ref %s", flakeRef)
+	}
+	deps := strings.Split(strings.Trim(string(res), " \n"), "\n")
+
+	return deps, nil
+}
+
+func (d *Driver) NixBuildFlake(executable string, flakeRef string, flakeSha string) error {
+	flakeHost := strings.Split(flakeRef, "#")
+
+	if len(flakeHost) != 2 {
+		return fmt.Errorf("Invalid flake ref.")
+	}
+
+	nixShaCmd := &exec.Cmd {
+		Path: executable,
+		Args: []string{
+			executable,
+			"flake",
+			"metadata",
+			"--json",
+			flakeHost[0],
+		},
+	}
+	nixSha, err := nixShaCmd.Output()
+	if err != nil {
+		return fmt.Errorf("failed to get sha for flake-ref %s with %s:\n %s", flakeRef, err, string(nixSha))
+	}
+
+	var shaJson map[string]interface{}
+	err = json.Unmarshal(nixSha, &shaJson)
+
+	if err != nil {
+		return fmt.Errorf("failed to parse json %s", err)
+	}
+
+	lockedVal, ok := shaJson["locked"].(map[string]interface{})
+	if !ok {
+		return fmt.Errorf("failed to parse `nix flake metadata` output")
+	}
+	fetchedSha, ok := lockedVal["narHash"].(string)
+	if !ok {
+		return fmt.Errorf("failed to parse `nix flake metadata` output")
+	}
+
+	if string(fetchedSha) != flakeSha {
+		return fmt.Errorf("pinned flake sha doesn't match: \"%s\" != \"%s\"", flakeSha, fetchedSha)
+	}
+
+	nixBuildCmd := &exec.Cmd {
+		Path: executable,
+		Args: []string{
+			executable,
+			"build",
+			"--no-link",
+			flakeRef,
+		},
+	}
+	res, err := nixBuildCmd.Output()
+	if err != nil {
+		return fmt.Errorf("failed to build flake-ref %s with %s:\n %s", flakeRef, err, string(res))
+	}
+
+	return nil
+}
+
+func (d *Driver) NixGetStorePath(executable string, flakeRef string) (string, error) {
+	nixEvalCmd := exec.Cmd {
+		Path: executable,
+		Args: []string{
+			executable,
+			"eval",
+			"--raw",
+			flakeRef + ".outPath",
+		},
+	}
+
+	storePath, err := nixEvalCmd.Output()
+	if err != nil {
+		return "", fmt.Errorf("failed to get store path of %s", flakeRef)
+	}
+	return string(storePath), nil
+}
+
+func (d *Driver) GetGCRoot(containerName string, allocationId string) string {
+	return fmt.Sprintf("%s/%s-%s", d.config.GCRootsRoot, containerName, allocationId)
+}
+
+func (d *Driver) GetRootFSPath(containerName string, allocationId string) string {
+	return fmt.Sprintf("%s/%s-%s", d.config.RootFSRoot, containerName, allocationId)
+}
+
+func (d *Driver) SetupRootFS(flakeRef string, containerName string, allocationId string, flakeSha string) (string, string, error) {
+	nixExecutable, err := exec.LookPath("nix")
+	if err != nil {
+		return "", "", fmt.Errorf("failed to find `nix` executable")
+	}
+
+	err = d.NixBuildFlake(nixExecutable, flakeRef, flakeSha)
+	if err != nil {
+		return "", "", err
+	}
+
+	deps, err := d.NixGetDeps(nixExecutable, flakeRef)
+	if err != nil {
+		return "", "", err
+	}
+
+	rootFS := d.GetRootFSPath(containerName, allocationId)
+	os.MkdirAll(rootFS, 0755)
+
+	for _, dep := range deps {
+		target := fmt.Sprintf("%s%s", rootFS, dep)
+
+		info, err := os.Stat(dep)
+		if os.IsNotExist(err) {
+			return "", "", fmt.Errorf("store path reported as dep but does no exist %s: %s", dep, err)
+		}
+		if info.IsDir() {
+			os.MkdirAll(target, 0755)
+		} else {
+			os.Create(target)
+		}
+
+		err32 := syscall.Mount(dep, target, "", syscall.MS_BIND, "")
+		if err32 != nil {
+			return "", "", fmt.Errorf("failed to bind mount store path %s: %v", dep, err)
+		}
+	}
+
+	storePath, err := d.NixGetStorePath(nixExecutable, flakeRef)
+	if err != nil {
+		return "", "", err
+	}
+
+	// Create GC-Root
+	gcRoot := d.GetGCRoot(containerName, allocationId)
+	os.MkdirAll(d.config.GCRootsRoot, 0755)
+
+	os.Symlink(storePath, gcRoot)
+
+	return rootFS, deps[len(deps)-1], nil
+}
+
 // StartTask returns a task handle and a driver network if necessary.
 func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) {
 	if _, ok := d.tasks.Get(cfg.ID); ok {
@@ -444,13 +627,20 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 	}
 	containerConfig.ContainerName = containerName
 
+	// var err error
+	// containerConfig.Image, err = d.pullImage(driverConfig.Image, driverConfig.ImagePullTimeout, &driverConfig.Auth)
+	// if err != nil {
+	//	return nil, nil, fmt.Errorf("Error in pulling image %s: %v", driverConfig.Image, err)
+	// }
+
 	var err error
-	containerConfig.Image, err = d.pullImage(driverConfig.Image, driverConfig.ImagePullTimeout, &driverConfig.Auth)
+	rootFSPath, mainStorePath, err := d.SetupRootFS(driverConfig.FlakeRef, cfg.Name, cfg.AllocID, driverConfig.FlakeSha)
 	if err != nil {
-		return nil, nil, fmt.Errorf("Error in pulling image %s: %v", driverConfig.Image, err)
+		return nil, nil, fmt.Errorf("Error in building rootfs from flake-ref %s: %v", driverConfig.FlakeRef, err)
 	}
+	containerConfig.RootFSPath = rootFSPath
 
-	d.logger.Info(fmt.Sprintf("Successfully pulled %s image\n", containerConfig.Image.Name()))
+	d.logger.Info(fmt.Sprintf("Successfully created rootfs from %s flake-ref\n", driverConfig.FlakeRef))
 
 	// Setup environment variables.
 	for key, val := range cfg.Env {
@@ -483,7 +673,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 
 	containerConfig.User = cfg.User
 
-	container, err := d.createContainer(&containerConfig, &driverConfig)
+	container, err := d.createContainer(&containerConfig, &driverConfig, mainStorePath)
 	if err != nil {
 		return nil, nil, fmt.Errorf("Error in creating container: %v", err)
 	}
@@ -647,6 +837,37 @@ func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) e
 		return fmt.Errorf("Shutdown failed: %v", err)
 	}
 
+	var driverConfig TaskConfig
+	if err := handle.taskConfig.DecodeDriverConfig(&driverConfig); err != nil {
+		return fmt.Errorf("failed to decode driver config: %v", err)
+	}
+
+	nixExecutable, err := exec.LookPath("nix")
+	if err != nil {
+		return fmt.Errorf("failed to find `nix` executable")
+	}
+
+	deps, err := d.NixGetDeps(nixExecutable, driverConfig.FlakeRef)
+	if err != nil {
+		return err
+	}
+
+	rootFSPath := d.GetRootFSPath(handle.taskConfig.Name, handle.taskConfig.AllocID)
+
+	for _, dep := range deps {
+		err := syscall.Unmount(rootFSPath + "/" + dep, 0)
+		if err != nil {
+			return err
+		}
+	}
+
+
+	// os.RemoveAll(rootFSPath)
+
+	// gcRoot := d.GetGCRoot(handle.taskConfig.Name, handle.taskConfig.AllocID)
+
+	// os.Remove(gcRoot)
+
 	return nil
 }
 

example/website.nomad

@@ -0,0 +1,29 @@
+job "hello" {
+  datacenters = ["dc1"]
+
+  group "hello-group" {
+    # network {
+    #   mode = "bridge"
+    #   port "http" {
+    # 	static = 80
+    # 	to = 80
+    #   }
+    # }
+    task "hello-task" {
+      driver = "containerd-driver"
+
+      config {
+	flake_ref = "git+http://gitea.redalder.org/RedAlder/systems#nixngSystems.website.config.system.build.toplevel"
+	flake_sha = "sha256-+muNSb2JOG1Gps5Xm7BsOSHoJPMEERdUXkp3UM4mhJA="
+	entrypoint = [ "init" ]
+      }
+
+
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+  }
+}

extra_config.hcl

@@ -0,0 +1,7 @@
+plugin "nomad-driver-containerd" {
+  config {
+    enabled = true
+    containerd_runtime = "io.containerd.runc.v2"
+    stats_interval = "5s"
+  }
+}