Volumemgr support for kubevirt eve

    Implements the volumehandler interface functions like CreateVolume, DestroyVolume etc. for PersistentVolumes (Kubernetes based volumes)
    Though longhorn can support filemode volumes we only create blockmode volumes since eve only deals with block volumes.

    CreateVolume() does the following:
     1) A blank volume create request, creates a blockmode PVC
     2) A downloaded volume qcow2 is converted to blockmode PVC
     3) For container images, we follow the same path of extracting rootfs to /persist/vault/volumes/*container directory
        once the rootfs is laid out, the files required for runx-initrd are generated and copied into /persist/vault/volumes/*container
        The directory content will look like this:
        4b6da250-6608-4686-a9fd-89e16f39ded6:/persist/vault/volumes/fe328d99-a380-45a7-99af-612b6264f628#0.container# ls
        cmdline            environment        image-config.json  modules            rootfs             ug
        This entire directory is first copied to a qcow2 using copy-image-to-qcow.sh script
        The qcow2 is copied to PVC, which will be used as bootdisk for the VM.
        The qcow2 will be deleted to save space.

    Volume snapshots are not supported yet and a corresponding error is returned.
    Kubenertes naming convention does not support "#", so all PVC are keyed with -pvc- instead of "#"
    Added docker.io/ prefix to the container images inserted into the containerd repository

    Addressed all review comments and squashed to one commit

Signed-off-by: Pramodh Pallapothu <pramodh@zededa.com>

pkg/pillar/Dockerfile

@@ -157,6 +157,7 @@ ADD scripts/device-steps.sh \
     scripts/handlezedserverconfig.sh \
     scripts/veth.sh \
     scripts/dhcpcd.sh \
+    scripts/copy-image-to-qcow.sh \
   /out/opt/zededa/bin/
 ADD conf/lisp.config.base /out/var/tmp/zededa/lisp.config.base
 

pkg/pillar/Makefile

@@ -94,7 +94,7 @@ test: build-docker-test
 		--entrypoint /final/opt/gotestsum $(DOCKER_TAG) \
 		--jsonfile /pillar/results.json \
 		--junitfile /pillar/results.xml \
-		--raw-command -- go test -coverprofile=coverage.txt -covermode=atomic -race -json \
+		--raw-command -- go test -tags kubevirt -coverprofile=coverage.txt -covermode=atomic -race -json \
 		./...
 	docker run --platform linux/$(ZARCH) -w /pillar \
 		--entrypoint /bin/sh $(DOCKER_TAG) \

pkg/pillar/cas/containerd.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -18,6 +19,7 @@ import (
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/mount"
 	"github.com/lf-edge/edge-containers/pkg/resolver"
+	"github.com/lf-edge/eve/pkg/pillar/base"
 	"github.com/lf-edge/eve/pkg/pillar/containerd"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 	"github.com/moby/sys/mountinfo"
@@ -679,13 +681,6 @@ func (c *containerdCAS) PrepareContainerRootDir(rootPath, reference, rootBlobSha
 		logrus.Errorf(err.Error())
 		return err
 	}
-	mountpoints := clientImageSpec.Config.Volumes
-	execpath := clientImageSpec.Config.Entrypoint
-	cmd := clientImageSpec.Config.Cmd
-	workdir := clientImageSpec.Config.WorkingDir
-	unProcessedEnv := clientImageSpec.Config.Env
-	logrus.Infof("PrepareContainerRootDir: mountPoints %+v execpath %+v cmd %+v workdir %+v env %+v",
-		mountpoints, execpath, cmd, workdir, unProcessedEnv)
 	clientImageSpecJSON, err := getJSON(clientImageSpec)
 	if err != nil {
 		err = fmt.Errorf("PrepareContainerRootDir: Could not build json of image: %v. %v",
@@ -705,6 +700,122 @@ func (c *containerdCAS) PrepareContainerRootDir(rootPath, reference, rootBlobSha
 		logrus.Errorf(err.Error())
 		return err
 	}
+	if base.IsHVTypeKube() {
+		err := c.prepareContainerRootDirForKubevirt(clientImageSpec, snapshotID, rootPath)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *containerdCAS) prepareContainerRootDirForKubevirt(clientImageSpec *spec.Image, snapshotID string, rootPath string) error {
+
+	// On kubevirt eve we also write the mountpoints and cmdline files to the rootPath.
+	// Once rootPath is populated with all necessary files, this rootPath directory will be
+	// converted to a PVC and passed in to domainmgr to attach to VM as rootdisk
+	// NOTE: For kvm eve these files are generated and passed to bootloader in pkg/pillar/containerd/oci.go:AddLoader()
+
+	mountpoints := clientImageSpec.Config.Volumes
+	execpath := clientImageSpec.Config.Entrypoint
+	cmd := clientImageSpec.Config.Cmd
+	workdir := clientImageSpec.Config.WorkingDir
+	unProcessedEnv := clientImageSpec.Config.Env
+	user := clientImageSpec.Config.User
+	logrus.Infof("PrepareContainerRootDir: mountPoints %+v execpath %+v cmd %+v workdir %+v env %+v user %+v",
+		mountpoints, execpath, cmd, workdir, unProcessedEnv, user)
+
+	// Mount the snapshot
+	// Do not unmount the rootfs in this code path, we need this containerdir to generate a qcow2->PVC from it in
+	// pkg/pillar/volumehandlers/csihandler.go
+	// Once qcow2 is successfully created, this directory will be unmounted in csihandler.go
+	if err := c.MountSnapshot(snapshotID, GetRoofFsPath(rootPath)); err != nil {
+		return fmt.Errorf("PrepareContainerRootDir error mount of snapshot: %s. %s", snapshotID, err)
+	}
+
+	err := c.writeKubevirtMountpointsFile(mountpoints, rootPath)
+	if err != nil {
+		return err
+	}
+
+	// create env manifest
+	envContent := ""
+	if workdir != "" {
+		envContent = fmt.Sprintf("export WORKDIR=\"%s\"\n", workdir)
+	} else {
+		envContent = fmt.Sprintf("export WORKDIR=/\n")
+	}
+	for _, e := range unProcessedEnv {
+		keyAndValueSlice := strings.SplitN(e, "=", 2)
+		if len(keyAndValueSlice) == 2 {
+			//handles Key=Value case
+			envContent = envContent + fmt.Sprintf("export %s=\"%s\"\n", keyAndValueSlice[0], keyAndValueSlice[1])
+		} else {
+			//handles Key= case
+			envContent = envContent + fmt.Sprintf("export %s\n", e)
+		}
+	}
+	if err := os.WriteFile(filepath.Join(rootPath, "environment"), []byte(envContent), 0644); err != nil {
+		return err
+	}
+
+	// create cmdline manifest
+	// each item needs to be independently quoted for initrd
+	execpathQuoted := make([]string, 0)
+
+	// This loop is just to pick the execpath and eliminate any square braces around it.
+	// Just pick /entrypoint.sh from [/entrypoint.sh]
+	for _, c := range execpath {
+		execpathQuoted = append(execpathQuoted, fmt.Sprintf("\"%s\"", c))
+	}
+	for _, s := range cmd {
+		execpathQuoted = append(execpathQuoted, fmt.Sprintf("\"%s\"", s))
+	}
+	command := strings.Join(execpathQuoted, " ")
+	if err := os.WriteFile(filepath.Join(rootPath, "cmdline"),
+		[]byte(command), 0644); err != nil {
+		return err
+	}
+
+	// Userid and GID are same
+	ug := "0 0"
+	if user != "" {
+		uid, converr := strconv.Atoi(user)
+		if converr != nil {
+			return converr
+		}
+		ug = fmt.Sprintf("%d %d", uid, uid)
+	}
+	if err := os.WriteFile(filepath.Join(rootPath, "ug"),
+		[]byte(ug), 0644); err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(filepath.Join(rootPath, "modules"), 0600)
+	return err
+}
+
+func (*containerdCAS) writeKubevirtMountpointsFile(mountpoints map[string]struct{}, rootPath string) error {
+	if len(mountpoints) > 0 {
+		f, err := os.OpenFile(filepath.Join(rootPath, "mountPoints"), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			err = fmt.Errorf("PrepareContainerRootDir: Exception while creating file  mountpoints to %v %v",
+				rootPath, err)
+			logrus.Error(err.Error())
+			return err
+		}
+
+		defer f.Close()
+		for m := range mountpoints {
+			m += "\n"
+			if _, err := f.WriteString(m); err != nil {
+				err = fmt.Errorf("PrepareContainerRootDir: Exception while writing mountpoints to %v %v",
+					rootPath, err)
+				logrus.Error(err.Error())
+				return err
+			}
+		}
+	}
 	return nil
 }
 

pkg/pillar/cas/containerd_test.go

@@ -0,0 +1,43 @@
+package cas
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestWriteKubevirtMountpointsFile(t *testing.T) {
+	cas := containerdCAS{}
+	mountPoints := map[string]struct{}{
+		"/media/floppy": {},
+		"/media/cdrom":  {},
+	}
+
+	dname, err := os.MkdirTemp("", "prefix")
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = cas.writeKubevirtMountpointsFile(mountPoints, dname)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	contentBytes, err := os.ReadFile(filepath.Join(dname, "mountPoints"))
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	content := string(contentBytes)
+
+	for mountPoint := range mountPoints {
+		if !strings.Contains(content, mountPoint) {
+			t.Fatalf("mountPoint %s is missing", mountPoint)
+		}
+	}
+
+	os.RemoveAll(filepath.Join(dname, "mountPoints"))
+}

pkg/pillar/cmd/volumemgr/handlecontent.go

@@ -10,6 +10,7 @@ import (
 
 	v1types "github.com/google/go-containerregistry/pkg/v1/types"
 	zconfig "github.com/lf-edge/eve-api/go/config"
+	"github.com/lf-edge/eve/pkg/pillar/base"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 	"github.com/lf-edge/eve/pkg/pillar/utils"
 	"github.com/lf-edge/eve/pkg/pillar/vault"
@@ -182,6 +183,7 @@ func createContentTreeStatus(ctx *volumemgrContext, config types.ContentTreeConf
 			DisplayName:       config.DisplayName,
 			State:             types.INITIAL,
 			Blobs:             []string{},
+			HVTypeKube:        base.IsHVTypeKube(),
 			// LastRefCountChangeTime: time.Now(),
 		}
 		populateDatastoreFields(ctx, config, status)
@@ -325,7 +327,8 @@ func doDeleteContentTree(ctx *volumemgrContext, status *types.ContentTreeStatus)
 	log.Functionf("doDeleteContentTree for %v", status.ContentID)
 	RemoveAllBlobsFromContentTreeStatus(ctx, status, status.Blobs...)
 	//We create a reference when we load the blobs. We should remove that reference when we delete the contentTree.
-	if err := ctx.casClient.RemoveImage(status.ReferenceID()); err != nil {
+	refName := status.ReferenceID()
+	if err := ctx.casClient.RemoveImage(refName); err != nil {
 		log.Errorf("doDeleteContentTree: exception while deleting image %s: %s",
 			status.RelativeURL, err.Error())
 	}

pkg/pillar/cmd/volumemgr/handlevolumeref.go

@@ -38,6 +38,7 @@ func handleVolumeRefCreate(ctxArg interface{}, key string,
 			VerifyOnly:             config.VerifyOnly,
 			Target:                 vs.Target,
 			CustomMeta:             vs.CustomMeta,
+			ReferenceName:          vs.ReferenceName,
 		}
 		if vs.HasError() {
 			description := vs.ErrorDescription
@@ -187,6 +188,7 @@ func updateVolumeRefStatus(ctx *volumemgrContext, vs *types.VolumeStatus) {
 				status.Target = vs.Target
 				status.CustomMeta = vs.CustomMeta
 				status.WWN = vs.WWN
+				status.ReferenceName = vs.ReferenceName
 				if vs.HasError() {
 					description := vs.ErrorDescription
 					description.ErrorEntities = []*types.ErrorEntity{{
@@ -215,6 +217,7 @@ func updateVolumeRefStatus(ctx *volumemgrContext, vs *types.VolumeStatus) {
 				WWN:                    vs.WWN,
 				VerifyOnly:             config.VerifyOnly,
 				Target:                 vs.Target,
+				ReferenceName:          vs.ReferenceName,
 			}
 			if vs.HasError() {
 				description := vs.ErrorDescription

pkg/pillar/cmd/volumemgr/handlework.go

@@ -235,8 +235,10 @@ func casIngestWorker(ctxPtr interface{}, w worker.Work) worker.WorkResult {
 		}
 	}
 
+	appImgName := status.ReferenceID()
+
 	// load the blobs
-	loadedBlobs, err := ctx.casClient.IngestBlobsAndCreateImage(status.ReferenceID(), *root, loadBlobs...)
+	loadedBlobs, err := ctx.casClient.IngestBlobsAndCreateImage(appImgName, *root, loadBlobs...)
 	// loadedBlobs are BlobStatus for the ones we loaded
 	for _, blob := range loadedBlobs {
 		d.loaded = append(d.loaded, blob.Sha256)

pkg/pillar/cmd/volumemgr/initialvolumestatus.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 
 	zconfig "github.com/lf-edge/eve-api/go/config"
+	"github.com/lf-edge/eve/pkg/pillar/kubeapi"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 	"github.com/lf-edge/eve/pkg/pillar/volumehandlers"
 	"github.com/lf-edge/eve/pkg/pillar/zfs"
@@ -66,6 +67,28 @@ func populateExistingVolumesFormatDatasets(_ *volumemgrContext, dataset string)
 	log.Functionf("populateExistingVolumesFormatDatasets(%s) Done", dataset)
 }
 
+// populateExistingVolumesFormatPVC iterates over the namespace and takes format
+// from the name of the volume/PVC and prepares map of it
+func populateExistingVolumesFormatPVC(_ *volumemgrContext) {
+
+	log.Functionf("populateExistingVolumesFormatPVC")
+	pvlist, err := kubeapi.GetPVCList(log)
+	if err != nil {
+		log.Errorf("populateExistingVolumesFormatPVC: GetPVCList failed: %v", err)
+		return
+	}
+	for _, pvcName := range pvlist {
+		tempStatus, err := getVolumeStatusByPVC(pvcName)
+		if err != nil {
+			log.Error(err)
+			continue
+		}
+		volumeFormat[tempStatus.Key()] = tempStatus.ContentFormat
+	}
+	log.Functionf("populateExistingVolumesFormatPVC Done")
+
+}
+
 // Periodic garbage collection looking at RefCount=0 files in the unknown
 // Others have their delete handler.
 func gcObjects(ctx *volumemgrContext, dirName string) {
@@ -117,6 +140,38 @@ func gcVolumes(ctx *volumemgrContext, locations []string) {
 	}
 }
 
+func getVolumeStatusByPVC(pvcName string) (*types.VolumeStatus, error) {
+	var encrypted bool
+	var parsedFormat int32
+	var volumeIDAndGeneration string
+
+	volumeIDAndGeneration = pvcName
+	parsedFormat = int32(zconfig.Format_PVC)
+
+	generation := strings.Split(volumeIDAndGeneration, "-pvc-")
+	volUUID, err := uuid.FromString(generation[0])
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse VolumeID: %s", err)
+	}
+	if len(generation) == 1 {
+		return nil, fmt.Errorf("cannot extract generation from PVC %s", pvcName)
+	}
+	// we cannot extract LocalGenerationCounter from the PVC name
+	// assume it is zero
+	generationCounter, err := strconv.ParseInt(generation[1], 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse GenerationCounter: %s", err)
+	}
+	vs := types.VolumeStatus{
+		VolumeID:          volUUID,
+		Encrypted:         encrypted,
+		GenerationCounter: generationCounter,
+		ContentFormat:     zconfig.Format(parsedFormat),
+		FileLocation:      pvcName,
+	}
+	return &vs, nil
+}
+
 func getVolumeStatusByLocation(location string) (*types.VolumeStatus, error) {
 	var encrypted bool
 	var parsedFormat int32

pkg/pillar/cmd/volumemgr/updatestatus.go

@@ -434,11 +434,12 @@ func doUpdateContentTree(ctx *volumemgrContext, status *types.ContentTreeStatus)
 		log.Functionf("doUpdateContentTree(%s) successfully loaded all blobs into CAS", status.Key())
 
 		// check if the image was created
-		if !lookupImageCAS(status.ReferenceID(), ctx.casClient) {
+		imgName := status.ReferenceID()
+		if !lookupImageCAS(imgName, ctx.casClient) {
 			log.Functionf("doUpdateContentTree(%s): image does not yet exist in CAS", status.Key())
 			return changed, false
 		}
-		log.Functionf("doUpdateContentTree(%s): image exists in CAS, Content Tree load is completely LOADED", status.Key())
+		log.Functionf("doUpdateContentTree(%s): image %v exists in CAS, Content Tree load is completely LOADED", status.Key(), imgName)
 		status.State = types.LOADED
 		status.CreateTime = time.Now()
 		// ContentTreeStatus.FileLocation has no meaning once everything is loaded
@@ -592,7 +593,8 @@ func doUpdateVol(ctx *volumemgrContext, status *types.VolumeStatus) (bool, bool)
 			status.State != types.CREATING_VOLUME &&
 			status.SubState == types.VolumeSubStateInitial {
 
-			_, err := ctx.casClient.GetImageHash(ctStatus.ReferenceID())
+			imgName := ctStatus.ReferenceID()
+			_, err := ctx.casClient.GetImageHash(imgName)
 			if err != nil {
 				log.Functionf("doUpdateVol(%s): waiting for image create: %s", status.Key(), err.Error())
 				return changed, false
@@ -614,7 +616,7 @@ func doUpdateVol(ctx *volumemgrContext, status *types.VolumeStatus) (bool, bool)
 					status.Key(), status.DisplayName)
 				return changed, false
 			}
-			status.ReferenceName = ctStatus.ReferenceID()
+			status.ReferenceName = imgName
 			status.ContentFormat = ctStatus.Format
 			log.Noticef("doUpdateVol(%s): setting VolumeStatus.ContentFormat by ContentTree to %s", status.Key(), status.ContentFormat)
 			changed = true