Init okteto persistent volume from image content (#1541)

Signed-off-by: Pablo Chico de Guzman <pchico83@gmail.com>
okteto · May 18, 2021 · dfd7ad6 · dfd7ad6
1 parent 1026e05
commit dfd7ad6
Show file tree

Hide file tree

Showing 7 changed files with 128 additions and 194 deletions.
diff --git a/cmd/up/syncthing.go b/cmd/up/syncthing.go
@@ -98,11 +98,9 @@ func (up *upContext) startSyncthing(ctx context.Context) error {
 		return err
 	}
 
-	if !up.Dev.PersistentVolumeEnabled() || up.Dev.PersistentVolumeInitFromImage() {
-		if err := up.Sy.WaitForScanning(ctx, up.Dev, false); err != nil {
-			log.Infof("failed to wait for syncthing scanning: %s", err.Error())
-			return up.checkOktetoStartError(ctx, "Failed to connect to the synchronization service")
-		}
+	if err := up.Sy.WaitForScanning(ctx, up.Dev, false); err != nil {
+		log.Infof("failed to wait for syncthing scanning: %s", err.Error())
+		return up.checkOktetoStartError(ctx, "Failed to connect to the synchronization service")
 	}
 
 	return up.Sy.WaitForConnected(ctx, up.Dev)

diff --git a/pkg/k8s/deployments/translate.go b/pkg/k8s/deployments/translate.go
@@ -17,6 +17,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	okLabels "github.com/okteto/okteto/pkg/k8s/labels"
@@ -37,8 +38,8 @@ const (
 	revisionAnnotation         = "deployment.kubernetes.io/revision"
 	//OktetoBinName name of the okteto bin init container
 	OktetoBinName = "okteto-bin"
-	//OktetoInitFromImage name of the okteto init from image container
-	OktetoInitDataName = "okteto-init-data"
+	//OktetoInitVolumeContainerName name of the okteto init container that initializes the persistent colume from image content
+	OktetoInitVolumeContainerName = "okteto-init-volume"
 
 	//syncthing
 	oktetoSyncSecretVolume = "okteto-sync-secret" // skipcq GSC-G101  not a secret
@@ -110,9 +111,6 @@ func translate(t *model.Translation, c *kubernetes.Clientset, isOktetoNamespace
 
 		if rule.Image == "" {
 			rule.Image = devContainer.Image
-			if rule.InitFromImageContainer != nil {
-				rule.InitFromImageContainer.Image = devContainer.Image
-			}
 		}
 
 		TranslateDevContainer(devContainer, rule)
@@ -123,7 +121,7 @@ func translate(t *model.Translation, c *kubernetes.Clientset, isOktetoNamespace
 		if rule.IsMainDevContainer() {
 			TranslateOktetoBinVolumeMounts(devContainer)
 			TranslateOktetoInitBinContainer(rule.InitContainer, &t.Deployment.Spec.Template.Spec)
-			TranslateOktetoInitFromImageContainer(rule.InitFromImageContainer, &t.Deployment.Spec.Template.Spec)
+			TranslateOktetoInitFromImageContainer(&t.Deployment.Spec.Template.Spec, rule)
 			TranslateDinDContainer(&t.Deployment.Spec.Template.Spec, rule)
 			TranslateOktetoBinVolume(&t.Deployment.Spec.Template.Spec)
 		}
@@ -574,35 +572,44 @@ func TranslateOktetoInitBinContainer(initContainer model.InitContainer, spec *ap
 }
 
 //TranslateOktetoInitFromImageContainer translates the init from image container of a pod
-func TranslateOktetoInitFromImageContainer(initContainer *model.InitFromImageContainer, spec *apiv1.PodSpec) {
-	if initContainer == nil {
+func TranslateOktetoInitFromImageContainer(spec *apiv1.PodSpec, rule *model.TranslationRule) {
+	if !rule.PersistentVolume {
 		return
 	}
-	c := apiv1.Container{
-		Name:            OktetoInitDataName,
-		Image:           initContainer.Image,
+
+	if spec.InitContainers == nil {
+		spec.InitContainers = []apiv1.Container{}
+	}
+
+	c := &apiv1.Container{
+		Name:            OktetoInitVolumeContainerName,
+		Image:           rule.Image,
 		ImagePullPolicy: apiv1.PullIfNotPresent,
-		Command:         initContainer.Command,
 		VolumeMounts:    []apiv1.VolumeMount{},
 	}
-
-	for i := range initContainer.Volumes {
+	command := "echo initializing volume..."
+	iVolume := 1
+	for _, v := range rule.Volumes {
+		if !strings.HasPrefix(v.SubPath, model.SourceCodeSubPath) && !strings.HasPrefix(v.SubPath, model.DataSubPath) {
+			continue
+		}
 		c.VolumeMounts = append(
 			c.VolumeMounts,
 			apiv1.VolumeMount{
-				Name:      initContainer.Volumes[i].Name,
-				MountPath: initContainer.Volumes[i].MountPath,
-				SubPath:   initContainer.Volumes[i].SubPath,
+				Name:      v.Name,
+				MountPath: fmt.Sprintf("/init-volume/%d", iVolume),
+				SubPath:   v.SubPath,
 			},
 		)
+		mounPath := filepath.Join(v.MountPath, ".")
+		command = fmt.Sprintf("%s && (cp -Rv %s/. /init-volume/%d || true)", command, mounPath, iVolume)
+		iVolume++
 	}
 
-	translateInitResources(&c, initContainer.Resources)
-
-	if spec.InitContainers == nil {
-		spec.InitContainers = []apiv1.Container{}
-	}
-	spec.InitContainers = append(spec.InitContainers, c)
+	c.Command = []string{"sh", "-c", command}
+	translateInitResources(c, rule.InitContainer.Resources)
+	TranslateContainerSecurityContext(c, rule.SecurityContext)
+	spec.InitContainers = append(spec.InitContainers, *c)
 }
 
 //TranslateOktetoSyncSecret translates the syncthing secret container of a pod

diff --git a/pkg/k8s/deployments/translate_test.go b/pkg/k8s/deployments/translate_test.go
@@ -62,6 +62,9 @@ serviceAccount: sa
 sync:
   - .:/app
   - sub:/path
+volumes:
+  - /go/pkg/
+  - /root/.cache/go-build
 secrets:
   - %s:/remote
 resources:
@@ -188,6 +191,42 @@ services:
 								},
 							},
 						},
+						{
+							Name:            OktetoInitVolumeContainerName,
+							Image:           "web:latest",
+							ImagePullPolicy: apiv1.PullIfNotPresent,
+							Command:         []string{"sh", "-c", "echo initializing volume... && (cp -Rv /go/pkg/. /init-volume/1 || true) && (cp -Rv /root/.cache/go-build/. /init-volume/2 || true) && (cp -Rv /app/. /init-volume/3 || true) && (cp -Rv /path/. /init-volume/4 || true)"},
+							SecurityContext: &apiv1.SecurityContext{
+								RunAsUser:  &runAsUser,
+								RunAsGroup: &runAsGroup,
+							},
+							VolumeMounts: []apiv1.VolumeMount{
+								{
+									Name:      dev.GetVolumeName(),
+									ReadOnly:  false,
+									MountPath: "/init-volume/1",
+									SubPath:   path.Join(model.DataSubPath, "go/pkg"),
+								},
+								{
+									Name:      dev.GetVolumeName(),
+									ReadOnly:  false,
+									MountPath: "/init-volume/2",
+									SubPath:   path.Join(model.DataSubPath, "root/.cache/go-build"),
+								},
+								{
+									Name:      dev.GetVolumeName(),
+									ReadOnly:  false,
+									MountPath: "/init-volume/3",
+									SubPath:   model.SourceCodeSubPath,
+								},
+								{
+									Name:      dev.GetVolumeName(),
+									ReadOnly:  false,
+									MountPath: "/init-volume/4",
+									SubPath:   path.Join(model.SourceCodeSubPath, "sub"),
+								},
+							},
+						},
 					},
 					Containers: []apiv1.Container{
 						{
@@ -232,6 +271,18 @@ services:
 									MountPath: model.RemoteMountPath,
 									SubPath:   model.RemoteSubPath,
 								},
+								{
+									Name:      dev.GetVolumeName(),
+									ReadOnly:  false,
+									MountPath: "/go/pkg/",
+									SubPath:   path.Join(model.DataSubPath, "go/pkg"),
+								},
+								{
+									Name:      dev.GetVolumeName(),
+									ReadOnly:  false,
+									MountPath: "/root/.cache/go-build",
+									SubPath:   path.Join(model.DataSubPath, "root/.cache/go-build"),
+								},
 								{
 									Name:      dev.GetVolumeName(),
 									ReadOnly:  false,
@@ -564,7 +615,6 @@ namespace: n
 image: web:latest
 sync:
   - .:/app
-  - sub:/path
 docker:
   enabled: true
   image: docker:19
@@ -656,6 +706,25 @@ docker:
 								},
 							},
 						},
+						{
+							Name:            OktetoInitVolumeContainerName,
+							Image:           "web:latest",
+							ImagePullPolicy: apiv1.PullIfNotPresent,
+							Command:         []string{"sh", "-c", "echo initializing volume... && (cp -Rv /app/. /init-volume/1 || true)"},
+							SecurityContext: &apiv1.SecurityContext{
+								RunAsUser:    &rootUser,
+								RunAsGroup:   &rootUser,
+								RunAsNonRoot: &falseBoolean,
+							},
+							VolumeMounts: []apiv1.VolumeMount{
+								{
+									Name:      dev.GetVolumeName(),
+									ReadOnly:  false,
+									MountPath: "/init-volume/1",
+									SubPath:   model.SourceCodeSubPath,
+								},
+							},
+						},
 					},
 					Containers: []apiv1.Container{
 						{
@@ -716,12 +785,6 @@ docker:
 									MountPath: "/app",
 									SubPath:   model.SourceCodeSubPath,
 								},
-								{
-									Name:      dev.GetVolumeName(),
-									ReadOnly:  false,
-									MountPath: "/path",
-									SubPath:   path.Join(model.SourceCodeSubPath, "sub"),
-								},
 								{
 									Name:      oktetoSyncSecretVolume,
 									ReadOnly:  false,
@@ -766,12 +829,6 @@ docker:
 									MountPath: "/app",
 									SubPath:   model.SourceCodeSubPath,
 								},
-								{
-									Name:      dev.GetVolumeName(),
-									ReadOnly:  false,
-									MountPath: "/path",
-									SubPath:   path.Join(model.SourceCodeSubPath, "sub"),
-								},
 							},
 							LivenessProbe:  nil,
 							ReadinessProbe: nil,

diff --git a/pkg/model/dev.go b/pkg/model/dev.go
@@ -161,6 +161,7 @@ type Dev struct {
 	Services             []*Dev                `json:"services,omitempty" yaml:"services,omitempty"`
 	PersistentVolumeInfo *PersistentVolumeInfo `json:"persistentVolume,omitempty" yaml:"persistentVolume,omitempty"`
 	InitContainer        InitContainer         `json:"initContainer,omitempty" yaml:"initContainer,omitempty"`
+	InitFromImage        bool                  `json:"initFromImage,omitempty" yaml:"initFromImage,omitempty"`
 	Timeout              time.Duration         `json:"timeout,omitempty" yaml:"timeout,omitempty"`
 	Docker               DinDContainer         `json:"docker,omitempty" yaml:"docker,omitempty"`
 	Divert               *Divert               `json:"divert,omitempty" yaml:"divert,omitempty"`
@@ -222,10 +223,9 @@ type ExternalVolume struct {
 
 // PersistentVolumeInfo info about the persistent volume
 type PersistentVolumeInfo struct {
-	Enabled       bool   `json:"enabled,omitempty" yaml:"enabled,omitempty"`
-	InitFromImage bool   `json:"initFromImage,omitempty" yaml:"initFromImage,omitempty"`
-	StorageClass  string `json:"storageClass,omitempty" yaml:"storageClass,omitempty"`
-	Size          string `json:"size,omitempty" yaml:"size,omitempty"`
+	Enabled      bool   `json:"enabled,omitempty" yaml:"enabled,omitempty"`
+	StorageClass string `json:"storageClass,omitempty" yaml:"storageClass,omitempty"`
+	Size         string `json:"size,omitempty" yaml:"size,omitempty"`
 }
 
 // InitContainer represents the initial container
@@ -241,14 +241,6 @@ type DinDContainer struct {
 	Resources ResourceRequirements `json:"resources,omitempty" yaml:"resources,omitempty"`
 }
 
-// InitContainer represents the initial container
-type InitFromImageContainer struct {
-	Image     string               `json:"image,omitempty" yaml:"image,omitempty"`
-	Command   []string             `json:"command,omitempty" yaml:"command,omitempty"`
-	Volumes   []VolumeMount        `json:"volumes,omitempty" yaml:"volumes,omitempty"`
-	Resources ResourceRequirements `json:"resources,omitempty" yaml:"resources,omitempty"`
-}
-
 // SecurityContext represents a pod security context
 type SecurityContext struct {
 	RunAsUser    *int64        `json:"runAsUser,omitempty" yaml:"runAsUser,omitempty"`
@@ -982,38 +974,6 @@ func (dev *Dev) ToTranslationRule(main *Dev, reset bool) *TranslationRule {
 		}
 	}
 
-	if main == dev && main.PersistentVolumeInitFromImage() {
-		rule.InitFromImageContainer = &InitFromImageContainer{
-			Image:     main.Image.Name,
-			Volumes:   []VolumeMount{},
-			Resources: main.InitContainer.Resources,
-		}
-		command := "echo initializing volume..."
-		for i := range dev.Volumes {
-			rule.InitFromImageContainer.Volumes = append(
-				rule.InitFromImageContainer.Volumes,
-				VolumeMount{
-					Name:      main.GetVolumeName(),
-					MountPath: fmt.Sprintf("/initData-%d", i),
-					SubPath:   getDataSubPath(dev.Volumes[i].RemotePath),
-				},
-			)
-			command = fmt.Sprintf("%s && (cp -Rv %s/* /initData-%d || true)", command, dev.Volumes[i].RemotePath, i)
-		}
-		for i := range dev.Sync.Folders {
-			rule.InitFromImageContainer.Volumes = append(
-				rule.InitFromImageContainer.Volumes,
-				VolumeMount{
-					Name:      main.GetVolumeName(),
-					MountPath: fmt.Sprintf("/initSync-%d", i),
-					SubPath:   main.getSourceSubPath(dev.Sync.Folders[i].LocalPath),
-				},
-			)
-			command = fmt.Sprintf("%s && (cp -Rv %s/* /initSync-%d || true)", command, dev.Sync.Folders[i].RemotePath, i)
-		}
-		rule.InitFromImageContainer.Command = []string{"sh", "-c", command}
-	}
-
 	for _, v := range dev.ExternalVolumes {
 		rule.Volumes = append(
 			rule.Volumes,

diff --git a/pkg/model/translation.go b/pkg/model/translation.go
@@ -32,28 +32,27 @@ type Translation struct {
 
 // TranslationRule represents how to apply a container translation in a deployment
 type TranslationRule struct {
-	Marker                 string                  `json:"marker"`
-	OktetoBinImageTag      string                  `json:"oktetoBinImageTag"`
-	Node                   string                  `json:"node,omitempty"`
-	Container              string                  `json:"container,omitempty"`
-	Image                  string                  `json:"image,omitempty"`
-	ImagePullPolicy        apiv1.PullPolicy        `json:"imagePullPolicy,omitempty" yaml:"imagePullPolicy,omitempty"`
-	Environment            Environment             `json:"environment,omitempty"`
-	Secrets                []Secret                `json:"secrets,omitempty"`
-	Command                []string                `json:"command,omitempty"`
-	Args                   []string                `json:"args,omitempty"`
-	WorkDir                string                  `json:"workdir"`
-	Healthchecks           bool                    `json:"healthchecks" yaml:"healthchecks"`
-	PersistentVolume       bool                    `json:"persistentVolume" yaml:"persistentVolume"`
-	Volumes                []VolumeMount           `json:"volumes,omitempty"`
-	SecurityContext        *SecurityContext        `json:"securityContext,omitempty"`
-	ServiceAccount         string                  `json:"serviceAccount,omitempty" yaml:"serviceAccount,omitempty"`
-	Resources              ResourceRequirements    `json:"resources,omitempty"`
-	InitContainer          InitContainer           `json:"initContainers,omitempty"`
-	InitFromImageContainer *InitFromImageContainer `json:"initFromImage" yaml:"initFromImage"`
-	Probes                 *Probes                 `json:"probes" yaml:"probes"`
-	Lifecycle              *Lifecycle              `json:"lifecycle" yaml:"lifecycle"`
-	Docker                 DinDContainer           `json:"docker" yaml:"docker"`
+	Marker            string               `json:"marker"`
+	OktetoBinImageTag string               `json:"oktetoBinImageTag"`
+	Node              string               `json:"node,omitempty"`
+	Container         string               `json:"container,omitempty"`
+	Image             string               `json:"image,omitempty"`
+	ImagePullPolicy   apiv1.PullPolicy     `json:"imagePullPolicy,omitempty" yaml:"imagePullPolicy,omitempty"`
+	Environment       Environment          `json:"environment,omitempty"`
+	Secrets           []Secret             `json:"secrets,omitempty"`
+	Command           []string             `json:"command,omitempty"`
+	Args              []string             `json:"args,omitempty"`
+	WorkDir           string               `json:"workdir"`
+	Healthchecks      bool                 `json:"healthchecks" yaml:"healthchecks"`
+	PersistentVolume  bool                 `json:"persistentVolume" yaml:"persistentVolume"`
+	Volumes           []VolumeMount        `json:"volumes,omitempty"`
+	SecurityContext   *SecurityContext     `json:"securityContext,omitempty"`
+	ServiceAccount    string               `json:"serviceAccount,omitempty" yaml:"serviceAccount,omitempty"`
+	Resources         ResourceRequirements `json:"resources,omitempty"`
+	InitContainer     InitContainer        `json:"initContainers,omitempty"`
+	Probes            *Probes              `json:"probes" yaml:"probes"`
+	Lifecycle         *Lifecycle           `json:"lifecycle" yaml:"lifecycle"`
+	Docker            DinDContainer        `json:"docker" yaml:"docker"`
 }
 
 // IsMainDevContainer returns true if the translation rule applies to the main dev container of the okteto manifest