Merge pull request #839 from keyingliu/reboot

Keep VM data during node restart
Mirantis · Jan 14, 2019 · 07b5608 · 07b5608
2 parents 8b439ae + 2d79639
commit 07b5608
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 20 deletions.
diff --git a/deploy/data/virtlet-ds.yaml b/deploy/data/virtlet-ds.yaml
@@ -138,6 +138,8 @@ spec:
         command:
         - /libvirt.sh
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /sys/fs/cgroup
           name: cgroup
         - mountPath: /lib/modules
@@ -177,6 +179,8 @@ spec:
         # In case we inject local virtlet image we want to use it not officially available one
         imagePullPolicy: IfNotPresent
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /run
           # Don't add "mountPropagation: Bidirectional", it will lack mount entry
           name: run
@@ -299,6 +303,9 @@ spec:
       - hostPath:
           path: /var/run/netns
         name: netns-dir
+      - hostPath:
+          path: /etc/libvirt/qemu
+        name: qemu
       - configMap:
           name: virtlet-image-translations
         name: image-name-translations

diff --git a/pkg/libvirttools/gc.go b/pkg/libvirttools/gc.go
@@ -96,22 +96,12 @@ func (v *VirtualizationTool) checkSandboxNetNs(sandbox metadata.PodSandboxMetada
 	}
 
 	if !v.mountPointChecker.IsPathAnNs(sinfo.ContainerSideNetwork.NsPath) {
-		containers, err := v.metadataStore.ListPodContainers(sandbox.GetID())
-		if err != nil {
-			return err
-		}
-		for _, container := range containers {
-			// remove container from metadata store
-			if err := container.Save(func(*types.ContainerInfo) (*types.ContainerInfo, error) {
-				return nil, nil
-			}); err != nil {
-				return err
+		// NS didn't found, need RunSandbox again
+		if err := sandbox.Save(func(s *types.PodSandboxInfo) (*types.PodSandboxInfo, error) {
+			if s != nil {
+				s.State = types.PodSandboxState_SANDBOX_NOTREADY
 			}
-		}
-
-		// remove sandbox from metadata store
-		if err := sandbox.Save(func(*types.PodSandboxInfo) (*types.PodSandboxInfo, error) {
-			return nil, nil
+			return s, nil
 		}); err != nil {
 			return err
 		}

diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go
@@ -173,14 +173,16 @@ func (v *VirtletManager) Stop() {
 // garbage collection for both libvirt and the image store.
 func (v *VirtletManager) recoverAndGC() error {
 	var errors []string
-	for _, err := range v.recoverNetworkNamespaces() {
-		errors = append(errors, fmt.Sprintf("* error recovering VM network namespaces: %v", err))
-	}
 
 	for _, err := range v.virtTool.GarbageCollect() {
 		errors = append(errors, fmt.Sprintf("* error performing libvirt GC: %v", err))
 	}
 
+	// recover network namespace after VM GC
+	for _, err := range v.recoverNetworkNamespaces() {
+		errors = append(errors, fmt.Sprintf("* error recovering VM network namespaces: %v", err))
+	}
+
 	if err := v.imageStore.GC(); err != nil {
 		errors = append(errors, fmt.Sprintf("* error during image GC: %v", err))
 	}
@@ -213,6 +215,10 @@ OUTER:
 			allErrors = append(allErrors, fmt.Errorf("inconsistent database. Found pod %q sandbox but can not retrive its metadata", s.GetID()))
 			continue
 		}
+		// Don't recover if sandbox is not ready
+		if psi.State != types.PodSandboxState_SANDBOX_READY {
+			continue
+		}
 
 		haveRunningContainers := false
 		containers, err := v.metadataStore.ListPodContainers(s.GetID())

diff --git a/pkg/tools/TestGenCommand__compat.out.yaml b/pkg/tools/TestGenCommand__compat.out.yaml
@@ -41,6 +41,8 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /sys/fs/cgroup
           name: cgroup
         - mountPath: /lib/modules
@@ -76,6 +78,8 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /run:shared
           name: run
         - mountPath: /lib/modules
@@ -278,6 +282,9 @@ spec:
       - hostPath:
           path: /var/run/netns
         name: netns-dir
+      - hostPath:
+          path: /etc/libvirt/qemu
+        name: qemu
       - configMap:
           name: virtlet-image-translations
         name: image-name-translations

diff --git a/pkg/tools/TestGenCommand__compat_dev.out.yaml b/pkg/tools/TestGenCommand__compat_dev.out.yaml
@@ -41,6 +41,8 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /sys/fs/cgroup
           name: cgroup
         - mountPath: /lib/modules
@@ -78,6 +80,8 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /run:shared
           name: run
         - mountPath: /lib/modules
@@ -286,6 +290,9 @@ spec:
       - hostPath:
           path: /var/run/netns
         name: netns-dir
+      - hostPath:
+          path: /etc/libvirt/qemu
+        name: qemu
       - configMap:
           name: virtlet-image-translations
         name: image-name-translations

diff --git a/pkg/tools/TestGenCommand__dev.out.yaml b/pkg/tools/TestGenCommand__dev.out.yaml
@@ -41,6 +41,8 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /sys/fs/cgroup
           name: cgroup
         - mountPath: /lib/modules
@@ -78,6 +80,8 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /run
           name: run
         - mountPath: /lib/modules
@@ -288,6 +292,9 @@ spec:
       - hostPath:
           path: /var/run/netns
         name: netns-dir
+      - hostPath:
+          path: /etc/libvirt/qemu
+        name: qemu
       - configMap:
           name: virtlet-image-translations
         name: image-name-translations

diff --git a/pkg/tools/TestGenCommand__plain.out.yaml b/pkg/tools/TestGenCommand__plain.out.yaml
@@ -41,6 +41,8 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /sys/fs/cgroup
           name: cgroup
         - mountPath: /lib/modules
@@ -76,6 +78,8 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /run
           name: run
         - mountPath: /lib/modules
@@ -280,6 +284,9 @@ spec:
       - hostPath:
           path: /var/run/netns
         name: netns-dir
+      - hostPath:
+          path: /etc/libvirt/qemu
+        name: qemu
       - configMap:
           name: virtlet-image-translations
         name: image-name-translations

diff --git a/pkg/tools/TestGenCommand__tag.out.yaml b/pkg/tools/TestGenCommand__tag.out.yaml
@@ -41,6 +41,8 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /sys/fs/cgroup
           name: cgroup
         - mountPath: /lib/modules
@@ -76,6 +78,8 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /etc/libvirt/qemu
+          name: qemu
         - mountPath: /run
           name: run
         - mountPath: /lib/modules
@@ -280,6 +284,9 @@ spec:
       - hostPath:
           path: /var/run/netns
         name: netns-dir
+      - hostPath:
+          path: /etc/libvirt/qemu
+        name: qemu
       - configMap:
           name: virtlet-image-translations
         name: image-name-translations

diff --git a/pkg/tools/bindata.go b/pkg/tools/bindata.go