From 3a1cf2d52ad8a09af34132f16ab81dc8386ae0f1 Mon Sep 17 00:00:00 2001 From: Jing Xu Date: Mon, 5 Dec 2016 10:17:54 -0800 Subject: [PATCH 1/2] Fix GCI mounter script to run garbage collection multiple times Remove break in the mounter script to make sure gc run multiple times --- cluster/gce/gci/mounter/mounter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/gce/gci/mounter/mounter b/cluster/gce/gci/mounter/mounter index 6f45a56abf43..2fba3255f2bf 100755 --- a/cluster/gce/gci/mounter/mounter +++ b/cluster/gce/gci/mounter/mounter @@ -31,7 +31,7 @@ function gc { # Rkt pods end up creating new copies of mounts on the host. Hence it is ideal to clean them up right away. attempt=0 until [ $attempt -ge 5 ]; do - ${RKT_BINARY} gc --grace-period=0s &> /dev/null && break + ${RKT_BINARY} gc --grace-period=0s &> /dev/null attempt=$[$attempt+1] sleep 1 done From 896e0b867e8db0b0d4ed94e1763fc8583aa66c0c Mon Sep 17 00:00:00 2001 From: Jing Xu Date: Mon, 5 Dec 2016 14:52:09 -0800 Subject: [PATCH 2/2] Fix unmount issue cuased by GCI mounter this is a workaround for the unmount device issue caused by gci mounter. In GCI cluster, if gci mounter is used for mounting, the container started by mounter script will cause additional mounts created in the container. Since these mounts are irrelavant to the original mounts, they should be not considered when checking the mount references. By comparing the mount path prefix, those additional mounts can be filtered out. Plan to work on better approach to solve this issue. --- pkg/util/mount/mount.go | 10 +++++++++- pkg/util/mount/mount_linux_test.go | 4 ++-- pkg/volume/util/util.go | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go index 8796d6a52d5b..e561892bcd9d 100644 --- a/pkg/util/mount/mount.go +++ b/pkg/util/mount/mount.go @@ -130,13 +130,21 @@ func GetMountRefs(mounter Interface, mountPath string) ([]string, error) { } } + // TODO: this is a workaround for the unmount device issue caused by gci mounter. + // In GCI cluster, if gci mounter is used for mounting, the container started by mounter + // script will cause additional mounts created in the container. Since these mounts are + // irrelavant to the original mounts, they should be not considered when checking the + // mount references. Current solution is to filter out those mount paths that contain + // the string of original mount path. + // Plan to work on better approach to solve this issue. + // Find all references to the device. var refs []string if deviceName == "" { glog.Warningf("could not determine device for path: %q", mountPath) } else { for i := range mps { - if mps[i].Device == deviceName && mps[i].Path != slTarget { + if mps[i].Device == deviceName && !strings.Contains(mps[i].Path, slTarget) { refs = append(refs, mps[i].Path) } } diff --git a/pkg/util/mount/mount_linux_test.go b/pkg/util/mount/mount_linux_test.go index 4c7acbe4393e..19cdb0ba22ca 100644 --- a/pkg/util/mount/mount_linux_test.go +++ b/pkg/util/mount/mount_linux_test.go @@ -98,7 +98,7 @@ func TestGetMountRefs(t *testing.T) { {Device: "/dev/sdb", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd"}, {Device: "/dev/sdb", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd-in-pod"}, {Device: "/dev/sdc", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2"}, - {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod"}, + {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1"}, {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2"}, }, } @@ -114,7 +114,7 @@ func TestGetMountRefs(t *testing.T) { }, }, { - "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod", + "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1", []string{ "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2", "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2", diff --git a/pkg/volume/util/util.go b/pkg/volume/util/util.go index cc5c25d41a76..8078869d6dc1 100644 --- a/pkg/volume/util/util.go +++ b/pkg/volume/util/util.go @@ -94,7 +94,7 @@ func UnmountPath(mountPath string, mounter mount.Interface) error { return err } if notMnt { - glog.V(4).Info("%q is unmounted, deleting the directory", mountPath) + glog.V(4).Infof("%q is unmounted, deleting the directory", mountPath) return os.Remove(mountPath) } return nil