This patch adds a test that reproduces a bug in snap-confine/snapd that
affects Ubuntu 16.04 running an LXD container with snapd. Inside such
container the root directory is not implicitly rshared by systemd and
after running any snap command snaps cannot be removed/unmounted
correctly.

The error is addressed by a little trick in snap-confine. The existing
approach that bind-mounts /snap over itself and then rshares it is
replaced with a loop that rshares each mounted snap. In effect /snap
bind mount doesn't exist anymore and all application and hook processes
start with the sharing already in place.

With-Kind-Regards-To: Kyle Fazzari
Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>

cmd/snap-confine/mount-support.c

@@ -679,37 +679,32 @@ void sc_populate_mount_ns(const char *base_snap_name, const char *snap_name)
 	}
 }
 
-static bool is_mounted_with_shared_option(const char *dir)
-    __attribute__ ((nonnull(1)));
-
-static bool is_mounted_with_shared_option(const char *dir)
+void sc_ensure_shared_snap_mount(void)
 {
+	struct sc_mountinfo_entry *entry = NULL;
 	struct sc_mountinfo *sm SC_CLEANUP(sc_cleanup_mountinfo) = NULL;
-	sm = sc_parse_mountinfo(NULL);
-	if (sm == NULL) {
+
+	if ((sm = sc_parse_mountinfo(NULL)) == NULL) {
 		die("cannot parse /proc/self/mountinfo");
 	}
-	struct sc_mountinfo_entry *entry = sc_first_mountinfo_entry(sm);
-	while (entry != NULL) {
-		const char *mount_dir = entry->mount_dir;
-		if (sc_streq(mount_dir, dir)) {
-			const char *optional_fields = entry->optional_fields;
-			if (strstr(optional_fields, "shared:") != NULL) {
-				return true;
-			}
-		}
-		entry = sc_next_mountinfo_entry(entry);
-	}
-	return false;
-}
 
-void sc_ensure_shared_snap_mount(void)
-{
-	if (!is_mounted_with_shared_option("/")
-	    && !is_mounted_with_shared_option(SNAP_MOUNT_DIR)) {
-		sc_do_mount(SNAP_MOUNT_DIR, SNAP_MOUNT_DIR, "none",
-			    MS_BIND | MS_REC, 0);
-		sc_do_mount("none", SNAP_MOUNT_DIR, NULL, MS_SHARED | MS_REC,
-			    NULL);
+	// Make all the snaps mounted in SNAP_MOUNT_DIR shared.
+	for (entry = sc_first_mountinfo_entry(sm); entry != NULL;
+	     entry = sc_next_mountinfo_entry(entry)) {
+		// Is it in the snap mount dir?
+		if (strstr(entry->mount_dir, SNAP_MOUNT_DIR) !=
+		    entry->mount_dir) {
+			continue;
+		}
+		// Is it a squashfs or one of that fuse squashfs helpers?
+		if (!sc_streq(entry->fs_type, "squashfs")
+		    || !sc_streq(entry->fs_type, "fuse.squashfuse")) {
+			continue;
+		}
+		// Is it shared? We don't need to fix shared mounts.
+		if (strstr(entry->optional_fields, "shared:") == NULL) {
+			continue;
+		}
+		sc_do_mount("none", entry->mount_dir, NULL, MS_SHARED, NULL);
 	}
 }

tests/main/lxd-refresh-cycle/task.yaml

@@ -0,0 +1,64 @@
+summary: Ensure that we can refresh/remove snaps in LXD
+details: >
+    There is a bug affecting snapd in environments when the root filesystem is
+    not initially "rshared" (in the sense of mount --make-rshared). This test
+    reproduces the issue inside LXD (the issue also affects Ubuntu 14.04
+    without systemd running as init). The issue only happens after we invoke
+    snap-confine at least once, otherwise it is hidden.
+systems: [ubuntu-16*, ubuntu-core-*]
+kill-timeout: 25m # lxd downloads can be quite slow
+restore: |
+    if  [ $(ls -1 "$GOHOME"/snapd_*.deb | wc -l || echo 0) -eq 0 ]; then
+        exit
+    fi
+    lxd.lxc stop my-ubuntu
+    lxd.lxc delete my-ubuntu
+    rm -f mountinfo.*
+debug: |
+    journalctl -u snap.lxd.daemon.service # debug output from lxd
+execute: |
+    if  [[ $(ls -1 "$GOHOME"/snapd_*.deb | wc -l || echo 0) -eq 0 ]]; then
+        echo "No run lxd test when there are not .deb files built"
+        exit
+    fi
+    wait_for_lxd(){
+        while ! printf "GET / HTTP/1.0\n\n" | nc -U /var/snap/lxd/common/lxd/unix.socket | MATCH "200 OK"; do sleep 1; done
+    }
+    echo "Install lxd"
+    snap install lxd
+    echo "Create a trivial container using the lxd snap"
+    wait_for_lxd
+    lxd init --auto
+    lxd.lxc launch ubuntu:16.04 my-ubuntu
+    echo "Ensure we can run things inside"
+    lxd.lxc exec my-ubuntu echo hello | MATCH hello
+    echo "Ensure we can get network"
+    lxd.lxc network create testbr0
+    lxd.lxc network attach testbr0 my-ubuntu eth0
+    lxd.lxc exec my-ubuntu dhclient eth0
+    echo "Install locally built snapd inside the container"
+    lxd.lxc exec my-ubuntu -- apt autoremove --purge -y snapd ubuntu-core-launcher
+    lxd.lxc exec my-ubuntu -- cat /proc/self/mountinfo > mountinfo.after-purge
+    lxd.lxc exec my-ubuntu -- mkdir -p "$GOHOME"
+    lxd.lxc file push "$GOHOME"/snapd_*.deb my-ubuntu/$GOPATH/
+    lxd.lxc exec my-ubuntu -- dpkg -i "$GOHOME"/snapd_*.deb
+    lxd.lxc exec my-ubuntu -- cat /proc/self/mountinfo > mountinfo.after-install
+    # FIXME: workaround for missing squashfuse
+    lxd.lxc exec my-ubuntu -- apt update
+    lxd.lxc exec my-ubuntu -- apt install -y squashfuse
+    echo "Download and side-load a test snap three times" 
+    # Revision cannot be globbed remotely so download and glob here.
+    snap download test-snapd-tools
+    lxd.lxc exec my-ubuntu -- snap download test-snapd-tools
+    snap_file=$(ls test-snapd-tools_*.snap)
+    echo "We can refresh the snap as many times for now..."
+    for i in $(seq 5); do
+        lxd.lxc exec my-ubuntu -- snap install --dangerous "$snap_file"
+    done
+    echo "We can also remove it successfully"
+    lxd.lxc exec my-ubuntu -- snap remove test-snapd-tools
+    echo "Running an installed snap will fix the sharing of the mount point"
+    lxd.lxc exec my-ubuntu -- snap install --dangerous "$snap_file"
+    lxd.lxc exec my-ubuntu -- snap run test-snapd-tools.success
+    echo "But not in a way that breaks removal"
+    lxd.lxc exec my-ubuntu -- snap remove test-snapd-tools