This patch adds a test that reproduces a bug in snap-confine/snapd that
affects Ubuntu 16.04 running an LXD container with snapd. Inside such
container the root directory is not implicitly rshared by systemd and
after running any snap command snaps cannot be removed/unmounted
correctly.

The error is addressed by a little trick in snap-confine. All the snap
bind mounts are stopped. The /snap directory is bind-mounted over
itself, rshare is applied and then all the snap mount units are started
again. We must rely on this trick as there is insufficient information
in the mount table for to correctly stop and then restart fuse
(squashfuse) mounts.

With-Kind-Regards-To: Kyle Fazzari
Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>

cmd/snap-confine/mount-support.c

@@ -19,6 +19,8 @@
 
 #include <errno.h>
 #include <fcntl.h>
+#include <fnmatch.h>
+#include <libgen.h>
 #include <libgen.h>
 #include <limits.h>
 #include <mntent.h>
@@ -33,7 +35,6 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <unistd.h>
-#include <libgen.h>
 
 #include "../libsnap-confine-private/classic.h"
 #include "../libsnap-confine-private/cleanup-funcs.h"
@@ -705,11 +706,56 @@ static bool is_mounted_with_shared_option(const char *dir)
 
 void sc_ensure_shared_snap_mount(void)
 {
-	if (!is_mounted_with_shared_option("/")
-	    && !is_mounted_with_shared_option(SNAP_MOUNT_DIR)) {
-		sc_do_mount(SNAP_MOUNT_DIR, SNAP_MOUNT_DIR, "none",
-			    MS_BIND | MS_REC, 0);
-		sc_do_mount("none", SNAP_MOUNT_DIR, NULL, MS_SHARED | MS_REC,
-			    NULL);
+	if (is_mounted_with_shared_option("/")) {
+		debug("the / directory is shared, no need to do anything");
+		return;
+	}
+	if (is_mounted_with_shared_option(SNAP_MOUNT_DIR)) {
+		debug("the %s directory is shared, no need to do anything",
+		      SNAP_MOUNT_DIR);
+		return;
+	}
+
+	debug("snap directory needs to be `mount --make-rshared'");
+
+	const char *dirname = "/etc/systemd/system";
+	DIR *d SC_CLEANUP(sc_cleanup_closedir) = NULL;
+	struct dirent *dent;
+	char buf[PATH_MAX] = { 0 };
+
+	if ((d = opendir(dirname)) == NULL) {
+		die("cannot open directory %s", dirname);
+	}
+
+	while ((dent = readdir(d)) != NULL) {
+		if (fnmatch("snap-*.mount", dent->d_name, 0) != 0) {
+			continue;
+		}
+		sc_must_snprintf(buf, sizeof buf, "systemctl stop %s",
+				 dent->d_name);
+		if (system(buf) < 0) {
+			die("cannot stop mount unit %s", dent->d_name);
+		}
+		debug("stoped snap mount unit %s", dent->d_name);
+	}
+
+	// Bind mount /snap over itself so that we can do mount --make-rshared it.
+	sc_do_mount(SNAP_MOUNT_DIR, SNAP_MOUNT_DIR, "none",
+		    MS_BIND | MS_REC, 0);
+	sc_do_mount("none", SNAP_MOUNT_DIR, NULL, MS_SHARED | MS_REC, NULL);
+	debug("made %s mount --make-rshared", SNAP_MOUNT_DIR);
+
+	// Rewind the directory and start all the mount units again.
+	rewinddir(d);
+	while ((dent = readdir(d)) != NULL) {
+		if (fnmatch("snap-*.mount", dent->d_name, 0) != 0) {
+			continue;
+		}
+		sc_must_snprintf(buf, sizeof buf, "systemctl start %s",
+				 dent->d_name);
+		if (system(buf) < 0) {
+			die("cannot start mount unit %s", dent->d_name);
+		}
+		debug("started snap mount unit %s", dent->d_name);
 	}
 }

cmd/snap-confine/snap-confine.apparmor.in

@@ -120,6 +120,8 @@
     # LP: #1668659
     mount options=(rw rbind) /snap/ -> /snap/,
     mount options=(rw rshared) -> /snap/,
+    /etc/systemd/system/ r,
+    /bin/systemctl Ux,
 
     # boostrapping the mount namespace
     mount options=(rw rshared) -> /,

tests/main/lxd-refresh-cycle/task.yaml

@@ -0,0 +1,64 @@
+summary: Ensure that we can refresh/remove snaps in LXD
+details: >
+    There is a bug affecting snapd in environments when the root filesystem is
+    not initially "rshared" (in the sense of mount --make-rshared). This test
+    reproduces the issue inside LXD (the issue also affects Ubuntu 14.04
+    without systemd running as init). The issue only happens after we invoke
+    snap-confine at least once, otherwise it is hidden.
+systems: [ubuntu-16*, ubuntu-core-*]
+kill-timeout: 25m # lxd downloads can be quite slow
+restore: |
+    if  [ $(ls -1 "$GOHOME"/snapd_*.deb | wc -l || echo 0) -eq 0 ]; then
+        exit
+    fi
+    lxd.lxc stop my-ubuntu
+    lxd.lxc delete my-ubuntu
+    rm -f mountinfo.*
+debug: |
+    journalctl -u snap.lxd.daemon.service # debug output from lxd
+execute: |
+    if  [[ $(ls -1 "$GOHOME"/snapd_*.deb | wc -l || echo 0) -eq 0 ]]; then
+        echo "No run lxd test when there are not .deb files built"
+        exit
+    fi
+    wait_for_lxd(){
+        while ! printf "GET / HTTP/1.0\n\n" | nc -U /var/snap/lxd/common/lxd/unix.socket | MATCH "200 OK"; do sleep 1; done
+    }
+    echo "Install lxd"
+    snap install lxd
+    echo "Create a trivial container using the lxd snap"
+    wait_for_lxd
+    lxd init --auto
+    lxd.lxc launch ubuntu:16.04 my-ubuntu
+    echo "Ensure we can run things inside"
+    lxd.lxc exec my-ubuntu echo hello | MATCH hello
+    echo "Ensure we can get network"
+    lxd.lxc network create testbr0
+    lxd.lxc network attach testbr0 my-ubuntu eth0
+    lxd.lxc exec my-ubuntu dhclient eth0
+    echo "Install locally built snapd inside the container"
+    lxd.lxc exec my-ubuntu -- apt autoremove --purge -y snapd ubuntu-core-launcher
+    lxd.lxc exec my-ubuntu -- cat /proc/self/mountinfo > mountinfo.after-purge
+    lxd.lxc exec my-ubuntu -- mkdir -p "$GOHOME"
+    lxd.lxc file push "$GOHOME"/snapd_*.deb my-ubuntu/$GOPATH/
+    lxd.lxc exec my-ubuntu -- dpkg -i "$GOHOME"/snapd_*.deb
+    lxd.lxc exec my-ubuntu -- cat /proc/self/mountinfo > mountinfo.after-install
+    # FIXME: workaround for missing squashfuse
+    lxd.lxc exec my-ubuntu -- apt update
+    lxd.lxc exec my-ubuntu -- apt install -y squashfuse
+    echo "Download and side-load a test snap three times"
+    # Revision cannot be globbed remotely so download and glob here.
+    snap download test-snapd-tools
+    lxd.lxc exec my-ubuntu -- snap download test-snapd-tools
+    snap_file=$(ls test-snapd-tools_*.snap)
+    echo "We can refresh the snap as many times for now..."
+    for i in $(seq 5); do
+        lxd.lxc exec my-ubuntu -- snap install --dangerous "$snap_file"
+    done
+    echo "We can also remove it successfully"
+    lxd.lxc exec my-ubuntu -- snap remove test-snapd-tools
+    echo "Running an installed snap will fix the sharing of the mount point"
+    lxd.lxc exec my-ubuntu -- snap install --dangerous "$snap_file"
+    lxd.lxc exec my-ubuntu -- snap run test-snapd-tools.success
+    echo "But not in a way that breaks removal"
+    lxd.lxc exec my-ubuntu -- snap remove test-snapd-tools