cgfsng: add cgfsng_monitor_destroy()

Since we switched to the new cgroup scoping scheme that places the
container payload into lxc.payload/<container-name> and
lxc.monitor/<container-name> deletion becomes slightly more complicated.
The monitor will be able to rm_rf(lxc.payload/<container-name>) but will
not be able to rm_rf(lxc.monitor/<container-name>) since it will be
located in that cgroup and it will thus be populated.
My current solution to this is to create a lxc.pivot cgroup that only
exists so that the monitor process on container stop can pivot into it,
call rm_rf(lxc.monitor/<container-name>) and can then exit. This group
has not function whatsoever apart from this and can thus be shared by
all monitor processes.

Signed-off-by: Christian Brauner <christian.brauner@ubuntu.com>

src/lxc/cgroups/cgfsng.c

@@ -1111,8 +1111,8 @@ static int cgroup_rmdir_wrapper(void *data)
 	return cgroup_rmdir(arg->hierarchies, arg->container_cgroup);
 }
 
-__cgfsng_ops__ static void cgfsng_payload_destroy(struct cgroup_ops *ops,
-		struct lxc_handler *handler)
+__cgfsng_ops static void cgfsng_payload_destroy(struct cgroup_ops *ops,
+						struct lxc_handler *handler)
 {
 	int ret;
 	struct generic_userns_exec_data wrap;
@@ -1133,6 +1133,60 @@ __cgfsng_ops__ static void cgfsng_payload_destroy(struct cgroup_ops *ops,
 	}
 }
 
+__cgfsng_ops static void cgfsng_monitor_destroy(struct cgroup_ops *ops,
+						struct lxc_handler *handler)
+{
+	int len;
+	char *pivot_path;
+	struct lxc_conf *conf = handler->conf;
+	char pidstr[INTTYPE_TO_STRLEN(pid_t)];
+
+	if (!ops->hierarchies)
+		return;
+
+	len = snprintf(pidstr, sizeof(pidstr), "%d", handler->monitor_pid);
+	if (len < 0 || (size_t)len >= sizeof(pidstr))
+		return;
+
+	for (int i = 0; ops->hierarchies[i]; i++) {
+		int ret;
+		struct hierarchy *h = ops->hierarchies[i];
+
+		if (!h->monitor_full_path)
+			continue;
+
+		if (conf && conf->cgroup_meta.dir)
+			pivot_path = must_make_path(h->mountpoint,
+						    h->container_base_path,
+						    conf->cgroup_meta.dir,
+						    "lxc.pivot",
+						    "cgroup.procs", NULL);
+		else
+			pivot_path = must_make_path(h->mountpoint,
+						    h->container_base_path,
+						    "lxc.pivot",
+						    "cgroup.procs", NULL);
+
+		ret = mkdir_p(pivot_path, 0755);
+		if (ret < 0 && errno != EEXIST)
+			goto next;
+
+		/* Move ourselves into the pivot cgroup to delete our own
+		 * cgroup.
+		 */
+		ret = lxc_write_to_file(pivot_path, pidstr, len, false, 0666);
+		if (ret != 0)
+			goto next;
+
+		ret = recursive_destroy(h->monitor_full_path);
+		if (ret < 0)
+			WARN("Failed to destroy \"%s\"", h->monitor_full_path);
+
+	next:
+		free(pivot_path);
+	}
+}
+
 static bool cg_unified_create_cgroup(struct hierarchy *h, char *cgname)
 {
 	size_t i, parts_len;
@@ -2682,7 +2736,8 @@ struct cgroup_ops *cgfsng_ops_init(struct lxc_conf *conf)
 	}
 
 	cgfsng_ops->data_init = cgfsng_data_init;
-	cgfsng_ops->destroy = cgfsng_payload_destroy;
+	cgfsng_ops->payload_destroy = cgfsng_payload_destroy;
+	cgfsng_ops->monitor_destroy = cgfsng_monitor_destroy;
 	cgfsng_ops->monitor_create = cgfsng_monitor_create;
 	cgfsng_ops->monitor_enter = cgfsng_monitor_enter;
 	cgfsng_ops->payload_create = cgfsng_payload_create;

src/lxc/cgroups/cgroup.h

@@ -129,7 +129,8 @@ struct cgroup_ops {
 	cgroup_layout_t cgroup_layout;
 
 	bool (*data_init)(struct cgroup_ops *ops);
-	void (*destroy)(struct cgroup_ops *ops, struct lxc_handler *handler);
+	void (*payload_destroy)(struct cgroup_ops *ops, struct lxc_handler *handler);
+	void (*monitor_destroy)(struct cgroup_ops *ops, struct lxc_handler *handler);
 	bool (*monitor_create)(struct cgroup_ops *ops, struct lxc_handler *handler);
 	bool (*monitor_enter)(struct cgroup_ops *ops, pid_t pid);
 	bool (*payload_create)(struct cgroup_ops *ops, struct lxc_handler *handler);

src/lxc/start.c

@@ -725,6 +725,8 @@ int lxc_init(const char *name, struct lxc_handler *handler)
 	const char *loglevel;
 	struct lxc_conf *conf = handler->conf;
 
+	handler->monitor_pid = lxc_raw_getpid();
+
 	lsm_init();
 	TRACE("Initialized LSM");
 
@@ -857,7 +859,8 @@ int lxc_init(const char *name, struct lxc_handler *handler)
 	return 0;
 
 out_destroy_cgroups:
-	handler->cgroup_ops->destroy(handler->cgroup_ops, handler);
+	handler->cgroup_ops->payload_destroy(handler->cgroup_ops, handler);
+	handler->cgroup_ops->monitor_destroy(handler->cgroup_ops, handler);
 
 out_delete_terminal:
 	lxc_terminal_delete(&handler->conf->console);
@@ -951,7 +954,8 @@ void lxc_fini(const char *name, struct lxc_handler *handler)
 
 	lsm_process_cleanup(handler->conf, handler->lxcpath);
 
-	cgroup_ops->destroy(cgroup_ops, handler);
+	cgroup_ops->payload_destroy(cgroup_ops, handler);
+	cgroup_ops->monitor_destroy(cgroup_ops, handler);
 	cgroup_exit(cgroup_ops);
 
 	if (handler->conf->reboot == REBOOT_NONE) {
@@ -1971,7 +1975,7 @@ int __lxc_start(const char *name, struct lxc_handler *handler,
 		goto out_fini_nonet;
 	}
 
-	if (!cgroup_ops->monitor_enter(cgroup_ops, lxc_raw_getpid())) {
+	if (!cgroup_ops->monitor_enter(cgroup_ops, handler->monitor_pid)) {
 		ERROR("Failed to enter monitor cgroup");
 		goto out_fini_nonet;
 	}

src/lxc/start.h

@@ -103,6 +103,9 @@ struct lxc_handler {
 	/* The child's pid. */
 	pid_t pid;
 
+	/* The monitor's pid. */
+	pid_t monitor_pid;
+
 	/* Whether the child has already exited. */
 	bool init_died;