Mount points continue to increase without decreasing, exhausting MANAGER_MAX_NAMES and causing DoS

[taskset]

We have encountered a problem. All systemctl commands cannot be executed.
Some errors similar to the following are reported:

Jul 13 19:37:17 e99g07484.et2 dbus[2155]: [system] Activating via systemd: service name='org.freedesktop.PolicyKit1' unit='polkit.service'
Jul 13 19:37:17 e99g07484.et2 dbus[2155]: [system] Activation via systemd failed for unit 'polkit.service': Argument list too long
Jul 13 19:37:17 e99g07484.et2 dbus[2155]: [system] Activating via systemd: service name='org.freedesktop.PolicyKit1' unit='polkit.service'
Jul 13 19:37:17 e99g07484.et2 dbus[2155]: [system] Activation via systemd failed for unit 'polkit.service': Argument list too long

I collected a coredump for analysis and found that the number of n_entries in m->units reached 131072.

(gdb) p m->units
$2 = (Hashmap *) 0x56087eba5290
(gdb) p *m->units
$3 = {b = {hash_ops = 0x56087e59b6e0 <string_hash_ops>, {indirect = {storage = 0x7f87a35fc010 "0\222\275\205\bV",
        hash_key = "H\206\243\250\273$\033\275\224\213\207\025\326p\214\300", n_entries = 131072, n_buckets = 246723,
        idx_lowest_entry = 0, _pad = "\000\000"}, direct = {
        storage = "\020\300_\243\207\177\000\000H\206\243\250\273$\033\275\224\213\207\025\326p\214\300\000\000\002\000\303\303\003\000\000\000\000\000\000\000"}}, type = HASHMAP_TYPE_PLAIN, has_indirect = true, n_direct_entries = 0, from_pool = false}}
(gdb)


 * n_entries = 131072 *

#define MANAGER_MAX_NAMES 131072 /* 128K */

I went on to parse the units details and found that most of the units (13W +) are mounts.

Use the following GDB command to traverse the linked list:

$ cat .gdbinit
define dump_mount_list
set $_node = (Unit *)$arg0
set $_num = 0
while ($_node)
printf "addr: %p, mount->id: %s, source_path: %s\n", $_node, $_node->id, $_node->source_path
set $_node = $_node->units_by_type_next
set $_num = $_num + 1
end
printf "num is %d\n", $_num
end

enum UnitType {
        UNIT_SERVICE = 0,
        UNIT_SOCKET,
        UNIT_BUSNAME,
        UNIT_TARGET,
        UNIT_SNAPSHOT,
        UNIT_DEVICE,
        UNIT_MOUNT,
        UNIT_AUTOMOUNT,

(gdb) p m->units_by_type[6]
$1 = (Unit *) 0x5608a73630f0

130,000 + mount points will be printed:

addr: 0x5608a73630f0, mount->id: home-t4-pouch-containers-5b1dce60939b18d5661d9b6d498c65d08178121f7b95c1481920379acb45dcec-rootfs.mount, source_path: /proc/self/mountinfo
addr: 0x5608a73829f0, mount->id: home-t4-pouch-containerd-state-io.containerd.runtime.v1.linux-default-48b9c8cefd5c953bbf3303e8b4ea7b04a777ccfc789e05e5adf5ebddb834b958-rootfs.mount, source_path: /proc/self/mountinfo
addr: 0x5608a739c240, mount->id: home-t4-pouch-containers-48b9c8cefd5c953bbf3303e8b4ea7b04a777ccfc789e05e5adf5ebddb834b958-rootfs.mount, source_path: /proc/self/mountinfo
addr: 0x5608a7390680, mount->id: home-t4-pouch-containerd-state-io.containerd.runtime.v1.linux-default-8356eb46281e7fbe2c5da86d1a62eb4f93658cb7e3a4c4c854b656921649e1a4-rootfs.mount, source_path: /proc/self/mountinfo
addr: 0x5608a7377040, mount->id: home-t4-pouch-containers-8356eb46281e7fbe2c5da86d1a62eb4f93658cb7e3a4c4c854b656921649e1a4-rootfs.mount, source_path: /proc/self/mountinfo
addr: 0x5608a7267b40, mount->id: home-t4-pouch-containerd-state-io.containerd.runtime.v1.linux-default-6b3ce6d5a5f2126b6c0df9ac3663f8a4e3fc553e8952aa7665f622f662f7f154-rootfs.mount, source_path: /proc/self/mountinfo
addr: 0x5608a732e180, mount->id: home-t4-pouch-containers-6b3ce6d5a5f2126b6c0df9ac3663f8a4e3fc553e8952aa7665f622f662f7f154-rootfs.mount, source_path: /proc/self/mountinfo
……

After analyzing the code, I found the following possible bugs：

static int mount_dispatch_io(sd_event_source *source, int fd, uint32_t revents, void *userdata) {
……
        r = mount_load_proc_self_mountinfo(m, true);         -->Add the data from /proc/self/mountinfo to m->units
        if (r < 0) {
                /* Reset flags, just in case, for later calls */
                LIST_FOREACH(units_by_type, u, m->units_by_type[UNIT_MOUNT]) {
                        Mount *mount = MOUNT(u);

                        mount->is_mounted = mount->just_mounted = mount->just_changed = false;
                }

                return 0;             -->If returned here, the data in m->units will only increase, not decrease
        }

        manager_dispatch_load_queue(m);

        LIST_FOREACH(units_by_type, u, m->units_by_type[UNIT_MOUNT]) {
…                             -->The code here will clean up the residual data in m->units
        } 
…
}

I also constructed a use case to reproduce the bug.

A, Construct a path greater than 256 characters (so mount_load_proc_self_mountinfo () returns an error code)：

# mkdir -p /run/kata-containers/shared/sandboxes/f0ea3efdb417f442128830e86118cf216d1d236d6f970205a680972bcd062f74/f0ea3efdb417f442128830e86118cf216d1d236d6f970205a680972bcd062f74-a1bed3c11a474518-aaaaaa_xxxx_mix_xxxx_container_role_20200310112829109807.yyyy_container_role_20200310112829109807_15_81
# mkdir -p /tmp/test
# mount --bind /tmp/test   /run/kata-containers/shared/sandboxes/f0ea3efdb417f442128830e86118cf216d1d236d6f970205a680972bcd062f74/f0ea3efdb417f442128830e86118cf216d1d236d6f970205a680972bcd062f74-a1bed3c11a474518-aaaaaa_xxxx_mix_xxxx_container_role_20200310112829109807.yyyy_container_role_20200310112829109807_15_81

B, Mount some directories, then umount them

# mkdir -p ./a1 ./b1
# mount --bind a1 b1
# 
# mkdir -p ./a2 ./b2
# mount --bind a2 b2
# 
# mkdir -p ./a3 ./b3
# mount --bind a3 b3
# 
# umount  b1
# umount  b2
# umount  b3

C, Finally, through GDB analysis, it can be found that these mount points are still in m->units:

Breakpoint 1, mount_dispatch_io (source=0x55a04d0f4910, fd=9, revents=10, userdata=0x55a04d0ec040) at src/core/mount.c:1711
1711	static int mount_dispatch_io(sd_event_source *source, int fd, uint32_t revents, void *userdata) {
(gdb) bt
#0  mount_dispatch_io (source=0x55a04d0f4910, fd=9, revents=10, userdata=0x55a04d0ec040) at src/core/mount.c:1711
#1  0x000055a04c7ab6f0 in source_dispatch (s=s@entry=0x55a04d0f4910) at src/libsystemd/sd-event/sd-event.c:2115
#2  0x000055a04c7ac78a in sd_event_dispatch (e=0x55a04d0ec5e0) at src/libsystemd/sd-event/sd-event.c:2472
#3  0x000055a04c7ac92f in sd_event_run (e=<optimized out>, timeout=<optimized out>) at src/libsystemd/sd-event/sd-event.c:2501
#4  0x000055a04c70c2c3 in manager_loop (m=0x55a04d0ec040) at src/core/manager.c:2274
#5  0x000055a04c7006b1 in main (argc=5, argv=0x7ffe1edfaae8) at src/core/main.c:1819
(gdb) frame 5
#5  0x000055a04c7006b1 in main (argc=5, argv=0x7ffe1edfaae8) at src/core/main.c:1819
1819	                r = manager_loop(m);
(gdb) p  m->units_by_type[6]
$1 = (Unit *) 0x55a04d108c80
(gdb) dump_mount_list  0x55a04d108c80
addr: 0x55a04d108c80, mount->id: mnt-work-issues-systemd_maxunits-b3.mount, source_path: /proc/self/mountinfo
addr: 0x55a04d15e120, mount->id: mnt-work-issues-systemd_maxunits-b2.mount, source_path: /proc/self/mountinfo
...
addr: 0x55a04d0ef580, mount->id: run-user-0.mount, source_path: /proc/self/mountinfo
addr: 0x55a04d17ca10, mount->id: mnt-work-issues-systemd_maxunits-b1.mount, source_path: /proc/self/mountinfo
....

Similar bugs：
https://access.redhat.com/solutions/4620671
systemd/systemd#15221
kubernetes/kubernetes#57345

[taskset]

The following patch may fix it:
ba0d56f （"mount: don't propagate errors from mount_setup_unit() further up"）

[taskset]

systemd/systemd#14294

[taskset]

@kyle-walker @poettering @dtardon @keszybz
Please take a look at it. Thank you.

[dtardon]

@w-simon Please report systemd bugs in Red Hat Enterprise Linux to https://bugzilla.redhat.com .

[fuweid]

/cc