podman unable to limit memory (-m flag) on Ubuntu/Debian distros #6365
Comments
openshift-ci-robot
added
the
kind/bug
|
Did you try adding I think I remember having the sample problem a while ago and fixing it this way |
|
@geverartsdev Geeze have I just been spoiled by the RPM distros lol. I totally forgot about the kernal params that aren't switched on by default in other distros. Yes this totally worked thank you. A follow up question though. Are there changes that can be made to not have this problem out-of-the-box. Docker doesn't have this problem, so I'm guessing they are doing something to bypass ( I say bypass because I just tested installing docker on a Debian machine and doing memory limiting without issue but then installed podman and got the same issue until I set the params) or maybe they're only limiting pure memory and not swap+mem? |
|
@richpeaua I am pleased to know that it worked for you. I guess it could be possible to avoid running into this issue out of the box yes. When you take a look at the configuration file ( An example of the config.json file with memory limitation (see linux.resources.memory){
"ociVersion":"1.0.1-dev",
"process":{
"terminal":true,
"user":{
"uid":0,
"gid":0,
"additionalGids":[
0,
1,
2,
3,
4,
6,
10,
11,
20,
26,
27
]
},
"args":[
"sh"
],
"env":[
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"TERM=xterm",
"HOSTNAME=1d3498689c56",
"container=podman",
"HOME=/root"
],
"cwd":"/",
"capabilities":{
"bounding":[
"CAP_AUDIT_WRITE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_MKNOD",
"CAP_NET_BIND_SERVICE",
"CAP_NET_RAW",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYS_CHROOT"
],
"effective":[
"CAP_AUDIT_WRITE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_MKNOD",
"CAP_NET_BIND_SERVICE",
"CAP_NET_RAW",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYS_CHROOT"
],
"inheritable":[
"CAP_AUDIT_WRITE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_MKNOD",
"CAP_NET_BIND_SERVICE",
"CAP_NET_RAW",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYS_CHROOT"
],
"permitted":[
"CAP_AUDIT_WRITE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_MKNOD",
"CAP_NET_BIND_SERVICE",
"CAP_NET_RAW",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYS_CHROOT"
],
"ambient":[
"CAP_AUDIT_WRITE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_MKNOD",
"CAP_NET_BIND_SERVICE",
"CAP_NET_RAW",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYS_CHROOT"
]
},
"rlimits":[
{
"type":"RLIMIT_NOFILE",
"hard":1048576,
"soft":1048576
},
{
"type":"RLIMIT_NPROC",
"hard":32768,
"soft":32768
}
],
"oomScoreAdj":0
},
"root":{
"path":"/var/lib/containers/storage/aufs/mnt/4b6754a223c81dcc18b836a5a0ddd3742dbe18331e9d8c1cf444deaca3101577"
},
"hostname":"1d3498689c56",
"mounts":[
{
"destination":"/proc",
"type":"proc",
"source":"proc",
"options":[
"nosuid",
"noexec",
"nodev"
]
},
{
"destination":"/dev",
"type":"tmpfs",
"source":"tmpfs",
"options":[
"nosuid",
"strictatime",
"mode=755",
"size=65536k"
]
},
{
"destination":"/sys",
"type":"sysfs",
"source":"sysfs",
"options":[
"nosuid",
"noexec",
"nodev",
"ro"
]
},
{
"destination":"/dev/pts",
"type":"devpts",
"source":"devpts",
"options":[
"nosuid",
"noexec",
"newinstance",
"ptmxmode=0666",
"mode=0620",
"gid=5"
]
},
{
"destination":"/dev/mqueue",
"type":"mqueue",
"source":"mqueue",
"options":[
"nosuid",
"noexec",
"nodev"
]
},
{
"destination":"/etc/resolv.conf",
"type":"bind",
"source":"/var/run/containers/storage/aufs-containers/1d3498689c569473c6e0c905abaf0e705efe11243e053a2cc71dbd605eee9d88/userdata/resolv.conf",
"options":[
"bind",
"private"
]
},
{
"destination":"/etc/hosts",
"type":"bind",
"source":"/var/run/containers/storage/aufs-containers/1d3498689c569473c6e0c905abaf0e705efe11243e053a2cc71dbd605eee9d88/userdata/hosts",
"options":[
"bind",
"private"
]
},
{
"destination":"/dev/shm",
"type":"bind",
"source":"/var/lib/containers/storage/aufs-containers/1d3498689c569473c6e0c905abaf0e705efe11243e053a2cc71dbd605eee9d88/userdata/shm",
"options":[
"bind",
"private"
]
},
{
"destination":"/etc/hostname",
"type":"bind",
"source":"/var/run/containers/storage/aufs-containers/1d3498689c569473c6e0c905abaf0e705efe11243e053a2cc71dbd605eee9d88/userdata/hostname",
"options":[
"bind",
"private"
]
},
{
"destination":"/run/.containerenv",
"type":"bind",
"source":"/var/run/containers/storage/aufs-containers/1d3498689c569473c6e0c905abaf0e705efe11243e053a2cc71dbd605eee9d88/userdata/.containerenv",
"options":[
"bind",
"private"
]
},
{
"destination":"/sys/fs/cgroup",
"type":"cgroup",
"source":"cgroup",
"options":[
"rprivate",
"nosuid",
"noexec",
"nodev",
"relatime",
"ro"
]
}
],
"annotations":{
"io.container.manager":"libpod",
"io.kubernetes.cri-o.Created":"2020-05-24T10:29:21.571932948+02:00",
"io.kubernetes.cri-o.TTY":"true",
"io.podman.annotations.autoremove":"TRUE",
"io.podman.annotations.init":"FALSE",
"io.podman.annotations.privileged":"FALSE",
"io.podman.annotations.publish-all":"FALSE",
"org.opencontainers.image.stopSignal":"15"
},
"linux":{
"resources":{
"devices":[
{
"allow":false,
"access":"rwm"
}
],
"memory":{
"limit":1073741824,
"swap":2147483648,
"disableOOMKiller":false
},
"cpu":{
"quota":100000,
"period":100000
},
"pids":{
"limit":4096
}
},
"cgroupsPath":"machine.slice:libpod:1d3498689c569473c6e0c905abaf0e705efe11243e053a2cc71dbd605eee9d88",
"namespaces":[
{
"type":"pid"
},
{
"type":"network",
"path":"/var/run/netns/cni-e21d40f8-6b16-fd55-07fe-b82b80c8a071"
},
{
"type":"ipc"
},
{
"type":"uts"
},
{
"type":"mount"
},
{
"type":"cgroup"
}
],
"seccomp":{
"defaultAction":"SCMP_ACT_ERRNO",
"architectures":[
"SCMP_ARCH_X86_64",
"SCMP_ARCH_X86",
"SCMP_ARCH_X32"
],
"syscalls":[...]
},
"maskedPaths":[
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware",
"/sys/fs/selinux"
],
"readonlyPaths":[
"/proc/asound",
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
}
}An example of the config.json file without memory limitation (see linux.resources.memory){
"ociVersion":"1.0.1-dev",
"process":{
"terminal":true,
"user":{
"uid":0,
"gid":0,
"additionalGids":[
0,
1,
2,
3,
4,
6,
10,
11,
20,
26,
27
]
},
"args":[
"sh"
],
"env":[
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"TERM=xterm",
"HOSTNAME=618b302f1e2c",
"container=podman",
"HOME=/root"
],
"cwd":"/",
"capabilities":{
"bounding":[
"CAP_AUDIT_WRITE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_MKNOD",
"CAP_NET_BIND_SERVICE",
"CAP_NET_RAW",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYS_CHROOT"
],
"effective":[
"CAP_AUDIT_WRITE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_MKNOD",
"CAP_NET_BIND_SERVICE",
"CAP_NET_RAW",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYS_CHROOT"
],
"inheritable":[
"CAP_AUDIT_WRITE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_MKNOD",
"CAP_NET_BIND_SERVICE",
"CAP_NET_RAW",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYS_CHROOT"
],
"permitted":[
"CAP_AUDIT_WRITE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_MKNOD",
"CAP_NET_BIND_SERVICE",
"CAP_NET_RAW",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYS_CHROOT"
],
"ambient":[
"CAP_AUDIT_WRITE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_MKNOD",
"CAP_NET_BIND_SERVICE",
"CAP_NET_RAW",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYS_CHROOT"
]
},
"rlimits":[
{
"type":"RLIMIT_NOFILE",
"hard":1048576,
"soft":1048576
},
{
"type":"RLIMIT_NPROC",
"hard":32768,
"soft":32768
}
],
"oomScoreAdj":0
},
"root":{
"path":"/var/lib/containers/storage/aufs/mnt/87f43a4d823ebc7bb385944aa1761d5ec6093e98e667a13f67a2f208996de12e"
},
"hostname":"618b302f1e2c",
"mounts":[
{
"destination":"/proc",
"type":"proc",
"source":"proc",
"options":[
"nosuid",
"noexec",
"nodev"
]
},
{
"destination":"/dev",
"type":"tmpfs",
"source":"tmpfs",
"options":[
"nosuid",
"strictatime",
"mode=755",
"size=65536k"
]
},
{
"destination":"/sys",
"type":"sysfs",
"source":"sysfs",
"options":[
"nosuid",
"noexec",
"nodev",
"ro"
]
},
{
"destination":"/dev/pts",
"type":"devpts",
"source":"devpts",
"options":[
"nosuid",
"noexec",
"newinstance",
"ptmxmode=0666",
"mode=0620",
"gid=5"
]
},
{
"destination":"/dev/mqueue",
"type":"mqueue",
"source":"mqueue",
"options":[
"nosuid",
"noexec",
"nodev"
]
},
{
"destination":"/etc/resolv.conf",
"type":"bind",
"source":"/var/run/containers/storage/aufs-containers/618b302f1e2c2b7e6c0d88f6a68b7385c73004689f4c860b50406758f2fe653f/userdata/resolv.conf",
"options":[
"bind",
"private"
]
},
{
"destination":"/etc/hosts",
"type":"bind",
"source":"/var/run/containers/storage/aufs-containers/618b302f1e2c2b7e6c0d88f6a68b7385c73004689f4c860b50406758f2fe653f/userdata/hosts",
"options":[
"bind",
"private"
]
},
{
"destination":"/dev/shm",
"type":"bind",
"source":"/var/lib/containers/storage/aufs-containers/618b302f1e2c2b7e6c0d88f6a68b7385c73004689f4c860b50406758f2fe653f/userdata/shm",
"options":[
"bind",
"private"
]
},
{
"destination":"/etc/hostname",
"type":"bind",
"source":"/var/run/containers/storage/aufs-containers/618b302f1e2c2b7e6c0d88f6a68b7385c73004689f4c860b50406758f2fe653f/userdata/hostname",
"options":[
"bind",
"private"
]
},
{
"destination":"/run/.containerenv",
"type":"bind",
"source":"/var/run/containers/storage/aufs-containers/618b302f1e2c2b7e6c0d88f6a68b7385c73004689f4c860b50406758f2fe653f/userdata/.containerenv",
"options":[
"bind",
"private"
]
},
{
"destination":"/sys/fs/cgroup",
"type":"cgroup",
"source":"cgroup",
"options":[
"rprivate",
"nosuid",
"noexec",
"nodev",
"relatime",
"ro"
]
}
],
"annotations":{
"io.container.manager":"libpod",
"io.kubernetes.cri-o.Created":"2020-05-24T10:38:46.60949562+02:00",
"io.kubernetes.cri-o.TTY":"true",
"io.podman.annotations.autoremove":"TRUE",
"io.podman.annotations.init":"FALSE",
"io.podman.annotations.privileged":"FALSE",
"io.podman.annotations.publish-all":"FALSE",
"org.opencontainers.image.stopSignal":"15"
},
"linux":{
"resources":{
"devices":[
{
"allow":false,
"access":"rwm"
}
],
"memory":{
"disableOOMKiller":false
},
"cpu":{
"quota":100000,
"period":100000
},
"pids":{
"limit":4096
}
},
"cgroupsPath":"machine.slice:libpod:618b302f1e2c2b7e6c0d88f6a68b7385c73004689f4c860b50406758f2fe653f",
"namespaces":[
{
"type":"pid"
},
{
"type":"network",
"path":"/var/run/netns/cni-5c2ac273-08c9-663e-8dbc-5d333ee21a8c"
},
{
"type":"ipc"
},
{
"type":"uts"
},
{
"type":"mount"
},
{
"type":"cgroup"
}
],
"seccomp":{
"defaultAction":"SCMP_ACT_ERRNO",
"architectures":[
"SCMP_ARCH_X86_64",
"SCMP_ARCH_X86",
"SCMP_ARCH_X32"
],
"syscalls":[ ...]
},
"maskedPaths":[
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware",
"/sys/fs/selinux"
],
"readonlyPaths":[
"/proc/asound",
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
}
} |
|
We have this issue with Minikube on Ubuntu as well, there is a difference between Docker and Podman. As a workaround, we have to disable all memory limits (not only swap) for Podman... i.e. with docker we get a warning, but with podman it is an error. It's acceptable for the memory-swap limit to not be enforced, but it is unfortunate that we also have to disable memory limit without it. Here is our workaround, in some Go code: runArgs = append(runArgs, fmt.Sprintf("--cpus=%s", p.CPUs))
memcgSwap := true
if runtime.GOOS == "linux" {
if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); os.IsNotExist(err) {
// requires CONFIG_MEMCG_SWAP_ENABLED or cgroup_enable=memory in grub
glog.Warning("Your kernel does not support swap limit capabilities or the cgroup is not mounted.")
memcgSwap = false
}
}
if p.OCIBinary == Podman && memcgSwap { // swap is required for memory
runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory))
}
if p.OCIBinary == Docker { // swap is only required for --memory-swap
runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory))
}And here is the command line tool output: |
|
In the original code it was unconditionally disabled, due to some confusion with rootless: That is , forgot to run So unfortunately this was not reported upstream (cc @medyagh) And then we just did the workaround. |
|
@geverartsdev That config.json provided great information, thank you. I took the liberty in doing something similar with docker to see the difference, but instead of memory limited vs non-limited, here is the output of hostconfig.json for memory-limited w/ swap enabled and w/out swap enabled. As you can see, Docker is able to detect and conditionally change limiting from swap+mem to just mem (and outputting the swap limit warning seen in @afbjorklund posts). So the next steps would be to adjust the code in podman to do the same detect and switch (also seen in @afbjorklund post) hostconfig.json w/ swap limiting enabled (MemorySwap)hostconfig.json w/out swap limiting enabled (MemorySwap) |
|
I know where we set the swap limit in the code (I was the one who added it to Podman v2.0), so if there's an easy way to resolve if the host supports swap limiting, it should be easy enough to add. |
|
@giuseppe Is this something we can throw in the libpod cgroups package? |
|
@mheon Awesome. When swap limiting is disabled, the cgroup resource control file to control swap+mem, i.e For the fix. We would need to switch from using |
|
@richpeaua Just note that this would only work for cgroupv1, not cgroupv2. But the same principle can be applied to cgroupv2 (looking for file |
Why? This file is actually present even when no control over swap is supported |
|
It doesn't look like that file check works - it's likely fine for cgroups v1, but my F31 laptop on v2 does not have it (and definitely supports swap limits) |
|
I'm not sure if kubernetes supports v2 yet, so we just tell people to turn it off (or use a VM, if needed) We really need some better warnings/suggestions, for people running with SELinux and Cgroups V2 |
|
@geverartsdev Yes you're correct. Those are cgroupv1 control files. So maybe we can also use this to future proof as well by adding checks and conditions for v2? Regarding the presence of the memsw file, if you look at my first post concerning the issue I posted the differences in the swap limit enabled and disabled directories (albeit one from Debian and the other from Centos8) but as you can see memsw.limit_in_bytes is not present in swap limit disabled. And the workaround used by @afbjorklund for minikube takes that into account as well. The potential fix would be something like (psuedo code). Or something along those lines. @mheon I would suggest trying this out on a Debian/Ubuntu machine if you can. I haven't run into this issue on any RPM distro due to most, if not all, having swap limiting enabled by default. |
|
@richpeaua Right, I think I now see what you meant by saying "switch from using The thing is, if my understaning of the situation is correct, Podman does not have to deal with those files directly, this is actually the underlying container runtime (runc, crun, ...) which will, through the mean of systemd or cgroupfs, edit those files accordingly. What I showed you in the config.json files is the input given to those container runtimes, they will see that we require some resource management for the container to create, and they will take care of it for us. So the only thing that podman would have to do in order to avoid your problem out of the box, is to not ask to the container runtime to setup such limit that the system does not support. The relevance of finding a good detection mechanism doesn't change is the same. Alternatively the container runtime could also detect this error and decide to ignore the requirement of the user, but this is another debate. |
|
@geverartsdev You're right. That would fall to the runtime... and I believe you're correct again in stating that the fix would lie, not in changing the control file to use, but what is written to the This can be seen in a post I made with the docker version of Interestingly, the error outputted by podman is that it is attempting to write |
|
@mheon can we perform the check on the presence of the For example, if the file is not present then exclude that field and have the value of the |
|
As I said before, we cannot gate on the presence of that file, because it does not work for both cgroups v1 and v2. This may not be relevant for Minikube, but Podman has to support both cgroup versions. With a reliable testing function, yes, this is a trivial change. |
|
Seems like the Kata-Containers folks had a discussion about this (not exactly the same problem but related), this might give some ideas kata-containers/packaging#104 @mheon How bad would it be to use two different tricks for cgroupv1 and cgroupv2? As long as in the end it works for both of course. Btw, do you know what does podman do to detect the current cgroup version? |
|
We can already detect whether we're on a v1 or v2 system, and we already have a viable way to detect whether swap limits are available on v1, so we just need a way to detect whether support is available on v2 and we should be good. |
Ow, didn't know that, what is it? Couldn't this inspire something similar for v2? |
|
FYI the kernel config settings Just tested this on Debian and Ubuntu machines. Setting both to They do, however, indicate if swap limiting is enabled by default (which is the case with the RPM distros). |
I guess ultimately Kubernetes will support v2, and distributions (and even docker) will start using it. And then it would of course be nice if the user was able to run crio-in-podman with minikube on those. But since that is not yet the case, we require our Fedora users to either disable cgroups v2 or to run a KVM VM (with cgroups v1). As described elsewhere, this is more of a chicken and egg problem... kubernetes/minikube#6795 (comment) https://medium.com/nttlabs/cgroup-v2-596d035be4d7 As long as nobody uses v2, nothing will support it. And when nothing supports it, nobody will use it. It would still be nice if podman was able to limit memory, when memory swap limits are unavailable. And even better if this also worked with cgroups v2, or at least when back to failing in the old way.
Assuming that they are available when on v2 would probably work out (and only check on v1). Currently it happily assumes that they are available on both, even when not needed, and errors out. |
|
I think this is perfectly reasonable to do for us to support limiting memory only when swap limits aren't available, but on the Podman side, I need to make sure this does work on both v1 and v2 environments. My current theory is that, if we can identify the cgroup that the current process is part of (alternatively we can use |
|
@mheon It seems that rootless podman run container cgroups are in the i.e Here are some outputs from some containers I've run. podman run -d hello-world sleep 120[ROOTLESS]: sudo podman run -d hello-world sleep 120 [ROOT]: |
|
memory swap is always present on cgroup v2, so we don't have to worry. For cgroup v1, we need something like cri-o/cri-o@7bdf938 If someone is interested in fixing it and opens a PR, I can review it |
On my machine, if I don't add
|
|
I'll try and get to this one this week |
|
@mheon Any update on the this? |
|
Not yet - fell through the cracks (I forgot to self-assign so I didn't see it when I reviewed open issues - oops). I'll try and get to it tomorrow. |
|
@mheon Not a worries! Thank you! |
|
I haven't double-checked, but I'm fairly certain this was fixed by the 2.0 release, which checks sysinfo to verify swap limits are possible |
|
Please reopen if @mheon is mistaken. |
@rhatdan @mheon It doesn't look to me it's possible to run containers with a memory limit, but without swap accounting ( |
|
Using sysinfo doesn't seem sufficient. It's only working in conjunction with Would it be reasonable to check |
|
I don't have permissions to reopen this issue, fwiw. |
|
Why can't you just verify that the file memory.swap.max exists before trying to write it? |
|
Podman does not currently have logic for this, and it could get complicated supporting CGroups V1 and V2. |
|
podman (either with docker (with same runc as podman above): Also note that |
|
@giuseppe WDYT? |
|
Podman 2.0 seems to behave differently: Is this the desired behaviour? Even if compatible with what Docker does, I am not sure we should ignore a configuration that was explicitly set by the user. For cgroup v2 we don't have this check in place (even if |
|
opened a PR to add the same check on cgroup v2: #8197 |
I believe so, yes. For the record, it seems podman 1.8.2 had the issue described above (cgroup v1, swapaccount=0): podman 2.1.1 works (cgroup v1, swapaccount=0): |
|
This is fixed in master. |
|
Thanks all! |
github-actions
bot
added
the
locked - please file new issue/PR
/kind bug
Description
Was trying to test drive Hashicorp Nomad podman driver on Debian/Ubuntu machines and I ran into an interesting issue. Deployment of containers, via Nomad, fail, yet if I just ran
podman run <image>it worked fine.The errors I was getting were "failed to write -1 to /sys/fs/cgroup/memory/machine.slice/libpod-2e61c7b46bc2aeed6dadecb07583e97e03ffcc694.scope/memory.memsw.limit_in_bytes".
I looked into that libpod-*-.scope cgroup directory and it's missing the memory.memsw.limit_in_bytes resource control file
I then deployed to a Centos8 machine and it worked just fine. I followed up by checking to see if that file was present and sure enough it was.
Due to this difference in the cgroup control files podman, and thus Nomad, is unable to limit memory in the Debian/Ubuntu distros
Steps to reproduce the issue:
Describe the results you received:
Your kernel does not support swap limit capabilities,or the cgroup is not mounted. Memory limited without swap. Error: container_linux.go:349: starting container process caused "process_linux.go:449: container init caused \"process_linux.go:415: setting cgroup config for procHooks process caused \\\"failed to write \\\\\\\"-1\\\\\\\" to \\\\\\\"/sys/fs/cgroup/memory/machine.slice/libpod-8639bb6811f6bec5282ee72dac48abb6a647ce7daf84dd000b195dc4fe6f4df7.scope/memory.memsw.limit_in_bytes\\\\\\\": open /sys/fs/cgroup/memory/machine.slice/libpod-8639bb6811f6bec5282ee72dac48abb6a647ce7daf84dd000b195dc4fe6f4df7.scope/memory.memsw.limit_in_bytes: permission denied\\\"\"": OCI runtime permission denied errorDescribe the results you expected:
Output of hello-world container/successful job deployment status on Nomad
Additional information you deem important (e.g. issue happens only occasionally):
I've tried using multiple versions of podman. I've looked into all the Ubuntu distros 16.04+ and the same issue in all of those releases, so it seems to be just a weird divergence in cgroup resource control interfaces between the RPM and Deb distros.
And I want to repeat, running podman without -m flag WORKS. both rootless and with root.
Output of
podman version:Output of
podman info --debug:Package info (e.g. output of
rpm -q podmanorapt list podman):Additional environment details (AWS, VirtualBox, physical, etc.):
VirtualBox, GCP compute instances
The text was updated successfully, but these errors were encountered: