Skip to content

Commit

Permalink
Limit memory usage based on cgroup info
Browse files Browse the repository at this point in the history
We try to convert OOM-kill at the cgroup level into python MemoryError.

The method employed has a number of caveats:
- setrlimit only acts on one process, when cgroup is a group of process
  ; other process in the container could also consume memory
- the OOM-killer might still kick in before Python raising MemoryError,
  depending on the particular of the Python allocator
  • Loading branch information
VannTen committed Jan 10, 2023
1 parent 1992d58 commit 0207c59
Showing 1 changed file with 28 additions and 0 deletions.
28 changes: 28 additions & 0 deletions thoth/solver/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@

import click
import logging
import os
import resource
import time

from thoth.analyzer import print_command_result
Expand Down Expand Up @@ -67,6 +69,30 @@ def cli(ctx=None, verbose=0):
_LOG.debug("Debug mode is on")


def _limit_memory() -> None:
"""Limit memory to cgroup limit if we're inside a container.
This turn OOM killer errors into python MemoryError exceptions.
"""
for file in [
'/sys/fs/cgroup/memory/memory.soft_limit_in_bytes', # cgroups v1
'/sys/fs/cgroup/memory/memory.limit_in_bytes',
'/sys/fs/cgroup/memory.high', # cgroups v2
'/sys/fs/cgroup/memory.max',
]:
try:
with open(file) as limit:
memory = limit.read()
if memory == "max": # memory.high might contain "max" -> indicates to use the memory.max value
continue
value = int(memory)
resource.setrlimit(resource.RLIMIT_AS, (value, value))
_LOG.info("Limiting memory to %i bytes based on cgroup limit", value)
break
except FileNotFoundError:
pass


@cli.command()
@click.pass_context
@click.option(
Expand Down Expand Up @@ -155,6 +181,8 @@ def python(
_LOG.error("No requirements specified, exiting")
sys.exit(1)

_limit_memory()

index_urls = index.split(",") if index else ("https://pypi.org/simple",)
dependency_index_urls = dependency_index.split(",") if dependency_index else index_urls

Expand Down

0 comments on commit 0207c59

Please sign in to comment.