Skip to content

Commit

Permalink
Make ShellcmdLrms handle app.requested_memory as total app memory.
Browse files Browse the repository at this point in the history
`Application.requested_memory` is the *total* amount of memory requested by the application independently of the number of cores, as documented [in the `Application` docstring][1].

Now, different backends handle `Application.requested_memory` differently; see for example the behaviour of `Application.sbatch()` where `Application..requested_memory` is handled as a per-node requirement (see the documentation for the '--mem' option [in the SLURM man pages][2])

So the way to solve this problem is to make sure that `Application.requested_memory` is handled consistently for all backends, hence, change `ShellcmdLrms` accordingly.

[1]: https://github.com/uzh/gc3pie/blob/master/gc3libs/__init__.py#L765
[2]: https://slurm.schedmd.com/sbatch.html
  • Loading branch information
smaffiol authored and riccardomurri committed Mar 5, 2018
1 parent 3541a1a commit 5d6b1f0
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 7 deletions.
9 changes: 4 additions & 5 deletions gc3libs/backends/shellcmd.py
Expand Up @@ -1357,7 +1357,7 @@ def submit_job(self, app):
self.free_slots -= app.requested_cores
self.user_run += 1
if app.requested_memory:
self.available_memory -= (app.requested_memory * app.requested_cores)
self.available_memory -= app.requested_memory
self._job_infos[pid] = {
'requested_cores': app.requested_cores,
'requested_memory': app.requested_memory,
Expand Down Expand Up @@ -1388,14 +1388,13 @@ def _check_app_requirements(self, app, update=True):
(self.name, self.max_cores))

if app.requested_memory:
total_requested_memory = app.requested_cores * app.requested_memory
if self.available_memory < total_requested_memory:
if self.available_memory < app.requested_memory:
raise gc3libs.exceptions.MaximumCapacityReached(
"Resource {0} does not have enough available memory:"
" {1} requested total, but only {2} available."
.format(
self.name,
total_requested_memory.to_str('%g%s', unit=Memory.MB),
app.requested_memory.to_str('%g%s', unit=Memory.MB),
self.available_memory.to_str('%g%s', unit=Memory.MB))
)

Expand Down Expand Up @@ -1629,7 +1628,7 @@ def _cleanup_terminating_task(self, app, pid, termstatus=None):
self.free_slots += app.requested_cores
self.user_run -= 1
if app.requested_memory is not None:
self.available_memory += (app.requested_memory * app.requested_cores)
self.available_memory += app.requested_memory
wrapper_filename = posixpath.join(
app.execution.lrms_execdir,
ShellcmdLrms.PRIVATE_DIR,
Expand Down
3 changes: 1 addition & 2 deletions gc3libs/backends/tests/test_shellcmd.py
Expand Up @@ -220,8 +220,7 @@ def test_resource_usage(self):
cores_after = self.backend.free_slots
mem_after = self.backend.available_memory
assert cores_before == cores_after + 2
assert mem_before == mem_after + (app.requested_cores * app.requested_memory)

assert mem_before == mem_after + app.requested_memory
self.run_until_terminating(app)
assert self.backend.free_slots == cores_before
assert self.backend.available_memory == mem_before
Expand Down

0 comments on commit 5d6b1f0

Please sign in to comment.