Disable ShellcmdLrms resources if capacity is exceeded.

This is implemented by having the resource raise a
`MaximumCapacityExceeded` exception, to which the scheduler reacts by
dropping the resource for the current cycle.

gc3libs/backends/shellcmd.py

@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-# Copyright (C) 2009-2017 University of Zurich. All rights reserved.
+# Copyright (C) 2009-2018 University of Zurich. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -1378,19 +1378,19 @@ def _check_app_requirements(self, app, update=True):
 
         if self.free_slots == 0:  # or free_slots == 0:
             if self.override:
-                raise gc3libs.exceptions.LRMSSubmitError(
+                raise gc3libs.exceptions.MaximumCapacityReached(
                     "Resource {0} already running maximum allowed number of jobs"
                     .format(self.name))
             else:
-                raise gc3libs.exceptions.LRMSSubmitError(
+                raise gc3libs.exceptions.MaximumCapacityReached(
                     "Resource %s already running maximum allowed number of jobs"
                     " (%s). Increase 'max_cores' to raise." %
                     (self.name, self.max_cores))
 
         if app.requested_memory:
             total_requested_memory = app.requested_cores * app.requested_memory
             if self.available_memory < total_requested_memory:
-                raise gc3libs.exceptions.LRMSSubmitError(
+                raise gc3libs.exceptions.MaximumCapacityReached(
                     "Resource {0} does not have enough available memory:"
                     " {1} requested total, but only {2} available."
                     .format(

gc3libs/core.py

@@ -1101,13 +1101,14 @@ def first_come_first_serve(tasks, resources, matchmaker=MatchMaker()):
             try:
                 # result = yield (task_idx, target.name)
                 yield (task_idx, target.name)
-            except gc3libs.exceptions.ResourceNotReady:
+            except (gc3libs.exceptions.ResourceNotReady,
+                    gc3libs.exceptions.MaximumCapacityReached) as exc:
                 # this is not a real error: the resource is adapting
                 # for the task and will actually accept it sometime in
                 # the future, so disable resource and try next one
                 gc3libs.log.debug(
-                    "Disabling resource `%s` for this scheduling cycle",
-                    target.name)
+                    "Disabling resource `%s` for this scheduling cycle: %s",
+                    target.name, exc)
                 resources.remove(target)
                 continue
             # pylint: disable=broad-except

gc3libs/exceptions.py

@@ -111,7 +111,7 @@ def __init__(self, msg, do_log=True):
         Exception.__init__(self, msg)
 
 
-# derived exceptions
+## derived exceptions
 
 class AuthError(Error):
 
@@ -322,6 +322,7 @@ def __init__(self, msg, do_log=False):
             DeprecationWarning, 2)
         super(LRMSSkipSubmissionToNextIteration, self).__init__(msg, do_log)
 
+
 class MaximumCapacityReached(LRMSSubmitError, RecoverableError):
 
     """