diff --git a/Dockerfile b/Dockerfile index e6a4b2d..191a0b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,4 +8,4 @@ RUN pip install -r requirements.txt COPY . . -CMD ["python", "autoscale.py"] +CMD ["python", "scale.py"] diff --git a/autoscaler/autoscaler.py b/autoscaler/autoscaler.py index 5ef92b1..d10290d 100644 --- a/autoscaler/autoscaler.py +++ b/autoscaler/autoscaler.py @@ -28,7 +28,6 @@ from __future__ import division from __future__ import print_function -import re import timeit import logging @@ -41,32 +40,30 @@ class Autoscaler(object): Args: redis_client: Redis Client Connection object. scaling_config: string, joined lists of autoscaling configurations - secondary_scaling_config: string, defines initial pod counts deployment_delim: string, character delimiting deployment configs. param_delim: string, character delimiting deployment config parameters. """ - def __init__(self, redis_client, scaling_config, secondary_scaling_config, - deployment_delim=';', param_delim='|'): + def __init__(self, + redis_client, + scaling_config, + deployment_delim=';', + param_delim='|'): if deployment_delim == param_delim: raise ValueError('`deployment_delim` and `param_delim` must be ' 'different. Got "{}" and "{}".'.format( deployment_delim, param_delim)) + self.redis_client = redis_client self.logger = logging.getLogger(str(self.__class__.__name__)) self.completed_statuses = {'done', 'failed'} - self.autoscaling_params = self._get_primary_autoscaling_params( + self.autoscaling_params = self._get_autoscaling_params( scaling_config=scaling_config.rstrip(), deployment_delim=deployment_delim, param_delim=param_delim) - self.secondary_autoscaling_params = self._get_secondary_autoscaling_params( - scaling_config=secondary_scaling_config.rstrip(), - deployment_delim=deployment_delim, - param_delim=param_delim) - self.redis_keys = { 'predict': 0, 'train': 0, @@ -77,17 +74,16 @@ def __init__(self, redis_client, scaling_config, secondary_scaling_config, self.reference_pods = {} - def _get_primary_autoscaling_params(self, scaling_config, - deployment_delim=';', - param_delim='|'): + def _get_autoscaling_params(self, + scaling_config, + deployment_delim=';', + param_delim='|'): raw_params = [x.split(param_delim) for x in scaling_config.split(deployment_delim)] params = {} for entry in raw_params: try: - print(len(entry)) - print(entry) namespace_resource_type_name = ( str(entry[3]).strip(), str(entry[4]).strip(), @@ -110,33 +106,20 @@ def _get_primary_autoscaling_params(self, scaling_config, return params - def _get_secondary_autoscaling_params(self, scaling_config, - deployment_delim=';', - param_delim='|'): - return [x.split(param_delim) - for x in scaling_config.split(deployment_delim)] - - def tally_keys(self): + def tally_queues(self): + """Update counts of all redis queues""" start = timeit.default_timer() - # reset the key tallies to 0 - for k in self.redis_keys: - self.redis_keys[k] = 0 - self.logger.debug('Tallying keys in redis matching: `%s`', - ', '.join(self.redis_keys.keys())) + for q in self.redis_keys: + self.logger.debug('Tallying items in queue `%s`.', q) - for key in self.redis_client.scan_iter(count=1000): - if any(re.match(k, key) for k in self.redis_keys): - if self.redis_client.type(key) != 'hash': - continue + num_items = self.redis_client.llen(q) - status = self.redis_client.hget(key, 'status') + processing_q = 'processing-{}:*'.format(q) + scan = self.redis_client.scan_iter(match=processing_q, count=1000) + num_in_progress = len([x for x in scan]) - # add up each type of key that is "in-progress" or "new" - if status is not None and status not in self.completed_statuses: - for k in self.redis_keys: - if re.match(k, key): - self.redis_keys[k] += 1 + self.redis_keys[q] = num_items + num_in_progress self.logger.debug('Finished tallying redis keys in %s seconds.', timeit.default_timer() - start) @@ -220,10 +203,21 @@ def patch_namespaced_job(self, name, namespace, body): def get_current_pods(self, namespace, resource_type, name, only_running=False): - """Find the number of current pods deployed for the given resource""" + """Find the number of current pods deployed for the given resource. + + Args: + name: str, name of the resource. + namespace: str, namespace of the resource. + resource_type: str, type of resource to count. + only_running: bool, Only count pods with status `Running`. + + Returns: + int, number of pods for the given resource. + """ if resource_type not in self.managed_resource_types: - raise ValueError('The resource_type of {} is unsuitable. Use either' - '`deployment` or `job`'.format(resource_type)) + raise ValueError( + '`resource_type` must be one of {}. Got {}.'.format( + self.managed_resource_types, resource_type)) current_pods = 0 if resource_type == 'deployment': @@ -235,8 +229,8 @@ def get_current_pods(self, namespace, resource_type, name, else: current_pods = d.spec.replicas - self.logger.debug("Deployment {} has {} pods".format( - name, current_pods)) + self.logger.debug('Deployment %s has %s pods', + name, current_pods) break elif resource_type == 'job': @@ -253,6 +247,7 @@ def get_current_pods(self, namespace, resource_type, name, def clip_pod_count(self, desired_pods, min_pods, max_pods, current_pods): # set `desired_pods` to inside the max/min boundaries. + _original = desired_pods if desired_pods > max_pods: desired_pods = max_pods elif desired_pods < min_pods: @@ -263,6 +258,9 @@ def clip_pod_count(self, desired_pods, min_pods, max_pods, current_pods): if 0 < desired_pods < current_pods: desired_pods = current_pods + if desired_pods != _original: + self.logger.debug('Clipped pods from %s to %s', + _original, desired_pods) return desired_pods def get_desired_pods(self, key, keys_per_pod, @@ -271,12 +269,6 @@ def get_desired_pods(self, key, keys_per_pod, return self.clip_pod_count(desired_pods, min_pods, max_pods, current_pods) - def get_secondary_desired_pods(self, reference_pods, pods_per_reference_pod, - min_pods, max_pods, current_pods): - desired_pods = current_pods + pods_per_reference_pod * reference_pods - return self.clip_pod_count(desired_pods, min_pods, - max_pods, current_pods) - def scale_resource(self, desired_pods, current_pods, resource_type, namespace, name): if desired_pods == current_pods: @@ -296,52 +288,50 @@ def scale_resource(self, desired_pods, current_pods, namespace, current_pods, desired_pods) return True - def scale_primary_resources(self): + def scale_resources(self): """Scale each resource defined in `autoscaling_params`""" - self.logger.debug("Scaling primary resources.") - for ((namespace, resource_type, name), - entries) in self.autoscaling_params.items(): - # iterate through all entries with this - # (namespace, resource_type, name) entry. We sum up the current - # and desired pods over all entries + self.logger.debug('Scaling primary resources.') + for fingerprint, entries in self.autoscaling_params.items(): + # iterate through all entries with this fingerprint + namespace, resource_type, name = fingerprint + self.logger.debug('Scaling %s `%s.%s` with %s config entries.', + resource_type, namespace, name, len(entries)) + + # Sum up the current and desired pods over all entries current_pods = self.get_current_pods(namespace, resource_type, name) desired_pods = 0 - self.logger.debug("Scaling {}".format((namespace, resource_type, name))) - min_pods_for_all_entries = [] max_pods_for_all_entries = [] + if entries == []: # this is the most conservative bound + self.logger.warning('%s `%s.%s` has no autoscaling entry.', + str(resource_type).capitalize(), + namespace, name) + continue + for entry in entries: - min_pods = entry["min_pods"] - max_pods = entry["max_pods"] - keys_per_pod = entry["keys_per_pod"] - prefix = entry["prefix"] + min_pods = entry['min_pods'] + max_pods = entry['max_pods'] + keys_per_pod = entry['keys_per_pod'] + prefix = entry['prefix'] min_pods_for_all_entries.append(min_pods) max_pods_for_all_entries.append(max_pods) - self.logger.debug("Inspecting entry {}.".format(entry)) - desired_pods += self.get_desired_pods(prefix, keys_per_pod, min_pods, max_pods, current_pods) - self.logger.debug("desired_pods now = {}".format(desired_pods)) - - # this is the most conservative bound - if entries == []: - return - min_pods = max(min_pods_for_all_entries) max_pods = min(max_pods_for_all_entries) desired_pods = self.clip_pod_count(desired_pods, min_pods, max_pods, current_pods) - self.logger.debug("desired_pods clamped to {}".format(desired_pods)) - self.logger.debug('Scaling %s `%s`', resource_type, name) + if desired_pods > 0 and resource_type != 'job': + desired_pods = current_pods if current_pods > 0 else 1 self.logger.debug('%s `%s` in namespace `%s` has a current state ' 'of %s pods and a desired state of %s pods.', @@ -351,65 +341,6 @@ def scale_primary_resources(self): self.scale_resource(desired_pods, current_pods, resource_type, namespace, name) - def scale_secondary_resources(self): - for entry in self.secondary_autoscaling_params: - # redis-consumer-deployment|deployment|deepcell| - # tf-serving-deployment|deployment|deepcell|7|0|200 - try: - resource_name = str(entry[0]).strip() - resource_type = str(entry[1]).strip() - resource_namespace = str(entry[2]).strip() - reference_resource_name = str(entry[3]).strip() - reference_resource_type = str(entry[4]).strip() - reference_resource_namespace = str(entry[5]).strip() - pods_per_other_pod = int(entry[6]) - min_pods = int(entry[7]) - max_pods = int(entry[8]) - except (IndexError, ValueError): - self.logger.error('Secondary autoscaling entry %s is ' - 'malformed.', entry) - continue - - self.logger.debug('Scaling secondary %s `%s`', - resource_type, resource_name) - - # keep track of how many reference pods we're working with - if resource_name not in self.reference_pods: - self.reference_pods[resource_name] = 0 - - current_reference_pods = self.get_current_pods( - reference_resource_namespace, - reference_resource_type, - reference_resource_name, - only_running=True) - - new_reference_pods = current_reference_pods - \ - self.reference_pods[resource_name] - - # update reference pod count - self.reference_pods[resource_name] = current_reference_pods - - self.logger.debug('Secondary scaling: %s `%s` references %s `%s` ' - 'which has %s pods (%s new pods).', - str(resource_type).capitalize(), resource_name, - reference_resource_type, reference_resource_name, - current_reference_pods, new_reference_pods) - - # only scale secondary deployments if there are new reference pods - if new_reference_pods > 0: - - # compute desired pods for this deployment - current_pods = self.get_current_pods( - resource_namespace, resource_type, resource_name) - - desired_pods = self.get_secondary_desired_pods( - new_reference_pods, pods_per_other_pod, - min_pods, max_pods, current_pods) - - self.scale_resource(desired_pods, current_pods, resource_type, - resource_namespace, resource_name) - def scale(self): - self.tally_keys() - self.scale_primary_resources() - self.scale_secondary_resources() + self.tally_queues() + self.scale_resources() diff --git a/autoscaler/autoscaler_test.py b/autoscaler/autoscaler_test.py index be4db1d..268b65e 100644 --- a/autoscaler/autoscaler_test.py +++ b/autoscaler/autoscaler_test.py @@ -28,6 +28,9 @@ from __future__ import division from __future__ import print_function +import random +import string + import pytest import kubernetes @@ -41,59 +44,12 @@ def __init__(self, **kwds): class DummyRedis(object): - def __init__(self, prefix='predict', status='new'): - self.prefix = '/'.join(x for x in prefix.split('/') if x) - self.status = status - self.fail_count = 0 - - def keys(self): - return [ - '{}_{}_{}'.format(self.prefix, self.status, 'x.tiff'), - '{}_{}_{}'.format(self.prefix, 'failed', 'x.zip'), - '{}_{}_{}'.format('train', self.status, 'x.TIFF'), - '{}_{}_{}'.format(self.prefix, self.status, 'x.ZIP'), - '{}_{}_{}'.format(self.prefix, 'done', 'x.tiff'), - '{}_{}_{}'.format('train', self.status, 'x.zip'), - ] + def llen(self, queue_name): + return len(queue_name) def scan_iter(self, match=None, count=None): - keys = [ - '{}_{}_{}'.format(self.prefix, self.status, 'x.tiff'), - '{}_{}_{}'.format(self.prefix, 'failed', 'x.zip'), - '{}_{}_{}'.format('train', self.status, 'x.TIFF'), - '{}_{}_{}'.format(self.prefix, self.status, 'x.ZIP'), - '{}_{}_{}'.format(self.prefix, 'done', 'x.tiff'), - '{}_{}_{}'.format('train', self.status, 'x.zip'), - 'malformedKey' - ] - if match: - return (k for k in keys if k.startswith(match[:-1])) - return (k for k in keys) - - def expected_keys(self, suffix=None): - for k in self.keys(): - v = k.split('_') - if v[0] == self.prefix: - if v[1] == self.status: - if suffix: - if v[-1].lower().endswith(suffix): - yield k - else: - yield k - - def hget(self, rhash, field): - if field == 'status': - return rhash.split('_')[1] - elif field == 'file_name': - return rhash.split('_')[-1] - elif field == 'input_file_name': - return rhash.split('_')[-1] - elif field == 'output_file_name': - return rhash.split('_')[-1] - return False - - def type(self, rhash): - return 'hash' + match = match if match else '' + yield match + random.choice(string.ascii_letters) class DummyKubernetes(object): @@ -138,25 +94,18 @@ def patch_namespaced_job(self, *_, **__): class TestAutoscaler(object): - def test__get_primary_autoscaling_params(self): + def test__get_autoscaling_params(self): primary_params = """ 1|2|3|namespace|resource_1|predict|name_1; 4|5|6|namespace|resource_1|track|name_1; 7|8|9|namespace|resource_2|train|name_1 """.strip() - secondary_params = """ - redis-consumer-deployment|deployment|deepcell|tf-serving-deployment|deployment|deepcell|7|0|0; - tracking-consumer-deployment|deployment|deepcell|tf-serving-deployment|deployment|deepcell|7|0|0; - data-processing-deployment|deployment|deepcell|tf-serving-deployment|deployment|deepcell|1|0|0 - """.strip() redis_client = DummyRedis() scaler = autoscaler.Autoscaler(redis_client, - primary_params, - secondary_params) - print("printing primary the nsecondary") + primary_params) + print("printing primary autoscaling parameters") print(scaler.autoscaling_params) - print(scaler.secondary_autoscaling_params) print("done printing") assert scaler.autoscaling_params == { ('namespace', 'resource_1', 'name_1'): [ @@ -201,31 +150,6 @@ def test_get_desired_pods(self): desired_pods = scaler.get_desired_pods('predict', 10, 0, 5, 3) assert desired_pods == 3 - def test_get_secondary_desired_pods(self): - # reference_pods, pods_per_reference_pod, - # min_pods, max_pods, current_pods - redis_client = DummyRedis() - scaler = autoscaler.Autoscaler(redis_client, 'None', 'None') - - # desired_pods is > max_pods - desired_pods = scaler.get_secondary_desired_pods(1, 10, 0, 4, 4) - assert desired_pods == 4 - # desired_pods is < min_pods - desired_pods = scaler.get_secondary_desired_pods(1, 1, 2, 4, 0) - assert desired_pods == 2 - # desired_pods is in range - desired_pods = scaler.get_secondary_desired_pods(1, 3, 0, 4, 0) - assert desired_pods == 3 - # desired_pods is in range with current pods and max limit - desired_pods = scaler.get_secondary_desired_pods(1, 3, 0, 4, 2) - assert desired_pods == 4 - # desired_pods is in range with current pods and no max limit - desired_pods = scaler.get_secondary_desired_pods(1, 3, 0, 10, 2) - assert desired_pods == 5 - # desired_pods is less than current_pods but current > max - desired_pods = scaler.get_secondary_desired_pods(1, 0, 0, 4, 10) - assert desired_pods == 10 - def test_list_namespaced_deployment(self): redis_client = DummyRedis() scaler = autoscaler.Autoscaler(redis_client, 'None', 'None') @@ -289,13 +213,13 @@ def test_get_current_pods(self): deployed_pods = scaler.get_current_pods('ns', 'job', 'pod2') assert deployed_pods == 2 - def test_tally_keys(self): + def test_tally_queues(self): redis_client = DummyRedis() scaler = autoscaler.Autoscaler(redis_client, 'None', 'None') - scaler.tally_keys() - assert scaler.redis_keys == {'predict': 2, 'track': 0, 'train': 2} + scaler.tally_queues() + assert scaler.redis_keys == {'predict': 8, 'track': 6, 'train': 6} - def test_scale_primary_resources(self): + def test_scale_resources(self): redis_client = DummyRedis() deploy_params = ['0', '1', '3', 'ns', 'deployment', 'predict', 'name'] job_params = ['1', '2', '1', 'ns', 'job', 'train', 'name'] @@ -308,30 +232,30 @@ def test_scale_primary_resources(self): # non-integer values will warn, but not raise (or autoscale) bad_params = ['f0', 'f1', 'f3', 'ns', 'job', 'train', 'name'] p = deployment_delim.join([param_delim.join(bad_params)]) - scaler = autoscaler.Autoscaler(redis_client, p, 'None', - deployment_delim, param_delim) + scaler = autoscaler.Autoscaler(redis_client, p, deployment_delim, + param_delim) scaler.get_apps_v1_client = DummyKubernetes scaler.get_batch_v1_client = DummyKubernetes - scaler.scale_primary_resources() + scaler.scale_resources() # not enough params will warn, but not raise (or autoscale) bad_params = ['0', '1', '3', 'ns', 'job', 'train'] p = deployment_delim.join([param_delim.join(bad_params)]) - scaler = autoscaler.Autoscaler(redis_client, p, 'None', - deployment_delim, param_delim) + scaler = autoscaler.Autoscaler(redis_client, p, deployment_delim, + param_delim) scaler.get_apps_v1_client = DummyKubernetes scaler.get_batch_v1_client = DummyKubernetes - scaler.scale_primary_resources() + scaler.scale_resources() # test bad resource_type with pytest.raises(ValueError): bad_params = ['0', '1', '3', 'ns', 'bad_type', 'train', 'name'] p = deployment_delim.join([param_delim.join(bad_params)]) - scaler = autoscaler.Autoscaler(redis_client, p, 'None', - deployment_delim, param_delim) + scaler = autoscaler.Autoscaler(redis_client, p, deployment_delim, + param_delim) scaler.get_apps_v1_client = DummyKubernetes scaler.get_batch_v1_client = DummyKubernetes - scaler.scale_primary_resources() + scaler.scale_resources() # test good delimiters and scaling params, bad resource_type deploy_params = ['0', '5', '1', 'ns', 'deployment', 'predict', 'name'] @@ -339,135 +263,23 @@ def test_scale_primary_resources(self): params = [deploy_params, job_params] p = deployment_delim.join([param_delim.join(p) for p in params]) - scaler = autoscaler.Autoscaler(redis_client, p, 'None', - deployment_delim, param_delim) - - scaler.get_apps_v1_client = DummyKubernetes - scaler.get_batch_v1_client = DummyKubernetes - - scaler.scale_primary_resources() - # test desired_pods == current_pods - scaler.get_desired_pods = lambda *x: 4 - scaler.scale_primary_resources() - - # same delimiter throws an error; - with pytest.raises(ValueError): - param_delim = '|' - deployment_delim = '|' - p = deployment_delim.join([param_delim.join(p) for p in params]) - autoscaler.Autoscaler(None, p, 'None', - deployment_delim, param_delim) - - def test_scale_secondary_resources(self): - # redis-deployment|deployment|namespace|tf-serving-deployment| - # deployment|namespace2|podRatio|minPods|maxPods - - redis_client = DummyRedis() - deploy_params = ['name', 'deployment', 'ns', - 'primary', 'deployment', 'ns', - '7', '1', '10'] - job_params = ['name', 'job', 'ns', - 'primary', 'job', 'ns', - '7', '1', '10'] - - params = [deploy_params, job_params] - - param_delim = '|' - deployment_delim = ';' - - # non-integer values will warn, but not raise (or autoscale) - bad_params = ['name', 'bad_type', 'ns', - 'primary', 'bad_type', 'ns', - 'f7', 'f1', 'f10'] - p = deployment_delim.join([param_delim.join(bad_params)]) - scaler = autoscaler.Autoscaler(redis_client, 'None', p, - deployment_delim, param_delim) - scaler.get_apps_v1_client = DummyKubernetes - scaler.get_batch_v1_client = DummyKubernetes - scaler.scale_secondary_resources() - - # not enough params will warn, but not raise (or autoscale) - bad_params = ['name', 'job', 'ns', 'primary', 'job', '7', '1', '10'] - p = deployment_delim.join([param_delim.join(bad_params)]) - scaler = autoscaler.Autoscaler(redis_client, 'None', p, - deployment_delim, param_delim) - scaler.get_apps_v1_client = DummyKubernetes - scaler.get_batch_v1_client = DummyKubernetes - scaler.scale_secondary_resources() - - # test bad resource_type - with pytest.raises(ValueError): - bad_params = ['name', 'bad_type', 'ns', - 'primary', 'bad_type', 'ns', - '7', '1', '10'] - p = deployment_delim.join([param_delim.join(bad_params)]) - scaler = autoscaler.Autoscaler(redis_client, 'None', p, - deployment_delim, param_delim) - scaler.get_apps_v1_client = DummyKubernetes - scaler.get_batch_v1_client = DummyKubernetes - scaler.scale_secondary_resources() - - # test good delimiters and scaling params, bad resource_type - params = [deploy_params, job_params] - p = deployment_delim.join([param_delim.join(p) for p in params]) - - scaler = autoscaler.Autoscaler(redis_client, 'None', p, - deployment_delim, param_delim) + scaler = autoscaler.Autoscaler(redis_client, p, deployment_delim, + param_delim) scaler.get_apps_v1_client = DummyKubernetes scaler.get_batch_v1_client = DummyKubernetes - scaler.scale_secondary_resources() + scaler.scale_resources() # test desired_pods == current_pods scaler.get_desired_pods = lambda *x: 4 - scaler.scale_secondary_resources() - - # test nothing happens if no new pods - params = [deploy_params, job_params] - p = deployment_delim.join([param_delim.join(p) for p in params]) - - scaler = autoscaler.Autoscaler(redis_client, 'None', p, - deployment_delim, param_delim) - - scaler.get_apps_v1_client = DummyKubernetes - scaler.get_batch_v1_client = DummyKubernetes - - def curr_pods(*args, **kwargs): - if args[2] == 'primary': - return 0 - return 2 - - scaler.get_current_pods = curr_pods - - global counter - counter = 0 - - def dummy_scale(*args, **kwargs): - global counter - counter += 1 - - scaler.scale_resource = dummy_scale - scaler.scale_secondary_resources() - # `scale_deployments` should not be called. - assert counter == 0 - - # test scaling does occur with current reference pods - def curr_pods_2(*args, **kwargs): - if args[2] == 'primary': - return 1 - return 2 - scaler.get_current_pods = curr_pods_2 - scaler.scale_secondary_resources() - # `scale_deployments` should be called. - assert counter == 1 + scaler.scale_resources() # same delimiter throws an error; with pytest.raises(ValueError): param_delim = '|' deployment_delim = '|' p = deployment_delim.join([param_delim.join(p) for p in params]) - autoscaler.Autoscaler(None, 'None', p, - deployment_delim, param_delim) + autoscaler.Autoscaler(None, p, deployment_delim, param_delim) def test_scale(self): redis_client = DummyRedis() @@ -484,9 +296,8 @@ def dummy_scale_resources(): global counter counter += 1 - scaler.tally_keys = dummy_tally - scaler.scale_primary_resources = dummy_scale_resources - scaler.scale_secondary_resources = dummy_scale_resources + scaler.tally_queues = dummy_tally + scaler.scale_resources = dummy_scale_resources scaler.scale() - assert counter == 3 + assert counter == 2 diff --git a/autoscale.py b/scale.py similarity index 90% rename from autoscale.py rename to scale.py index 0e99c6c..5ae58e1 100644 --- a/autoscale.py +++ b/scale.py @@ -23,10 +23,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -""" -Sort the key in the Redis base according to name and scale different -Kubernetes deployments as appropriate. -""" +"""Turn on and off k8s resources based on items in the Redis queue.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -79,15 +76,13 @@ def initialize_logger(debug_mode=True): SCALER = autoscaler.Autoscaler( redis_client=REDIS_CLIENT, - scaling_config=os.getenv('AUTOSCALING'), - secondary_scaling_config=os.getenv('SECONDARY_AUTOSCALING')) + scaling_config=os.getenv('AUTOSCALING')) INTERVAL = int(os.getenv('INTERVAL', '5')) while True: try: SCALER.scale() - _logger.debug('Sleeping for %s seconds.', INTERVAL) time.sleep(INTERVAL) except Exception as err: # pylint: disable=broad-except _logger.critical('Fatal Error: %s: %s', type(err).__name__, err)