From ff7a1d995a1cef845f71a730f3a0fe568e040dbd Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Thu, 7 Mar 2019 17:09:02 +0000 Subject: [PATCH] py3: respond to feedback(2) --- bin/cylc-cat-state | 4 +- bin/cylc-check-software | 2 +- bin/cylc-check-versions | 2 +- bin/cylc-client | 5 +- doc/src/appendices/suiterc-config-ref.rst | 4 +- doc/src/conf.py | 14 ++-- doc/src/installation.rst | 1 - lib/cylc/log_diagnosis.py | 7 +- lib/cylc/network/client.py | 10 ++- lib/cylc/network/scan.py | 26 +++---- lib/cylc/network/server.py | 68 ++++--------------- lib/cylc/suite_srv_files_mgr.py | 2 +- lib/cylc/tests/test_suite_srv_files_mgr.py | 2 +- lib/cylc/tests/test_task_outputs.py | 17 +++++ lib/parsec/fileparse.py | 5 +- tests/cylc-scan/01-hosts/suite.rc | 10 --- tests/cylc-trigger/07-edit-run-abort/suite.rc | 2 +- tests/restart/00-pre-initial.t | 3 - 18 files changed, 67 insertions(+), 117 deletions(-) delete mode 100644 tests/cylc-scan/01-hosts/suite.rc diff --git a/bin/cylc-cat-state b/bin/cylc-cat-state index 6dbe18ba18f..1a1d296e07f 100755 --- a/bin/cylc-cat-state +++ b/bin/cylc-cat-state @@ -27,7 +27,7 @@ if remrun(): sys.exit(0) import os -import pickle +import json import re import sqlite3 import traceback @@ -119,7 +119,7 @@ def extract_lines(state): yield r"time : %(time_str)s (%(time_since_epoch)s)" % state yield r"initial cycle : %(initial_point)s" % state yield r"final cycle : %(final_point)s" % state - yield pickle.dumps(state["broadcast_states"]).splitlines() + yield json.dumps(state["broadcast_states"]).splitlines() yield "Begin task states" for item in state["task_pool"]: yield ( diff --git a/bin/cylc-check-software b/bin/cylc-check-software index 3937ef5090c..48f46bf2988 100755 --- a/bin/cylc-check-software +++ b/bin/cylc-check-software @@ -49,7 +49,7 @@ latter grouped as Python, TeX or 'other' (neither). 'opt_spec' item format: : [, , , <'OTHER' TUPLE>] with <'OTHER' TUPLE> = ([], , , ). """ -req_py_ver_range = ((3,),) +req_py_ver_range = ((3,6),) opt_spec = { 'EmPy': [None, 'TEMPLATING', 'PY'], 'sphinx': [(1, 5, 3), 'HTMLDOCS', 'PY'], diff --git a/bin/cylc-check-versions b/bin/cylc-check-versions index 3c1ce039d0a..9c408755c09 100755 --- a/bin/cylc-check-versions +++ b/bin/cylc-check-versions @@ -133,7 +133,7 @@ def main(): print("All", contacted, "accounts have cylc-" + CYLC_VERSION) else: print("WARNING: failed to invoke cylc-%s on %d accounts:" % ( - CYLC_VERSION, len(list(warn.keys())))) + CYLC_VERSION, len(warn))) m = max(len(ac) for ac in warn) for ac, warning in warn.items(): print(' ', ac.ljust(m), warning) diff --git a/bin/cylc-client b/bin/cylc-client index 7150024a188..b1a8acfc0ba 100755 --- a/bin/cylc-client +++ b/bin/cylc-client @@ -60,7 +60,4 @@ def main(): if __name__ == '__main__': - try: - main() - except Exception as exc: - sys.exit(exc) + main() diff --git a/doc/src/appendices/suiterc-config-ref.rst b/doc/src/appendices/suiterc-config-ref.rst index 122cd4f1a25..24e6ed30b4c 100644 --- a/doc/src/appendices/suiterc-config-ref.rst +++ b/doc/src/appendices/suiterc-config-ref.rst @@ -953,8 +953,8 @@ Allows tasks to spawn out to ``max active cycle points`` submitted before its successor can be spawned. *Important*: This should be used with care given the potential impact of -additional task proxies in terms of memory and cpu for the cylc daemon. -Also, use +additional task proxies in terms of memory and cpu for the cylc server +program. Also, use of the setting may highlight any issues with suite design relying on the default behaviour where downstream tasks would otherwise be waiting on ones upstream submitting and the suite would have stalled e.g. a housekeeping task diff --git a/doc/src/conf.py b/doc/src/conf.py index 853d7ecab17..332e4e6cd7b 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -114,12 +114,14 @@ 'pointsize': '11pt', } +# Title for the cylc documentation section +CYLC_DOC_TITLE = 'Cylc Documentation' + # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'cylc.tex', 'Cylc Documentation', - '2008-2019 NIWA & British Crown (Met Office) & Contributors', 'manual'), + ('index', 'cylc.tex', CYLC_DOC_TITLE, copyright, 'manual'), ] # Image file to place at the top of the title page. @@ -134,8 +136,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'cylc', 'cylc Documentation', - '2008-2019 NIWA & British Crown (Met Office) & Contributors', 1), + ('index', 'cylc', CYLC_DOC_TITLE, copyright, 1), ] # If true, show URL addresses after external links. @@ -148,9 +149,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'cylc', 'Cylc Documentation', - '2008-2019 NIWA & British Crown (Met Office) & Contributors', - 'cylc', 'The Cylc Suite Engine', 'Miscellaneous'), + ('index', 'cylc', CYLC_DOC_TITLE, copyright, 'cylc', project, + 'Miscellaneous'), ] # How to display URL addresses. diff --git a/doc/src/installation.rst b/doc/src/installation.rst index 3147be06a5d..914dc29f80b 100644 --- a/doc/src/installation.rst +++ b/doc/src/installation.rst @@ -20,7 +20,6 @@ Requirements: The following packages are necessary for running tests in Cylc: -- `mock `_ - `pytest `_ To generate the HTML User Guide, you will need: diff --git a/lib/cylc/log_diagnosis.py b/lib/cylc/log_diagnosis.py index 1016a608849..f0f423f4d61 100644 --- a/lib/cylc/log_diagnosis.py +++ b/lib/cylc/log_diagnosis.py @@ -24,12 +24,7 @@ class LogAnalyserError(Exception): - def __init__(self, msg): - Exception.__init__(self, msg) - self.msg = msg - - def __str__(self): - return self.msg + pass class LogSpec(object): diff --git a/lib/cylc/network/client.py b/lib/cylc/network/client.py index 55d6e693f81..16045849ae1 100644 --- a/lib/cylc/network/client.py +++ b/lib/cylc/network/client.py @@ -28,7 +28,7 @@ from cylc import LOG import cylc.flags -from cylc.hostuserutil import get_host, get_fqdn_by_host +from cylc.hostuserutil import get_fqdn_by_host from cylc.network import encrypt, decrypt, get_secret from cylc.suite_srv_files_mgr import ( SuiteSrvFilesManager, SuiteServiceFileError) @@ -286,11 +286,9 @@ def get_location(cls, suite, owner, host): # exc.args = (cls.NOT_RUNNING % suite,) # raise - if host and host.split('.')[0] == 'localhost': - host = get_host() - elif host and '.' not in host: # Not IP and no domain - host = get_fqdn_by_host(host) - else: + if not host: host = contact[SuiteSrvFilesManager.KEY_HOST] + host = get_fqdn_by_host(host) + port = int(contact[SuiteSrvFilesManager.KEY_PORT]) return host, port diff --git a/lib/cylc/network/scan.py b/lib/cylc/network/scan.py index a6f59c470fa..f8dcfb28682 100644 --- a/lib/cylc/network/scan.py +++ b/lib/cylc/network/scan.py @@ -42,7 +42,7 @@ def async_map(coroutine, iterator): """Map iterator iterator onto a coroutine. - * Yields results in order as an when they are ready + * Yields results in order as and when they are ready. * Slow workers can block. Args: @@ -52,7 +52,7 @@ def async_map(coroutine, iterator): Should yield tuples to be passed into the coroutine. Yields: - list - List of results + list - List of results. Example: >>> async def square(number): return number ** 2 @@ -70,20 +70,18 @@ def async_map(coroutine, iterator): awaiting.append(task) index = 0 - buff = [] + completed_tasks = {} while awaiting: completed, awaiting = loop.run_until_complete( asyncio.wait(awaiting, return_when=asyncio.FIRST_COMPLETED)) - buff.extend(completed) + completed_tasks.update({t.ind: t.result() for t in completed}) - old_len = -1 - while len(buff) != old_len: - old_len = len(buff) - for task in buff: - if task.ind == index: - index += 1 - buff.remove(task) - yield task.result() + changed = True + while changed and completed_tasks: + if index in completed_tasks: + yield completed_tasks.pop(index) + changed = True + index += 1 def async_unordered_map(coroutine, iterator): @@ -196,7 +194,7 @@ def re_compile_filters(patterns_owner=None, patterns_name=None): return (cres['owner'], cres['name']) -def get_scan_items_from_fs(owner_pattern=None, reg_pattern=None, updater=None): +def get_scan_items_from_fs(owner_pattern=None, reg_pattern=None): """Scrape list of suites from the filesystem. Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for @@ -231,8 +229,6 @@ def get_scan_items_from_fs(owner_pattern=None, reg_pattern=None, updater=None): item[1] is not None))) for run_d, owner in run_dirs: for dirpath, dnames, _ in os.walk(run_d, followlinks=True): - if updater and updater.quit: - return # Always descend for top directory, but # don't descend further if it has a .service/ or log/ dir if dirpath != run_d and ( diff --git a/lib/cylc/network/server.py b/lib/cylc/network/server.py index 0ee28936403..8615c349cf5 100644 --- a/lib/cylc/network/server.py +++ b/lib/cylc/network/server.py @@ -18,7 +18,6 @@ """Server for suite runtime API.""" import getpass -import re from queue import Queue from time import sleep from threading import Thread @@ -32,7 +31,6 @@ KEY_META, KEY_NAME, KEY_OWNER, KEY_STATES, KEY_TASKS_BY_STATE, KEY_UPDATE_TIME, KEY_VERSION) from cylc.version import CYLC_VERSION -from cylc.wallclock import RE_DATE_TIME_FORMAT_EXTENDED class ZMQServer(object): @@ -106,7 +104,7 @@ def start(self, ports): self.port = port break else: - raise Exception('No room at the inn, all ports occupied.') + raise IOError('No room at the inn, all ports occupied.') # start accepting requests self.register_endpoints() @@ -152,7 +150,6 @@ def _listener(self): # process try: message = self.decode(msg, self.secret()) - LOG.debug('zmq:recv %s', message) except Exception as exc: # purposefully catch generic exception # failed to decode message, possibly resulting from failed # authentication @@ -160,6 +157,7 @@ def _listener(self): {'error': {'message': str(exc)}}, self.secret()) else: # success case - serve the request + LOG.debug('zmq:recv %s', message) res = self._receiver(message) response = self.encode(res, self.secret()) LOG.debug('zmq:send %s', res) @@ -191,7 +189,7 @@ def _receiver(self, message): response = method(**args) except Exception as exc: # includes incorrect arguments (TypeError) - LOG.error(exc) # note the error server side + LOG.exception(exc) # note the error server side import traceback return {'error': { 'message': str(exc), 'traceback': traceback.format_exc()}} @@ -214,6 +212,14 @@ def authorise(req_priv_level): Args: req_priv_level (cylc.network.Priv): A privilege level for the method. + Wrapped function args: + user + The authenticated user (determined server side) + host + The client host (if provided by client) - non trustworthy + prog + The client program name (if provided by client) - non trustworthy + """ def wrapper(fcn): def _authorise(self, *args, user='?', meta=None, **kwargs): @@ -222,7 +228,7 @@ def _authorise(self, *args, user='?', meta=None, **kwargs): usr_priv_level = self.get_priv_level(user) if usr_priv_level < req_priv_level: - LOG.info( + LOG.warn( "[client-connect] DENIED (privilege '%s' < '%s') %s@%s:%s", usr_priv_level, req_priv_level, user, host, prog) raise Exception('Authorisation failure') @@ -238,15 +244,6 @@ class SuiteRuntimeServer(ZMQServer): This class contains the cylc endpoints. - Note the following argument names are protected: - - user - The authenticated user (determined server side) - host - The client host (if provided by client) - non trustworthy - prog - The client program name (if provided by client) - non trustworthy - """ API = 4 # cylc API version @@ -306,8 +303,6 @@ def dry_run_tasks(self, items, check_syntax=True): items[0] is an identifier for matching a task proxy. """ - if not isinstance(items, list): - items = [items] self.schd.command_queue.put(('dry_run_tasks', (items,), {'check_syntax': check_syntax})) return (True, 'Command queued') @@ -338,8 +333,6 @@ def get_graph_raw(self, start_point_string, stop_point_string, ungroup_all=False): """Return raw suite graph.""" # Ensure that a "None" str is converted to the None value. - if stop_point_string is not None: - stop_point_string = str(stop_point_string) return self.schd.info_get_graph_raw( start_point_string, stop_point_string, group_nodes=group_nodes, @@ -364,8 +357,6 @@ def get_suite_state_summary(self): @ZMQServer.expose def get_task_info(self, names): """Return info of a task.""" - if not isinstance(names, list): - names = [names] return self.schd.info_get_task_info(names) @authorise(Priv.READ) @@ -378,10 +369,8 @@ def get_task_jobfile_path(self, task_id): @ZMQServer.expose def get_task_requisites(self, items=None, list_prereqs=False): """Return prerequisites of a task.""" - if not isinstance(items, list): - items = [items] return self.schd.info_get_task_requisites( - items, list_prereqs=(list_prereqs in [True, 'True'])) + items, list_prereqs=list_prereqs) @authorise(Priv.CONTROL) @ZMQServer.expose @@ -405,8 +394,6 @@ def hold_tasks(self, items): items is a list of identifiers for matching task proxies. """ - if not isinstance(items, list): - items = [items] self.schd.command_queue.put(("hold_tasks", (items,), {})) return (True, 'Command queued') @@ -441,15 +428,10 @@ def insert_tasks(self, items, stop_point_string=None, no_check=False): items is a list of identifiers of (families of) task instances. """ - if not isinstance(items, list): - items = [items] - if stop_point_string == "None": - stop_point_string = None self.schd.command_queue.put(( "insert_tasks", (items,), - {"stop_point_string": stop_point_string, - "no_check": no_check in ['True', True]})) + {"stop_point_string": stop_point_string, "no_check": no_check})) return (True, 'Command queued') @authorise(Priv.CONTROL) @@ -459,8 +441,6 @@ def kill_tasks(self, items): items is a list of identifiers for matching task proxies. """ - if not isinstance(items, list): - items = [items] self.schd.command_queue.put(("kill_tasks", (items,), {})) return (True, 'Command queued') @@ -490,11 +470,8 @@ def poll_tasks(self, items=None, poll_succ=False): items is a list of identifiers for matching task proxies. """ - if items is not None and not isinstance(items, list): - items = [items] self.schd.command_queue.put( - ("poll_tasks", (items,), - {"poll_succ": poll_succ in ['True', True]})) + ("poll_tasks", (items,), {"poll_succ": poll_succ})) return (True, 'Command queued') @authorise(Priv.CONTROL) @@ -554,8 +531,6 @@ def release_tasks(self, items): items is a list of identifiers for matching task proxies. """ - if not isinstance(items, list): - items = [items] self.schd.command_queue.put(("release_tasks", (items,), {})) return (True, 'Command queued') @@ -566,8 +541,6 @@ def remove_tasks(self, items, spawn=False): items is a list of identifiers for matching task proxies. """ - if not isinstance(items, list): - items = [items] self.schd.command_queue.put( ("remove_tasks", (items,), {"spawn": spawn})) return (True, 'Command queued') @@ -579,10 +552,6 @@ def reset_task_states(self, items, state=None, outputs=None): items is a list of identifiers for matching task proxies. """ - if not isinstance(items, list): - items = [items] - if outputs and not isinstance(outputs, list): - outputs = [outputs] self.schd.command_queue.put(( "reset_task_states", (items,), {"state": state, "outputs": outputs})) @@ -634,8 +603,6 @@ def spawn_tasks(self, items): items is a list of identifiers for matching task proxies. """ - if not isinstance(items, list): - items = [items] self.schd.command_queue.put(("spawn_tasks", (items,), {})) return (True, 'Command queued') @@ -653,8 +620,6 @@ def take_checkpoints(self, items): items[0] is the name of the checkpoint. """ - if not isinstance(items, list): - items = [items] self.schd.command_queue.put(("take_checkpoints", (items,), {})) return (True, 'Command queued') @@ -665,9 +630,6 @@ def trigger_tasks(self, items, back_out=False): items is a list of identifiers for matching task proxies. """ - if not isinstance(items, list): - items = [items] - items = [str(item) for item in items] self.schd.command_queue.put( ("trigger_tasks", (items,), {"back_out": back_out})) return (True, 'Command queued') diff --git a/lib/cylc/suite_srv_files_mgr.py b/lib/cylc/suite_srv_files_mgr.py index 90de94cd627..0be8586a120 100644 --- a/lib/cylc/suite_srv_files_mgr.py +++ b/lib/cylc/suite_srv_files_mgr.py @@ -496,7 +496,7 @@ def register(self, reg=None, source=None, redirect=False): source_str = source os.symlink(source_str, target) - print(('REGISTERED %s -> %s' % (reg, source))) + print('REGISTERED %s -> %s' % (reg, source)) return reg def create_auth_files(self, reg): diff --git a/lib/cylc/tests/test_suite_srv_files_mgr.py b/lib/cylc/tests/test_suite_srv_files_mgr.py index aeaec51b62e..279756ade4f 100644 --- a/lib/cylc/tests/test_suite_srv_files_mgr.py +++ b/lib/cylc/tests/test_suite_srv_files_mgr.py @@ -178,7 +178,7 @@ def setUp(self): def test_register(self, mocked_os): """Test the SuiteSrvFilesManager register function.""" def mkdirs_standin(_, exist_ok=False): - True + return True # we do not need to mock these functions mocked_os.path.basename.side_effect = os.path.basename diff --git a/lib/cylc/tests/test_task_outputs.py b/lib/cylc/tests/test_task_outputs.py index c6e68408e01..6345a254c9d 100644 --- a/lib/cylc/tests/test_task_outputs.py +++ b/lib/cylc/tests/test_task_outputs.py @@ -1,3 +1,20 @@ +#!/bin/bash +# +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . import random import unittest diff --git a/lib/parsec/fileparse.py b/lib/parsec/fileparse.py index 4cb5787ca15..6def8a9a62e 100644 --- a/lib/parsec/fileparse.py +++ b/lib/parsec/fileparse.py @@ -289,7 +289,7 @@ def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False): LOG.debug('Processing with Jinja2') try: flines = jinja2process(flines, fdir, template_vars) - except (Exception, TemplateError, UndefinedError) as exc: + except Exception as exc: # Extract diagnostic info from the end of the Jinja2 traceback. exc_lines = traceback.format_exc().splitlines() suffix = [] @@ -302,8 +302,7 @@ def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False): lineno = None if hasattr(exc, 'lineno'): lineno = exc.lineno - elif (isinstance(exc, Exception) or - isinstance(exc, UndefinedError)): + else: match = re.search(r'File "