From c722b40a6e1910ba963c53f8bf11f1e340a295b9 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 26 Apr 2018 16:57:22 -0700 Subject: [PATCH 01/35] Committing recommendations from futurize. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index 26304c9..6954737 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -663,7 +663,7 @@ def _get_file_stats(base_value, ignore_list, ignore_directories): base_value or nested in base value that are not otherwise ignored by the input parameters. """ - if isinstance(base_value, types.StringTypes): + if isinstance(base_value, str): try: if base_value not in ignore_list and ( not os.path.isdir(base_value) or From 331f7c3a307cefaf3a0ec593ebf42e95eb75bdeb Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 26 Apr 2018 17:11:06 -0700 Subject: [PATCH 02/35] Adding relevant __future__ imports. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/taskgraph/__init__.py b/taskgraph/__init__.py index c930e78..b15afaf 100644 --- a/taskgraph/__init__.py +++ b/taskgraph/__init__.py @@ -1,4 +1,8 @@ """taskgraph module.""" +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import import pkg_resources from taskgraph.Task import * From 9e516e587ba492b7ee6fc807084b95ffcd9af959 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 26 Apr 2018 17:11:25 -0700 Subject: [PATCH 03/35] Changing Queue import. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index 6954737..ee63a63 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -11,7 +11,11 @@ import multiprocessing import threading import errno -import Queue +try: + import queue as queue +except ImportError: + # Python3 renamed queue as queue + import queue import inspect import abc @@ -83,7 +87,7 @@ def __init__(self, taskgraph_cache_dir_path, n_workers): # used to synchronize a pass through potential tasks to add to the # work queue - self.work_queue = Queue.Queue() + self.work_queue = queue.queue() self.worker_semaphore = threading.Semaphore(max(1, n_workers)) # launch threads to manage the workers for thread_id in xrange(max(1, n_workers)): @@ -94,7 +98,7 @@ def __init__(self, taskgraph_cache_dir_path, n_workers): worker_thread.start() # tasks that get passed add_task get put in this queue for scheduling - self.waiting_task_queue = Queue.Queue() + self.waiting_task_queue = queue.queue() waiting_task_scheduler = threading.Thread( target=self._process_waiting_tasks, name='_waiting_task_scheduler') @@ -103,7 +107,7 @@ def __init__(self, taskgraph_cache_dir_path, n_workers): # tasks in the work ready queue have dependencies satisfied but need # priority scheduling - self.work_ready_queue = Queue.Queue() + self.work_ready_queue = queue.queue() priority_task_scheduler = threading.Thread( target=self._schedule_priority_tasks, name='_priority_task_scheduler') @@ -237,7 +241,7 @@ def _schedule_priority_tasks(self): break # push task to priority queue heapq.heappush(priority_queue, task) - except Queue.Empty: + except queue.Empty: # this triggers when work_ready_queue is empty and # there's something in the work_ready_queue break @@ -259,7 +263,7 @@ def _schedule_priority_tasks(self): def _process_waiting_tasks(self): """Process any tasks that are waiting on dependencies. - This worker monitors the self.waiting_task_queue Queue and looks for + This worker monitors the self.waiting_task_queue queue and looks for (task, 'wait'), or (task, 'done') tuples. If mode is 'wait' the task is indexed locally with reference to From 3d58b8e82f6e17d4bde7b20f9e9ae20b6c667f74 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 26 Apr 2018 17:19:07 -0700 Subject: [PATCH 04/35] Accidentally over-replaced Queue with queue. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index ee63a63..9220c9a 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -87,7 +87,7 @@ def __init__(self, taskgraph_cache_dir_path, n_workers): # used to synchronize a pass through potential tasks to add to the # work queue - self.work_queue = queue.queue() + self.work_queue = queue.Queue() self.worker_semaphore = threading.Semaphore(max(1, n_workers)) # launch threads to manage the workers for thread_id in xrange(max(1, n_workers)): @@ -98,7 +98,7 @@ def __init__(self, taskgraph_cache_dir_path, n_workers): worker_thread.start() # tasks that get passed add_task get put in this queue for scheduling - self.waiting_task_queue = queue.queue() + self.waiting_task_queue = queue.Queue() waiting_task_scheduler = threading.Thread( target=self._process_waiting_tasks, name='_waiting_task_scheduler') @@ -107,7 +107,7 @@ def __init__(self, taskgraph_cache_dir_path, n_workers): # tasks in the work ready queue have dependencies satisfied but need # priority scheduling - self.work_ready_queue = queue.queue() + self.work_ready_queue = queue.Queue() priority_task_scheduler = threading.Thread( target=self._schedule_priority_tasks, name='_priority_task_scheduler') From 700ea0cf63d23b64cfc7b4f072a50b8135e8f256 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 26 Apr 2018 17:21:25 -0700 Subject: [PATCH 05/35] Using python3 keys/values instead of iterkeys/itervalues(). RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index 9220c9a..e7ce723 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -28,6 +28,11 @@ LOGGER = logging.getLogger('Task') +# range is an iterator in python3. +if 'xrange' not in __builtins__: + xrange = range + + class TaskGraph(object): """Encapsulates the worker and tasks states for parallel processing.""" @@ -338,7 +343,7 @@ def join(self, timeout=None): return True try: timedout = False - for task in self.task_id_map.itervalues(): + for task in self.task_id_map.values(): timedout = not task.join(timeout) # if the last task timed out then we want to timeout for all # of the task graph @@ -373,7 +378,7 @@ def _terminate(self): self.close() if self.n_workers > 0: self.worker_pool.terminate() - for task in self.task_id_map.itervalues(): + for task in self.task_id_map.values(): task._terminate() self.terminated = True @@ -677,7 +682,7 @@ def _get_file_stats(base_value, ignore_list, ignore_directories): except OSError: pass elif isinstance(base_value, collections.Mapping): - for key in sorted(base_value.iterkeys()): + for key in sorted(base_value.keys()): value = base_value[key] for stat in _get_file_stats( value, ignore_list, ignore_directories): From 03227ffdfbc8632663b96f6495fd60f9ccbf6f62 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 26 Apr 2018 20:42:11 -0700 Subject: [PATCH 06/35] Correcting import of "reload" for python 3. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- tests/test_task.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_task.py b/tests/test_task.py index 6540d4d..e9c27e6 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -13,6 +13,10 @@ logging.basicConfig(level=logging.DEBUG) +if 'reload' not in __builtins__: + import imp + reload = imp.reload + def _long_running_function(): """Wait for 5 seconds.""" From f4c6f4a38e66709d144176a80ed38b5b8e9669c1 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 26 Apr 2018 21:03:11 -0700 Subject: [PATCH 07/35] Correcting a string and adding __hash__ implementation for Task. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index e7ce723..e7eaa02 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -12,7 +12,7 @@ import threading import errno try: - import queue as queue + import Queue as queue except ImportError: # Python3 renamed queue as queue import queue @@ -467,10 +467,13 @@ def __init__( self.target_path_list+self.ignore_path_list, self.ignore_directories)) - task_string = '%s:%s:%s:%s:%s:%s' % ( - self.func.__name__, pickle.dumps(self.args), - json.dumps(self.kwargs, sort_keys=True), source_code, - self.target_path_list, str(file_stat_list)) + task_string = "{0}:{1}:{2}:{3}:{4}:{5}".format( + self.func.__name__, + pickle.dumps(self.args), + json.dumps(self.kwargs, sort_keys=True), + source_code, + self.target_path_list, + file_stat_list).encode('utf-8') self.task_hash = hashlib.sha1(task_string).hexdigest() @@ -489,6 +492,10 @@ def __eq__(self, other): return self.task_hash == other.task_hash return False + def __hash__(self): + """Return the base-16 integer hash of this hash string.""" + return int(self.task_hash, 16) + def __ne__(self, other): """Inverse of __eq__.""" return not self.__eq__(other) From 16adb9fbe5fabb9301fd1085138e06f2fdf71f28 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 26 Apr 2018 21:11:26 -0700 Subject: [PATCH 08/35] Explicitly encoding a couple of strings and adding abd.ABC superclass. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index e7eaa02..3dfd9ba 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -635,7 +635,7 @@ def _terminate(self, exception_object=None): self._task_complete_event.set() -class EncapsulatedTaskOp: +class EncapsulatedTaskOp(abc.ABC): """Used as a superclass for Task operations that need closures. This class will automatically hash the subclass's __call__ method source @@ -647,11 +647,11 @@ class EncapsulatedTaskOp: def __init__(self, *args, **kwargs): # try to get the source code of __call__ so task graph will recompute # if the function has changed - args_as_str = str([args, kwargs]) + args_as_str = str([args, kwargs]).encode('utf-8') try: # hash the args plus source code of __call__ id_hash = hashlib.sha1(args_as_str + inspect.getsource( - self.__class__.__call__)).hexdigest() + self.__class__.__call__).encode('utf-8')).hexdigest() except IOError: # this will fail if the code is compiled, that's okay just do # the args From 55d01df231bb0889cfa2d6690d4ff0701a732274 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 10:45:26 -0700 Subject: [PATCH 09/35] Handling ABC superclasses in python 2/3. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index 3dfd9ba..f2321af 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -19,6 +19,12 @@ import inspect import abc +try: + ABCSuperclass = abc.ABC +except AttributeError: + # Python 2 doesn't define abc.ABC. + ABCSuperclass = object + try: import psutil HAS_PSUTIL = True @@ -635,7 +641,7 @@ def _terminate(self, exception_object=None): self._task_complete_event.set() -class EncapsulatedTaskOp(abc.ABC): +class EncapsulatedTaskOp(ABCSuperclass): """Used as a superclass for Task operations that need closures. This class will automatically hash the subclass's __call__ method source From 5ec373ec8cc00bf1f576373efd9f12112774c112 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 10:54:36 -0700 Subject: [PATCH 10/35] Defining ABC superclass for python 2/3 compatibility. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index f2321af..250fd1b 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -19,11 +19,11 @@ import inspect import abc -try: - ABCSuperclass = abc.ABC -except AttributeError: - # Python 2 doesn't define abc.ABC. - ABCSuperclass = object +# Superclass for ABCs, compatible with python 2.7+ that replaces __metaclass__ +# usage that is no longer clearly documented in python 3 (if it's even present +# at all ... __metaclass__ has been removed from the python data model docs) +# Taken from https://stackoverflow.com/a/38668373/299084 +ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) try: import psutil @@ -641,15 +641,13 @@ def _terminate(self, exception_object=None): self._task_complete_event.set() -class EncapsulatedTaskOp(ABCSuperclass): +class EncapsulatedTaskOp(ABC): """Used as a superclass for Task operations that need closures. This class will automatically hash the subclass's __call__ method source as well as the arguments to its __init__ function to calculate the Task's unique hash. """ - __metaclass__ = abc.ABCMeta - def __init__(self, *args, **kwargs): # try to get the source code of __call__ so task graph will recompute # if the function has changed From aefe27cda3ae8ec57e88575269f274f677db5020 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 11:01:57 -0700 Subject: [PATCH 11/35] Adding a basestring-equivalent comparator for python 2/3. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index 250fd1b..bd2044f 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -38,6 +38,11 @@ if 'xrange' not in __builtins__: xrange = range +# In python2, basestring is the common superclass of str and unicode. In +# python3, we'll probably only be dealing with str objects. +if 'basestring' not in __builtins__: + basestring = str + class TaskGraph(object): """Encapsulates the worker and tasks states for parallel processing.""" @@ -683,7 +688,7 @@ def _get_file_stats(base_value, ignore_list, ignore_directories): base_value or nested in base value that are not otherwise ignored by the input parameters. """ - if isinstance(base_value, str): + if isinstance(base_value, basestring): try: if base_value not in ignore_list and ( not os.path.isdir(base_value) or From 7d4629f75b9df559fa3c51147321ed5c060f63e9 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 11:03:57 -0700 Subject: [PATCH 12/35] Commenting a python 2/3 compatibility condition. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- tests/test_task.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_task.py b/tests/test_task.py index e9c27e6..209699e 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -13,6 +13,8 @@ logging.basicConfig(level=logging.DEBUG) + +# Python 3 relocated the reload function to imp. if 'reload' not in __builtins__: import imp reload = imp.reload From 1ab0c390e1aa1bb8c11886173ddfb077d98570dd Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 11:08:05 -0700 Subject: [PATCH 13/35] Slight pylinting. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- tests/test_task.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_task.py b/tests/test_task.py index 209699e..dc4abe7 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -8,22 +8,18 @@ import pickle import logging -import taskgraph import mock -logging.basicConfig(level=logging.DEBUG) - +import taskgraph -# Python 3 relocated the reload function to imp. -if 'reload' not in __builtins__: - import imp - reload = imp.reload +logging.basicConfig(level=logging.DEBUG) def _long_running_function(): """Wait for 5 seconds.""" time.sleep(5) + def _create_list_on_disk(value, length, target_path): """Create a numpy array on disk filled with value of `size`.""" target_list = [value] * length @@ -44,6 +40,7 @@ def _div_by_zero(): """Divide by zero to raise an exception.""" return 1/0 + class TaskGraphTests(unittest.TestCase): """Tests for the taskgraph.""" @@ -71,6 +68,10 @@ def test_version_not_loaded(self): from pkg_resources import DistributionNotFound import taskgraph + # Python 3 relocated the reload function to imp. + if 'reload' not in __builtins__: + from imp import reload + with mock.patch('taskgraph.pkg_resources.get_distribution', side_effect=DistributionNotFound('taskgraph')): with self.assertRaises(RuntimeError): From 0758dae20e529e1a64478320839413155d6e9c86 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 11:14:49 -0700 Subject: [PATCH 14/35] Moving the redefinition of reload() to the outer scope. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- tests/test_task.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_task.py b/tests/test_task.py index dc4abe7..68e5500 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -15,6 +15,11 @@ logging.basicConfig(level=logging.DEBUG) +# Python 3 relocated the reload function to imp. +if 'reload' not in __builtins__: + from imp import reload + + def _long_running_function(): """Wait for 5 seconds.""" time.sleep(5) @@ -68,10 +73,6 @@ def test_version_not_loaded(self): from pkg_resources import DistributionNotFound import taskgraph - # Python 3 relocated the reload function to imp. - if 'reload' not in __builtins__: - from imp import reload - with mock.patch('taskgraph.pkg_resources.get_distribution', side_effect=DistributionNotFound('taskgraph')): with self.assertRaises(RuntimeError): From 3ce1343cbb19a48776819800d4a8ed0fe6ea390d Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 11:17:10 -0700 Subject: [PATCH 15/35] Adding tox spec for testing taskgraph on python 2/3. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- tox.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..6b2c2ae --- /dev/null +++ b/tox.ini @@ -0,0 +1,9 @@ +[tox] +envlist = py27,py35 + +[testenv] +deps = + nose + mock + coverage +commands=nosetests -vsP --with-coverage --cover-package=taskgraph From 4b71ec2cd50e987d92b9cdeebfddd4a6ff664b8d Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 11:38:08 -0700 Subject: [PATCH 16/35] Adding dependency matrix testing for psutil. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- tox.ini | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 6b2c2ae..4b9da3c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,12 @@ [tox] -envlist = py27,py35 +envlist = {py27,py35}-{base,psutil} [testenv] +commands = nosetests -vsP --with-coverage --cover-package=taskgraph + +# only install psutil to the environments where we're testing psutil. deps = nose mock coverage -commands=nosetests -vsP --with-coverage --cover-package=taskgraph + {py27,py35}-psutil: psutil From 2c294c5e3e540766955094bd2eb37712d26d4777 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 11:51:37 -0700 Subject: [PATCH 17/35] Commenting niceness. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index bd2044f..de5fc23 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -28,6 +28,14 @@ try: import psutil HAS_PSUTIL = True + if psutil.WINDOWS: + # Windows' scheduler doesn't use POSIX niceness. + PROCESS_LOW_PRIORITY = psutil.BELOW_NORMAL_PRIORITY_CLASS + else: + # On POSIX, use system niceness. + # -20 is low priority, 0 is normal priority, 19 is low priority. + # 10 here is an abritrary selection that's probably nice enough. + PROCESS_LOW_PRIORITY = 10 except ImportError: HAS_PSUTIL = False @@ -91,10 +99,10 @@ def __init__(self, taskgraph_cache_dir_path, n_workers): self.worker_pool = multiprocessing.Pool(n_workers) if HAS_PSUTIL: parent = psutil.Process() - parent.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS) + parent.nice(PROCESS_LOW_PRIORITY) for child in parent.children(): try: - child.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS) + child.nice(PROCESS_LOW_PRIORITY) except psutil.NoSuchProcess: LOGGER.warn( "NoSuchProcess exception encountered when trying " From bc25c09b4a3502a82a3f216297b9cb59ea882e04 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 11:56:28 -0700 Subject: [PATCH 18/35] Noting changes in HISTORY.rst. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- HISTORY.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 21f0b1c..085c03d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,11 @@ .. :changelog: -.. Unreleased Changes +Unreleased Changes +------------------ +* Taskgraph now supports python 3. +* Fixed an issue with ``taskgraph.TaskGraph`` where process priority could not + be set on linux (but it could be set on Windows). +* Adding a matrix of tests to automate testing via ``tox``. 0.4.0 (2018-04-18) ------------------ From 16d6693f499836abff593f549f09668e45c5609a Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 11:57:53 -0700 Subject: [PATCH 19/35] Correcting RST syntax for hierarchical section headings. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 135d9bf..cf29e40 100644 --- a/README.rst +++ b/README.rst @@ -2,22 +2,22 @@ TaskGraph: ================================================= About TaskGraph -=============== +--------------- TaskGraph is great. TaskGraph Dependencies -====================== +---------------------- Task Graph is written in pure Python, but if the ``psutils`` package is installed the distributed multiprocessing processes will be ``nice``\d. Example Use -=========== +----------- Install taskgraph with -`pip install taskgraph` +``pip install taskgraph`` Then From a9720777d49a181d742229d3cb898d0a863cb6ec Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 12:04:39 -0700 Subject: [PATCH 20/35] Noting how to run tests in taskgraph README. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- README.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.rst b/README.rst index cf29e40..3cda868 100644 --- a/README.rst +++ b/README.rst @@ -64,3 +64,23 @@ Then # expect that result is a list `list_len` long with `value_a+value_b` in it result = pickle.load(open(result_path, 'rb')) + +Running Tests +------------- + +Taskgraph includes a ``tox`` configuration for automating builds across +multiple python versions and whether ``psutil`` is installed. To +execute all tests, run:: + + $ tox + +Alternatively, if you're only trying to run tests on a single configuration +(say, python 3.5 without ``psutil``), you'd run:: + + $ tox -e py35-base + +Or if you'd like to run the tests for the combination of Python 2.7 with +``psutil``, you'd run:: + + $ tox -e py27-psutil + From 7e3a02d2f7abba6e7574384e0de1f5fccb3efe74 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 12:05:47 -0700 Subject: [PATCH 21/35] Noting specifically which versions of python taskgraph is tested with. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 085c03d..a809ee9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,7 +2,7 @@ Unreleased Changes ------------------ -* Taskgraph now supports python 3. +* Taskgraph now supports python 3 (tested with python 2.7, 3.5). * Fixed an issue with ``taskgraph.TaskGraph`` where process priority could not be set on linux (but it could be set on Windows). * Adding a matrix of tests to automate testing via ``tox``. From a5e52a89662b76ede573cafdcb4d666eea2d1ea4 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 12:13:35 -0700 Subject: [PATCH 22/35] Simplifying a string modification I had made. What's in this commit is a more straightforward change. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index de5fc23..b9f7823 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -486,15 +486,12 @@ def __init__( self.target_path_list+self.ignore_path_list, self.ignore_directories)) - task_string = "{0}:{1}:{2}:{3}:{4}:{5}".format( - self.func.__name__, - pickle.dumps(self.args), - json.dumps(self.kwargs, sort_keys=True), - source_code, - self.target_path_list, - file_stat_list).encode('utf-8') - - self.task_hash = hashlib.sha1(task_string).hexdigest() + task_string = '%s:%s:%s:%s:%s:%s' % ( + self.func.__name__, pickle.dumps(self.args), + json.dumps(self.kwargs, sort_keys=True), source_code, + self.target_path_list, str(file_stat_list)) + + self.task_hash = hashlib.sha1(task_string.encode('utf-8')).hexdigest() # get ready to make a directory and target based on hashname # take the first 3 characters of the hash and make a subdirectory From 02400b319c731e9416befdc004860310e0ab7262 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 12:17:08 -0700 Subject: [PATCH 23/35] Removing __future__ statements we don't use and using absolute_import. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/taskgraph/__init__.py b/taskgraph/__init__.py index b15afaf..99c5a20 100644 --- a/taskgraph/__init__.py +++ b/taskgraph/__init__.py @@ -1,10 +1,8 @@ """taskgraph module.""" from __future__ import unicode_literals -from __future__ import print_function -from __future__ import division from __future__ import absolute_import import pkg_resources -from taskgraph.Task import * +from .Task import * try: __version__ = pkg_resources.get_distribution(__name__).version From 81d03a9a5cfeb037b6c38e11ba6a14acb5134e6d Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 12:20:37 -0700 Subject: [PATCH 24/35] Specifying what gets exported to ``taskgraph`` namespace. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/taskgraph/__init__.py b/taskgraph/__init__.py index 99c5a20..9e6913d 100644 --- a/taskgraph/__init__.py +++ b/taskgraph/__init__.py @@ -2,7 +2,11 @@ from __future__ import unicode_literals from __future__ import absolute_import import pkg_resources -from .Task import * +from .Task import TaskGraph, Task, EncapsulatedTaskOp + + +__all__ = ['TaskGraph', 'Task', 'EncapsulatedTaskOp'] + try: __version__ = pkg_resources.get_distribution(__name__).version From 3abfd4f248e043ba653eb4fc2fec6267d3d7fbd7 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 12:24:28 -0700 Subject: [PATCH 25/35] Clarifying the HISTORY note about test automation. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- HISTORY.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index a809ee9..8f840a4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,7 +5,8 @@ Unreleased Changes * Taskgraph now supports python 3 (tested with python 2.7, 3.5). * Fixed an issue with ``taskgraph.TaskGraph`` where process priority could not be set on linux (but it could be set on Windows). -* Adding a matrix of tests to automate testing via ``tox``. +* Adding matrix-based test automation (python 2.7, python 3.5, with/without + ``psutil``) via ``tox. 0.4.0 (2018-04-18) ------------------ From f2c2a04e79bb4ddf10cd10df5c7c840567e1f419 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 12:37:39 -0700 Subject: [PATCH 26/35] Adding generator functions as a replacement for itervalues()/values(). RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index b9f7823..4cfa3b1 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -1,7 +1,6 @@ """Task graph framework.""" import heapq import pprint -import types import collections import hashlib import json @@ -42,14 +41,23 @@ LOGGER = logging.getLogger('Task') -# range is an iterator in python3. -if 'xrange' not in __builtins__: +try: + dict.itervalues +except AttributeError: + # Python 3 + # range is an iterator in python3. xrange = range - -# In python2, basestring is the common superclass of str and unicode. In -# python3, we'll probably only be dealing with str objects. -if 'basestring' not in __builtins__: + # In python2, basestring is the common superclass of str and unicode. In + # python3, we'll probably only be dealing with str objects. basestring = str + def itervalues(d): + """Python 2/3 compatibility iterator over d.values()""" + return iter(d.values()) +else: + # Python 2 + def itervalues(d): + """Python 2/3 compatibility alias for d.itervalues()""" + return d.itervalues() class TaskGraph(object): @@ -362,7 +370,7 @@ def join(self, timeout=None): return True try: timedout = False - for task in self.task_id_map.values(): + for task in itervalues(self.task_id_map): timedout = not task.join(timeout) # if the last task timed out then we want to timeout for all # of the task graph @@ -397,7 +405,7 @@ def _terminate(self): self.close() if self.n_workers > 0: self.worker_pool.terminate() - for task in self.task_id_map.values(): + for task in itervalues(self.task_id_map): task._terminate() self.terminated = True From f147d4b5a6c746e2dd9f057ab5ec8ef4c8a25ce7 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 27 Apr 2018 12:54:56 -0700 Subject: [PATCH 27/35] Improving comments and correcting RST syntax in HISTORY. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- HISTORY.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8f840a4..11cbbdc 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,11 +2,11 @@ Unreleased Changes ------------------ -* Taskgraph now supports python 3 (tested with python 2.7, 3.5). -* Fixed an issue with ``taskgraph.TaskGraph`` where process priority could not - be set on linux (but it could be set on Windows). +* Taskgraph now supports python versions 2 and 3 (tested with python 2.7, 3.5). +* Fixed an issue with ``taskgraph.TaskGraph`` that prevented a multiprocessed + graph from executing on POSIX systems when ``psutil`` was installed. * Adding matrix-based test automation (python 2.7, python 3.5, with/without - ``psutil``) via ``tox. + ``psutil``) via ``tox``. 0.4.0 (2018-04-18) ------------------ From 3813e071aadcb6cfbedab93de32fef9ff6f0e2d4 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Sat, 28 Apr 2018 14:16:39 -0700 Subject: [PATCH 28/35] Correcting inline comment about priority. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- taskgraph/Task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskgraph/Task.py b/taskgraph/Task.py index 4cfa3b1..bb6a4ea 100644 --- a/taskgraph/Task.py +++ b/taskgraph/Task.py @@ -32,7 +32,7 @@ PROCESS_LOW_PRIORITY = psutil.BELOW_NORMAL_PRIORITY_CLASS else: # On POSIX, use system niceness. - # -20 is low priority, 0 is normal priority, 19 is low priority. + # -20 is high priority, 0 is normal priority, 19 is low priority. # 10 here is an abritrary selection that's probably nice enough. PROCESS_LOW_PRIORITY = 10 except ImportError: From d58ff044ee5aaef55892fdf7b6584457af8b533a Mon Sep 17 00:00:00 2001 From: James Douglass Date: Sat, 28 Apr 2018 15:12:17 -0700 Subject: [PATCH 29/35] Updating repo path to use the natcap organization. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- HISTORY.rst | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 11cbbdc..7ae7147 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,7 @@ Unreleased Changes graph from executing on POSIX systems when ``psutil`` was installed. * Adding matrix-based test automation (python 2.7, python 3.5, with/without ``psutil``) via ``tox``. +* Updating repository path to ``https://bitbucket.org/natcap/taskgraph``. 0.4.0 (2018-04-18) ------------------ diff --git a/setup.py b/setup.py index 96184bf..96c98cc 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ long_description=README, maintainer='Rich Sharp', maintainer_email='richpsharp@gmail.com', - url='https://bitbucket.org/richsharp/taskgraph', + url='https://bitbucket.org/natcap/taskgraph', packages=['taskgraph'], license='BSD', keywords='parallel multiprocessing distributed computing', From 62a8136fa38e44598d620e164980e47f4254fc1b Mon Sep 17 00:00:00 2001 From: James Douglass Date: Sat, 28 Apr 2018 15:19:21 -0700 Subject: [PATCH 30/35] Using pytest instead of nosetests to avoid process hang. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 4b9da3c..ef315ed 100644 --- a/tox.ini +++ b/tox.ini @@ -2,11 +2,11 @@ envlist = {py27,py35}-{base,psutil} [testenv] -commands = nosetests -vsP --with-coverage --cover-package=taskgraph +commands = pytest --cov-report=term --cov # only install psutil to the environments where we're testing psutil. deps = - nose mock - coverage + pytest + pytest-cov {py27,py35}-psutil: psutil From 97136f179fd266bdc2eeafb2bd7ee0ef2c2c178b Mon Sep 17 00:00:00 2001 From: James Douglass Date: Sat, 28 Apr 2018 15:48:49 -0700 Subject: [PATCH 31/35] Adding test results as junit file. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ef315ed..13a213d 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ envlist = {py27,py35}-{base,psutil} [testenv] -commands = pytest --cov-report=term --cov +commands = pytest --cov-report=term --cov --junitxml=testresults.xml # only install psutil to the environments where we're testing psutil. deps = From 2e8fae43a8b0decf47cb3670c4c2f31e5b367b60 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Mon, 30 Apr 2018 10:38:30 -0700 Subject: [PATCH 32/35] Bumping python3 version to 3.6 and noting change in HISTORY. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- HISTORY.rst | 4 ++-- tox.ini | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7ae7147..6393752 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,10 +2,10 @@ Unreleased Changes ------------------ -* Taskgraph now supports python versions 2 and 3 (tested with python 2.7, 3.5). +* Taskgraph now supports python versions 2 and 3 (tested with python 2.7, 3.6). * Fixed an issue with ``taskgraph.TaskGraph`` that prevented a multiprocessed graph from executing on POSIX systems when ``psutil`` was installed. -* Adding matrix-based test automation (python 2.7, python 3.5, with/without +* Adding matrix-based test automation (python 2.7, python 3.6, with/without ``psutil``) via ``tox``. * Updating repository path to ``https://bitbucket.org/natcap/taskgraph``. diff --git a/tox.ini b/tox.ini index 13a213d..27810ac 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35}-{base,psutil} +envlist = {py27,py36}-{base,psutil} [testenv] commands = pytest --cov-report=term --cov --junitxml=testresults.xml @@ -9,4 +9,4 @@ deps = mock pytest pytest-cov - {py27,py35}-psutil: psutil + {py27,py36}-psutil: psutil From 646ba78861ba8ca618d2c13ff813185985265269 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 1 May 2018 11:58:02 -0700 Subject: [PATCH 33/35] Updating Trove classifiers. RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 96c98cc..c12d08c 100644 --- a/setup.py +++ b/setup.py @@ -21,11 +21,13 @@ }, classifiers=[ 'Intended Audience :: Developers', + 'Topic :: System :: Distributed Computing', 'Development Status :: 5 - Production/Stable', 'Natural Language :: English', 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft', 'Operating System :: POSIX', - 'Programming Language :: Python :: 2 :: Only', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: BSD License' ]) From 82a4d3ac09ab03b56003cdf1c9095ff43843a3e8 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 1 May 2018 11:58:29 -0700 Subject: [PATCH 34/35] Adding XML coverage report (needed for jenkins). RE:#BITBUCKET-12 Branch: feature/TASKGRAPH-12-python3 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 27810ac..a5ffe37 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ envlist = {py27,py36}-{base,psutil} [testenv] -commands = pytest --cov-report=term --cov --junitxml=testresults.xml +commands = pytest --cov-report=term --cov-report=xml --cov --junitxml=testresults.xml # only install psutil to the environments where we're testing psutil. deps = From c6234e8331aba3ad40de2f0d61d3ee5c960215c7 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 4 May 2018 15:32:50 -0700 Subject: [PATCH 35/35] Noting release in HISTORY. RE:#BITBUCKET-13 Branch: release/0.5.0 --- HISTORY.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6393752..65a154b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,8 @@ .. :changelog: -Unreleased Changes +.. Unreleased Changes + +0.5.0 (2018-05-04) ------------------ * Taskgraph now supports python versions 2 and 3 (tested with python 2.7, 3.6). * Fixed an issue with ``taskgraph.TaskGraph`` that prevented a multiprocessed