Merge remote-tracking branch 'upstream/master' into add-toml

.github/workflows/ci.yml

@@ -0,0 +1,52 @@
+name: CI
+
+on:
+  push:
+    branches: [master, test]
+  pull_request:
+    branches: [master, test]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu, windows, macos]
+        python-version: ['3.5', '3.6', '3.7', '3.8', '3.9', 'pypy3']
+        include:
+          # https://github.com/pydoit/doit/issues/372
+          - os: macos
+            pytest-args: -k 'not(cmd_auto or TestFileWatcher)'
+          - os: macos
+            python-version: pypy3
+            pytest-args: -k 'not(cmd_auto or TestFileWatcher or remove_all or ForgetAll)'
+        exclude:
+          - os: windows
+            python-version: pypy3
+    steps:
+      - if: ${{ matrix.os == 'ubuntu' }}
+        run: sudo apt-get install strace
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install --user -U pip wheel setuptools
+      - id: pip-cache
+        run: echo "::set-output name=dir::$(pip cache dir)"
+      - uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('setup.py', 'dev_requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{ matrix.python-version }}-
+            ${{ runner.os }}-pip-
+      - run: pip install . -r dev_requirements.txt python-coveralls
+      - run: pip freeze
+      - run: pip check
+      - run: doit pyflakes
+      - run: py.test -vv ${{ matrix.pytest-args }}
+      - if: ${{ matrix.os == 'ubuntu' && matrix.python-version == '3.8' }}
+        run: |
+          doit coverage
+          coveralls || echo "TODO: remove when run on a repo that is set up"

.github/workflows/website.yml

@@ -0,0 +1,25 @@
+name: Website
+
+on:
+  push:
+    branches: [master, test]
+  pull_request:
+    branches: [master, test]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - run: sudo apt-get install hunspell hunspell-en-us
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - run: pip install -U pip wheel setuptools
+      - run: pip install . -r doc_requirements.txt -r dev_requirements.txt
+      - run: pip freeze
+      - run: doit -v2 website
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Website
+          path: ./doc/_build

CHANGES

@@ -3,6 +3,18 @@
 Changes
 =======
 
+0.34.0 (*unreleased*)
+=====================
+
+- Fix #381: Allow Delayed tasks to be executed multiple times in same process.
+- Fix #382: TaskResult make sure `started` include microsecond as decimal number.
+
+
+0.33.1 (*2020-09-04*)
+=====================
+
+ - fix sdist distribution to include all files
+
 0.33.0 (*2020-09-01*)
 =====================
 

README.rst

@@ -16,6 +16,9 @@ README
 .. image:: https://coveralls.io/repos/pydoit/doit/badge.png?branch=master
   :target: https://coveralls.io/r/pydoit/doit?branch=master
 
+.. image:: https://xscode.com/assets/promo-banner.svg
+   :target: https://xscode.com/schettino72/doit
+
 
 doit - automation tool
 ======================
@@ -227,3 +230,6 @@ contributing
 ==============
 
 On github create pull requests using a named feature branch.
+
+Financial contribution to support maitanance welcome.
+https://xscode.com/schettino72/doit

dev_requirements.txt

@@ -3,6 +3,6 @@
 
 pyflakes
 pytest>=5.4.1
-coverage>=4.0
+coverage>=4.0,<5
 doit-py>=0.4.0
 toml>=0.10.1

doc/conf.py

@@ -24,9 +24,9 @@
 author = 'Eduardo Schettino (schettino72)'
 
 # The short X.Y version
-version = '0.33'
+version = '0.34'
 # The full version, including alpha/beta/rc tags
-release = '0.33'
+release = '0.34'
 
 
 # -- General configuration ---------------------------------------------------

doc/stories.rst

@@ -89,15 +89,15 @@ our papers, mentioning it in our funding requests, etc).
 Thanks for developing `doit`, it's just wonderful for computational biology (and
 for many other tasks, of course, but this is our research field:)!
 
-`Cheminformatics data processing @ Atomwise <https://www.atomwise.com>`_ 
+`Cheminformatics data processing @ Atomwise <https://www.atomwise.com>`_
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 by `Jon Sorenson <https://github.com/drkeoni>`_ (2018-12-14)
 
 I came to `doit` from the loose requirement that I wanted a task-dependency/DAG type
 of workflow solution for the various pipelines that our team is constructing.
 I come from computational biology originally, and the similarity of data processing pipelines
-to build systems has long been appreciated.  More than a decade ago 
+to build systems has long been appreciated.  More than a decade ago
 many of us were writing bioinformatics
 pipelines in `make` because it gave us so many features "for free."
 
@@ -112,7 +112,7 @@ from an intermediate stage.
 
 I knew that build-system based frameworks would do exactly what I wanted and not
 hand me too much cruft, and on that note I found `doit`.  It's worked perfectly
-for my needs: 
+for my needs:
 
 - The order of tasks is derived from dependencies
 
@@ -184,7 +184,7 @@ Each sound should be loaded from PCM, converted into MP3 or OGG and linked with
 
 
 `MetalK8s @ Scality <https://www.scality.com/>`_
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 by `Sylvain Laperche <https://github.com/slaperche-scality>`_ (2019-05-06)
 

dodo.py

@@ -132,7 +132,7 @@ def task_package():
     yield pkg.revision_git()
     yield pkg.manifest_git()
     yield pkg.sdist()
-    yield pkg.sdist_upload()
+    # yield pkg.sdist_upload()
 
 
 

doit/loader.py

@@ -2,6 +2,7 @@
 
 import os
 import sys
+import copy
 import inspect
 import importlib
 from collections import OrderedDict
@@ -137,7 +138,7 @@ def load_tasks(namespace, command_names=(), allow_delayed=False):
     def _process_gen():
         task_list.extend(generate_tasks(name, ref(), ref.__doc__))
     def _add_delayed(tname):
-        task_list.append(Task(tname, None, loader=delayed,
+        task_list.append(Task(tname, None, loader=copy.copy(delayed),
                               doc=delayed.creator.__doc__))
 
     for name, ref, _ in funcs:

doit/reporter.py

@@ -177,7 +177,7 @@ def to_dict(self):
         """convert result data to dictionary"""
         if self._started_on is not None:
             started = datetime.datetime.utcfromtimestamp(self._started_on)
-            self.started = str(started)
+            self.started = str(started.strftime('%Y-%m-%d %H:%M:%S.%f'))
             self.elapsed = self._finished_on - self._started_on
         return {'name': self.task.name,
                 'result': self.result,

doit/version.py

@@ -1,2 +1,2 @@
 """doit version, defined out of __init__.py to avoid circular reference"""
-VERSION = (0, 33, 0)
+VERSION = (0, 34, 'dev0')

setup.py

@@ -26,7 +26,7 @@
 
 setup(name = 'doit',
       description = 'doit - Automation Tool',
-      version = '0.33.0',
+      version = '0.34.dev0',
       license = 'MIT',
       author = 'Eduardo Naufel Schettino',
       author_email = 'schettino72@gmail.com',

tests/test_action.py

@@ -25,7 +25,7 @@
 
 @pytest.fixture
 def tmpfile(request):
-    temp = tempfile.TemporaryFile('w+')
+    temp = tempfile.TemporaryFile('w+', encoding="utf-8")
     request.addfinalizer(temp.close)
     return temp
 

tests/test_dependency.py

@@ -17,6 +17,7 @@
 PROGRAM = "%s %s/sample_process.py" % (executable, TEST_PATH)
 
 
+
 def test_unicode_md5():
     data = "我"
     # no exception is raised

tests/test_loader.py

@@ -128,9 +128,15 @@ def task_zzz3(): # pragma: no cover
         f_task = tasks['foo']
         assert f_task.loader.task_dep == 'yyy2'
         assert f_task.loader.creator == task_zzz3
-        assert tasks['bar'].loader is tasks['foo'].loader
+        assert tasks['bar'].loader.task_dep == tasks['foo'].loader.task_dep
         assert tasks['foo'].doc == 'not loaded task doc'
 
+        # make sure doit can be executed more then once in single process GH#381
+        list2 = load_tasks(dodo, allow_delayed=True)
+        tasks2 = {t.name:t for t in list2}
+        assert tasks['bar'].loader is not tasks2['bar'].loader
+
+
     def testNameInBlacklist(self):
         dodo_module = {'task_cmd_name': lambda:None}
         pytest.raises(InvalidDodoFile, load_tasks, dodo_module, ['cmd_name'])