From 52c5001bdba6d8717bc1cd97070ba1e95d25c90b Mon Sep 17 00:00:00 2001 From: Eduardo Schettino Date: Fri, 4 Sep 2020 14:34:59 +0800 Subject: [PATCH 1/7] bump revision to 0.33.1, create manifest before generating sdist. --- CHANGES | 5 +++++ dodo.py | 2 +- doit/version.py | 2 +- setup.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 51df891b..bd1d395e 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,11 @@ Changes ======= +0.33.1 (*2020-09-04*) +===================== + + - fix sdist distribution to include all files + 0.33.0 (*2020-09-01*) ===================== diff --git a/dodo.py b/dodo.py index 2a978560..3b5b2d14 100755 --- a/dodo.py +++ b/dodo.py @@ -132,7 +132,7 @@ def task_package(): yield pkg.revision_git() yield pkg.manifest_git() yield pkg.sdist() - yield pkg.sdist_upload() + # yield pkg.sdist_upload() diff --git a/doit/version.py b/doit/version.py index 6bb42d52..9c9ff3ec 100644 --- a/doit/version.py +++ b/doit/version.py @@ -1,2 +1,2 @@ """doit version, defined out of __init__.py to avoid circular reference""" -VERSION = (0, 33, 0) +VERSION = (0, 33, 1) diff --git a/setup.py b/setup.py index 8a3ceb97..8fbd3a92 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup(name = 'doit', description = 'doit - Automation Tool', - version = '0.33.0', + version = '0.33.1', license = 'MIT', author = 'Eduardo Naufel Schettino', author_email = 'schettino72@gmail.com', From cb91b9e3bf51ca6977cd17c2329edf309d353326 Mon Sep 17 00:00:00 2001 From: Tobias Kohlmaier <32921797+kohtoa15@users.noreply.github.com> Date: Tue, 23 Feb 2021 12:38:53 +0100 Subject: [PATCH 2/7] Use uniform date format for TaskResult to_dict() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standard datetime to string conversion leaves open the possibility that in case of µs = 0, no decimals are displayed, which could break conversion back to datetime based on standard format. --- doit/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doit/reporter.py b/doit/reporter.py index 8831e9a9..6ac59f69 100644 --- a/doit/reporter.py +++ b/doit/reporter.py @@ -177,7 +177,7 @@ def to_dict(self): """convert result data to dictionary""" if self._started_on is not None: started = datetime.datetime.utcfromtimestamp(self._started_on) - self.started = str(started) + self.started = str(started.strftime('%Y-%m-%d %H:%M:%S.%f')) self.elapsed = self._finished_on - self._started_on return {'name': self.task.name, 'result': self.result, From 3ec2e808b51131cc084aa5095590da347851ee50 Mon Sep 17 00:00:00 2001 From: Eduardo Schettino Date: Fri, 5 Mar 2021 00:41:13 +0800 Subject: [PATCH 3/7] docs; add link to sponsoring through xscode. --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 5cdb808b..660a4018 100644 --- a/README.rst +++ b/README.rst @@ -16,6 +16,9 @@ README .. image:: https://coveralls.io/repos/pydoit/doit/badge.png?branch=master :target: https://coveralls.io/r/pydoit/doit?branch=master +.. image:: https://xscode.com/assets/promo-banner.svg + :target: https://xscode.com/schettino72/doit + doit - automation tool ====================== @@ -227,3 +230,6 @@ contributing ============== On github create pull requests using a named feature branch. + +Financial contribution to support maitanance welcome. +https://xscode.com/schettino72/doit From e0dfd09b9c4bcf3bf81ee0245633f865f536ab86 Mon Sep 17 00:00:00 2001 From: Eduardo Schettino Date: Fri, 5 Mar 2021 00:51:57 +0800 Subject: [PATCH 4/7] start 0.34 --- CHANGES | 5 +++++ doc/conf.py | 4 ++-- doit/version.py | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index bd1d395e..4d419017 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,11 @@ Changes ======= +0.34.0 (*unreleased*) +===================== + + + 0.33.1 (*2020-09-04*) ===================== diff --git a/doc/conf.py b/doc/conf.py index 728892c7..900cf8a8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -24,9 +24,9 @@ author = 'Eduardo Schettino (schettino72)' # The short X.Y version -version = '0.33' +version = '0.34' # The full version, including alpha/beta/rc tags -release = '0.33' +release = '0.34' # -- General configuration --------------------------------------------------- diff --git a/doit/version.py b/doit/version.py index 9c9ff3ec..73ee0f54 100644 --- a/doit/version.py +++ b/doit/version.py @@ -1,2 +1,2 @@ """doit version, defined out of __init__.py to avoid circular reference""" -VERSION = (0, 33, 1) +VERSION = (0, 34, 'dev0') diff --git a/setup.py b/setup.py index 8fbd3a92..9f4d52b6 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup(name = 'doit', description = 'doit - Automation Tool', - version = '0.33.1', + version = '0.34.dev0', license = 'MIT', author = 'Eduardo Naufel Schettino', author_email = 'schettino72@gmail.com', From 2155d403db420da16c94c6c39f01de5bcdb7b30f Mon Sep 17 00:00:00 2001 From: Eduardo Schettino Date: Fri, 5 Mar 2021 01:43:53 +0800 Subject: [PATCH 5/7] Fix #381: Allow Delayed tasks to be executed multiple times in same process. --- CHANGES | 1 + doit/loader.py | 3 ++- tests/test_loader.py | 8 +++++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 4d419017..ca80178f 100644 --- a/CHANGES +++ b/CHANGES @@ -6,6 +6,7 @@ Changes 0.34.0 (*unreleased*) ===================== +- Fix #381: Allow Delayed tasks to be executed multiple times in same process. 0.33.1 (*2020-09-04*) diff --git a/doit/loader.py b/doit/loader.py index 5b7f2566..76ec8afd 100644 --- a/doit/loader.py +++ b/doit/loader.py @@ -2,6 +2,7 @@ import os import sys +import copy import inspect import importlib from collections import OrderedDict @@ -137,7 +138,7 @@ def load_tasks(namespace, command_names=(), allow_delayed=False): def _process_gen(): task_list.extend(generate_tasks(name, ref(), ref.__doc__)) def _add_delayed(tname): - task_list.append(Task(tname, None, loader=delayed, + task_list.append(Task(tname, None, loader=copy.copy(delayed), doc=delayed.creator.__doc__)) for name, ref, _ in funcs: diff --git a/tests/test_loader.py b/tests/test_loader.py index 63d77b64..b8c6cb72 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -128,9 +128,15 @@ def task_zzz3(): # pragma: no cover f_task = tasks['foo'] assert f_task.loader.task_dep == 'yyy2' assert f_task.loader.creator == task_zzz3 - assert tasks['bar'].loader is tasks['foo'].loader + assert tasks['bar'].loader.task_dep == tasks['foo'].loader.task_dep assert tasks['foo'].doc == 'not loaded task doc' + # make sure doit can be executed more then once in single process GH#381 + list2 = load_tasks(dodo, allow_delayed=True) + tasks2 = {t.name:t for t in list2} + assert tasks['bar'].loader is not tasks2['bar'].loader + + def testNameInBlacklist(self): dodo_module = {'task_cmd_name': lambda:None} pytest.raises(InvalidDodoFile, load_tasks, dodo_module, ['cmd_name']) From 50e1f7555559c4595a4e9248e01ab4a136c48fec Mon Sep 17 00:00:00 2001 From: Eduardo Schettino Date: Fri, 5 Mar 2021 02:03:37 +0800 Subject: [PATCH 6/7] Changelog for #382 --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index ca80178f..829fd3f6 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,7 @@ Changes ===================== - Fix #381: Allow Delayed tasks to be executed multiple times in same process. +- Fix #382: TaskResult make sure `started` include microsecond as decimal number. 0.33.1 (*2020-09-04*) From 0aa4cc55fdd90b4f4e3480fb3ca089be572053c5 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sun, 14 Mar 2021 08:16:47 -0400 Subject: [PATCH 7/7] Add github actions (#379) * add github actions * install pip, setuptools with --user --- .github/workflows/ci.yml | 52 +++++++++++++++++++++++++++++++++++ .github/workflows/website.yml | 25 +++++++++++++++++ dev_requirements.txt | 2 +- doc/stories.rst | 8 +++--- tests/test_action.py | 2 +- tests/test_dependency.py | 1 + 6 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/website.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..4978b629 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,52 @@ +name: CI + +on: + push: + branches: [master, test] + pull_request: + branches: [master, test] + +jobs: + test: + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu, windows, macos] + python-version: ['3.5', '3.6', '3.7', '3.8', '3.9', 'pypy3'] + include: + # https://github.com/pydoit/doit/issues/372 + - os: macos + pytest-args: -k 'not(cmd_auto or TestFileWatcher)' + - os: macos + python-version: pypy3 + pytest-args: -k 'not(cmd_auto or TestFileWatcher or remove_all or ForgetAll)' + exclude: + - os: windows + python-version: pypy3 + steps: + - if: ${{ matrix.os == 'ubuntu' }} + run: sudo apt-get install strace + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - run: pip install --user -U pip wheel setuptools + - id: pip-cache + run: echo "::set-output name=dir::$(pip cache dir)" + - uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('setup.py', 'dev_requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.python-version }}- + ${{ runner.os }}-pip- + - run: pip install . -r dev_requirements.txt python-coveralls + - run: pip freeze + - run: pip check + - run: doit pyflakes + - run: py.test -vv ${{ matrix.pytest-args }} + - if: ${{ matrix.os == 'ubuntu' && matrix.python-version == '3.8' }} + run: | + doit coverage + coveralls || echo "TODO: remove when run on a repo that is set up" diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml new file mode 100644 index 00000000..0bd3a47b --- /dev/null +++ b/.github/workflows/website.yml @@ -0,0 +1,25 @@ +name: Website + +on: + push: + branches: [master, test] + pull_request: + branches: [master, test] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - run: sudo apt-get install hunspell hunspell-en-us + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: "3.8" + - run: pip install -U pip wheel setuptools + - run: pip install . -r doc_requirements.txt -r dev_requirements.txt + - run: pip freeze + - run: doit -v2 website + - uses: actions/upload-artifact@v2 + with: + name: Website + path: ./doc/_build diff --git a/dev_requirements.txt b/dev_requirements.txt index f16ba864..ba479a00 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -3,5 +3,5 @@ pyflakes pytest>=5.4.1 -coverage>=4.0 +coverage>=4.0,<5 doit-py>=0.4.0 diff --git a/doc/stories.rst b/doc/stories.rst index 95eb347d..a9857c47 100644 --- a/doc/stories.rst +++ b/doc/stories.rst @@ -89,7 +89,7 @@ our papers, mentioning it in our funding requests, etc). Thanks for developing `doit`, it's just wonderful for computational biology (and for many other tasks, of course, but this is our research field:)! -`Cheminformatics data processing @ Atomwise `_ +`Cheminformatics data processing @ Atomwise `_ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ by `Jon Sorenson `_ (2018-12-14) @@ -97,7 +97,7 @@ by `Jon Sorenson `_ (2018-12-14) I came to `doit` from the loose requirement that I wanted a task-dependency/DAG type of workflow solution for the various pipelines that our team is constructing. I come from computational biology originally, and the similarity of data processing pipelines -to build systems has long been appreciated. More than a decade ago +to build systems has long been appreciated. More than a decade ago many of us were writing bioinformatics pipelines in `make` because it gave us so many features "for free." @@ -112,7 +112,7 @@ from an intermediate stage. I knew that build-system based frameworks would do exactly what I wanted and not hand me too much cruft, and on that note I found `doit`. It's worked perfectly -for my needs: +for my needs: - The order of tasks is derived from dependencies @@ -184,7 +184,7 @@ Each sound should be loaded from PCM, converted into MP3 or OGG and linked with `MetalK8s @ Scality `_ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ by `Sylvain Laperche `_ (2019-05-06) diff --git a/tests/test_action.py b/tests/test_action.py index 1b9f18b5..fdbfc191 100644 --- a/tests/test_action.py +++ b/tests/test_action.py @@ -25,7 +25,7 @@ @pytest.fixture def tmpfile(request): - temp = tempfile.TemporaryFile('w+') + temp = tempfile.TemporaryFile('w+', encoding="utf-8") request.addfinalizer(temp.close) return temp diff --git a/tests/test_dependency.py b/tests/test_dependency.py index f9fe22de..17927ce7 100644 --- a/tests/test_dependency.py +++ b/tests/test_dependency.py @@ -17,6 +17,7 @@ PROGRAM = "%s %s/sample_process.py" % (executable, TEST_PATH) + def test_unicode_md5(): data = "我" # no exception is raised