Merging with upstream

.github/workflows/autoupdate.yaml

@@ -0,0 +1,24 @@
+# Auto-update PRs when other PRs are merged.
+# See https://github.com/marketplace/actions/auto-update
+
+name: autoupdate
+on:
+  push:
+    branches:
+      - master
+jobs:
+  autoupdate:
+    name: autoupdate
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: docker://chinthakagodawita/autoupdate-action:v1
+        env:
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+          # Work on PRs against branches that are protected (so they need to be
+          # up to date)
+          PR_FILTER: "protected"
+          # Don't work on draft PRs
+          PR_READY_STATE: "ready_for_review"
+          # Don't fail on the triggering commit if it makes a merge conflict
+          # with a PR and the PR can't update
+          MERGE_CONFLICT_ACTION: "ignore"

.readthedocs.yaml

@@ -8,6 +8,8 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.9"
+  apt_packages:
+    - graphviz
 
 sphinx:
    configuration: docs/conf.py

LICENSE

@@ -202,3 +202,28 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+All code in this repository excluding src/toil/statsAndLogging.py::install_log_color is under the Apache License as outlined directly above.
+Some code in src/toil/statsAndLogging.py::install_log_color is under the MiniWDL MIT License as outlined directly below.
+
+MIT License
+
+Copyright (c) 2018 Chan Zuckerberg Initiative
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -15,7 +15,7 @@ include common.mk
 
 define help
 
-Supported targets: prepare, develop, docs, sdist, clean, test, docker, and push_docker.
+Supported targets: prepare, develop, docs, dist, clean, test, docker, and push_docker.
 
 Please note that all build targets require a virtualenv to be active.
 
@@ -29,10 +29,11 @@ list of supported extras. To install Toil in develop mode with all extras, run
 
 	make develop extras=[all]
 
-The 'sdist' target creates a source distribution of Toil. It is used for some unit tests and for
-installing the currently checked out version of Toil into the appliance image.
+The 'dist' target creates a source distribution and a wheel of Toil. It is used
+for some unit tests and for installing the currently checked out version of Toil
+into the appliance image.
 
-The 'clean' target cleans up the side effects of 'develop', 'sdist', 'docs', and 'docker'
+The 'clean' target cleans up the side effects of 'develop', 'dist', 'docs', and 'docker'
 on this machine. It does not undo externally visible effects like removing packages already
 uploaded to PyPI.
 
@@ -91,7 +92,7 @@ extras=
 # You can say make develop packages=xxx to install packages in the same Python
 # environment as Toil itself without creating dependency conflicts with Toil
 packages=
-sdist_name:=toil-$(shell python version_template.py distVersion).tar.gz
+sdist_name:=toil-$(shell python3 version_template.py distVersion).tar.gz
 
 green=\033[0;32m
 normal=\033[0m
@@ -114,11 +115,12 @@ clean_develop: check_venv
 uninstall:
 	- pip uninstall -y toil
 
+dist: sdist
 sdist: dist/$(sdist_name)
 
-dist/$(sdist_name): check_venv
+dist/$(sdist_name):
 	@test -f dist/$(sdist_name) && mv dist/$(sdist_name) dist/$(sdist_name).old || true
-	python setup.py sdist
+	python3 -m build
 	@test -f dist/$(sdist_name).old \
 	    && ( cmp -s <(tar -xOzf dist/$(sdist_name)) <(tar -xOzf dist/$(sdist_name).old) \
 	         && mv dist/$(sdist_name).old dist/$(sdist_name) \
@@ -133,20 +135,25 @@ clean_sdist:
 # Setting SET_OWNER_TAG will tag cloud resources so that UCSC's cloud murder bot won't kill them.
 test: check_venv check_build_reqs
 	TOIL_OWNER_TAG="shared" \
-	    python -m pytest --durations=0 --strict-markers --log-level DEBUG --log-cli-level INFO -r s $(cov) -n $(threads) --dist loadscope $(tests) -m "$(marker)"
+	    python -m pytest --log-format="%(asctime)s %(levelname)s %(message)s" --durations=0 --strict-markers --log-level DEBUG --log-cli-level INFO -r s $(cov) -n $(threads) --dist loadscope $(tests) -m "$(marker)" --color=yes
+
+test_debug: check_venv check_build_reqs
+	TOIL_OWNER_TAG="$(whoami)" \
+	    python -m pytest --log-format="%(asctime)s %(levelname)s %(message)s" --durations=0 --strict-markers --log-level DEBUG -s -o log_cli=true --log-cli-level DEBUG -r s $(tests) -m "$(marker)" --tb=native --maxfail=1 --color=yes
 
 
 # This target will skip building docker and all docker based tests
 # these are our travis tests; rename?
 test_offline: check_venv check_build_reqs
 	@printf "$(cyan)All docker related tests will be skipped.$(normal)\n"
 	TOIL_SKIP_DOCKER=True \
-	    python -m pytest -vv --timeout=600 --strict-markers --log-level DEBUG --log-cli-level INFO $(cov) -n $(threads) --dist loadscope $(tests) -m "$(marker)"
+	TOIL_SKIP_ONLINE=True \
+	    python -m pytest --log-format="%(asctime)s %(levelname)s %(message)s" -vv --timeout=600 --strict-markers --log-level DEBUG --log-cli-level INFO $(cov) -n $(threads) --dist loadscope $(tests) -m "$(marker)" --color=yes
 
 # This target will run about 1 minute of tests, and stop at the first failure
 test_1min: check_venv check_build_reqs
 	TOIL_SKIP_DOCKER=True \
-	    python -m pytest -vv --timeout=10 --strict-markers --log-level DEBUG --log-cli-level INFO --maxfail=1 src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::CWLWorkflowTest::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior -m "$(marker)"
+	    python -m pytest --log-format="%(asctime)s %(levelname)s %(message)s" -vv --timeout=10 --strict-markers --log-level DEBUG --log-cli-level INFO --maxfail=1 src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::CWLWorkflowTest::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior -m "$(marker)" --color=yes
 
 ifdef TOIL_DOCKER_REGISTRY
 
@@ -232,7 +239,7 @@ push_docker: docker
 
 load_docker: docker
 	cd docker ; docker buildx build --platform $(arch) --load --tag=$(docker_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../.docker-cache/toil -f Dockerfile .
-	cd dashboard/prometheus ; docker buildx build --platform $(arch) --load --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/prometheus -f Dockerfile . 
+	cd dashboard/prometheus ; docker buildx build --platform $(arch) --load --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/prometheus -f Dockerfile .
 	cd dashboard/grafana ; docker buildx build --platform $(arch) --load --tag=$(grafana_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/grafana -f Dockerfile .
 	cd dashboard/mtail ; docker buildx build --platform $(arch) --load --tag=$(mtail_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/mtail -f Dockerfile .
 
@@ -245,7 +252,6 @@ endif
 
 
 docs: check_venv check_build_reqs
-	# Strange, but seemingly benign Sphinx warning floods stderr if not filtered:
 	cd docs && ${MAKE} html
 
 clean_docs: check_venv
@@ -254,7 +260,7 @@ clean_docs: check_venv
 clean: clean_develop clean_sdist clean_docs
 
 check_build_reqs:
-	@(python -c 'import mock; import pytest' && which sphinx-build >/dev/null) \
+	@(python -c 'import pytest' && which sphinx-build >/dev/null) \
 		|| ( printf "$(red)Build requirements are missing. Run 'make prepare' to install them.$(normal)\n" ; false )
 
 prepare: check_venv
@@ -295,13 +301,11 @@ remove_unused_imports: $(PYSOURCES)
 remove_trailing_whitespace:
 	$(CURDIR)/contrib/admin/remove_trailing_whitespace.py
 
-format: $(wildcard src/toil/cwl/*.py)
+format: $(PYSOURCES)
 	black $^ contrib/mypy-stubs
 
 mypy:
-	mypy --ignore-missing-imports --no-strict-optional \
-		--warn-redundant-casts --warn-unused-ignores \
-		$(CURDIR)/src/toil/cwl/cwltoil.py
+	MYPYPATH=$(CURDIR)/contrib/mypy-stubs mypy --strict $(CURDIR)/src/toil/{cwl/cwltoil.py,test/cwl/cwlTest.py}
 	$(CURDIR)/contrib/admin/mypy-with-ignore.py
 
 # This target will check any modified files for pylint errors.
@@ -321,8 +325,8 @@ diff_pydocstyle_report: pydocstyle_report.txt
 	diff-quality --compare-branch=master --violations=pycodestyle --fail-under=100 $^
 
 diff_mypy:
-	mypy --cobertura-xml-report . src/toil || true
-	diff-cover --fail-under=100 cobertura.xml
+	mypy --cobertura-xml-report . src/toil > /dev/null || true
+	diff-cover --fail-under=100 --compare-branch origin/master cobertura.xml
 
 pyupgrade: $(PYSOURCES)
 	pyupgrade --exit-zero-even-if-changed --py37-plus $^

README.rst

@@ -4,7 +4,9 @@
 
 Toil is a scalable, efficient, cross-platform (Linux & macOS) pipeline management system,
 written entirely in Python, and designed around the principles of functional
-programming.
+programming. It supports running workflows written in either Common Workflow Language (`CWL`_) or 
+Workflow Description Language (`WDL`_), as well as having its own rich Python API for writing workflows against. 
+It supports running workflows locally on your system (e.g. a laptop), on an HPC cluster, or in the cloud. 
 
 * Check the `website`_ for a description of Toil and its features.
 * Full documentation for the latest stable release can be found at
@@ -20,8 +22,10 @@ programming.
 .. _forum: https://groups.google.com/forum/#!forum/toil-community
 .. _blog: https://toilpipelines.wordpress.com/
 .. _biostars: https://www.biostars.org/t/toil/
+.. _CWL: https://www.commonwl.org/
+.. _WDL: https://openwdl.org/
 
 Notes:
 
 * Toil moved from https://github.com/BD2KGenomics/toil to https://github.com/DataBiosphere/toil on July 5th, 2018.
-* Toil dropped python 2.7 support on February 13, 2020 (last working py2.7 version is 3.24.0).
+* Toil dropped Python 2.7 support on February 13, 2020 (the last working py2.7 version is 3.24.0).

attic/README.md

@@ -1,5 +1,5 @@
 #Toil
-Python based pipeline management software for clusters that makes running recursive and dynamically scheduled computations straightforward. So far works with gridEngine, lsf, parasol and on multi-core machines.
+Python based pipeline management software for clusters that makes running recursive and dynamically scheduled computations straightforward. So far works with gridEngine, lsf, and on multi-core machines.
 
 ##Authors
 [Benedict Paten](https://github.com/benedictpaten/), [Dent Earl](https://github.com/dentearl/), [Daniel Zerbino](https://github.com/dzserbino/), [Glenn Hickey](https://github.com/glennhickey/), other UCSC people.
@@ -28,9 +28,9 @@ The following walks through running a toil script and using the command-line too
 
 Once toil is installed, running a toil script is performed by executing the script from the command-line, e.g. (using the file sorting toy example in **tests/sort/scriptTreeTest_Sort.py**):
 
-<code>[]$ scriptTreeTest_Sort.py --fileToSort foo --toil bar/toil --batchSystem parasol --logLevel INFO --stats</code>
+<code>[]$ scriptTreeTest_Sort.py --fileToSort foo --toil bar/toil --batchSystem slurm --logLevel INFO --stats</code>
 
-Which in this case uses the parasol batch system, and INFO level logging and where foo is the file to sort and bar/toil is the location of a directory (which should not already exist) from which the batch will be managed. Details of the toil options are described below; the stats option is used to gather statistics about the jobs in a run.
+Which in this case uses the slurm batch system, and INFO level logging and where foo is the file to sort and bar/toil is the location of a directory (which should not already exist) from which the batch will be managed. Details of the toil options are described below; the stats option is used to gather statistics about the jobs in a run.
 
 The script will return a zero exit value if the toil system is successfully able to run to completion, else it will create an exception. If the script fails because a job failed then the log file information of the job will be reported to std error.
 The toil directory (here 'bar/toil') is not automatically deleted regardless of success or failure, and contains a record of the jobs run, which can be enquired about using the **toilStatus** command. e.g.
@@ -150,17 +150,14 @@ The important arguments to **toilStats** are:
 
     --batchSystem=BATCHSYSTEM
                         The type of batch system to run the job(s) with,
-                        currently can be
-                        'singleMachine'/'parasol'/'acidTest'/'gridEngine'/'lsf'.
+                        currently can be 'singleMachine'/'gridEngine'/'lsf'.
                         default=singleMachine
     --maxThreads=MAXTHREADS
                         The maximum number of threads (technically processes
                         at this point) to use when running in single machine
                         mode. Increasing this will allow more jobs to run
                         concurrently when running on a single machine.
                         default=4
-    --parasolCommand=PARASOLCOMMAND
-                        The command to run the parasol program default=parasol
 
     Options to specify default cpu/memory requirements (if not
     specified by the jobs themselves), and to limit the total amount of
@@ -202,7 +199,7 @@ The important arguments to **toilStats** are:
     --bigBatchSystem=BIGBATCHSYSTEM
                         The batch system to run for jobs with larger
                         memory/cpus requests, currently can be
-                        'singleMachine'/'parasol'/'acidTest'/'gridEngine'.
+                        'singleMachine'/'gridEngine'.
                         default=none
     --bigMemoryThreshold=BIGMEMORYTHRESHOLD
                         The memory threshold above which to submit to the big
@@ -240,7 +237,7 @@ The important arguments to **toilStats** are:
 
 The following sections are for people creating toil scripts and as general information. The presentation **[docs/toilSlides.pdf](https://github.com/benedictpaten/toil/blob/master/doc/toilSlides.pdf)** is also a quite useful, albeit slightly out of date, guide to using toil. -
 
-Most batch systems (such as LSF, Parasol, etc.) do not allow jobs to spawn
+Most batch systems (such as LSF) do not allow jobs to spawn
 other jobs in a simple way.
 
 The basic pattern provided by toil is as follows:

attic/toil-sort-example.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 from six.moves import xrange
-from argparse import ArgumentParser
+from configargparse import ArgumentParser
 import os
 import logging
 import random
@@ -14,7 +14,7 @@ def setup(job, input_file_id, n, down_checkpoints):
     Returns the FileID of the sorted file
     """
     # Write the input file to the file store
-    job.fileStore.logToMaster("Starting the merge sort")
+    job.fileStore.log_to_leader("Starting the merge sort")
     return job.addChildJobFn(down,
                              input_file_id, n,
                              down_checkpoints=down_checkpoints,
@@ -33,7 +33,7 @@ def down(job, input_file_id, n, down_checkpoints):
     length = os.path.getsize(input_file)
     if length > n:
         # We will subdivide the file
-        job.fileStore.logToMaster("Splitting file: %s of size: %s"
+        job.fileStore.log_to_leader("Splitting file: %s of size: %s"
                                   % (input_file_id, length), level=logging.CRITICAL)
         # Split the file into two copies
         mid_point = get_midpoint(input_file, 0, length)
@@ -52,7 +52,7 @@ def down(job, input_file_id, n, down_checkpoints):
                                                       memory='600M').rv()).rv()
     else:
         # We can sort this bit of the file
-        job.fileStore.logToMaster("Sorting file: %s of size: %s"
+        job.fileStore.log_to_leader("Sorting file: %s of size: %s"
                                   % (input_file_id, length), level=logging.CRITICAL)
         # Sort the copy and write back to the fileStore
         output_file = job.fileStore.getLocalTempFile()
@@ -66,7 +66,7 @@ def up(job, input_file_id_1, input_file_id_2):
     with job.fileStore.writeGlobalFileStream() as (fileHandle, output_id):
         with job.fileStore.readGlobalFileStream(input_file_id_1) as inputFileHandle1:
             with job.fileStore.readGlobalFileStream(input_file_id_2) as inputFileHandle2:
-                job.fileStore.logToMaster("Merging %s and %s to %s"
+                job.fileStore.log_to_leader("Merging %s and %s to %s"
                                           % (input_file_id_1, input_file_id_2, output_id))
                 merge(inputFileHandle1, inputFileHandle2, fileHandle)
 

common.mk

@@ -69,6 +69,7 @@ export TOIL_AWS_NODE_DEBUG?=False  # Don't shut down EC2 instances that fail so
 export TOIL_TEST_INTEGRATIVE?=False  # If ``True``, this allows the integration tests to run.
 export TOIL_TEST_QUICK?=False  # If ``True``, long running tests are skipped.
 export TOIL_SKIP_DOCKER?=False  # Skip docker dependent tests
+export TOIL_SKIP_ONLINE?=False  # Skip Internet-dependent tests
 
 export TOIL_AWS_KEYNAME?=id_rsa  # SSH key to use for tests in AWS.
 export TOIL_GOOGLE_KEYNAME?=id_rsa  # SSH key to use for tests in google.

contrib/admin/mypy-with-ignore.py

@@ -33,25 +33,15 @@ def main():
         'src/toil/__init__.py',
         'src/toil/deferred.py',
         'src/toil/version.py',
-        'src/toil/wdl/utils.py',
-        'src/toil/wdl/wdl_synthesis.py',
-        'src/toil/wdl/wdl_analysis.py',
-        'src/toil/wdl/wdl_functions.py',
-        'src/toil/wdl/toilwdl.py',
-        'src/toil/wdl/versions/draft2.py',
-        'src/toil/wdl/versions/v1.py',
-        'src/toil/wdl/versions/dev.py',
         'src/toil/provisioners/abstractProvisioner.py',
         'src/toil/provisioners/gceProvisioner.py',
         'src/toil/provisioners/__init__.py',
         'src/toil/provisioners/node.py',
         'src/toil/provisioners/aws/boto2Context.py',
-        'src/toil/provisioners/aws/awsProvisioner.py',
         'src/toil/provisioners/aws/__init__.py',
         'src/toil/batchSystems/slurm.py',
         'src/toil/batchSystems/gridengine.py',
         'src/toil/batchSystems/singleMachine.py',
-        'src/toil/batchSystems/parasol.py',
         'src/toil/batchSystems/torque.py',
         'src/toil/batchSystems/options.py',
         'src/toil/batchSystems/registry.py',
@@ -104,7 +94,6 @@ def ignore(file_path):
             if file_path.startswith(prefix):
                 return True
         return False
-
 
     filtered_files_to_check = []
     for file_path in all_files_to_check:

contrib/mypy-stubs/configargparse/__init__.pyi

@@ -0,0 +1,5 @@
+from .configargparse import ArgParser as ArgParser
+from .configargparse import ArgumentParser as ArgumentParser
+from .configargparse import Namespace as Namespace
+from .configargparse import SUPPRESS as SUPPRESS
+from .configargparse import YAMLConfigFileParser as YAMLConfigFileParser