Merge remote-tracking branch 'upstream/dev' into workflow_intermediat…

…es_ux

.ci/flake8_blacklist.txt

@@ -1,8 +1,7 @@
 .venv/
 .tox/
-eggs/
-client/
+client/node_modules/
 database/
-doc/patch.py
 doc/source/conf.py
+eggs/
 lib/galaxy/util/jstree.py

.ci/flake8_docstrings_include_list.txt

@@ -0,0 +1,2 @@
+lib/galaxy/jobs/metrics
+lib/galaxy/exceptions

.ci/flake8_lint_include_list.txt

@@ -1,11 +1,13 @@
+client/galaxy/style/source_material/circle.py
 contrib/
 cron/add_manual_builds.py
 cron/cleanup_datasets.py
 cron/parse_builds_3_sites.py
 cron/parse_builds.py
+doc/parse_gx_xsd.py
+doc/patch.py
 lib/galaxy/actions/
-lib/galaxy/auth/__init__.py
-lib/galaxy/auth/providers/__init__.py
+lib/galaxy/auth/
 lib/galaxy/config.py
 lib/galaxy/dataset_collections/builder.py
 lib/galaxy/dataset_collections/__init__.py
@@ -14,6 +16,7 @@ lib/galaxy/dataset_collections/subcollections.py
 lib/galaxy/dataset_collections/type_description.py
 lib/galaxy/datatypes/assembly.py
 lib/galaxy/datatypes/binary.py
+lib/galaxy/datatypes/checkers.py
 lib/galaxy/datatypes/constructive_solid_geometry.py
 lib/galaxy/datatypes/converters/bcf_bgzip_to_bcf_converter.py
 lib/galaxy/datatypes/converters/bcf_to_bcf_bgzip_converter.py
@@ -46,14 +49,17 @@ lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py
 lib/galaxy/datatypes/coverage.py
 lib/galaxy/datatypes/dataproviders/base.py
 lib/galaxy/datatypes/dataproviders/exceptions.py
+lib/galaxy/datatypes/dataproviders/__init__.py
 lib/galaxy/datatypes/data.py
 lib/galaxy/datatypes/display_applications/__init__.py
 lib/galaxy/datatypes/display_applications/util.py
 lib/galaxy/datatypes/images.py
 lib/galaxy/datatypes/__init__.py
+lib/galaxy/datatypes/metadata.py
 lib/galaxy/datatypes/msa.py
 lib/galaxy/datatypes/ngsindex.py
 lib/galaxy/datatypes/proteomics.py
+lib/galaxy/datatypes/sequence.py
 lib/galaxy/datatypes/tabular.py
 lib/galaxy/datatypes/text.py
 lib/galaxy/datatypes/tracks.py
@@ -68,14 +74,25 @@ lib/galaxy/external_services/result_handlers/__init__.py
 lib/galaxy_ext/
 lib/galaxy/forms/__init__.py
 lib/galaxy/jobs/actions/
+lib/galaxy/jobs/command_factory.py
 lib/galaxy/jobs/datasets.py
 lib/galaxy/jobs/deferred/
 lib/galaxy/jobs/error_level.py
 lib/galaxy/jobs/__init__.py
 lib/galaxy/jobs/manager.py
 lib/galaxy/jobs/metrics/
 lib/galaxy/jobs/rules/
+lib/galaxy/jobs/runners/cli.py
+lib/galaxy/jobs/runners/condor.py
+lib/galaxy/jobs/runners/drmaa.py
+lib/galaxy/jobs/runners/godocker.py
+lib/galaxy/jobs/runners/kubernetes.py
+lib/galaxy/jobs/runners/local.py
+lib/galaxy/jobs/runners/pbs.py
+lib/galaxy/jobs/runners/pulsar.py
+lib/galaxy/jobs/runners/slurm.py
 lib/galaxy/jobs/runners/state_handlers/
+lib/galaxy/jobs/runners/tasks.py
 lib/galaxy/jobs/runners/util/cli/factory.py
 lib/galaxy/jobs/runners/util/cli/job/__init__.py
 lib/galaxy/jobs/runners/util/cli/job/slurm_torque.py
@@ -85,6 +102,7 @@ lib/galaxy/jobs/runners/util/drmaa/
 lib/galaxy/jobs/runners/util/env.py
 lib/galaxy/jobs/runners/util/external.py
 lib/galaxy/jobs/runners/util/__init__.py
+lib/galaxy/jobs/runners/util/job_script/
 lib/galaxy/jobs/splitters/basic.py
 lib/galaxy/jobs/splitters/__init__.py
 lib/galaxy/jobs/stock_rules.py
@@ -93,6 +111,7 @@ lib/galaxy/main.py
 lib/galaxy/managers/annotatable.py
 lib/galaxy/managers/api_keys.py
 lib/galaxy/managers/base.py
+lib/galaxy/managers/collections_util.py
 lib/galaxy/managers/context.py
 lib/galaxy/managers/deletable.py
 lib/galaxy/managers/__init__.py
@@ -101,6 +120,9 @@ lib/galaxy/managers/libraries.py
 lib/galaxy/managers/secured.py
 lib/galaxy/managers/taggable.py
 lib/galaxy/model/__init__.py
+lib/galaxy/model/item_attrs.py
+lib/galaxy/model/mapping.py
+lib/galaxy/model/metadata.py
 lib/galaxy/model/migrate/__init__.py
 lib/galaxy/model/migrate/versions/0001_initial_tables.py
 lib/galaxy/model/migrate/versions/0002_metadata_file_table.py
@@ -242,22 +264,29 @@ lib/galaxy/sample_tracking/__init__.py
 lib/galaxy/sample_tracking/sample.py
 lib/galaxy/security/validate_user_input.py
 lib/galaxy/tags/
+lib/galaxy/tools/actions/__init__.py
 lib/galaxy/tools/actions/metadata.py
-lib/galaxy/tools/cwl
+lib/galaxy/tools/cwl/
 lib/galaxy/tools/data_manager/__init__.py
-lib/galaxy/tools/deps
+lib/galaxy/tools/deps/
+lib/galaxy/tools/exception_handling.py
+lib/galaxy/tools/execute.py
 lib/galaxy/tools/filters/
 lib/galaxy/tools/imp_exp/export_history.py
 lib/galaxy/tools/imp_exp/__init__.py
 lib/galaxy/tools/linters/
 lib/galaxy/tools/lint.py
 lib/galaxy/tools/lint_util.py
 lib/galaxy/tools/loader_directory.py
+lib/galaxy/tools/loader.py
+lib/galaxy/tools/parameters/dataset_matcher.py
 lib/galaxy/tools/parameters/history_query.py
+lib/galaxy/tools/parameters/__init__.py
 lib/galaxy/tools/parameters/input_translation.py
 lib/galaxy/tools/parameters/sanitize.py
 lib/galaxy/tools/parameters/validation.py
 lib/galaxy/tools/parameters/wrapped_json.py
+lib/galaxy/tools/parameters/wrapped.py
 lib/galaxy/tools/parser/
 lib/galaxy/tools/special_tools.py
 lib/galaxy/tools/test.py
@@ -287,6 +316,24 @@ lib/galaxy/visualization/__init__.py
 lib/galaxy/visualization/plugins/__init__.py
 lib/galaxy/visualization/plugins/utils.py
 lib/galaxy/visualization/tracks/
+lib/galaxy/web/base/controllers/__init__.py
+lib/galaxy/web/base/__init__.py
+lib/galaxy/web/buildapp.py
+lib/galaxy/web/formatting.py
+lib/galaxy/web/framework/base.py
+lib/galaxy/web/framework/decorators.py
+lib/galaxy/web/framework/helpers/grids.py
+lib/galaxy/web/framework/__init__.py
+lib/galaxy/web/framework/middleware/error.py
+lib/galaxy/web/framework/middleware/__init__.py
+lib/galaxy/web/framework/middleware/request_id.py
+lib/galaxy/web/framework/middleware/sentry.py
+lib/galaxy/web/framework/middleware/static.py
+lib/galaxy/web/framework/middleware/statsd.py
+lib/galaxy/web/framework/middleware/translogger.py
+lib/galaxy/web/framework/middleware/xforwardedhost.py
+lib/galaxy/web/__init__.py
+lib/galaxy/web/params.py
 lib/galaxy/webapps/galaxy/api/genomes.py
 lib/galaxy/webapps/galaxy/api/__init__.py
 lib/galaxy/webapps/galaxy/api/jobs.py
@@ -296,6 +343,7 @@ lib/galaxy/webapps/galaxy/api/roles.py
 lib/galaxy/webapps/galaxy/api/samples.py
 lib/galaxy/webapps/galaxy/api/tools.py
 lib/galaxy/webapps/galaxy/api/tours.py
+lib/galaxy/webapps/galaxy/api/workflows.py
 lib/galaxy/webapps/galaxy/controllers/async.py
 lib/galaxy/webapps/galaxy/controllers/data_manager.py
 lib/galaxy/webapps/galaxy/controllers/error.py
@@ -310,6 +358,7 @@ lib/galaxy/webapps/galaxy/controllers/tool_runner.py
 lib/galaxy/webapps/galaxy/controllers/userskeys.py
 lib/galaxy/webapps/galaxy/__init__.py
 lib/galaxy/webapps/__init__.py
+lib/galaxy/webapps/reports/config.py
 lib/galaxy/webapps/reports/controllers/__init__.py
 lib/galaxy/webapps/reports/controllers/query.py
 lib/galaxy/webapps/reports/__init__.py
@@ -329,24 +378,11 @@ lib/galaxy/webapps/tool_shed/util/__init__.py
 lib/galaxy/webapps/tool_shed/util/ratings_util.py
 lib/galaxy/webapps/tool_shed/util/shed_statistics.py
 lib/galaxy/webapps/util.py
-lib/galaxy/web/base/controllers/__init__.py
-lib/galaxy/web/base/__init__.py
-lib/galaxy/web/buildapp.py
-lib/galaxy/web/formatting.py
-lib/galaxy/web/framework/decorators.py
-lib/galaxy/web/framework/helpers/grids.py
-lib/galaxy/web/framework/__init__.py
-lib/galaxy/web/framework/middleware/__init__.py
-lib/galaxy/web/framework/middleware/request_id.py
-lib/galaxy/web/framework/middleware/sentry.py
-lib/galaxy/web/framework/middleware/static.py
-lib/galaxy/web/framework/middleware/statsd.py
-lib/galaxy/web/framework/middleware/translogger.py
-lib/galaxy/web/framework/middleware/xforwardedhost.py
-lib/galaxy/web/params.py
+lib/galaxy/workflow/extract.py
 lib/galaxy/workflow/__init__.py
 lib/galaxy/workflow/render.py
-lib/galaxy/workflow/schedulers/__init__.py
+lib/galaxy/workflow/run.py
+lib/galaxy/workflow/schedulers/
 lib/galaxy/workflow/steps.py
 lib/galaxy/work/__init__.py
 lib/mimeparse.py
@@ -439,130 +475,105 @@ scripts/tool_shed/migrate_tools_to_repositories.py
 scripts/tool_shed/show_tool_dependency_installation_dir_contents.py
 scripts/tools/re_escape_output.py
 test/api/__init__.py
+test/api/test_dataset_collections.py
 test/api/test_datasets.py
 test/api/test_datatypes.py
 test/api/test_framework.py
-test/api/test_histories.py
 test/api/test_history_contents_provenance.py
 test/api/test_jobs.py
 test/api/test_libraries.py
+test/api/test_page_revisions.py
 test/api/test_search.py
+test/api/test_tool_data.py
+test/api/test_tools.py
 test/api/test_tours.py
+test/api/test_workflow_extraction.py
 test/api/test_workflows_from_yaml.py
-test/api/workflows_format_2/interface.py
-test/api/workflows_format_2/main.py
+test/api/workflows_format_2/
 test/base/api_asserts.py
+test/base/api.py
 test/base/api_util.py
 test/base/__init__.py
 test/base/instrument.py
+test/base/integration_util.py
+test/base/interactor.py
 test/base/nose_util.py
+test/base/test_logging.py
 test/base/tool_shed_util.py
+test/base/twilltestcase.py
 test/casperjs/
 test/functional/database_contexts.py
 test/functional/__init__.py
+test/functional/test_data_managers.py
+test/functional/test_toolbox.py
+test/functional/tool-data/
 test/functional/tools/
-test/integration/__init__.py
-test/integration/test_pulsar_embedded.py
+test/functional/workflow.py
+test/integration/
 test/manual/__init__.py
 test/shed_functional/base/common.py
 test/shed_functional/base/__init__.py
 test/shed_functional/functional/
+test/shed_functional/functional_tests.py
 test/shed_functional/__init__.py
-test/unit/dataset_collections/__init__.py
+test/unit/dataset_collections/
 test/unit/datatypes/dataproviders/__init__.py
+test/unit/datatypes/dataproviders/test_line_dataproviders.py
 test/unit/datatypes/__init__.py
 test/unit/datatypes/test_data.py
-test/unit/jobs/dynamic_tool_destination/__init__.py
-test/unit/jobs/dynamic_tool_destination/mockGalaxy.py
-test/unit/jobs/dynamic_tool_destination/ymltests.py
+test/unit/jobs/dynamic_tool_destination/
 test/unit/jobs/__init__.py
+test/unit/jobs/test_command_factory.py
 test/unit/jobs/test_datasets.py
+test/unit/jobs/test_job_configuration.py
+test/unit/jobs/test_job_output_checker.py
+test/unit/jobs/test_mapper.py
+test/unit/jobs/test_rule_helper.py
 test/unit/jobs/test_rules/
 test/unit/jobs/test_runner_params.py
 test/unit/managers/__init__.py
+test/unit/managers/test_CollectionManager.py
+test/unit/managers/test_DatasetManager.py
 test/unit/managers/test_HDAManager.py
+test/unit/managers/test_HDCAManager.py
 test/unit/managers/test_HistoryContentsManager.py
-test/unit/managers/test_HistoryManager.py
 test/unit/managers/test_UserManager.py
 test/unit/shed_unit/__init__.py
 test/unit/shed_unit/test_fabric_util.py
+test/unit/shed_unit/test_td_common_util.py
+test/unit/test_galaxy_transactions.py
 test/unit/test_lazy_process.py
+test/unit/test_routes.py
 test/unit/test_security_helper.py
 test/unit/test_sockets.py
 test/unit/test_sqlite_utils.py
 test/unit/test_topsort.py
 test/unit/tools/filter_modules/
 test/unit/tools/__init__.py
-test/unit/tools/test_data_parameters.py
-test/unit/tools/test_tool_dependency_description.py
-test/unit/tools/test_tool_panel.py
+test/unit/tools/test_actions.py
+test/unit/tools/test_citations.py
+test/unit/tools/test_collect_primary_datasets.py
+test/unit/tools/test_conda_resolution.py
+test/unit/tools/test_dataset_matcher.py
+test/unit/tools/test_parameter_parsing.py
 test/unit/tools/test_parsing.py
+test/unit/tools/test_select_parameters.py
+test/unit/tools/test_toolbox_filters.py
+test/unit/tools/test_tool_dependency_description.py
 test/unit/tools/test_tool_deps.py
+test/unit/tools/test_tool_external_files.py
 test/unit/tools/test_tool_loader.py
-test/unit/tools/test_toolbox_filters.py
+test/unit/tools/test_tool_panel.py
 test/unit/tools/test_watcher.py
-test/unit/unittest_utils/__init__.py
-test/unit/unittest_utils/utility.py
+test/unit/tools/test_wrappers.py
+test/unit/tools_support.py
+test/unit/unittest_utils/
 test/unit/visualizations/__init__.py
 test/unit/visualizations/plugins/__init__.py
 test/unit/web/base/__init__.py
 test/unit/web/framework/__init__.py
 test/unit/web/__init__.py
-test/unit/workflows/__init__.py
-test/unit/workflows/test_extract_summary.py
-test/unit/workflows/test_modules.py
-test/unit/workflows/test_render.py
-test/unit/workflows/test_workflow_progress.py
+test/unit/workflows/
 test/unit/test_objectstore.py
 tool_list.py
-tools/data_source/fetch.py
-tools/data_source/genbank.py
-tools/data_source/hbvar_filter.py
-tools/data_source/import.py
-tools/data_source/microbial_import_code.py
-tools/data_source/microbial_import.py
-tools/data_source/upload.py
-tools/evolution/
-tools/extract/liftOver_wrapper.py
-tools/filters/axt_to_concat_fasta.py
-tools/filters/axt_to_fasta.py
-tools/filters/axt_to_lav_code.py
-tools/filters/axt_to_lav.py
-tools/filters/bed_to_gff_converter.py
-tools/filters/catWrapper.py
-tools/filters/convert_characters.py
-tools/filters/gff/
-tools/filters/gff_to_bed_converter.py
-tools/filters/gtf_to_bedgraph_converter.py
-tools/filters/join.py
-tools/filters/joinWrapper.py
-tools/filters/lav_to_bed_code.py
-tools/filters/lav_to_bed.py
-tools/filters/mergeCols.py
-tools/filters/randomlines.py
-tools/filters/random_lines_two_pass.py
-tools/filters/secure_hash_message_digest.py
-tools/filters/sff_extract.py
-tools/filters/sorter.py
-tools/filters/trimmer.py
-tools/filters/ucsc_gene_bed_to_exon_bed.py
-tools/filters/ucsc_gene_bed_to_intron_bed.py
-tools/filters/ucsc_gene_table_to_intervals.py
-tools/filters/uniq.py
-tools/filters/wiggle_to_simple.py
-tools/genomespace/
-tools/maf/
-tools/meme/
-tools/metag_tools/
-tools/next_gen_conversion/fastq_conversions.py
-tools/next_gen_conversion/fastq_gen_conv.py
-tools/next_gen_conversion/solid_to_fastq.py
-tools/ngs_simulation/
-tools/phenotype_association/
-tools/plotting/
-tools/solid_tools/
-tools/sr_assembly/
-tools/sr_mapping/
-tools/stats/grouping.py
-tools/stats/gsummary.py
-tools/visualization/
+tools/

.ci/flake8_wrapper.sh

@@ -5,4 +5,4 @@ set -e
 flake8 --exclude `paste -sd, .ci/flake8_blacklist.txt` .
 
 # Apply stricter rules for the directories shared with Pulsar
-flake8 --ignore= --max-line-length=150 lib/galaxy/jobs/runners/util/
+flake8 --ignore=D --max-line-length=150 lib/galaxy/jobs/runners/util/