Merge branch 'dev' into auto_tour_generation

# Conflicts:
#	static/maps/mvc/tool/tool-form-base.js.map
#	static/maps/mvc/tool/tool-form.js.map
#	static/scripts/bundled/analysis.bundled.js
#	static/scripts/bundled/analysis.bundled.js.map
#	static/scripts/bundled/libs.bundled.js
#	static/scripts/bundled/libs.bundled.js.map
#	static/scripts/bundled/login.bundled.js
#	static/scripts/bundled/login.bundled.js.map
#	static/scripts/mvc/tool/tool-form-base.js
#	static/scripts/mvc/tool/tool-form.js

.ci/flake8_blacklist.txt

@@ -5,3 +5,4 @@ database/
 doc/source/conf.py
 eggs/
 lib/galaxy/util/jstree.py
+lib/galaxy/web/proxy/js/node_modules/

.ci/flake8_lint_include_list.txt

@@ -14,6 +14,7 @@ lib/galaxy/dataset_collections/__init__.py
 lib/galaxy/dataset_collections/structure.py
 lib/galaxy/dataset_collections/subcollections.py
 lib/galaxy/dataset_collections/type_description.py
+lib/galaxy/dataset_collections/types/__init__.py
 lib/galaxy/datatypes/assembly.py
 lib/galaxy/datatypes/binary.py
 lib/galaxy/datatypes/checkers.py
@@ -53,8 +54,10 @@ lib/galaxy/datatypes/dataproviders/__init__.py
 lib/galaxy/datatypes/data.py
 lib/galaxy/datatypes/display_applications/__init__.py
 lib/galaxy/datatypes/display_applications/util.py
+lib/galaxy/datatypes/genetics.py
 lib/galaxy/datatypes/images.py
 lib/galaxy/datatypes/__init__.py
+lib/galaxy/datatypes/interval.py
 lib/galaxy/datatypes/metadata.py
 lib/galaxy/datatypes/msa.py
 lib/galaxy/datatypes/ngsindex.py
@@ -63,8 +66,7 @@ lib/galaxy/datatypes/sequence.py
 lib/galaxy/datatypes/tabular.py
 lib/galaxy/datatypes/text.py
 lib/galaxy/datatypes/tracks.py
-lib/galaxy/datatypes/util/generic_util.py
-lib/galaxy/datatypes/util/__init__.py
+lib/galaxy/datatypes/util/
 lib/galaxy/eggs/
 lib/galaxy/exceptions/__init__.py
 lib/galaxy/external_services/__init__.py
@@ -93,16 +95,7 @@ lib/galaxy/jobs/runners/pulsar.py
 lib/galaxy/jobs/runners/slurm.py
 lib/galaxy/jobs/runners/state_handlers/
 lib/galaxy/jobs/runners/tasks.py
-lib/galaxy/jobs/runners/util/cli/factory.py
-lib/galaxy/jobs/runners/util/cli/job/__init__.py
-lib/galaxy/jobs/runners/util/cli/job/slurm_torque.py
-lib/galaxy/jobs/runners/util/cli/job/torque.py
-lib/galaxy/jobs/runners/util/cli/shell/__init__.py
-lib/galaxy/jobs/runners/util/drmaa/
-lib/galaxy/jobs/runners/util/env.py
-lib/galaxy/jobs/runners/util/external.py
-lib/galaxy/jobs/runners/util/__init__.py
-lib/galaxy/jobs/runners/util/job_script/
+lib/galaxy/jobs/runners/util/
 lib/galaxy/jobs/splitters/basic.py
 lib/galaxy/jobs/splitters/__init__.py
 lib/galaxy/jobs/stock_rules.py
@@ -123,134 +116,7 @@ lib/galaxy/model/__init__.py
 lib/galaxy/model/item_attrs.py
 lib/galaxy/model/mapping.py
 lib/galaxy/model/metadata.py
-lib/galaxy/model/migrate/__init__.py
-lib/galaxy/model/migrate/versions/0001_initial_tables.py
-lib/galaxy/model/migrate/versions/0002_metadata_file_table.py
-lib/galaxy/model/migrate/versions/0003_security_and_libraries.py
-lib/galaxy/model/migrate/versions/0004_indexes_and_defaults.py
-lib/galaxy/model/migrate/versions/0006_change_qual_datatype.py
-lib/galaxy/model/migrate/versions/0007_sharing_histories.py
-lib/galaxy/model/migrate/versions/0008_galaxy_forms.py
-lib/galaxy/model/migrate/versions/0009_request_table.py
-lib/galaxy/model/migrate/versions/0010_hda_display_at_authz_table.py
-lib/galaxy/model/migrate/versions/0011_v0010_mysql_index_fix.py
-lib/galaxy/model/migrate/versions/0012_user_address.py
-lib/galaxy/model/migrate/versions/0013_change_lib_item_templates_to_forms.py
-lib/galaxy/model/migrate/versions/0014_pages.py
-lib/galaxy/model/migrate/versions/0015_tagging.py
-lib/galaxy/model/migrate/versions/0016_v0015_mysql_index_fix.py
-lib/galaxy/model/migrate/versions/0017_library_item_indexes.py
-lib/galaxy/model/migrate/versions/0018_ordered_tags_and_page_tags.py
-lib/galaxy/model/migrate/versions/0019_request_library_folder.py
-lib/galaxy/model/migrate/versions/0020_library_upload_job.py
-lib/galaxy/model/migrate/versions/0021_user_prefs.py
-lib/galaxy/model/migrate/versions/0022_visualization_tables.py
-lib/galaxy/model/migrate/versions/0023_page_published_and_deleted_columns.py
-lib/galaxy/model/migrate/versions/0024_page_slug_unique_constraint.py
-lib/galaxy/model/migrate/versions/0025_user_info.py
-lib/galaxy/model/migrate/versions/0026_cloud_tables.py
-lib/galaxy/model/migrate/versions/0027_request_events.py
-lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py
-lib/galaxy/model/migrate/versions/0029_user_actions.py
-lib/galaxy/model/migrate/versions/0030_history_slug_column.py
-lib/galaxy/model/migrate/versions/0031_community_and_workflow_tags.py
-lib/galaxy/model/migrate/versions/0032_stored_workflow_slug_column.py
-lib/galaxy/model/migrate/versions/0033_published_cols_for_histories_and_workflows.py
-lib/galaxy/model/migrate/versions/0034_page_user_share_association.py
-lib/galaxy/model/migrate/versions/0035_item_annotations_and_workflow_step_tags.py
-lib/galaxy/model/migrate/versions/0036_add_deleted_column_to_library_template_assoc_tables.py
-lib/galaxy/model/migrate/versions/0037_samples_library.py
-lib/galaxy/model/migrate/versions/0038_add_inheritable_column_to_library_template_assoc_tables.py
-lib/galaxy/model/migrate/versions/0039_add_synopsis_column_to_library_table.py
-lib/galaxy/model/migrate/versions/0040_page_annotations.py
-lib/galaxy/model/migrate/versions/0041_workflow_invocation.py
-lib/galaxy/model/migrate/versions/0042_workflow_invocation_fix.py
-lib/galaxy/model/migrate/versions/0043_visualization_sharing_tagging_annotating.py
-lib/galaxy/model/migrate/versions/0044_add_notify_column_to_request_table.py
-lib/galaxy/model/migrate/versions/0045_request_type_permissions_table.py
-lib/galaxy/model/migrate/versions/0046_post_job_actions.py
-lib/galaxy/model/migrate/versions/0047_job_table_user_id_column.py
-lib/galaxy/model/migrate/versions/0048_dataset_instance_state_column.py
-lib/galaxy/model/migrate/versions/0049_api_keys_table.py
-lib/galaxy/model/migrate/versions/0050_drop_cloud_tables.py
-lib/galaxy/model/migrate/versions/0051_imported_col_for_jobs_table.py
-lib/galaxy/model/migrate/versions/0052_sample_dataset_table.py
-lib/galaxy/model/migrate/versions/0053_item_ratings.py
-lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py
-lib/galaxy/model/migrate/versions/0055_add_pja_assoc_for_jobs.py
-lib/galaxy/model/migrate/versions/0056_workflow_outputs.py
-lib/galaxy/model/migrate/versions/0057_request_notify.py
-lib/galaxy/model/migrate/versions/0058_history_import_export.py
-lib/galaxy/model/migrate/versions/0059_sample_dataset_file_path.py
-lib/galaxy/model/migrate/versions/0060_history_archive_import.py
-lib/galaxy/model/migrate/versions/0061_tasks.py
-lib/galaxy/model/migrate/versions/0062_user_openid_table.py
-lib/galaxy/model/migrate/versions/0063_sequencer_table.py
-lib/galaxy/model/migrate/versions/0064_add_run_and_sample_run_association_tables.py
-lib/galaxy/model/migrate/versions/0066_deferred_job_and_transfer_job_tables.py
-lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py
-lib/galaxy/model/migrate/versions/0068_rename_sequencer_to_external_services.py
-lib/galaxy/model/migrate/versions/0069_rename_sequencer_form_type.py
-lib/galaxy/model/migrate/versions/0070_add_info_column_to_deferred_job_table.py
-lib/galaxy/model/migrate/versions/0071_add_history_and_workflow_to_sample.py
-lib/galaxy/model/migrate/versions/0072_add_pid_and_socket_columns_to_transfer_job_table.py
-lib/galaxy/model/migrate/versions/0073_add_ldda_to_implicit_conversion_table.py
-lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py
-lib/galaxy/model/migrate/versions/0075_add_subindex_column_to_run_table.py
-lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py
-lib/galaxy/model/migrate/versions/0077_create_tool_tag_association_table.py
-lib/galaxy/model/migrate/versions/0078_add_columns_for_disk_usage_accounting.py
-lib/galaxy/model/migrate/versions/0079_input_library_to_job_table.py
-lib/galaxy/model/migrate/versions/0080_quota_tables.py
-lib/galaxy/model/migrate/versions/0081_add_tool_version_to_hda_ldda.py
-lib/galaxy/model/migrate/versions/0082_add_tool_shed_repository_table.py
-lib/galaxy/model/migrate/versions/0083_add_prepare_files_to_task.py
-lib/galaxy/model/migrate/versions/0084_add_ldda_id_to_implicit_conversion_table.py
-lib/galaxy/model/migrate/versions/0085_add_task_info.py
-lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py
-lib/galaxy/model/migrate/versions/0087_tool_id_guid_map_table.py
-lib/galaxy/model/migrate/versions/0088_add_installed_changeset_revison_column.py
-lib/galaxy/model/migrate/versions/0089_add_object_store_id_columns.py
-lib/galaxy/model/migrate/versions/0090_add_tool_shed_repository_table_columns.py
-lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py
-lib/galaxy/model/migrate/versions/0092_add_migrate_tools_table.py
-lib/galaxy/model/migrate/versions/0093_add_job_params_col.py
-lib/galaxy/model/migrate/versions/0094_add_job_handler_col.py
-lib/galaxy/model/migrate/versions/0095_hda_subsets.py
-lib/galaxy/model/migrate/versions/0096_openid_provider.py
-lib/galaxy/model/migrate/versions/0097_add_ctx_rev_column.py
-lib/galaxy/model/migrate/versions/0098_genome_index_tool_data_table.py
-lib/galaxy/model/migrate/versions/0099_add_tool_dependency_table.py
-lib/galaxy/model/migrate/versions/0100_alter_tool_dependency_table_version_column.py
-lib/galaxy/model/migrate/versions/0101_drop_installed_changeset_revision_column.py
-lib/galaxy/model/migrate/versions/0102_add_tool_dependency_status_columns.py
-lib/galaxy/model/migrate/versions/0103_add_tool_shed_repository_status_columns.py
-lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py
-lib/galaxy/model/migrate/versions/0105_add_cleanup_event_table.py
-lib/galaxy/model/migrate/versions/0106_add_missing_indexes.py
-lib/galaxy/model/migrate/versions/0107_add_exit_code_to_job_and_task.py
-lib/galaxy/model/migrate/versions/0108_add_extended_metadata.py
-lib/galaxy/model/migrate/versions/0109_add_repository_dependency_tables.py
-lib/galaxy/model/migrate/versions/0110_add_dataset_uuid.py
-lib/galaxy/model/migrate/versions/0111_add_job_destinations.py
-lib/galaxy/model/migrate/versions/0112_add_data_manager_history_association_and_data_manager_job_association_tables.py
-lib/galaxy/model/migrate/versions/0113_update_migrate_tools_table.py
-lib/galaxy/model/migrate/versions/0114_update_migrate_tools_table_again.py
-lib/galaxy/model/migrate/versions/0115_longer_user_password_field.py
-lib/galaxy/model/migrate/versions/0116_drop_update_available_col_add_tool_shed_status_col.py
-lib/galaxy/model/migrate/versions/0117_add_user_activation.py
-lib/galaxy/model/migrate/versions/0118_add_hda_extended_metadata.py
-lib/galaxy/model/migrate/versions/0119_job_metrics.py
-lib/galaxy/model/migrate/versions/0120_dataset_collections.py
-lib/galaxy/model/migrate/versions/0121_workflow_uuids.py
-lib/galaxy/model/migrate/versions/0123_add_workflow_request_tables.py
-lib/galaxy/model/migrate/versions/0124_job_state_history.py
-lib/galaxy/model/migrate/versions/0125_workflow_step_tracking.py
-lib/galaxy/model/migrate/versions/0126_password_reset.py
-lib/galaxy/model/migrate/versions/0127_output_collection_adjustments.py
-lib/galaxy/model/migrate/versions/0128_session_timeout.py
-lib/galaxy/model/migrate/versions/0129_job_external_output_metadata_validity.py
-lib/galaxy/model/migrate/versions/0130_change_pref_datatype.py
+lib/galaxy/model/migrate/
 lib/galaxy/model/orm/
 lib/galaxy/model/tool_shed_install/migrate/__init__.py
 lib/galaxy/model/tool_shed_install/migrate/versions/
@@ -264,46 +130,8 @@ lib/galaxy/sample_tracking/__init__.py
 lib/galaxy/sample_tracking/sample.py
 lib/galaxy/security/validate_user_input.py
 lib/galaxy/tags/
-lib/galaxy/tools/actions/
-lib/galaxy/tools/cwl/
-lib/galaxy/tools/data_manager/__init__.py
-lib/galaxy/tools/deps/
-lib/galaxy/tools/exception_handling.py
-lib/galaxy/tools/execute.py
-lib/galaxy/tools/filters/
-lib/galaxy/tools/imp_exp/export_history.py
-lib/galaxy/tools/imp_exp/__init__.py
-lib/galaxy/tools/linters/
-lib/galaxy/tools/lint.py
-lib/galaxy/tools/lint_util.py
-lib/galaxy/tools/loader_directory.py
-lib/galaxy/tools/loader.py
-lib/galaxy/tools/parameters/dataset_matcher.py
-lib/galaxy/tools/parameters/history_query.py
-lib/galaxy/tools/parameters/__init__.py
-lib/galaxy/tools/parameters/input_translation.py
-lib/galaxy/tools/parameters/sanitize.py
-lib/galaxy/tools/parameters/validation.py
-lib/galaxy/tools/parameters/wrapped_json.py
-lib/galaxy/tools/parameters/wrapped.py
-lib/galaxy/tools/parser/
-lib/galaxy/tools/special_tools.py
-lib/galaxy/tools/test.py
-lib/galaxy/tools/toolbox/
-lib/galaxy/tools/util/galaxyops/
-lib/galaxy/tools/util/__init__.py
-lib/galaxy/tools/verify/
+lib/galaxy/tools/
 lib/galaxy/util/
-lib/galaxy_utils/__init__.py
-lib/galaxy/util/sleeper.py
-lib/galaxy/util/specs.py
-lib/galaxy_utils/sequence/fasta.py
-lib/galaxy_utils/sequence/fastq.py
-lib/galaxy_utils/sequence/__init__.py
-lib/galaxy_utils/sequence/transform.py
-lib/galaxy_utils/sequence/vcf.py
-lib/galaxy/util/template.py
-lib/galaxy/util/ucsc.py
 lib/galaxy/version.py
 lib/galaxy/visualization/data_providers/basic.py
 lib/galaxy/visualization/data_providers/cigar.py
@@ -334,6 +162,7 @@ lib/galaxy/web/framework/middleware/xforwardedhost.py
 lib/galaxy/web/__init__.py
 lib/galaxy/web/params.py
 lib/galaxy/webapps/galaxy/api/genomes.py
+lib/galaxy/webapps/galaxy/api/histories.py
 lib/galaxy/webapps/galaxy/api/__init__.py
 lib/galaxy/webapps/galaxy/api/jobs.py
 lib/galaxy/webapps/galaxy/api/lda_datasets.py
@@ -355,6 +184,7 @@ lib/galaxy/webapps/galaxy/controllers/requests.py
 lib/galaxy/webapps/galaxy/controllers/search.py
 lib/galaxy/webapps/galaxy/controllers/tool_runner.py
 lib/galaxy/webapps/galaxy/controllers/userskeys.py
+lib/galaxy/webapps/galaxy/config_watchers.py
 lib/galaxy/webapps/galaxy/__init__.py
 lib/galaxy/webapps/__init__.py
 lib/galaxy/webapps/reports/config.py
@@ -472,7 +302,6 @@ scripts/tools/re_escape_output.py
 test/api/__init__.py
 test/api/test_dataset_collections.py
 test/api/test_datasets.py
-test/api/test_datatypes.py
 test/api/test_framework.py
 test/api/test_history_contents_provenance.py
 test/api/test_jobs.py
@@ -483,7 +312,6 @@ test/api/test_tool_data.py
 test/api/test_tours.py
 test/api/test_workflow_extraction.py
 test/api/test_workflows_from_yaml.py
-test/api/workflows_format_2/
 test/base/api_asserts.py
 test/base/api.py
 test/base/api_util.py
@@ -495,6 +323,7 @@ test/base/nose_util.py
 test/base/test_logging.py
 test/base/tool_shed_util.py
 test/base/twilltestcase.py
+test/base/workflows_format_2/
 test/casperjs/
 test/functional/database_contexts.py
 test/functional/__init__.py

.ci/jenkins/selenium/run_galaxy.bash

@@ -2,9 +2,12 @@
 
 set -e
 
-sleep 30  # TODO: wait on something instead of just sleeping...
-
-echo `df`
+echo "Waiting for postgres to become available"
+while ! nc -z postgres 5432;
+do
+    sleep 1
+    printf "."
+done
 
 echo "Creating postgres database for Galaxy"
 createdb -w -U postgres -h postgres galaxy

.ci/jenkins/selenium/run_tests.sh

@@ -36,6 +36,12 @@ docker-compose down | true
 docker-compose build galaxy
 docker-compose up -d
 
+function tear_down {
+    docker-compose down
+}
+
+trap tear_down EXIT
+
 for service_name in postgres galaxy selenium
 do
     echo "Waiting on service ${service_name}"
@@ -79,7 +85,6 @@ do
                 docker logs "${container_id}"
                 echo "---"
             done
-            docker-compose down | true
             exit 1
         fi
     done
@@ -105,6 +110,4 @@ exit_code=$?
 
 cd $TEST_DIRECTORY
 
-docker-compose down
-
 exit $exit_code

.ci/py3_sources.txt

@@ -13,7 +13,9 @@ lib/galaxy/datatypes/constructive_solid_geometry.py
 lib/galaxy/datatypes/converters/
 lib/galaxy/datatypes/dataproviders/
 lib/galaxy/datatypes/data.py
+lib/galaxy/datatypes/genetics.py
 lib/galaxy/datatypes/images.py
+lib/galaxy/datatypes/interval.py
 lib/galaxy/datatypes/msa.py
 lib/galaxy/datatypes/ngsindex.py
 lib/galaxy/datatypes/proteomics.py
@@ -40,35 +42,19 @@ lib/galaxy/quota/
 lib/galaxy/sample_tracking/
 lib/galaxy/security/
 lib/galaxy/tags/
-lib/galaxy/tools/actions/
-lib/galaxy/tools/cwl/
-lib/galaxy/tools/deps/
-lib/galaxy/tools/exception_handling.py
-lib/galaxy/tools/execute.py
-lib/galaxy/tools/lint.py
-lib/galaxy/tools/lint_util.py
-lib/galaxy/tools/linters/
-lib/galaxy/tools/loader.py
-lib/galaxy/tools/loader_directory.py
-lib/galaxy/tools/parameters/dataset_matcher.py
-lib/galaxy/tools/parameters/__init__.py
-lib/galaxy/tools/parameters/input_translation.py
-lib/galaxy/tools/parameters/sanitize.py
-lib/galaxy/tools/parameters/validation.py
-lib/galaxy/tools/parameters/wrapped_json.py
-lib/galaxy/tools/parameters/wrapped.py
-lib/galaxy/tools/parser/
-lib/galaxy/tools/test.py
-lib/galaxy/tools/toolbox/
+lib/galaxy/tools/
 lib/galaxy/tours/
 lib/galaxy/util/
 lib/galaxy/visualization/
 lib/galaxy/web/buildapp.py
+lib/galaxy/web/framework/base.py
+lib/galaxy/web/framework/decorators.py
 lib/galaxy/web/framework/__init__.py
 lib/galaxy/web/framework/middleware/error.py
 lib/galaxy/web/framework/middleware/static.py
 lib/galaxy/web/framework/middleware/statsd.py
 lib/galaxy/web/__init__.py
+lib/galaxy/webapps/galaxy/api/histories.py
 lib/galaxy/webapps/galaxy/api/tours.py
 lib/galaxy/webapps/galaxy/api/workflows.py
 lib/galaxy/webapps/galaxy/controllers/external_services.py
@@ -84,10 +70,9 @@ lib/galaxy/webapps/tool_shed/util/ratings_util.py
 lib/galaxy/work/
 lib/galaxy/workflow/extract.py
 lib/galaxy/workflow/run.py
-lib/galaxy/workflow/schedulers/core.py
+lib/galaxy/workflow/schedulers/
 lib/galaxy/workflow/steps.py
 lib/galaxy_ext/
-lib/galaxy_utils/
 lib/log_tempfile.py
 lib/psyco_full.py
 lib/tool_shed/