Merge branch 'dev' of https://github.com/galaxyproject/galaxy into dev

.ci/check_py3_compatibility.sh

@@ -7,15 +7,16 @@ else
 fi
 
 export ACK_OPTIONS=" --type python \
---ignore-dir=.git/ \
---ignore-dir=.tox/ \
---ignore-dir=.venv/ \
---ignore-dir=client/node_modules/ \
---ignore-dir=database/ \
---ignore-dir=doc/build/ \
---ignore-dir=eggs/ \
---ignore-dir=static/maps/ \
---ignore-dir=static/scripts/"
+--ignore-dir=.git \
+--ignore-dir=.tox \
+--ignore-dir=.venv \
+--ignore-dir=.venv3 \
+--ignore-dir=client/node_modules \
+--ignore-dir=database \
+--ignore-dir=doc/build \
+--ignore-dir=eggs \
+--ignore-dir=static/maps \
+--ignore-dir=static/scripts"
 
 PYTHON2_ONLY_MODULES="__builtin__ _winreg BaseHTTPServer CGIHTTPServer \
 ConfigParser Cookie cookielib copy_reg cPickle cStringIO Dialog dummy_thread \

.ci/flake8_blacklist.txt

@@ -1,8 +1,12 @@
-.venv/
-.tox/
-client/node_modules/
-database/
+.git
+.tox
+.venv
+.venv3
+client/node_modules
+database
+doc/build
 doc/source/conf.py
-eggs/
+eggs
 lib/galaxy/util/jstree.py
-lib/galaxy/web/proxy/js/node_modules/
+static/maps
+static/scripts

.ci/flake8_wrapper.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-flake8 --exclude `paste -sd, .ci/flake8_blacklist.txt` .
+flake8 --exclude $(paste -sd, .ci/flake8_blacklist.txt) .
 
 # Apply stricter rules for the directories shared with Pulsar
 flake8 --ignore=D --max-line-length=150 lib/galaxy/jobs/runners/util/

.ci/flake8_wrapper_docstrings.sh

@@ -15,7 +15,7 @@ args="--ignore=D --select=D100,D201,D202,D206,D207,D208,D209,D211,D3,D403"
 
 if [ "$1" = "--include" ];
 then
-    flake8 $args `paste .ci/flake8_docstrings_include_list.txt`
+    flake8 $args $(paste -sd' ' .ci/flake8_docstrings_include_list.txt)
 else
-    flake8 $args --exclude `paste -sd, .ci/flake8_blacklist.txt` .
+    flake8 $args --exclude $(paste -sd, .ci/flake8_blacklist.txt) .
 fi

.ci/jenkins/api/run_tests.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc -api "$@"
+./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --skip_flakey_fails -api "$@"

.ci/jenkins/selenium-compose/run_tests.sh

@@ -23,8 +23,8 @@ find test -iname '*pyc' -exec rm -rf {} \;
 pip install docker-compose
 
 # TODO: Let docker-compose pick these at random.
-export GALAXY_PORT=`python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'`
-export SELENIUM_PORT=`python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'`
+export GALAXY_PORT=`python -c 'from __future__ import print_function; import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'`
+export SELENIUM_PORT=`python -c 'from __future__ import print_function; import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'`
 
 export TARGET_ROOT=`pwd`
 export TARGET_PATH=/galaxy

.ci/jenkins/selenium/run_tests.sh

@@ -2,6 +2,7 @@
 
 # Enable retries on tests to reduce chances of transient failures.
 : ${GALAXY_TEST_SELENIUM_RETRIES:=1}
+: ${GALAXY_TEST_CLIENT_BUILD_IMAGE:='node:9.4.0'}
 
 # If in Jenkins environment, use it for artifacts.
 if [ -n "$BUILD_NUMBER" ];
@@ -16,6 +17,8 @@ fi
 mkdir -p "$GALAXY_TEST_ERRORS_DIRECTORY"
 mkdir -p "$GALAXY_TEST_SCREENSHOTS_DIRECTORY"
 
+docker run -v `pwd`:`pwd`:rw -w `pwd` -u $UID $GALAXY_TEST_CLIENT_BUILD_IMAGE /bin/bash -c 'make client-production-maps'
+
 # Start Selenium server in the test Docker container.
 DOCKER_RUN_EXTRA_ARGS="-e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY} ${DOCKER_RUN_EXTRA_ARGS}"
 export DOCKER_RUN_EXTRA_ARGS

.gitignore

@@ -7,6 +7,7 @@ scripts/scramble/archives
 
 # Python virtualenv
 .venv
+.venv3
 
 # Database stuff
 database/beaker_sessions
@@ -123,16 +124,18 @@ tool-data/shared/jars/
 sprite-*.less
 
 # JS, Local node_modules, and bower_components directories
-static/scripts/bundled/[0-9]*
+static/scripts
 static/maps
-static/scripts/bundled/*.map
 node_modules
 bower_components
 client/galaxy/scripts/libs/*
 
 # Documentation build files.
 doc/build
 doc/schema.md
+doc/source/admin/config_logging_default_yaml.rst
+doc/source/api/api.rst
+doc/source/api/ts_api.rst
 doc/source/dev/schema.rst
 
 # Misc

.travis.yml

@@ -8,13 +8,13 @@ env:
   - TOX_ENV=py27-lint
   - TOX_ENV=py27-unit
   - TOX_ENV=qunit
-  - TOX_ENV=first_startup
-  - TOX_ENV=py27-lint-docstring-include-list
-  - TOX_ENV=check-python-dependencies
+  - TOX_ENV=py27-first_startup
+  - TOX_ENV=py27-lint_docstring_include_list
+  - TOX_ENV=check_python_dependencies
 
 matrix:
   include:
-    - env: TOX_ENV=validate-test-tools
+    - env: TOX_ENV=validate_test_tools
       addons:
         apt:
           packages:
@@ -25,12 +25,19 @@ matrix:
           packages:
             - ack-grep
   allow_failures:
-  - env: TOX_ENV=check-python-dependencies
+    - env: TOX_ENV=check_python_dependencies
 
 
 install:
+  - set -e
   - pip install tox
-  - if [ "$TOX_ENV" == "first_startup" ]; then bash -c "bash scripts/common_startup.sh && wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0127.sqlite && mv db_gx_rev_0127.sqlite database/universe.sqlite && bash manage_db.sh -c ./config/galaxy.ini.sample upgrade"; fi
+  - |
+    if [ "$TOX_ENV" == "py27-first_startup" ]; then
+        sh scripts/common_startup.sh
+        wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0127.sqlite
+        mv db_gx_rev_0127.sqlite database/universe.sqlite
+        sh manage_db.sh -c ./config/galaxy.yml.sample upgrade
+    fi
 
 script: tox -e $TOX_ENV
 

Makefile

@@ -17,7 +17,6 @@ DOC_SOURCE_DIR=$(DOCS_DIR)/source
 SLIDESHOW_DIR=$(DOC_SOURCE_DIR)/slideshow
 OPEN_RESOURCE=bash -c 'open $$0 || xdg-open $$0'
 SLIDESHOW_TO_PDF?=bash -c 'docker run --rm -v `pwd`:/cwd astefanutti/decktape /cwd/$$0 /cwd/`dirname $$0`/`basename -s .html $$0`.pdf'
-CLIENT_COMMIT_WARNING="Please remember to 'make client-production' when finished developing, before a commit!"
 YARN := $(shell command -v yarn 2> /dev/null)
 
 all: help
@@ -72,9 +71,6 @@ tool-shed-config-convert-dry-run: ## convert old style tool shed ini to yaml (dr
 tool-shed-config-convert: ## convert old style tool shed ini to yaml
 	$(CONFIG_MANAGE) convert tool_shed
 
-tool-shed-config-rebuild-sample: ## Rebuild sample tool shed yaml file from schema
-	$(CONFIG_MANAGE) build_sample_yaml tool_shed --add-comments
-
 reports-config-validate: ## validate reports YAML configuration file
 	$(CONFIG_MANAGE) validate reports
 
@@ -84,14 +80,27 @@ reports-config-convert-dry-run: ## convert old style reports ini to yaml (dry ru
 reports-config-convert: ## convert old style reports ini to yaml
 	$(CONFIG_MANAGE) convert reports
 
-reports-config-rebuild-sample: ## Rebuild sample reports yaml file from schema
-	$(CONFIG_MANAGE) build_sample_yaml reports --add-comments
-
 reports-config-lint: ## lint reports YAML configuration file
 	$(CONFIG_MANAGE) lint reports
 
-reports-config-rebuild-rst: ## Rebuild sample reports RST docs
+config-validate: ## validate galaxy YAML configuration file
+	$(CONFIG_MANAGE) validate galaxy
+
+config-convert-dry-run: ## convert old style galaxy ini to yaml (dry run)
+	$(CONFIG_MANAGE) convert galaxy --dry-run
+
+config-convert: ## convert old style galaxy ini to yaml
+	$(CONFIG_MANAGE) convert galaxy
+
+config-rebuild: ## Rebuild all sample YAML and RST files from config schema
+	$(CONFIG_MANAGE) build_sample_yaml galaxy --add-comments
+	$(CONFIG_MANAGE) build_rst galaxy > doc/source/admin/galaxy_options.rst
+	$(CONFIG_MANAGE) build_sample_yaml reports --add-comments
 	$(CONFIG_MANAGE) build_rst reports > doc/source/admin/reports_options.rst
+	$(CONFIG_MANAGE) build_sample_yaml tool_shed --add-comments
+
+config-lint: ## lint galaxy YAML configuration file
+	$(CONFIG_MANAGE) lint galaxy
 
 release-ensure-upstream: ## Ensure upstream branch for release commands setup
 ifeq (shell git remote -v | grep $(RELEASE_UPSTREAM), )
@@ -138,7 +147,6 @@ endif
 
 client: node-deps ## Rebuild client-side artifacts for local development.
 	cd client && yarn run build
-	@echo $(CLIENT_COMMIT_WARNING)
 
 client-production: node-deps ## Rebuild client-side artifacts for a production deployment (or committing to the repository).
 	cd client && yarn run build-production
@@ -151,7 +159,6 @@ client-format: node-deps ## Reformat client code
 
 client-watch: node-deps ## A useful target for parallel development building.
 	cd client && yarn run watch
-	@echo $(CLIENT_COMMIT_WARNING)
 
 client-test: client ## Run qunit tests via Karma
 	cd client && yarn run test

client/galaxy/scripts/apps/analysis.js

@@ -1,5 +1,6 @@
 import jQuery from "jquery";
 var $ = jQuery;
+import * as _ from "underscore";
 import GalaxyApp from "galaxy";
 import Router from "layout/router";
 import ToolPanel from "./panels/tool-panel";
@@ -14,16 +15,21 @@ import GridView from "mvc/grid/grid-view";
 import GridShared from "mvc/grid/grid-shared";
 import Workflows from "mvc/workflow/workflow";
 import HistoryImport from "components/HistoryImport.vue";
+import HistoryView from "components/HistoryView.vue";
 import HistoryList from "mvc/history/history-list";
 import ToolFormComposite from "mvc/tool/tool-form-composite";
 import QueryStringParsing from "utils/query-string-parsing";
 import Utils from "utils/utils";
 import Ui from "mvc/ui/ui-misc";
 import DatasetError from "mvc/dataset/dataset-error";
 import DatasetEditAttributes from "mvc/dataset/dataset-edit-attributes";
+import Visualization from "mvc/visualization/visualization-view";
 import Citations from "components/Citations.vue";
+import DisplayStructure from "components/DisplayStructured.vue";
 import Vue from "vue";
 
+/* global Galaxy */
+
 /** define the 'Analyze Data'/analysis/main/home page for Galaxy
  *  * has a masthead
  *  * a left tool menu to allow the user to load tools in the center panel
@@ -52,6 +58,8 @@ window.app = function app(options, bootstrapped) {
             "(/)pages(/)edit(/)": "show_pages_edit",
             "(/)pages(/)(:action_id)": "show_pages",
             "(/)visualizations(/)edit(/)": "show_visualizations_edit",
+            "(/)visualizations/show/(:visualization_id)": "show_visualizations_client",
+            "(/)visualizations/dataset_id=(:dataset_id)": "show_visualizations_selector",
             "(/)visualizations/(:action_id)": "show_visualizations",
             "(/)workflows/import_workflow": "show_import_workflow",
             "(/)workflows/run(/)": "show_run",
@@ -62,6 +70,8 @@ window.app = function app(options, bootstrapped) {
             "(/)histories(/)rename(/)": "show_histories_rename",
             "(/)histories(/)import(/)": "show_histories_import",
             "(/)histories(/)permissions(/)": "show_histories_permissions",
+            "(/)histories/view": "show_history_view",
+            "(/)histories/show_structure": "show_history_structure",
             "(/)histories(/)(:action_id)": "show_histories",
             "(/)datasets(/)list(/)": "show_datasets",
             "(/)custom_builds": "show_custom_builds",
@@ -113,14 +123,37 @@ window.app = function app(options, bootstrapped) {
             );
         },
 
+        show_visualizations_selector: function(dataset_id) {
+            this.page.display(
+                new Visualization.View({
+                    dataset_id: dataset_id
+                })
+            );
+        },
+
         show_workflows_published: function() {
             this.page.display(
                 new GridView({
-                    url_base: `${Galaxy.root}workflow/list_published`
+                    url_base: `${Galaxy.root}workflow/list_published`,
+                    active_tab: "shared"
                 })
             );
         },
 
+        show_history_view: function() {
+            var historyInstance = Vue.extend(HistoryView);
+            var vm = document.createElement("div");
+            this.page.display(vm);
+            new historyInstance({ propsData: { id: QueryStringParsing.get("id") } }).$mount(vm);
+        },
+
+        show_history_structure: function() {
+            let displayStructureInstance = Vue.extend(DisplayStructure);
+            let vm = document.createElement("div");
+            this.page.display(vm);
+            new displayStructureInstance({ propsData: { id: QueryStringParsing.get(" id: ") } }).$mount(vm);
+        },
+
         show_histories: function(action_id) {
             this.page.display(new HistoryList.View({ action_id: action_id }));
         },
@@ -160,15 +193,17 @@ window.app = function app(options, bootstrapped) {
         show_openids: function() {
             this.page.display(
                 new GridView({
-                    url_base: `${Galaxy.root}user/openids_list`
+                    url_base: `${Galaxy.root}user/openids_list`,
+                    active_tab: "user"
                 })
             );
         },
 
         show_datasets: function() {
             this.page.display(
                 new GridView({
-                    url_base: `${Galaxy.root}dataset/list`
+                    url_base: `${Galaxy.root}dataset/list`,
+                    active_tab: "user"
                 })
             );
         },
@@ -302,6 +337,10 @@ window.app = function app(options, bootstrapped) {
 
     // render and start the router
     $(() => {
+        options.config = _.extend(options.config, {
+            hide_panels: Galaxy.params.hide_panels,
+            hide_masthead: Galaxy.params.hide_masthead
+        });
         Galaxy.page = new Page.View(
             _.extend(options, {
                 Left: ToolPanel,

client/galaxy/scripts/apps/chart.js

@@ -0,0 +1,20 @@
+import * as _ from "libs/underscore";
+import Client from "mvc/visualization/chart/chart-client";
+import Datasets from "mvc/visualization/chart/utilities/datasets";
+import Series from "mvc/visualization/chart/utilities/series";
+import Jobs from "mvc/visualization/chart/utilities/jobs";
+
+window.bundleEntries = window.bundleEntries || {};
+
+export const bundleEntries = {
+    chart: function(options) {
+        return new Client(options);
+    },
+    chartUtilities: {
+        Datasets: Datasets,
+        Jobs: Jobs,
+        Series: Series
+    }
+};
+
+_.extend(window.bundleEntries, bundleEntries);