Merging with remove_mako_user_pref

.ci/first_startup.sh

@@ -1,6 +1,16 @@
+#!/bin/sh
+TRIES=120
+URL=http://localhost:8080
+EXIT_CODE=1
+i=0
 echo "Testing for correct startup:"
-bash run.sh --daemon && sleep 30s && curl -I localhost:8080
-EXIT_CODE=$?
+bash run.sh --daemon && \
+    while [ "$i" -le $TRIES ]; do
+        curl "$URL" && EXIT_CODE=0 && break
+        sleep 1
+        i=$((i + 1))
+    done
+kill -9 "$(cat paster.pid)"
 echo "exit code:$EXIT_CODE, showing startup log:"
 cat paster.log
 exit $EXIT_CODE

.ci/flake8_blacklist.txt

@@ -1,6 +1,7 @@
 .venv/
-client/
+.tox/
+client/node_modules/
 database/
-doc/patch.py
 doc/source/conf.py
+eggs/
 lib/galaxy/util/jstree.py

.ci/flake8_docstrings_include_list.txt

@@ -0,0 +1,2 @@
+lib/galaxy/jobs/metrics
+lib/galaxy/exceptions

.ci/flake8_wrapper.sh

@@ -5,4 +5,4 @@ set -e
 flake8 --exclude `paste -sd, .ci/flake8_blacklist.txt` .
 
 # Apply stricter rules for the directories shared with Pulsar
-flake8 --ignore= --max-line-length=150 lib/galaxy/jobs/runners/util/
+flake8 --ignore=D --max-line-length=150 lib/galaxy/jobs/runners/util/

.ci/flake8_wrapper_docstrings.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+# D100 - Missing docstring in public module.
+# D2XX - Whitespace issues.
+# D3XX - Quoting issues.
+# D401 - First line should be in imperative mood 
+# D403 - First word of the first line should be properly capitalized
+args="--ignore=D --select=D100,D201,D202,D206,D207,D208,D209,D211,D3,D401,D403"
+
+# If the first argument is --include, lint the modules expected to pass. If
+# the first argument is --exclude, lint all modules the full Galaxy linter lints
+# (this will fail).
+
+if [ "$1" = "--include" ];
+then
+    flake8 $args `paste .ci/flake8_docstrings_include_list.txt`
+else
+    flake8 $args --exclude `paste -sd, .ci/flake8_blacklist.txt` .
+fi

.ci/flake8_wrapper_imports.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+
+flake8 `paste .ci/flake8_lint_include_list.txt`

.ci/py3_sources.txt

@@ -1,12 +1,22 @@
+client/galaxy/style/source_material/circle.py
 contrib/
 cron/
+doc/parse_gx_xsd.py
+doc/patch.py
 lib/galaxy/actions/
 lib/galaxy/auth/
 lib/galaxy/config.py
 lib/galaxy/dataset_collections/
+lib/galaxy/datatypes/assembly.py
 lib/galaxy/datatypes/binary.py
+lib/galaxy/datatypes/constructive_solid_geometry.py
 lib/galaxy/datatypes/converters/
 lib/galaxy/datatypes/dataproviders/
+lib/galaxy/datatypes/data.py
+lib/galaxy/datatypes/images.py
+lib/galaxy/datatypes/msa.py
+lib/galaxy/datatypes/ngsindex.py
+lib/galaxy/datatypes/proteomics.py
 lib/galaxy/datatypes/sequence.py
 lib/galaxy/datatypes/sniff.py
 lib/galaxy/datatypes/tabular.py
@@ -19,7 +29,11 @@ lib/galaxy/forms/
 lib/galaxy/jobs/
 lib/galaxy/managers/
 lib/galaxy/model/__init__.py
+lib/galaxy/model/item_attrs.py
+lib/galaxy/model/mapping.py
+lib/galaxy/model/metadata.py
 lib/galaxy/model/migrate/
+lib/galaxy/model/orm/now.py
 lib/galaxy/objectstore/
 lib/galaxy/openid/
 lib/galaxy/quota/
@@ -28,30 +42,42 @@ lib/galaxy/security/
 lib/galaxy/tags/
 lib/galaxy/tools/cwl/
 lib/galaxy/tools/deps/
+lib/galaxy/tools/exception_handling.py
+lib/galaxy/tools/execute.py
 lib/galaxy/tools/lint.py
 lib/galaxy/tools/lint_util.py
 lib/galaxy/tools/linters/
 lib/galaxy/tools/loader.py
 lib/galaxy/tools/loader_directory.py
+lib/galaxy/tools/parameters/dataset_matcher.py
+lib/galaxy/tools/parameters/__init__.py
+lib/galaxy/tools/parameters/wrapped_json.py
+lib/galaxy/tools/parameters/wrapped.py
 lib/galaxy/tools/parser/
 lib/galaxy/tools/toolbox/
 lib/galaxy/tours/
 lib/galaxy/util/
 lib/galaxy/visualization/
+lib/galaxy/web/buildapp.py
+lib/galaxy/web/framework/__init__.py
+lib/galaxy/web/framework/middleware/error.py
+lib/galaxy/web/framework/middleware/static.py
+lib/galaxy/web/__init__.py
+lib/galaxy/webapps/galaxy/api/tours.py
+lib/galaxy/webapps/galaxy/api/workflows.py
+lib/galaxy/webapps/galaxy/controllers/userskeys.py
+lib/galaxy/webapps/reports/__init__.py
+lib/galaxy/webapps/tool_shed/controllers/user.py
+lib/galaxy/webapps/tool_shed/__init__.py
 lib/galaxy/work/
+lib/galaxy/workflow/extract.py
+lib/galaxy/workflow/run.py
+lib/galaxy/workflow/schedulers/core.py
 lib/galaxy_ext/
 lib/galaxy_utils/
 lib/log_tempfile.py
 lib/psyco_full.py
-lib/tool_shed/capsule/
-lib/tool_shed/dependencies/
-lib/tool_shed/grids/
-lib/tool_shed/managers/
-lib/tool_shed/metadata/
-lib/tool_shed/repository_types/
-lib/tool_shed/tools/
-lib/tool_shed/util/
-lib/tool_shed/utility_containers/
+lib/tool_shed/
 scripts/api/
 scripts/auth/
 scripts/bootstrap_history.py
@@ -72,8 +98,4 @@ scripts/db_shell.py
 scripts/drmaa_external_runner.py
 test/
 tool_list.py
-tools/data_source/
-tools/evolution/
-tools/sr_mapping/
-tools/stats/aggregate_scores_in_intervals.py
-tools/visualization/
+tools/

.ci/validate_test_tools.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+cd `dirname $0`/..
+
+xsd_path="lib/galaxy/tools/xsd/galaxy.xsd"
+# Lint the XSD
+xmllint --noout "$xsd_path"
+
+test_tools_path='test/functional/tools'
+tool_files_list=$(ls "$test_tools_path"/*xml | grep -v '_conf.xml$')
+sh scripts/validate_tools.sh $tool_files_list

.gitignore

@@ -20,6 +20,7 @@ database/pbs
 database/tmp
 database/*.sqlite
 database/openid_consumer_cache
+database/dependencies
 
 # Python bytecode
 *.pyc
@@ -41,6 +42,9 @@ reports_webapp.lock
 reports_webapp.log
 reports_webapp.pid
 
+# Jupyter Runtime Files
+.ipynb_checkpoints
+
 # Config files
 universe_wsgi.ini
 reports_wsgi.ini
@@ -123,6 +127,8 @@ bower_components
 
 # Documentation build files.
 doc/build
+doc/schema.md
+doc/source/dev/schema.rst
 
 # Misc
 *.orig

.travis.yml

@@ -8,6 +8,11 @@ env:
   - TOX_ENV=py27-unit
   - TOX_ENV=qunit
   - TOX_ENV=first_startup
+  - TOX_ENV=py27-lint-imports
+  - TOX_ENV=py27-lint-imports-include-list
+  - TOX_ENV=validate-test-tools
+  - TOX_ENV=py27-lint-docstring-include-list
+
 matrix:
   include:
   - os: osx
@@ -16,12 +21,12 @@ matrix:
   - os: osx
     env: TOX_ENV=py27-unit
     language: generic
-
-before_install:
-  - if [ `uname` == "Darwin"  ]; then bash -c "brew update && brew install python"; fi
+  allow_failures:
+  - env: TOX_ENV=py27-lint-imports
 
 install:
   - pip install tox
+  - if [ "$TOX_ENV" == "validate-test-tools" ]; then sudo apt-get install libxml2-utils; fi
   - if [ "$TOX_ENV" == "qunit" ]; then bash -c 'cd test/qunit && npm install'; fi
   - if [ "$TOX_ENV" == "first_startup" ]; then bash -c "bash scripts/common_startup.sh && wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0127.sqlite && mv db_gx_rev_0127.sqlite database/universe.sqlite && bash manage_db.sh -c ./config/galaxy.ini.sample upgrade"; fi
 

CONTRIBUTORS.md

@@ -48,6 +48,7 @@ The following individuals have contributed code to Galaxy:
 * Eric Enns <eric.enns@gmail.com>
 * fescudie <fescudie@toulouse.inra.fr>
 * Dorine Francheteau <dorine@bx.psu.edu>
+* Jean-Frédéric (@JeanFred on Github)
 * Jaime Frey <jfrey@cs.wisc.edu>
 * Carrie Ganote <cganote@iu.edu>
 * Ryan Golhar <ngsbioinformatics@gmail.com>
@@ -77,6 +78,7 @@ The following individuals have contributed code to Galaxy:
 * Simone Leo <simone.leo@gmail.com>
 * Kanwei Li <kanwei@gmail.com>
 * Michael Li <michael.li@uwaterloo.ca>
+* Pierre Lindenbaum <plindenbaum@yahoo.fr>
 * Mikael Loaec <mikael.loaec@versailles.inra.fr>
 * Philip Mabon <philipmabon@gmail.com>
 * Remi Marenco <remi.marenco@gmail.com> <remimarenco@gmail.com>

LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2005-2015 Galaxy Contributors (see CONTRIBUTORS.md)
+Copyright (c) 2005-2016 Galaxy Contributors (see CONTRIBUTORS.md)
 
 Licensed under the Academic Free License version 3.0
 

Makefile

@@ -13,6 +13,11 @@ IN_VENV=if [ -f $(VENV)/bin/activate ]; then . $(VENV)/bin/activate; fi;
 PROJECT_URL?=https://github.com/galaxyproject/galaxy
 GRUNT_DOCKER_NAME:=galaxy/client-builder:16.01
 GRUNT_EXEC?=node_modules/grunt-cli/bin/grunt
+DOCS_DIR=doc
+DOC_SOURCE_DIR=$(DOCS_DIR)/source
+SLIDESHOW_DIR=$(DOC_SOURCE_DIR)/slideshow
+OPEN_RESOURCE=bash -c 'open $$0 || xdg-open $$0'
+SLIDESHOW_TO_PDF?=bash -c 'docker run --rm -v `pwd`:/cwd astefanutti/decktape /cwd/$$0 /cwd/`dirname $$0`/`basename -s .html $$0`.pdf'
 
 all: help
 	@echo "This makefile is primarily used for building Galaxy's JS client. A sensible all target is not yet implemented."
@@ -21,19 +26,31 @@ docs: ## generate Sphinx HTML documentation, including API docs
 	$(IN_VENV) $(MAKE) -C doc clean
 	$(IN_VENV) $(MAKE) -C doc html
 
+docs-slides-ready:
+	test -f plantuml.jar ||  wget http://jaist.dl.sourceforge.net/project/plantuml/plantuml.jar
+	java -jar plantuml.jar -c $(DOC_SOURCE_DIR)/slideshow/architecture/images/plantuml_options.txt -tsvg $(SLIDESHOW_DIR)/architecture/images/ *.plantuml.txt
+	$(IN_VENV) python scripts/slideshow/build_slideshow.py 'Galaxy Architecture' $(SLIDESHOW_DIR)/architecture/galaxy_architecture.md
+
+docs-slides-export: docs-slides-ready
+	$(SLIDESHOW_TO_PDF) $(SLIDESHOW_DIR)/galaxy_architecture/galaxy_architecture.html
+
 _open-docs:
-	open doc/_build/html/index.html || xdg-open doc/_build/html/index.html
+	$(OPEN_RESOURCE) $(DOCS_DIR)/_build/html/index.html
 
 open-docs: docs _open-docs ## generate Sphinx HTML documentation and open in browser
 
 open-project: ## open project on github
-	open $(PROJECT_URL) || xdg-open $(PROJECT_URL)
+	$(OPEN_RESOURCE) $(PROJECT_URL)
 
 lint: ## check style using tox and flake8 for Python 2 and Python 3
 	$(IN_VENV) tox -e py27-lint && tox -e py34-lint
 
 release-ensure-upstream: ## Ensure upstream branch for release commands setup
-	if [ ! `git remote -v | grep -q $(RELEASE_UPSTREAM)` ]; then git remote add $(RELEASE_UPSTREAM) git@github.com:galaxyproject/galaxy.git; fi
+ifeq (shell git remote -v | grep $(RELEASE_UPSTREAM), )
+	git remote add $(RELEASE_UPSTREAM) git@github.com:galaxyproject/galaxy.git
+else
+	@echo "Remote $(RELEASE_UPSTREAM) already exists."
+endif
 
 release-merge-stable-to-next: release-ensure-upstream ## Merge last release into dev
 	git fetch $(RELEASE_UPSTREAM) && git checkout dev && git merge --ff-only $(RELEASE_UPSTREAM)/dev && git merge $(RELEASE_UPSTREAM)/$(RELEASE_PREVIOUS)
@@ -79,6 +96,18 @@ grunt-docker: grunt-docker-image ## Run grunt inside docker
 clean-grunt-docker-image: ## Remove grunt docker image
 	docker rmi ${GRUNT_DOCKER_NAME}
 
+grunt-watch-style: npm-deps ## Execute watching style builder for dev purposes
+	cd client && $(GRUNT_EXEC) watch-style
+
+grunt-watch-develop: npm-deps ## Execute watching grunt builder for dev purposes (unpacked, allows debugger statements)
+	cd client && $(GRUNT_EXEC) watch --develop
+
+webpack-watch: npm-deps ## Execute watching webpack for dev purposes
+	cd client && ./node_modules/webpack/bin/webpack.js --watch
+
+client-develop: grunt-watch-style grunt-watch-develop webpack-watch  ## A useful target for parallel development building.
+	@echo "Remember to rerun `make client` before committing!"
+
 
 # Release Targets
 release-create-rc: release-ensure-upstream ## Create a release-candidate branch
@@ -89,14 +118,16 @@ release-create-rc: release-ensure-upstream ## Create a release-candidate branch
 	git push $(MY_UPSTREAM) release_$(RELEASE_CURR)
 	git push $(RELEASE_UPSTREAM) release_$(RELEASE_CURR)
 	git checkout -b version-$(RELEASE_CURR)
-	sed -i "s/^VERSION_MAJOR = .*/VERSION_MAJOR = \"$(RELEASE_CURR)\"/" lib/galaxy/version.py
-	sed -i "s/^VERSION_MINOR = .*/VERSION_MINOR = \"rc1\"/" lib/galaxy/version.py
+	sed -i.bak -e "s/^VERSION_MAJOR = .*/VERSION_MAJOR = \"$(RELEASE_CURR)\"/" lib/galaxy/version.py
+	sed -i.bak -e "s/^VERSION_MINOR = .*/VERSION_MINOR = \"rc1\"/" lib/galaxy/version.py
+	rm -f lib/galaxy/version.py.bak
 	git add lib/galaxy/version.py
 	git commit -m "Update version to $(RELEASE_CURR).rc1"
 	git checkout dev
 
 	git checkout -b version-$(RELEASE_NEXT).dev
-	sed -i "s/^VERSION_MAJOR = .*/VERSION_MAJOR = \"$(RELEASE_NEXT)\"/" lib/galaxy/version.py
+	sed -i.bak -e "s/^VERSION_MAJOR = .*/VERSION_MAJOR = \"$(RELEASE_NEXT)\"/" lib/galaxy/version.py
+	rm -f lib/galaxy/version.py.bak
 	git add lib/galaxy/version.py
 	git commit -m "Update version to $(RELEASE_NEXT).dev"
 
@@ -106,6 +137,7 @@ release-create-rc: release-ensure-upstream ## Create a release-candidate branch
 	git commit -m "Merge branch 'version-$(RELEASE_CURR)' into version-$(RELEASE_NEXT).dev"
 	git push $(MY_UPSTREAM) version-$(RELEASE_CURR):version-$(RELEASE_CURR)
 	git push $(MY_UPSTREAM) version-$(RELEASE_NEXT).dev:version-$(RELEASE_NEXT).dev
+	git checkout dev
 	git branch -d version-$(RELEASE_CURR)
 	git branch -d version-$(RELEASE_NEXT).dev
 	# TODO: Use hub to automate these PR creations or push directly.
@@ -125,7 +157,8 @@ release-create: release-ensure-upstream ## Create a release branch
 	# Test run of merging. If there are conflicts, it will fail here.
 	git merge release_$(RELEASE_CURR)
 	git checkout release_$(RELEASE_CURR)
-	sed -i "s/^VERSION_MINOR = .*/VERSION_MINOR = None/" lib/galaxy/version.py
+	sed -i.bak -e "s/^VERSION_MINOR = .*/VERSION_MINOR = None/" lib/galaxy/version.py
+	rm -f lib/galaxy/version.py.bak
 	git add lib/galaxy/version.py
 	git commit -m "Update version to $(RELEASE_CURR)"
 	git tag -m "Tag version $(RELEASE_CURR)" v$(RELEASE_CURR)
@@ -137,10 +170,10 @@ release-create: release-ensure-upstream ## Create a release branch
 	git commit -m "Merge branch 'release_$(RELEASE_CURR)' into dev"
 	git checkout master
 	git merge release_$(RELEASE_CURR)
-	#git push $(RELEASE_UPSTREAM) release_$(RELEASE_CURR):release_$(RELEASE_CURR)
-	#git push $(RELEASE_UPSTREAM) dev:dev
-	#git push $(RELEASE_UPSTREAM) master:master
-	#git push $(RELEASE_UPSTREAM) --tags
+	git push $(RELEASE_UPSTREAM) release_$(RELEASE_CURR):release_$(RELEASE_CURR)
+	git push $(RELEASE_UPSTREAM) dev:dev
+	git push $(RELEASE_UPSTREAM) master:master
+	git push $(RELEASE_UPSTREAM) --tags
 
 release-create-point: ## Create a point release
 	git pull --ff-only $(RELEASE_UPSTREAM) master
@@ -153,7 +186,8 @@ release-create-point: ## Create a point release
 	#git push $(MY_UPSTREAM) $(RELEASE_NEXT_BRANCH)
 	git merge release_$(RELEASE_CURR)
 	git checkout release_$(RELEASE_CURR)
-	sed -i "s/^VERSION_MINOR = .*/VERSION_MINOR = \"$(RELEASE_CURR_MINOR_NEXT)\"/" lib/galaxy/version.py
+	sed -i.bak -e "s/^VERSION_MINOR = .*/VERSION_MINOR = \"$(RELEASE_CURR_MINOR_NEXT)\"/" lib/galaxy/version.py
+	rm -f lib/galaxy/version.py.bak
 	git add lib/galaxy/version.py
 	git commit -m "Update version to $(RELEASE_CURR).$(RELEASE_CURR_MINOR_NEXT)"
 	git tag -m "Tag version $(RELEASE_CURR).$(RELEASE_CURR_MINOR_NEXT)" v$(RELEASE_CURR).$(RELEASE_CURR_MINOR_NEXT)

client/bower.json

@@ -29,7 +29,8 @@
     "wymeditor": "~1.0.0-rc.1",
     "jstree": "~3.0.9",
     "jquery-ui": "git://github.com/jquery/jquery-ui.git#~1.11.2",
-    "threedubmedia.jquery.event": "*",
+    "jquery.complexify": "git://github.com/danpalmer/jquery.complexify.js.git#~0.5.1",
+     "threedubmedia.jquery.event": "*",
     "jquery-migrate": "~1.4",
     "requirejs": "~2"
   },