Use pipenv to generate development dependencies.

- Rework pattern for linting dependencies to work for dev-requirements.txt.
- Get rid of dev-requirements.txt and replace it with a symbolic link to the latest pinned versions of dependencies.

Makefile

@@ -119,8 +119,11 @@ release-check-blocking-prs: ## Check github for release blocking PRs
 release-bootstrap-history: ## bootstrap history for a new release
 	$(IN_VENV) python scripts/bootstrap_history.py --release $(RELEASE_CURR)
 
-update-linting-requirements:  ## update linting dependencies
-	sh .ci/tox_envs/update.sh
+update-dependencies:  ## update linting + dev dependencies
+	sh lib/galaxy/dependencies/pipfiles/update.sh
+
+update-and-commit-dependencies:  ## update and commit linting + dev dependencies
+	sh lib/galaxy/dependencies/pipfiles/update.sh -c
 
 node-deps: ## Install NodeJS dependencies.
 	cd client && yarn install --check-files

lib/galaxy/dependencies/dev-requirements.txt

@@ -0,0 +1 @@
+pipfiles/develop/pinned-requirements.txt

lib/galaxy/dependencies/pipfiles/develop/Pipfile

@@ -0,0 +1,25 @@
+[[source]]
+
+url = "https://wheels.galaxyproject.org/simple"
+verify_ssl = true
+name = "galaxy"
+
+[[source]]
+
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+
+NoseHTML = "*"
+twill = "==0.9.1"
+mock = "*"
+selenium = "*"
+testfixtures = "*"
+watchdog = "*"
+"pygithub3" = "*"
+Sphinx = "*"
+sphinx_rtd_theme = "*"
+lxml = "*"
+recommonmark = "*"

lib/galaxy/dependencies/pipfiles/develop/pinned-hashed-requirements.txt

@@ -0,0 +1,35 @@
+funcsigs==1.0.2; python_version < '3.3' --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca  --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50
+recommonmark==0.4.0 --hash=sha256:cd8bf902e469dae94d00367a8197fb7b81fcabc9cfb79d520e0d22d0fbeaa8b7  --hash=sha256:6e29c723abcf5533842376d87c4589e62923ecb6002a8e059eb608345ddaff9d
+snowballstemmer==1.2.1 --hash=sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89  --hash=sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128
+nosehtml==0.4.4 --hash=sha256:056f5d88f80e47cc85f1ca8fb671899360537b2eae0ecd802daadc618871b7a2  --hash=sha256:fa01ac39070500623ccad3e63e9db660268f20c0b4eff24c8ac6f6bee71fab36
+pygments==2.2.0 --hash=sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d  --hash=sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc
+watchdog==0.8.3 --hash=sha256:7e65882adb7746039b6f3876ee174952f8eaaa34491ba34333ddf1fe35de4162
+lxml==4.1.0 --hash=sha256:7a8715539adb41c78129983ba69d852e0102a3f51d559eeb91dce1f6290c4ad0  --hash=sha256:d3a98dda9831a37ef7f55c5e69c0d276c278f24978f5b36b9fad7eac05a22bfc  --hash=sha256:1deacd52638da2d7fcb864c3949f0285638ec10e6aace93ce15c6a2e0ed91b95  --hash=sha256:1548247ea3b50014a3ea55ad9446108df191b6a6e51aa8f5953c95b663f382ff  --hash=sha256:5da6f5b31ea2b573cb20e88aefc6b49d849d07588ba60871342cae42f569b0d7  --hash=sha256:12e348eb57fb79ccf91a49b7b937c49a5bbe1d73ba75589674b76a56d064bda0  --hash=sha256:b9e1735918fc1e83c522b9f1048e6bc5af38af958e4efc843046e4b0075a021b  --hash=sha256:fafeb4b190bd63ba2bcee2496d99cb7345fafbace6b999403010abdff8c05b72  --hash=sha256:9007da6fb1b96fb1c9d7bd65e97bbbad60295abc19833d7e67e05314c1868f58  --hash=sha256:cfbf0b956f33cda3af2a1438a2541549b69a7a240e71de7d8ca819b8f1547aac  --hash=sha256:0a103253a94cdad86028d273aaebb8b30c75fdf009c23e52cdc8ce88429fd326  --hash=sha256:b3d5a0ecf0c2c31c404246b6706b2e477159ee07b73be5102389ab250dd67701  --hash=sha256:e92af0fd08c7d2176ad4be4a7c47fd800d6ee05046b41e36ed579c01fb106c25  --hash=sha256:feb2144c2ae4035ad57165dd22bdc93b1389158a985c0497a096d39e2b2cd67b  --hash=sha256:a4433655219b84a360dbdf2c34d9625c3988a272e6fc028222d528ad5902f6a2  --hash=sha256:db98287cb1488eb103930a64444542f6ffe83694ef392f801aa56d648d905663  --hash=sha256:307d325ee143b60b9c82912e96e9f4345200c33c8ae00b04b001e4c85fb5f146  --hash=sha256:5caec9b174dbf927034d588669c62d2a9d0ce447365b20a3463f4daab1e4f03b  --hash=sha256:fb816595494ce21191764572215f56edfbc6d9fbebd1491c8466502892989689  --hash=sha256:10399bececdb67f0d9251ecf2dda2abf6ddeee6096741754356f1a3715c8c830  --hash=sha256:c263fd15d27f3be93485fcd83a495cbbc35352512d9e31644d49a54504a1be2a  --hash=sha256:c10ad53216d5af2b3ba63e65db793cb7dd7e598e17826938045e32f38b0e4814  --hash=sha256:d42a5182d4b0953d02e5f46c9f0dc304be736fbaa1c0d2f11326182b9684b5f4  --hash=sha256:dd7c22bf890d266e72c5e5c8c44555ffbfe4ca2a329da785e7d8b1972fc3ff74  --hash=sha256:93df9805146980e83834ea9320baa6a56d8aea45f63d7d3cc721f71eb1a1bac6  --hash=sha256:7ba1b62fe9414d73d493241011df952b72074808debc3a2d6d8a64fb9944edf6  --hash=sha256:d2c121f5f77bed1e1eddeee23ee76fee8a3d48fa7a3aab589d12942f87778a9e  --hash=sha256:be3aaeb5f468a49f523f16736ccff7d82af2b4b303292ba3d052b5b28f3fbe47
+babel==2.5.1 --hash=sha256:f20b2acd44f587988ff185d8949c3e208b4b3d5d20fcab7d91fe481ffa435528  --hash=sha256:6007daf714d0cd5524bbe436e2d42b3c20e68da66289559341e48d2cd6d25811
+six==1.11.0 --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb  --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9
+selenium==3.6.0 --hash=sha256:fb6a2a5d80bf6f1de145d3a92a5d1331755e43aae07cff7b2588aa6e66fa8eaf  --hash=sha256:9563021c5cba084041e13c218eb531d9c6920dcfa0ba1024bb5bf2b6c0df1797
+commonmark==0.5.4 --hash=sha256:34d73ec8085923c023930dfc0bcd1c4286e28a2a82de094bb72fabcc0281cbe5
+docutils==0.14 --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6  --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6  --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274
+typing==3.6.2; python_version < '3.5' --hash=sha256:349b1f9c109c84b53ac79ac1d822eaa68fc91d63b321bd9392df15098f746f53  --hash=sha256:63a8255fe7c6269916baa440eb9b6a67139b0b97a01af632e7bd2842e1e02f15  --hash=sha256:d514bd84b284dd3e844f0305ac07511f097e325171f6cc4a20878d11ad771849
+certifi==2017.7.27.1 --hash=sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704  --hash=sha256:40523d2efb60523e113b44602298f0960e900388cf3bb6043f645cf57ea9e3f5
+mock==2.0.0 --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1  --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba
+testfixtures==5.2.0 --hash=sha256:6a85faa732ab9c2f2ac763775b4348f4d7fd2b9093d8cdb2324ed03b3770ee81  --hash=sha256:7427d4b63861c1896865ea7be078fa1e2b219ce9b1a9a0895f823475a5f76c85
+imagesize==0.7.1 --hash=sha256:6ebdc9e0ad188f9d1b2cdd9bc59cbe42bf931875e829e7a595e6b3abdc05cdfb  --hash=sha256:0ab2c62b87987e3252f89d30b7cedbec12a01af9274af9ffa48108f2c13c6062
+alabaster==0.7.10 --hash=sha256:2eef172f44e8d301d25aff8068fddd65f767a3f04b5f15b0f4922f113aa1c732  --hash=sha256:37cdcb9e9954ed60912ebc1ca12a9d12178c26637abdf124e3cde2341c257fe0
+urllib3==1.22 --hash=sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b  --hash=sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f
+argh==0.26.2 --hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3  --hash=sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65
+markupsafe==1.0 --hash=sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665
+pytz==2017.2 --hash=sha256:c883c2d6670042c7bc1688645cac73dd2b03193d1f7a6847b6154e96890be06d  --hash=sha256:03c9962afe00e503e2d96abab4e8998a0f84d4230fa57afe1e0528473698cdd9  --hash=sha256:487e7d50710661116325747a9cd1744d3323f8e49748e287bc9e659060ec6bf9  --hash=sha256:43f52d4c6a0be301d53ebd867de05e2926c35728b3260157d274635a0a947f1c  --hash=sha256:d1d6729c85acea5423671382868627129432fba9a89ecbb248d8d1c7a9f01c67  --hash=sha256:54a935085f7bf101f86b2aff75bd9672b435f51c3339db2ff616e66845f2b8f9  --hash=sha256:39504670abb5dae77f56f8eb63823937ce727d7cdd0088e6909e6dcac0f89043  --hash=sha256:ddc93b6d41cfb81266a27d23a79e13805d4a5521032b512643af8729041a81b4  --hash=sha256:f5c056e8f62d45ba8215e5cb8f50dfccb198b4b9fbea8500674f3443e4689589
+jinja2==2.9.6 --hash=sha256:2231bace0dfd8d2bf1e5d7e41239c06c9e0ded46e70cc1094a0aa64b0afeb054  --hash=sha256:ddaa01a212cd6d641401cb01b605f4a4d9f37bfc93043d7f760ec70fb99ff9ff
+twill==0.9.1 --hash=sha256:d2df51cc4eca5a6dec0a2af3817b236655558deff9542fc3d7653d16c8d5ba7e
+pygithub3==0.5.1 --hash=sha256:5669a4cdbf4b3192716442e2224db45ecd1639e42f990739ddfa8f2ca3f7ba87
+pyyaml==3.12 --hash=sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f  --hash=sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736  --hash=sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269  --hash=sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8  --hash=sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4  --hash=sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1  --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab  --hash=sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3  --hash=sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8  --hash=sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6  --hash=sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca  --hash=sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8  --hash=sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608  --hash=sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7
+sphinx==1.6.5 --hash=sha256:fdf77f4f30d84a314c797d67fe7d1b46665e6c48a25699d7bf0610e05a2221d4  --hash=sha256:c6de5dbdbb7a0d7d2757f4389cc00e8f6eb3c49e1772378967a12cfcf2cfe098
+pbr==3.1.1 --hash=sha256:60c25b7dfd054ef9bb0ae327af949dd4676aa09ac3a9471cdc871d8a9213f9ac  --hash=sha256:05f61c71aaefc02d8e37c0a3eeb9815ff526ea28b3b76324769e6158d7f95be1
+sphinxcontrib-websupport==1.0.1 --hash=sha256:f4932e95869599b89bf4f80fc3989132d83c9faa5bf633e7b5e0c25dffb75da2  --hash=sha256:7a85961326aa3a400cd4ad3c816d70ed6f7c740acd7ce5d78cd0a67825072eb9
+pathtools==0.1.2 --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0
+sphinx-rtd-theme==0.2.4 --hash=sha256:62ee4752716e698bad7de8a18906f42d33664128eea06c46b718fc7fbd1a9f5c  --hash=sha256:2df74b8ff6fae6965c527e97cca6c6c944886aae474b490e17f92adfbe843417
+chardet==3.0.4 --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691  --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae
+nose==1.3.7 --hash=sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a  --hash=sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac  --hash=sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98
+requests==2.18.4 --hash=sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b  --hash=sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e
+idna==2.6 --hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4  --hash=sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f

lib/galaxy/dependencies/pipfiles/develop/pinned-requirements.txt

@@ -0,0 +1,35 @@
+funcsigs==1.0.2; python_version < '3.3'   
+recommonmark==0.4.0   
+snowballstemmer==1.2.1   
+nosehtml==0.4.4   
+pygments==2.2.0   
+watchdog==0.8.3 
+lxml==4.1.0                                                       
+babel==2.5.1   
+six==1.11.0   
+selenium==3.6.0   
+commonmark==0.5.4 
+docutils==0.14     
+typing==3.6.2; python_version < '3.5'     
+certifi==2017.7.27.1   
+mock==2.0.0   
+testfixtures==5.2.0   
+imagesize==0.7.1   
+alabaster==0.7.10   
+urllib3==1.22   
+argh==0.26.2   
+markupsafe==1.0 
+pytz==2017.2                 
+jinja2==2.9.6   
+twill==0.9.1 
+pygithub3==0.5.1 
+pyyaml==3.12                           
+sphinx==1.6.5   
+pbr==3.1.1   
+sphinxcontrib-websupport==1.0.1   
+pathtools==0.1.2 
+sphinx-rtd-theme==0.2.4   
+chardet==3.0.4   
+nose==1.3.7     
+requests==2.18.4   
+idna==2.6   

lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt

@@ -0,0 +1,6 @@
+mccabe==0.6.1   
+pycodestyle==2.3.1   
+pyflakes==1.6.0   
+enum34==1.1.6; python_version < '3.4'       
+flake8==3.5.0   
+configparser==3.5.0; python_version < '3.2' 

lib/galaxy/dependencies/pipfiles/flake8_imports/pinned-requirements.txt

@@ -0,0 +1,7 @@
+mccabe==0.6.1   
+pycodestyle==2.3.1   
+pyflakes==1.6.0   
+enum34==1.1.6; python_version < '3.4'       
+flake8-import-order==0.14   
+flake8==3.5.0   
+configparser==3.5.0; python_version < '3.2' 

lib/galaxy/dependencies/pipfiles/update.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+commit=0
+
+usage() {
+cat << EOF
+Usage: ${0##*/} [-c] 
+
+Use pipenv to regenerate locked and hashed versions of Galaxy dependencies. Use -c to automatically commit these changes.
+
+EOF
+}
+
+while getopts ":i" opt; do
+    case "$opt" in
+        h)
+            usage
+            exit 0
+            ;;
+        i)
+            commit=1
+            ;;
+        '?')
+            usage >&2
+            exit 1
+            ;;
+    esac
+done
+
+THIS_DIRECTORY="$(dirname "$0")"
+cd "$THIS_DIRECTORY"
+ENVS=(develop flake8 flake8_imports)
+
+for env in "${ENVS[@]}"
+do
+        cd "$env"
+        pipenv lock
+        pipenv lock -r > pinned-hashed-requirements.txt
+        sed 's/--hash[^[:space:]]*//g' pinned-hashed-requirements.txt > pinned-requirements.txt
+        cd ".."
+done
+
+if [ "$commit" -eq "1" ];
+then
+	git add -u .
+	git commit -m "Rev and re-lock Galaxy dependencies."
+fi

tox.ini

@@ -5,23 +5,23 @@ skipsdist = True
 [testenv:py27-lint]
 commands = bash .ci/flake8_wrapper.sh
 whitelist_externals = bash
-deps = -r.ci/tox_envs/flake8/requirements.txt
+deps = -rlib/galaxy/dependencies/pipfiles/flake8/requirements.txt
 
 [testenv:py33-lint]
 commands = bash .ci/flake8_wrapper.sh
 whitelist_externals = bash
-deps = -r.ci/tox_envs/flake8/requirements.txt
+deps = -rlib/galaxy/dependencies/pipfiles/flake8/requirements.txt
 
 
 [testenv:py34-lint]
 commands = bash .ci/flake8_wrapper.sh
 whitelist_externals = bash
-deps = -r.ci/tox_envs/flake8/requirements.txt
+deps = -rlib/galaxy/dependencies/pipfiles/flake8/requirements.txt
 
 [testenv:py35-lint]
 commands = bash .ci/flake8_wrapper.sh
 whitelist_externals = bash
-deps = -r.ci/tox_envs/flake8/requirements.txt
+deps = -rlib/galaxy/dependencies/pipfiles/flake8/requirements.txt
 
 [testenv:py27-unit]
 commands = bash run_tests.sh --no-create-venv -u
@@ -41,13 +41,13 @@ deps =
 commands = bash .ci/flake8_wrapper.sh
 whitelist_externals = bash
 skip_install = True
-deps = -r.ci/tox_envs/flake8_imports/requirements.txt
+deps = -rlib/galaxy/dependencies/pipfiles/flake8_imports/requirements.txt
 
 [testenv:py27-lint-imports-include-list]
 commands = bash .ci/flake8_wrapper_imports.sh
 whitelist_externals = bash
 skip_install = True
-deps = -r.ci/tox_envs/flake8_imports/requirements.txt
+deps = -rlib/galaxy/dependencies/pipfiles/flake8_imports/requirements.txt
 
 [testenv:qunit]
 commands = bash run_tests.sh -q
@@ -73,13 +73,13 @@ whitelist_externals = bash
 commands = bash .ci/flake8_wrapper_docstrings.sh --exclude
 whitelist_externals = bash
 skip_install = True
-deps = -r.ci/tox_envs/flake8/requirements.txt
+deps = -rlib/galaxy/dependencies/pipfiles/flake8/requirements.txt
 
 [testenv:py27-lint-docstring-include-list]
 commands = bash .ci/flake8_wrapper_docstrings.sh --include
 whitelist_externals = bash
 skip_install = True
-deps = -r.ci/tox_envs/flake8/requirements.txt
+deps = -rlib/galaxy/dependencies/pipfiles/flake8/requirements.txt
 
 [testenv:check-python-dependencies]
 commands = make list-dependency-updates # someday change exit code on this.