Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Absolute pins on linting dependencies. #4876

Merged
merged 4 commits into from Oct 31, 2017
Merged

Absolute pins on linting dependencies. #4876

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ccb5f61
Absolute pins on linting dependencies.
jmchilton Oct 26, 2017
b25de8c
Fix bashism (spotted by @nsoranzo).
jmchilton Oct 27, 2017
04dc147
Fix more bashims
nsoranzo Oct 31, 2017
0ad92e8
Rev and re-lock linting dependencies.
nsoranzo Oct 31, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
14 changes: 14 additions & 0 deletions .ci/tox_envs/flake8/Pipfile
View file
Open in desktop
@@ -0,0 +1,14 @@
[[source]]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For future reference these files can be recreated with:

$ rm -rf .ci/tox_envs/flake8
$ mkdir -p .ci/tox_envs/flake8
$ cd .ci/tox_envs/flake8/
$ pipenv --rm
$ pipenv install flake8
$ pipenv lock
$ pipenv lock -r > requirements.txt


url = "https://pypi.python.org/simple"
verify_ssl = true
name = "pypi"


[dev-packages]



[packages]

"flake8" = "*"
6 changes: 6 additions & 0 deletions .ci/tox_envs/flake8/requirements.txt
View file
Open in desktop
@@ -0,0 +1,6 @@
mccabe==0.6.1 --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f
pycodestyle==2.3.1 --hash=sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9 --hash=sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766
pyflakes==1.6.0 --hash=sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f --hash=sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805
enum34==1.1.6; python_version < '3.4' --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850
flake8==3.5.0 --hash=sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37 --hash=sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0
configparser==3.5.0; python_version < '3.2' --hash=sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a
15 changes: 15 additions & 0 deletions .ci/tox_envs/flake8_imports/Pipfile
View file
Open in desktop
@@ -0,0 +1,15 @@
[[source]]

url = "https://pypi.python.org/simple"
verify_ssl = true
name = "pypi"


[dev-packages]



[packages]

"flake8" = "*"
"flake8-import-order" = "*"
7 changes: 7 additions & 0 deletions .ci/tox_envs/flake8_imports/requirements.txt
View file
Open in desktop
@@ -0,0 +1,7 @@
mccabe==0.6.1 --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f
pycodestyle==2.3.1 --hash=sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9 --hash=sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766
pyflakes==1.6.0 --hash=sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f --hash=sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805
enum34==1.1.6; python_version < '3.4' --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850
flake8-import-order==0.14 --hash=sha256:69aa93a5bdb526310a0bd994e3603122673d1103d8063d3505e45118005044b3 --hash=sha256:77271feabb17d7cc286e9156531bb94781fdf2bbc1690e2fd6fa2776580b2209
flake8==3.5.0 --hash=sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37 --hash=sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0
configparser==3.5.0; python_version < '3.2' --hash=sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a
14 changes: 14 additions & 0 deletions .ci/tox_envs/update.sh
View file
Open in desktop
@@ -0,0 +1,14 @@
#!/bin/sh

THIS_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

THIS_DIRECTORY="$(dirname "$0")" should be enough here and without bashisms.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually I think that was needed. Your variant gives me a relative path which I can't use twice. I've reworked the logic a bit in #4891's version of this file (c3776ef#diff-95de76b45ef92f3f93bdb163d284571e) so this isn't a problem though.

Copy link
Member

@nsoranzo nsoranzo Oct 27, 2017 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right, but a mix of the 2 should work:

THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)"

ENVS=(flake8 flake8_imports)

for env in "${ENVS[@]}"
do
cd "$THIS_DIRECTORY/$env"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For what I can see, this isn't actually updating the requirements, just creating Pipfile and requirements.txt.
Should there be a:

pipenv update

here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was imaging more of less just keeping the Pipfile un-pinned except when it is absolutely needed (e.g. Twill, pysam, etc...) - the pinning would happen in the requirements and lock file creation here. Re-generating the lock file rebuilds a Pipfile.lock (and associated requirements files) with the newest compatible versions of the libraries in the Pipfile. If we generated new lock and requirements files and we discover they are incompatible during testing - we add version restrictions in the Pipfile and then re-run and generate the appropriate lock and requirement files.

I can't find good documentation on pipenv update and it doesn't work for me locally. I think it is for installing newer updates to the dependencies not for updating the Pipfile right? Even if it was for updating the Pipfile, I think for the above workflow it would be bad to update the locked dependencies automatically since we are only locking things in the Pipfile if we know we need to lock them.

I did notice that the Pipfile.lock files weren't added since that pattern is our .gitignore. I've added them to #4903 but I can back port them here too - but since we are still just using the generate requirements files with pip - it doesn't seem important to track the lock files at this time.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the explanation, I assumed that pipenv lock would create the lock file according to the versions installed in the virtualenv, not also update packages if they're not pinned.
It seems the documentation needs to be clearer on both commands.

pipenv lock
pipenv lock -r > requirements.txt
done

git add -u "$THIS_DIRECTORY"
git commit -m "Rev and re-lock linting dependencies."
3 changes: 3 additions & 0 deletions Makefile
View file
Open in desktop
Expand Up @@ -119,6 +119,9 @@ release-check-blocking-prs: ## Check github for release blocking PRs
release-bootstrap-history: ## bootstrap history for a new release
$(IN_VENV) python scripts/bootstrap_history.py --release $(RELEASE_CURR)

update-linting-requirements: ## update linting dependencies
sh .ci/tox_envs/update.sh

node-deps: ## Install NodeJS dependencies.
cd client && yarn install --check-files

Expand Down
1 change: 0 additions & 1 deletion scripts/bootstrap_history.py
View file
Open in desktop
Expand Up @@ -160,7 +160,6 @@
make release-create-rc RELEASE_CURR=${version} RELEASE_NEXT=${next_version}

- [ ] Open PRs from your fork of branch ``version-${version}`` to upstream ``release_${version}`` and of ``version-${next_version}.dev`` to ``dev``.
- [ ] Open PR against ``release_${version}`` branch to pin flake8 deps in tox.ini to the latest available version. See [example](https://github.com/galaxyproject/galaxy/pull/3476).
- [ ] Update ``next_milestone`` in [P4's configuration](https://github.com/galaxyproject/p4) to `${next_version}` so it properly tags new PRs.
- [ ] Set the ``release_${version}`` branch in GitHub [settings](https://github.com/galaxyproject/galaxy/settings/branches) as protected.

Expand Down
30 changes: 9 additions & 21 deletions tox.ini
View file
Open in desktop
Expand Up @@ -5,25 +5,23 @@ skipsdist = True
[testenv:py27-lint]
commands = bash .ci/flake8_wrapper.sh
whitelist_externals = bash
deps =
flake8>=3.4.1
flake8-docstrings>=1.1.0
pydocstyle>=2.1.1
deps = -r.ci/tox_envs/flake8/requirements.txt

[testenv:py33-lint]
commands = bash .ci/flake8_wrapper.sh
whitelist_externals = bash
deps = flake8>=3.4.1
deps = -r.ci/tox_envs/flake8/requirements.txt


[testenv:py34-lint]
commands = bash .ci/flake8_wrapper.sh
whitelist_externals = bash
deps = flake8>=3.4.1
deps = -r.ci/tox_envs/flake8/requirements.txt

[testenv:py35-lint]
commands = bash .ci/flake8_wrapper.sh
whitelist_externals = bash
deps = flake8>=3.4.1
deps = -r.ci/tox_envs/flake8/requirements.txt

[testenv:py27-unit]
commands = bash run_tests.sh --no-create-venv -u
Expand All @@ -43,17 +41,13 @@ deps =
commands = bash .ci/flake8_wrapper.sh
whitelist_externals = bash
skip_install = True
deps =
flake8>=3.4.1
flake8-import-order>=0.14
deps = -r.ci/tox_envs/flake8_imports/requirements.txt

[testenv:py27-lint-imports-include-list]
commands = bash .ci/flake8_wrapper_imports.sh
whitelist_externals = bash
skip_install = True
deps =
flake8>=3.4.1
flake8-import-order>=0.14
deps = -r.ci/tox_envs/flake8_imports/requirements.txt

[testenv:qunit]
commands = bash run_tests.sh -q
Expand All @@ -79,19 +73,13 @@ whitelist_externals = bash
commands = bash .ci/flake8_wrapper_docstrings.sh --exclude
whitelist_externals = bash
skip_install = True
deps =
flake8>=3.4.1
flake8-docstrings>=1.1.0
pydocstyle>=2.1.1
deps = -r.ci/tox_envs/flake8/requirements.txt

[testenv:py27-lint-docstring-include-list]
commands = bash .ci/flake8_wrapper_docstrings.sh --include
whitelist_externals = bash
skip_install = True
deps =
flake8>=3.4.1
flake8-docstrings>=1.1.0
pydocstyle>=2.1.1
deps = -r.ci/tox_envs/flake8/requirements.txt

[testenv:check-python-dependencies]
commands = make list-dependency-updates # someday change exit code on this.
Expand Down