ci: refactor dependency handling, update & pin all deps

This commit updates dependency handling in CI to be consistent with the
conventions emerging across various other python projects, specifically:

- py3 version is refreshed to 3.8
- pip-compile is used to pin all dependency versions in CI
- github actions is used to periodically update the versions

The motivation of doing this now is that some development crossing both
(pubtools-pulp, pubtools-pulplib) is otherwise difficult to handle
without breaking the CI of this project. Pinning dependencies ensures
stable CI.

.github/workflows/pip-compile-merge.yml

@@ -0,0 +1,23 @@
+on:
+  workflow_dispatch: {}
+  schedule:
+  - cron: "20 20 * * 2"
+
+name: "pip-compile: merge PR"
+jobs:
+ pip_compile_merge:
+   name: pip-compile automerge
+   runs-on: ubuntu-latest
+   steps:
+    - name: Look up pull request
+      uses: juliangruber/find-pull-request-action@v1
+      id: find-pull-request
+      with:
+        branch: deps/pip-compile
+    - name: Merge Pull Request
+      uses: juliangruber/merge-pull-request-action@v1
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        number: ${{ steps.find-pull-request.outputs.number }}
+        method: merge
+      if: ${{ steps.find-pull-request.outputs.number }}

.github/workflows/pip-compile.yml

@@ -0,0 +1,58 @@
+on:
+  workflow_dispatch: {}
+  schedule:
+  - cron: "20 20 * * 0"
+
+name: "pip-compile: create PR"
+jobs:
+ pip_compile:
+   name: pip-compile
+   runs-on: ubuntu-latest
+   steps:
+     - name: Setup Python
+       uses: actions/setup-python@v2
+       with:
+         python-version: "3.9"
+
+     - name: Install system dependencies
+       run: |
+         sudo apt-get update
+         sudo apt-get install -y libkrb5-dev
+
+     - name: Install tox
+       run: pip install tox
+
+     - name: pip-compile
+       uses: technote-space/create-pr-action@v2
+       with:
+         EXECUTE_COMMANDS: tox -e pip-compile
+         COMMIT_MESSAGE: 'chore: scheduled pip-compile'
+         COMMIT_NAME: 'GitHub Actions'
+         COMMIT_EMAIL: 'noreply@github.com'
+         GITHUB_TOKEN: ${{ secrets.PIP_COMPILE_TOKEN }}
+         PR_BRANCH_PREFIX: deps/
+         PR_BRANCH_NAME: 'pip-compile'
+         PR_TITLE: 'chore: scheduled pip-compile'
+         PR_BODY: '## Update dependencies
+
+  This is a scheduled update of Python dependencies within this repo managed by pip-compile.
+
+  This change will be submitted automatically within a few days if all checks have
+  succeeded.'
+
+
+     - name: Look up pull request
+       uses: juliangruber/find-pull-request-action@v1
+       id: find-pull-request
+       with:
+         branch: deps/pip-compile
+
+     # Immediately approve it if we can so it can be merged later
+     - name: Approve Pull Request
+       uses: juliangruber/approve-pull-request-action@v1
+       with:
+         github-token: ${{ secrets.APPROVAL_TOKEN }}
+         number: ${{ steps.find-pull-request.outputs.number }}
+       # allow errors because token may be unset.
+       continue-on-error: true
+       if: ${{ steps.find-pull-request.outputs.number }}

.travis.yml

@@ -9,11 +9,11 @@ matrix:
   include:
   - python: "2.7"
     env: TOX_ENV=py27
-  - python: "3.6"
+  - python: "3.8"
     env: TOX_ENV=static
-  - python: "3.6"
+  - python: "3.8"
     env: TOX_ENV=cov-travis DEPLOY=1
-  - python: "3.6"
+  - python: "3.8"
     env: TOX_ENV=docs
 script: tox -e $TOX_ENV
 after_success:

MANIFEST.in

@@ -1,4 +1,4 @@
 include README.md
 include CHANGELOG.md
 include LICENSE
-include requirements.txt
+include requirements.in

requirements.in

@@ -0,0 +1,6 @@
+six
+pubtools-pulplib>=2.8.0
+fastpurge
+more_executors>=2.7.0
+pushcollector>=1.2.0
+pushsource

setup.py

@@ -15,7 +15,7 @@ def get_long_description():
 
 
 def get_requirements():
-    with open("requirements.txt") as f:
+    with open("requirements.in") as f:
         return f.read().splitlines()
 
 

test-requirements.in

@@ -0,0 +1,14 @@
+pytest
+pytest-cov
+coveralls
+mock
+requests_mock
+rpm-py-installer
+
+# The following deps are not relevant for the legacy test env.
+black; python_version>='3'
+pre-commit; python_version>='3'
+pylint; python_version>='3'
+sphinx; python_version>='3'
+sphinx-argparse; python_version>='3'
+alabaster; python_version>='3'

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,py36,static,docs
+envlist = py27,py38,static,docs
 
 [testenv]
 deps=-rtest-requirements.txt
@@ -8,46 +8,46 @@ whitelist_externals=sh
 
 [testenv:py27]
 deps=
-	# Note: we need to explicitly list requirements.txt here
+	# Note: we need to explicitly list requirements.in here
 	# so it's processed at the same time as the constraints file
-	-rrequirements.txt
-	-cconstraints-legacy.txt
-	-rtest-requirements.txt
+	-rrequirements.in
+	-clegacy.constraints
+	-rtest-requirements.in
 
 [testenv:static]
 deps=
 	-rtest-requirements.txt
-	pylint==2.7.2
 commands=
 	sh -c 'pylint pubtools; test $(( $? & (1|2|4|32) )) = 0'
 
 [testenv:cov]
-deps=
-	-rtest-requirements.txt
-	pytest-cov
 usedevelop=true
 commands=
 	pytest --cov-report=html --cov=pubtools {posargs}
 
 [testenv:cov-travis]
 passenv = TRAVIS TRAVIS_*
-deps=
-	-rtest-requirements.txt
-	pytest-cov
-	coveralls
 usedevelop=true
 commands=
 	pytest --cov=pubtools {posargs}
 	coveralls
 
 [testenv:docs]
-deps=
-	sphinx
-	sphinx-argparse
-	alabaster
 use_develop=true
 commands=
 	sphinx-build -M html docs docs/_build
 
 [pytest]
 testpaths = tests
+
+[testenv:pip-compile]
+# Recompile all requirements .txt files using pip-compile.
+# Don't edit me - I'm deployed from a template.
+deps = pip-tools
+basepython = python3.8
+skip_install = true
+skipsdist = true
+commands =
+    pip-compile -U --generate-hashes requirements.in
+    pip-compile -U --generate-hashes requirements.in test-requirements.in -o test-requirements.txt
+# end pip-compile