Skip to content

Commit

Permalink
Improve Github action workflows (#7651)
Browse files Browse the repository at this point in the history 
* Remove restore keys
* Log pylint + astroid versions
* Reset cache versions
* Add check-latest to setup-python
* Use pyproject.toml for hash
* Update comment-hider version comment
* Pin additional actions
  • Loading branch information
@cdce8p @Pierre-Sassoulas
cdce8p authored and Pierre-Sassoulas committed Nov 17, 2022
1 parent 8b0bb16 commit d56f848
Show file tree
Hide file tree
Showing 10 changed files with 153 additions and 129 deletions.
24 changes: 13 additions & 11 deletions .github/workflows/changelog.yml
View file
Expand Up @@ -5,9 +5,12 @@ on:
types: [opened, synchronize, labeled, unlabeled, reopened]

env:
# Also change CACHE_VERSION in the other workflows
CACHE_VERSION: 31
DEFAULT_PYTHON: "3.10"
CACHE_VERSION: 1
KEY_PREFIX: base-venv
DEFAULT_PYTHON: "3.11"

permissions:
contents: read

jobs:
check-changelog:
Expand All @@ -17,32 +20,31 @@ jobs:
timeout-minutes: 10
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.0.2
uses: actions/checkout@v3.1.0
with:
# `towncrier check` runs `git diff --name-only origin/main...`, which
# needs a non-shallow clone.
fetch-depth: 0
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
id: python
uses: actions/setup-python@v4.2.0
uses: actions/setup-python@v4.3.0
with:
python-version: ${{ env.DEFAULT_PYTHON }}
check-latest: true
- name: Generate partial Python venv restore key
id: generate-python-key
run: >-
echo "::set-output name=key::base-venv-${{ env.CACHE_VERSION }}-${{
hashFiles('setup.cfg', 'requirements_test.txt', 'requirements_test_min.txt')
}}"
echo "key=${{ env.KEY_PREFIX }}-${{ env.CACHE_VERSION }}-${{
hashFiles('pyproject.toml', 'requirements_test.txt',
'requirements_test_min.txt') }}" >> $GITHUB_OUTPUT
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@v3.0.8
uses: actions/cache@v3.0.11
with:
path: venv
key: >-
${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
steps.generate-python-key.outputs.key }}
restore-keys: |
${{ runner.os }}-${{ steps.python.outputs.python-version }}-base-venv-${{ env.CACHE_VERSION }}-
- name: Create Python virtual environment
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
Expand Down
55 changes: 30 additions & 25 deletions .github/workflows/checks.yaml
View file
Expand Up @@ -8,14 +8,18 @@ on:
pull_request: ~

env:
CACHE_VERSION: 31
DEFAULT_PYTHON: "3.10"
CACHE_VERSION: 1
KEY_PREFIX: base-venv
DEFAULT_PYTHON: "3.11"
PRE_COMMIT_CACHE: ~/.cache/pre-commit

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

permissions:
contents: read

jobs:
prepare-base:
name: Prepare base dependencies
Expand All @@ -26,28 +30,27 @@ jobs:
pre-commit-key: ${{ steps.generate-pre-commit-key.outputs.key }}
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.0.2
uses: actions/checkout@v3.1.0
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
id: python
uses: actions/setup-python@v4.2.0
uses: actions/setup-python@v4.3.0
with:
python-version: ${{ env.DEFAULT_PYTHON }}
check-latest: true
- name: Generate partial Python venv restore key
id: generate-python-key
run: >-
echo "::set-output name=key::base-venv-${{ env.CACHE_VERSION }}-${{
hashFiles('setup.cfg', 'requirements_test.txt', 'requirements_test_min.txt')
}}"
echo "key=${{ env.KEY_PREFIX }}-${{ env.CACHE_VERSION }}-${{
hashFiles('pyproject.toml', 'requirements_test.txt',
'requirements_test_min.txt') }}" >> $GITHUB_OUTPUT
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@v3.0.8
uses: actions/cache@v3.0.11
with:
path: venv
key: >-
${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
steps.generate-python-key.outputs.key }}
restore-keys: |
${{ runner.os }}-${{ steps.python.outputs.python-version }}-base-venv-${{ env.CACHE_VERSION }}-
- name: Create Python virtual environment
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
Expand All @@ -59,17 +62,15 @@ jobs:
- name: Generate pre-commit restore key
id: generate-pre-commit-key
run: >-
echo "::set-output name=key::pre-commit-${{ env.CACHE_VERSION }}-${{
hashFiles('.pre-commit-config.yaml') }}"
echo "key=pre-commit-${{ env.CACHE_VERSION }}-${{
hashFiles('.pre-commit-config.yaml') }}" >> $GITHUB_OUTPUT
- name: Restore pre-commit environment
id: cache-precommit
uses: actions/cache@v3.0.8
uses: actions/cache@v3.0.11
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: >-
${{ runner.os }}-${{ steps.generate-pre-commit-key.outputs.key }}
restore-keys: |
${{ runner.os }}-pre-commit-${{ env.CACHE_VERSION }}-
- name: Install pre-commit dependencies
if: steps.cache-precommit.outputs.cache-hit != 'true'
run: |
Expand All @@ -83,15 +84,16 @@ jobs:
needs: prepare-base
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.0.2
uses: actions/checkout@v3.1.0
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
id: python
uses: actions/setup-python@v4.2.0
uses: actions/setup-python@v4.3.0
with:
python-version: ${{ env.DEFAULT_PYTHON }}
check-latest: true
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@v3.0.8
uses: actions/cache@v3.0.11
with:
path: venv
key:
Expand All @@ -104,7 +106,7 @@ jobs:
exit 1
- name: Restore pre-commit environment
id: cache-precommit
uses: actions/cache@v3.0.8
uses: actions/cache@v3.0.11
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ runner.os }}-${{ needs.prepare-base.outputs.pre-commit-key }}
Expand All @@ -121,6 +123,7 @@ jobs:
run: |
. venv/bin/activate
pip install -e .
pip list | grep 'astroid\|pylint'
pre-commit run --hook-stage manual pylint-with-spelling --all-files
spelling:
Expand All @@ -130,15 +133,16 @@ jobs:
needs: prepare-base
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.0.2
uses: actions/checkout@v3.1.0
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
id: python
uses: actions/setup-python@v4.2.0
uses: actions/setup-python@v4.3.0
with:
python-version: ${{ env.DEFAULT_PYTHON }}
check-latest: true
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@v3.0.8
uses: actions/cache@v3.0.11
with:
path: venv
key:
Expand All @@ -161,15 +165,16 @@ jobs:
needs: prepare-base
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.0.2
uses: actions/checkout@v3.1.0
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
id: python
uses: actions/setup-python@v4.2.0
uses: actions/setup-python@v4.3.0
with:
python-version: ${{ env.DEFAULT_PYTHON }}
check-latest: true
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@v3.0.8
uses: actions/cache@v3.0.11
with:
path: venv
key:
Expand Down
41 changes: 23 additions & 18 deletions .github/workflows/primer-test.yaml
View file
Expand Up @@ -13,46 +13,49 @@ on:
- ".github/workflows/primer-test.yaml"

env:
CACHE_VERSION: 31
CACHE_VERSION: 1
KEY_PREFIX: venv

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

permissions:
contents: read

jobs:
prepare-tests-linux:
name: prepare / ${{ matrix.python-version }} / Linux
runs-on: ubuntu-latest
timeout-minutes: 5
strategy:
matrix:
python-version: [3.8, 3.9, "3.10"]
python-version: [3.8, 3.9, "3.10", "3.11"]
outputs:
python-key: ${{ steps.generate-python-key.outputs.key }}
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.0.2
uses: actions/checkout@v3.1.0
- name: Set up Python ${{ matrix.python-version }}
id: python
uses: actions/setup-python@v4.2.0
uses: actions/setup-python@v4.3.0
with:
python-version: ${{ matrix.python-version }}
check-latest: true
- name: Generate partial Python venv restore key
id: generate-python-key
run: >-
echo "::set-output name=key::venv-${{ env.CACHE_VERSION }}-${{
hashFiles('setup.cfg', 'requirements_test.txt', 'requirements_test_min.txt')
}}"
echo "key=${{ env.KEY_PREFIX }}-${{ env.CACHE_VERSION }}-${{
hashFiles('pyproject.toml', 'requirements_test.txt',
'requirements_test_min.txt') }}" >> $GITHUB_OUTPUT
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@v3.0.8
uses: actions/cache@v3.0.11
with:
path: venv
key: >-
${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
steps.generate-python-key.outputs.key }}
restore-keys: |
${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ env.CACHE_VERSION }}-
- name: Create Python virtual environment
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
Expand All @@ -68,18 +71,19 @@ jobs:
needs: prepare-tests-linux
strategy:
matrix:
python-version: [3.8, 3.9, "3.10"]
python-version: [3.8, 3.9, "3.10", "3.11"]
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.0.2
uses: actions/checkout@v3.1.0
- name: Set up Python ${{ matrix.python-version }}
id: python
uses: actions/setup-python@v4.2.0
uses: actions/setup-python@v4.3.0
with:
python-version: ${{ matrix.python-version }}
check-latest: true
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@v3.0.8
uses: actions/cache@v3.0.11
with:
path: venv
key:
Expand All @@ -103,18 +107,19 @@ jobs:
needs: prepare-tests-linux
strategy:
matrix:
python-version: [3.8, 3.9, "3.10"]
python-version: [3.8, 3.9, "3.10", "3.11"]
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.0.2
uses: actions/checkout@v3.1.0
- name: Set up Python ${{ matrix.python-version }}
id: python
uses: actions/setup-python@v4.2.0
uses: actions/setup-python@v4.3.0
with:
python-version: ${{ matrix.python-version }}
check-latest: true
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@v3.0.8
uses: actions/cache@v3.0.11
with:
path: venv
key:
Expand Down
29 changes: 15 additions & 14 deletions .github/workflows/primer_comment.yaml
View file
Expand Up @@ -14,7 +14,9 @@ on:

env:
# This needs to be the SAME as in the Main and PR job
CACHE_VERSION: 31
CACHE_VERSION: 1
KEY_PREFIX: venv-primer
DEFAULT_PYTHON: "3.10"

permissions:
contents: read
Expand All @@ -27,35 +29,36 @@ jobs:
name: Run
runs-on: ubuntu-latest
steps:
- uses: actions/setup-node@v3
- uses: actions/setup-node@v3.5.1
with:
node-version: 16
- run: npm install @octokit/rest
- name: Check out code from GitHub
uses: actions/checkout@v3.0.2
- name: Set up Python 3.10
uses: actions/checkout@v3.1.0
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
id: python
uses: actions/setup-python@v4.2.0
uses: actions/setup-python@v4.3.0
with:
python-version: "3.10"
python-version: ${{ env.DEFAULT_PYTHON }}
check-latest: true

# Restore cached Python environment
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@v3.0.8
uses: actions/cache@v3.0.11
with:
path: venv
key:
${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-primer-${{
env.CACHE_VERSION }}
${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
env.KEY_PREFIX }}-${{ env.CACHE_VERSION }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python venv from cache"
exit 1
- name: Download outputs
uses: actions/github-script@v6
uses: actions/github-script@v6.3.3
with:
script: |
// Download workflow pylint output
Expand Down Expand Up @@ -110,9 +113,8 @@ jobs:
--new-file=output_${{ steps.python.outputs.python-version }}_pr.txt
- name: Post comment
id: post-comment
uses: actions/github-script@v6
uses: actions/github-script@v6.3.3
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const fs = require('fs')
const comment = fs.readFileSync('tests/.pylint_primer_tests/comment.txt', { encoding: 'utf8' })
Expand All @@ -128,8 +130,7 @@ jobs:
return prNumber
- name: Hide old comments
# Taken from mypy primer
# v0.3.0
uses: kanga333/comment-hider@bbdf5b562fbec24e6f60572d8f712017428b92e0
uses: kanga333/comment-hider@c12bb20b48aeb8fc098e35967de8d4f8018fffdf # v0.4.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
leave_visible: 1
Expand Down

0 comments on commit d56f848

Please sign in to comment.