Skip to content

Commit

Permalink
ci: Parallelize and speed up using workflows
Browse files Browse the repository at this point in the history
- Add support to Python 3.9
- Speed-up similar builds with ccache
- Remove tox.ini file and tox dependencies
  • Loading branch information
Synss committed Mar 6, 2020
1 parent b53e874 commit d05a09b
Show file tree
Hide file tree
Showing 10 changed files with 160 additions and 222 deletions.
238 changes: 132 additions & 106 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -1,135 +1,161 @@
# Python CircleCI 2.0 configuration file
# vi: et:ts=2:sw=2
#
# Check https://circleci.com/docs/2.0/language-python/ for more details
#
version: 2

version: 2.0

jobs:
build:
python_27:
docker:
- image: circleci/python:3-stretch
- image: circleci/python:2.7.17
environment: &env
VERSION: 2.16.5
CC: /usr/lib/ccache/cc
C_INCLUDE_PATH: /usr/local/include:/usr/include
LD_LIBRARY_PATH: /usr/local/lib:/usr/lib
working_directory: ~/python-mbedtls
steps:
steps: &steps
- checkout

- run:
name: Downgrade openssl for Python 3.4
- run: &deps
name: Install dependencies
command: |
sudo apt-get update
sudo apt-get install libssl1.0-dev
- run:
name: Install pyenv
sudo apt-get -qq update
sudo apt-get -qq install ccache
echo "export PATH=/usr/lib/ccache:$PATH" >> $BASH_ENV
- run: &prepare_cache_versioning
name: Prepare cache versioning
command: python --version > checksum
- restore_cache: &restore_cache_ccache
keys:
- ccache--v0--{{ checksum "checksum" }}
- run: &mbedtls
name: Install mbedtls
command: |
git clone https://github.com/pyenv/pyenv.git $HOME/python-mbedtls/.pyenv
cat << 'EOF' >> $BASH_ENV
export PYENV_ROOT="$HOME/python-mbedtls/.pyenv"
export PATH="$PYENV_ROOT/bin:$PATH"
EOF
sudo ./scripts/download-mbedtls.sh $VERSION /usr/local/src
sudo ./scripts/install-mbedtls.sh /usr/local/src
- run: &install
name: Install library
command: |
python setup.py bdist_wheel
python -m pip install --user --upgrade dist/*.whl
- save_cache: &save_cache_ccache
key: ccache--v0--{{ checksum "checksum" }}--{{ .BuildNum }}
paths:
- /home/circleci/.ccache
- run: &tests
name: Run tests
command: |
python -m pip install --user -r requirements/tests.txt
python -m pytest
- store_artifacts: &artifacts
path: out
destination: artifacts

- restore_cache:
key: py27-2.7.16-v0-{{ arch }}
- restore_cache:
key: py34-3.4.10-v0-{{ arch }}
- restore_cache:
key: py35-3.5.9-v0-{{ arch }}
- restore_cache:
key: py36-3.6.10-v0-{{ arch }}
- restore_cache:
key: py37-3.7.6-v0-{{ arch }}
- restore_cache:
key: py38-3.8.1-v0-{{ arch }}
python_34:
docker:
- image: circleci/python:3.4.10
environment: *env
working_directory: ~/python-mbedtls
steps: *steps

python_35:
docker:
- image: circleci/python:3.5.9
environment: *env
working_directory: ~/python-mbedtls
steps: *steps

python_36:
docker:
- image: circleci/python:3.6.10
environment: *env
working_directory: ~/python-mbedtls
steps: *steps

python_37:
docker:
- image: circleci/python:3.7.6
environment: *env
working_directory: ~/python-mbedtls
steps: *steps

python_38:
docker:
- image: circleci/python:3.8.2
environment: *env
working_directory: ~/python-mbedtls
steps: *steps

python_39:
docker:
- image: circleci/python:3.9-rc
environment: *env
working_directory: ~/python-mbedtls
steps: *steps

python_extra:
docker:
- image: circleci/python:3.7.6
environment: *env
working_directory: ~/python-mbedtls
steps:
- checkout
- run: *deps
- run: *prepare_cache_versioning
- restore_cache:
keys:
- ccache--v0--{{ checksum "checksum" }}--cov
- run: *mbedtls
- run:
name: Install Python versions
name: Install library
command: |
eval "$(pyenv init -)"
test -d ".pyenv/versions/2.7.16" || pyenv install 2.7.16
test -d ".pyenv/versions/3.4.10" || pyenv install 3.4.10
test -d ".pyenv/versions/3.5.9" || pyenv install 3.5.9
test -d ".pyenv/versions/3.6.10" || pyenv install 3.6.10
test -d ".pyenv/versions/3.7.6" || pyenv install 3.7.6
test -d ".pyenv/versions/3.8.1" || pyenv install 3.8.1
- save_cache:
key: py27-2.7.16-v0-{{ arch }}
paths:
- .pyenv/versions/2.7.16
- save_cache:
key: py34-3.4.10-v0-{{ arch }}
paths:
- .pyenv/versions/3.4.10
- save_cache:
key: py35-3.5.9-v0-{{ arch }}
paths:
- .pyenv/versions/3.5.9
- save_cache:
key: py36-3.6.10-v0-{{ arch }}
paths:
- .pyenv/versions/3.6.10
- save_cache:
key: py37-3.7.6-v0-{{ arch }}
paths:
- .pyenv/versions/3.7.6
python setup.py --with-coverage build_ext --inplace
- save_cache:
key: py38-3.8.1-v0-{{ arch }}
key: ccache--v0--{{ checksum "checksum" }}--cov--{{ .BuildNum }}
paths:
- .pyenv/versions/3.8.1

- /home/circleci/.ccache
- run:
name: Install mbedtls
name: Run tests
command: |
VERSION=2.16.5
DESTDIR=$HOME/lib/$VERSION
sudo apt-get install cmake
sudo ./scripts/download-mbedtls.sh $VERSION $DESTDIR/src
sudo ./scripts/install-mbedtls.sh $DESTDIR/src $DESTDIR
cat << EOF >> $BASH_ENV
export VERSION=$VERSION
export DESTDIR=$DESTDIR
export LD_LIBRARY_PATH=$DESTDIR/lib
export C_INCLUDE_PATH=$DESTDIR/include
export TOX_PARALLEL_NO_SPINNER=1
EOF
python -m pip install --user -r requirements/tests.txt
python -m pip install --user cython pytest-black pytest-cov
PYTHONPATH=$(pwd)/src/:$PYTHONPATH
export PYTHONPATH
python -m pytest --black --cov=src/ tests/
- run:
name: Create environment
name: Coverage
command: |
python -m venv venv
. venv/bin/activate
pip install -r requirements/circleci.txt
python -m pip install --user coveralls
PYTHONPATH=$(pwd)/src/:$PYTHONPATH
export PYTHONPATH
python -m coveralls
- run:
name: Run tests
name: Run doctest
command: |
eval "$(pyenv init -)"
pyenv shell 2.7.16 3.4.10 3.5.9 3.6.10 3.7.6 3.8.1
. venv/bin/activate
tox --parallel auto -e py27,py34,py35,py36,py37-coveralls,py38
tox -e py37 --run-command "{envpython} -m doctest {toxinidir}/README.rst"
PYTHONPATH=$(pwd)/src/:$PYTHONPATH
export PYTHONPATH
python -m doctest README.rst
- run:
name: Build API documentation
command: |
eval "$(pyenv init -)"
pyenv shell 3.7.6
. venv/bin/activate
tox -e py37-docs
python -m pip install --user sphinx sphinx_rtd_theme
sphinx-build docs/source docs_out/ --color -W -bhtml
- run:
name: Save docs
when: on_success
command: |
mkdir -p out/
cp -R .tox/docs_out out/docs
- run:
name: Save logs
when: on_fail
command: |
mkdir -p out/log
cp .tox/*/log/py*.log out/log || true
cp -R docs_out out/docs
- store_artifacts: *artifacts

- store_artifacts:
path: out
destination: artifacts
workflows:
version: 2
build:
jobs:
- python_extra
- python_39
- python_38
- python_37
- python_36
- python_35
- python_34
- python_27
4 changes: 4 additions & 0 deletions ChangeLog
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
[next]

* tls: Support Python 3.9

[1.1.0] - 2020-02-22

* tls: Add support to PSK for (D)TLS
Expand Down
2 changes: 0 additions & 2 deletions requirements/circleci.in

This file was deleted.

20 changes: 0 additions & 20 deletions requirements/circleci.txt

This file was deleted.

5 changes: 0 additions & 5 deletions requirements/tests.in
Original file line number Diff line number Diff line change
@@ -1,12 +1,7 @@
coverage < 5.0 # For Python 3.4
cython
pytest < 5.0.0 # For Python < 3.5
pytest-cov
pytest-repeat
readme_renderer

docutils < 0.16 # via readme_renderer; for Python 3.4
importlib-metadata < 1.2.0 # via pluggy, pytest; for Python 3.4
more-itertools < 6.0.0 # via pytest, zipp; for Python < 3.5
pygments < 2.4.0 # via readme-renderer; for Python 3.4
zipp < 2.0.0 # via importlib-metadata; for Python < 3.6
18 changes: 7 additions & 11 deletions requirements/tests.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,17 @@
atomicwrites==1.3.0 # via pytest
attrs==19.3.0 # via pytest
bleach==3.1.1 # via readme-renderer
coverage==4.5.4 # via -r requirements/tests.in (line 1), pytest-cov
cython==0.29.15 # via -r requirements/tests.in (line 2)
docutils==0.15.2 # via -r requirements/tests.in (line 8), readme-renderer
importlib-metadata==1.1.3 # via -r requirements/tests.in (line 9), pluggy, pytest
more-itertools==5.0.0 # via -r requirements/tests.in (line 10), pytest
docutils==0.15.2 # via -r requirements/tests.in, readme-renderer
importlib-metadata==1.1.3 # via -r requirements/tests.in
more-itertools==5.0.0 # via -r requirements/tests.in, pytest
packaging==20.1 # via pytest
pluggy==0.13.1 # via pytest
py==1.8.1 # via pytest
pygments==2.3.1 # via -r requirements/tests.in (line 11), readme-renderer
pygments==2.3.1 # via readme-renderer
pyparsing==2.4.6 # via packaging
pytest-cov==2.8.1 # via -r requirements/tests.in (line 4)
pytest-repeat==0.8.0 # via -r requirements/tests.in (line 5)
pytest==4.6.9 # via -r requirements/tests.in (line 3), pytest-cov, pytest-repeat
readme-renderer==24.0 # via -r requirements/tests.in (line 6)
pytest==4.6.9 # via -r requirements/tests.in
readme-renderer==24.0 # via -r requirements/tests.in
six==1.14.0 # via bleach, more-itertools, packaging, pytest, readme-renderer
wcwidth==0.1.8 # via pytest
webencodings==0.5.1 # via bleach
zipp==1.2.0 # via -r requirements/tests.in (line 12), importlib-metadata
zipp==1.2.0 # via -r requirements/tests.in, importlib-metadata
6 changes: 3 additions & 3 deletions scripts/download-mbedtls.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@

set -ex

if [ $# -eq 1 ] || [ $# -eq 2 ]; then
if [ $# -eq 2 ]; then
version="$1"
destdir="${2:-/usr/local/src}"
destdir="$2"
case $destdir in
/*) ;;
*) destdir="$PWD/$destdir";;
Expand All @@ -14,7 +14,7 @@ else
cat <<-EOF
usage:
$0 VERSION [DESTDIR]
$0 VERSION DESTDIR
Download a local copy mbedtls at VERSION.
Expand Down
18 changes: 9 additions & 9 deletions scripts/install-mbedtls.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,23 @@

set -ex

if [ $# -le 1 ]; then
if [ $# -eq 1 ] || [ $# -eq 2 ]; then
srcdir="$1"
destdir="${2:-/usr/local}"
case $destdir in
/*) ;;
*) destdir="$PWD/$destdir";;
esac
else
cat <<-EOF
usage:
$0 SRCDIR [DESTDIR]
Install a mbedtls from the sources in SRCDIR to DESTDIR.
EOF
exit 1
else
srcdir="$1"
destdir="${2:-/usr/local}"
case $destdir in
/*) ;;
*) destdir="$PWD/$destdir";;
esac
fi


Expand Down

0 comments on commit d05a09b

Please sign in to comment.