Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Make Python 3 the default for development #4544

Merged
merged 20 commits into from
Jul 2, 2019
Merged

Make Python 3 the default for development #4544

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
ca5a9fa
Make Python 3 the default for development
rmol Jun 11, 2019
0d8623b
Lint/CI fixes
rmol Jun 20, 2019
e1b4b49
Python 2 Docker layer caching. Try to fix safety/bandit.
rmol Jun 20, 2019
1916072
Fix virtualenv problem in fetch-tor-packages
rmol Jun 20, 2019
4a0b295
Set LC_ALL=C.UTF-8 for fetch-tor-packages
rmol Jun 20, 2019
abd323b
Have to request Python 2 to get the Docker image named correctly
rmol Jun 20, 2019
f257b92
Stop removing Docker images
rmol Jun 20, 2019
64d1be1
Remove unnecessary mypy appeasement
rmol Jun 21, 2019
d2efeec
Stop pinning pip version in venv creation
rmol Jun 21, 2019
ec25495
Compile pip requirements in containers
rmol Jun 21, 2019
3b7249c
Remove another "set -u" that breaks in virtualenv
rmol Jun 21, 2019
9dbdbe2
Use Python 2 in "make staging" step of staging-test-with-rebase
rmol Jun 21, 2019
3aaf77f
Remove a couple more instances of "set -u" and virtualenv
rmol Jun 21, 2019
7ab9e18
Fix file mode assertions in testinfra
rmol Jun 24, 2019
c171171
Revert testinfra locale test to work in CI
rmol Jun 24, 2019
ee3e80b
Remove automatic virtualenv management
rmol Jun 25, 2019
804a43e
Make sure lint has development requirements in the container
rmol Jun 25, 2019
c18743c
Use pip3 to install safety/bandit
rmol Jun 25, 2019
26c08b8
Makefile: stop installing safety/bandit; improve test-config target.
rmol Jun 27, 2019
f15b2b1
Update developer docs for Python 3.
rmol Jun 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 106 additions & 68 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ common-steps:
name: Rebase on top of GitHub target branch
command: ./devops/scripts/rebase-ci.sh

- &installenchant
run:
name: Install enchant
command: sudo apt-get install enchant

# Python 3 (default) Docker layer caching
- &createcachedir
run:
name: Ensure cache dir exists and permissions are good
Expand All @@ -13,7 +19,7 @@ common-steps:

- &restorecache
restore_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python2/Dockerfile" }}
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
paths:
- /caches/layers.tar.gz

Expand All @@ -22,58 +28,104 @@ common-steps:
name: Load image layer cache
command: |
set +o pipefail
docker load -i /caches/layers.tar |true
docker load -i /caches/layers.tar || true

- &dockerimagebuild
run:
name: Build Docker images
command: |
set +o pipefail
docker images
fromtag=$(docker images |grep securedrop-test-xenial-py2 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" ./bin/dev-shell true
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" ./bin/dev-shell true

- &saveimagelayers
run:
name: Save Docker image layer cache
command: |
docker images
docker save -o /caches/layers.tar securedrop-test-xenial-py2:latest
docker save -o /caches/layers.tar securedrop-test-xenial-py3:latest

- &savecache
save_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python2/Dockerfile" }}
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
paths:
- /caches/layers.tar

- &installenchant
# Python 2 Docker layer caching
- &p2createcachedir
run:
name: Install enchant
command: sudo apt-get install enchant
name: Ensure Python 2 cache dir exists and permissions are good
command: |
sudo mkdir -p /p2caches && sudo chown circleci: -R /p2caches

- &p2restorecache
restore_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python2/Dockerfile" }}
paths:
- /p2caches/layers.tar.gz

- &p2loadimagelayers
run:
name: Load Python 2 image layer cache
command: |
set +o pipefail
docker load -i /p2caches/layers.tar || true

- &p2dockerimagebuild
run:
name: Build Python 2 Docker images
command: |
set +o pipefail
docker images
fromtag=$(docker images |grep securedrop-test-xenial-py2 |head -n1 |awk '{print $2}')
cd securedrop && PYTHON_VERSION=2 DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" ./bin/dev-shell true

- &p2saveimagelayers
run:
name: Save Python 2 Docker image layer cache
command: |
docker images
docker save -o /p2caches/layers.tar securedrop-test-xenial-py2:latest

- &p2savecache
save_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python2/Dockerfile" }}
paths:
- /p2caches/layers.tar

version: 2
jobs:
lint:
machine:
enabled: true
environment:
DOCKER_API_VERSION: 1.23
BASE_OS: xenial
PYTHON_VERSION: 3
steps:
- checkout
- *rebaseontarget
- *createcachedir
- *restorecache
- *loadimagelayers
- *dockerimagebuild
- *saveimagelayers
- *savecache
- *installenchant

- run:
name: Install development dependencies
command: pip install -U -r securedrop/requirements/develop-requirements.txt
name: Run all linters but shellcheck
command: |
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "sudo pip3 install -r requirements/python3/develop-requirements.txt && make -C .. ansible-config-lint app-lint docs-lint flake8 html-lint typelint yamllint"

- run:
name: Run code linting
command: make lint
name: Run shellcheck
command: make shellcheck

- run:
name: Run documentation linting
command: make docs-lint

app-tests:
python2-app-tests:
machine:
enabled: true
environment:
Expand All @@ -83,12 +135,12 @@ jobs:
steps:
- checkout
- *rebaseontarget
- *createcachedir
- *restorecache
- *loadimagelayers
- *dockerimagebuild
- *saveimagelayers
- *savecache
- *p2createcachedir
- *p2restorecache
- *p2loadimagelayers
- *p2dockerimagebuild
- *p2saveimagelayers
- *p2savecache

- run:
name: Make test results directory
Expand All @@ -99,12 +151,10 @@ jobs:
no_output_timeout: 20m
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs|update-builder)")
echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
docker rm -f securedrop-test-xenial-py2 || true
fromtag=$(docker images |grep securedrop-test-xenial-py2 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_RUN_ARGUMENTS=$(bash <(curl -s https://codecov.io/env)) DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" make test
PYTHON_VERSION=2 DOCKER_RUN_ARGUMENTS=$(bash <(curl -s https://codecov.io/env)) DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" make test
rmol marked this conversation as resolved.
Show resolved Hide resolved

- store_test_results:
path: ~/test-results
Expand All @@ -124,32 +174,11 @@ jobs:
- checkout
- *rebaseontarget
- *createcachedir

- restore_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
paths:
- /caches/layers.tar.gz

- *restorecache
- *loadimagelayers

- run:
name: Build Docker images
command: |
set +o pipefail
docker images
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" ./bin/dev-shell true

- run:
name: Save Docker image layer cache
command: |
docker images
docker save -o /caches/layers.tar securedrop-test-xenial-py3:latest

- save_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
paths:
- /caches/layers.tar
- *dockerimagebuild
- *saveimagelayers
- *savecache

- run:
name: Make test results directory
Expand All @@ -159,12 +188,10 @@ jobs:
name: Run tests
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs|update-builder)")
echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
docker rm -f securedrop-test-xenial-py3 || true
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make test
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make test

- store_test_results:
path: ~/test-results
Expand Down Expand Up @@ -197,12 +224,10 @@ jobs:
name: Run tests
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^i18n")
echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
export TESTFILES=$(cd securedrop; circleci tests glob 'tests/pageslayout/test*py' |circleci tests split --split-by=timings |xargs echo)
docker rm -f securedrop-test-xenial-py2 || true
fromtag=$(docker images |grep securedrop-test-xenial-py2 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" make translation-test
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make translation-test

- store_test_results:
path: ~/test-results
Expand All @@ -218,23 +243,22 @@ jobs:
name: Run tests
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs|update-builder)")
echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
cd admin; make test

fetch-tor-debs:
docker:
- image: gcr.io/cloud-builders/docker
steps:
- run: apt-get install -y make virtualenv python-pip enchant jq
- run: apt-get install -y make virtualenv python3-pip enchant jq
- checkout
- setup_remote_docker
- run:
name: Fetch Tor packages
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs|update-builder)")
echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
export LC_ALL="C.UTF-8"
make fetch-tor-packages

updater-gui-tests:
Expand Down Expand Up @@ -263,23 +287,39 @@ jobs:
name: Run tests
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs|update-builder)")
echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
cd journalist_gui
xvfb-run -a pipenv run python3 test_gui.py

static-analysis-and-no-known-cves:
machine:
enabled: true
environment:
DOCKER_API_VERSION: 1.23
BASE_OS: xenial
PYTHON_VERSION: 3
steps:
- checkout
- *rebaseontarget
- *createcachedir
- *restorecache
- *loadimagelayers
- *dockerimagebuild
- *saveimagelayers
- *savecache

- run:
name: Check Python dependencies for CVEs
command: make safety
command: |
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "sudo pip3 install -q --upgrade safety && make -C .. safety"

- run:
name: Run static security testing on source code
command: make bandit
command: |
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "sudo pip3 install -q --upgrade pip && sudo pip3 install -q --upgrade bandit && make -C .. bandit"


staging-test-with-rebase:
machine:
Expand All @@ -295,7 +335,6 @@ jobs:
name: Run Staging tests on GCE
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs)")
echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
make ci-go
no_output_timeout: 20m
Expand All @@ -317,14 +356,13 @@ jobs:
docker:
- image: gcr.io/cloud-builders/docker
steps:
- run: apt-get install -y make virtualenv python-pip enchant jq
- run: apt-get install -y make virtualenv enchant jq
- checkout
- setup_remote_docker
- run:
name: Test Debian package build
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^update-builder")
echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
make ci-deb-tests

Expand All @@ -349,7 +387,7 @@ workflows:
securedrop_ci:
jobs:
- lint
- app-tests:
- python2-app-tests:
filters:
branches:
ignore:
Expand Down
3 changes: 2 additions & 1 deletion .flake8
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
max-line-length = 100
exclude =
config.py,
.venv/,
admin/.venv,
.venv*/,
journalist_gui/journalist_gui/updaterUI.py,
journalist_gui/journalist_gui/resources_rc.py,
.python3,
Expand Down
5 changes: 1 addition & 4 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ docs/_build/
.env

# virtualenv
.venv/
.venv*/
venv/
ENV/

Expand Down Expand Up @@ -140,9 +140,6 @@ junit/
raw-test-output/
*results.xml

# Virtualenv on tails
.venv
redshiftzero marked this conversation as resolved.
Show resolved Hide resolved

# molecule
.molecule

Expand Down
2 changes: 1 addition & 1 deletion .yamllint
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
extends: relaxed
rules:
line-length:
max: 200
max: 220
level: warning
Loading