Merge pull request #4544 from rmol/4519-py3-dev

Make Python 3 the default for development
freedomofpress · Jul 2, 2019 · 1c2a437 · 1c2a437
2 parents 3e128d2 + f15b2b1
commit 1c2a437
Show file tree

Hide file tree

Showing 103 changed files with 1,504 additions and 741 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -5,6 +5,12 @@ common-steps:
       name: Rebase on top of GitHub target branch
       command: ./devops/scripts/rebase-ci.sh
 
+  - &installenchant
+    run:
+      name: Install enchant
+      command: sudo apt-get install enchant
+
+  # Python 3 (default) Docker layer caching
   - &createcachedir
     run:
       name: Ensure cache dir exists and permissions are good
@@ -13,7 +19,7 @@ common-steps:
 
   - &restorecache
     restore_cache:
-      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python2/Dockerfile" }}
+      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
       paths:
         - /caches/layers.tar.gz
 
@@ -22,58 +28,104 @@ common-steps:
       name: Load image layer cache
       command: |
         set +o pipefail
-        docker load -i /caches/layers.tar |true
+        docker load -i /caches/layers.tar || true
 
   - &dockerimagebuild
     run:
       name: Build Docker images
       command: |
         set +o pipefail
         docker images
-        fromtag=$(docker images |grep securedrop-test-xenial-py2 |head -n1 |awk '{print $2}')
-        cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" ./bin/dev-shell true
+        fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
+        cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" ./bin/dev-shell true
 
   - &saveimagelayers
     run:
       name: Save Docker image layer cache
       command: |
         docker images
-        docker save -o /caches/layers.tar securedrop-test-xenial-py2:latest
+        docker save -o /caches/layers.tar securedrop-test-xenial-py3:latest
 
   - &savecache
     save_cache:
-      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python2/Dockerfile" }}
+      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
       paths:
         - /caches/layers.tar
 
-  - &installenchant
+  # Python 2 Docker layer caching
+  - &p2createcachedir
     run:
-      name: Install enchant
-      command: sudo apt-get install enchant
+      name: Ensure Python 2 cache dir exists and permissions are good
+      command: |
+        sudo mkdir -p /p2caches && sudo chown circleci: -R /p2caches
+
+  - &p2restorecache
+    restore_cache:
+      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python2/Dockerfile" }}
+      paths:
+        - /p2caches/layers.tar.gz
+
+  - &p2loadimagelayers
+    run:
+      name: Load Python 2 image layer cache
+      command: |
+        set +o pipefail
+        docker load -i /p2caches/layers.tar || true
+
+  - &p2dockerimagebuild
+    run:
+      name: Build Python 2 Docker images
+      command: |
+        set +o pipefail
+        docker images
+        fromtag=$(docker images |grep securedrop-test-xenial-py2 |head -n1 |awk '{print $2}')
+        cd securedrop && PYTHON_VERSION=2 DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" ./bin/dev-shell true
+
+  - &p2saveimagelayers
+    run:
+      name: Save Python 2 Docker image layer cache
+      command: |
+        docker images
+        docker save -o /p2caches/layers.tar securedrop-test-xenial-py2:latest
+
+  - &p2savecache
+    save_cache:
+      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python2/Dockerfile" }}
+      paths:
+        - /p2caches/layers.tar
 
 version: 2
 jobs:
   lint:
     machine:
       enabled: true
+    environment:
+      DOCKER_API_VERSION: 1.23
+      BASE_OS: xenial
+      PYTHON_VERSION: 3
     steps:
       - checkout
       - *rebaseontarget
+      - *createcachedir
+      - *restorecache
+      - *loadimagelayers
+      - *dockerimagebuild
+      - *saveimagelayers
+      - *savecache
       - *installenchant
 
       - run:
-          name: Install development dependencies
-          command: pip install -U -r securedrop/requirements/develop-requirements.txt
+          name: Run all linters but shellcheck
+          command: |
+            fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "sudo pip3 install -r requirements/python3/develop-requirements.txt && make -C .. ansible-config-lint app-lint docs-lint flake8 html-lint typelint yamllint"
 
       - run:
-          name: Run code linting
-          command: make lint
+          name: Run shellcheck
+          command: make shellcheck
 
-      - run:
-          name: Run documentation linting
-          command: make docs-lint
 
-  app-tests:
+  python2-app-tests:
     machine:
       enabled: true
     environment:
@@ -83,12 +135,12 @@ jobs:
     steps:
       - checkout
       - *rebaseontarget
-      - *createcachedir
-      - *restorecache
-      - *loadimagelayers
-      - *dockerimagebuild
-      - *saveimagelayers
-      - *savecache
+      - *p2createcachedir
+      - *p2restorecache
+      - *p2loadimagelayers
+      - *p2dockerimagebuild
+      - *p2saveimagelayers
+      - *p2savecache
 
       - run:
           name: Make test results directory
@@ -99,12 +151,10 @@ jobs:
           no_output_timeout: 20m
           command: |
             BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs|update-builder)")
-            echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
             if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
-            docker rm -f securedrop-test-xenial-py2 || true
             fromtag=$(docker images |grep securedrop-test-xenial-py2 |head -n1 |awk '{print $2}')
-            cd securedrop && DOCKER_RUN_ARGUMENTS=$(bash <(curl -s https://codecov.io/env)) DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" make test
+            PYTHON_VERSION=2 DOCKER_RUN_ARGUMENTS=$(bash <(curl -s https://codecov.io/env)) DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" make test
 
       - store_test_results:
           path: ~/test-results
@@ -124,32 +174,11 @@ jobs:
       - checkout
       - *rebaseontarget
       - *createcachedir
-
-      - restore_cache:
-          key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
-          paths:
-            - /caches/layers.tar.gz
-
+      - *restorecache
       - *loadimagelayers
-
-      - run:
-          name: Build Docker images
-          command: |
-            set +o pipefail
-            docker images
-            fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
-            cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" ./bin/dev-shell true
-
-      - run:
-          name: Save Docker image layer cache
-          command: |
-            docker images
-            docker save -o /caches/layers.tar securedrop-test-xenial-py3:latest
-
-      - save_cache:
-          key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
-          paths:
-            - /caches/layers.tar
+      - *dockerimagebuild
+      - *saveimagelayers
+      - *savecache
 
       - run:
           name: Make test results directory
@@ -159,12 +188,10 @@ jobs:
           name: Run tests
           command: |
             BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs|update-builder)")
-            echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
             if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
-            docker rm -f securedrop-test-xenial-py3 || true
             fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
-            cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make test
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make test
 
       - store_test_results:
           path: ~/test-results
@@ -197,12 +224,10 @@ jobs:
           name: Run tests
           command: |
             BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^i18n")
-            echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
             if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             export TESTFILES=$(cd securedrop; circleci tests glob 'tests/pageslayout/test*py' |circleci tests split --split-by=timings |xargs echo)
-            docker rm -f securedrop-test-xenial-py2 || true
-            fromtag=$(docker images |grep securedrop-test-xenial-py2 |head -n1 |awk '{print $2}')
-            cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" make translation-test
+            fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make translation-test
 
       - store_test_results:
           path: ~/test-results
@@ -218,23 +243,22 @@ jobs:
           name: Run tests
           command: |
             BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs|update-builder)")
-            echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
             if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             cd admin; make test
 
   fetch-tor-debs:
     docker:
       - image: gcr.io/cloud-builders/docker
     steps:
-      - run: apt-get install -y make virtualenv python-pip enchant jq
+      - run: apt-get install -y make virtualenv python3-pip enchant jq
       - checkout
       - setup_remote_docker
       - run:
           name: Fetch Tor packages
           command: |
             BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs|update-builder)")
-            echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
             if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
+            export LC_ALL="C.UTF-8"
             make fetch-tor-packages
 
   updater-gui-tests:
@@ -263,23 +287,39 @@ jobs:
           name: Run tests
           command: |
             BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs|update-builder)")
-            echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
             if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             cd journalist_gui
             xvfb-run -a pipenv run python3 test_gui.py
 
   static-analysis-and-no-known-cves:
     machine:
       enabled: true
+    environment:
+      DOCKER_API_VERSION: 1.23
+      BASE_OS: xenial
+      PYTHON_VERSION: 3
     steps:
       - checkout
+      - *rebaseontarget
+      - *createcachedir
+      - *restorecache
+      - *loadimagelayers
+      - *dockerimagebuild
+      - *saveimagelayers
+      - *savecache
+
       - run:
           name: Check Python dependencies for CVEs
-          command: make safety
+          command: |
+            fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "sudo pip3 install -q --upgrade safety && make -C .. safety"
 
       - run:
           name: Run static security testing on source code
-          command: make bandit
+          command: |
+            fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "sudo pip3 install -q --upgrade pip && sudo pip3 install -q --upgrade bandit && make -C .. bandit"
+
 
   staging-test-with-rebase:
     machine:
@@ -295,7 +335,6 @@ jobs:
           name: Run Staging tests on GCE
           command: |
             BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs)")
-            echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
             if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             make ci-go
           no_output_timeout: 20m
@@ -317,14 +356,13 @@ jobs:
     docker:
       - image: gcr.io/cloud-builders/docker
     steps:
-      - run: apt-get install -y make virtualenv python-pip enchant jq
+      - run: apt-get install -y make virtualenv enchant jq
       - checkout
       - setup_remote_docker
       - run:
           name: Test Debian package build
           command: |
             BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^update-builder")
-            echo "match-ci-branch.sh said: ${BRANCH_MATCH}"
             if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             make ci-deb-tests
 
@@ -349,7 +387,7 @@ workflows:
   securedrop_ci:
     jobs:
       - lint
-      - app-tests:
+      - python2-app-tests:
           filters:
             branches:
               ignore:

diff --git a/.flake8 b/.flake8
@@ -2,7 +2,8 @@
 max-line-length = 100
 exclude =
     config.py,
-    .venv/,
+    admin/.venv,
+    .venv*/,
     journalist_gui/journalist_gui/updaterUI.py,
     journalist_gui/journalist_gui/resources_rc.py,
     .python3,

diff --git a/.gitignore b/.gitignore
@@ -99,7 +99,7 @@ docs/_build/
 .env
 
 # virtualenv
-.venv/
+.venv*/
 venv/
 ENV/
 
@@ -140,9 +140,6 @@ junit/
 raw-test-output/
 *results.xml
 
-# Virtualenv on tails
-.venv
-
 # molecule
 .molecule
 

diff --git a/.yamllint b/.yamllint
@@ -2,5 +2,5 @@
 extends: relaxed
 rules:
   line-length:
-    max: 200
+    max: 220
     level: warning