Skip to content

Commit

Permalink
Merge pull request #2344 from jkfran/python-tests
Browse files Browse the repository at this point in the history 
Update CircleCI and tests
  • Loading branch information
@nottrobin
nottrobin committed Nov 7, 2019
2 parents de39e24 + 706bd02 commit c5ec769
Show file tree
Hide file tree
Showing 3 changed files with 161 additions and 55 deletions.
40 changes: 29 additions & 11 deletions .circleci/config.yml
View file
Expand Up @@ -7,7 +7,24 @@ defaults: &defaults

version: 2
jobs:
python-lint:
test-site:
<<: *defaults
steps:
- checkout
- run:
name: Install dependencies
command: yarn && pip3 install -r requirements.txt
- run:
name: Build the site resources
command: yarn run build
- run:
name: Run the site server
command: ./entrypoint 0.0.0.0:80
background: true
- run:
name: Check site is accessible
command: sleep 3 && curl --head --fail --retry-delay 5 --retry 10 --retry-connrefused http://localhost
lint-python:
<<: *defaults
steps:
- checkout
Expand All @@ -16,8 +33,8 @@ jobs:
command: pip3 install -r requirements.txt
- run:
name: Lint webapp with Black
command: yarn run lint-py
python-tests:
command: yarn run lint-python
test-python:
<<: *defaults
steps:
- checkout
Expand All @@ -32,7 +49,7 @@ jobs:
command: |
coverage run --source=. -m unittest discover tests
bash <(curl -s https://codecov.io/bash) -cF python
js-lint:
lint-js:
<<: *defaults
steps:
- checkout
Expand All @@ -42,7 +59,7 @@ jobs:
- run:
name: Lint statics with eslint
command: yarn run lint-js
js-tests:
test-js:
<<: *defaults
steps:
- checkout
Expand All @@ -54,7 +71,7 @@ jobs:
command: |
yarn run test-js
bash <(curl -s https://codecov.io/bash) -cF javascript
scss-lint:
lint-scss:
<<: *defaults
steps:
- checkout
Expand All @@ -68,8 +85,9 @@ workflows:
version: 2
build:
jobs:
- python-lint
- python-tests
- js-lint
- js-tests
- scss-lint
- test-site
- lint-python
- test-python
- lint-js
- test-js
- lint-scss
6 changes: 3 additions & 3 deletions package.json
View file
Expand Up @@ -2,11 +2,11 @@
"scripts": {
"lint-js": "eslint static/js",
"lint-scss": "sass-lint 'static/**/*.scss' --verbose --no-exit",
"lint-py": "flake8 webapp tests && black --check --line-length 79 tests webapp",
"lint-python": "flake8 webapp tests && black --check --line-length 79 webapp tests",
"test": "yarn run test-python && yarn run test-js-all && yarn run lint-scss",
"test-js": "jest",
"test-js-all": "yarn run lint-js && yarn run test-js",
"test-python": "yarn run lint-py && FLASK_DEBUG=0 python3 -m unittest discover tests",
"test-python": "yarn run lint-python && FLASK_DEBUG=0 python3 -m unittest discover tests",
"get-licenses": "if [ ! -f webapp/licenses.json ] ; then curl https://raw.githubusercontent.com/spdx/license-list-data/master/json/licenses.json -o webapp/licenses.json; fi",
"build": "yarn run get-licenses && yarn run build-js && yarn run build-css",
"build-css": "node-sass --include-path node_modules static/sass --output-style compressed --output static/css && postcss --use autoprefixer --no-map --replace 'static/css/**/*.css'",
Expand All @@ -26,7 +26,7 @@
},
"lint-staged": {
"linters": {
"**/*.py": "./run exec yarn lint-py",
"**/*.py": "./run exec yarn lint-python",
"**/*.js": "./run exec yarn lint-js",
"**/*.scss": "./run exec yarn lint-scss"
},
Expand Down
170 changes: 129 additions & 41 deletions run
View file
@@ -1,7 +1,7 @@
#! /usr/bin/env bash

# CAUTION:
# This file was generated by generator-canonical-webteam@3.2.2
# This file was generated by generator-canonical-webteam@3.4.2
# and should usually not be edited directly.
#
# This file was generated by the "canonical-webteam" Yeoman generator
Expand All @@ -14,7 +14,7 @@

set -euo pipefail

USAGE="How to use ./run v3.2.2
USAGE="How to use ./run v3.4.2
===
$ ./run \\
Expand All @@ -31,8 +31,11 @@ Commands
- watch [-s|--watch-site]: Run \`yarn run watch\` (for jekyll sites, watch for changes with \`--watch-site\`)
- build: Run \`yarn run build\`
- test: Run \`yarn run test\`
- test-python: Run \`yarn run test-python\`
- lint-python: Run \`yarn run lint-python\`
- stop: Stop any running containers
- exec [-p|--expose-port PORT] <args>: Run a command in the development container (optionally exposing a port to the host)
- exec [-r|--root] [-p|--expose-port PORT] <args>: Run a command in the development container (optionally exposing a port to the host)
- yarn [-r|--root] [-p|--expose-port PORT] <script-name>: Run a yarn script from package.json
- clean: Remove all images and containers, any installed dependencies and the .docker-project file
- clean-cache: Empty cache files, which are saved between projects (eg, yarn)
"
Expand Down Expand Up @@ -155,12 +158,45 @@ while [[ -n "${1:-}" ]] && [[ "${1:0:1}" == "-" ]]; do
shift
;;
-h|--help) echo "$USAGE"; exit ;;
-v|--version) echo "Generated from generator-canonical-webteam@3.2.2"; exit ;;
-v|--version) echo "Generated from generator-canonical-webteam@3.4.2"; exit ;;
*) invalid "Option '${key}' not recognised." ;;
esac
shift
done

start_django_db () {
# Run the database if necessary
if grep -q django.db.backends.postgresql_psycopg2 */settings.py 2> /dev/null; then
# Create isolated network
if ! docker network inspect ${network_name} &> /dev/null; then
docker network create ${network_name}
fi

# Start the database
trap "kill_container ${db_container}" EXIT;
if ! docker inspect -f {{.State.Running}} ${db_container} &>/dev/null; then
docker run \
--name ${db_container} `# Name the container` \
--rm `# Remove the container once it's finished` \
--volume "${db_volume}":/var/lib/postgresql/data `# Store dependencies in a docker volume` \
${http_proxy} `# Include HTTP proxy if needed` \
--network ${network_name} `# Use an isolated network` \
--network-alias db `# Call this container "db" on the network so it can be found` \
--detach `# Run in the background` \
postgres `# Use the image for node version 7`
fi

# Wait for it
wait_time=0
until docker exec ${db_container} pg_isready || [ $wait_time -eq 4 ]; do
sleep $(( wait_time++ ))
done

# Provision database
run_as_user "${network}" python3 manage.py migrate
fi
}

kill_container () {
container_name="${1}"

Expand All @@ -180,6 +216,12 @@ docker_run () {
container_name="${container_name// /_}" # Replace spaces with underscores
container_name=$(echo ${container_name} | tr -dc '[:alnum:]_.-') # Remove disallowed chars

# Use network if it's been setup
network=""
if docker network inspect ${network_name} &> /dev/null; then
network="--network ${network_name}"
fi

# Kill existing containers
kill_container "${container_name}"

Expand All @@ -196,6 +238,7 @@ docker_run () {
--volume ${usr_local_volume}:/usr/local/ `# Bind local folder to volume` \
--volume ${cache_volume}:/home/shared/.cache/ `# Bind cache to volume` \
--env COMMIT_ID=${commit_id} `# Pass through the commit ID` \
${network} `# Network settings, if needed` \
${env_file} `# Pass any files of environment variables to the container` \
${env_vars} `# Pass explicit environment variables to the container` \
${http_proxy} `# Include HTTP proxy if needed` \
Expand All @@ -212,41 +255,6 @@ run_as_user () {
docker_run "--user $(id -u):$(id -g) ${run_as_user_options}" $@
}

python_run () {
python_run_options="${1}"; shift

if grep -q django.db.backends.postgresql_psycopg2 */settings.py 2> /dev/null; then
# Create isolated network
if ! docker network inspect ${network_name} &> /dev/null; then
docker network create ${network_name}
fi
python_run_options="${python_run_options} --network ${network_name}"

# Start the database
if [[ "${python_run_options}" != *"--detach"* ]]; then trap "kill_container ${project}-db" EXIT; fi
if ! docker inspect -f {{.State.Running}} ${db_container} &>/dev/null; then
docker run \
--name ${db_container} `# Name the container` \
--rm `# Remove the container once it's finished` \
--volume "${db_volume}":/var/lib/postgresql/data `# Store dependencies in a docker volume` \
${http_proxy} `# Include HTTP proxy if needed` \
--network "${network_name}" `# Use an isolated network` \
--network-alias db `# Call this container "db" on the network so it can be found` \
--detach `# Run in the background` \
postgres `# Use the image for node version 7`
fi

# Provision database for django sites
if [ -f manage.py ]; then
docker_run "--network ${network_name}" python3 manage.py migrate
fi

fi

# Run the command in the python docker image
run_as_user "${python_run_options}" $@
}

create_etc_volume() {
# Create local user and group in the dev image
uid=$(id -u)
Expand All @@ -272,13 +280,14 @@ update_dependencies() {
if [ -f package.json ]; then
package_json_hash=$(${md5_command} package.json | cut -c1-8)
if [ -d node_modules ]; then
yarn_dependencies_hash=$(find node_modules -type f -print0 | sort -z | xargs -0 ${md5_command} | ${md5_command} | cut -c1-8)-${package_json_hash}
yarn_dependencies_hash=$(find node_modules -type f ! -wholename 'node_modules/.cache/*' -print0 | sort -z | xargs -0 ${md5_command} | ${md5_command} | cut -c1-8)-${package_json_hash}
fi
if [ -z "${yarn_dependencies_hash:-}" ] || [ ! -f .yarn.${project}.hash ] || [ "${yarn_dependencies_hash}" != "$(cat .yarn.${project}.hash)" ]; then
echo "Installing new Yarn dependencies"
run_as_user "" yarn install --force ${yarn_proxy}
yarn_dependencies_hash=$(find node_modules -type f -print0 | sort -z | xargs -0 ${md5_command} | ${md5_command} | cut -c1-8)-${package_json_hash}
yarn_dependencies_hash=$(find node_modules -type f ! -wholename 'node_modules/.cache/*' -print0 | sort -z | xargs -0 ${md5_command} | ${md5_command} | cut -c1-8)-${package_json_hash}
echo ${yarn_dependencies_hash} > .yarn.${project}.hash
echo "Saved ${yarn_dependencies_hash} to .yarn.${project}.hash"
else
echo "Yarn dependencies haven't changed. To force an update, delete .yarn.${project}.hash."
fi
Expand Down Expand Up @@ -390,6 +399,8 @@ case $run_command in
fi
done

start_django_db

# Run the serve container, publishing the port, and detaching if required
run_as_user "--env PORT=${PORT} --publish ${PORT}:${PORT} ${publish_forward_ports} ${publish_extra_ports} ${detach} ${run_serve_docker_opts} ${module_volumes}" yarn run serve $*
;;
Expand Down Expand Up @@ -461,6 +472,48 @@ case $run_command in
echo "==="
fi
;;
"test-python")
update_dependencies

test_error=false

# Run node tests
echo "- Running python tests"
run_as_user "" yarn run test-python || test_error=true

# Report success or failure
if ${test_error}; then
echo "==="
echo "Tests failed"
echo "==="
exit 1
else
echo "==="
echo "Tests succeeded"
echo "==="
fi
;;
"lint-python")
update_dependencies

lint_error=false

# Run node tests
echo "- Running python lint"
run_as_user "" yarn run lint-python || lint_error=true

# Report success or failure
if ${lint_error}; then
echo "==="
echo "Lint failed"
echo "==="
exit 1
else
echo "==="
echo "Lint succeeded"
echo "==="
fi
;;
"clean")
echo "Remove hash files"
rm -rf .*.hash
Expand Down Expand Up @@ -501,6 +554,7 @@ case $run_command in
"exec")
expose_ports=""
run_as_root=false

while [[ -n "${1:-}" ]] && [[ "${1:0:1}" == "-" ]]; do
key="$1"

Expand All @@ -517,11 +571,45 @@ case $run_command in
esac
shift
done

update_dependencies
start_django_db

if ${run_as_root}; then
docker_run "${expose_ports}" $@
else
run_as_user "${expose_ports}" $@
fi
;;
"yarn")
expose_ports=""
run_as_root=false

while [[ -n "${1:-}" ]] && [[ "${1:0:1}" == "-" ]]; do
key="$1"

case $key in
-r|--root)
run_as_root=true
;;
-p|--expose-port)
if [ -z "${2:-}" ]; then invalid "Missing port number. Usage: --expose-port XXXX"; fi
expose_ports="${expose_ports} --publish ${2}:${2}"
shift
;;
*) invalid "Option '${key}' not recognised." ;;
esac
shift
done

update_dependencies
start_django_db

if ${run_as_root}; then
docker_run "${expose_ports}" yarn run $@
else
run_as_user "${expose_ports}" yarn run $@
fi
;;
*) invalid "Command '${run_command}' not recognised." ;;
esac

0 comments on commit c5ec769

Please sign in to comment.