From 9b2d284ac2d614c5404a62d8d73fa6c9ea813915 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 13:54:06 +0200 Subject: [PATCH 01/11] Added test coverage through coveralls in the travis system --- .travis.yml | 1 + backend/settings.py | 6 +++++- test/travis_script.sh | 10 ++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc53665ad..e9d8926e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,5 +7,6 @@ before_install: - test/travis_before_install.sh install: - pip install -r backend/requirements.txt + - pip install coverage coveralls script: - test/travis_script.sh diff --git a/backend/settings.py b/backend/settings.py index 4cbd431b3..b0a62ebc4 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -1,6 +1,10 @@ import json -json_settings_fh = open("../settings.json") +try: + json_settings_fh = open("settings.json") +except FileNotFoundError: + json_settings_fh = open("../settings.json") + json_settings = json.load(json_settings_fh) json_settings_fh.close() diff --git a/test/travis_script.sh b/test/travis_script.sh index 1124cc748..f4ccef324 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -31,13 +31,14 @@ mysql -u swefreq -h 127.0.0.1 -P 3366 swefreq_test < test/data/load_dummy_data.s echo ">>> Test 3. Check that the backend starts" -cd backend +cd backend ../test/01_daemon_starts.sh +cd .. echo ">>> Test 4. the backend API" -python route.py --develop 1>http_log.txt 2>&1 & +coverage run backend/route.py --develop 1>http_log.txt 2>&1 & BACKEND_PID=$! function exit_handler() { @@ -51,3 +52,8 @@ trap exit_handler EXIT sleep 2 # Lets wait a little bit so the server has started python test.py -v + +kill -2 $BACKEND_PID +sleep 2 + +coveralls From 93e116b2f006435e358f4d7b9b05be687b6e1e5c Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 15:01:39 +0200 Subject: [PATCH 02/11] Updated Dockerfile-backend to run from base dir instea --- Dockerfile-backend | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile-backend b/Dockerfile-backend index c2f9361ba..2ccc0b728 100644 --- a/Dockerfile-backend +++ b/Dockerfile-backend @@ -6,8 +6,8 @@ RUN apt-get update && apt-get install -y \ libmysqlclient-dev ADD . /code -WORKDIR /code/backend +WORKDIR /code -RUN pip3 install -r requirements.txt +RUN pip3 install -r backend/requirements.txt -CMD ["python3", "route.py", "--develop"] +CMD ["python3", "backend/route.py", "--develop"] From 9dbc07d3f2961c22b694f578c2a3b3da33a90f66 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 15:02:21 +0200 Subject: [PATCH 03/11] Run from base directory in travis test --- test/travis_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/travis_script.sh b/test/travis_script.sh index f4ccef324..58e6dba3e 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -51,7 +51,7 @@ function exit_handler() { trap exit_handler EXIT sleep 2 # Lets wait a little bit so the server has started -python test.py -v +python backend/test.py -v kill -2 $BACKEND_PID sleep 2 From 6a520e9e9418c8d1338b42bb48bc5e68d7f7b415 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 14:12:59 +0200 Subject: [PATCH 04/11] More coverage settings and show result in log --- .coveragerc | 5 +++++ test/travis_script.sh | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..8db5982de --- /dev/null +++ b/.coveragerc @@ -0,0 +1,5 @@ +[run] +omit = /usr/local/*,/home/travis/virtualenv/* + +[report] +omit = /usr/local/*,/home/travis/virtualenv/* diff --git a/test/travis_script.sh b/test/travis_script.sh index 58e6dba3e..f6b7fe57e 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -38,7 +38,7 @@ cd .. echo ">>> Test 4. the backend API" -coverage run backend/route.py --develop 1>http_log.txt 2>&1 & +coverage run backend/route.py --port=4000 --develop 1>http_log.txt 2>&1 & BACKEND_PID=$! function exit_handler() { @@ -57,3 +57,4 @@ kill -2 $BACKEND_PID sleep 2 coveralls +coverage report From b0b5494245faf41b1a46f5fbfab3c81a8b9e4eec Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 14:31:33 +0200 Subject: [PATCH 05/11] Only run the test coverage stuff if we have the ".coverage" file --- test/travis_script.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/travis_script.sh b/test/travis_script.sh index f6b7fe57e..66d7f3104 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -56,5 +56,7 @@ python backend/test.py -v kill -2 $BACKEND_PID sleep 2 -coveralls -coverage report +if [ -f .coverage ]; then + coveralls + coverage report +fi From 902c171db6c705e92af0d19e021fd51c954a840d Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 14:40:01 +0200 Subject: [PATCH 06/11] Added a way to quit the webapp in developer mode --- backend/application.py | 7 +++++++ backend/route.py | 1 + test/travis_script.sh | 5 +++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/application.py b/backend/application.py index 4df01482d..49c0f90ce 100644 --- a/backend/application.py +++ b/backend/application.py @@ -8,6 +8,7 @@ import smtplib import socket import tornado.web +import tornado import random import string import uuid @@ -41,6 +42,12 @@ def build_dataset_structure(dataset_version, user=None, dataset=None): return r +class QuitHandler(handlers.UnsafeHandler): + def get(self): + ioloop = tornado.ioloop.IOLoop.instance() + ioloop.stop() + + class GetSchema(handlers.UnsafeHandler): """ Returns the schema.org, and bioschemas.org, annotation for a given diff --git a/backend/route.py b/backend/route.py index 200b3b7a6..3b0c6de06 100644 --- a/backend/route.py +++ b/backend/route.py @@ -92,6 +92,7 @@ def __init__(self, settings): ## Adding developer login handler if settings.get('develop', False): self.declared_handlers.insert(-1, ("/developer/login", auth.DeveloperLoginHandler)) + self.declared_handlers.insert(-1, ("/developer/quit", application.QuitHandler)) # google oauth key self.oauth_key = tornado_settings["google_oauth"]["key"] diff --git a/test/travis_script.sh b/test/travis_script.sh index 66d7f3104..b83c44003 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -52,10 +52,11 @@ trap exit_handler EXIT sleep 2 # Lets wait a little bit so the server has started python backend/test.py -v - -kill -2 $BACKEND_PID sleep 2 +# Quit the app +curl localhost:4000/developer/quit + if [ -f .coverage ]; then coveralls coverage report From fe3a6808689ee9dfd7d096c7d00bb7837b591c18 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 16:07:45 +0200 Subject: [PATCH 07/11] Added travis and coverage badges to readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 04b8d704e..9d563bfd4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ SweFreq - Swedish Frequency database ==================================== +[![Travis Status][travis-badge]][travis-link] +[![Coverage Status][coveralls-badge]][coverage-link] + Running on a production system ------------------------------ @@ -70,3 +73,8 @@ Quick development mode ```bash $ docker-compose up ``` + +[travis-badge]: https://travis-ci.org/NBISweden/swefreq.svg?branch=develop +[travis-link]: https://travis-ci.org/NBISweden/swefreq +[coveralls-badge]: https://coveralls.io/repos/github/NBISweden/swefreq/badge.svg?branch=develop +[coveralls-link]: https://coveralls.io/github/NBISweden/swefreq?branch=release From db0efb7e4349480c71550a7de3275a80413f443c Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 16:12:14 +0200 Subject: [PATCH 08/11] Updated coveralls badge --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9d563bfd4..a9591ac71 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ SweFreq - Swedish Frequency database ==================================== [![Travis Status][travis-badge]][travis-link] -[![Coverage Status][coveralls-badge]][coverage-link] +[![Coverage Status][coveralls-badge]][coveralls-link] Running on a production system @@ -77,4 +77,4 @@ $ docker-compose up [travis-badge]: https://travis-ci.org/NBISweden/swefreq.svg?branch=develop [travis-link]: https://travis-ci.org/NBISweden/swefreq [coveralls-badge]: https://coveralls.io/repos/github/NBISweden/swefreq/badge.svg?branch=develop -[coveralls-link]: https://coveralls.io/github/NBISweden/swefreq?branch=release +[coveralls-link]: https://coveralls.io/github/NBISweden/swefreq?branch=develop From a02753fe2cc8160dd92e7db7721e4e5a47a40f4d Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 5 Apr 2018 09:44:26 +0200 Subject: [PATCH 09/11] Moved the sleeps around --- test/travis_script.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/travis_script.sh b/test/travis_script.sh index b83c44003..b3a98b970 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -41,6 +41,8 @@ echo ">>> Test 4. the backend API" coverage run backend/route.py --port=4000 --develop 1>http_log.txt 2>&1 & BACKEND_PID=$! +sleep 2 # Lets wait a little bit so the server has started + function exit_handler() { kill -9 $BACKEND_PID @@ -50,12 +52,12 @@ function exit_handler() { trap exit_handler EXIT -sleep 2 # Lets wait a little bit so the server has started + python backend/test.py -v -sleep 2 # Quit the app curl localhost:4000/developer/quit +sleep 2 # Lets wait a little bit so the server has stopped if [ -f .coverage ]; then coveralls From 0ea9885713cf6e703c2cb9885a62f6eb7bdc01ff Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 5 Apr 2018 09:54:05 +0200 Subject: [PATCH 10/11] Better exit handler --- test/travis_script.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/travis_script.sh b/test/travis_script.sh index b3a98b970..a187e9604 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -44,10 +44,14 @@ BACKEND_PID=$! sleep 2 # Lets wait a little bit so the server has started function exit_handler() { + rv=$? + # Ignore errors in the exit handler + set +e kill -9 $BACKEND_PID echo "THE HTTP LOG WAS:" cat http_log.txt + exit $rv } trap exit_handler EXIT From 209ba62db381b953c39178d925c60d0d9741d589 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 5 Apr 2018 11:12:19 +0200 Subject: [PATCH 11/11] Updated travis script --- .pylintrc | 3 ++- test/travis_script.sh | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.pylintrc b/.pylintrc index 3e4f6d1d3..5b77e41e8 100644 --- a/.pylintrc +++ b/.pylintrc @@ -138,7 +138,8 @@ disable=print-statement, keyword-arg-before-vararg, arguments-differ, line-too-long, - import-error + import-error, + no-self-use # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/test/travis_script.sh b/test/travis_script.sh index a187e9604..34f904dc0 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -32,9 +32,7 @@ mysql -u swefreq -h 127.0.0.1 -P 3366 swefreq_test < test/data/load_dummy_data.s echo ">>> Test 3. Check that the backend starts" -cd backend -../test/01_daemon_starts.sh -cd .. +(cd backend && ../test/01_daemon_starts.sh) echo ">>> Test 4. the backend API" @@ -43,10 +41,12 @@ BACKEND_PID=$! sleep 2 # Lets wait a little bit so the server has started -function exit_handler() { +exit_handler() { rv=$? # Ignore errors in the exit handler set +e + # We want to make sure the background process has stopped, otherwise the + # travis test will stall for a long time. kill -9 $BACKEND_PID echo "THE HTTP LOG WAS:"