diff --git a/.gitignore b/.gitignore index b5432a7..4a65335 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,5 @@ ghostdriver.log __pycache__/ node_modules/ *.pyc -_test_config.yaml config.yaml data diff --git a/.travis/travis_install.sh b/.travis/travis_install.sh index 0b3b5b4..4e589ce 100755 --- a/.travis/travis_install.sh +++ b/.travis/travis_install.sh @@ -17,8 +17,6 @@ section "install.cesium_web.requirements" npm -g install npm@latest npm --version node --version -make dependencies -make check-js-updates if [[ ${TRIGGERED_FROM_REPO} == "cesium-ml/cesium" ]]; then mkdir cesium-clone @@ -35,11 +33,9 @@ pip list --format=columns section_end "install.cesium_web.requirements" -section "init.cesium_web" -make paths +section "init.db" make db_init -make bundle -section_end "init.cesium_web" +section_end "init.db" section "install.chromedriver" diff --git a/Makefile b/Makefile index e4f7db7..9691999 100644 --- a/Makefile +++ b/Makefile @@ -1,107 +1,39 @@ -SHELL = /bin/bash -SUPERVISORD=FLAGS=$$FLAGS supervisord -c baselayer/conf/supervisor/supervisor.conf -SUPERVISORCTL=FLAGS=$$FLAGS supervisorctl -c baselayer/conf/supervisor/supervisor.conf -ENV_SUMMARY=PYTHONPATH=. ./baselayer/tools/env_summary.py $$FLAGS -ESLINT=./node_modules/.bin/eslint - .DEFAULT_GOAL := run -bundle = ./static/build/bundle.js -webpack = ./node_modules/.bin/webpack -baselayer_branch = $(shell git config -f .gitmodules submodule.baselayer.branch) - baselayer/README.md: git submodule update --init --remote $(MAKE) baselayer-update -.PHONY: baselayer-update +.PHONY: baselayer-update run log baselayer-update: ./baselayer/tools/submodule_update.sh -dependencies: baselayer/README.md - @./baselayer/tools/silent_monitor.py pip install -r baselayer/requirements.txt - @./baselayer/tools/silent_monitor.py pip install -r requirements.txt - @./baselayer/tools/silent_monitor.py ./baselayer/tools/check_js_deps.sh - -db_init: - @PYTHONPATH=. ./baselayer/tools/silent_monitor.py ./baselayer/tools/db_init.py - -db_clear: - PYTHONPATH=. ./baselayer/tools/db_init.py -f - -$(bundle): webpack.config.js package.json - $(webpack) - -bundle: $(bundle) - -bundle-watch: - $(webpack) -w +log: + make -C baselayer log -paths: - @mkdir -p log run tmp - @mkdir -p log/sv_child - @mkdir -p ~/.local/cesium/logs +run: + make -C baselayer run -fill_conf_values: - PYTHONPATH=. ./baselayer/tools/fill_conf_values.py - -log: paths - ./baselayer/tools/watch_logs.py - -run: paths dependencies fill_conf_values - @echo "Supervisor will now fire up various micro-services." - @echo - @echo " - Run \`make log\` in another terminal to view logs" - @echo " - Run \`make monitor\` in another terminal to restart services" - @echo - @echo "The server is in debug mode:" - @echo " JavaScript and Python files will be reloaded upon change." - @echo - - @FLAGS="--debug" && \ - $(ENV_SUMMARY) && echo && \ - echo "Press Ctrl-C to abort the server" && \ - echo && \ - $(SUPERVISORD) +run_testing: + make -C baselayer run_testing run_production: - export FLAGS="--config config.yaml" && \ - $(ENV_SUMMARY) && \ - $(SUPERVISORD) - -run_testing: paths dependencies - export FLAGS="--config _test_config.yaml" && \ - $(ENV_SUMMARY) && \ - $(SUPERVISORD) - -monitor: - @echo "Entering supervisor control panel." - @echo " - Type \`status\` too see microservice status" - $(SUPERVISORCTL) -i status + make -C baselayer run_production -# Attach to terminal of running webserver; useful to, e.g., use pdb -attach: - $(SUPERVISORCTL) fg app +test: + make -C baselayer test -clean: - rm $(bundle) +test_headless: + make -C baselayer test_headless -test_headless: paths dependencies fill_conf_values - PYTHONPATH='.' xvfb-run ./tools/test_frontend.py - -test: paths dependencies fill_conf_values - PYTHONPATH='.' ./tools/test_frontend.py - -stop: - $(SUPERVISORCTL) stop all +db_init: + make -C baselayer db_init -status: - PYTHONPATH='.' ./baselayer/tools/supervisor_status.py +db_clear: + make -C baselayer db_clear docker-images: # Add --no-cache flag to rebuild from scratch docker build -t cesium/web . && docker push cesium/web -# Call this target to see which Javascript dependencies are not up to date -check-js-updates: - ./baselayer/tools/check_js_updates.sh +-include "baselayer/README.md" # always clone baselayer if it doesn't exist diff --git a/README.md b/README.md index d07a110..c4cbfd0 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,7 @@ The easiest way to try the web app is to run it through Docker: ``` and restart `postgresl` (`sudo service postgresql restart`). -2. Install Python and JavaScript dependencies with `make_dependencies` -3. Initialize the database with `make db_init` +2. Initialize the database with `make db_init` If you've run this script before, you may see warnings here about the database already existing. Ignore those. @@ -64,7 +63,7 @@ database already existing. Ignore those. ## Configuration -Copy `config.yaml.example` to `config.yaml` and customize. +Copy `config.yaml.defaults` to `config.yaml` and customize. - Under `server`, set 'multi_user' to True to enable logins. Also provide the Google credentials, obtained as described in the config file. diff --git a/baselayer b/baselayer index 3132182..f67b6c1 160000 --- a/baselayer +++ b/baselayer @@ -1 +1 @@ -Subproject commit 313218231ca0a9fa1f9b245668f6cce37682a862 +Subproject commit f67b6c1f52ff18582b84e1088d5520b2ae5ed1cf diff --git a/cesium_app/handlers/dataset.py b/cesium_app/handlers/dataset.py index afd8554..cb858bd 100644 --- a/cesium_app/handlers/dataset.py +++ b/cesium_app/handlers/dataset.py @@ -22,9 +22,10 @@ def post(self): return self.error('No tar file uploaded') zipfile = data['tarFile'] - tarball_content_type_str = 'data:application/gzip;base64,' + tarball_content_types = ('data:application/gzip;base64', + 'data:application/x-gzip;base64') - if not zipfile['body'].startswith(tarball_content_type_str): + if not zipfile['body'].startswith(tarball_content_types): return self.error('Invalid tar file - please ensure file is gzip ' 'format.') @@ -38,9 +39,10 @@ def post(self): util.secure_filename(zipfile['name'])) zipfile_path = pjoin(self.cfg['paths:upload_folder'], zipfile_name) + for prefix in tarball_content_types: + zipfile['body'] = zipfile['body'].replace(prefix, '') with open(zipfile_path, 'wb') as f: - f.write(base64.b64decode( - zipfile['body'].replace(tarball_content_type_str, ''))) + f.write(base64.b64decode(zipfile['body'])) try: tarfile.open(zipfile_path) except tarfile.ReadError: diff --git a/cesium_app/tests/conftest.py b/cesium_app/tests/conftest.py index cebaac9..f496fb8 100644 --- a/cesium_app/tests/conftest.py +++ b/cesium_app/tests/conftest.py @@ -13,7 +13,7 @@ from selenium.common.exceptions import TimeoutException from seleniumrequests.request import RequestMixin from pytest_factoryboy import register, LazyFixture -from baselayer.app.config import Config +from baselayer.app.config import load_config from baselayer.app.test_util import (driver, MyCustomWebDriver, reset_state, set_server_url) from cesium_app import models @@ -22,10 +22,10 @@ PredictionFactory) -print('Loading test configuration from _test_config.yaml') +print('Loading test configuration from test_config.yaml') basedir = pathlib.Path(os.path.dirname(__file__))/'../..' -cfg = Config([basedir/'config.yaml.example', basedir/'_test_config.yaml']) -set_server_url(cfg['server:url']) +cfg = load_config([basedir/'test_config.yaml']) +set_server_url(f'http://localhost:{cfg["ports:app"]}') print('Setting test database to:', cfg['database']) models.init_db(**cfg['database']) diff --git a/cesium_app/tests/frontend/test_predict.py b/cesium_app/tests/frontend/test_predict.py index e8384c1..2e9e2dd 100644 --- a/cesium_app/tests/frontend/test_predict.py +++ b/cesium_app/tests/frontend/test_predict.py @@ -234,7 +234,6 @@ def test_predict_specific_ts_name(driver, project, dataset, featureset, model): response = driver.request( 'POST', '{}/predictions'.format(driver.server_url), json=data).json() - print(response) assert response['status'] == 'success' for i in range(10): diff --git a/config.yaml.example b/config.yaml.defaults similarity index 83% rename from config.yaml.example rename to config.yaml.defaults index c6174b4..22f6c26 100644 --- a/config.yaml.example +++ b/config.yaml.defaults @@ -30,8 +30,6 @@ database: password: server: - url: http://localhost:5000 - # From https://console.developers.google.com/ # # - Create Client ID @@ -44,13 +42,3 @@ server: debug_login: True google_oauth2_key: google_oauth2_secret: - -ports: - websocket: 64000 - fake_oauth: 63000 - app: 5000 - app_http_proxy: 5001 - app_internal: 65000 - dask: 63500 - websocket_path_in: 'ipc:///tmp/message_flow_in' - websocket_path_out: 'ipc:///tmp/message_flow_out' diff --git a/test_config.yaml b/test_config.yaml new file mode 100644 index 0000000..6ea1fb7 --- /dev/null +++ b/test_config.yaml @@ -0,0 +1,11 @@ +database: + database: cesium_test + user: cesium + host: localhost + port: 5432 + +server: + auth: + debug_login: True + google_oauth2_key: + google_oauth2_secret: diff --git a/tools/test_frontend.py b/tools/test_frontend.py deleted file mode 100755 index 2246fcb..0000000 --- a/tools/test_frontend.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python - -import os -from os.path import join as pjoin -import pathlib -import sys -sys.path.insert(0, pjoin(os.path.dirname(__file__), '..')) - -import signal -import socket -import subprocess -import time -from baselayer.tools.supervisor_status import supervisor_status -try: - import http.client as http -except ImportError: - import httplib as http - -from baselayer.app.model_util import clear_tables -from cesium_app import models - -try: - import pytest_randomly - RAND_ARGS = '--randomly-seed=1' -except ImportError: - RAND_ARGS = '' - -TEST_CONFIG = '_test_config.yaml' - - -base_dir = os.path.abspath(pjoin(os.path.dirname(__file__), '..')) -if len(sys.argv) > 1: - test_spec = sys.argv[1] -else: - test_spec = pjoin(base_dir, 'cesium_app', 'tests') - - -def add_test_yaml(): - print(f'Creating {TEST_CONFIG}') - - from textwrap import dedent - with open(TEST_CONFIG, 'w') as f: - f.write(dedent(''' - database: - database: cesium_test - user: cesium - host: localhost - port: 5432 - - server: - url: http://localhost:5000 - multi_user: True - auth: - debug_login: True - google_oauth2_key: - google_oauth2_secret: - - ''')) - - -def delete_test_yaml(): - os.remove(TEST_CONFIG) - - -if __name__ == '__main__': - add_test_yaml() - - # Initialize the test database connection - from cesium_app.models import init_db - from baselayer.app.config import load_config - basedir = pathlib.Path(os.path.dirname(__file__))/'..' - cfg = load_config([basedir/'config.yaml.example', basedir/TEST_CONFIG]) - init_db(**cfg['database']) - - clear_tables() - - web_client = subprocess.Popen(['make', 'run_testing'], cwd=base_dir, - preexec_fn=os.setsid) - - print('[test_frontend] Waiting for supervisord to launch all server processes...') - - try: - timeout = 0 - while ((timeout < 30) and - (not all([b'RUNNING' in line for line in supervisor_status()]))): - time.sleep(1) - timeout += 1 - - if timeout == 10: - print('[test_frontend] Could not launch server processes; terminating') - sys.exit(-1) - - for timeout in range(10): - conn = http.HTTPConnection("localhost", 5000) - try: - conn.request('HEAD', '/') - status = conn.getresponse().status - if status == 200: - break - except socket.error: - pass - time.sleep(1) - else: - raise socket.error("Could not connect to localhost:5000.") - - if status != 200: - print('[test_frontend] Server status is {} instead of 200'.format( - status)) - sys.exit(-1) - else: - print('[test_frontend] Verified server availability') - - print('[test_frontend] Launching pytest on {}...'.format(test_spec)) - - status = subprocess.run(f'python -m pytest -v {test_spec} {RAND_ARGS}', - shell=True, check=True) - except: - raise - finally: - print('[test_frontend] Terminating supervisord...') - os.killpg(os.getpgid(web_client.pid), signal.SIGTERM) - delete_test_yaml()