Add shell scripts to install, run and test the API

.gitignore

@@ -8,3 +8,4 @@ bootstrap.json
 .cache
 .coverage
 coverage.xml
+/virtualenv

.travis.yml

@@ -2,18 +2,12 @@
 # "This computer doesn't have VT-X/AMD-v enabled."
 sudo: required
 dist: trusty
+services:
+  - mongodb
 install:
-  - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-  - sudo sh -c "echo 'deb https://apt.dockerproject.org/repo ubuntu-trusty main' > /etc/apt/sources.list.d/docker.list"
-  - sudo apt-get update -qq
-  - sudo apt-get -o Dpkg::Options::="--force-confnew" install -y -q docker-engine
-  - sudo curl -o /usr/local/bin/docker-compose -L https://github.com/docker/compose/releases/download/1.6.2/docker-compose-`uname -s`-`uname -m`
-  - sudo chmod +x /usr/local/bin/docker-compose
-before_script:
-  - sudo bin/install.sh --ci
+  - bin/install-ubuntu.sh
+  - test/bin/setup-integration-tests-ubuntu.sh
 script:
-  - bin/runtests.sh unit --ci
-  - bin/runtests.sh integration --ci
-  - ./test/lint.sh api
+  - SCITRAN_PERSISTENT_DB_PORT=27017 test/bin/run-tests-ubuntu.sh
 after_success:
   - coveralls

CONTRIBUTING.md

@@ -23,7 +23,7 @@ Changes to `requirements.txt` should always be by pull request.
 - Add docstrings to all functions with a one-line description of its purpose.
 
 ### Format
-- Ensure that `./test/lint.sh api` exits without errors.
+Ensure that `./test/bin/lint.sh api` exits without errors.
 
 ### Commit Messages
 1. The subject line should be a phrase describing the commit and limited to 50 characters

Dockerfile

@@ -73,8 +73,6 @@ RUN pip install --upgrade pip wheel setuptools \
 #
 COPY . /var/scitran/code/api/
 
-
-
 COPY docker/uwsgi-entrypoint.sh /var/scitran/
 COPY docker/uwsgi-config.ini /var/scitran/config/
 COPY docker/newrelic.ini /var/scitran/config/

README.md

@@ -19,10 +19,27 @@ SciTran Core is a RESTful HTTP API, written in Python and backed by MongoDB. It
 
 
 ### Usage
+**Currently Python 2 Only**  
+
+#### OSX
 ```
-./bin/run.sh [config file]
+$ ./bin/run-dev-osx.sh --help
+Run a development instance of scitran-core
+ Also starts mongo on port 9001 by default
+
+ Usage:
+
+ -C, --config-file <shell-script>: Source a shell script to set environemnt variables
+ -I, --no-install: Do not attempt install the application first
+ -R, --reload <interval>: Enable live reload, specifying interval in seconds
+ -T, --no-testdata: do not bootstrap testdata
+ -U, --no-user: do not bootstrap users and groups
 ```
-or
+
+#### Ubuntu
 ```
-PYTHONPATH=. uwsgi --http :8443 --virtualenv ./runtime --master --wsgi-file bin/api.wsgi
+mkvirtualenv scitran-core
+./bin/install-ubuntu.sh
+uwsgi --http :8080 --master --wsgi-file bin/api.wsgi -H $VIRTUAL_ENV \
+    --env SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:27017/scitran-core"
 ```

TESTING.md

@@ -1,3 +1,17 @@
+## Run the tests
+### OSX
+```
+./test/bin/run-tests-osx.sh
+```
+
+### Ubuntu
+```
+# Follow installation instructions in README first
+workon scitran-core
+./test/bin/setup-integration-tests-ubuntu.sh
+./test/bin/run-tests-ubuntu.sh
+```
+
 ### Tools
 - [abao](https://github.com/cybertk/abao/)
 - [postman](https://www.getpostman.com/docs/)
@@ -25,4 +39,3 @@ Postman Links
 - http://blog.getpostman.com/2014/03/07/writing-automated-tests-for-apis-using-postman/
 - https://www.getpostman.com/docs/environments
 - https://www.getpostman.com/docs/newman_intro
-

api/config.py

@@ -5,8 +5,8 @@
 import pymongo
 import datetime
 import elasticsearch
-from . import util
 
+from . import util
 
 logging.basicConfig(
     format='%(asctime)s %(name)16.16s %(filename)24.24s %(lineno)5d:%(levelname)4.4s %(message)s',

bin/api.wsgi

@@ -1,4 +1,9 @@
 # vim: filetype=python
+import sys
+import os.path
+
+repo_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
+sys.path.append(repo_path)
 
 from api import api
 

bin/bootstrap.py

@@ -3,126 +3,49 @@
 """This script helps bootstrap users and data"""
 
 import os
+import os.path
 import sys
 import json
 import logging
 import argparse
 import datetime
-import requests
 
-logging.basicConfig(
-    format='%(asctime)s %(levelname)8.8s %(message)s',
-    datefmt='%Y-%m-%d %H:%M:%S',
-    level=logging.DEBUG,
-)
-log = logging.getLogger('scitran.bootstrap')
+import jsonschema
 
-logging.getLogger('requests').setLevel(logging.WARNING) # silence Requests library
+repo_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
+sys.path.append(repo_path)
 
+from api import config, validators
 
-def _upsert_user(request_session, api_url, user_doc):
-    """
-    Insert user, or update if insert fails due to user already existing.
-
-    Returns:
-        requests.Response: API response.
+def bootstrap_users_and_groups(bootstrap_json_file_path):
+    """Loads users and groups directly into the database.
 
     Args:
-        request_session (requests.Session): Session to use for the request.
-        api_url (str): Base url for the API eg. 'https://localhost:8443/api'
-        user_doc (dict): Valid user doc defined in user input schema.
-    """
-    new_user_resp = request_session.post(api_url + '/users', json=user_doc)
-    if new_user_resp.status_code != 409:
-        return new_user_resp
-
-    # Already exists, update instead
-    return request_session.put(api_url + '/users/' + user_doc['_id'], json=user_doc)
-
-
-def _upsert_role(request_session, api_url, role_doc, group_id):
-    """
-    Insert group role, or update if insert fails due to group role already existing.
-
-    Returns:
-        requests.Response: API response.
-
-    Args:
-        request_session (requests.Session): Session to use for the request.
-        api_url -- (str): Base url for the API eg. 'https://localhost:8443/api'
-        role_doc -- (dict) Valid permission doc defined in permission input schema.
-    """
-    base_role_url = "{0}/groups/{1}/roles".format(api_url, group_id)
-    new_role_resp = request_session.post(base_role_url , json=role_doc)
-    if new_role_resp.status_code != 409:
-        return new_role_resp
-
-    # Already exists, update instead
-    full_role_url = "{0}/{1}/{2}".format(base_role_url, role_doc['site'], role_doc['_id'])
-    return request_session.put(full_role_url, json=role_doc)
-
-
-def users(filepath, api_url, http_headers, insecure):
+        bootstrap_json_file_path (str): Path to json file with users and groups
     """
-    Upserts the users/groups/roles defined in filepath parameter.
-
-    Raises:
-        requests.HTTPError: Upsert failed.
-    """
-    now = datetime.datetime.utcnow()
-    with open(filepath) as fd:
-        input_data = json.load(fd)
-    with requests.Session() as rs:
-        log.info('bootstrapping users...')
-        rs.verify = not insecure
-        rs.headers = http_headers
-        for u in input_data.get('users', []):
-            log.info('    {0}'.format(u['_id']))
-            r = _upsert_user(request_session=rs, api_url=api_url, user_doc=u)
-            r.raise_for_status()
-
-        log.info('bootstrapping groups...')
-        r = rs.get(api_url + '/config')
-        r.raise_for_status()
-        site_id = r.json()['site']['id']
-        for g in input_data.get('groups', []):
-            roles = g.pop('roles')
-            log.info('    {0}'.format(g['_id']))
-            r = rs.post(api_url + '/groups' , json=g)
-            r.raise_for_status()
-            for role in roles:
-                role.setdefault('site', site_id)
-                r = _upsert_role(request_session=rs, api_url=api_url, role_doc=role, group_id=g['_id'])
-                r.raise_for_status()
-    log.info('bootstrapping complete')
-
-
-ap = argparse.ArgumentParser()
-ap.description = 'Bootstrap SciTran users and groups'
-ap.add_argument('url', help='API URL')
-ap.add_argument('json', help='JSON file containing users and groups')
-ap.add_argument('--insecure', action='store_true', help='do not verify SSL connections')
-ap.add_argument('--secret', help='shared API secret')
-args = ap.parse_args()
-
-if args.insecure:
-    requests.packages.urllib3.disable_warnings()
-
-http_headers = {
-    'X-SciTran-Method': 'bootstrapper',
-    'X-SciTran-Name': 'Bootstrapper',
-}
-if args.secret:
-    http_headers['X-SciTran-Auth'] = args.secret
-# TODO: extend this to support oauth tokens
-
-try:
-    users(args.json, args.url, http_headers, args.insecure)
-except requests.HTTPError as ex:
-    log.error(ex)
-    log.error("request_body={0}".format(ex.response.request.body))
-    sys.exit(1)
-except Exception as ex:
-    log.error('Unexpected error:')
-    log.error(ex)
-    sys.exit(1)
+    log = logging.getLogger('scitran.bootstrap')
+    with open(bootstrap_json_file_path, "r") as bootstrap_data_file:
+        bootstrap_data = json.load(bootstrap_data_file)
+    user_schema_path = validators.schema_uri("mongo", "user.json")
+    user_schema, user_resolver = validators._resolve_schema(user_schema_path)
+    for user in bootstrap_data.get("users", []):
+        config.log.info("Bootstrapping user: {0}".format(user["email"]))
+        user["created"] = user["modified"] = datetime.datetime.utcnow()
+        if user.get("api_key"):
+            user["api_key"]["created"] = datetime.datetime.utcnow()
+        validators._validate_json(user, user_schema, user_resolver)
+        config.db.users.insert_one(user)
+    group_schema_path = validators.schema_uri("mongo", "group.json")
+    group_schema, group_resolver = validators._resolve_schema(group_schema_path)
+    for group in bootstrap_data.get("groups", []):
+        config.log.info("Bootstrapping group: {0}".format(group["name"]))
+        group["created"] = group["modified"] = datetime.datetime.utcnow()
+        validators._validate_json(group, group_schema, group_resolver)
+        config.db.groups.insert_one(group)
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.description = 'Bootstrap SciTran users and groups'
+    ap.add_argument('json', help='JSON file containing users and groups')
+    args = ap.parse_args()
+    bootstrap_users_and_groups(args.json)

bin/install-dev-osx.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+VIRTUALENV_PATH=${VIRTUALENV_PATH:-"./virtualenv"}
+
+if [ -f "`which brew`" ]; then
+    echo "Homebrew is installed"
+else
+    echo "Installing Homebrew"
+    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+    echo "Installed Homebrew"
+fi
+
+if brew list | grep -q openssl; then
+    echo "OpenSSL is installed"
+else
+    echo "Installing OpenSSL"
+    brew install openssl
+    echo "Installed OpenSSL"
+fi
+
+if brew list | grep -q python; then
+    echo "Python is installed"
+else
+    echo "Installing Python"
+    brew install python
+    echo "Installed Python"
+fi
+
+if [ -f "`which virtualenv`" ]; then
+    echo "Virtualenv is installed"
+else
+    echo "Installing Virtualenv"
+    pip install virtualenv
+    echo "Installed Virtualenv"
+fi
+
+if [ -d "$VIRTUALENV_PATH" ]; then
+    echo "Virtualenv exists at $VIRTUALENV_PATH"
+else
+    echo "Creating 'scitran' Virtualenv at $VIRTUALENV_PATH"
+    virtualenv -p `brew --prefix`/bin/python --prompt="(scitran) " $VIRTUALENV_PATH
+    echo "Created 'scitran' Virtualenv at $VIRTUALENV_PATH"
+fi
+
+echo "Activating Virtualenv"
+set -a
+. $VIRTUALENV_PATH/bin/activate
+
+pip install -U pip
+env LDFLAGS="-L$(brew --prefix openssl)/lib" \
+  CFLAGS="-I$(brew --prefix openssl)/include" \
+  pip install cryptography
+
+echo "Installing Python requirements"
+./bin/install-python-requirements.sh
+
+echo "Installing node and dev dependencies"
+if [ ! -f "$VIRTUALENV_PATH/bin/node" ]; then
+  # Node doesn't exist in the virtualenv, install
+  echo "Installing nodejs"
+  node_source_dir=`mktemp -d`
+  curl https://nodejs.org/dist/v6.4.0/node-v6.4.0-darwin-x64.tar.gz | tar xvz -C "$node_source_dir"
+  mv $node_source_dir/node-v6.4.0-darwin-x64/bin/* "$VIRTUALENV_PATH/bin"
+  mv $node_source_dir/node-v6.4.0-darwin-x64/lib/* "$VIRTUALENV_PATH/lib"
+  rm -rf "$node_source_dir"
+  npm config set prefix "$VIRTUALENV_PATH"
+fi
+
+pip install -U -r "test/integration_tests/requirements.txt"
+if [ ! -f "`which abao`" ]; then
+  npm install -g git+https://github.com/flywheel-io/abao.git#better-jsonschema-ref
+fi
+if [ ! -f "`which newman`" ]; then
+  npm install -g newman@3.0.1
+fi
+
+install_mongo() {
+    curl $MONGODB_URL | tar xz -C $VIRTUAL_ENV/bin --strip-components 2
+    echo "MongoDB version $MONGODB_VERSION installed"
+}
+
+MONGODB_VERSION=$(cat mongodb_version.txt)
+MONGODB_URL="https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-$MONGODB_VERSION.tgz"
+if [ -x "$VIRTUAL_ENV/bin/mongod" ]; then
+    INSTALLED_MONGODB_VERSION=$($VIRTUAL_ENV/bin/mongod --version | grep "db version" | cut -d "v" -f 3)
+    echo "MongoDB version $INSTALLED_MONGODB_VERSION is installed"
+    if [ "$INSTALLED_MONGODB_VERSION" != "$MONGODB_VERSION" ]; then
+        echo "Upgrading MongoDB to version $MONGODB_VERSION"
+        install_mongo
+    fi
+else
+    echo "Installing MongoDB"
+    install_mongo
+fi

bin/install-python-requirements.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+pip install -U pip wheel setuptools
+
+pip install -U -r requirements.txt