Merge pull request #158 from thebrianzeng/docker

Docker and Consul support

.travis.yml

@@ -45,6 +45,7 @@ before_script:
 script:
   - flake8 density/
   - flake8 density/tests
+  - source config/settings.dev
   - cd density && nosetests
 
 notifications:

Dockerfile

@@ -0,0 +1,15 @@
+# use Docker's provided python image
+FROM python:2.7
+MAINTAINER natebrennand <natebrennand@gmail.com>
+
+# install all packages
+COPY ./config/requirements.txt /requirements.txt
+RUN pip install -r /requirements.txt
+
+# add the application directories
+ADD ./density /density
+WORKDIR /density
+
+# expose the port and start the server
+EXPOSE 6002
+CMD gunicorn density:app -b 0.0.0.0:6002 --log-file /opt/logs/gunicorn.log --log-level debug

README.md

@@ -51,6 +51,55 @@ Be sure to also insert the [Oauth table](config/oauth_dev_dump.sql) so that you
 
 
 
+
+
+## Docker
+
+The Docker container requires that either the port for the Postgres instance is forward or that the host is set to an exact IP or domain.
+
+Using Docker:
+
+```bash
+# enter the project directory
+cd density
+
+# builds a Docker image
+#   -t dictates that the image is tagged as 'density'
+docker build -t density .
+
+# runs a docker container tagged as 'density'
+#   --net=host forwards all ports from the host to the container
+#       this allows docker to access the Postres port, 5432
+#   -e allows setting environment variables within the container
+#   -d detaches the process and runs the container like a daemon
+docker run --net=host -d density
+
+# ps shows all docker containers currently running
+docker ps
+```
+
+Running Density within Docker relies on an instance of Consul being run on `localhost:8500` which is used to configure the settings.
+
+
+
+
+
+## Consul
+
+[Consul](https://consul.io/) is a distributed key/value store that we are using to configure Density in production.
+You should not need to intereact with Consul unless you are attempting to deploy it.
+
+Consul is installed within the Vagrant VM automatically.
+It can be started with the script at `config/run_consul.sh`.
+The web UI is located at [port 8500](http://localhost:8500).
+
+
+
+
+
+
+
+
 ## Routes
 
 Supported routes currently include:

Vagrantfile

@@ -10,11 +10,12 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # please see the online documentation at vagrantup.com.
 
   # Every Vagrant virtual environment requires a box to build off of.
-  config.vm.box = "precise64"
-  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+  config.vm.box = "ubuntu/trusty64"
 
   # expose port 5000 for Flask
   config.vm.network :forwarded_port,  guest: 5000,    host: 5000
+  # expose port 8500 for Consul
+  config.vm.network :forwarded_port,  guest: 8500,    host: 8500
 
   # run the install script for dependencies
   config.vm.provision :shell, :path => "config/bootstrap.sh"

config/bootstrap.sh

@@ -1,5 +1,17 @@
 #!/usr/bin/env bash
 
+
+# add swap if DNE
+# swap is necessary for using Docker
+if [ $(sudo swapon -s | wc -l) -eq 1 ]
+then
+    fallocate -l 2G /swapfile
+    chmod 600 /swapfile
+    mkswap /swapfile
+    swapon /swapfile
+fi
+
+
 # installs the package passed in if it's not installed
 install () {
     package=$1
@@ -42,4 +54,28 @@ pip install flake8  # for local testing
 # install vim
 install vim
 
+# install docker
+install docker.io
+service restart docker.io
+
+install curl
+install unzip
+
+# install consul if it is not already present
+if [[ ! $(which consul) ]]
+then
+    mkdir -p /var/lib/consul
+    mkdir -p /usr/share/consul
+    mkdir -p /etc/consul/conf.d
+
+    curl -OL https://dl.bintray.com/mitchellh/consul/0.5.0_linux_amd64.zip
+    unzip 0.5.0_linux_amd64.zip
+    mv consul /usr/local/bin/consul
+
+    curl -OL https://dl.bintray.com/mitchellh/consul/0.5.0_web_ui.zip
+    unzip 0.5.0_web_ui.zip
+    mv dist /usr/share/consul/ui
+fi
+
+
 exit 0

config/requirements.txt

@@ -1,13 +1,26 @@
 Flask==0.10.1
+Flask-Mail==0.9.1
 Jinja2==2.7.3
 MarkupSafe==0.23
-Werkzeug==0.9.6
-flake8==2.2.3
+Werkzeug==0.10.1
+argparse==1.2.1
+blinker==1.3
+colorama==0.2.5
+flake8==2.3.0
+gunicorn==19.2.1
+html5lib==0.999
+httplib2==0.9
 itsdangerous==0.24
-mccabe==0.2.1
-pep8==1.5.7
-psycopg2==2.5.4
+mccabe==0.3
+oauth2client==1.4.6
+pep8==1.6.2
+psycopg2==2.6
+pyasn1==0.1.7
+pyasn1-modules==0.0.5
 pyflakes==0.8.1
+python-consul==0.3.10
+requests==2.3.0
+rsa==3.1.4
+six==1.9.0
+urllib3==1.7.1
 wsgiref==0.1.2
-oauth2client==1.4.1
-Flask-Mail==0.9.1

config/run_consul.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+# http://www.consul.io/docs/commands/index.html
+
+consul agent \
+    -server \
+    -bootstrap \
+    -client 0.0.0.0 \
+    -data-dir ~/consul \
+    -ui-dir /usr/share/consul/ui
+

config/settings.dev

@@ -13,3 +13,5 @@ export PG_HOST="localhost"
 export PG_PORT="5432"
 
 export GOOGLE_CLIENT_ID="859795907220-57lf7t8m19a1i3huaogqg546u5efjk8j.apps.googleusercontent.com"
+
+export USE_ENV_VARS="TRUE"

config/setup_consul_dev.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+curl -X PUT -d '5000' http://localhost:8500/v1/kv/density/flask_port
+curl -X PUT -d 'TRUE' http://localhost:8500/v1/kv/density/flask_debug
+curl -X PUT -d '859795907220-57lf7t8m19a1i3huaogqg546u5efjk8j.apps.googleusercontent.com' http://localhost:8500/v1/kv/density/google_client_id
+curl -X PUT -d 'density' http://localhost:8500/v1/kv/density/postgres_database
+curl -X PUT -d 'localhost' http://localhost:8500/v1/kv/density/postgres_host
+curl -X PUT -d 'adi' http://localhost:8500/v1/kv/density/postgres_password
+curl -X PUT -d '5432' http://localhost:8500/v1/kv/density/postgres_port
+curl -X PUT -d 'adi' http://localhost:8500/v1/kv/density/postgres_user
+curl -X PUT -d 'abc123' http://localhost:8500/v1/kv/density/secret_key

density/config/flask_config.py

@@ -1,4 +1,3 @@
-
 """
 Collects settings from the environment and adds them to the app configuration.
 
@@ -11,32 +10,72 @@
 from sys import exit
 from datetime import datetime
 
+from consul import Consul
+from flask.json import JSONEncoder
 
-try:
-    # flask settings
-    HOST = environ['HOST']
-    PORT = environ['PORT']
-    SECRET_KEY = environ['SECRET_KEY']
-    DEBUG = True if environ['DEBUG'] == 'TRUE' else False
-
-    # postgres settings
-    PG_USER = environ['PG_USER']
-    PG_PASSWORD = environ['PG_PASSWORD']
-    PG_DB = environ['PG_DB']
-    PG_HOST = environ['PG_HOST']
-    PG_PORT = environ['PG_PORT']
-
-    # oauth settings
-    GOOGLE_CLIENT_ID = environ['GOOGLE_CLIENT_ID']
-
-    if not DEBUG:
-        MAIL_SERVER = 'smtp.gmail.com'
-        MAIL_PORT = 465
-        MAIL_USE_SSL = True
-        MAIL_USE_TLS = False
-        MAIL_DEFAULT_SENDER = 'densitylogger@gmail.com'
-        MAIL_USERNAME = 'densitylogger@gmail.com'
-        MAIL_PASSWORD = environ['MAIL_PASSWORD']
+# dictionary the flask app configures itself from
+config = {
+    'HOST': '0.0.0.0',
+    'PORT': None,
+    'SECRET_KEY': None,
+    'PG_USER': None,
+    'PG_PASSWORD': None,
+    'PG_DB': None,
+    'PG_HOST': None,
+    'PG_PORT': None,
+    'GOOGLE_CLIENT_ID': None,
+    'DEBUG': None
+}
+
+# consul_configurations contains equivalent keys that will be used to extract
+# configuration values from Consul.
+consul_configurations = [  # consul key --> config key
+    ('flask_port', 'PORT'),
+    ('flask_debug', 'DEBUG'),
+    ('secret_key', 'SECRET_KEY'),
+    ('postgres_user', 'PG_USER'),
+    ('postgres_password', 'PG_PASSWORD'),
+    ('postgres_database', 'PG_DB'),
+    ('postgres_host', 'PG_HOST'),
+    ('postgres_port', 'PG_PORT'),
+    ('google_client_id', 'GOOGLE_CLIENT_ID'),
+]
+
+if environ.get('USE_ENV_VARS') == 'TRUE':
+    try:  # use local settings
+        for env_key, value in config.iteritems():
+            if not value:
+                config[env_key] = environ[env_key]
+
+    except KeyError as e:
+        """ Throw an error if a setting is missing """
+        print "ERR MSG: {}".format(e.message)
+        print ("Some of your settings aren't in the environment."
+               "You probably need to run:"
+               "\n\n\tsource config/<your settings file>")
+        exit(1)
+
+else:  # use consul
+    kv = Consul().kv  # initalize client to KV store
+
+    for consul_key, config_key in consul_configurations:
+        _, tmp = kv.get("density/{}".format(consul_key))
+        val = tmp.get('Value')
+        config[config_key] = val
+        if not val:
+            raise Exception(("no value found in Consul for key "
+                             "density/{}").format(consul_key))
+
+    # mail settings
+    MAIL_SERVER = 'smtp.gmail.com'
+    MAIL_PORT = 465
+    MAIL_USE_SSL = True
+    MAIL_USE_TLS = False
+    MAIL_DEFAULT_SENDER = 'densitylogger@gmail.com'
+    MAIL_USERNAME = 'densitylogger@gmail.com'
+    _, MAIL_PASSWORD = kv.get('density/mail_password')
+    if not MAIL_PASSWORD:
+        raise Exception("No password for Mail found in Consul")
 
     # administrator list
     ADMINS = [
@@ -48,18 +87,10 @@
         'jzf2101@columbia.edu'
     ]
 
-except KeyError as e:
-    """ Throw an error if a setting is missing """
-    print "ERR MSG: {}".format(e.message)
-    print ("Some of your settings aren't in the environment."
-           "You probably need to run:"
-           "\n\n\tsource config/<your settings file>")
-    exit(1)
-
+config['DEBUG'] = (config['DEBUG'] == 'TRUE')
 
 """ Creates a json encoder that returns ISO 8601 strings for datetimes
     http://flask.pocoo.org/snippets/119/ """
-from flask.json import JSONEncoder
 
 
 class ISO8601Encoder(JSONEncoder):

density/density.py

@@ -1,15 +1,6 @@
 from flask import Flask, g, jsonify, render_template, json, request
 from flask_mail import Message, Mail
-app = Flask(__name__)
-
-# do import early to check that all env variables are present
-if not app.debug:
-    mail = Mail(app)
-
-# change the default JSON encoder to handle datetime's properly
 from config import flask_config
-app.config.from_object('config.flask_config')
-app.json_encoder = flask_config.ISO8601Encoder
 
 # library imports
 import psycopg2
@@ -24,6 +15,14 @@
 import re
 from functools import wraps
 
+app = Flask(__name__)
+app.config.update(**flask_config.config)
+if not app.debug:
+    mail = Mail(app)
+
+# change the default JSON encoder to handle datetime's properly
+app.json_encoder = flask_config.ISO8601Encoder
+
 with open('data/capacity_group.json') as json_data:
     FULL_CAP_DATA = json.load(json_data)['data']
 
@@ -203,8 +202,8 @@ def auth():
         http = httplib2.Http()
         http = credentials.authorize(http)
 
-        h, content = http.request('https://www.googleapis.com/plus/v1/people/'
-                                  + gplus_id, 'GET')
+        h, content = http.request(
+            'https://www.googleapis.com/plus/v1/people/' + gplus_id, 'GET')
         data = json.loads(content)
         email = data["emails"][0]["value"]
 
@@ -216,8 +215,7 @@ def auth():
                                    success=False,
                                    reason="Please log in with your " +
                                    "Columbia or Barnard email. You logged " +
-                                   "in with: " +
-                                   email)
+                                   "in with: " + email)
 
         # Get UNI and ask database for code.
         uni = regex.group('uni')