Adds experiment driver and gh-webhook handler

fixes getpopper/popper#49
fixes getpopper/popper#51
fixes getpopper/popper#75
fixes getpopper/popper#84
fixes getpopper/popper#87
fixes getpopper/popper#88

.gitignore

@@ -1,3 +1,4 @@
+*.sw?
 *.pyc
 frontend/applications/__init__.py
 frontend/httpserver.log
@@ -7,4 +8,4 @@ frontend/welcome.w2p
 frontend/httpserver.pid
 .ropeproject
 frontend/.idea/
-.idea/
+.idea/

webhook/README.md

@@ -0,0 +1,40 @@
+# Hook handler for PopperCI
+
+The hook expects:
+
+  * A `WORKSPACE` variable with the path to the workspace folder.
+  * A `CREDENTIALS` variable with the path to the credentials folder.
+  * A `SQLITEDB` path pointing to a sqlite3 database.
+
+We have a convenience container for this, to run:
+
+```bash
+docker run --name webhook-handler \
+  -v `pwd`/frontend/gh-webhook:/app/hooks \
+  -v /path/to/credentials:/path/to/credentials \
+  -v /path/to/workspace:path/to/workspace \
+  -v /path/to/db.sqlite:/path/to/db.sqlite \
+  -v /usr/bin/docker:/usr/bin/docker \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -e WORKSPACE=/path/to/workspace \
+  -e CREDENTIALS=/path/to/credentials \
+  -e SQLITEDB=/path/to/db.sqlite \
+  -p 5000:5000
+  ivotron/webhook-handler
+```
+
+The above hook container launches other docker containers (as 
+explained 
+[here](https://jpetazzo.github.io/2015/09/03/do-not-use-docker-in-docker-for-ci/)), 
+so the `docker` binary has to be available inside the 
+`webhook-handler` container. To test, create a `payload.json` file 
+based on [GitHub's 
+schema](https://developer.github.com/v3/activity/events/types/#pushevent) 
+and then:
+
+```bash
+curl \
+  -vX POST http://localhost:5000/webhooks \
+  -d @payload.json \
+  --header "Content-Type: application/json"
+```

webhook/push

@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+
+import logging
+import json
+import os
+import os.path
+import sys
+
+from sqlalchemy import *
+from subprocess import Popen, PIPE
+
+workspace_path = os.environ['WORKSPACE']
+credentials_path = os.environ['CREDENTIALS']
+sqlite_file = os.environ['SQLITEDB']
+
+logger = logging.getLogger('webhook-handler')
+logger.setLevel(logging.DEBUG)
+ch = logging.StreamHandler()
+logger.addHandler(ch)
+
+
+def sh(cmd):
+    process = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
+    out, err = process.communicate()
+    ecode = process.returncode
+
+    return ecode, out, err
+
+
+def shf(cmd):
+    # run sh and fail if non-zero exit
+    ecode, _, _ = sh(cmd)
+    if ecode != 0:
+        raise Exception("Non-zero exit ({}) for '{}'".format(ecode, cmd))
+
+
+def popper_check_experiment(workspace, environment):
+    cmd = (
+        "docker run --rm "
+        "  --volume `which docker`:/usr/bin/docker "
+        "  --volume /var/run/docker.sock:/var/run/docker.sock "
+        "  --volume {0}:{0} "
+        "  --workdir {0} "
+        "  {1} "
+        "  ivotron/popperci"
+    ).format(workspace, ' '.join(environment))
+
+    logger.info("Executing: " + cmd)
+
+    ecode, _, _ = sh(cmd)
+
+    return ecode
+
+
+def get_project_info(db, payload):
+    project = payload['repository']['name']
+    url = payload['repository']['url']
+    uname = payload['repository']['owner']['name']
+
+    users = Table('auth_user', MetaData(db), autoload=True)
+    user = users.select(users.c.username == uname).execute().fetchone()
+
+    if not user:
+        raise Exception("User {} not registered.".format(uname))
+
+    return user.id, uname, project, url
+
+
+def insert_execution(db, uid, project, payload):
+    executions = Table('build', MetaData(db), autoload=True)
+    res = executions.insert().execute(user_id=uid, project=project,
+                                      meta=json.dumps(payload))
+    return res.inserted_primary_key[0]
+
+
+def get_credentials(db, uid):
+    creds = Table('credentials', MetaData(db), autoload=True)
+    return creds.select(creds.c.owner_id == uid).execute()
+
+
+def insert_result(db, exec_id, ecode, experiment, workspace):
+    if ecode != 0:
+        experiment_status = 'FAIL'
+    else:
+        with open('{}/popper_status'.format(workspace), 'r') as f:
+            experiment_status = f.read().strip()
+
+    results = Table('experiment', MetaData(db), autoload=True)
+    results.insert().execute(build_id=exec_id, experiment_name=experiment,
+                             status=experiment_status)
+
+
+def popper_check_project(payload):
+
+    db = create_engine('sqlite:///{}'.format(sqlite_file))
+
+    uid, uname, project, url = get_project_info(db, payload)
+
+    exec_id = insert_execution(db, uid, project, payload)
+
+    workspace = '{}/{}/{}/{}'.format(workspace_path, uname, project, exec_id)
+
+    shf("mkdir -p " + workspace)
+    shf("git clone --recursive {} {}".format(url, workspace))
+
+    environment = []
+
+    for c in get_credentials(db, uid):
+        if c.cred_text:
+            env_item = '-e {}={}'.format(c.name, c.cred_text)
+        else:
+            cred_path = credentials_path + '/' + c.cred_file
+            env_item = '-v {}:{}'.format(cred_path, c.name)
+
+        environment += [env_item]
+
+    for experiment in os.listdir(workspace + '/experiments'):
+
+        experiment_path = workspace + '/experiments/' + experiment
+        ecode = popper_check_experiment(experiment_path, environment)
+
+        insert_result(db, exec_id, ecode, experiment, experiment_path)
+
+    db.dispose()
+
+
+if __name__ == '__main__':
+    if not sys.argv[1] or not sys.argv[2]:
+        raise Exception("Usage: ./push <json-file> event ")
+
+    if sys.argv[2] != "push":
+        print("Only 'push' is supported")
+        sys.exit(0)
+
+    if not os.path.isfile(sqlite_file):
+        raise Exception("Can't find file {}".format(sqlite_file))
+    if not os.path.isdir(credentials_path):
+        raise Exception("Can't find folder {}".format(credentials_path))
+    if not os.path.isdir(workspace_path):
+        raise Exception("Can't find folder {}".format(workspace_path))
+
+    with open(sys.argv[1], 'r') as f:
+        payload = json.load(f)
+
+    popper_check_project(payload)