Asynchronous refactoring using RQ

Procfile

@@ -0,0 +1,3 @@
+web: gunicorn -w 4 invenio_kwalitee:app
+broker: redis-server
+worker: python -u -m invenio_kwalitee.worker

README.rst

@@ -23,6 +23,40 @@ or (to also show test coverage) ::
 
     python setup.py nosetests
 
+Deployment
+==========
+
+Upstart (Ubuntu)
+----------------
+
+The web application can be served using nginx_ + uWSGI_ or gunicorn_ and the
+worker can also be handled using upstart_. Here is the configuration for it.
+VirtualEnv_ is a clean way to set everything up and is recommended.::
+
+    # /etc/init/<myservice>.conf
+    description "Kwalitee RQ worker"
+
+    respawn
+    respawn limit 15 5
+    console log
+    setuid <USER>
+    setgid <GROUP>
+
+    exec /usr/bin/python -m invenio_kwalitee.worker
+    # Or if you've set it up in a virtualenv
+    #exec <VIRTUALENV>/bin/python -m invenio_kwalitee.worker
+
+Then, you can manage it using upstart like anything else.::
+
+    $ sudo start <myservice>
+    $ sudo stop <myservice>
+
+.. _nginx: http://gunicorn-docs.readthedocs.org/en/latest/deploy.html
+.. _uWSGI: http://uwsgi-docs.readthedocs.org/en/latest/Upstart.html
+.. _gunicorn: http://gunicorn-docs.readthedocs.org/en/latest/deploy.html#upstart
+.. _upstart: http://upstart.ubuntu.com/
+.. _VirtualEnv: http://virtualenv.readthedocs.org/en/latest/virtualenv.html
+
 License
 =======
 Copyright (C) 2014 CERN.

invenio_kwalitee/__init__.py

@@ -24,53 +24,81 @@
 import os
 import operator
 
-from flask import Flask, jsonify, render_template, request, make_response
+from flask import (Flask, json, jsonify, make_response, render_template,
+                   request, url_for)
+from rq import Queue
 
-from .kwalitee import Kwalitee
+from .kwalitee import pull_request
+from .worker import conn
 
-app = Flask(__name__, template_folder='templates', static_folder='static',
+app = Flask(__name__, template_folder="templates", static_folder="static",
             instance_relative_config=True)
 
 # Load default configuration
-app.config.from_object('invenio_kwalitee.config')
+app.config.from_object("invenio_kwalitee.config")
 
 # Load invenio_kwalitee.cfg from instance folder
-app.config.from_pyfile('invenio_kwalitee.cfg', silent=True)
-app.config.from_envvar('INVENIO_KWALITEE_CONFIG', silent=True)
+app.config.from_pyfile("invenio_kwalitee.cfg", silent=True)
+app.config.from_envvar("INVENIO_KWALITEE_CONFIG", silent=True)
+app.config["queue"] = Queue(connection=conn)
 
-# Create kwalitee instance
-kw = Kwalitee(app)
 
 # Create instance path
 try:
     if not os.path.exists(app.instance_path):
         os.makedirs(app.instance_path)  # pragma: no cover
-except Exception: # pragma: no cover
+except Exception:  # pragma: no cover
     pass
 
 
-@app.route('/status/<commit_sha>')
+@app.route("/status/<commit_sha>")
 def status(commit_sha):
     with app.open_instance_resource(
-            'status_{sha}.txt'.format(sha=commit_sha), 'r') as f:
+            "status_{sha}.txt".format(sha=commit_sha), "r") as f:
         status = f.read()
-    status = status if len(status) > 0 else commit_sha + ': Everything OK'
-    return render_template('status.html', status=status)
+    status = status if len(status) > 0 else commit_sha + ": Everything OK"
+    return render_template("status.html", status=status)
 
 
-@app.route('/', methods=['GET'])
+@app.route("/", methods=["GET"])
 def index():
     key = lambda x: os.path.getctime(os.path.join(app.instance_path, x))
-    test = operator.methodcaller('startswith', 'status_')
+    test = operator.methodcaller("startswith", "status_")
     files = map(lambda x: x[7:-4], filter(test, sorted(
         os.listdir(app.instance_path), key=key, reverse=True)))
-    return render_template('index.html', files=files)
+    return render_template("index.html", files=files)
 
 
-@app.route('/payload', methods=['POST'])
+@app.route("/payload", methods=["POST"])
 def payload():
+    q = app.config["queue"]
     try:
-        return jsonify(payload=kw(request))
+        event = None
+        if "X-GitHub-Event" in request.headers:
+            event = request.headers["X-GitHub-Event"]
+        else:
+            raise ValueError("No X-GitHub-Event HTTP header found")
+
+        if event == "ping":
+            payload = {"message": "pong"}
+        elif event == "pull_request":
+            data = json.loads(request.data)
+            pull_request_url = data["pull_request"]["url"]
+            commit_sha = data["pull_request"]["head"]["sha"]
+            status_url = url_for("status", commit_sha=commit_sha,
+                                 _external=True)
+            config = dict(app.config, instance_path=app.instance_path)
+            del config["queue"]
+            q.enqueue(pull_request, pull_request_url, status_url, config)
+            payload = {
+                "state": "pending",
+                "target_url": status_url,
+                "description": "kwalitee is working this commit out"
+            }
+        else:
+            raise ValueError("Event {0} is not supported".format(event))
+
+        return jsonify(payload=payload)
     except Exception as e:
         import traceback
         # Uncomment to help you debug

invenio_kwalitee/kwalitee.py

@@ -21,10 +21,11 @@
 ## granted to it by virtue of its status as an Intergovernmental Organization
 ## or submit itself to any jurisdiction.
 
+import os
 import re
 import requests
 import operator
-from flask import current_app, json, url_for
+from flask import json
 
 
 # Max number of errors to be sent back
@@ -90,7 +91,7 @@ def _check_signatures(lines, signatures, trusted, **kwargs):
     errors = []
     test = operator.methodcaller('startswith', signatures)
     for i, line in lines:
-        if test(line):
+        if signatures and test(line):
             matching.append(line)
         else:
             errors.append('Unrecognized bullet/signature on line {0}: "{1}"'
@@ -120,87 +121,52 @@ def check_message(message, **kwargs):
     return errors
 
 
-class Kwalitee(object):
-    def __init__(self, app=None, **kwargs):
-        if app is not None:
-            kwargs.update({
-                "components": app.config["COMPONENTS"],
-                "signatures": app.config["SIGNATURES"],
-                "trusted": app.config["TRUSTED_DEVELOPERS"],
-            })
-        self.config = kwargs
-        # This is your Github personal API token, our advice is to
-        # put it into instance/invenio_kwalitee.cfg so it won't be
-        # versioned ever. Keep it safe.
-        self.token = app.config.get("ACCESS_TOKEN", None)
-
-    @property
-    def token(self):
-        return self._token
-
-    @token.setter
-    def token(self, value):
-        self._token = value
-
-    def __headers(self):
-        headers = {"Content-Type": "application/json"}
-        if self._token is not None:
-            headers["Authorization"] = "token {0}".format(self._token)
-        return headers
-
-    def __call__(self, request):
-        self.request = request
-        if "X-GitHub-Event" in request.headers:
-            event = request.headers["X-GitHub-Event"]
-        else:
-            raise ValueError("No X-GitHub-Event HTTP header found")
-
-        data = json.loads(request.data)
-        fn = getattr(self, "on_{0}".format(event))
-
-        return fn(data)
-
-    def __getattr__(self, command):
-        raise NotImplementedError("{0}.{1} method is missing"
-                                  .format(self.__class__.__name__, command))
-
-    def on_ping(self, data):
-        return dict(message="Hi there!")
-
-    def on_pull_request(self, data):
-        errors = []
-
-        commits_url = data['pull_request']['commits_url']
-        commit_sha = data["pull_request"]["head"]["sha"]
-
-        # Check only if the title does not contain 'wip'.
-        must_check = re.search(r"\bwip\b",
-                               data['pull_request']['title'],
-                               re.IGNORECASE) is None
-
-        if must_check is True:
-            response = requests.get(commits_url)
-            commits = json.loads(response.content)
-            for commit in commits:
-                sha = commit["sha"]
-                message = commit["commit"]["message"]
-                errs = check_message(message, **self.config)
-
-                requests.post(commit["comments_url"],
-                              data=json.dumps({"body": "\n".join(errs)}),
-                              headers=self.__headers())
-                errors += map(lambda x: "%s: %s" % (sha, x), errs)
-
-            filename = "status_{0}.txt".format(commit_sha)
-            with current_app.open_instance_resource(filename, "w+") as f:
-                f.write("\n".join(errors))
-
-        state = "error" if len(errors) > 0 else "success"
-        body = dict(state=state,
-                    target_url=url_for("status", commit_sha=commit_sha,
-                                       _external=True),
-                    description="\n".join(errors)[:MAX])
-        requests.post(data["pull_request"]["statuses_url"],
-                      data=json.dumps(body),
-                      headers=self.__headers())
-        return body
+def pull_request(pull_request_url, status_url, config):
+    errors = []
+    pull_request = requests.get(pull_request_url)
+    data = json.loads(pull_request.content)
+    kwargs = {
+        "components": config.get("COMPONENTS", []),
+        "signatures": config.get("SIGNATURES", []),
+        "trusted": config.get("TRUSTED_DEVELOPERS", [])
+    }
+    headers = {
+        "Content-Type": "application/json",
+        # This is required to post comments on GitHub on yours behalf.
+        # Please update your configuration accordingly.
+        "Authorization": "token {0}".format(config["ACCESS_TOKEN"])
+    }
+    instance_path = config["instance_path"]
+
+    commits_url = data["commits_url"]
+    commit_sha = data["head"]["sha"]
+
+    # Check only if the title does not contain 'wip'.
+    must_check = re.search(r"\bwip\b",
+                           data["title"],
+                           re.IGNORECASE) is None
+
+    if must_check is True:
+        response = requests.get(commits_url)
+        commits = json.loads(response.content)
+        for commit in commits:
+            sha = commit["sha"]
+            message = commit["commit"]["message"]
+            errs = check_message(message, **kwargs)
+
+            requests.post(commit["comments_url"],
+                          data=json.dumps({"body": "\n".join(errs)}),
+                          headers=headers)
+            errors += map(lambda x: "%s: %s" % (sha, x), errs)
+
+        filename = "status_{0}.txt".format(commit_sha)
+        with open(os.path.join(instance_path, filename), "w+") as f:
+            f.write("\n".join(errors))
+
+    state = "error" if len(errors) > 0 else "success"
+    body = dict(state=state,
+                target_url=status_url,
+                description="\n".join(errors)[:MAX])
+    requests.post(data["statuses_url"],
+                  data=json.dumps(body),
+                  headers=headers)

invenio_kwalitee/worker.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio-Kwalitee
+## Copyright (C) 2014 CERN.
+##
+## Invenio-Kwalitee is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio-Kwalitee is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio-Kwalitee; if not, write to the Free Software Foundation,
+## Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+##
+## In applying this licence, CERN does not waive the privileges and immunities
+## granted to it by virtue of its status as an Intergovernmental Organization
+## or submit itself to any jurisdiction.
+
+import sys
+from redis import Redis
+from rq import Worker, Queue, Connection
+
+conn = Redis()
+
+
+def main(argv):  # pragma: no cover
+    with Connection(conn):
+        worker = Worker(list(map(Queue, ('high', 'default', 'low'))))
+        worker.work()
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main(sys.argv))

setup.py

@@ -49,7 +49,8 @@
     install_requires=[
         'Flask',
         'requests',
-        'six',
+        'rq',
+        'six'
     ],
     classifiers=[
         'Environment :: Web Environment',

tests/test_ping.py

@@ -39,19 +39,6 @@ def test_ping(self):
                                                        "than fast."}))
         self.assertEqual(200, response.status_code)
 
-    def test_ping_fail(self):
-        """POST /payload (ping) rejects non-JSON content"""
-        tester = app.test_client(self)
-        response = tester.post("/payload",
-                               headers=(("X-GitHub-Event", "ping"),
-                                        ("X-GitHub-Delivery", "1")),
-                               data="not JSON")
-        body = json.loads(response.data)
-        self.assertEqual(500, response.status_code)
-        self.assertEqual(u"No JSON object could be decoded",
-                         body["exception"])
-        self.assertEqual(u"failure", body["status"])
-
     def test_ping_no_headers(self):
         """POST /payload (ping) expects a X-GitHub-Event header"""
         tester = app.test_client(self)
@@ -76,6 +63,6 @@ def test_not_a_ping(self):
                                                        "than fast."}))
         body = json.loads(response.data)
         self.assertEqual(500, response.status_code)
-        self.assertEqual(u"Kwalitee.on_pong method is missing",
+        self.assertEqual(u"Event pong is not supported",
                          body["exception"])
         self.assertEqual(u"failure", body["status"])