Permalink
Browse files

Merge pull request #113 from MichaelMayorov/112_github_hooks

Real time updating pull-information on review site
  • Loading branch information...
2 parents 5762d1a + c7230c4 commit abfb09e51689ccee204154a453271a94e178fb89 @certik certik committed Jul 13, 2012
Showing with 278 additions and 13 deletions.
  1. +40 −0 README.rst
  2. +1 −1 web/app.yaml
  3. +154 −1 web/app/main.py
  4. +5 −2 web/app/models.py
  5. +0 −7 web/cron.yaml
  6. +6 −0 web/index.yaml
  7. +6 −2 web/templates/index.html
  8. +66 −0 web/templates/upload_url.html
View
@@ -95,3 +95,43 @@ of the commit you want.
This is also useful for bisecting problems with SymPy Bot. Simply use
git to bisect in your local SymPy repository and pass the SHA1's it
picks to ``sympy-bot -n -m``.
+
+Web interface integration with github
+-------------------------------------
+
+This way is a bit complicated in set up than previous (poll github for new pulls),
+but that will update information about pulls in real time.
+
+SymPy Bot web-interface (which located in under web/) supports integration with
+github via mechanism called hooks http://developer.github.com/v3/repos/hooks/
+
+To use that feature you need to follow these steps:
+
+1. Go to ``http://example.com/upload_pull``, sign in as administrator and press
+ ``generate`` button. After that, all admins will recieve notification with
+ secret URL (you can see a log of all generations in table on that page)
+2. You need to tell github to use this URL, so here steps (replace ``username``
+ and ``repo`` with you values):
+ - Go to https://github.com/user/repo/admin/hooks
+ - Click on ``WebHook URLs`` and add secret URL there.
+ - Find the hook that you want to modify by::
+
+ curl -u username https://api.github.com/repos/username/repo/hooks
+
+ the ``id`` field gives the hook ID, copy and paste the path in the
+ "url" field into the command::
+
+ curl -u username -d '{ "events": [ "pull_request" ] }'
+ https://api.github.com/repos/username/repo/hooks/ID
+
+ You will see that the "events" part::
+
+ "events": [
+ "push"
+ ],
+
+ changed to::
+
+ "events": [
+ "pull_request"
+ ],
View
@@ -1,5 +1,5 @@
application: sympy3
-version: 3
+version: 4
runtime: python
api_version: 1
View
@@ -4,6 +4,7 @@
import sys
import traceback
from random import random
+import urllib2
from google.appengine.dist import use_library
use_library("django", "1.2")
@@ -17,7 +18,7 @@
from jsonrpc_client import JSONRPCService, JSONRPCError
from jsonrpc_server import JSONRPCServer
-from models import PullRequest, Task, User
+from models import PullRequest, Task, User, UploadURL
from github import (github_get_pull_request_all_v2,
github_get_pull_request_all_v3, github_get_pull_request,
github_get_user)
@@ -29,6 +30,10 @@
else:
url_base = "http://reviews.sympy.org"
+# default github user and repo
+polled_user = "sympy"
+polled_repo = "sympy"
+
class RequestHandler(webapp.RequestHandler):
def render(self, temp, data=None):
@@ -256,6 +261,152 @@ def user():
else:
raise ValueError("wrong type")
+class UploadPull(RequestHandler):
+
+ def post(self, url_path):
+ last_row = UploadURL.all().order("-created_at").get()
+ if last_row:
+ if last_row.url_path == url_path:
+ try:
+ payload = json.loads(self.request.get("payload"))
+ logging.info(payload)
+ except json.JSONDecodeError:
+ self.error(400)
+ self.response.out.write("Incorrect request format\n")
+ user_repo = payload["repository"]["full_name"]
+ # Download complete pull request with information about mergeability
+ pull_request = github_get_pull_request(user_repo, payload["number"])
+ num = payload["number"]
+ # Get the old entity or create a new one:
+ p = PullRequest.all()
+ p.filter("num =", int(num))
+ p = p.get()
+ if p is None:
+ p = PullRequest(num=num)
+ # Update all data that we can from GitHub:
+ p.url = pull_request["html_url"]
+ p.state = pull_request["state"]
+ p.title = pull_request["title"]
+ p.body = pull_request["body"]
+ p.mergeable = pull_request["mergeable"]
+ if pull_request["head"]["repo"]:
+ p.repo = pull_request["head"]["repo"]["url"]
+ p.branch = pull_request["head"]["ref"]
+ p.author_name = pull_request["user"].get("name", "")
+ p.author_email = pull_request["user"].get("email", "")
+ created_at = pull_request["created_at"]
+ created_at = datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
+ p.created_at = created_at
+ u = User.all()
+ u.filter("login =", pull_request["user"]["login"])
+ u = u.get()
+ if u is None:
+ u = User(login=pull_request["user"]["login"])
+ u.id = pull_request["user"]["id"]
+ u.avatar_url = pull_request["user"]['avatar_url']
+ u.url = pull_request["user"]["url"]
+ u.put()
+ p.author = u
+ p.put()
+ else:
+ self.error(404)
+ self.response.out.write("Requesting URL doesn't exist\n")
+ else:
+ self.error(500)
+ self.response.out.write("URL for posting data not defined yet\n")
+
+ def get(self, url_path):
+
+ def notify_admins(user, new_url):
+ from google.appengine.api.mail import send_mail_to_admins
+ mail = user.email()
+ subject = "SymPy bot notification"
+ body = "New upload URL " + new_url
+ send_mail_to_admins(sender=mail, subject=subject, body=body)
+
+ from google.appengine.api import users
+ user = users.get_current_user()
+ is_admin = users.is_current_user_admin()
+ rows = []
+ upload_url = ""
+ if user:
+ if is_admin:
+ if self.request.get("generate"):
+ import sha
+ rand_string = os.urandom(10)
+ sha_hash = sha.new(rand_string)
+ new_record = UploadURL(url_path=sha_hash.hexdigest(), user=user.nickname())
+ new_record.put()
+ new_url = self.request.host_url + "/upload_pull/" + \
+ sha_hash.hexdigest()
+ notify_admins(user, new_url)
+ if self.request.get("populate"):
+ taskqueue.add(url="/worker-ng", queue_name="github")
+ rows = UploadURL.all()
+ last_row = rows.order("-created_at").get()
+ if last_row:
+ upload_url = (last_row.url_path)
+ else:
+ upload_url = ("")
+
+ self.render("upload_url.html", {"user": user,
+ "upload_url": upload_url,
+ "is_admin": is_admin,
+ "rows": rows,
+ "login_url": users.create_login_url("/upload_pull"),
+ "logout_url": users.create_logout_url("/upload_pull"),
+ }
+ )
+
+
+class WorkerNG(webapp.RequestHandler):
+ """
+ This class using for populating pull requests database and users
+ (calls when admin press "Populate" button)
+ """
+ def post(self):
+ user_repo = polled_user + "/" + polled_repo
+ payload = github_get_pull_request_all_v3(user_repo)
+ # checkout mergeability
+ for pos in xrange(len(payload)):
+ pull = github_get_pull_request(user_repo, payload[pos]["number"])
+ payload[pos]["mergeable"] = pull["mergeable"]
+ # Process each pull request from payload
+ for pull in payload:
+ p = PullRequest.all()
+ num = pull["number"]
+ p.filter("num =", num)
+ p = p.get()
+ if p is None:
+ p = PullRequest(num=num)
+ p.url = pull["html_url"]
+ p.state = pull["state"]
+ p.title = pull["title"]
+ p.body = pull["body"]
+ p.mergeable = pull["mergeable"]
+ if pull["head"]["repo"]:
+ p.repo = pull["head"]["repo"]["url"]
+ p.branch = pull["head"]["ref"]
+ created_at = pull["created_at"]
+ created_at = datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
+ p.created_at = created_at
+
+ # Collect public information about user
+ u = User.all()
+ login = pull["user"]["login"]
+ u.filter("login =", login)
+ u = u.get()
+ if u is None:
+ u = User(login=login)
+ u.id = pull["user"]["id"]
+ u.avatar_url = pull["user"]["avatar_url"]
+ u.url = pull["user"]["url"]
+ u.put()
+
+ p.author = u
+ p.put()
+
+
def main():
urls = [
('/', MainPage),
@@ -266,6 +417,8 @@ def main():
('/update/?', UpdatePage),
('/quickupdate/?', QuickUpdatePage),
('/worker/?', Worker),
+ ('/upload_pull/?(.*)/?', UploadPull),
+ ('/worker-ng/?', WorkerNG),
]
application = webapp.WSGIApplication(urls, debug=True)
run_wsgi_app(application)
View
@@ -5,8 +5,6 @@ class User(db.Model):
id = db.IntegerProperty()
url = db.StringProperty()
avatar_url = db.StringProperty()
- name = db.StringProperty()
- email = db.StringProperty()
class PullRequest(db.Model):
num = db.IntegerProperty(required=True)
@@ -28,3 +26,8 @@ class Task(db.Model):
interpreter = db.StringProperty()
testcommand = db.StringProperty()
uploaded_at = db.DateTimeProperty(auto_now_add=True)
+
+class UploadURL(db.Model):
+ url_path = db.StringProperty()
+ user = db.StringProperty()
+ created_at = db.DateTimeProperty(auto_now_add=True)
View
@@ -1,7 +0,0 @@
-cron:
-- description: update from GitHub (quick)
- url: /quickupdate
- schedule: every 12 hours
-- description: update from GitHub (full)
- url: /update
- schedule: every day 00:00
View
@@ -38,3 +38,9 @@ indexes:
properties:
- name: pullrequest
- name: uploaded_at
+
+- kind: UploadURL
+ properties:
+ - name: created_at
+ direction: desc
+ - name: url_path
View
@@ -46,7 +46,9 @@ <h2>Mergeable Pullrequests</h2>
{% for p in pullrequests_mergeable %}
<tr>
<td><a href="/pullrequest/{{ p.num }}/">#{{ p.num }}</a></td>
- <td>{{ p.author.name }}</td>
+ <td>
+ <a href="https://github.com/{{ p.author.login }}">{{ p.author.login }}</a>
+ </td>
<td>{{ p.created_at|date:"r" }}</td>
<td>{{ p.title }}</td>
</tr>
@@ -71,7 +73,9 @@ <h2>Non-Mergeable Pullrequests</h2>
{% for p in pullrequests_nonmergeable %}
<tr>
<td><a href="/pullrequest/{{ p.num }}/">#{{ p.num }}</a></td>
- <td>{{ p.author.name }}</td>
+ <td>
+ <a href="https://github.com/{{ p.author.login }}">{{ p.author.login }}</a>
+ </td>
<td>{{ p.created_at|date:"r" }}</td>
<td>{{ p.title }}</td>
</tr>
@@ -0,0 +1,66 @@
+{% extends "base.html" %}
+
+{% block title %}
+Generate new URL-path for Github API
+{% endblock %}
+
+{% block runtime %}
+$(document).ready(function() {
+ $('#url_changes').dataTable({
+ "sScrollY": "400px",
+ "bPaginate": false,
+ "aaSorting": [[ 0, "desc" ]]
+ });
+});
+{% endblock %}
+
+{% block content %}
+<form action="/upload_pull/" method="get">
+<h2>Administrator's access required</h2>
+{% if user %}
+ {%if is_admin %}
+ <b>Upload URL:</b>
+ <p>
+ {% if upload_url %}
+ <a href="/upload_pull/{{ upload_url }}">{{ upload_url }}</a>
+ {% else %}
+ There is no upload URL yet, press "Generate" to create new
+ {% endif %}
+ </p>
+ <b>NOTE:</b>
+ <p>
+ If you will press button below, then review site will generate new URL
+ for uploading pull request information and will send an email with new URL
+ to all admins.
+ </p>
+ <input type="submit" name="generate" value="Generate">
+ <input type="submit" name="populate" value="Populate">
+ <table id="url_changes" class="display">
+ <thead>
+ <tr>
+ <th>URL-path</th>
+ <th>Generated by</th>
+ <th>Time</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for row in rows %}
+ <tr>
+ <td>{{ row.url_path }}</a></td>
+ <td>{{ row.user }}</td>
+ <td>{{ row.created_at|date:"r" }}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+
+ {% else %}
+ Your account <b>{{ user.nickname }}</b> has no rights to access this page
+ <br />
+ Maybe <a href="{{ logout_url }}">logout</a> and try again?
+ {% endif %}
+{% else %}
+ <a href="{{ login_url }}">Sign in</a>.
+{% endif %}
+</form>
+{% endblock %}

0 comments on commit abfb09e

Please sign in to comment.