github_jira_sync.py

#!/usr/bin/env python

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Utility for updating JIRA's with information about Github pull requests

import json
import os
import re
import sys
import urllib2

try:
    import jira.client
except ImportError:
    print "This tool requires the jira-python library"
    print "Install using 'sudo pip install jira'"
    sys.exit(-1)

# User facing configs
GITHUB_API_BASE = os.environ.get("GITHUB_API_BASE", "https://api.github.com/repos/apache/spark")
JIRA_PROJECT_NAME = os.environ.get("JIRA_PROJECT_NAME", "SPARK")
JIRA_API_BASE = os.environ.get("JIRA_API_BASE", "https://issues.apache.org/jira")
JIRA_USERNAME = os.environ.get("JIRA_USERNAME", "apachespark")
JIRA_PASSWORD = os.environ.get("JIRA_PASSWORD", "XXX")
# Maximum number of updates to perform in one run
MAX_UPDATES = int(os.environ.get("MAX_UPDATES", "100000"))
# Cut-off for oldest PR on which to comment. Useful for avoiding
# "notification overload" when running for the first time.
MIN_COMMENT_PR = int(os.environ.get("MIN_COMMENT_PR", "1496"))

# File used as an opitimization to store maximum previously seen PR
# Used mostly because accessing ASF JIRA is slow, so we want to avoid checking
# the state of JIRA's that are tied to PR's we've already looked at.
MAX_FILE = ".github-jira-max"

def get_url(url):
    try:
        return urllib2.urlopen(url)
    except urllib2.HTTPError as e:
        print "Unable to fetch URL, exiting: %s" % url
        sys.exit(-1)

def get_json(urllib_response):
    return json.load(urllib_response)

# Return a list of (JIRA id, JSON dict) tuples:
# e.g. [('SPARK-1234', {.. json ..}), ('SPARK-5687', {.. json ..})}
def get_jira_prs():
    result = []
    has_next_page = True
    page_num = 0
    while has_next_page:
	page = get_url(GITHUB_API_BASE + "/pulls?page=%s&per_page=100" % page_num)
	page_json = get_json(page)

	for pull in page_json:
	    jiras = re.findall(JIRA_PROJECT_NAME + "-[0-9]{4,5}", pull['title'])
	    for jira in jiras:
		result = result + [(jira,  pull)]

	# Check if there is another page
	link_header = filter(lambda k: k.startswith("Link"), page.info().headers)[0]
	if not "next"in link_header:
	    has_next_page = False
	else:
	    page_num = page_num + 1
    return result

def set_max_pr(max_val):
    f = open(MAX_FILE, 'w')
    f.write("%s" % max_val)
    f.close()
    print "Writing largest PR number seen: %s" % max_val

def get_max_pr():
    if os.path.exists(MAX_FILE):
        result = int(open(MAX_FILE, 'r').read())
        print "Read largest PR number previously seen: %s" % result
        return result
    else:
        return 0

jira_client = jira.client.JIRA({'server': JIRA_API_BASE},
                                basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))

jira_prs = get_jira_prs()

previous_max = get_max_pr()
print "Retrieved %s JIRA PR's from Github" % len(jira_prs)
jira_prs = [(k, v) for k, v in jira_prs if int(v['number']) > previous_max]
print "%s PR's remain after excluding visted ones" % len(jira_prs)

num_updates = 0
considered = []
for issue, pr in sorted(jira_prs, key=lambda (k, v): int(v['number'])):
    if num_updates >= MAX_UPDATES:
      break
    pr_num = int(pr['number'])

    print "Checking issue %s" % issue
    considered = considered + [pr_num]

    url = pr['html_url']
    title = "[Github] Pull Request #%s (%s)" % (pr['number'], pr['user']['login']) 
    try:
      existing_links = map(lambda l: l.raw['object']['url'], jira_client.remote_links(issue))
    except:
      print "Failure reading JIRA %s (does it exist?)" % issue
      print sys.exc_info()[0]
      continue

    if url in existing_links:
        continue

    icon = {"title": "Pull request #%s" % pr['number'], 
      "url16x16": "https://assets-cdn.github.com/favicon.ico"}
    destination = {"title": title, "url": url, "icon": icon}
    # For all possible fields see:
    # https://developer.atlassian.com/display/JIRADEV/Fields+in+Remote+Issue+Links     
    # application = {"name": "Github pull requests", "type": "org.apache.spark.jira.github"} 
    jira_client.add_remote_link(issue, destination)
    
    comment = "User '%s' has created a pull request for this issue:" % pr['user']['login']
    comment = comment + ("\n%s" % pr['html_url'])
    if pr_num >= MIN_COMMENT_PR:
        jira_client.add_comment(issue, comment)
    
    print "Added link %s <-> PR #%s" % (issue, pr['number'])
    num_updates = num_updates + 1

if len(considered) > 0:
    set_max_pr(max(considered))