Skip to content
This repository has been archived by the owner on Jan 31, 2023. It is now read-only.

Commit

Permalink
Browse files Browse the repository at this point in the history
more plugins, http cache, status filter
  • Loading branch information
robertwb committed Jun 4, 2011
1 parent 6d4fc98 commit 601bd02
Show file tree
Hide file tree
Showing 7 changed files with 152 additions and 39 deletions.
64 changes: 47 additions & 17 deletions buildbot.py
@@ -1,3 +1,4 @@
import signal
import re, os, sys, subprocess, time, traceback
import bz2, urllib2, urllib, json
from optparse import OptionParser
Expand Down Expand Up @@ -27,9 +28,9 @@ def contains_any(key, values):
return {'$or': clauses}

def get_ticket(server, return_all=False, **conf):
query = "raw"
query = "raw&status=open&todo"
if 'trusted_authors' in conf:
query += "&authors=" + ':'.join(conf['trusted_authors'])
query += "&authors=" + urllib.quote_plus(':'.join(conf['trusted_authors']), safe=':')
try:
handle = urllib2.urlopen(server + "/ticket/?" + query)
all = json.load(handle)
Expand Down Expand Up @@ -68,6 +69,8 @@ def rate_ticket(ticket, **conf):
rating = 0
if ticket['spkgs']:
return # can't handle these yet
elif not ticket['patches']:
return # nothing to do
for dep in ticket['depends_on']:
if isinstance(dep, basestring) and '.' in dep:
if compare_version(conf['base'], dep) < 0:
Expand All @@ -83,8 +86,9 @@ def rate_ticket(ticket, **conf):
# TODO: remove condition
if 'component' in ticket:
rating += conf['bonus'].get(ticket['component'], 0)
rating += conf['bonus'].get(ticket['status'], 0)
rating += conf['bonus'].get(ticket['priority'], 0)
rating += conf['bonus'].get(ticket['id'], 0)
rating += conf['bonus'].get(str(ticket['id']), 0)
redundancy = (100,)
prune_pending(ticket)
if not ticket.get('retry'):
Expand All @@ -102,19 +106,19 @@ def parse_datetime(s):
# The one thing Python can't do is parse dates...
return time.mktime(time.strptime(s[:-5].strip(), DATE_FORMAT[:-3])) + 60*int(s[-5:].strip())

def prune_pending(ticket, machine=None):
def prune_pending(ticket, machine=None, timeout=6*60*60):
if 'reports' in ticket:
reports = ticket['reports']
else:
return []
# TODO: is there a better way to handle time zones?
now = time.time() + 60 * int(time.strftime('%z')) + 1000000000000
now = time.time() + 60 * int(time.strftime('%z'))
for report in list(reports):
if report['status'] == 'Pending':
t = parse_datetime(report['time'])
if report['machine'] == machine:
reports.remove(report)
elif now - t > 6 * 60 * 60:
elif now - t > timeout:
reports.remove(report)
return reports

Expand All @@ -140,10 +144,17 @@ def report_ticket(server, ticket, status, base, machine, log, plugins=[]):
except:
traceback.print_exc()

class TimeOut(Exception):
pass

def alarm_handler(signum, frame):
raise Alarm

class Tee:
def __init__(self, filepath, time=False):
def __init__(self, filepath, time=False, timeout=60*60*24):
self.filepath = filepath
self.time = time
self.timeout = timeout

def __enter__(self):
self._saved = os.dup(sys.stdout.fileno()), os.dup(sys.stderr.fileno())
Expand All @@ -165,7 +176,14 @@ def __exit__(self, exc_type, exc_val, exc_tb):
os.dup2(self._saved[0], sys.stdout.fileno())
os.dup2(self._saved[1], sys.stderr.fileno())
time.sleep(1)
self.tee.wait()
try:
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(self.timeout)
self.tee.wait()
signal.alarm(0)
except TimeOut:
traceback.print_exc()
raise
return False


Expand Down Expand Up @@ -227,10 +245,10 @@ def test_a_ticket(sage_root, server, ticket=None, nodocs=False):
if not os.path.exists(log_dir):
os.mkdir(log_dir)
log = '%s/%s-log.txt' % (log_dir, ticket['id'])
#report_ticket(server, ticket, status='Pending', base=base, machine=conf['machine'], log=None)
report_ticket(server, ticket, status='Pending', base=base, machine=conf['machine'], log=None)
plugins_results = []
try:
with Tee(log, time=True):
with Tee(log, time=True, timeout=conf['timeout']):
t = Timer()
start_time = time.time()

Expand All @@ -246,11 +264,12 @@ def test_a_ticket(sage_root, server, ticket=None, nodocs=False):
state = 'built'

working_dir = "%s/devel/sage-%s" % (sage_root, ticket['id'])
patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().split('\n')
# Only the ones on this ticket.
patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):]
kwds = {
"original_dir": "%s/devel/sage-0" % sage_root,
"patched_dir": working_dir,
"patches": ["%s/devel/sage-%s/.hg/patches/%s" % (sage_root, ticket['id'], p) for p in patches],
"patches": ["%s/devel/sage-%s/.hg/patches/%s" % (sage_root, ticket['id'], p) for p in patches if p],
}
for name, plugin in conf['plugins']:
try:
Expand All @@ -266,8 +285,8 @@ def test_a_ticket(sage_root, server, ticket=None, nodocs=False):
plugins_results.append((name, passed))

test_dirs = ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs]
#do_or_die("$SAGE_ROOT/sage -tp %s -sagenb %s" % (conf['parallelism'], ' '.join(test_dirs)))
do_or_die("$SAGE_ROOT/sage -t $SAGE_ROOT/devel/sage-%s/sage/rings/integer.pyx" % ticket['id'])
do_or_die("$SAGE_ROOT/sage -tp %s -sagenb %s" % (conf['parallelism'], ' '.join(test_dirs)))
#do_or_die("$SAGE_ROOT/sage -t $SAGE_ROOT/devel/sage-%s/sage/rings/integer.pyx" % ticket['id'])
#do_or_die('sage -testall')
t.finish("Tests")
state = 'tested'
Expand All @@ -288,11 +307,23 @@ def get_conf(path, **overrides):
conf = {
"idle": 300,
"parallelism": 3,
"plugins": ["plugins.coverage", "plugins.docbuild"],
"timeout": 3 * 60 * 60,
"plugins": ["plugins.commit_messages", "plugins.coverage", "plugins.docbuild"],
"bonus": {},
}
default_bonus = {
"needs_review": 1000,
"positive_review": 500,
"blocker": 100,
"critical": 50,
}
for key, value in unicode_conf.items():
conf[str(key)] = value
for key, value in default_bonus.items():
if key not in conf['bonus']:
conf['bonus'][key] = value
conf.update(overrides)

def locate_plugin(name):
ix = name.rindex('.')
module = name[:ix]
Expand Down Expand Up @@ -333,8 +364,7 @@ def locate_plugin(name):
clean = lookup_ticket(options.server, 0)
def good(report):
return report['machine'] == conf['machine'] and report['status'] == 'TestsPassed'
# TODO: fix to not have to re-start as often
if False and not any(good(report) for report in current_reports(clean, base=get_base(options.sage_root))):
if not any(good(report) for report in current_reports(clean, base=get_base(options.sage_root))):
res = test_a_ticket(ticket=0, sage_root=options.sage_root, server=options.server)
if res != 'TestsPassed':
print "\n\n"
Expand Down
33 changes: 32 additions & 1 deletion plugins.py
Expand Up @@ -10,16 +10,47 @@
ticket -- a dictionary of all the ticket informaton
original_dir -- pristine sage-main directory
patched_dir -- patched sage-branch directory for this ticket
patch_list -- a list of absolute paths to the patch files for this ticket
patchs -- a list of absolute paths to the patch files for this ticket
It is recommended that a plugin ignore extra keywords to be
compatible with future options.
"""

import re, os

from trac import do_or_die


def coverage(ticket, **kwds):
do_or_die('$SAGE_ROOT/sage -coverageall')

def docbuild(ticket, **kwds):
do_or_die('$SAGE_ROOT/sage -docbuild --jsmath reference html')

def commit_messages(ticket, patches, **kwds):
for patch_path in patches:
patch = os.path.basename(patch_path)
print "Looking at", patch
header = []
for line in open(patch_path):
if line.startswith('diff '):
break
header.append(line)
else:
print ''.join(header[:10])
raise ValueError("Not a valid patch file: " + patch)
print ''.join(header)
if header[0].strip() != "# HG changeset patch":
raise ValueError("Not a mercurial patch file: " + patch)
for line in header:
if not line.startswith('# '):
# First description line
if not re.search(r"\b%s\b" % ticket['id'], line):
raise ValueError("Ticket number not in first line of comments:" + patch)
elif line.startswith('[mq]'):
raise ValueError("Mercurial queue boilerplate")
break
else:
raise ValueError("No patch comments:" + patch)
print
print "All patches good."
62 changes: 53 additions & 9 deletions serve.py
Expand Up @@ -24,7 +24,18 @@ def ticket_list():
if 'query' in request.args:
query = json.loads(request.args.get('query'))
else:
query = {'status': 'needs_review'}
status = request.args.get('status', 'needs_review')
if status == 'all':
query = {}
elif status in ('new', 'closed'):
query = {'status': {'$regex': status + '.*' }}
elif status in ('open'):
query = {'status': {'$regex': 'needs_.*|positive_review' }}
else:
query = {'status': status}
if 'todo' in request.args:
query['patches'] = {'$not': {'$size': 0}}
query['spkgs'] = {'$size': 0}
if 'authors' in request.args:
authors = request.args.get('authors').split(':')
query['authors'] = {'$in': authors}
Expand Down Expand Up @@ -60,9 +71,10 @@ def preprocess(all):
summary[ticket['report_status']] += 1
yield ticket
ticket0 = db.lookup_ticket(0)
versions = set(report['base'] for report in ticket0['reports'])
versions = [(v, get_ticket_status(ticket0, v)) for v in sorted(versions)]
return render_template("ticket_list.html", tickets=preprocess(all), summary=summary, base=base, base_status=get_ticket_status(db.lookup_ticket(0), base), versions=versions)
versions = list(set(report['base'] for report in ticket0['reports']))
versions.sort(trac.compare_version)
versions = [(v, get_ticket_status(ticket0, v)) for v in versions if v != '4.7.']
return render_template("ticket_list.html", tickets=preprocess(all), summary=summary, base=base, base_status=get_ticket_status(db.lookup_ticket(0), base), versions=versions, status_order=status_order)

def format_patches(ticket, patches, deps=None, required=None):
if deps is None:
Expand Down Expand Up @@ -108,11 +120,17 @@ def render_ticket(ticket):
info = tickets.find_one({'id': ticket})
if 'kick' in request.args:
info['retry'] = True
db.save_ticket(info)
db.save_ticket(info)
if 'reports' in info:
info['reports'].sort(lambda a, b: -cmp(a['time'], b['time']))
else:
info['reports'] = []

old_reports = list(info['reports'])
buildbot.prune_pending(info)
if old_reports != info['reports']:
db.save_ticket(info)

def format_info(info):
new_info = {}
for key, value in info.items():
Expand Down Expand Up @@ -152,6 +170,7 @@ def render_ticket_status(ticket):
status = get_ticket_status(info, base=base)[1]
response = make_response(open('images/%s-blob.png' % status_colors[status]).read())
response.headers['Content-type'] = 'image/png'
response.headers['Cache-Control'] = 'no-cache'
return response

def get_or_set(ticket, key, default):
Expand Down Expand Up @@ -212,21 +231,42 @@ def shorten(lines):
if prev is not None:
yield prev

def extract_plugin_log(data, plugin):
from buildbot import plugin_boundary
start = plugin_boundary(plugin) + "\n"
end = plugin_boundary(plugin, end=True) + "\n"
all = []
include = False
for line in StringIO(data):
if line == start:
include = True
if include:
all.append(line)
if line == end:
break
return ''.join(all)

@app.route("/ticket/<id>/log/<path:log>")
def get_ticket_log(id, log):
return get_log(log)

@app.route("/log/<path:log>")
def get_log(log):
path = "/log/" + log
if not logs.exists(path):
data = "No such log!"
else:
data = bz2.decompress(logs.get(path).read())
if 'plugin' in request.args:
data = extract_plugin_log(data, request.args.get('plugin'))
if 'short' in request.args:
response = Response(shorten(data), direct_passthrough=True)
else:
response = make_response(data)
response.headers['Content-type'] = 'text/plain'
return response

status_order = ['New', 'ApplyFailed', 'BuildFailed', 'TestsFailed', 'PluginFailed', 'TestsPassed', 'Pending', 'Spkg']
status_order = ['New', 'ApplyFailed', 'BuildFailed', 'TestsFailed', 'PluginFailed', 'TestsPassed', 'Pending', 'NoPatch', 'Spkg']
# TODO: cleanup old records
# status_order += ['started', 'applied', 'built', 'tested']

Expand All @@ -238,13 +278,15 @@ def get_log(log):
'TestsPassed': 'green',
'PluginFailed': 'blue',
'Pending' : 'white',
'Spkg' : 'purple',
'NoPatch' : 'purple',
'Spkg' : 'purple',
}

@app.route("/blob/<status>")
def status_image(status):
response = make_response(open('images/%s-blob.png' % status_colors[status]).read())
response.headers['Content-type'] = 'image/png'
response.headers['Cache-Control'] = 'max-age=3600'
return response

@app.route("/robots.txt")
Expand All @@ -253,7 +295,7 @@ def robots():
User-agent: *
Disallow: /ticket/1303/status.png
Disallow: /blob/
Crawl-delay: 10
Crawl-delay: 5
""".lstrip()

@app.route("/favicon.ico")
Expand All @@ -267,8 +309,10 @@ def get_ticket_status(ticket, base=None):
if len(all):
index = min(status_order.index(report['status']) for report in all)
return len(all), status_order[index]
elif ticket['spkgs'] or not ticket['patches']:
elif ticket['spkgs']:
return 0, 'Spkg'
elif not ticket['patches']:
return 0, 'NoPatch'
else:
return 0, 'New'

Expand Down
2 changes: 1 addition & 1 deletion start_all.sh
Expand Up @@ -4,5 +4,5 @@ SAGE=/levi/scratch/robertwb/buildbot/sage-4.6/sage
PYTHON="$SAGE -python"

mongod --port=21000 --dbpath=../data &> mongod.log &
exec $PYTHON serve.py --base=4.6.2 --port=21100
exec $PYTHON serve.py --base=4.7 --port=21100

0 comments on commit 601bd02

Please sign in to comment.