Permalink
Browse files

more plugins, http cache, status filter

  • Loading branch information...
1 parent 6d4fc98 commit 601bd024373caa9f67abfbbf1305a1fdec9005c5 @robertwb robertwb committed Jun 4, 2011
Showing with 152 additions and 39 deletions.
  1. +47 −17 buildbot.py
  2. +32 −1 plugins.py
  3. +53 −9 serve.py
  4. +1 −1 start_all.sh
  5. +14 −6 templates/ticket.html
  6. +3 −3 templates/ticket_list.html
  7. +2 −2 trac.py
View
@@ -1,3 +1,4 @@
+import signal
import re, os, sys, subprocess, time, traceback
import bz2, urllib2, urllib, json
from optparse import OptionParser
@@ -27,9 +28,9 @@ def contains_any(key, values):
return {'$or': clauses}
def get_ticket(server, return_all=False, **conf):
- query = "raw"
+ query = "raw&status=open&todo"
if 'trusted_authors' in conf:
- query += "&authors=" + ':'.join(conf['trusted_authors'])
+ query += "&authors=" + urllib.quote_plus(':'.join(conf['trusted_authors']), safe=':')
try:
handle = urllib2.urlopen(server + "/ticket/?" + query)
all = json.load(handle)
@@ -68,6 +69,8 @@ def rate_ticket(ticket, **conf):
rating = 0
if ticket['spkgs']:
return # can't handle these yet
+ elif not ticket['patches']:
+ return # nothing to do
for dep in ticket['depends_on']:
if isinstance(dep, basestring) and '.' in dep:
if compare_version(conf['base'], dep) < 0:
@@ -83,8 +86,9 @@ def rate_ticket(ticket, **conf):
# TODO: remove condition
if 'component' in ticket:
rating += conf['bonus'].get(ticket['component'], 0)
+ rating += conf['bonus'].get(ticket['status'], 0)
rating += conf['bonus'].get(ticket['priority'], 0)
- rating += conf['bonus'].get(ticket['id'], 0)
+ rating += conf['bonus'].get(str(ticket['id']), 0)
redundancy = (100,)
prune_pending(ticket)
if not ticket.get('retry'):
@@ -102,19 +106,19 @@ def parse_datetime(s):
# The one thing Python can't do is parse dates...
return time.mktime(time.strptime(s[:-5].strip(), DATE_FORMAT[:-3])) + 60*int(s[-5:].strip())
-def prune_pending(ticket, machine=None):
+def prune_pending(ticket, machine=None, timeout=6*60*60):
if 'reports' in ticket:
reports = ticket['reports']
else:
return []
# TODO: is there a better way to handle time zones?
- now = time.time() + 60 * int(time.strftime('%z')) + 1000000000000
+ now = time.time() + 60 * int(time.strftime('%z'))
for report in list(reports):
if report['status'] == 'Pending':
t = parse_datetime(report['time'])
if report['machine'] == machine:
reports.remove(report)
- elif now - t > 6 * 60 * 60:
+ elif now - t > timeout:
reports.remove(report)
return reports
@@ -140,10 +144,17 @@ def report_ticket(server, ticket, status, base, machine, log, plugins=[]):
except:
traceback.print_exc()
+class TimeOut(Exception):
+ pass
+
+def alarm_handler(signum, frame):
+ raise Alarm
+
class Tee:
- def __init__(self, filepath, time=False):
+ def __init__(self, filepath, time=False, timeout=60*60*24):
self.filepath = filepath
self.time = time
+ self.timeout = timeout
def __enter__(self):
self._saved = os.dup(sys.stdout.fileno()), os.dup(sys.stderr.fileno())
@@ -165,7 +176,14 @@ def __exit__(self, exc_type, exc_val, exc_tb):
os.dup2(self._saved[0], sys.stdout.fileno())
os.dup2(self._saved[1], sys.stderr.fileno())
time.sleep(1)
- self.tee.wait()
+ try:
+ signal.signal(signal.SIGALRM, alarm_handler)
+ signal.alarm(self.timeout)
+ self.tee.wait()
+ signal.alarm(0)
+ except TimeOut:
+ traceback.print_exc()
+ raise
return False
@@ -227,10 +245,10 @@ def test_a_ticket(sage_root, server, ticket=None, nodocs=False):
if not os.path.exists(log_dir):
os.mkdir(log_dir)
log = '%s/%s-log.txt' % (log_dir, ticket['id'])
- #report_ticket(server, ticket, status='Pending', base=base, machine=conf['machine'], log=None)
+ report_ticket(server, ticket, status='Pending', base=base, machine=conf['machine'], log=None)
plugins_results = []
try:
- with Tee(log, time=True):
+ with Tee(log, time=True, timeout=conf['timeout']):
t = Timer()
start_time = time.time()
@@ -246,11 +264,12 @@ def test_a_ticket(sage_root, server, ticket=None, nodocs=False):
state = 'built'
working_dir = "%s/devel/sage-%s" % (sage_root, ticket['id'])
- patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().split('\n')
+ # Only the ones on this ticket.
+ patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):]
kwds = {
"original_dir": "%s/devel/sage-0" % sage_root,
"patched_dir": working_dir,
- "patches": ["%s/devel/sage-%s/.hg/patches/%s" % (sage_root, ticket['id'], p) for p in patches],
+ "patches": ["%s/devel/sage-%s/.hg/patches/%s" % (sage_root, ticket['id'], p) for p in patches if p],
}
for name, plugin in conf['plugins']:
try:
@@ -266,8 +285,8 @@ def test_a_ticket(sage_root, server, ticket=None, nodocs=False):
plugins_results.append((name, passed))
test_dirs = ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs]
- #do_or_die("$SAGE_ROOT/sage -tp %s -sagenb %s" % (conf['parallelism'], ' '.join(test_dirs)))
- do_or_die("$SAGE_ROOT/sage -t $SAGE_ROOT/devel/sage-%s/sage/rings/integer.pyx" % ticket['id'])
+ do_or_die("$SAGE_ROOT/sage -tp %s -sagenb %s" % (conf['parallelism'], ' '.join(test_dirs)))
+ #do_or_die("$SAGE_ROOT/sage -t $SAGE_ROOT/devel/sage-%s/sage/rings/integer.pyx" % ticket['id'])
#do_or_die('sage -testall')
t.finish("Tests")
state = 'tested'
@@ -288,11 +307,23 @@ def get_conf(path, **overrides):
conf = {
"idle": 300,
"parallelism": 3,
- "plugins": ["plugins.coverage", "plugins.docbuild"],
+ "timeout": 3 * 60 * 60,
+ "plugins": ["plugins.commit_messages", "plugins.coverage", "plugins.docbuild"],
+ "bonus": {},
+ }
+ default_bonus = {
+ "needs_review": 1000,
+ "positive_review": 500,
+ "blocker": 100,
+ "critical": 50,
}
for key, value in unicode_conf.items():
conf[str(key)] = value
+ for key, value in default_bonus.items():
+ if key not in conf['bonus']:
+ conf['bonus'][key] = value
conf.update(overrides)
+
def locate_plugin(name):
ix = name.rindex('.')
module = name[:ix]
@@ -333,8 +364,7 @@ def locate_plugin(name):
clean = lookup_ticket(options.server, 0)
def good(report):
return report['machine'] == conf['machine'] and report['status'] == 'TestsPassed'
-# TODO: fix to not have to re-start as often
- if False and not any(good(report) for report in current_reports(clean, base=get_base(options.sage_root))):
+ if not any(good(report) for report in current_reports(clean, base=get_base(options.sage_root))):
res = test_a_ticket(ticket=0, sage_root=options.sage_root, server=options.server)
if res != 'TestsPassed':
print "\n\n"
View
@@ -10,16 +10,47 @@
ticket -- a dictionary of all the ticket informaton
original_dir -- pristine sage-main directory
patched_dir -- patched sage-branch directory for this ticket
- patch_list -- a list of absolute paths to the patch files for this ticket
+ patchs -- a list of absolute paths to the patch files for this ticket
It is recommended that a plugin ignore extra keywords to be
compatible with future options.
"""
+import re, os
+
from trac import do_or_die
+
def coverage(ticket, **kwds):
do_or_die('$SAGE_ROOT/sage -coverageall')
def docbuild(ticket, **kwds):
do_or_die('$SAGE_ROOT/sage -docbuild --jsmath reference html')
+
+def commit_messages(ticket, patches, **kwds):
+ for patch_path in patches:
+ patch = os.path.basename(patch_path)
+ print "Looking at", patch
+ header = []
+ for line in open(patch_path):
+ if line.startswith('diff '):
+ break
+ header.append(line)
+ else:
+ print ''.join(header[:10])
+ raise ValueError("Not a valid patch file: " + patch)
+ print ''.join(header)
+ if header[0].strip() != "# HG changeset patch":
+ raise ValueError("Not a mercurial patch file: " + patch)
+ for line in header:
+ if not line.startswith('# '):
+ # First description line
+ if not re.search(r"\b%s\b" % ticket['id'], line):
+ raise ValueError("Ticket number not in first line of comments:" + patch)
+ elif line.startswith('[mq]'):
+ raise ValueError("Mercurial queue boilerplate")
+ break
+ else:
+ raise ValueError("No patch comments:" + patch)
+ print
+ print "All patches good."
View
@@ -24,7 +24,18 @@ def ticket_list():
if 'query' in request.args:
query = json.loads(request.args.get('query'))
else:
- query = {'status': 'needs_review'}
+ status = request.args.get('status', 'needs_review')
+ if status == 'all':
+ query = {}
+ elif status in ('new', 'closed'):
+ query = {'status': {'$regex': status + '.*' }}
+ elif status in ('open'):
+ query = {'status': {'$regex': 'needs_.*|positive_review' }}
+ else:
+ query = {'status': status}
+ if 'todo' in request.args:
+ query['patches'] = {'$not': {'$size': 0}}
+ query['spkgs'] = {'$size': 0}
if 'authors' in request.args:
authors = request.args.get('authors').split(':')
query['authors'] = {'$in': authors}
@@ -60,9 +71,10 @@ def preprocess(all):
summary[ticket['report_status']] += 1
yield ticket
ticket0 = db.lookup_ticket(0)
- versions = set(report['base'] for report in ticket0['reports'])
- versions = [(v, get_ticket_status(ticket0, v)) for v in sorted(versions)]
- return render_template("ticket_list.html", tickets=preprocess(all), summary=summary, base=base, base_status=get_ticket_status(db.lookup_ticket(0), base), versions=versions)
+ versions = list(set(report['base'] for report in ticket0['reports']))
+ versions.sort(trac.compare_version)
+ versions = [(v, get_ticket_status(ticket0, v)) for v in versions if v != '4.7.']
+ return render_template("ticket_list.html", tickets=preprocess(all), summary=summary, base=base, base_status=get_ticket_status(db.lookup_ticket(0), base), versions=versions, status_order=status_order)
def format_patches(ticket, patches, deps=None, required=None):
if deps is None:
@@ -108,11 +120,17 @@ def render_ticket(ticket):
info = tickets.find_one({'id': ticket})
if 'kick' in request.args:
info['retry'] = True
- db.save_ticket(info)
+ db.save_ticket(info)
if 'reports' in info:
info['reports'].sort(lambda a, b: -cmp(a['time'], b['time']))
else:
info['reports'] = []
+
+ old_reports = list(info['reports'])
+ buildbot.prune_pending(info)
+ if old_reports != info['reports']:
+ db.save_ticket(info)
+
def format_info(info):
new_info = {}
for key, value in info.items():
@@ -152,6 +170,7 @@ def render_ticket_status(ticket):
status = get_ticket_status(info, base=base)[1]
response = make_response(open('images/%s-blob.png' % status_colors[status]).read())
response.headers['Content-type'] = 'image/png'
+ response.headers['Cache-Control'] = 'no-cache'
return response
def get_or_set(ticket, key, default):
@@ -212,21 +231,42 @@ def shorten(lines):
if prev is not None:
yield prev
+def extract_plugin_log(data, plugin):
+ from buildbot import plugin_boundary
+ start = plugin_boundary(plugin) + "\n"
+ end = plugin_boundary(plugin, end=True) + "\n"
+ all = []
+ include = False
+ for line in StringIO(data):
+ if line == start:
+ include = True
+ if include:
+ all.append(line)
+ if line == end:
+ break
+ return ''.join(all)
+
+@app.route("/ticket/<id>/log/<path:log>")
+def get_ticket_log(id, log):
+ return get_log(log)
+
@app.route("/log/<path:log>")
def get_log(log):
path = "/log/" + log
if not logs.exists(path):
data = "No such log!"
else:
data = bz2.decompress(logs.get(path).read())
+ if 'plugin' in request.args:
+ data = extract_plugin_log(data, request.args.get('plugin'))
if 'short' in request.args:
response = Response(shorten(data), direct_passthrough=True)
else:
response = make_response(data)
response.headers['Content-type'] = 'text/plain'
return response
-status_order = ['New', 'ApplyFailed', 'BuildFailed', 'TestsFailed', 'PluginFailed', 'TestsPassed', 'Pending', 'Spkg']
+status_order = ['New', 'ApplyFailed', 'BuildFailed', 'TestsFailed', 'PluginFailed', 'TestsPassed', 'Pending', 'NoPatch', 'Spkg']
# TODO: cleanup old records
# status_order += ['started', 'applied', 'built', 'tested']
@@ -238,13 +278,15 @@ def get_log(log):
'TestsPassed': 'green',
'PluginFailed': 'blue',
'Pending' : 'white',
- 'Spkg' : 'purple',
+ 'NoPatch' : 'purple',
+ 'Spkg' : 'purple',
}
@app.route("/blob/<status>")
def status_image(status):
response = make_response(open('images/%s-blob.png' % status_colors[status]).read())
response.headers['Content-type'] = 'image/png'
+ response.headers['Cache-Control'] = 'max-age=3600'
return response
@app.route("/robots.txt")
@@ -253,7 +295,7 @@ def robots():
User-agent: *
Disallow: /ticket/1303/status.png
Disallow: /blob/
-Crawl-delay: 10
+Crawl-delay: 5
""".lstrip()
@app.route("/favicon.ico")
@@ -267,8 +309,10 @@ def get_ticket_status(ticket, base=None):
if len(all):
index = min(status_order.index(report['status']) for report in all)
return len(all), status_order[index]
- elif ticket['spkgs'] or not ticket['patches']:
+ elif ticket['spkgs']:
return 0, 'Spkg'
+ elif not ticket['patches']:
+ return 0, 'NoPatch'
else:
return 0, 'New'
View
@@ -4,5 +4,5 @@ SAGE=/levi/scratch/robertwb/buildbot/sage-4.6/sage
PYTHON="$SAGE -python"
mongod --port=21000 --dbpath=../data &> mongod.log &
-exec $PYTHON serve.py --base=4.6.2 --port=21100
+exec $PYTHON serve.py --base=4.7 --port=21100
Oops, something went wrong.

0 comments on commit 601bd02

Please sign in to comment.