Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 812 lines (671 sloc) 30.146 kB
5bfa25b @choller Initial code commit
choller authored
1 #!/usr/bin/env python
2 # ***** BEGIN LICENSE BLOCK *****
3 # Version: MPL 2.0
4 #
5 # This Source Code Form is subject to the terms of the Mozilla Public
6 # License, v. 2.0. If a copy of the MPL was not distributed with this
7 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 #
9 # The Original Code is ADBFuzz.
10 #
11 # The Initial Developer of the Original Code is Christian Holler (decoder).
12 #
13 # Contributors:
14 # Christian Holler <decoder@mozilla.com> (Original Developer)
15 #
16 # ***** END LICENSE BLOCK *****
17
18 import base64
19 import itertools
20 import os
21 import argparse
22 import re
23 import platform
24 import subprocess
4a74ca9 @choller Now ultimately better than the last version
choller authored
25 import traceback
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
26 import time
5bfa25b @choller Initial code commit
choller authored
27
28 from optparse import OptionParser
29
30 from compileShell import makeShell, shellName, testBinary
31 from subprocesses import captureStdout
32
33 from bugzilla.models import Bug, Attachment, Flag, User, Comment
34 from bugzilla.agents import BugzillaAgent
35 from bugzilla.utils import urljoin, qs, get_credentials, FILE_TYPES
36
37 def enum(*sequential, **named):
38 enums = dict(zip(sequential, range(len(sequential))), **named)
39 return type('Enum', (), enums)
40
41 def parseOpts():
42 usage = 'Usage: %prog [options] bugid [bugid ..]'
43 parser = OptionParser(usage)
44 # See http://docs.python.org/library/optparse.html#optparse.OptionParser.disable_interspersed_args
45 parser.disable_interspersed_args()
46
47 # Define the repository base.
48 parser.add_option('-r', '--repobase',
49 dest='repobase',
50 default=None,
51 help='Repository base directory, mandatory.')
52
53 parser.add_option('-v', '--verbose',
54 dest='verbose',
55 action='store_true',
56 default=False,
57 help='Be verbose. Defaults to "False"')
58
f6bd5eb @choller Add functionality to mark verify fixed.
choller authored
59 parser.add_option('-V', '--verify-fixed',
60 dest='verifyfixed',
61 action='store_true',
62 default=False,
63 help='Verify fix and comment. Defaults to "False"')
64
0d6924b Add more awesomeness to bugmon.py
decoder authored
65 parser.add_option('-C', '--confirm',
66 dest='confirm',
67 action='store_true',
68 default=False,
69 help='Attempt to confirm (or deny) open bugs. Defaults to "False"')
70
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
71 parser.add_option('-p', '--process',
72 dest='process',
73 action='store_true',
74 default=False,
75 help='Process commands on whiteboard of the bug. Defaults to "False"')
76
4a74ca9 @choller Now ultimately better than the last version
choller authored
77 parser.add_option('-U', '--update-bug',
78 dest='updatebug',
79 action='store_true',
80 default=False,
81 help='Update the bug. Defaults to "False"')
82
0d6924b Add more awesomeness to bugmon.py
decoder authored
83 parser.add_option('-P', '--update-bug-positive',
84 dest='updatebugpositive',
85 action='store_true',
86 default=False,
87 help='Update the bug also when it not changes state. Defaults to "False"')
88
4a74ca9 @choller Now ultimately better than the last version
choller authored
89 parser.add_option('-G', '--guess-opts',
90 dest='guessopts',
91 action='store_true',
92 default=False,
93 help='Force guessing the JS shell options. Defaults to "False"')
94
5bfa25b @choller Initial code commit
choller authored
95 (options, args) = parser.parse_args()
96
97 if len(args) < 1:
98 parser.error('Not enough arguments')
99
100 return (options, args)
101
102 def main():
103 # Script options
104 (options, args) = parseOpts()
105
106 # Get the API root, default to bugzilla.mozilla.org
107 API_ROOT = os.environ.get('BZ_API_ROOT',
108 'https://api-dev.bugzilla.mozilla.org/latest/')
109
110 # Authenticate
111 username, password = get_credentials()
112
113 # Sample run
f6bd5eb @choller Add functionality to mark verify fixed.
choller authored
114 bugmon = BugMonitor(API_ROOT, username, password, options.repobase, options)
5bfa25b @choller Initial code commit
choller authored
115
116 for bug_id in args:
117 print "====== Analyzing bug " + str(bug_id) + " ======"
118 try:
f6bd5eb @choller Add functionality to mark verify fixed.
choller authored
119 if options.verifyfixed:
4a74ca9 @choller Now ultimately better than the last version
choller authored
120 bugmon.verifyFixedBug(bug_id, options.updatebug)
0d6924b Add more awesomeness to bugmon.py
decoder authored
121 elif options.confirm:
122 bugmon.confirmOpenBug(bug_id, options.updatebug, options.updatebugpositive)
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
123 elif options.process:
124 bugmon.processCommand(bug_id)
f6bd5eb @choller Add functionality to mark verify fixed.
choller authored
125 else:
126 result = bugmon.reproduceBug(bug_id)
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
127 except BugException as b:
128 print "Cannot process bug: " + str(b)
5bfa25b @choller Initial code commit
choller authored
129 except Exception as e:
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
130 print "Caught exception: " + str(e)
131 print traceback.format_exc()
132
5bfa25b @choller Initial code commit
choller authored
133
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
134 class BugException(Exception):
135 pass
5bfa25b @choller Initial code commit
choller authored
136
4a74ca9 @choller Now ultimately better than the last version
choller authored
137 class BugMonitorResult:
138 # Different result states:
139 # FAILED - Unable to reproduce on original revision
140 # REPRODUCED_FIXED - Reproduced on original revision but not on tip (fixed on tip)
141 # REPRODUCED_TIP - Reproduced on both revisions
142 # REPRODUCED_SWITCHED - Reproduced on tip, but with a different crash/signal
143 statusCodes = enum('FAILED', 'REPRODUCED_FIXED', 'REPRODUCED_TIP', 'REPRODUCED_SWITCHED')
144
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
145 def __init__(self, branchName, origRev, tipRev, testFlags, testPath, arch, status):
4a74ca9 @choller Now ultimately better than the last version
choller authored
146 self.branchName = branchName
147 self.origRev = origRev
148 self.tipRev = tipRev
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
149 self.testFlags = testFlags
150 self.testPath = testPath
151 self.arch = arch
4a74ca9 @choller Now ultimately better than the last version
choller authored
152 self.status = status
153
5bfa25b @choller Initial code commit
choller authored
154 class BugMonitor:
155
f6bd5eb @choller Add functionality to mark verify fixed.
choller authored
156 def __init__(self, apiroot, username, password, repoBase, options):
157 self.apiroot = apiroot
158 self.bz = BugzillaAgent(apiroot, username, password)
5bfa25b @choller Initial code commit
choller authored
159
160 self.repoBase = repoBase
161
162 # Here we store the tip revision per repository for caching purposes
163 self.tipRev = {}
164
0d6924b Add more awesomeness to bugmon.py
decoder authored
165 self.guessopts = {}
166 self.guessopts['mozilla-central'] = ['-m -n', '-m -n -a', '-m', '-j', '-j -m', '-j -m -a', None]
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
167 #self.guessopts['ionmonkey'] = ['--ion -n -m', '--ion -n -m --ion-eager', None, '--ion-eager']
168 self.guessopts['ionmonkey'] = ['--ion -n -m', '--ion -n -m --ion-eager']
0d6924b Add more awesomeness to bugmon.py
decoder authored
169
f6bd5eb @choller Add functionality to mark verify fixed.
choller authored
170 # Misc options
171 self.options = options
172
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
173 def fetchBug(self, bug_id):
174 bug = self.bz.get_bug(bug_id)
175 if len(bug.depends_on) > 0:
176 if isinstance(bug.depends_on[0], str):
177 bug.depends_on = [ int("".join(bug.depends_on)) ]
178 return bug
179
f6bd5eb @choller Add functionality to mark verify fixed.
choller authored
180 def postComment(self, bugnum, comment):
181 url = urljoin(self.apiroot, 'bug/%s/comment?%s' % (bugnum, self.bz.qs()))
182 return Comment(text=comment).post_to(url)
183
4a74ca9 @choller Now ultimately better than the last version
choller authored
184 def verifyFixedBug(self, bugnum, updateBug):
f6bd5eb @choller Add functionality to mark verify fixed.
choller authored
185 # Fetch the bug
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
186 bug = self.fetchBug(bugnum)
f6bd5eb @choller Add functionality to mark verify fixed.
choller authored
187
188 if (bug.status == "RESOLVED" and bug.resolution == "FIXED"):
189 result = self.reproduceBug(bugnum)
190
4a74ca9 @choller Now ultimately better than the last version
choller authored
191 if (result.status == BugMonitorResult.statusCodes.REPRODUCED_FIXED):
192 if updateBug:
193 print "Marking bug " + str(bugnum) + " as verified fixed..."
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
194 while True:
195 try:
196 bug = self.fetchBug(bugnum)
197 # Mark VERIFIED FIXED now
198 bug.status = "VERIFIED"
199 bug.put()
200 break
201 except:
202 print "Failed to submit bug change, sleeping one second and retrying..."
203 time.sleep(1)
204
4a74ca9 @choller Now ultimately better than the last version
choller authored
205 # Add a comment
206 self.postComment(bugnum, "JSBugMon: This bug has been automatically verified fixed.")
207 else:
208 print "Would mark bug " + str(bugnum) + " as verified fixed..."
f6bd5eb @choller Add functionality to mark verify fixed.
choller authored
209
210 return
211
0d6924b Add more awesomeness to bugmon.py
decoder authored
212 def confirmOpenBug(self, bugnum, updateBug, updateBugPositive):
4a74ca9 @choller Now ultimately better than the last version
choller authored
213 # Fetch the bug
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
214 bug = self.fetchBug(bugnum)
4a74ca9 @choller Now ultimately better than the last version
choller authored
215
216 if (bug.status != "RESOLVED" and bug.status != "VERIFIED"):
0d6924b Add more awesomeness to bugmon.py
decoder authored
217 bugUpdateRequested = False
218 bugConfirmRequested = False
219 bugCloseRequested = False
220 bugUpdated = False
221
222 closeBug = False
223
224 wbOpts = []
225 if (bug.whiteboard != None):
226 ret = re.compile('\[jsbugmon:([^\]]+)\]').search(bug.whiteboard)
227 if (ret != None and ret.groups > 1):
228 wbOpts = ret.group(1).split(",")
229
230 # Explicitly marked to ignore this bug
231 if ('ignore' in wbOpts):
232 return
233
234 if ('update' in wbOpts):
235 bugUpdateRequested = True
236
237 if ('reconfirm' in wbOpts):
238 bugConfirmRequested = True
239
240 if ('close' in wbOpts):
241 bugCloseRequested = True
242
4a74ca9 @choller Now ultimately better than the last version
choller authored
243 result = self.reproduceBug(bugnum)
244
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
245 comments = []
246
4a74ca9 @choller Now ultimately better than the last version
choller authored
247 if (result.status == BugMonitorResult.statusCodes.REPRODUCED_TIP):
0d6924b Add more awesomeness to bugmon.py
decoder authored
248 if updateBugPositive or bugConfirmRequested:
4a74ca9 @choller Now ultimately better than the last version
choller authored
249 print "Marking bug " + str(bugnum) + " as confirmed on tip..."
250 # Add a comment
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
251 comments.append("JSBugMon: This bug has been automatically confirmed to be still valid (reproduced on revision " + result.tipRev + ").")
0d6924b Add more awesomeness to bugmon.py
decoder authored
252 bugUpdated = True
4a74ca9 @choller Now ultimately better than the last version
choller authored
253 else:
254 print "Would mark bug " + str(bugnum) + " as confirmed on tip..."
0d6924b Add more awesomeness to bugmon.py
decoder authored
255 elif (result.status == BugMonitorResult.statusCodes.REPRODUCED_FIXED):
256 if updateBug or bugUpdateRequested:
257 print "Marking bug " + str(bugnum) + " as non-reproducing on tip..."
258 # Add a comment
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
259 comments.append("JSBugMon: The testcase found in this bug no longer reproduces (tried revision " + result.tipRev + ").")
0d6924b Add more awesomeness to bugmon.py
decoder authored
260 bugUpdated = True
261
262 # Close bug only if requested to do so
263 closeBug = bugCloseRequested
264 else:
265 print "Would mark bug " + str(bugnum) + " as non-reproducing on tip..."
266
267 if bugUpdated:
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
268 wbOpts.append('ignore')
269 wbParts = filter(lambda x: len(x) > 0, map(str.rstrip, map(str.lstrip, re.split('\[jsbugmon:[^\]]+\]', bug.whiteboard))))
270 wbParts.append("[jsbugmon:" + ",".join(wbOpts) + "]")
271
272 while True:
273 try:
274 # Fetch the bug again for updating
275 bug = self.fetchBug(bugnum)
276
277 # We add "ignore" to our bugmon options so we don't update the bug a second time
278 bug.whiteboard = " ".join(wbParts)
279
280 # Mark bug as WORKSFORME if confirmed to no longer reproduce
281 if closeBug:
282 bug.status = "RESOLVED"
283 bug.resolution = "WORKSFORME"
284
285 bug.put()
286 break
287 except:
288 print "Failed to submit bug change, sleeping one second and retrying..."
289 time.sleep(1)
290
291 if (len(comments) > 0):
292 comment = "\n".join(comments)
293 print "Posting comment: "
294 print comment
295 print ""
296 self.postComment(bugnum, comment)
297
298 return
299
300 def processCommand(self, bugnum):
301 # Fetch the bug
302 bug = self.fetchBug(bugnum)
303
304 bugUpdateRequested = False
305 bugConfirmRequested = False
306 bugCloseRequested = False
307 bugVerifyRequested = False
308 bugBisectRequested = False
309 bugBisectFixRequested = False
310 bugUpdated = False
311
312 closeBug = False
313 verifyBug = False
314
315 wbOpts = []
316 if (bug.whiteboard != None):
317 ret = re.compile('\[jsbugmon:([^\]]+)\]').search(bug.whiteboard)
318 if (ret != None and ret.groups > 1):
319 wbOpts = ret.group(1).split(",")
320
321 # Explicitly marked to ignore this bug
322 if ('ignore' in wbOpts):
323 return
324
325 if ('update' in wbOpts):
326 bugUpdateRequested = True
327
328 if ('reconfirm' in wbOpts):
329 bugConfirmRequested = True
0d6924b Add more awesomeness to bugmon.py
decoder authored
330
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
331 if ('close' in wbOpts):
332 bugCloseRequested = True
333
334 if ('verify' in wbOpts):
335 bugVerifyRequested = True
336
337 if ('bisect' in wbOpts):
338 bugBisectRequested = True
339
340 if ('bisectfix' in wbOpts):
341 bugBisectFixRequested = True
342
343 print wbOpts
344
345 comments = []
346
347 # Keep bisect comments separate so we can remove bisect/bisectfix commands separately
348 bisectComments = []
349 bisectFixComments = []
350
351 result = None
352
353 for opt in wbOpts:
354 if (opt.find("=") > 0):
355 (cmd,param) = opt.split('=')
356 if (cmd != None and param != None):
357 if (cmd == "verify-branch"):
358 branches = param.split(';');
359 for branch in branches:
360 print "Branch " + branch
361 branchResult = self.reproduceBug(bugnum, branch)
362 if (branchResult.status == BugMonitorResult.statusCodes.REPRODUCED_TIP):
363 print "Marking bug " + str(bugnum) + " as reproducing on branch " + branch + " ..."
364 # Add a comment
365 comments.append("JSBugMon: This bug has been automatically confirmed to be still valid on branch " + branch + " (reproduced on revision " + branchResult.tipRev + ").")
366 elif (branchResult.status == BugMonitorResult.statusCodes.REPRODUCED_FIXED):
367 print "Marking bug " + str(bugnum) + " as non-reproducing on branch " + branch + " ..."
368 comments.append("JSBugMon: The testcase found in this bug does not reproduce on branch " + branch + " (tried revision " + branchResult.tipRev + ").")
369 else:
370 print "Marking bug " + str(bugnum) + " as not processable ..."
371 comments.append("JSBugMon: Command failed during processing this bug: " + opt + " (branch " + branch + ")")
372
373 if bugVerifyRequested:
374 if bug.status == "RESOLVED":
375 if result == None:
376 result = self.reproduceBug(bugnum)
377 if (result.status == BugMonitorResult.statusCodes.REPRODUCED_TIP):
378 print "Marking bug " + str(bugnum) + " as cannot verify fixed..."
379 # Add a comment
380 comments.append("JSBugMon: Cannot confirm fix, issue is still valid. (tried revision " + result.tipRev + ").")
381 elif (result.status == BugMonitorResult.statusCodes.REPRODUCED_FIXED):
382 print "Marking bug " + str(bugnum) + " as verified fixed..."
383 comments.append("JSBugMon: This bug has been automatically verified fixed. (tried revision " + result.tipRev + ").")
384 verifyBug = True
385 else:
386 print "Marking bug " + str(bugnum) + " as not processable ..."
387 comments.append("JSBugMon: Command failed during processing this bug: verify")
388
389 if bugUpdateRequested:
390 if bug.status != "RESOLVED" and bug.status != "VERIFIED":
391 if result == None:
392 result = self.reproduceBug(bugnum)
393 if (result.status == BugMonitorResult.statusCodes.REPRODUCED_TIP):
394 if bugConfirmRequested:
395 print "Marking bug " + str(bugnum) + " as confirmed on tip..."
396 # Add a comment
397 comments.append("JSBugMon: This bug has been automatically confirmed to be still valid (reproduced on revision " + result.tipRev + ").")
398
399 elif (result.status == BugMonitorResult.statusCodes.REPRODUCED_FIXED):
400 print "Marking bug " + str(bugnum) + " as non-reproducing on tip..."
401 # Add a comment
402 comments.append("JSBugMon: The testcase found in this bug no longer reproduces (tried revision " + result.tipRev + ").")
403 if bugCloseRequested:
404 closeBug = True
405
406 if bugBisectRequested and bug.status != "RESOLVED" and bug.status != "VERIFIED":
407 if result == None:
408 result = self.reproduceBug(bugnum)
409 if (result.status == BugMonitorResult.statusCodes.REPRODUCED_TIP):
410 print "Bisecting bug " + str(bugnum) + " ..."
411 bisectComment = self.bisectBug(bugnum, result)
412 print bisectComment
413 if len(bisectComment) > 0:
414 bisectComments.append("JSBugMon: Bisection requested, result:")
415 bisectComments.extend(bisectComment)
416 else:
417 bisectComments.append("JSBugMon: Bisection requested, failed due to error (try manually).")
418 bisectComments.append("");
419
420 if bugBisectFixRequested:
421 if result == None:
422 result = self.reproduceBug(bugnum)
423 if (result.status == BugMonitorResult.statusCodes.REPRODUCED_FIXED):
424 print "Bisecting fix for bug " + str(bugnum) + " ..."
425 bisectComment = self.bisectBug(bugnum, result, True)
426 print bisectComment
427 if len(bisectComment) > 0:
428 bisectFixComments.append("JSBugMon: Fix Bisection requested, result:")
429 bisectFixComments.extend(bisectComment)
430 else:
431 bisectFixComments.append("JSBugMon: Fix Bisection requested, failed due to error (try manually).")
432 bisectFixComments.append("");
433
434 wbParts = []
435 whiteBoardModified = False
436 if closeBug or verifyBug or len(comments) > 0:
437 whiteBoardModified = True
0d6924b Add more awesomeness to bugmon.py
decoder authored
438 wbOpts.append('ignore')
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
439
440 if bugBisectRequested:
441 whiteBoardModified = True
442 wbOpts.remove('bisect')
443 comments.extend(bisectComments)
444
445 if bugBisectFixRequested and len(bisectFixComments) > 0:
446 whiteBoardModified = True
447 wbOpts.remove('bisectfix')
448 comments.extend(bisectFixComments)
449
450 if whiteBoardModified:
0d6924b Add more awesomeness to bugmon.py
decoder authored
451 wbParts = filter(lambda x: len(x) > 0, map(str.rstrip, map(str.lstrip, re.split('\[jsbugmon:[^\]]+\]', bug.whiteboard))))
452 wbParts.append("[jsbugmon:" + ",".join(wbOpts) + "]")
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
453
454 while True:
455 # Fetch the bug again
456 bug = self.fetchBug(bugnum)
0d6924b Add more awesomeness to bugmon.py
decoder authored
457
458 # Mark bug as WORKSFORME if confirmed to no longer reproduce
459 if closeBug:
460 bug.status = "RESOLVED"
461 bug.resolution = "WORKSFORME"
462
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
463 # Mark bug as VERIFIED if we verified it successfully
464 if verifyBug:
465 bug.status = "VERIFIED"
0d6924b Add more awesomeness to bugmon.py
decoder authored
466
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
467 if whiteBoardModified:
468 # We add "ignore" to our bugmon options so we don't update the bug a second time
469 bug.whiteboard = " ".join(wbParts)
470
471 try:
472 bug.put()
473 break
474 except Exception as e:
475 print "Caught exception: " + str(e)
476 print traceback.format_exc()
477 time.sleep(1)
478 except:
479 print "Failed to submit bug change, sleeping one second and retrying..."
480 time.sleep(1)
481
482 if (len(comments) > 0):
483 comment = "\n".join(comments)
484 print "Posting comment: "
485 print comment
486 print ""
487 self.postComment(bugnum, comment)
4a74ca9 @choller Now ultimately better than the last version
choller authored
488
489 return
490
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
491 def bisectBug(self, bugnum, reproductionResult, bisectForFix=False):
492 # By default, bisect for the regressing changeset
493 revFlag = '-e'
494 if bisectForFix:
495 revFlag = '-s'
496
497 cmd = [ 'python', '/home/decoder/LangFuzz/fuzzing/autobisect-js/autoBisect.py', '-d', os.path.join(self.repoBase, reproductionResult.branchName), '-p', '-a', reproductionResult.arch, revFlag, reproductionResult.origRev, '--flags', ",".join(reproductionResult.testFlags), '-i', reproductionResult.testPath, 'crashes', '10' ]
498 outLines = subprocess.check_output(cmd).split("\n");
499 retLines = []
500 found = False
501 for outLine in outLines:
502 if not found and (outLine.find("autoBisect shows this is probably related") != -1 or outLine.find("Due to skipped revisions") != -1):
503 found = True
504
505 if found:
506 # Remove possible email address
507 if outLine.find("user:") != -1:
508 outLine = re.sub("\s*<.+>", "", outLine)
4a74ca9 @choller Now ultimately better than the last version
choller authored
509
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
510 # autobisect emits a date at the end, skip that
511 if (re.match("^\w+:", outLine) == None) and re.search("\s+\d{1,2}:\d{1,2}:\d{1,2}\s+", outLine) != None:
512 continue
513
514 retLines.append(outLine)
515
516 return retLines
517
518 def reproduceBug(self, bugnum, tipBranch=None):
5bfa25b @choller Initial code commit
choller authored
519 # Fetch the bug
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
520 bug = self.fetchBug(bugnum)
5bfa25b @choller Initial code commit
choller authored
521
522 # Look for the first comment
523 comment = bug.comments[0] if len(bug.comments) > 0 else None
524
525 if (comment == None):
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
526 raise BugException("Error: Specified bug does not have any comments")
5bfa25b @choller Initial code commit
choller authored
527
528 text = comment.text
529
530 # Isolate revision to test for
531 rev = self.extractRevision(text)
532
533 if (rev == None):
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
534 raise BugException("Error: Failed to isolate original revision for test")
5bfa25b @choller Initial code commit
choller authored
535
4a74ca9 @choller Now ultimately better than the last version
choller authored
536 opts = None
f1cf707 @choller More awesomeness...
choller authored
537 tipOpts = None
5bfa25b @choller Initial code commit
choller authored
538
4a74ca9 @choller Now ultimately better than the last version
choller authored
539 # Isolate options for testing, not explicitly instructed to guess
540 if not self.options.guessopts:
541 opts = self.extractOptions(text)
542 if (opts == None):
543 print "Warning: No options found, will try to guess"
5bfa25b @choller Initial code commit
choller authored
544
545 arch = None
f1cf707 @choller More awesomeness...
choller authored
546 archList = None
5bfa25b @choller Initial code commit
choller authored
547 if (bug.platform == "x86_64"):
548 arch = "64"
549 elif (bug.platform == "x86"):
550 arch = "32"
551 elif (bug.platform == "All"):
f1cf707 @choller More awesomeness...
choller authored
552 arch = "64"
553 archList = [ "64", "32" ] # TODO: Detect native platform here
5bfa25b @choller Initial code commit
choller authored
554 else:
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
555 raise BugException("Error: Unsupported architecture \"" + bug.platform + "\" required by bug")
5bfa25b @choller Initial code commit
choller authored
556
557 if (bug.version == "Trunk"):
558 reponame = "mozilla-central"
0d6924b Add more awesomeness to bugmon.py
decoder authored
559 elif (bug.version == "Other Branch"):
560 reponame = "ionmonkey"
5bfa25b @choller Initial code commit
choller authored
561 else:
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
562 raise BugException("Error: Unsupported branch \"" + bug.version + "\" required by bug")
563
564 if (tipBranch == None):
565 tipBranch = reponame
5bfa25b @choller Initial code commit
choller authored
566
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
567 print "Repobase: " + self.repoBase
568 print "Reponame: " + reponame
5bfa25b @choller Initial code commit
choller authored
569 repoDir = os.path.join(self.repoBase, reponame)
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
570 tipRepoDir = os.path.join(self.repoBase, tipBranch)
5bfa25b @choller Initial code commit
choller authored
571
4a74ca9 @choller Now ultimately better than the last version
choller authored
572 # We need at least some shell to extract the test from the bug,
573 # so we build a debug tip shell here already
5bfa25b @choller Initial code commit
choller authored
574 updated = False
575 if not self.tipRev.has_key(repoDir):
576 # If we don't know the tip revision for this branch, update and get it
577 self.tipRev[repoDir] = self.hgUpdate(repoDir)
578 updated = True
4a74ca9 @choller Now ultimately better than the last version
choller authored
579 (tipShell, tipRev) = self.getShell("cache/", arch, "dbg", 0, self.tipRev[repoDir], updated, repoDir)
5bfa25b @choller Initial code commit
choller authored
580
581 # If the file already exists, then we can reuse it
582 testFile = "bug" + str(bugnum) + ".js"
583
584 if (os.path.exists(testFile)):
585 print "Using existing (cached) testfile " + testFile
586 else:
4a74ca9 @choller Now ultimately better than the last version
choller authored
587
5bfa25b @choller Initial code commit
choller authored
588 # We need to detect where our test is.
589 blocks = text.split("\n\n")
590 found = False
591 cnt = 0
f1cf707 @choller More awesomeness...
choller authored
592 for i,block in enumerate(blocks):
5bfa25b @choller Initial code commit
choller authored
593 # Write our test to file
594 outFile = open(testFile, "w")
595 outFile.write(block)
596 outFile.close()
0d6924b Add more awesomeness to bugmon.py
decoder authored
597 (err, ret) = testBinary(tipShell, testFile, [], 0, timeout=30)
5bfa25b @choller Initial code commit
choller authored
598
599 if (err.find("SyntaxError") < 0):
f1cf707 @choller More awesomeness...
choller authored
600 # We have found the test (or maybe only the start of the test)
601 # Try adding more code until we hit an error or are out of
602 # blocks.
603 oldBlock = block
604 curBlock = block
605 for j,block in enumerate(blocks):
606 if j > i:
607 curBlock = curBlock + "\n" + block
608 # Write our test to file
609 outFile = open(testFile, "w")
610 outFile.write(curBlock)
611 outFile.close()
0d6924b Add more awesomeness to bugmon.py
decoder authored
612 (err, ret) = testBinary(tipShell, testFile, [], 0, timeout=30)
f1cf707 @choller More awesomeness...
choller authored
613 if (err.find("SyntaxError") >= 0):
614 # Too much, write oldBlock and break
615 outFile = open(testFile, "w")
616 outFile.write(oldBlock)
617 outFile.close()
618 break
619 else:
620 oldBlock = curBlock
621
5bfa25b @choller Initial code commit
choller authored
622 found = True
f1cf707 @choller More awesomeness...
choller authored
623 print "Isolated possible testcase starting in textblock " + str(cnt)
5bfa25b @choller Initial code commit
choller authored
624 break
625 cnt += 1
626 if not found:
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
627 # First try to find a suitable attachment
628 attachments = [a for a in bug.attachments if not bool(int(a.is_obsolete))]
629 for attachment in attachments:
630 # Seriously, we don't need anything larger than 512kb here^^
631 if (attachment.size <= 512*1024):
632 # Refetch attachment with content
633 url = urljoin(self.apiroot, 'attachment/%s/?%s&attachmentdata=1' % (attachment.id, self.bz.qs()))
634 attachment = attachment.get(url)
635
636 try:
637 rawData = base64.b64decode(attachment.data)
638 # Write our data to file
639 outFile = open(testFile, "w")
640 outFile.write(rawData)
641 outFile.close()
642 (err, ret) = testBinary(tipShell, testFile, [], 0, timeout=30)
643 if (err.find("SyntaxError") < 0):
644 # Found something that looks like JS :)
645 found = True
646 break
647 except TypeError:
648 pass
649
650 # If we still haven't found any test, give up here...
651 if not found:
652 # Ensure we don't cache the wrong test
653 os.remove(testFile)
654 raise BugException("Error: Failed to isolate test from comment")
5bfa25b @choller Initial code commit
choller authored
655
4a74ca9 @choller Now ultimately better than the last version
choller authored
656 (oouterr, oret) = (None, None)
657 (origShell, origRev) = (None, None)
5bfa25b @choller Initial code commit
choller authored
658
f1cf707 @choller More awesomeness...
choller authored
659 # If we have an exact architecture, we will only test that
660 if (archList == None):
661 archList = [ arch ]
4a74ca9 @choller Now ultimately better than the last version
choller authored
662
f1cf707 @choller More awesomeness...
choller authored
663 for compileType in ['dbg', 'opt']:
664 for archType in archList:
665 # Update to tip and cache result:
666 updated = False
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
667 if not self.tipRev.has_key(tipRepoDir):
f1cf707 @choller More awesomeness...
choller authored
668 # If we don't know the tip revision for this branch, update and get it
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
669 self.tipRev[tipRepoDir] = self.hgUpdate(tipRepoDir)
f1cf707 @choller More awesomeness...
choller authored
670 updated = True
671
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
672 (tipShell, tipRev) = self.getShell("cache/", archType, compileType, 0, self.tipRev[tipRepoDir], updated, tipRepoDir)
f1cf707 @choller More awesomeness...
choller authored
673 (origShell, origRev) = self.getShell("cache/", archType, compileType, 0, rev, False, repoDir)
674
675
676 if (opts != None):
0d6924b Add more awesomeness to bugmon.py
decoder authored
677 (oouterr, oret) = testBinary(origShell, testFile, opts , 0, verbose=self.options.verbose, timeout=30)
f1cf707 @choller More awesomeness...
choller authored
678 else:
679 print "Guessing options...",
0d6924b Add more awesomeness to bugmon.py
decoder authored
680 guessopts = self.guessopts[reponame]
f1cf707 @choller More awesomeness...
choller authored
681 for opt in guessopts:
0d6924b Add more awesomeness to bugmon.py
decoder authored
682 topts = []
683 if opt == None:
684 print " no options",
685 else:
686 print " " + opt,
687 topts = opt.split(' ')
688 (oouterr, oret) = testBinary(origShell, testFile, topts , 0, verbose=self.options.verbose, timeout=30)
f1cf707 @choller More awesomeness...
choller authored
689 if (oret < 0):
690 opts = topts
691 break;
692
693 # If we reproduced with one arch, then we don't need to try the others
694 if (oret < 0):
695 break;
4a74ca9 @choller Now ultimately better than the last version
choller authored
696
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
697 print ""
698
4a74ca9 @choller Now ultimately better than the last version
choller authored
699 # If we reproduced with dbg, then we don't need to try opt
700 if (oret < 0):
701 break;
702
703 # Check if we reproduced at all (dbg or opt)
5bfa25b @choller Initial code commit
choller authored
704 if (oret < 0):
705 print ""
706 print "Successfully reproduced bug (exit code " + str(oret) + ") on original revision " + rev + ":"
f1cf707 @choller More awesomeness...
choller authored
707 errl = oouterr.split("\n")
708 if len(errl) > 2: errl = errl[-2:]
709 for err in errl:
710 print err
5bfa25b @choller Initial code commit
choller authored
711
4a74ca9 @choller Now ultimately better than the last version
choller authored
712 if (opts != None):
713 # Try running on tip now
714 print "Testing bug on tip..."
f1cf707 @choller More awesomeness...
choller authored
715 if self.options.guessopts:
0d6924b Add more awesomeness to bugmon.py
decoder authored
716 guessopts = self.guessopts[reponame]
f1cf707 @choller More awesomeness...
choller authored
717 for opt in guessopts:
0d6924b Add more awesomeness to bugmon.py
decoder authored
718 tipOpts = []
719 if opt == None:
720 print " no options",
721 else:
722 print " " + opt,
723 tipOpts = opt.split(' ')
724 (touterr, tret) = testBinary(tipShell, testFile, tipOpts , 0, verbose=self.options.verbose, timeout=30)
f1cf707 @choller More awesomeness...
choller authored
725 if (tret < 0):
726 break;
727 else:
728 tipOpts = opts
0d6924b Add more awesomeness to bugmon.py
decoder authored
729 (touterr, tret) = testBinary(tipShell, testFile, tipOpts , 0, verbose=self.options.verbose, timeout=30)
4a74ca9 @choller Now ultimately better than the last version
choller authored
730 else:
731 print ""
5bfa25b @choller Initial code commit
choller authored
732
733 if (tret < 0):
734 if (tret == oret):
f1cf707 @choller More awesomeness...
choller authored
735 if (opts == tipOpts):
736 print "Result: Bug still reproduces"
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
737 return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, BugMonitorResult.statusCodes.REPRODUCED_TIP)
f1cf707 @choller More awesomeness...
choller authored
738 else:
739 print "Result: Bug still reproduces, but with different options: " + " ".join(tipOpts) # TODO need another code here in the future
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
740 return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, BugMonitorResult.statusCodes.REPRODUCED_TIP)
5bfa25b @choller Initial code commit
choller authored
741 else:
742 # Unlikely but possible, switched signal
743 print "Result: Bug now reproduces with signal " + str(tret) + " (previously " + str(oret) + ")"
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
744 return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, BugMonitorResult.statusCodes.REPRODUCED_SWITCHED)
5bfa25b @choller Initial code commit
choller authored
745 else:
746 print "Result: Bug no longer reproduces"
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
747 return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, BugMonitorResult.statusCodes.REPRODUCED_FIXED)
5bfa25b @choller Initial code commit
choller authored
748 else:
749 print "Error: Failed to reproduce bug on original revision"
5eda3a5 @choller2 Countless stuff, I lost track ^_^
choller2 authored
750 return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, BugMonitorResult.statusCodes.FAILED)
5bfa25b @choller Initial code commit
choller authored
751
752 def extractOptions(self, text):
753 ret = re.compile('((?: \-[a-z])+)', re.DOTALL).search(text)
754 if (ret != None and ret.groups > 1):
755 return ret.group(1).lstrip().split(" ")
756
757 return None
758
759 def extractRevision(self, text):
0d6924b Add more awesomeness to bugmon.py
decoder authored
760 if (text == None):
761 return None
5bfa25b @choller Initial code commit
choller authored
762 tokens = text.split(' ')
763 for token in tokens:
764 if (re.match('^[a-f0-9]{12}[^a-f0-9]?', token)):
4a74ca9 @choller Now ultimately better than the last version
choller authored
765 return token[0:12]
5bfa25b @choller Initial code commit
choller authored
766 return None
767
768 def hgUpdate(self, repoDir, rev=None):
769 print "Running hg update..."
770 if (rev != None):
771 captureStdout(['hg', 'update', '-r', rev], ignoreStderr=True, currWorkingDir=repoDir)
772 else:
773 captureStdout(['hg', 'update'], ignoreStderr=True, currWorkingDir=repoDir)
774
775 hgIdCmdList = ['hg', 'identify', repoDir]
776 # In Windows, this throws up a warning about failing to set color mode to win32.
777 if platform.system() == 'Windows':
778 hgIdFull = captureStdout(hgIdCmdList, currWorkingDir=repoDir, ignoreStderr=True)[0]
779 else:
780 hgIdFull = captureStdout(hgIdCmdList, currWorkingDir=repoDir)[0]
781 hgIdChangesetHash = hgIdFull.split(' ')[0]
782
783 #os.chdir(savedPath)
784 return hgIdChangesetHash
785
786 def getCachedShell(self, shellCacheDir, archNum, compileType, valgrindSupport, rev):
787 cachedShell = os.path.join(shellCacheDir, shellName(archNum, compileType, rev, valgrindSupport))
788 if os.path.exists(cachedShell):
789 return cachedShell
790 return None
791
4a74ca9 @choller Now ultimately better than the last version
choller authored
792 def getShell(self, shellCacheDir, archNum, compileType, valgrindSupport, rev, updated, repoDir):
793 shell = self.getCachedShell(shellCacheDir, archNum, compileType, valgrindSupport, rev)
794 updRev = None
795 if (shell == None):
796 if updated:
797 updRev = rev
798 else:
799 updRev = self.hgUpdate(repoDir, rev)
800
801
802 if (rev == None):
803 print "Compiling a new shell for tip (revision " + updRev + ")"
804 else:
805 print "Compiling a new shell for revision " + updRev
0d6924b Add more awesomeness to bugmon.py
decoder authored
806 shell = makeShell(shellCacheDir, repoDir, archNum, compileType, valgrindSupport, updRev, True)
4a74ca9 @choller Now ultimately better than the last version
choller authored
807
808 return (shell, updRev)
809
5bfa25b @choller Initial code commit
choller authored
810 if __name__ == '__main__':
811 main()
Something went wrong with that request. Please try again.