Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Merging from default\n

changeset:   2942:88d4e6cc2057
parent:      2940:4d5f49af57ab
user:        Rail Aliiev <rail@mozilla.com>
date:        Mon Jan 21 14:39:22 2013 -0500
files:       common.py process/factory.py
description:
Bug 820238 - PuppetAgain config for new Linux64 testing reference platform. r=armenzg


changeset:   2943:c65fd7add8d6
user:        Chris AtLee <catlee@mozilla.com>
date:        Mon Jan 21 19:45:01 2013 -0500
files:       bin/hgpoller.py bin/log_uploader.py bin/postrun.py bin/try_mailer.py bin/update_from_files.py changes/ftppoller.py changes/hgpoller.py changes/mobileftppoller.py changes/tinderboxpoller.py common.py env.py l10n.py log.py misc.py misc_scheduler.py process/factory.py process/release.py scheduler.py status/db/jsoncol.py status/db/model.py status/db/status.py status/errors.py status/generators.py status/log_handlers.py status/mail.py status/pulse.py status/queued_command.py status/tinderboxmailnotifier.py steps/base.py steps/l10n.py steps/misc.py steps/mobile.py steps/mock.py steps/release.py steps/signing.py steps/source.py steps/talos.py steps/test.py steps/unittest.py steps/updates.py test/test_hgpoller.py test/test_misc_important.py test/test_misc_nextslaves.py test/test_misc_scheduler_nightly.py test/test_misc_scheduler_propfuncs.py test/test_misc_scheduler_propscheduler.py test/test_process_factory.py test/test_test_order.py test/test_try_parser.py try_parser.py
description:
nobug: PEP8! r=bitrotallthethings


changeset:   2944:4c7c7df48595
user:        Steve Fink <sfink@mozilla.com>
date:        Tue Jan 22 12:50:07 2013 -0800
files:       misc.py
description:
Bug 830466 - Turn off tipsOnly for try. r=catlee


changeset:   2945:78189628f1e2
tag:         tip
user:        Armen Zambrano Gasparnian <armenzg@mozilla.com>
date:        Tue Jan 22 16:03:32 2013 -0500
files:       misc.py process/factory.py
description:
Bug 710840: Track peak virtual memory usage of link.exe process during libxul
PGO link on graph server. r=bhearsum

--HG--
branch : production-0.8
  • Loading branch information...
commit 97050edc161868c6bc66a7eee7605ca05a1d171f 2 parents a470055 + 53cbf81
Kim Moir authored January 22, 2013

Showing 50 changed files with 4,659 additions and 3,621 deletions. Show diff stats Hide diff stats

  1. 47  bin/hgpoller.py
  2. 117  bin/log_uploader.py
  3. 101  bin/postrun.py
  4. 44  bin/try_mailer.py
  5. 77  bin/update_from_files.py
  6. 41  changes/ftppoller.py
  7. 42  changes/hgpoller.py
  8. 73  changes/mobileftppoller.py
  9. 104  changes/tinderboxpoller.py
  10. 15  common.py
  11. 96  env.py
  12. 33  l10n.py
  13. 12  log.py
  14. 889  misc.py
  15. 83  misc_scheduler.py
  16. 4,026  process/factory.py
  17. 505  process/release.py
  18. 84  scheduler.py
  19. 1  status/db/jsoncol.py
  20. 189  status/db/model.py
  21. 76  status/db/status.py
  22. 16  status/errors.py
  23. 3  status/generators.py
  24. 16  status/log_handlers.py
  25. 36  status/mail.py
  26. 59  status/pulse.py
  27. 18  status/queued_command.py
  28. 22  status/tinderboxmailnotifier.py
  29. 10  steps/base.py
  30. 19  steps/l10n.py
  31. 116  steps/misc.py
  32. 19  steps/mobile.py
  33. 35  steps/mock.py
  34. 15  steps/release.py
  35. 6  steps/signing.py
  36. 2  steps/source.py
  37. 34  steps/talos.py
  38. 78  steps/test.py
  39. 150  steps/unittest.py
  40. 29  steps/updates.py
  41. 136  test/test_hgpoller.py
  42. 21  test/test_misc_important.py
  43. 44  test/test_misc_nextslaves.py
  44. 77  test/test_misc_scheduler_nightly.py
  45. 35  test/test_misc_scheduler_propfuncs.py
  46. 17  test/test_misc_scheduler_propscheduler.py
  47. 3  test/test_process_factory.py
  48. 68  test/test_test_order.py
  49. 442  test/test_try_parser.py
  50. 99  try_parser.py
47  bin/hgpoller.py
... ...
@@ -1,16 +1,22 @@
1 1
 #!/usr/bin/env python
2  
-import urlparse, urllib, time
  2
+import urlparse
  3
+import urllib
  4
+import time
3 5
 try:
4 6
     import json
5 7
 except:
6 8
     import simplejson as json
7 9
 
8  
-import httplib, urllib2, socket, ssl
  10
+import httplib
  11
+import urllib2
  12
+import socket
  13
+import ssl
9 14
 
10 15
 import subprocess
11 16
 from buildbotcustom.changes.hgpoller import _parse_changes
12 17
 import logging as log
13 18
 
  19
+
14 20
 def buildValidatingOpener(ca_certs):
15 21
     class VerifiedHTTPSConnection(httplib.HTTPSConnection):
16 22
         def connect(self):
@@ -45,6 +51,7 @@ def https_open(self, req):
45 51
 
46 52
     return url_opener
47 53
 
  54
+
48 55
 def validating_https_open(url, ca_certs, username=None, password=None):
49 56
     url_opener = buildValidatingOpener(ca_certs)
50 57
     req = urllib2.Request(url)
@@ -55,17 +62,18 @@ def validating_https_open(url, ca_certs, username=None, password=None):
55 62
         req.add_header("Authorization", "Basic %s" % pw)
56 63
     return url_opener.open(req)
57 64
 
  65
+
58 66
 def getChanges(base_url, last_changeset=None, tips_only=False, ca_certs=None,
59  
-        username=None, password=None):
  67
+               username=None, password=None):
60 68
     bits = urlparse.urlparse(base_url)
61 69
     if bits.scheme == 'https':
62 70
         assert ca_certs, "you must specify ca_certs"
63 71
 
64 72
     params = [('full', '1')]
65 73
     if last_changeset:
66  
-        params.append( ('fromchange', last_changeset) )
  74
+        params.append(('fromchange', last_changeset))
67 75
     if tips_only:
68  
-        params.append( ('tipsonly', '1') )
  76
+        params.append(('tipsonly', '1'))
69 77
     url = "%s/json-pushes?%s" % (base_url, urllib.urlencode(params))
70 78
 
71 79
     log.debug("Fetching %s", url)
@@ -78,22 +86,25 @@ def getChanges(base_url, last_changeset=None, tips_only=False, ca_certs=None,
78 86
     data = handle.read()
79 87
     return _parse_changes(data)
80 88
 
  89
+
81 90
 def sendchange(master, branch, change):
82  
-    log.info("Sendchange %s to %s on branch %s", change['changeset'], master, branch)
  91
+    log.info("Sendchange %s to %s on branch %s", change['changeset'],
  92
+             master, branch)
83 93
     cmd = ['retry.py', '-r', '5', '-s', '5', '-t', '30',
84  
-            '--stdout-regexp', 'change sent successfully']
  94
+           '--stdout-regexp', 'change sent successfully']
85 95
     cmd.extend(
86  
-          ['buildbot', 'sendchange',
  96
+        ['buildbot', 'sendchange',
87 97
             '--master', master,
88 98
             '--branch', branch,
89 99
             '--comments', change['comments'].encode('ascii', 'replace'),
90 100
             '--revision', change['changeset'],
91 101
             '--user', change['author'].encode('ascii', 'replace'),
92 102
             '--when', str(change['updated']),
93  
-            ])
  103
+         ])
94 104
     cmd.extend(change['files'])
95 105
     subprocess.check_call(cmd)
96 106
 
  107
+
97 108
 def processBranch(branch, state, config):
98 109
     log.debug("Processing %s", branch)
99 110
     master = config.get('main', 'master')
@@ -116,13 +127,14 @@ def processBranch(branch, state, config):
116 127
 
117 128
     try:
118 129
         changes = getChanges(url, tips_only=tips_only,
119  
-                last_changeset=last_changeset, ca_certs=ca_certs,
120  
-                username=username, password=password)
  130
+                             last_changeset=last_changeset, ca_certs=ca_certs,
  131
+                             username=username, password=password)
121 132
         # Do sendchanges!
122 133
         for c in changes:
123 134
             # Ignore off-default branches
124 135
             if c['branch'] != 'default' and config.getboolean(branch, 'default_branch_only'):
125  
-                log.info("Skipping %s on branch %s", c['changeset'], c['branch'])
  136
+                log.info(
  137
+                    "Skipping %s on branch %s", c['changeset'], c['branch'])
126 138
                 continue
127 139
             # Change the comments to include the url to the revision
128 140
             c['comments'] += ' %s/rev/%s' % (url, c['changeset'])
@@ -150,11 +162,12 @@ def processBranch(branch, state, config):
150 162
 
151 163
     parser = OptionParser()
152 164
     parser.set_defaults(
153  
-            config_file="hgpoller.ini",
154  
-            verbosity=log.INFO,
155  
-            )
  165
+        config_file="hgpoller.ini",
  166
+        verbosity=log.INFO,
  167
+    )
156 168
     parser.add_option("-f", "--config-file", dest="config_file")
157  
-    parser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=log.DEBUG)
  169
+    parser.add_option("-v", "--verbose", dest="verbosity",
  170
+                      action="store_const", const=log.DEBUG)
158 171
 
159 172
     options, args = parser.parse_args()
160 173
 
@@ -168,7 +181,7 @@ def processBranch(branch, state, config):
168 181
         'interval': 300,
169 182
         'state_file': 'state.json',
170 183
         'default_branch_only': "yes",
171  
-        })
  184
+    })
172 185
     config.read(options.config_file)
173 186
 
174 187
     try:
117  bin/log_uploader.py
@@ -3,7 +3,10 @@
3 3
 
4 4
 Uploads logs from build to the given host.
5 5
 """
6  
-import os, cPickle, gzip, subprocess
  6
+import os
  7
+import cPickle
  8
+import gzip
  9
+import subprocess
7 10
 from datetime import datetime
8 11
 import time
9 12
 
@@ -17,13 +20,14 @@
17 20
 retries = 5
18 21
 retry_sleep = 30
19 22
 
  23
+
20 24
 def do_cmd(cmd):
21 25
     "Runs the command, and returns output"
22 26
     devnull = open(os.devnull)
23 27
     proc = subprocess.Popen(cmd,
24  
-            stdout=subprocess.PIPE,
25  
-            stderr=subprocess.STDOUT,
26  
-            stdin=devnull,)
  28
+                            stdout=subprocess.PIPE,
  29
+                            stderr=subprocess.STDOUT,
  30
+                            stdin=devnull,)
27 31
 
28 32
     retcode = proc.wait()
29 33
     output = proc.stdout.read().strip()
@@ -32,13 +36,15 @@ def do_cmd(cmd):
32 36
     raise Exception("Command %s returned non-zero exit code %i:\n%s" % (
33 37
         cmd, retcode, output))
34 38
 
  39
+
35 40
 def ssh(user, identity, host, remote_cmd, port=22):
36 41
     cmd = ['ssh', '-l', user]
37 42
     if identity:
38 43
         cmd.extend(['-i', identity])
39 44
     cmd.extend(['-p', str(port), host, remote_cmd])
40 45
 
41  
-    return retry(do_cmd, attempts=retries+1, sleeptime=retry_sleep, args=(cmd,))
  46
+    return retry(do_cmd, attempts=retries + 1, sleeptime=retry_sleep, args=(cmd,))
  47
+
42 48
 
43 49
 def scp(user, identity, host, files, remote_dir, port=22):
44 50
     cmd = ['scp']
@@ -50,6 +56,7 @@ def scp(user, identity, host, files, remote_dir, port=22):
50 56
 
51 57
     return retry(do_cmd, attempts=retries, sleeptime=retry_sleep, args=(cmd,))
52 58
 
  59
+
53 60
 def getBuild(builder_path, build_number):
54 61
     build_path = os.path.join(builder_path, build_number)
55 62
 
@@ -64,6 +71,7 @@ class FakeBuilder:
64 71
     build.builder = FakeBuilder()
65 72
     return build
66 73
 
  74
+
67 75
 def getAuthor(build):
68 76
     props = build.getProperties()
69 77
     if 'who' in props:
@@ -73,6 +81,7 @@ def getAuthor(build):
73 81
     if changes:
74 82
         return changes[0].who
75 83
 
  84
+
76 85
 def getBuildId(build):
77 86
     try:
78 87
         buildid = build.getProperty('buildid')
@@ -90,6 +99,7 @@ def getBuildId(build):
90 99
 
91 100
     return buildid
92 101
 
  102
+
93 103
 def isNightly(build):
94 104
     try:
95 105
         if build.getProperty('nightly_build'):
@@ -97,6 +107,7 @@ def isNightly(build):
97 107
     except:
98 108
         return False
99 109
 
  110
+
100 111
 def formatLog(tmpdir, build, master_name, builder_suffix=''):
101 112
     """
102 113
     Returns a filename with the contents of the build log
@@ -104,9 +115,11 @@ def formatLog(tmpdir, build, master_name, builder_suffix=''):
104 115
     """
105 116
     builder_name = build.builder.name
106 117
     if master_name:
107  
-        build_name = "%s%s-%s-build%s.txt.gz" % (builder_name, builder_suffix, master_name, build_number)
  118
+        build_name = "%s%s-%s-build%s.txt.gz" % (
  119
+            builder_name, builder_suffix, master_name, build_number)
108 120
     else:
109  
-        build_name = "%s%s-build%s.txt.gz" % (builder_name, builder_suffix, build_number)
  121
+        build_name = "%s%s-build%s.txt.gz" % (
  122
+            builder_name, builder_suffix, build_number)
110 123
 
111 124
     logFile = gzip.GzipFile(os.path.join(tmpdir, build_name), "w")
112 125
 
@@ -136,7 +149,6 @@ def formatLog(tmpdir, build, master_name, builder_suffix=''):
136 149
 
137 150
     logFile.write("\n")
138 151
 
139  
-
140 152
     # Steps
141 153
     for step in build.getSteps():
142 154
         times = step.getTimes()
@@ -151,9 +163,11 @@ def formatLog(tmpdir, build, master_name, builder_suffix=''):
151 163
         if results == (None, []):
152 164
             results = "not started"
153 165
 
154  
-        shortText = ' '.join(step.getText()) + ' (results: %s, elapsed: %s)' % (results, elapsed)
  166
+        shortText = ' '.join(step.getText(
  167
+        )) + ' (results: %s, elapsed: %s)' % (results, elapsed)
155 168
         if times and times[0]:
156  
-            logFile.write("========= Started %s (at %s) =========\n" % (shortText, datetime.fromtimestamp(times[0])))
  169
+            logFile.write("========= Started %s (at %s) =========\n" %
  170
+                          (shortText, datetime.fromtimestamp(times[0])))
157 171
         else:
158 172
             logFile.write("========= Skipped %s =========\n" % shortText)
159 173
             continue
@@ -165,7 +179,8 @@ def formatLog(tmpdir, build, master_name, builder_suffix=''):
165 179
                 logFile.write("\n")
166 180
 
167 181
         if times and times[1]:
168  
-            logFile.write("========= Finished %s (at %s) =========\n\n" % (shortText, datetime.fromtimestamp(times[1])))
  182
+            logFile.write("========= Finished %s (at %s) =========\n\n" %
  183
+                          (shortText, datetime.fromtimestamp(times[1])))
169 184
         else:
170 185
             logFile.write("========= Finished %s =========\n\n" % shortText)
171 186
     logFile.close()
@@ -173,38 +188,41 @@ def formatLog(tmpdir, build, master_name, builder_suffix=''):
173 188
 
174 189
 if __name__ == "__main__":
175 190
     from optparse import OptionParser
176  
-    import tempfile, shutil
  191
+    import tempfile
  192
+    import shutil
177 193
 
178 194
     parser = OptionParser(__doc__)
179 195
     parser.set_defaults(
180  
-            nightly=False,
181  
-            release=None,
182  
-            trybuild=False,
183  
-            shadowbuild=False,
184  
-            l10n=False,
185  
-            user=os.environ.get("USER"),
186  
-            product="firefox",
187  
-            retries=retries,
188  
-            retry_sleep=retry_sleep,
189  
-            master_name=None,
190  
-            )
  196
+        nightly=False,
  197
+        release=None,
  198
+        trybuild=False,
  199
+        shadowbuild=False,
  200
+        l10n=False,
  201
+        user=os.environ.get("USER"),
  202
+        product="firefox",
  203
+        retries=retries,
  204
+        retry_sleep=retry_sleep,
  205
+        master_name=None,
  206
+    )
191 207
     parser.add_option("-u", "--user", dest="user", help="upload user name")
192 208
     parser.add_option("-i", "--identity", dest="identity", help="ssh identity")
193 209
     parser.add_option("-b", "--branch", dest="branch", help="branch")
194 210
     parser.add_option("-p", "--platform", dest="platform", help="platform")
195  
-    parser.add_option("-r", "--retries", dest="retries", help="number of times to try", type="int")
196  
-    parser.add_option("-t", "--retrytime", dest="retry_sleep", help="time to sleep between tries", type="int")
  211
+    parser.add_option("-r", "--retries", dest="retries",
  212
+                      help="number of times to try", type="int")
  213
+    parser.add_option("-t", "--retrytime", dest="retry_sleep",
  214
+                      help="time to sleep between tries", type="int")
197 215
     parser.add_option("--product", dest="product", help="product directory")
198 216
     parser.add_option("--nightly", dest="nightly", action="store_true",
199  
-            help="upload to nightly dir")
  217
+                      help="upload to nightly dir")
200 218
     parser.add_option("--release", dest="release",
201  
-            help="upload to release candidates dir")
  219
+                      help="upload to release candidates dir")
202 220
     parser.add_option("--l10n", dest="l10n", action="store_true",
203  
-            help="include locale value in log filename")
  221
+                      help="include locale value in log filename")
204 222
     parser.add_option("--try", dest="trybuild", action="store_true",
205  
-            help="upload to try build directory")
  223
+                      help="upload to try build directory")
206 224
     parser.add_option("--shadow", dest="shadowbuild", action="store_true",
207  
-            help="upload to shadow build directory")
  225
+                      help="upload to shadow build directory")
208 226
     parser.add_option("--master-name", dest="master_name")
209 227
 
210 228
     options, args = parser.parse_args()
@@ -233,27 +251,30 @@ def formatLog(tmpdir, build, master_name, builder_suffix=''):
233 251
                 suffix = '-%s' % build.getProperty('locale')
234 252
             except KeyError:
235 253
                 suffix = '-unknown'
236  
-            logfile = formatLog(local_tmpdir, build, options.master_name, suffix)
  254
+            logfile = formatLog(
  255
+                local_tmpdir, build, options.master_name, suffix)
237 256
         else:
238 257
             logfile = formatLog(local_tmpdir, build, options.master_name)
239 258
 
240 259
         # Now....upload it!
241  
-        remote_tmpdir = ssh(user=options.user, identity=options.identity, host=host,
242  
-                remote_cmd="mktemp -d")
  260
+        remote_tmpdir = ssh(
  261
+            user=options.user, identity=options.identity, host=host,
  262
+            remote_cmd="mktemp -d")
243 263
         try:
244 264
             # Release logs go into the 'logs' directory
245 265
             if options.release:
246 266
                 # Create the logs directory
247 267
                 ssh(user=options.user, identity=options.identity, host=host,
248  
-                        remote_cmd="mkdir -p %s/logs" % remote_tmpdir)
  268
+                    remote_cmd="mkdir -p %s/logs" % remote_tmpdir)
249 269
                 scp(user=options.user, identity=options.identity, host=host,
250  
-                        files=[logfile], remote_dir='%s/logs' % remote_tmpdir)
  270
+                    files=[logfile], remote_dir='%s/logs' % remote_tmpdir)
251 271
                 remote_files = [os.path.join(remote_tmpdir, 'logs', os.path.basename(f)) for f in [logfile]]
252 272
             else:
253 273
                 scp(user=options.user, identity=options.identity, host=host,
254  
-                        files=[logfile], remote_dir=remote_tmpdir)
  274
+                    files=[logfile], remote_dir=remote_tmpdir)
255 275
 
256  
-                remote_files = [os.path.join(remote_tmpdir, os.path.basename(f)) for f in [logfile]]
  276
+                remote_files = [os.path.join(
  277
+                    remote_tmpdir, os.path.basename(f)) for f in [logfile]]
257 278
 
258 279
             uploadArgs = dict(
259 280
                 branch=options.branch,
@@ -275,7 +296,7 @@ def formatLog(tmpdir, build, master_name, builder_suffix=''):
275 296
                     who=getAuthor(build),
276 297
                     revision=build.getProperty('revision')[:12],
277 298
                     builddir="%s-%s" % (options.branch, platform),
278  
-                    ))
  299
+                ))
279 300
             else:
280 301
                 buildid = getBuildId(build)
281 302
 
@@ -294,22 +315,24 @@ def formatLog(tmpdir, build, master_name, builder_suffix=''):
294 315
                         # Don't upload to the latest directory - the logs are
295 316
                         # already in the dated directory and we should keep the
296 317
                         # latest-* directory clean.
297  
-                        #uploadArgs['to_latest'] = True
  318
+                        # uploadArgs['to_latest'] = True
298 319
                     else:
299 320
                         uploadArgs['to_tinderbox_builds'] = True
300 321
                         uploadArgs['upload_dir'] = uploadArgs['branch']
301 322
 
302 323
                 else:
303  
-                    uploadArgs['upload_dir'] = "%s-%s" % (options.branch, platform)
  324
+                    uploadArgs[
  325
+                        'upload_dir'] = "%s-%s" % (options.branch, platform)
304 326
 
305 327
                     if options.nightly or isNightly(build):
306 328
                         uploadArgs['to_dated'] = True
307 329
                         # Don't upload to the latest directory - the logs are
308 330
                         # already in the dated directory and we should keep the
309 331
                         # latest-* directory clean.
310  
-                        #uploadArgs['to_latest'] = True
  332
+                        # uploadArgs['to_latest'] = True
311 333
                         if 'mobile' in options.product:
312  
-                            uploadArgs['branch'] = options.branch + '-' + platform
  334
+                            uploadArgs[
  335
+                                'branch'] = options.branch + '-' + platform
313 336
                         else:
314 337
                             uploadArgs['branch'] = options.branch
315 338
 
@@ -325,17 +348,17 @@ def formatLog(tmpdir, build, master_name, builder_suffix=''):
325 348
 
326 349
                 props = build.getProperties()
327 350
                 if props.getProperty('got_revision') is not None:
328  
-                    revision=props['got_revision']
  351
+                    revision = props['got_revision']
329 352
                 elif props.getProperty('revision') is not None:
330  
-                    revision=props['revision']
  353
+                    revision = props['revision']
331 354
                 else:
332  
-                    revision=None
  355
+                    revision = None
333 356
                 uploadArgs.update(dict(
334 357
                     to_try=False,
335 358
                     who=None,
336 359
                     revision=revision,
337 360
                     buildid=buildid,
338  
-                    ))
  361
+                ))
339 362
             post_upload_cmd = postUploadCmdPrefix(**uploadArgs)
340 363
             post_upload_cmd += [remote_tmpdir]
341 364
             post_upload_cmd += remote_files
@@ -346,7 +369,7 @@ def formatLog(tmpdir, build, master_name, builder_suffix=''):
346 369
             print ssh(user=options.user, identity=options.identity, host=host, remote_cmd=post_upload_cmd)
347 370
         finally:
348 371
             ssh(user=options.user, identity=options.identity, host=host,
349  
-                    remote_cmd="rm -rf %s" % remote_tmpdir)
  372
+                remote_cmd="rm -rf %s" % remote_tmpdir)
350 373
 
351 374
     finally:
352 375
         shutil.rmtree(local_tmpdir)
101  bin/postrun.py
@@ -23,7 +23,8 @@
23 23
 command queue entries.
24 24
 
25 25
 """
26  
-import os, sys
  26
+import os
  27
+import sys
27 28
 import re
28 29
 import cPickle as pickle
29 30
 from datetime import datetime
@@ -41,6 +42,7 @@
41 42
 
42 43
 from util.commands import get_output
43 44
 
  45
+
44 46
 class PostRunner(object):
45 47
     def __init__(self, config):
46 48
         self.config = config
@@ -57,12 +59,14 @@ def uploadLog(self, build):
57 59
         product = info['product']
58 60
         platform = info['platform']
59 61
 
60  
-        upload_args = ['-r', '2', '-t', '10', '--master-name', self.config['statusdb.master_name']]
  62
+        upload_args = ['-r', '2', '-t', '10', '--master-name',
  63
+                       self.config['statusdb.master_name']]
61 64
         if "nightly" in builder.name:
62 65
             upload_args.append("--nightly")
63 66
         if builder.name.startswith("release-"):
64 67
             upload_args.append("--release")
65  
-            upload_args.append("%s/%s" % (info.get('version'), info.get('build_number')))
  68
+            upload_args.append(
  69
+                "%s/%s" % (info.get('version'), info.get('build_number')))
66 70
 
67 71
         if branch and 'try' in branch:
68 72
             upload_args.append("--try")
@@ -106,7 +110,8 @@ def mailResults(self, build, log_url):
106 110
                "--log-url", log_url,
107 111
                ]
108 112
 
109  
-        cmd.extend(['-f', self.config.get('mail_notifier_sender', 'tryserver@build.mozilla.org')])
  113
+        cmd.extend(['-f', self.config.get(
  114
+            'mail_notifier_sender', 'tryserver@build.mozilla.org')])
110 115
         if self.config.get('mail_real_author'):
111 116
             cmd.append('--to-author')
112 117
 
@@ -190,12 +195,12 @@ def getBuildInfo(self, build):
190 195
     def writePulseMessage(self, options, build, build_id):
191 196
         builder_name = build.builder.name
192 197
         msg = {
193  
-                'event': 'build.%s.%s.log_uploaded' % (builder_name, build.number),
194  
-                'payload': {"build": build.asDict()},
195  
-                'master_name': options.master_name,
196  
-                'master_incarnation': options.master_incarnation,
197  
-                'id': build_id,
198  
-            }
  198
+            'event': 'build.%s.%s.log_uploaded' % (builder_name, build.number),
  199
+            'payload': {"build": build.asDict()},
  200
+            'master_name': options.master_name,
  201
+            'master_incarnation': options.master_incarnation,
  202
+            'id': build_id,
  203
+        }
199 204
         self.pulse_queue.add(json.dumps([msg]))
200 205
 
201 206
     def updateStatusDB(self, build, request_ids):
@@ -219,15 +224,16 @@ def updateStatusDB(self, build, request_ids):
219 224
 
220 225
         log.debug("searching for build")
221 226
         q = session.query(model.Build).filter_by(
222  
-                master_id=master.id,
223  
-                builder=db_builder,
224  
-                buildnumber=build.number,
225  
-                starttime=starttime,
226  
-                )
  227
+            master_id=master.id,
  228
+            builder=db_builder,
  229
+            buildnumber=build.number,
  230
+            starttime=starttime,
  231
+        )
227 232
         db_build = q.first()
228 233
         if not db_build:
229 234
             log.debug("creating new build")
230  
-            db_build = model.Build.fromBBBuild(session, build, builder_name, master.id)
  235
+            db_build = model.Build.fromBBBuild(
  236
+                session, build, builder_name, master.id)
231 237
         else:
232 238
             log.debug("updating old build")
233 239
             db_build.updateFromBBBuild(session, build)
@@ -241,23 +247,24 @@ def updateStatusDB(self, build, request_ids):
241 247
         for i in request_ids:
242 248
             # See if we already have this row
243 249
             q = model.schedulerdb_requests.select()
244  
-            q = q.where(model.schedulerdb_requests.c.status_build_id==db_build.id)
245  
-            q = q.where(model.schedulerdb_requests.c.scheduler_request_id==i)
  250
+            q = q.where(
  251
+                model.schedulerdb_requests.c.status_build_id == db_build.id)
  252
+            q = q.where(model.schedulerdb_requests.c.scheduler_request_id == i)
246 253
             q = q.limit(1).execute()
247 254
             if not q.fetchone():
248 255
                 # Find the schedulerdb build id for this
249 256
                 bid = schedulerdb.execute(
250  
-                        sa.text('select id from builds where brid=:brid and number=:number'),
251  
-                        brid=i, number=build.number
252  
-                        ).fetchone()
  257
+                    sa.text('select id from builds where brid=:brid and number=:number'),
  258
+                    brid=i, number=build.number
  259
+                ).fetchone()
253 260
                 if bid is not None:
254 261
                     bid = bid[0]
255 262
                     log.debug("bid for %s is %s", i, bid)
256 263
                     model.schedulerdb_requests.insert().execute(
257  
-                            status_build_id=db_build.id,
258  
-                            scheduler_request_id=i,
259  
-                            scheduler_build_id=bid,
260  
-                            )
  264
+                        status_build_id=db_build.id,
  265
+                        scheduler_request_id=i,
  266
+                        scheduler_build_id=bid,
  267
+                    )
261 268
         log.debug("build id is %s", db_build.id)
262 269
         return db_build.id
263 270
 
@@ -267,9 +274,11 @@ def getRequestTimes(self, request_ids):
267 274
         schedulerdb = sa.create_engine(self.config['schedulerdb.url'])
268 275
         retval = {}
269 276
         for i in request_ids:
270  
-            submitted_at = schedulerdb.execute(sa.text("select submitted_at from buildrequests where id=:brid"),
271  
-                    brid=i,
272  
-                    ).fetchone()
  277
+            submitted_at = schedulerdb.execute(
  278
+                sa.text(
  279
+                    "select submitted_at from buildrequests where id=:brid"),
  280
+                brid=i,
  281
+            ).fetchone()
273 282
             if submitted_at is not None:
274 283
                 retval[i] = submitted_at[0]
275 284
         return retval
@@ -294,11 +303,14 @@ def processBuild(self, options, build_path, request_ids):
294 303
                 log_url = None
295 304
             log.debug("adding properties")
296 305
             build.properties.setProperty('log_url', log_url, 'postrun.py')
297  
-            build.properties.setProperty('request_ids', [int(i) for i in request_ids], 'postrun.py')
298  
-            build.properties.setProperty('request_times', self.getRequestTimes(request_ids), 'postrun.py')
  306
+            build.properties.setProperty(
  307
+                'request_ids', [int(i) for i in request_ids], 'postrun.py')
  308
+            build.properties.setProperty('request_times', self.getRequestTimes(
  309
+                request_ids), 'postrun.py')
299 310
             build_id = self.updateStatusDB(build, request_ids)
300 311
 
301  
-            cmd = [sys.executable] + sys.argv + ["--statusdb-id", str(build_id)]
  312
+            cmd = [sys.executable] + sys.argv + [
  313
+                "--statusdb-id", str(build_id)]
302 314
             self.command_queue.add(json.dumps(cmd))
303 315
         else:
304 316
             log.info("publishing to pulse")
@@ -306,24 +318,29 @@ def processBuild(self, options, build_path, request_ids):
306 318
             build_id = options.statusdb_id
307 319
             build.properties.setProperty('log_url', log_url, 'postrun.py')
308 320
             build.properties.setProperty('statusdb_id', build_id, 'postrun.py')
309  
-            build.properties.setProperty('request_ids', [int(i) for i in request_ids], 'postrun.py')
310  
-            build.properties.setProperty('request_times', self.getRequestTimes(request_ids), 'postrun.py')
  321
+            build.properties.setProperty(
  322
+                'request_ids', [int(i) for i in request_ids], 'postrun.py')
  323
+            build.properties.setProperty('request_times', self.getRequestTimes(
  324
+                request_ids), 'postrun.py')
311 325
             self.writePulseMessage(options, build, build_id)
312 326
 
  327
+
313 328
 def main():
314 329
     from optparse import OptionParser
315 330
     parser = OptionParser()
316 331
     parser.set_defaults(
317  
-            config=None,
318  
-            loglevel=logging.INFO,
319  
-            log_url=None,
320  
-            statusdb_id=None,
321  
-            master_name=None,
322  
-            master_incarnation=None,
323  
-            )
  332
+        config=None,
  333
+        loglevel=logging.INFO,
  334
+        log_url=None,
  335
+        statusdb_id=None,
  336
+        master_name=None,
  337
+        master_incarnation=None,
  338
+    )
324 339
     parser.add_option("-c", "--config", dest="config")
325  
-    parser.add_option("-v", "--verbose", dest="loglevel", const=logging.DEBUG, action="store_const")
326  
-    parser.add_option("-q", "--quiet", dest="loglevel", const=logging.WARNING, action="store_const")
  340
+    parser.add_option("-v", "--verbose", dest="loglevel",
  341
+                      const=logging.DEBUG, action="store_const")
  342
+    parser.add_option("-q", "--quiet", dest="loglevel",
  343
+                      const=logging.WARNING, action="store_const")
327 344
     parser.add_option("--log-url", dest="log_url")
328 345
     parser.add_option("--statusdb-id", dest="statusdb_id", type="int")
329 346
     parser.add_option("--master-name", dest="master_name")
44  bin/try_mailer.py
@@ -3,13 +3,17 @@
3 3
 
4 4
 Uploads logs to the given host, and then sends an email to the build's owner
5 5
 """
6  
-import subprocess, sys, os, re
  6
+import subprocess
  7
+import sys
  8
+import os
  9
+import re
7 10
 import cPickle
8 11
 from email.message import Message
9 12
 from email.utils import formatdate
10 13
 
11 14
 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION, RETRY
12 15
 
  16
+
13 17
 def getBuild(builder_path, build_number):
14 18
     build_path = os.path.join(builder_path, build_number)
15 19
 
@@ -24,6 +28,7 @@ class FakeBuilder:
24 28
     build.builder = FakeBuilder()
25 29
     return build
26 30
 
  31
+
27 32
 def uploadLog(args):
28 33
     """Uploads the build log, and returns the URL to it"""
29 34
     my_dir = os.path.abspath(os.path.dirname(__file__))
@@ -33,9 +38,9 @@ def uploadLog(args):
33 38
     print "Running", cmd
34 39
 
35 40
     proc = subprocess.Popen(cmd,
36  
-            stdout=subprocess.PIPE,
37  
-            stderr=subprocess.STDOUT,
38  
-            stdin=devnull,)
  41
+                            stdout=subprocess.PIPE,
  42
+                            stderr=subprocess.STDOUT,
  43
+                            stdin=devnull,)
39 44
 
40 45
     retcode = proc.wait()
41 46
     output = proc.stdout.read().strip()
@@ -47,6 +52,7 @@ def uploadLog(args):
47 52
         return url.group(), retcode
48 53
     return None, retcode
49 54
 
  55
+
50 56
 def makeTryMessage(build, log_url):
51 57
     builder = build.builder.name
52 58
 
@@ -64,7 +70,7 @@ def makeTryMessage(build, log_url):
64 70
     branch = props['branch']
65 71
     tree = "Try"
66 72
     if 'comm' in branch:
67  
-       tree = "Thunderbird-Try" 
  73
+        tree = "Thunderbird-Try"
68 74
 
69 75
     if 'got_revision' in props:
70 76
         revision = props['got_revision'][:12]
@@ -125,8 +131,8 @@ def makeTryMessage(build, log_url):
125 131
     text = re.sub("\n", "<br>\n", text)
126 132
 
127 133
     headers = {"In-Reply-To": "<%(branch)s-%(revision)s>" % locals(),
128  
-                "References": "<%(branch)s-%(revision)s>" % locals(),
129  
-              }
  134
+               "References": "<%(branch)s-%(revision)s>" % locals(),
  135
+               }
130 136
 
131 137
     return dict(
132 138
         subject=subject,
@@ -134,7 +140,8 @@ def makeTryMessage(build, log_url):
134 140
         headers=headers,
135 141
         author=who,
136 142
         type='html',
137  
-        )
  143
+    )
  144
+
138 145
 
139 146
 def formatMessage(msgdict, from_, to):
140 147
     m = Message()
@@ -144,7 +151,7 @@ def formatMessage(msgdict, from_, to):
144 151
     m['Subject'] = msgdict['subject']
145 152
     m['From'] = from_
146 153
     m['To'] = ", ".join(to)
147  
-    for k,v in msgdict['headers'].items():
  154
+    for k, v in msgdict['headers'].items():
148 155
         if k not in m:
149 156
             m[k] = v
150 157
     return m
@@ -153,16 +160,19 @@ def formatMessage(msgdict, from_, to):
153 160
     from argparse import ArgumentParser
154 161
     from smtplib import SMTP
155 162
     parser = ArgumentParser()
156  
-    parser.add_argument("-f", "--from", dest="from_", help="from email address", required=True)
157  
-    parser.add_argument("-t", "--to", dest="to", help="to email address", action='append')
  163
+    parser.add_argument("-f", "--from", dest="from_",
  164
+                        help="from email address", required=True)
  165
+    parser.add_argument(
  166
+        "-t", "--to", dest="to", help="to email address", action='append')
158 167
     parser.add_argument("--to-author", dest="to_author", help="send mail to build's owner", action="store_true")
159  
-    parser.add_argument("--log-url", dest="log_url", help="url to uploaded log")
  168
+    parser.add_argument(
  169
+        "--log-url", dest="log_url", help="url to uploaded log")
160 170
     parser.set_defaults(
161 171
         to_author=False,
162 172
         to=[],
163 173
         from_=None,
164 174
         log_url=None
165  
-        )
  175
+    )
166 176
 
167 177
     options, args = parser.parse_known_args()
168 178
 
@@ -177,12 +187,13 @@ def formatMessage(msgdict, from_, to):
177 187
     print
178 188
 
179 189
     tm_parser = ArgumentParser()
180  
-    tm_parser.add_argument("-e", "--all-emails", dest="all_emails", help="request all emails", action="store_true")
  190
+    tm_parser.add_argument("-e", "--all-emails", dest="all_emails",
  191
+                           help="request all emails", action="store_true")
181 192
     tm_parser.add_argument("-f", "--failure-emails", dest="failure", help="request failure emails only", action="store_true")
182 193
     tm_parser.set_defaults(
183 194
         all_emails=False,
184 195
         failure=False,
185  
-        )
  196
+    )
186 197
 
187 198
     builder_path, build_number = args[-2:]
188 199
     build = getBuild(builder_path, build_number)
@@ -191,7 +202,8 @@ def formatMessage(msgdict, from_, to):
191 202
     match = re.search("try: ", build.source.changes[-1].comments)
192 203
     comment_args = ""
193 204
     if match:
194  
-        comment_args = build.source.changes[-1].comments.split("try: ")[1].split()
  205
+        comment_args = build.source.changes[-1].comments.split(
  206
+            "try: ")[1].split()
195 207
     tm_options, args = tm_parser.parse_known_args(comment_args)
196 208
 
197 209
     # Let's check the results to see if we need the message
77  bin/update_from_files.py
... ...
@@ -1,34 +1,46 @@
1 1
 import buildbotcustom.status.db.model as model
2  
-import cPickle, os, re, time, sys
  2
+import cPickle
  3
+import os
  4
+import re
  5
+import time
  6
+import sys
3 7
 from datetime import datetime
4 8
 from buildbot.status.builder import BuilderStatus, BuildStepStatus
5 9
 
6 10
 # Monkey patching!
7 11
 # These are various replacement functions for __setstate__, which is
8 12
 # called when unpickling files.
  13
+
  14
+
9 15
 def monkeypatch(orig, new):
10 16
     orig_setstate = orig.__setstate__
  17
+
11 18
     def wrapper(self, state):
12 19
         return new(self, orig_setstate, state)
13 20
     orig.__setstate__ = wrapper
14 21
 
  22
+
15 23
 def builder_setstate_noevents(self, orig, state):
16 24
     slaves = state['slavenames']
17 25
     state['events'] = []
18 26
     orig(self, state)
19 27
     self.slavenames = slaves
20 28
 
  29
+
21 30
 def builder_setstate_events(self, orig, state):
22 31
     slaves = state['slavenames']
23 32
     orig(self, state)
24 33
     self.slavenames = slaves
25 34
 
  35
+
26 36
 def buildstep_setstate(self, orig, state):
27 37
     state['logs'] = []
28 38
     orig(self, state)
29 39
 
  40
+
30 41
 def getBuildNumbers(builder, last_time):
31 42
     files = os.listdir(builder)
  43
+
32 44
     def _sortfunc(x):
33 45
         try:
34 46
             return int(x)
@@ -45,16 +57,19 @@ def _sortfunc(x):
45 57
             retval.append(f)
46 58
     return retval
47 59
 
  60
+
48 61
 def getBuild(builder, number):
49 62
     try:
50 63
         return cPickle.load(open(os.path.join(builder, number)))
51 64
     except:
52 65
         return None
53 66
 
  67
+
54 68
 def getBuilder(builder):
55 69
     builder = cPickle.load(open(os.path.join(builder, 'builder')))
56 70
     return builder
57 71
 
  72
+
58 73
 def updateBuilderSlaves(session, builder, db_builder):
59 74
     bb_slaves = set(s for s in builder.slavenames)
60 75
     db_slaves = set()
@@ -69,7 +84,8 @@ def updateBuilderSlaves(session, builder, db_builder):
69 84
     old_slaves = db_slaves - bb_slaves
70 85
 
71 86
     for s in new_slaves:
72  
-        bs = model.BuilderSlave(added=datetime.now(), slave=model.Slave.get(session, s))
  87
+        bs = model.BuilderSlave(
  88
+            added=datetime.now(), slave=model.Slave.get(session, s))
73 89
         db_builder.slaves.append(bs)
74 90
         session.add(bs)
75 91
 
@@ -80,6 +96,7 @@ def updateBuilderSlaves(session, builder, db_builder):
80 96
 
81 97
     session.commit()
82 98
 
  99
+
83 100
 def updateSlaveTimes(session, master, builder, db_builder, last_time):
84 101
     db_slaves = {}
85 102
     for builder_slave in db_builder.slaves:
@@ -87,10 +104,12 @@ def updateSlaveTimes(session, master, builder, db_builder, last_time):
87 104
 
88 105
     # Fetch all the events from the database for these slaves
89 106
     events = session.query(model.MasterSlave).\
90  
-                filter(model.MasterSlave.slave_id.in_([slave.slave.id for slave in db_builder.slaves]))
  107
+        filter(model.MasterSlave.slave_id.in_(
  108
+               [slave.slave.id for slave in db_builder.slaves]))
91 109
 
92 110
     if last_time:
93  
-        events = events.filter(model.MasterSlave.connected > datetime.utcfromtimestamp(last_time))
  111
+        events = events.filter(model.MasterSlave.connected >
  112
+                               datetime.utcfromtimestamp(last_time))
94 113
 
95 114
     events = events.order_by(model.MasterSlave.connected.asc()).all()
96 115
 
@@ -114,7 +133,8 @@ def updateSlaveTimes(session, master, builder, db_builder, last_time):
114 133
             t = datetime.utcfromtimestamp(int(e.started))
115 134
             if e.text[0] == "connect":
116 135
                 # This slave just connected to this builder
117  
-                # Check if we've got an entry earlier than this that hasn't been disconnected yet
  136
+                # Check if we've got an entry earlier than this that hasn't
  137
+                # been disconnected yet
118 138
                 found = False
119 139
                 for event in reversed(slave_events):
120 140
                     if event.connected < t and not event.disconnected:
@@ -130,17 +150,19 @@ def updateSlaveTimes(session, master, builder, db_builder, last_time):
130 150
                     if event:
131 151
                         print t
132 152
                         for e in reversed(slave_events):
133  
-                            print e.connected, e.connected-t, e.connected == t
  153
+                            print e.connected, e.connected - t, e.connected == t
134 154
                         raise ValueError("Shouldn't be here!")
135  
-                    event = model.MasterSlave(connected=t, slave=db_slaves[name].slave, master=master)
  155
+                    event = model.MasterSlave(connected=t, slave=db_slaves[
  156
+                                              name].slave, master=master)
136 157
                     session.add(event)
137 158
 
138 159
                     slave_events.append(event)
139 160
                     events.append(event)
140  
-                    slave_events.sort(key=lambda x:x.connected)
141  
-                    events.sort(key=lambda x:x.connected)
  161
+                    slave_events.sort(key=lambda x: x.connected)
  162
+                    events.sort(key=lambda x: x.connected)
142 163
             else:
143  
-                # If this is a disconnect event, find the last connect event and mark it as disconnected
  164
+                # If this is a disconnect event, find the last connect event
  165
+                # and mark it as disconnected
144 166
                 found = False
145 167
                 for event in reversed(slave_events):
146 168
                     if event.connected < t:
@@ -186,7 +208,8 @@ def updateFromFiles(session, master_url, master_name, builders, last_time, updat
186 208
 
187 209
         updateBuilderSlaves(session, bb_builder, db_builder)
188 210
         if update_times:
189  
-            updateSlaveTimes(session, master, bb_builder, db_builder, last_time)
  211
+            updateSlaveTimes(
  212
+                session, master, bb_builder, db_builder, last_time)
190 213
 
191 214
         for j, buildNumber in enumerate(builds):
192 215
             master = session.merge(master)
@@ -196,8 +219,8 @@ def updateFromFiles(session, master_url, master_name, builders, last_time, updat
196 219
                 eta = 0
197 220
             else:
198 221
                 eta = (time.time() - s) / (complete)
199  
-                eta = (1-complete) * eta
200  
-            print builder, buildNumber, "%i/%i" % (j+1, bn), "%.2f%% complete" % (100* complete), "ETA in %i seconds" % eta
  222
+                eta = (1 - complete) * eta
  223
+            print builder, buildNumber, "%i/%i" % (j + 1, bn), "%.2f%% complete" % (100 * complete), "ETA in %i seconds" % eta
201 224
             i += 1
202 225
             build = getBuild(builder, buildNumber)
203 226
             if not build:
@@ -207,14 +230,15 @@ def updateFromFiles(session, master_url, master_name, builders, last_time, updat
207 230
                 starttime = datetime.utcfromtimestamp(build.started)
208 231
 
209 232
             q = session.query(model.Build).filter_by(
210  
-                    master_id=master.id,
211  
-                    builder=db_builder,
212  
-                    buildnumber=build.number,
213  
-                    starttime=starttime,
214  
-                    )
  233
+                master_id=master.id,
  234
+                builder=db_builder,
  235
+                buildnumber=build.number,
  236
+                starttime=starttime,
  237
+            )
215 238
             db_build = q.first()
216 239
             if not db_build:
217  
-                db_build = model.Build.fromBBBuild(session, build, builder_name, master.id)
  240
+                db_build = model.Build.fromBBBuild(
  241
+                    session, build, builder_name, master.id)
218 242
             else:
219 243
                 db_build.updateFromBBBuild(session, build)
220 244
             session.commit()
@@ -226,10 +250,12 @@ def updateFromFiles(session, master_url, master_name, builders, last_time, updat
226 250
 
227 251
     parser = OptionParser("%prog [options] builders")
228 252
     parser.add_option("-d", "--database", dest="database", help="database url")
229  
-    parser.add_option("-m", "--master", dest="master", help="master url (buildbotURL in the master.cfg file)")
230  
-    parser.add_option("-n", "--description", dest="name", help="human friendly name for master")
  253
+    parser.add_option("-m", "--master", dest="master",
  254
+                      help="master url (buildbotURL in the master.cfg file)")
  255
+    parser.add_option("-n", "--description", dest="name",
  256
+                      help="human friendly name for master")
231 257
     parser.add_option("", "--times", dest="times", help="update slave connect/disconnect times", action="store_true", default=False)
232  
-    parser.add_option("-c", "--config", dest="config", 
  258
+    parser.add_option("-c", "--config", dest="config",
233 259
                       help="read configurations from a file")
234 260
 
235 261
     options, args = parser.parse_args()
@@ -249,7 +275,7 @@ def updateFromFiles(session, master_url, master_name, builders, last_time, updat
249 275
             for param in supported_params:
250 276
                 # Rewrite empty CLI params if we have them in config
251 277
                 if not getattr(options, param, None) and \
252  
-                   config.has_option('DEFAULT', param):
  278
+                        config.has_option('DEFAULT', param):
253 279
                     setattr(options, param, config.get('DEFAULT', param))
254 280
 
255 281
         except (ParsingError, MissingSectionHeaderError):
@@ -294,10 +320,11 @@ def updateFromFiles(session, master_url, master_name, builders, last_time, updat
294 320
     except:
295 321
         last_time = 0
296 322
 
297  
-    print "\n" + "-"*75
  323
+    print "\n" + "-" * 75
298 324
     print "Starting update at", time.ctime(started)
299 325
 
300  
-    updated = updateFromFiles(session, options.master, options.name, builders, last_time, options.times)
  326
+    updated = updateFromFiles(session, options.master, options.name,
  327
+                              builders, last_time, options.times)
301 328
 
302 329
     print "Updated", updated, "builds in:"
303 330
 
41  changes/ftppoller.py
@@ -8,13 +8,14 @@
8 8
 from buildbot.changes import base, changes
9 9
 from buildbotcustom.l10n import ParseLocalesFile
10 10
 
  11
+
11 12
 class FtpPollerBase(base.ChangeSource):
12 13
     """This source will poll an ftp directory searching for a specific file and when found
13 14
     trigger a change to the change master."""
14 15
 
15 16
     compare_attrs = ["ftpURLs", "pollInterval", "branch"]
16 17
 
17  
-    parent = None # filled in when we're added
  18
+    parent = None  # filled in when we're added
18 19
     loop = None
19 20
     volatile = ['loop']
20 21
     working = 0
@@ -72,11 +73,11 @@ def _get_changes(self, url):
72 73
 
73 74
     def _process_changes(self, pageContents, url):
74 75
         if self.parseContents(pageContents):
75  
-            c = changes.Change(who = url,
76  
-                           comments = "success",
77  
-                           files = [],
78  
-                           properties={'who': url},
79  
-                           branch = self.branch)
  76
+            c = changes.Change(who=url,
  77
+                               comments="success",
  78
+                               files=[],
  79
+                               properties={'who': url},
  80
+                               branch=self.branch)
80 81
             self.parent.addChange(c)
81 82
 
82 83
 
@@ -99,7 +100,8 @@ def __init__(self, searchString="", **kwargs):
99 100
     def parseContents(self, pageContents):
100 101
         """ Check through lines to see if file exists """
101 102
         # scenario 1:
102  
-        # buildbot restarts or file already exists, so we don't want to trigger anything
  103
+        # buildbot restarts or file already exists, so we don't want to trigger
  104
+        # anything
103 105
         if self.gotFile == 1:
104 106
             if re.search(self.searchString, pageContents):
105 107
                 self.stopService()
@@ -151,7 +153,8 @@ def _get_ftp(self, locales, url):
151 153
         """Poll the ftp page with the given url. Return the page as a string
152 154
            along with the list of locales from the previous callback"""
153 155
         d = self._get_page(url)
154  
-        d.addCallback(lambda result: {'pageContents': result, 'locales': locales})
  156
+        d.addCallback(
  157
+            lambda result: {'pageContents': result, 'locales': locales})
155 158
         return d
156 159
 
157 160
     def poll(self):
@@ -174,18 +177,19 @@ def _get_locales(self, pageContents):
174 177
         parsedLocales = ParseLocalesFile(pageContents)
175 178
         return [re.compile(re.escape("%s/" % l)) for l in parsedLocales if len(parsedLocales[l]) == 0 or self.sl_platform_map[self.platform] in parsedLocales[l]]
176 179
 
177  
-
178 180
     def searchAllStrings(self, pageContents, locales):
179 181
         """match the ftp page against the locales list"""
180 182
         req_matches = len(locales)
181  
-        #count number of strings with at least one positive match
182  
-        matches = sum([1 for regex in locales if re.search(regex, pageContents)])
  183
+        # count number of strings with at least one positive match
  184
+        matches = sum(
  185
+            [1 for regex in locales if re.search(regex, pageContents)])
183 186
         return matches == req_matches
184 187
 
185 188
     def parseContents(self, pageContents, locales):
186 189
         """ Check through lines to see if file exists """
187 190
         # scenario 1:
188  
-        # buildbot restarts or all files already exist, so we don't want to trigger anything
  191
+        # buildbot restarts or all files already exist, so we don't want to
  192
+        # trigger anything
189 193
         if self.gotAllFiles:
190 194
             if self.searchAllStrings(pageContents, locales):
191 195
                 self.stopService()
@@ -207,14 +211,15 @@ def _process_changes(self, results, url):
207 211
         pageContents = results['pageContents']
208 212
         locales = results['locales']
209 213
         if self.parseContents(pageContents, locales):
210  
-            c = changes.Change(who = url,
211  
-                           comments = "success",
212  
-                           files = [],
213  
-                           branch = self.branch)
  214
+            c = changes.Change(who=url,
  215
+                               comments="success",
  216
+                               files=[],
  217
+                               branch=self.branch)
214 218
             self.parent.addChange(c)
215  
-        #return the locales list for the next ftp poller in the callback chain
  219
+        # return the locales list for the next ftp poller in the callback chain
216 220
         return locales
217 221
 
  222
+
218 223
 class UrlPoller(FtpPollerBase):
219 224
     compare_attrs = FtpPollerBase.compare_attrs + ['url']
220 225
     gotFile = True
@@ -225,7 +230,7 @@ def __init__(self, url, **kwargs):
225 230
 
226 231
     def poll(self):
227 232
         if self.working > 0:
228  
-            log.msg("Not polling UrlPoller because last poll is still working (%s)" \
  233
+            log.msg("Not polling UrlPoller because last poll is still working (%s)"
229 234
                     % self.working)
230 235
         else:
231 236
             self.working = self.working + 1
42  changes/hgpoller.py
@@ -116,19 +116,23 @@ def __init__(self, d):
116 116
         self.d = defer.Deferred()
117 117
         self.dead = False
118 118
         d.addCallbacks(self.succeeded, self.failed)
  119
+
119 120
     def succeeded(self, result):
120 121
         if self.dead:
121 122
             log.msg("Dead pluggable got called")
122 123
         else:
123 124
             self.d.callback(result)
124  
-    def failed(self, fail = None):
  125
+
  126
+    def failed(self, fail=None):
125 127
         if self.dead:
126 128
             log.msg("Dead pluggable got errbacked")
127 129
         else:
128 130
             self.d.errback(fail)
129 131
 
  132
+
130 133
 class BasePoller(object):
131 134