/
srcloner.py
370 lines (310 loc) · 14.3 KB
/
srcloner.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.
import ftplib
import logging
import os
import requests
import shutil
import sys
import tempfile
import stoneridge
LINUX_SUBDIRS = ('try-linux64',) # We only do 64-bit linux tests
MAC_SUBDIRS = ('try-macosx64',) # There is only one OS X build
WINDOWS_SUBDIRS = ('try-win32',) # win64 is unsupported, so ignore it for now
EMAIL_MESSAGE = '''Hello, %s
This is the Stone Ridge service. Unfortunately, I have had to cancel your test
run for the following reason:
%s
I hope this doesn't impact the happiness of your day too significantly.
My sincerest (for a computer) apologies,
-Stone Ridge
'''
class StoneRidgeCloner(object):
"""This runs on the central stone ridge server, and downloads releases from
ftp.m.o to a local directory that is served up to the clients by a plain
ol' web server. Those clients use stoneridge_downloader.py to get the files
they need from the central server.
"""
def __init__(self, nightly, srid, operating_systems, netconfigs,
ldap, sha, attempt):
self.host = stoneridge.get_config('cloner', 'host')
self.nightly = nightly
self.outroot = stoneridge.get_config('cloner', 'output')
self.srid = srid
self.outdir = os.path.join(self.outroot, srid)
self.keep = stoneridge.get_config_int('cloner', 'keep', default=50)
self.max_attempts = stoneridge.get_config_int('cloner', 'attempts')
self.operating_systems = operating_systems
self.netconfigs = netconfigs
self.ldap = ldap
self.sha = sha
self.attempt = attempt
if not os.path.exists(self.outroot):
os.mkdir(self.outroot)
root = stoneridge.get_config('cloner', 'root')
if nightly:
self.path = '/'.join([root, 'nightly', 'latest-mozilla-central'])
else:
self.path = '/'.join([root, 'try-builds', '%s-%s' % (ldap, sha)])
logging.debug('host: %s' % (self.host,))
logging.debug('path: %s' % (self.path,))
logging.debug('nightly: %s' % (self.nightly,))
logging.debug('srid: %s' % (self.srid,))
logging.debug('output root: %s' % (self.outroot,))
logging.debug('output directory: %s' % (self.outdir,))
logging.debug('keep history: %s' % (self.keep,))
logging.debug('max attempts: %s' % (self.max_attempts,))
logging.debug('operating systems: %s' % (self.operating_systems,))
logging.debug('netconfigs: %s' % (self.netconfigs,))
logging.debug('ldap: %s' % (self.ldap,))
logging.debug('sha: %s' % (self.sha,))
logging.debug('attempt: %s' % (self.attempt,))
self.prefix = ''
def _gather_filelist(self, path):
"""Get the list of files available on our FTP server
Returns: list of filenames relative to the path on the server
"""
logging.debug('gathering files from ftp server')
try:
ftp = ftplib.FTP(self.host)
ftp.login()
ftp.cwd(path)
files = ftp.nlst()
ftp.quit()
except:
# We blanket-catch exceptions here, because we want the error
# handling in the top level to take precedence for ANY problem that
# happens while listing the directory. Logging helps us track down
# unexpected errors that may occur.
logging.exception('Unable to list files in %s' % (path,))
return []
logging.debug('files in %s: %s' % (path, files))
return files
def _build_dl_url(self, try_subdir, fname):
"""Create a download (https) URL for a particular file
Returns: a URL string
"""
logging.debug('creating download url for %s' % (fname,))
remotefile = self.path
if not self.nightly:
remotefile = '/'.join([remotefile, try_subdir])
remotefile = '/'.join([remotefile, fname])
logging.debug('remote filename: %s' % (remotefile,))
url = 'https://%s%s' % (self.host, remotefile)
logging.debug('url: %s' % (url,))
return url
def _get_prefix(self, files):
"""Get the filename prefix that is common to all the files we'll need
to download
Returns: <prefix (string)>
"""
logging.debug('getting filename prefix')
prefixfile = [f for f in files if f.endswith('.checksums.asc')][-1]
prefix = prefixfile.replace('.checksums.asc', '')
prefix = prefix.rsplit('.', 1)[0] # Strip off the platform information
logging.debug('filename prefix: %s' % (prefix,))
return prefix
def _ensure_outdir(self, platform):
"""Ensure the output directory for a platform exists
"""
logging.debug('ensuring output directory for %s exists' % (platform,))
if not os.path.exists(self.outdir):
logging.debug('creating outdir %s' % (self.outdir,))
os.mkdir(self.outdir)
platdir = os.path.join(self.outdir, platform)
logging.debug('platform directory: %s' % (platdir,))
if not os.path.exists(platdir):
logging.debug('creating platform directory %s' % (platdir,))
os.mkdir(platdir)
def _dl_to_file(self, url, outfile):
"""Download the file at <url> and save it to the file
at <outfile>
"""
logging.debug('downloading %s => %s' % (url, outfile))
resp = requests.get(url, timeout=30000)
with file(outfile, 'wb') as f:
logging.debug('writing file contents')
f.write(resp.content)
def _dl_test_zip(self, try_subdir, archid, outdir):
"""Download the test zip for a particular architecture id (<archid>)
and save it at <outdir>/tests.zip
"""
logging.debug('downloading test zip for %s to %s' % (archid, outdir))
srcfile = '%s.%s.tests.zip' % (self.prefix, archid)
logging.debug('zip source filename: %s' % (srcfile,))
url = self._build_dl_url(try_subdir, srcfile)
outfile = os.path.join(self.outdir, outdir, 'tests.zip')
logging.debug('zip dest filename: %s' % (outfile,))
self._dl_to_file(url, outfile)
def _clone_mac(self):
"""Clone the dmg and tests zip for the mac build
"""
logging.debug('cloning mac build')
self._ensure_outdir('mac')
logging.debug('downloading firefox dmg')
dmg = '%s.mac.dmg' % (self.prefix,)
logging.debug('dmg source filename: %s' % (dmg,))
url = self._build_dl_url(MAC_SUBDIRS[0], dmg)
outfile = os.path.join(self.outdir, 'mac', 'firefox.dmg')
logging.debug('dmg dest filename: %s' % (outfile,))
self._dl_to_file(url, outfile)
self._dl_test_zip(MAC_SUBDIRS[0], 'mac', 'mac')
def _clone_linux(self):
"""Clone the .tar.bz2 and tests zip for both 32-bit and 64-bit linux
builds
"""
logging.debug('cloning linux builds')
archids = ('x86_64',)
outdirs = ('linux64',)
for archid, outdir, subdir in zip(archids, outdirs, LINUX_SUBDIRS):
logging.debug('architecture: %s' % (archid,))
logging.debug('outdir: %s' % (outdir,))
self._ensure_outdir(outdir)
logging.debug('downloading firefox tarball')
srcfile = '%s.linux-%s.tar.bz2' % (self.prefix, archid)
logging.debug('tarball source filename: %s' % (srcfile,))
url = self._build_dl_url(subdir, srcfile)
outfile = os.path.join(self.outdir, outdir, 'firefox.tar.bz2')
logging.debug('tarball dest filename: %s' % (outfile,))
self._dl_to_file(url, outfile)
self._dl_test_zip(subdir, 'linux-%s' % (archid,), outdir)
def _clone_win(self):
"""Clone the firefox zip and tests zip for both 32-bit and 64-bit
windows builds
"""
logging.debug('cloning windows build')
self._ensure_outdir('win32')
logging.debug('downloading firefox zip')
srcfile = '%s.win32.zip' % (self.prefix,)
logging.debug('zip source filename: %s' % (srcfile,))
url = self._build_dl_url(WINDOWS_SUBDIRS[0], srcfile)
outfile = os.path.join(self.outdir, 'win32', 'firefox.zip')
logging.debug('zip dest filename: %s' % (outfile,))
self._dl_to_file(url, outfile)
self._dl_test_zip(WINDOWS_SUBDIRS[0], 'win32', 'win32')
def _cleanup_old_directories(self):
"""We only keep around so many directories of historical firefoxen.
This gets rid of ones we don't care about any more
"""
logging.debug('cleaning up old directories')
with stoneridge.cwd(self.outroot):
listing = os.listdir('.')
logging.debug('candidate files: %s' % (listing,))
# We want to make sure that we're not looking at anything that's
# not a directory that may have somehow gotten into our directory.
# We also need to ignore dotfiles.
directories = [l for l in listing
if os.path.isdir(l) and not l.startswith('.')]
logging.debug('directories: %s' % (directories,))
# Find out when the directories were last modified, and sort the
# list by that, so we can delete the oldest ones.
times = [(d, os.stat(d).st_mtime) for d in directories]
times.sort(key=lambda x: x[1])
# Now we can figure out which directories to delete!
delete_us = times[:-self.keep]
logging.debug('directories to delete: %s' % (delete_us,))
for d in delete_us:
logging.debug('removing %s' % (d,))
shutil.rmtree(d)
def defer(self):
args = ['srdeferrer.py',
'--srid', self.srid,
'--config', stoneridge.get_config_file(),
'--log', '/dev/null',
'--pidfile', tempfile.mktemp(),
'--attempt', self.attempt + 1]
if self.nightly:
args.append('--nightly')
else:
args.extend(['--ldap', self.ldap])
args.extend(['--sha', self.sha])
for ops in self.operating_systems:
args.append('--%s' % (ops,))
for nc in self.netconfigs:
args.append('--%s' % (nc,))
stoneridge.run_process(*args)
def email(self, failure_message):
if not self.ldap:
return
message = EMAIL_MESSAGE % (self.ldap, failure_message)
stoneridge.sendmail(self.ldap, 'Stone Ridge Run Cancelled', message)
def exit_and_maybe_defer(self, deferred_message):
next_attempt = self.attempt + 1
if next_attempt > self.max_attempts:
logging.error('Unable to get build results for %s after %s '
'attempts. Cancelling run.' %
(self.srid, self.max_attempts))
self.email(deferred_message)
else:
logging.debug(deferred_message)
self.defer()
sys.exit(1)
def run(self):
files = self._gather_filelist(self.path)
if not self.nightly:
# For some ungodly reason, try builds have a different directory
# structure than nightly builds, so we have to handle them
# differently. Instead of all output being at the same level,
# they are separated out by platform for try builds. Le sigh.
subdirs = []
dist_files = None
if 'linux' in self.operating_systems:
subdirs.extend(LINUX_SUBDIRS)
if 'mac' in self.operating_systems:
subdirs.extend(MAC_SUBDIRS)
if 'windows' in self.operating_systems:
subdirs.extend(WINDOWS_SUBDIRS)
# Be reasonably sure the try run is complete, such that everything
# is ready for us to download.
for d in subdirs:
if d not in files:
self.exit_and_maybe_defer(
'Run %s not available' % (d,))
dist_path = '/'.join([self.path, subdirs[0]])
dist_files = self._gather_filelist(dist_path)
if not dist_files:
# We didn't get any files listed, but we should have. Just drop
# this run on the floor
self.email('No dist files found for srid %s' % (self.srid,))
logging.error('No files found! Dropping srid %s' %
(self.srid,))
sys.exit(1)
files = dist_files
if not files:
self.exit_and_maybe_defer(
'No files found for %s' % (self.srid,))
self.prefix = self._get_prefix(files)
# Make sure our output directory exists
if not os.path.exists(self.outdir):
logging.debug('creating output directory')
os.mkdir(self.outdir)
# Now download all the builds and test zipfiles
if self.nightly or 'mac' in self.operating_systems:
self._clone_mac()
if self.nightly or 'linux' in self.operating_systems:
self._clone_linux()
if self.nightly or 'windows' in self.operating_systems:
self._clone_win()
self._cleanup_old_directories()
@stoneridge.main
def main():
parser = stoneridge.ArgumentParser()
parser.add_argument('--nightly', dest='nightly', action='store_true',
default=False)
parser.add_argument('--srid', dest='srid', required=True)
for ops in stoneridge.OPERATING_SYSTEMS:
parser.add_argument('--%s' % (ops,), dest='operating_systems',
action='append_const', const=ops, default=[])
for nc in stoneridge.NETCONFIGS:
parser.add_argument('--%s' % (nc,), dest='netconfigs',
action='append_const', const=nc, default=[])
parser.add_argument('--attempt', dest='attempt', required=True, type=int)
parser.add_argument('--ldap', dest='ldap', default='')
parser.add_argument('--sha', dest='sha', default='')
args = parser.parse_args()
cloner = StoneRidgeCloner(args.nightly, args.srid, args.operating_systems,
args.netconfigs, args.ldap, args.sha,
args.attempt)
cloner.run()