Skip to content
Newer
Older
100644 270 lines (235 sloc) 10.6 KB
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 # You can obtain one at http://mozilla.org/MPL/2.0/.
4
5 import base64
6 import datetime
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
7 import ftplib
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
8 import logging
9 import os
10 import pytz
11 import re
12 import shutil
6251b23 @markrcote trigger_runs.py can trigger jobs for tinderbox builds and from local …
markrcote authored
13 import tempfile
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
14 import urllib
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
15 import urlparse
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
16 import zipfile
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
17
18 class NightlyBranch(object):
19
20 nightly_dirnames = [re.compile('(.*)-mozilla-central-android$')]
21
22 def nightly_ftpdir(self, year, month):
23 return 'ftp://ftp.mozilla.org/pub/mobile/nightly/%d/%02d/' % (year,
24 month)
25
26 def ftpdirs(self, start_time, end_time):
27 dirs = []
28 y = start_time.year
29 m = start_time.month
30 while y < end_time.year or m <= end_time.month:
31 dirs.append(self.nightly_ftpdir(y, m))
32 if m == 12:
33 y += 1
34 m = 1
35 else:
36 m += 1
37 return dirs
38
39 def build_info_from_ftp(self, ftpline):
40 srcdir = ftpline.split(' ')[-1].strip()
41 build_time = None
42 dirnamematch = None
43 logging.debug('matching dir names')
44 for r in self.nightly_dirnames:
45 dirnamematch = r.match(srcdir)
46 if dirnamematch:
47 break
48 if dirnamematch:
49 logging.debug('build time')
50 build_time = datetime.datetime.strptime(dirnamematch.group(1),
51 '%Y-%m-%d-%H-%M-%S')
52 build_time = build_time.replace(tzinfo=pytz.timezone('US/Pacific'))
53 logging.debug('got build time')
54 logging.debug('got info')
55 return (srcdir, build_time)
56
57 def build_date_from_url(self, url):
58 # nightly urls are of the form
59 # ftp://ftp.mozilla.org/pub/mobile/nightly/<year>/<month>/<year>-
60 # <month>-<day>-<hour>-<minute>-<second>-<branch>-android/
61 # <buildfile>
62 m = re.search('nightly\/[\d]{4}\/[\d]{2}\/([\d]{4}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2})-', url)
63 if not m:
64 return None
65 return datetime.datetime.strptime(m.group(1), '%Y-%m-%d-%H-%M-%S')
66
67
68 class TinderboxBranch(object):
69
70 main_ftp_url = 'ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/'
71
72 def ftpdirs(self, start_time, end_time):
73 # FIXME: Can we be certain that there's only one buildID (unique
74 # timestamp) regardless of branch (at least m-i vs m-c)?
75 return [self.main_ftp_url + 'mozilla-inbound-android/',
76 self.main_ftp_url + 'mozilla-central-android/']
77
78 def build_info_from_ftp(self, ftpline):
79 srcdir = ftpline.split()[8].strip()
80 build_time = datetime.datetime.fromtimestamp(int(srcdir), pytz.timezone('US/Pacific'))
81 return (srcdir, build_time)
82
83 def build_date_from_url(self, url):
84 # tinderbox urls are of the form
85 # ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/
86 # <branch>-android/<build timestamp>/<buildfile>
87 m = re.search('tinderbox-builds\/.*-android\/[\d]+\/', url)
88 if not m:
89 return None
90 return datetime.datetime.fromtimestamp(int(m.group(1)),
91 pytz.timezone('US/Pacific'))
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
92
93
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
94 class BuildCacheException(Exception):
95 pass
96
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
97 class BuildCache(object):
98
6251b23 @markrcote trigger_runs.py can trigger jobs for tinderbox builds and from local …
markrcote authored
99 MAX_NUM_BUILDS = 20
100 EXPIRE_AFTER_SECONDS = 60*60*24
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
101
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
102 class FtpLineCache(object):
103 def __init__(self):
104 self.lines = []
105 def __call__(self, line):
106 self.lines.append(line)
107
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
108 def __init__(self, cache_dir='builds', override_build_dir = None, enable_unittests = False):
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
109 self.cache_dir = cache_dir
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
110 self.enable_unittests = enable_unittests
111 self.override_build_dir = override_build_dir
112 if override_build_dir:
113 if not os.path.exists(override_build_dir):
114 raise BuildCacheException('Override Build Directory does not exist')
115
116 build_path = os.path.join(override_build_dir, 'build.apk')
117 if not os.path.exists(build_path):
118 raise BuildCacheException('Override Build Directory %s does not contain a build.apk.' %
119 override_build_dir)
120
121 tests_path = os.path.join(override_build_dir, 'tests')
122 if self.enable_unittests and not os.path.exists(tests_path):
123 raise BuildCacheException('Override Build Directory %s does not contain a tests directory.' %
124 override_build_dir)
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
125 if not os.path.exists(self.cache_dir):
126 os.mkdir(self.cache_dir)
127
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
128 @classmethod
129 def branch(cls, s):
130 if 'nightly' in s:
131 return NightlyBranch()
132 if 'tinderbox' in s:
133 return TinderboxBranch()
134 return None
135
136 def find_latest_build(self, branch_name='nightly'):
56f5c82 @markrcote Bug 783585 Improve latest-build downloader and handle build errors in…
markrcote authored
137 window = datetime.timedelta(days=3)
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
138 now = datetime.datetime.now()
56f5c82 @markrcote Bug 783585 Improve latest-build downloader and handle build errors in…
markrcote authored
139 builds = self.find_builds(now - window, now, branch_name)
140 if not builds:
141 logging.error('Could not find any nightly builds in the last '
142 '%d days!' % window.days)
143 return None
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
144 builds.sort()
145 return builds[-1]
146
147 def find_builds(self, start_time, end_time, branch_name='nightly'):
148 branch = self.branch(branch_name)
149 if not branch:
56f5c82 @markrcote Bug 783585 Improve latest-build downloader and handle build errors in…
markrcote authored
150 logging.error('unsupported branch "%s"' % branch_name)
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
151 return []
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
152
6251b23 @markrcote trigger_runs.py can trigger jobs for tinderbox builds and from local …
markrcote authored
153 if not start_time.tzinfo:
154 start_time = start_time.replace(tzinfo=pytz.timezone('US/Pacific'))
155 if not end_time.tzinfo:
156 end_time = end_time.replace(tzinfo=pytz.timezone('US/Pacific'))
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
157
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
158 builds = []
159 fennecregex = re.compile("fennec.*\.android-arm\.apk")
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
160
161 for d in branch.ftpdirs(start_time, end_time):
162 url = urlparse.urlparse(d)
163 logging.debug('logging into %s...' % url.netloc)
164 f = ftplib.FTP(url.netloc)
165 f.login()
166 logging.debug('looking for builds in %s...' % url.path)
167 lines = self.FtpLineCache()
168 f.dir(url.path, lines)
169 file('lines.out', 'w').write('\n'.join(lines.lines))
170 for line in lines.lines:
171 srcdir, build_time = branch.build_info_from_ftp(line)
172
173 if not build_time:
174 continue
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
175
6251b23 @markrcote trigger_runs.py can trigger jobs for tinderbox builds and from local …
markrcote authored
176 if build_time and (build_time < start_time or
177 build_time > end_time):
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
178 continue
179
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
180 newpath = url.path + srcdir
181 lines2 = self.FtpLineCache()
182 f.dir(newpath, lines2)
183 for l2 in lines2.lines:
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
184 filename = l2.split(' ')[-1].strip()
185 if fennecregex.match(filename):
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
186 buildurl = url.scheme + '://' + url.netloc + newpath + "/" + filename
187 builds.append(buildurl)
56f5c82 @markrcote Bug 783585 Improve latest-build downloader and handle build errors in…
markrcote authored
188 if not builds:
189 logging.error('No builds found.')
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
190 return builds
191
192 def build_date(self, url):
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
193 branch = self.branch(url)
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
194 builddate = None
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
195 if branch:
196 builddate = branch.build_date_from_url(url)
197 if not builddate:
198 logging.error('bad URL "%s"' % url)
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
199 return builddate
200
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
201 def get(self, buildurl, enable_unittests, force=False):
202 if self.override_build_dir:
203 return self.override_build_dir
204 build_dir = base64.b64encode(buildurl)
6251b23 @markrcote trigger_runs.py can trigger jobs for tinderbox builds and from local …
markrcote authored
205 self.clean_cache([build_dir])
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
206 cache_build_dir = os.path.join(self.cache_dir, build_dir)
207 build_path = os.path.join(cache_build_dir, 'build.apk')
208 if not os.path.exists(cache_build_dir):
209 os.makedirs(cache_build_dir)
210 if force or not os.path.exists(build_path):
6251b23 @markrcote trigger_runs.py can trigger jobs for tinderbox builds and from local …
markrcote authored
211 # retrieve to temporary file then move over, so we don't end
212 # up with half a file if it aborts
213 tmpf = tempfile.NamedTemporaryFile(delete=False)
214 tmpf.close()
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
215 urllib.urlretrieve(buildurl, tmpf.name)
216 os.rename(tmpf.name, build_path)
217 file(os.path.join(cache_build_dir, 'lastused'), 'w')
218 if enable_unittests:
219 tests_path = os.path.join(cache_build_dir, 'tests')
220 if (force or not os.path.exists(tests_path)) and enable_unittests:
221 tmpf = tempfile.NamedTemporaryFile(delete=False)
222 tmpf.close()
223 # XXX: assumes fixed buildurl-> tests_url mapping
224 tests_url = re.sub('.apk$', '.tests.zip', buildurl)
225 urllib.urlretrieve(tests_url, tmpf.name)
226 tests_zipfile = zipfile.ZipFile(tmpf.name)
227 tests_zipfile.extractall(tests_path)
228 tests_zipfile.close()
229 os.unlink(tmpf.name)
230 # XXX: assumes fixed buildurl-> robocop mapping
231 robocop_url = urlparse.urljoin(buildurl, 'robocop.apk')
232 robocop_path = os.path.join(cache_build_dir, 'robocop.apk')
233 tmpf = tempfile.NamedTemporaryFile(delete=False)
234 tmpf.close()
235 urllib.urlretrieve(robocop_url, tmpf.name)
236 os.rename(tmpf.name, robocop_path)
237 # XXX: assumes fixed buildurl-> fennec_ids.txt mapping
238 fennec_ids_url = urlparse.urljoin(buildurl, 'fennec_ids.txt')
239 fennec_ids_path = os.path.join(cache_build_dir, 'fennec_ids.txt')
240 tmpf = tempfile.NamedTemporaryFile(delete=False)
241 tmpf.close()
242 urllib.urlretrieve(fennec_ids_url, tmpf.name)
243 os.rename(tmpf.name, fennec_ids_path)
244 return cache_build_dir
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
245
6251b23 @markrcote trigger_runs.py can trigger jobs for tinderbox builds and from local …
markrcote authored
246 def clean_cache(self, preserve=[]):
247 def lastused_path(d):
248 return os.path.join(self.cache_dir, d, 'lastused')
249 def keep_build(d):
250 if preserve and d in preserve:
251 # specifically keep this build
252 return True
253 if not os.path.exists(lastused_path(d)):
254 # probably not a build dir
255 return True
256 if ((datetime.datetime.now() -
257 datetime.datetime.fromtimestamp(os.stat(lastused_path(d)).st_mtime) <=
258 datetime.timedelta(microseconds=1000*1000*self.EXPIRE_AFTER_SECONDS))):
259 # too new
260 return True
261 return False
262
263 builds = [(x, os.stat(lastused_path(x)).st_mtime) for x in
264 os.listdir(self.cache_dir) if not keep_build(x)]
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
265 builds.sort(key=lambda x: x[1])
266 while len(builds) > self.MAX_NUM_BUILDS:
6251b23 @markrcote trigger_runs.py can trigger jobs for tinderbox builds and from local …
markrcote authored
267 b = builds.pop(0)[0]
268 logging.info('Expiring %s' % b)
269 shutil.rmtree(os.path.join(self.cache_dir, b))
Something went wrong with that request. Please try again.