/
utils.py
201 lines (163 loc) · 7.52 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from future.utils import lrange
from future.utils import string_types
from collections import UserList
from twisted.internet import defer
from twisted.python import log
from buildbot.data import resultspec
from buildbot.process.properties import renderer
from buildbot.process.results import RETRY
from buildbot.util import flatten
@defer.inlineCallbacks
def getPreviousBuild(master, build):
# naive n-1 algorithm. Still need to define what we should skip
# SKIP builds? forced builds? rebuilds?
# don't hesitate to contribute improvements to that algorithm
n = build['number'] - 1
while n >= 0:
prev = yield master.data.get(("builders", build['builderid'], "builds", n))
if prev and prev['results'] != RETRY:
return prev
n -= 1
return None
@defer.inlineCallbacks
def getDetailsForBuildset(master, bsid, wantProperties=False, wantSteps=False,
wantPreviousBuild=False, wantLogs=False):
# Here we will do a bunch of data api calls on behalf of the reporters
# We do try to make *some* calls in parallel with the help of gatherResults, but don't commit
# to much in that. The idea is to do parallelism while keeping the code readable
# and maintainable.
# first, just get the buildset and all build requests for our buildset id
dl = [master.data.get(("buildsets", bsid)),
master.data.get(('buildrequests', ),
filters=[resultspec.Filter('buildsetid', 'eq', [bsid])])]
(buildset, breqs) = yield defer.gatherResults(dl)
# next, get the bdictlist for each build request
dl = [master.data.get(("buildrequests", breq['buildrequestid'], 'builds'))
for breq in breqs]
builds = yield defer.gatherResults(dl)
builds = flatten(builds, types=(list, UserList))
if builds:
yield getDetailsForBuilds(master, buildset, builds, wantProperties=wantProperties,
wantSteps=wantSteps, wantPreviousBuild=wantPreviousBuild, wantLogs=wantLogs)
return dict(buildset=buildset, builds=builds)
@defer.inlineCallbacks
def getDetailsForBuild(master, build, wantProperties=False, wantSteps=False,
wantPreviousBuild=False, wantLogs=False):
buildrequest = yield master.data.get(("buildrequests", build['buildrequestid']))
buildset = yield master.data.get(("buildsets", buildrequest['buildsetid']))
build['buildrequest'], build['buildset'] = buildrequest, buildset
ret = yield getDetailsForBuilds(master, buildset, [build],
wantProperties=wantProperties, wantSteps=wantSteps,
wantPreviousBuild=wantPreviousBuild, wantLogs=wantLogs)
return ret
@defer.inlineCallbacks
def getDetailsForBuilds(master, buildset, builds, wantProperties=False, wantSteps=False,
wantPreviousBuild=False, wantLogs=False):
builderids = {build['builderid'] for build in builds}
builders = yield defer.gatherResults([master.data.get(("builders", _id))
for _id in builderids])
buildersbyid = {builder['builderid']: builder
for builder in builders}
if wantProperties:
buildproperties = yield defer.gatherResults(
[master.data.get(("builds", build['buildid'], 'properties'))
for build in builds])
else: # we still need a list for the big zip
buildproperties = lrange(len(builds))
if wantPreviousBuild:
prev_builds = yield defer.gatherResults(
[getPreviousBuild(master, build) for build in builds])
else: # we still need a list for the big zip
prev_builds = lrange(len(builds))
if wantSteps:
buildsteps = yield defer.gatherResults(
[master.data.get(("builds", build['buildid'], 'steps'))
for build in builds])
if wantLogs:
for s in flatten(buildsteps, types=(list, UserList)):
logs = yield master.data.get(("steps", s['stepid'], 'logs'))
s['logs'] = list(logs)
for l in s['logs']:
l['content'] = yield master.data.get(("logs", l['logid'], 'contents'))
else: # we still need a list for the big zip
buildsteps = lrange(len(builds))
# a big zip to connect everything together
for build, properties, steps, prev in zip(builds, buildproperties, buildsteps, prev_builds):
build['builder'] = buildersbyid[build['builderid']]
build['buildset'] = buildset
build['url'] = getURLForBuild(
master, build['builderid'], build['number'])
if wantProperties:
build['properties'] = properties
if wantSteps:
build['steps'] = list(steps)
if wantPreviousBuild:
build['prev_build'] = prev
# perhaps we need data api for users with sourcestamps/:id/users
@defer.inlineCallbacks
def getResponsibleUsersForSourceStamp(master, sourcestampid):
changesd = master.data.get(("sourcestamps", sourcestampid, "changes"))
sourcestampd = master.data.get(("sourcestamps", sourcestampid))
changes, sourcestamp = yield defer.gatherResults([changesd, sourcestampd])
blamelist = set()
# normally, we get only one, but just assume there might be several
for c in changes:
blamelist.add(c['author'])
# Add patch author to blamelist
if 'patch' in sourcestamp and sourcestamp['patch'] is not None:
blamelist.add(sourcestamp['patch']['author'])
blamelist = list(blamelist)
blamelist.sort()
return blamelist
# perhaps we need data api for users with builds/:id/users
@defer.inlineCallbacks
def getResponsibleUsersForBuild(master, buildid):
dl = [
master.data.get(("builds", buildid, "changes")),
master.data.get(("builds", buildid, 'properties'))
]
changes, properties = yield defer.gatherResults(dl)
blamelist = set()
# add users from changes
for c in changes:
blamelist.add(c['author'])
# add owner from properties
if 'owner' in properties:
owner = properties['owner'][0]
if isinstance(owner, string_types):
blamelist.add(owner)
else:
blamelist.update(owner)
log.msg(
"Warning: owner property is a list for buildid {}. ".format(buildid))
log.msg("Please report a bug: changes: {}. properties: {}".format(
changes, properties))
# add owner from properties
if 'owners' in properties:
blamelist.update(properties['owners'][0])
blamelist = list(blamelist)
blamelist.sort()
return blamelist
def getURLForBuild(master, builderid, build_number):
prefix = master.config.buildbotURL
return prefix + "#builders/%d/builds/%d" % (
builderid,
build_number)
@renderer
def URLForBuild(props):
build = props.getBuild()
return build.getUrl()