diff --git a/master/buildbot/changes/changes.py b/master/buildbot/changes/changes.py index 3a2df332177..e7792f2c9ea 100644 --- a/master/buildbot/changes/changes.py +++ b/master/buildbot/changes/changes.py @@ -171,7 +171,7 @@ def asDict(self): result['revlink'] = getattr(self, 'revlink', None) result['properties'] = self.properties.asList() result['repository'] = getattr(self, 'repository', None) - result['codebase'] = getattr(self, 'codebase', None) + result['codebase'] = getattr(self, 'codebase', '') result['project'] = getattr(self, 'project', None) return result diff --git a/master/buildbot/db/migrate/versions/021_add_codebase.py b/master/buildbot/db/migrate/versions/021_add_codebase.py index a5b88368f85..7f03efc0cbd 100644 --- a/master/buildbot/db/migrate/versions/021_add_codebase.py +++ b/master/buildbot/db/migrate/versions/021_add_codebase.py @@ -25,8 +25,10 @@ def upgrade(migrate_engine): # Add codebase to tables - ss_codebase = sa.Column('codebase', sa.String(length=256)) + ss_codebase = sa.Column('codebase', sa.String(length=256), nullable=False, + server_default=sa.DefaultClause("")) ss_codebase.create(sourcestamps_table) - c_codebase = sa.Column('codebase', sa.String(length=256)) + c_codebase = sa.Column('codebase', sa.String(length=256), nullable=False, + server_default=sa.DefaultClause("")) c_codebase.create(changes_table) diff --git a/master/buildbot/db/model.py b/master/buildbot/db/model.py index ddb6fc4658f..e1dc1757a60 100644 --- a/master/buildbot/db/model.py +++ b/master/buildbot/db/model.py @@ -197,7 +197,8 @@ class Model(base.DBConnectorComponent): sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), # codebase is a logical name to specify what is in the repository - sa.Column('codebase', sa.String(256)), + sa.Column('codebase', sa.String(256), nullable=False, + server_default=sa.DefaultClause("")), # project names the project this source code represents. It is used # later to filter changes @@ -259,7 +260,8 @@ class Model(base.DBConnectorComponent): sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), # codebase is a logical name to specify what is in the repository - sa.Column('codebase', sa.String(256)), + sa.Column('codebase', sa.String(256), nullable=False, + server_default=sa.DefaultClause("")), # the project this source code represents sa.Column('project', sa.String(length=512), nullable=False, server_default=''), diff --git a/master/buildbot/master.py b/master/buildbot/master.py index c570a7e9d77..d3051569450 100644 --- a/master/buildbot/master.py +++ b/master/buildbot/master.py @@ -449,17 +449,22 @@ def handle_deprec(oldname, old, newname, new, default=None, properties[n] = (properties[n], 'Change') if self.config.codebaseGenerator is not None: - newchange = changes.Change(who=author, files=files, - comments=comments, isdir=is_dir, - revision=revision, - when=datetime2epoch(when_timestamp), - branch=branch, category=category, - revlink=revlink, properties=properties, - repository=repository, - project=project) - codebase = self.config.codebaseGenerator(newchange.asDict()) + chdict = {} + chdict['who']=author + chdict['files']=files + chdict['comments']=comments + chdict['isdir']=is_dir + chdict['revision']=revision + chdict['when']=when_timestamp + chdict['branch']=branch + chdict['category']=category + chdict['revlink']=revlink + chdict['properties']=properties + chdict['repository']=repository + chdict['project']=project + codebase = self.config.codebaseGenerator(chdict) else: - codebase = repository + codebase = '' d = defer.succeed(None) if src: diff --git a/master/buildbot/process/build.py b/master/buildbot/process/build.py index 574c29c9bae..29a679433f2 100644 --- a/master/buildbot/process/build.py +++ b/master/buildbot/process/build.py @@ -91,12 +91,7 @@ def setLocks(self, locks): def setSlaveEnvironment(self, env): self.slaveEnvironment = env - def getSourceStamp(self, codebase=None): - if codebase is None: - if self.sources: - return self.sources[0] - else: - return None + def getSourceStamp(self, codebase): for source in self.sources: if source.codebase == codebase: return source @@ -314,11 +309,7 @@ def setupBuild(self, expectations): step.setBuild(self) step.setBuildSlave(self.slavebuilder.slave) if callable (self.workdir): - if len(self.sources) == 1: - sources = self.sources[0] - else: - sources = self.sources - step.setDefaultWorkdir (self.workdir (sources)) + step.setDefaultWorkdir (self.workdir (self.sources)) else: step.setDefaultWorkdir (self.workdir) name = step.name @@ -359,6 +350,9 @@ def setupBuild(self, expectations): self.progress.setExpectationsFrom(expectations) # we are now ready to set up our BuildStatus. + # at the moment that schedulers start to deliver sourcestamps per codebase + # buildstatus should support multiple sourcestamps. + # until now schedulers deliver exactly one sourcestamp self.build_status.setSourceStamp(self.sources[0]) self.build_status.setReason(self.reason) self.build_status.setBlamelist(self.blamelist()) diff --git a/master/buildbot/process/buildrequest.py b/master/buildbot/process/buildrequest.py index 8801f2414c8..c1997b09d91 100644 --- a/master/buildbot/process/buildrequest.py +++ b/master/buildbot/process/buildrequest.py @@ -151,8 +151,8 @@ def store_source(source): def canBeMergedWith(self, other): #get all codebases from both requests - all_codebases = set(s.codebase for s in self.sources.itervalues()) - all_codebases |= set(s.codebase for s in other.sources.itervalues()) + all_codebases = set(self.sources.iterkeys()) + all_codebases &= set(other.sources.iterkeys()) for c in all_codebases: if not self.sources[c].canBeMergedWith(other.sources[c]): @@ -162,9 +162,9 @@ def canBeMergedWith(self, other): def mergeSourceStampsWith(self, others): """ Returns one merged sourcestamp for every codebase """ #get all codebases from all requests - all_codebases = set(s.codebase for s in self.sources.itervalues()) + all_codebases = set(self.sources.iterkeys()) for other in others: - all_codebases |= set(s.codebase for s in other.sources.itervalues()) + all_codebases |= set(other.sources.iterkeys()) all_merged_sources = {} # walk along the codebases diff --git a/master/buildbot/process/properties.py b/master/buildbot/process/properties.py index 9065b528174..afa24fbe2c2 100644 --- a/master/buildbot/process/properties.py +++ b/master/buildbot/process/properties.py @@ -288,6 +288,145 @@ def getRenderingFor(self, build): pmap.clear_temporary_values() return s +class InterpolateMap(object): + colon_minus_re = re.compile(r"(.*):-(.*)") + colon_tilde_re = re.compile(r"(.*):~(.*)") + colon_plus_re = re.compile(r"(.*):\+(.*)") + def __init__(self, properties, kwargs): + self.properties = properties + self.kwargs = kwargs + + def __getitem__(self, item): + properties = self.properties + key = '' + arg = '' + res = re.search('(?P[^:\[]+)(?P:|\[.*\])(?P.*)', item) + if res: + key = res.group('key') + index = res.group('index') + if index != ':': + #add src indirection to arg + arg = index + res.group('arg') + else: + arg = res.group('arg') + + def prop_fn(arg): + try: + prop, repl = arg.split(":", 1) + except ValueError: + prop, repl = arg, None + return properties, prop, repl + + def src_fn(arg): + ## TODO: Handle changes + codebase = attr = repl = None + + res = re.search('\[(?P.*)\]:(?P[^\]\]:]+):(?P.*)', + arg) + if res: + codebase = res.group('codebase') + attr = res.group('attr') + repl = res.group('repl') + else: + res = re.search('\[(?P.*)\]:(?P.*)', arg) + if res: + codebase = res.group('codebase') + attr = res.group('attr') + repl = None + ss = self.properties.getBuild().getSourceStamp(codebase) + if ss: + return ss.asDict(), attr, repl + else: + return {}, attr, repl + + def kw_fn(arg): + try: + kw, repl = arg.split(":", 1) + except ValueError: + kw, repl = arg, None + return self.kwargs, kw, repl + + for (keys, fn) in [ + (("p", "prop"), prop_fn), + (("s", "src"), src_fn), + (("kw", "kw"), kw_fn), + ]: + if key in keys: + d, kw, repl = fn(arg) + break; + + def colon_minus(d, kw, repl): + # %(prop:-repl)s + # if prop exists, use it; otherwise, use repl + if d.has_key(kw): + return d[kw] + else: + return repl % self + + def colon_tilde(d, kw, repl): + # %(prop:~repl)s + # if prop exists and is true (nonempty), use it; otherwise, use repl + if d.has_key(kw) and d[kw]: + return d[kw] + else: + return repl % self + + def colon_plus(d, kw, repl): + # %(prop:+repl)s + # if prop exists, use repl; otherwise, an empty string + if d.has_key(kw): + return repl % self + else: + return '' + + if repl is not None: + good_colon = False + for char, fn in [ + ( "-", colon_minus ), + ( "~", colon_tilde ), + ( "+", colon_plus ), + ]: + if repl[0] == char: + rv = fn(d, kw, repl[1:]) + good_colon = True + break + if not good_colon: + raise ValueError + else: + if d.has_key(kw): + rv = d[kw] + else: + rv = None + + # translate 'None' to an empty string + if rv is None: rv = '' + return rv + +class Interpolate(util.ComparableMixin): + """ + This is a marker class, used fairly widely to indicate that we + want to interpolate build properties. + """ + + implements(IRenderable) + compare_attrs = ('fmtstring', 'args', 'kwargs') + + def __init__(self, fmtstring, *args, **kwargs): + self.fmtstring = fmtstring + self.args = args + self.kwargs = kwargs + if self.args and self.kwargs: + raise ValueError('Interpolate takes either positional or keyword substitutions, not both.') + + def getRenderingFor(self, props): + props = props.getProperties() + if self.args: + args = props.render(self.args) + return self.fmtstring % tuple(args) + else: + kwargs = props.render(self.kwargs) + return self.fmtstring % InterpolateMap(props, kwargs) + return s class Property(util.ComparableMixin): """ @@ -363,7 +502,7 @@ def getRenderingFor(self, props): if name in sourceDict: strings.append(sourceDict[name]) else: - raise ValueError('Sourcestamp does not know "%s"' % name) + raise ValueError('Sourcestamp does not known "%s"' % name) s = self.fmtstring % tuple(strings) else: for k,v in self.lambda_subs.iteritems(): diff --git a/master/buildbot/sourcestamp.py b/master/buildbot/sourcestamp.py index d3db15fe166..26e99b29e6e 100644 --- a/master/buildbot/sourcestamp.py +++ b/master/buildbot/sourcestamp.py @@ -169,7 +169,7 @@ def canBeMergedWith(self, other): # this algorithm implements the "compatible" mergeRequests defined in # detail in cfg-buidlers.texinfo; change that documentation if the # algorithm changes! - if self.codebase != self.codebase: + if other.codebase != self.codebase: return False if other.repository != self.repository: return False diff --git a/master/buildbot/steps/source/oldsource.py b/master/buildbot/steps/source/oldsource.py index c4567ef4c20..548ceaa1b84 100644 --- a/master/buildbot/steps/source/oldsource.py +++ b/master/buildbot/steps/source/oldsource.py @@ -21,7 +21,7 @@ from zope.interface import implements from buildbot.process.buildstep import LoggingBuildStep, RemoteCommand from buildbot.interfaces import BuildSlaveTooOldError, IRenderable -from buildbot.status.builder import SKIPPED +from buildbot.status.builder import SKIPPED, FAILURE class _ComputeRepositoryURL(object): implements(IRenderable) @@ -194,6 +194,8 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, self.workdir = workdir self.sourcestamp = None + # Codebase cannot be set yet + self.codebase = '' self.alwaysUseLatest = alwaysUseLatest @@ -243,10 +245,6 @@ def computeSourceRevision(self, changes): self.checkoutDelay value.""" return None - def getCodebase(self): - """ Identify the unique repository for this step """ - return None - def start(self): if self.notReally: log.msg("faking %s checkout/update" % self.name) @@ -260,17 +258,11 @@ def start(self): self.args['workdir'] = self.workdir if not self.alwaysUseLatest: - # what source stamp would this build like to use? - codebase = self.getCodebase() - try: - s = self.build.getSourceStamp(codebase) - except KeyError: - log.msg("The source step cannot get the sourcestamp for codebase '%s'" % codebase) - self.step_status.setText(["Cannot get sourcestamp for codebase '%s'" % codebase]) - self.addCompleteLog("log","Step failed in getting the correct sourcestamp for codebase '%s' from build" % codebase) - return FAILURE + # what source stamp would this step like to use? + s = self.build.getSourceStamp(self.codebase) self.sourcestamp = s - if s is not None: + + if self.sourcestamp: # if branch is None, then use the Step's "default" branch branch = s.branch or self.branch # if revision is None, use the latest sources (-rHEAD) @@ -290,8 +282,12 @@ def start(self): if patch: self.addCompleteLog("patch", patch[1]) else: - self.step_status.setText(["No sourcestamp for repository '%s'" % id]) - self.addCompleteLog("log","No sourcestamp for repository '%s'" % id) + log.msg("No sourcestamp found in build for codebase '%s'" % self.codebase) + self.step_status.setText("Codebase '%s' not in build" % self.codebase) + self.addCompleteLog("log", + "No sourcestamp found in build for codebase '%s'" \ + % self.codebase) + self.finished(FAILURE) return FAILURE else: revision = None diff --git a/master/buildbot/test/unit/test_db_migrate_versions_021_add_codebase.py b/master/buildbot/test/unit/test_db_migrate_versions_021_add_codebase.py index c0dd73ad1b4..daffa8cab51 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_021_add_codebase.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_021_add_codebase.py @@ -19,7 +19,7 @@ from twisted.python import log from twisted.trial import unittest from buildbot.test.util import migration -from buildbot.util import UTC +from buildbot.util import UTC, datetime2epoch class Migration(migration.MigrateTestMixin, unittest.TestCase): @@ -81,6 +81,7 @@ def insert_sourcestamps_changes(self, conn, sourcestampid, repository, codebase, project='', codebase=codebase) + dt_when = datetime.datetime(1978, 6, 15, 12, 31, 15, tzinfo=UTC) conn.execute(self.changes.insert(), changeid = changeid, author = 'develop', @@ -89,7 +90,7 @@ def insert_sourcestamps_changes(self, conn, sourcestampid, repository, codebase, branch = 'default', revision = 'FD56A89', revling = None, - when_timestamp = datetime.datetime(1978, 6, 15, 12, 31, 15, tzinfo=UTC), + when_timestamp = datetime2epoch(dt_when), category = None, repository = repository, codebase = codebase, diff --git a/master/buildbot/test/unit/test_process_build.py b/master/buildbot/test/unit/test_process_build.py index 1140c2c05eb..221fbb7a4b8 100644 --- a/master/buildbot/test/unit/test_process_build.py +++ b/master/buildbot/test/unit/test_process_build.py @@ -482,22 +482,6 @@ def test_buildReturnSourceStamp(self): self.assertEqual( [source1.repository, source1.revision], ["repoA", "12345"]) self.assertEqual( [source2.repository, source2.revision], ["repoB", "67890"]) - def test_buildReturnSourceStamp_noArgs(self): - """ - Test that a build returns the first sourcestamp if no arg is passed - """ - source1 = self.build.getSourceStamp() - self.assertEqual( [source1.repository, source1.revision], ["repoA", "12345"]) - - def test_buildReturnSourceStamp_codebase(self): - """ - Test that a build returns the correct sourcestamp that belongs to the codebase - """ - codebase = 'B' - source2 = self.build.getSourceStamp(codebase) - self.assertTrue(source2 is not None) - self.assertEqual( [source2.repository, source2.revision], ["repoB", "67890"]) - def test_buildReturnSourceStamp_empty_codebase(self): """ Test that a build returns the correct sourcestamp if codebase is empty @@ -507,34 +491,6 @@ def test_buildReturnSourceStamp_empty_codebase(self): self.assertTrue(source3 is not None) self.assertEqual( [source3.repository, source3.revision], ["repoC", "111213"]) -class TestSingleSourceStamps(unittest.TestCase): - - def setUp(self): - r = FakeRequest() - s1 = FakeSource() - s1.repository = "repoA" - s1.changes = [FakeChange(10), FakeChange(11)] - s1.revision = "12345" - - r.sources.extend([s1]) - self.build = Build([r]) - - def test_buildReturnSourceStamp_noArgs(self): - """ - Test that a build returns the one and only sourcestamp - """ - source1 = self.build.getSourceStamp() - self.assertTrue(source1 is not None) - self.assertEqual( [source1.repository, source1.revision], ["repoA", "12345"]) - - def test_buildReturnSourceStamp_no_codebase(self): - """ - Test that a build returns the one and only sourcestamp - """ - codebase = '' - source1 = self.build.getSourceStamp(codebase) - self.assertTrue(source1 is not None) - self.assertEqual( [source1.repository, source1.revision], ["repoA", "12345"]) class TestBuildBlameList(unittest.TestCase): diff --git a/master/buildbot/test/unit/test_process_buildrequest.py b/master/buildbot/test/unit/test_process_buildrequest.py index 6e8c660de68..dc166678c5e 100644 --- a/master/buildbot/test/unit/test_process_buildrequest.py +++ b/master/buildbot/test/unit/test_process_buildrequest.py @@ -18,6 +18,14 @@ from buildbot.process import buildrequest from buildbot.process.build import Build +class FakeSource: + def __init__(self, mergeable = True): + self.codebase = '' + self.mergeable = mergeable + + def canBeMergedWith(self, other): + return self.mergeable + class TestBuildRequest(unittest.TestCase): def test_fromBrdict(self): @@ -327,3 +335,28 @@ def check(_): d.addCallback(check) return d + def test_build_can_be_merged_with_mergables_same_codebases(self): + r1 = buildrequest.BuildRequest() + r1.sources = {"A": FakeSource()} + r2 = buildrequest.BuildRequest() + r2.sources = {"A": FakeSource()} + mergeable = r1.canBeMergedWith(r2) + self.assertTrue(mergeable, "Both request should be able to merge") + + def test_build_can_be_merged_with_non_mergable_same_codebases(self): + r1 = buildrequest.BuildRequest() + r1.sources = {"A": FakeSource(mergeable = False)} + r2 = buildrequest.BuildRequest() + r2.sources = {"A": FakeSource(mergeable = False)} + mergeable = r1.canBeMergedWith(r2) + self.assertFalse(mergeable, "Both request should not be able to merge") + + def test_build_can_be_merged_with_non_mergables_different_codebases(self): + r1 = buildrequest.BuildRequest() + r1.sources = {"A": FakeSource(mergeable = False)} + r2 = buildrequest.BuildRequest() + r2.sources = {"B": FakeSource(mergeable = False)} + mergeable = r1.canBeMergedWith(r2) + self.assertTrue(mergeable, "Request containing different codebases " + + "should always be able to merge") + \ No newline at end of file diff --git a/master/buildbot/test/unit/test_process_properties.py b/master/buildbot/test/unit/test_process_properties.py index 9a7aefca471..9a774c54e72 100644 --- a/master/buildbot/test/unit/test_process_properties.py +++ b/master/buildbot/test/unit/test_process_properties.py @@ -18,21 +18,49 @@ from twisted.trial import unittest from twisted.python import components from buildbot.process.properties import PropertyMap, Properties, WithProperties +from buildbot.process.properties import Interpolate from buildbot.process.properties import Property, PropertiesMixin from buildbot.interfaces import IRenderable, IProperties class FakeProperties(object): def __init__(self, **kwargs): self.dict = kwargs + self.build = None def __getitem__(self, k): return self.dict[k] def has_key(self, k): return self.dict.has_key(k) - + def getBuild(self): + return self.build + +class FakeSource: + def __init__(self): + self.branch = None + self.codebase = '' + self.project = '' + self.repository = '' + self.revision = None + + def asDict(self): + ds = {} + ds['branch'] = self.branch + ds['codebase'] = self.codebase + ds['project'] = self.project + ds['repository'] = self.repository + ds['revision'] = self.revision + return ds + class FakeBuild(PropertiesMixin): def __init__(self, properties): + self.sources = {} + properties.build = self self.properties = properties + def getSourceStamp(self, codebase): + if codebase in self.sources: + return self.sources[codebase] + return None + components.registerAdapter( lambda build : IProperties(build.properties), FakeBuild, IProperties) @@ -250,6 +278,197 @@ def testTempValuePlusUnsetSet(self): self.assertEqual(self.pm['prop_nosuch:+set'], 'set') +class TestInterpolateProperties(unittest.TestCase): + def setUp(self): + self.props = Properties() + self.build = FakeBuild(self.props) + + def test_invalid_params(self): + self.assertRaises(ValueError, lambda : + Interpolate("%s %(foo)s", 1, foo=2)) + + def test_properties(self): + self.props.setProperty("buildername", "winbld", "test") + command = Interpolate("echo buildby-%(prop:buildername)s") + self.failUnlessEqual(self.build.render(command), + "echo buildby-winbld") + + def test_property_not_set(self): + command = Interpolate("echo buildby-%(prop:buildername)s") + self.failUnlessEqual(self.build.render(command), + "echo buildby-") + + def test_property_colon_minus(self): + command = Interpolate("echo buildby-%(prop:buildername:-blddef)s") + self.failUnlessEqual(self.build.render(command), + "echo buildby-blddef") + + def test_property_colon_tilde_true(self): + self.props.setProperty("buildername", "winbld", "test") + command = Interpolate("echo buildby-%(prop:buildername:~blddef)s") + self.failUnlessEqual(self.build.render(command), + "echo buildby-winbld") + + def test_property_colon_tilde_false(self): + self.props.setProperty("buildername", "", "test") + command = Interpolate("echo buildby-%(prop:buildername:~blddef)s") + self.failUnlessEqual(self.build.render(command), + "echo buildby-blddef") + + def test_property_colon_plus(self): + self.props.setProperty("project", "proj1", "test") + command = Interpolate("echo %(prop:project:+projectdefined)s") + self.failUnlessEqual(self.build.render(command), + "echo projectdefined") + +class TestInterpolateSrc(unittest.TestCase): + def setUp(self): + self.props = Properties() + self.build = FakeBuild(self.props) + sa = FakeSource() + sb = FakeSource() + sc = FakeSource() + sd = FakeSource() + + sa.repository = 'cvs://A..' + sa.codebase = 'cbA' + sa.project = "Project" + self.build.sources['cbA'] = sa + + sb.repository = 'cvs://B..' + sb.codebase = 'cbB' + sb.project = "Project" + self.build.sources['cbB'] = sb + + sd.repository = 'cvs://D..' + sd.codebase = '[cbD]:' + sd.project = None + self.build.sources['[cbD]:'] = sd + + def test_src(self): + command = Interpolate("echo %(src[cbB]:repository)s") + self.failUnlessEqual(self.build.render(command), + "echo cvs://B..") + + def test_src_src(self): + command = Interpolate("echo %(src[cbB]:repository)s %(src[cbB]:project)s") + self.failUnlessEqual(self.build.render(command), + "echo cvs://B.. Project") + + def test_src_complex_codebase(self): + command = Interpolate("echo %(src[[cbD]:]:repository)s") + self.failUnlessEqual(self.build.render(command), + "echo cvs://D..") + + def test_src_attr_empty(self): + command = Interpolate("echo %(src[cbC]:project)s") + self.failUnlessEqual(self.build.render(command), + "echo ") + + def test_src_colon_minus(self): + command = Interpolate("echo %(src[cbB]:nonattr:-defaultrepo)s") + self.failUnlessEqual(self.build.render(command), + "echo defaultrepo") + + def test_src_colon_minus_false(self): + command = Interpolate("echo '%(src[cbC]:project:-noproject)s'") + self.failUnlessEqual(self.build.render(command), + "echo 'noproject'") + + def test_src_colon_minus_codebase_notfound(self): + command = Interpolate("echo '%(src[unknown_codebase]:project:-noproject)s'") + self.failUnlessEqual(self.build.render(command), + "echo 'noproject'") + + def test_src_colon_tilde_true(self): + command = Interpolate("echo '%(src[cbB]:project:~noproject)s'") + self.failUnlessEqual(self.build.render(command), + "echo 'Project'") + + def test_src_colon_tilde_false(self): + command = Interpolate("echo '%(src[cbC]:project:~noproject)s'") + self.failUnlessEqual(self.build.render(command), + "echo 'noproject'") + + def test_src_colon_tilde_false_src_as_replacement(self): + command = Interpolate("echo '%(src[cbC]:project:~%(src[cbA]:project)s)s'") + self.failUnlessEqual(self.build.render(command), + "echo 'Project'") + + def test_src_colon_tilde_codebase_notfound(self): + command = Interpolate("echo '%(src[unknown_codebase]:project:~noproject)s'") + self.failUnlessEqual(self.build.render(command), + "echo 'noproject'") + + +class TestInterpolateKwargs(unittest.TestCase): + def setUp(self): + self.props = Properties() + self.build = FakeBuild(self.props) + sa = FakeSource() + + sa.repository = 'cvs://A..' + sa.codebase = 'cbA' + sa.project = None + sa.branch = "default" + self.build.sources['cbA'] = sa + + def test_kwarg(self): + command = Interpolate("echo %(kw:repository)s", repository = "cvs://A..") + self.failUnlessEqual(self.build.render(command), + "echo cvs://A..") + + def test_kwarg_kwarg(self): + command = Interpolate("echo %(kw:repository)s %(kw:branch)s", + repository = "cvs://A..", branch = "default") + self.failUnlessEqual(self.build.render(command), + "echo cvs://A.. default") + + def test_kwarg_not_mapped(self): + command = Interpolate("echo %(kw:repository)s", project = "projectA") + self.failUnlessEqual(self.build.render(command), + "echo ") + + def test_kwarg_colon_minus_not_available(self): + command = Interpolate("echo %(kw:repository:-cvs://A..)s", project = "projectA") + self.failUnlessEqual(self.build.render(command), + "echo cvs://A..") + + def test_kwarg_colon_minus_available(self): + command = Interpolate("echo %(kw:repository:-cvs://A..)s", repository = "cvs://B..") + self.failUnlessEqual(self.build.render(command), + "echo cvs://B..") + + def test_kwarg_colon_tilde_true(self): + command = Interpolate("echo %(kw:repository:~cvs://B..)s", repository = "cvs://A..") + self.failUnlessEqual(self.build.render(command), + "echo cvs://A..") + + def test_kwarg_colon_tilde_false(self): + command = Interpolate("echo %(kw:repository:~cvs://B..)s", repository = "") + self.failUnlessEqual(self.build.render(command), + "echo cvs://B..") + + def test_kwarg_colon_tilde_none(self): + command = Interpolate("echo %(kw:repository:~cvs://B..)s", repository = None) + self.failUnlessEqual(self.build.render(command), + "echo cvs://B..") + + def test_kwarg_colon_plus_false(self): + command = Interpolate("echo %(kw:repository:+cvs://B..)s", project = "project") + self.failUnlessEqual(self.build.render(command), + "echo ") + + def test_kwarg_colon_plus_true(self): + command = Interpolate("echo %(kw:repository:+cvs://B..)s", repository = None) + self.failUnlessEqual(self.build.render(command), + "echo cvs://B..") + + def test_kwargs_colon_minus_false_src_as_replacement(self): + command = Interpolate("echo '%(kw:text:-%(src[cbA]:branch)s)s'", notext='ddd') + self.failUnlessEqual(self.build.render(command), + "echo 'default'") + class TestWithProperties(unittest.TestCase): def setUp(self): diff --git a/master/buildbot/test/unit/test_process_withsource.py b/master/buildbot/test/unit/test_process_properties_withsource.py similarity index 98% rename from master/buildbot/test/unit/test_process_withsource.py rename to master/buildbot/test/unit/test_process_properties_withsource.py index 048f75b967b..b150db9bf0b 100644 --- a/master/buildbot/test/unit/test_process_withsource.py +++ b/master/buildbot/test/unit/test_process_properties_withsource.py @@ -56,7 +56,7 @@ def __init__(self, requests): for source in request.sources: self.sources.append(source) - def getSourceStamp(self, codebase=None): + def getSourceStamp(self, codebase): for source in self.sources: if source.codebase == codebase: return source diff --git a/master/docs/developer/config.rst b/master/docs/developer/config.rst index 0f1a6dbeaf3..247f4f00599 100644 --- a/master/docs/developer/config.rst +++ b/master/docs/developer/config.rst @@ -98,7 +98,7 @@ described in :ref:`developer-Reconfiguration`. A callable, or None, used to determine the codebase from an incomming :py:class:`~buildbot.changes.changes.Change`, - from :bb:cfg:`codebaseGenerator` + from :bb:cfg:`codebaseGenerator` .. py:attribute:: slavePortnum diff --git a/master/docs/manual/cfg-global.rst b/master/docs/manual/cfg-global.rst index 9484b5799b5..58ccfd90303 100644 --- a/master/docs/manual/cfg-global.rst +++ b/master/docs/manual/cfg-global.rst @@ -730,14 +730,14 @@ callables, and tries each in turn, returning the first successful match. .. _TwistedConch: http://twistedmatrix.com/trac/wiki/TwistedConch -.. bb:cfg:: codebase +.. bb:cfg:: codebaseGenerator Codebase Generator ~~~~~~~~~~~~~~~~~~ :: - def codebaseGenerator(change): + def codebaseGenerator(chdict): all_repositories = { r'https://hg/hg/mailsuite/mailclient': 'mailexe', r'https://hg/hg/mailsuite/mapilib': 'mapilib', @@ -746,11 +746,24 @@ Codebase Generator r'https://github.com/mailinc/mailsuite/mapilib': 'mapilib', r'https://github.com/mailinc/mailsuite/imaplib': 'imaplib', } - if change['repository'] in all_repositories: - return all_repositories[change['repository']] + if chdict['repository'] in all_repositories: + return all_repositories[chdict['repository']] else: - return change['repository'] + return '' c['codebaseGenerator'] = codebaseGenerator -For any incomming change a :ref:`codebase` is set based on the repository of the change. This codebase value is sufficient if all changes come from one VCS. If changes come from more than one VCS, extra processing will be needed to determine the codebase for the incomming change. This codebase will then be a logical name for the combination of repository and or branch etc. \ No newline at end of file +For any incomming change a :ref:`codebase` is set to ''. This codebase value is sufficient if all changes come from the same repository (or clones). If changes come from different repositories, extra processing will be needed to determine the codebase for the incomming change. This codebase will then be a logical name for the combination of repository and or branch etc. The codebaseGenerator accepts a dictionary containing the following change attributes: + + * who + * files + * comments + * isdir + * revision + * when + * branch + * category + * revlink + * properties + * repository + * project diff --git a/master/docs/manual/concepts.rst b/master/docs/manual/concepts.rst index 965d9ac0b1a..8963ab78991 100644 --- a/master/docs/manual/concepts.rst +++ b/master/docs/manual/concepts.rst @@ -324,8 +324,7 @@ hint for the build steps to figure out which code to check out. Codebase ++++++++ -The codebase is derived from a change. A complete software product may be composed of more than one repository. Each repository has its own unique position inside the product design (i.e. main module, shared library, resources, documentation). To be able to start builds from different CVS's and still distinquish the different repositories `codebase`'s are used. By default the codebase is equal to the repository of a change. The `master.cfg` may contain a callable that determines the codebase from an incomming change and replaces the default value(see. :bb:cfg:`Codebase Generator`). - +The codebase is derived from a change. A complete software product may be composed of more than one repository. Each repository has its own unique position inside the product design (i.e. main module, shared library, resources, documentation). To be able to start builds from different VCS's and still distinquish the different repositories `codebase`'s are used. By default the codebase is ''. The `master.cfg` may contain a callable that determines the codebase from an incomming change and replaces the default value(see. :bb:cfg:`codebaseGenerator`). A codebase is not allowed to contain ':'. .. _Attr-Revision: diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index 2897e4356ca..bbcf62e63e1 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -68,6 +68,10 @@ Deprecations, Removals, and Non-Compatible Changes ``hgbuildbot.baseurl`` to an empty string as suggested in :ref:`the Buildbot manual `. +* The configurable callable build.workdir has changed his parameterlist. Instead + of a single sourcestamp a list of sourcestamps is passed. Each sourcestamp in + the list has a different :ref:`codebase` + Changes for Developers ~~~~~~~~~~~~~~~~~~~~~~