merged with branch multirepo2a

master/buildbot/changes/changes.py

@@ -171,7 +171,7 @@ def asDict(self):
         result['revlink'] = getattr(self, 'revlink', None)
         result['properties'] = self.properties.asList()
         result['repository'] = getattr(self, 'repository', None)
-        result['codebase'] = getattr(self, 'codebase', None)
+        result['codebase'] = getattr(self, 'codebase', '')
         result['project'] = getattr(self, 'project', None)
         return result
 

master/buildbot/db/migrate/versions/021_add_codebase.py

@@ -25,8 +25,10 @@ def upgrade(migrate_engine):
 
 
     # Add codebase to tables
-    ss_codebase = sa.Column('codebase', sa.String(length=256))
+    ss_codebase = sa.Column('codebase', sa.String(length=256), nullable=False, 
+        server_default=sa.DefaultClause(""))
     ss_codebase.create(sourcestamps_table)
 
-    c_codebase = sa.Column('codebase', sa.String(length=256))
+    c_codebase = sa.Column('codebase', sa.String(length=256), nullable=False,
+        server_default=sa.DefaultClause(""))
     c_codebase.create(changes_table)

master/buildbot/db/model.py

@@ -197,7 +197,8 @@ class Model(base.DBConnectorComponent):
         sa.Column('repository', sa.String(length=512), nullable=False, server_default=''),
 
         # codebase is a logical name to specify what is in the repository
-        sa.Column('codebase', sa.String(256)),
+        sa.Column('codebase', sa.String(256), nullable=False,
+            server_default=sa.DefaultClause("")),
 
         # project names the project this source code represents.  It is used
         # later to filter changes
@@ -259,7 +260,8 @@ class Model(base.DBConnectorComponent):
         sa.Column('repository', sa.String(length=512), nullable=False, server_default=''),
 
         # codebase is a logical name to specify what is in the repository
-        sa.Column('codebase', sa.String(256)),
+        sa.Column('codebase', sa.String(256), nullable=False,
+            server_default=sa.DefaultClause("")),
 
         # the project this source code represents
         sa.Column('project', sa.String(length=512), nullable=False, server_default=''),

master/buildbot/master.py

@@ -449,17 +449,22 @@ def handle_deprec(oldname, old, newname, new, default=None,
             properties[n] = (properties[n], 'Change')
 
         if self.config.codebaseGenerator is not None:
-            newchange = changes.Change(who=author, files=files,
-                        comments=comments, isdir=is_dir,
-                        revision=revision,
-                        when=datetime2epoch(when_timestamp),
-                        branch=branch, category=category,
-                        revlink=revlink, properties=properties,
-                        repository=repository,
-                        project=project)
-            codebase = self.config.codebaseGenerator(newchange.asDict())
+            chdict = {}
+            chdict['who']=author
+            chdict['files']=files
+            chdict['comments']=comments
+            chdict['isdir']=is_dir
+            chdict['revision']=revision
+            chdict['when']=when_timestamp
+            chdict['branch']=branch
+            chdict['category']=category
+            chdict['revlink']=revlink
+            chdict['properties']=properties
+            chdict['repository']=repository
+            chdict['project']=project
+            codebase = self.config.codebaseGenerator(chdict)
         else:
-            codebase = repository
+            codebase = ''
 
         d = defer.succeed(None)
         if src:

master/buildbot/process/build.py

@@ -91,12 +91,7 @@ def setLocks(self, locks):
     def setSlaveEnvironment(self, env):
         self.slaveEnvironment = env
 
-    def getSourceStamp(self, codebase=None):
-        if codebase is None:
-            if self.sources:
-                return self.sources[0]
-            else:
-                return None
+    def getSourceStamp(self, codebase):
         for source in self.sources:
             if source.codebase == codebase:
                 return source
@@ -314,11 +309,7 @@ def setupBuild(self, expectations):
             step.setBuild(self)
             step.setBuildSlave(self.slavebuilder.slave)
             if callable (self.workdir):
-                if len(self.sources) == 1:
-                    sources = self.sources[0]
-                else:
-                    sources = self.sources
-                step.setDefaultWorkdir (self.workdir (sources))
+                step.setDefaultWorkdir (self.workdir (self.sources))
             else:
                 step.setDefaultWorkdir (self.workdir)
             name = step.name
@@ -359,6 +350,9 @@ def setupBuild(self, expectations):
                 self.progress.setExpectationsFrom(expectations)
 
         # we are now ready to set up our BuildStatus.
+        # at the moment that schedulers start to deliver sourcestamps per codebase
+        # buildstatus should support multiple sourcestamps.
+        # until now schedulers deliver exactly one sourcestamp
         self.build_status.setSourceStamp(self.sources[0])
         self.build_status.setReason(self.reason)
         self.build_status.setBlamelist(self.blamelist())

master/buildbot/process/buildrequest.py

@@ -151,8 +151,8 @@ def store_source(source):
 
     def canBeMergedWith(self, other):
         #get all codebases from both requests
-        all_codebases = set(s.codebase for s in self.sources.itervalues())
-        all_codebases |= set(s.codebase for s in other.sources.itervalues())
+        all_codebases = set(self.sources.iterkeys())
+        all_codebases &= set(other.sources.iterkeys())
 
         for c in all_codebases:
             if not self.sources[c].canBeMergedWith(other.sources[c]):
@@ -162,9 +162,9 @@ def canBeMergedWith(self, other):
     def mergeSourceStampsWith(self, others):
         """ Returns one merged sourcestamp for every codebase """
         #get all codebases from all requests
-        all_codebases = set(s.codebase for s in self.sources.itervalues())
+        all_codebases = set(self.sources.iterkeys())
         for other in others:
-            all_codebases |= set(s.codebase for s in other.sources.itervalues())
+            all_codebases |= set(other.sources.iterkeys())
 
         all_merged_sources = {}
         # walk along the codebases

master/buildbot/process/properties.py

@@ -288,6 +288,145 @@ def getRenderingFor(self, build):
             pmap.clear_temporary_values()
         return s
 
+class InterpolateMap(object): 
+    colon_minus_re = re.compile(r"(.*):-(.*)") 
+    colon_tilde_re = re.compile(r"(.*):~(.*)") 
+    colon_plus_re = re.compile(r"(.*):\+(.*)") 
+    def __init__(self, properties, kwargs): 
+        self.properties = properties 
+        self.kwargs = kwargs 
+
+    def __getitem__(self, item): 
+        properties = self.properties 
+        key = ''
+        arg = ''
+        res = re.search('(?P<key>[^:\[]+)(?P<index>:|\[.*\])(?P<arg>.*)', item)
+        if res:
+            key = res.group('key')
+            index = res.group('index')
+            if index != ':':
+                #add src indirection to arg
+                arg = index + res.group('arg')
+            else:
+                arg = res.group('arg')
+
+        def prop_fn(arg): 
+            try: 
+                prop, repl = arg.split(":", 1) 
+            except ValueError: 
+                prop, repl = arg, None 
+            return properties, prop, repl 
+
+        def src_fn(arg): 
+            ## TODO: Handle changes 
+            codebase = attr = repl = None
+
+            res = re.search('\[(?P<codebase>.*)\]:(?P<attr>[^\]\]:]+):(?P<repl>.*)',
+                          arg)
+            if res:
+                codebase = res.group('codebase')
+                attr = res.group('attr')
+                repl = res.group('repl')
+            else:
+                res = re.search('\[(?P<codebase>.*)\]:(?P<attr>.*)', arg)
+                if res:
+                    codebase = res.group('codebase')
+                    attr = res.group('attr')
+                    repl = None
+            ss = self.properties.getBuild().getSourceStamp(codebase)
+            if ss:
+                return ss.asDict(), attr, repl 
+            else:
+                return {}, attr, repl
+
+        def kw_fn(arg): 
+            try: 
+                kw, repl = arg.split(":", 1) 
+            except ValueError: 
+                kw, repl = arg, None 
+            return self.kwargs, kw, repl 
+
+        for (keys, fn) in [ 
+            (("p", "prop"), prop_fn), 
+            (("s", "src"), src_fn), 
+            (("kw", "kw"), kw_fn), 
+            ]: 
+            if key in keys: 
+                d, kw, repl = fn(arg) 
+                break;
+
+        def colon_minus(d, kw, repl): 
+            # %(prop:-repl)s 
+            # if prop exists, use it; otherwise, use repl 
+            if d.has_key(kw): 
+                return d[kw] 
+            else: 
+                return repl % self 
+
+        def colon_tilde(d, kw, repl): 
+            # %(prop:~repl)s 
+            # if prop exists and is true (nonempty), use it; otherwise, use repl 
+            if d.has_key(kw) and d[kw]: 
+                return d[kw] 
+            else: 
+                return repl % self 
+
+        def colon_plus(d, kw, repl): 
+            # %(prop:+repl)s 
+            # if prop exists, use repl; otherwise, an empty string 
+            if d.has_key(kw): 
+                return repl % self 
+            else: 
+                return '' 
+
+        if repl is not None: 
+            good_colon = False 
+            for char, fn in [ 
+                ( "-", colon_minus ), 
+                ( "~", colon_tilde ), 
+                ( "+", colon_plus ), 
+                ]: 
+                if repl[0] == char: 
+                    rv = fn(d, kw, repl[1:]) 
+                    good_colon = True 
+                    break 
+            if not good_colon: 
+                raise ValueError 
+        else: 
+            if d.has_key(kw): 
+                rv = d[kw] 
+            else: 
+                rv = None 
+
+        # translate 'None' to an empty string 
+        if rv is None: rv = '' 
+        return rv 
+
+class Interpolate(util.ComparableMixin): 
+    """ 
+    This is a marker class, used fairly widely to indicate that we 
+    want to interpolate build properties. 
+    """ 
+
+    implements(IRenderable) 
+    compare_attrs = ('fmtstring', 'args', 'kwargs') 
+
+    def __init__(self, fmtstring, *args, **kwargs): 
+        self.fmtstring = fmtstring 
+        self.args = args 
+        self.kwargs = kwargs 
+        if self.args and self.kwargs: 
+            raise ValueError('Interpolate takes either positional or keyword substitutions, not both.') 
+
+    def getRenderingFor(self, props): 
+        props = props.getProperties() 
+        if self.args: 
+            args = props.render(self.args) 
+            return self.fmtstring % tuple(args) 
+        else: 
+            kwargs = props.render(self.kwargs) 
+            return self.fmtstring % InterpolateMap(props, kwargs) 
+        return s 
 
 class Property(util.ComparableMixin):
     """
@@ -363,7 +502,7 @@ def getRenderingFor(self, props):
                 if name in sourceDict:
                     strings.append(sourceDict[name])
                 else:
-                    raise ValueError('Sourcestamp does not know "%s"' % name)
+                    raise ValueError('Sourcestamp does not known "%s"' % name)
             s = self.fmtstring % tuple(strings)
         else:
             for k,v in self.lambda_subs.iteritems():

master/buildbot/sourcestamp.py

@@ -169,7 +169,7 @@ def canBeMergedWith(self, other):
         # this algorithm implements the "compatible" mergeRequests defined in
         # detail in cfg-buidlers.texinfo; change that documentation if the
         # algorithm changes!
-        if self.codebase != self.codebase:
+        if other.codebase != self.codebase:
             return False
         if other.repository != self.repository:
             return False

master/buildbot/steps/source/oldsource.py

@@ -21,7 +21,7 @@
 from zope.interface import implements
 from buildbot.process.buildstep import LoggingBuildStep, RemoteCommand
 from buildbot.interfaces import BuildSlaveTooOldError, IRenderable
-from buildbot.status.builder import SKIPPED
+from buildbot.status.builder import SKIPPED, FAILURE
 
 class _ComputeRepositoryURL(object):
     implements(IRenderable)
@@ -194,6 +194,8 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
         self.workdir = workdir
 
         self.sourcestamp = None
+        # Codebase cannot be set yet
+        self.codebase = ''
 
         self.alwaysUseLatest = alwaysUseLatest
 
@@ -243,10 +245,6 @@ def computeSourceRevision(self, changes):
         self.checkoutDelay value."""
         return None
 
-    def getCodebase(self):
-        """ Identify the unique repository for this step """
-        return None
-
     def start(self):
         if self.notReally:
             log.msg("faking %s checkout/update" % self.name)
@@ -260,17 +258,11 @@ def start(self):
         self.args['workdir'] = self.workdir
 
         if not self.alwaysUseLatest:
-            # what source stamp would this build like to use?
-            codebase = self.getCodebase()
-            try:
-                s = self.build.getSourceStamp(codebase)
-            except KeyError:
-                log.msg("The source step cannot get the sourcestamp for codebase '%s'" % codebase)
-                self.step_status.setText(["Cannot get sourcestamp for codebase '%s'" % codebase])
-                self.addCompleteLog("log","Step failed in getting the correct sourcestamp for codebase '%s' from build" % codebase)
-                return FAILURE
+            # what source stamp would this step like to use?
+            s = self.build.getSourceStamp(self.codebase)
             self.sourcestamp = s
-            if s is not None:
+
+            if self.sourcestamp:
                 # if branch is None, then use the Step's "default" branch
                 branch = s.branch or self.branch
                 # if revision is None, use the latest sources (-rHEAD)
@@ -290,8 +282,12 @@ def start(self):
                 if patch:
                     self.addCompleteLog("patch", patch[1])
             else:
-                self.step_status.setText(["No sourcestamp for repository '%s'" % id])
-                self.addCompleteLog("log","No sourcestamp for repository '%s'" % id)
+                log.msg("No sourcestamp found in build for codebase '%s'" % self.codebase)
+                self.step_status.setText("Codebase '%s' not in build" % self.codebase)
+                self.addCompleteLog("log",
+                                    "No sourcestamp found in build for codebase '%s'" \
+                                    % self.codebase)
+                self.finished(FAILURE)                                    
                 return FAILURE
         else:
             revision = None

master/buildbot/test/unit/test_db_migrate_versions_021_add_codebase.py

@@ -19,7 +19,7 @@
 from twisted.python import log
 from twisted.trial import unittest
 from buildbot.test.util import migration
-from buildbot.util import UTC
+from buildbot.util import UTC, datetime2epoch
 
 class Migration(migration.MigrateTestMixin, unittest.TestCase):
 
@@ -81,6 +81,7 @@ def insert_sourcestamps_changes(self, conn, sourcestampid, repository, codebase,
                 project='',
                 codebase=codebase)
 
+        dt_when = datetime.datetime(1978, 6, 15, 12, 31, 15, tzinfo=UTC)
         conn.execute(self.changes.insert(),
                 changeid = changeid,
                 author = 'develop',
@@ -89,7 +90,7 @@ def insert_sourcestamps_changes(self, conn, sourcestampid, repository, codebase,
                 branch = 'default',
                 revision = 'FD56A89',
                 revling = None,
-                when_timestamp = datetime.datetime(1978, 6, 15, 12, 31, 15, tzinfo=UTC),
+                when_timestamp = datetime2epoch(dt_when),
                 category = None,
                 repository = repository,
                 codebase = codebase,