Add MercurialInRepo-branch support + testcase (fixes buildbot#187) an…

…d support spaces in vcexe-path (fixes buildbot#398)

buildbot/slave/commands.py

@@ -2246,15 +2246,14 @@ def sourcedirIsUpdateable(self):
 
     def doVCUpdate(self):
         d = os.path.join(self.builder.basedir, self.srcdir)
-        command = [self.vcexe, 'pull', '--update', '--verbose']
-        if self.args.get('revision'):
-            command.extend(['--rev', self.args['revision']])
+        command = [self.vcexe, 'pull', '--verbose', self.repourl]
         c = ShellCommand(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
                          keepStdout=True)
         self.command = c
         d = c.start()
         d.addCallback(self._handleEmptyUpdate)
+        d.addCallback(self._update)
         return d
 
     def _handleEmptyUpdate(self, res):
@@ -2269,37 +2268,83 @@ def _handleEmptyUpdate(self, res):
 
     def doVCFull(self):
         d = os.path.join(self.builder.basedir, self.srcdir)
-        command = [self.vcexe, 'clone', '-U']
-        command.extend([self.repourl, d])
+        command = [self.vcexe, 'init', d]
         c = ShellCommand(self.builder, command, self.builder.basedir,
                          sendRC=False, timeout=self.timeout)
         self.command = c
         cmd1 = c.start()
 
-        def _update(res):
-            updatecmd=[self.vcexe, 'update', '--repository', d]
-            if self.args.get('revision'):
-                updatecmd.extend(['--rev', self.args['revision']])
-            else:
-                updatecmd.extend(['--rev', self.args.get('branch',  'default')])
-            self.command = ShellCommand(self.builder, updatecmd,
-                self.builder.basedir, sendRC=False, timeout=self.timeout)
-            return self.command.start()
-
-        cmd1.addCallback(_update)
+        def _vcupdate(res):
+            return self.doVCUpdate()
+
+        cmd1.addCallback(_vcupdate)
         return cmd1
 
-    def _updateToDesiredRevision(self, res):
-        assert self.args.get('revision')
-        newdir = os.path.join(self.builder.basedir, self.srcdir)
-        # hg-0.9.1 and earlier (which need this fallback) also want to see
-        # 'hg update REV' instead of 'hg update --rev REV'. Note that this is
-        # the only place we use 'hg update', since what most VC tools mean
-        # by, say, 'cvs update' is expressed as 'hg pull --update' instead.
-        command = [self.vcexe, 'update', self.args['revision']]
-        c = ShellCommand(self.builder, command, newdir,
-                         sendRC=False, timeout=self.timeout)
-        return c.start()
+    def _update(self, res):
+        if res != 0:
+            return res
+
+        # compare current branch to update
+        self.update_branch = self.args.get('branch',  'default')
+
+        d = os.path.join(self.builder.basedir, self.srcdir)
+        parentscmd = [self.vcexe, 'identify', '--num', '--branch']
+        cmd = ShellCommand(self.builder, parentscmd, d,
+                           sendStdout=False, sendStderr=False, keepStdout=True, keepStderr=True)
+
+        def _parse(res):
+            if res != 0:
+                msg = "'hg identify' failed: %s\n%s" % (cmd.stdout, cmd.stderr)
+                self.sendStatus({'header': msg + "\n"})
+                log.msg(msg)
+                return res
+
+            log.msg('Output: %s' % cmd.stdout)
+
+            match = re.search(r'^(.+) (.+)$', cmd.stdout)
+            assert match
+
+            rev = match.group(1)
+            current_branch = match.group(2)
+
+            if rev == '-1':
+                msg = "Fresh hg repo, don't worry about branch"
+                log.msg(msg)
+
+            elif self.update_branch != current_branch:
+                msg = "Working dir is on branch '%s' and build needs '%s'. Clobbering." % (current_branch, self.update_branch)
+                self.sendStatus({'header': msg + "\n"})
+                log.msg(msg)
+
+                def _vcfull(res):
+                    return self.doVCFull()
+
+                d = self.doClobber(None, self.srcdir)                
+                d.addCallback(_vcfull)
+                return d
+
+            else:
+                msg = "Working dir on same branch as build (%s)." % (current_branch)
+                log.msg(msg)
+
+            return 0            
+
+        c = cmd.start()                
+        c.addCallback(_parse)
+        c.addCallback(self._update2)
+        return c
+
+    def _update2(self, res):                        
+        d = os.path.join(self.builder.basedir, self.srcdir)
+
+        updatecmd=[self.vcexe, 'update', '--clean', '--repository', d]
+        if self.args.get('revision'):
+            updatecmd.extend(['--rev', self.args['revision']])
+        else:
+            updatecmd.extend(['--rev', self.args.get('branch',  'default')])
+        self.command = ShellCommand(self.builder, updatecmd,
+            self.builder.basedir, sendRC=False, timeout=self.timeout)
+        return self.command.start()
 
     def parseGotRevision(self):
         # we use 'hg identify' to find out what we wound up with

buildbot/steps/source.py

@@ -846,33 +846,44 @@ class Mercurial(Source):
     name = "hg"
 
     def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
-                 **kwargs):
+                 branchType='dirname', **kwargs):
         """
         @type  repourl: string
         @param repourl: the URL which points at the Mercurial repository.
-                        This is used as the default branch. Using C{repourl}
-                        does not enable builds of alternate branches: use
-                        C{baseURL} to enable this. Use either C{repourl} or
-                        C{baseURL}, not both.
+                        This uses the 'default' branch unless defaultBranch is
+                        specified below and the C{branchType} is set to
+                        'inrepo'.  It is an error to specify a branch without
+                        setting the C{branchType} to 'inrepo'.
 
-        @param baseURL: if branches are enabled, this is the base URL to
-                        which a branch name will be appended. It should
-                        probably end in a slash. Use exactly one of
-                        C{repourl} and C{baseURL}.
+        @param baseURL: if 'dirname' branches are enabled, this is the base URL
+                        to which a branch name will be appended. It should
+                        probably end in a slash.  Use exactly one of C{repourl}
+                        and C{baseURL}.
 
         @param defaultBranch: if branches are enabled, this is the branch
                               to use if the Build does not specify one
-                              explicitly. It will simply be appended to
-                              C{baseURL} and the result handed to the
-                              'hg clone' command.
+                              explicitly.
+                              For 'dirname' branches, It will simply be
+                              appended to C{baseURL} and the result handed to
+                              the 'hg update' command.
+                              For 'inrepo' branches, this specifies the named
+                              revision to which the tree will update after a
+                              clone.
+
+        @param branchType: either 'dirname' or 'inrepo' depending on whether
+                           the branch name should be appended to the C{baseURL}
+                           or the branch is a mercurial named branch and can be
+                           found within the C{repourl}
         """
         self.repourl = repourl
         self.baseURL = baseURL
         self.branch = defaultBranch
+        self.branchType = branchType
         Source.__init__(self, **kwargs)
         self.addFactoryArguments(repourl=repourl,
                                  baseURL=baseURL,
                                  defaultBranch=defaultBranch,
+                                 branchType=branchType,
                                  )
         if (not repourl and not baseURL) or (repourl and baseURL):
             raise ValueError("you must provide exactly one of repourl and"
@@ -885,8 +896,11 @@ def startVC(self, branch, revision, patch):
                                         "about hg")
 
         if self.repourl:
-            assert not branch # we need baseURL= to use branches
+            # we need baseURL= to use dirname branches
+            assert self.branchType == 'inrepo' or not branch
             self.args['repourl'] = self.repourl
+            if branch:
+                self.args['branch'] = branch
         else:
             self.args['repourl'] = self.baseURL + branch
         self.args['revision'] = revision

buildbot/test/test_vc.py

@@ -355,7 +355,7 @@ def do(self, basedir, command, failureIsOk=False, stdin=None, env=None):
     def dovc(self, basedir, command, failureIsOk=False, stdin=None, env=None):
         """Like do(), but the VC binary will be prepended to COMMAND."""
         if isinstance(command, (str, unicode)):
-            command = self.vcexe + " " + command
+            command = [self.vcexe] + command.split(' ')
         else:
             # command is a list
             command = [self.vcexe] + command
@@ -2515,6 +2515,191 @@ def testTry(self):
 
 VCS.registerVC(Mercurial.vc_name, MercurialHelper())
 
+class MercurialInRepoHelper(MercurialHelper):
+    branchname = "the_branch"
+    try_branchname = "the_branch"
+
+
+    def createRepository(self):
+        self.createBasedir()
+        self.hg_base = os.path.join(self.repbase, "Mercurial-Repository")
+        self.repo = os.path.join(self.hg_base, "inrepobranch")
+        tmp = os.path.join(self.hg_base, "hgtmp")
+
+        os.makedirs(self.repo)
+        w = self.dovc(self.repo, "init")
+        yield w; w.getResult()
+
+        self.populate(tmp)
+        w = self.dovc(tmp, "init")
+        yield w; w.getResult()
+        w = self.dovc(tmp, "add")
+        yield w; w.getResult()
+        w = self.dovc(tmp, ['commit', '-m', 'initial_import'])
+        yield w; w.getResult()
+        w = self.dovc(tmp, ['push', self.repo])
+        # note that hg-push does not actually update the working directory
+        yield w; w.getResult()
+        w = self.dovc(tmp, "identify")
+        yield w; out = w.getResult()
+        self.addTrunkRev(self.extract_id(out))
+
+        self.populate_branch(tmp)
+        w = self.dovc(tmp, ['branch', self.branchname])
+        yield w; w.getResult()
+        w = self.dovc(tmp, ['commit', '-m', 'commit_on_branch'])
+        yield w; w.getResult()
+        w = self.dovc(tmp, ['push', '-f', self.repo])
+        yield w; w.getResult()
+        w = self.dovc(tmp, "identify")
+        yield w; out = w.getResult()
+        self.addBranchRev(self.extract_id(out))
+        rmdirRecursive(tmp)
+    createRepository = deferredGenerator(createRepository)
+
+    def vc_revise(self):
+        tmp = os.path.join(self.hg_base, "hgtmp2")
+        w = self.dovc(self.hg_base, ['clone', self.repo, tmp])
+        yield w; w.getResult()
+        w = self.dovc(tmp, ['update', '--clean', '--rev', 'default'])
+        yield w; w.getResult()  
+
+        self.version += 1
+        version_c = VERSION_C % self.version
+        version_c_filename = os.path.join(tmp, "version.c")
+        open(version_c_filename, "w").write(version_c)
+        # hg uses timestamps to distinguish files which have changed, so we
+        # force the mtime forward a little bit
+        future = time.time() + 2*self.version
+        os.utime(version_c_filename, (future, future))
+        w = self.dovc(tmp, ['commit', '-m', 'revised_to_%d' % self.version])
+        yield w; w.getResult()
+        w = self.dovc(tmp, ['push', '--force', self.repo])
+        yield w; w.getResult()
+        w = self.dovc(tmp, "identify")
+        yield w; out = w.getResult()
+        self.addTrunkRev(self.extract_id(out))
+        rmdirRecursive(tmp)
+    vc_revise = deferredGenerator(vc_revise)
+
+    def vc_try_checkout(self, workdir, rev, branch=None):
+        assert os.path.abspath(workdir) == workdir
+        if os.path.exists(workdir):
+            rmdirRecursive(workdir)
+        w = self.dovc(self.hg_base, ['clone', self.repo, workdir])
+        yield w; w.getResult()
+        w = self.dovc(workdir, ['update', '--clean', '--rev', branch if branch else 'default'])
+        yield w; w.getResult()
+
+        try_c_filename = os.path.join(workdir, "subdir", "subdir.c")
+        open(try_c_filename, "w").write(TRY_C)
+        future = time.time() + 2*self.version
+        os.utime(try_c_filename, (future, future))
+    vc_try_checkout = deferredGenerator(vc_try_checkout)
+
+    def vc_try_finish(self, workdir):
+#        rmdirRecursive(workdir)
+        pass
+
+
+class MercurialInRepo(Mercurial):
+    vc_name = 'MercurialInRepo'
+
+    def default_args(self): 
+        return  { 'repourl': self.helper.repo,
+                  'branchType': 'inrepo',
+                  'defaultBranch': 'default' }
+
+    def testCheckout(self):
+        self.helper.vcargs = self.default_args()
+        d = self.do_vctest(testRetry=False)
+
+        # TODO: testRetry has the same problem with Mercurial as it does for
+        # Arch
+        return d
+
+    def testPatch(self):
+        self.helper.vcargs = self.default_args() 
+        d = self.do_patch()
+        return d
+
+    def testCheckoutBranch(self):
+        self.helper.vcargs = self.default_args()
+        d = self.do_branch()
+        return d
+
+    def serveHTTP(self):
+        # the easiest way to publish hg over HTTP is by running 'hg serve' as
+        # a child process while the test is running. (you can also use a CGI
+        # script, which sounds difficult, or you can publish the files
+        # directly, which isn't well documented).
+
+        # grr.. 'hg serve' doesn't let you use --port=0 to mean "pick a free
+        # port", instead it uses it as a signal to use the default (port
+        # 8000). This means there is no way to make it choose a free port, so
+        # we are forced to make it use a statically-defined one, making it
+        # harder to avoid collisions.
+        self.httpPort = 8300 + (os.getpid() % 200)
+        args = [self.helper.vcexe,
+                "serve", "--port", str(self.httpPort), "--verbose"]
+
+        # in addition, hg doesn't flush its stdout, so we can't wait for the
+        # "listening at" message to know when it's safe to start the test.
+        # Instead, poll every second until a getPage works.
+
+        pp = MercurialServerPP() # logs+discards everything
+        # this serves one tree at a time, so we serve trunk. TODO: test hg's
+        # in-repo branches, for which a single tree will hold all branches.
+        self._hg_server = reactor.spawnProcess(pp, self.helper.vcexe, args,
+                                               os.environ,
+                                               self.helper.repo)
+        log.msg("waiting for hg serve to start")
+        done_d = defer.Deferred()
+        def poll():
+            d = client.getPage("http://localhost:%d/" % self.httpPort)
+            def success(res):
+                log.msg("hg serve appears to have started")
+                self._wait_for_server_poller.stop()
+                done_d.callback(None)
+            def ignore_connection_refused(f):
+                f.trap(error.ConnectionRefusedError)
+            d.addCallbacks(success, ignore_connection_refused)
+            d.addErrback(done_d.errback)
+            return d
+        self._wait_for_server_poller = task.LoopingCall(poll)
+        self._wait_for_server_poller.start(0.5, True)
+        return done_d
+
+    def tearDown(self):
+        if self._wait_for_server_poller:
+            if self._wait_for_server_poller.running:
+                self._wait_for_server_poller.stop()
+        if self._hg_server:
+            try:
+                self._hg_server.signalProcess("KILL")
+            except error.ProcessExitedAlready:
+                pass
+            self._hg_server = None
+        return VCBase.tearDown(self)
+
+    def testCheckoutHTTP(self):
+        d = self.serveHTTP()
+        def _started(res):
+            repourl = "http://localhost:%d/" % self.httpPort
+            self.helper.vcargs = self.default_args()
+            self.helper.vcargs['repourl'] = repourl 
+            return self.do_vctest(testRetry=False)
+        d.addCallback(_started)
+        return d
+
+    def testTry(self):
+        self.helper.vcargs = self.default_args()
+        d = self.do_getpatch()
+        return d
+
+VCS.registerVC(MercurialInRepo.vc_name, MercurialInRepoHelper())
+
+
 class GitHelper(BaseHelper):
     branchname = "branch"
     try_branchname = "branch"