diff --git a/master/buildbot/steps/source/repo.py b/master/buildbot/steps/source/repo.py index 29dd41a5be1..805bca4265e 100644 --- a/master/buildbot/steps/source/repo.py +++ b/master/buildbot/steps/source/repo.py @@ -18,29 +18,82 @@ from twisted.internet import defer, reactor +from buildbot import util from buildbot.process import buildstep from buildbot.steps.source.base import Source +from buildbot.interfaces import IRenderable +from zope.interface import implements -def default_sync_all_branches(properties): - # in case of manifest_override, we have no other choice to download all branches - # each project can indeed point on arbitrary commit - return properties.getProperty("manifest_override_url","")!="" -def default_update_tarball(properties,sync_all_branch_done): - # dont create a too big tarball in case we synched all branches - if not sync_all_branch_done: - return 7*24.0*3600.0 - return None +class RepoDownloadsFromProperties(util.ComparableMixin, object): + implements(IRenderable) + parse_download_re = (re.compile(r"repo download ([^ ]+) ([0-9]+/[0-9]+)"), + re.compile(r"([^ ]+) ([0-9]+/[0-9]+)"), + re.compile(r"([^ ]+)/([0-9]+/[0-9]+)"), + ) + + compare_attrs = ('names',) + + def __init__(self, names): + self.names = names + + def getRenderingFor(self, props): + downloads = [] + for propName in self.names: + s = props.getProperty(propName) + if s is not None: + downloads.extend(self.parseDownloadProperty(s)) + return downloads + + def parseDownloadProperty(self, s): + """ + lets try to be nice in the format we want + can support several instances of "repo download proj number/patch" (direct copy paste from gerrit web site) + or several instances of "proj number/patch" (simpler version) + This feature allows integrator to build with several pending interdependant changes. + returns list of repo downloads sent to the buildslave + """ + if s is None: + return [] + ret = [] + for cur_re in self.parse_download_re: + res = cur_re.search(s) + while res: + ret.append("%s %s" % (res.group(1), res.group(2))) + s = s[:res.start(0)] + s[res.end(0):] + res = cur_re.search(s) + return ret + + +class RepoDownloadsFromChangeSource(util.ComparableMixin, object): + implements(IRenderable) + compare_attrs = ('codebase',) + + def __init__(self, codebase=None): + self.codebase = codebase + + def getRenderingFor(self, props): + downloads = [] + if self.codebase is None: + changes = props.getBuild().allChanges() + else: + changes = props.getBuild().getSourceStamp(self.codebase).changes + for change in changes: + if ("event.type" in change.properties and + change.properties["event.type"] == "patchset-created"): + downloads.append("%s %s/%s" % (change.properties["event.change.project"], + change.properties["event.change.number"], + change.properties["event.patchSet.number"])) + return downloads + class Repo(Source): """ Class for Repo with all the smarts """ - name='repo' - renderables = ["manifest_url","manifest_branch","manifest_file", "tarball", "jobs"] + name = 'repo' + renderables = ["manifestURL", "manifestFile", "tarball", "jobs", + "syncAllBranches", "updateTarballAge", "manifestOverrideUrl", + "repoDownloads"] - parse_download_re = (re.compile(r"repo download ([^ ]+) ([0-9]+/[0-9]+)"), - re.compile(r"([^ ]+) ([0-9]+/[0-9]+)"), - re.compile(r"([^ ]+)/([0-9]+/[0-9]+)"), - ) ref_not_found_re = re.compile(r"fatal: Couldn't find remote ref") cherry_pick_error_re = re.compile(r"|".join([r"Automatic cherry-pick failed", r"error: " @@ -48,107 +101,67 @@ class Repo(Source): r"possibly due to conflict resolution."])) re_change = re.compile(r".* refs/changes/\d\d/(\d+)/(\d+) -> FETCH_HEAD$") re_head = re.compile(r"^HEAD is now at ([0-9a-f]+)...") - mirror_sync_retry = 10 # number of retries, if we detect mirror desynchronization - mirror_sync_sleep = 60 # wait 1min between retries (thus default total retry time is 10min) + mirror_sync_retry = 10 # number of retries, if we detect mirror desynchronization + mirror_sync_sleep = 60 # wait 1min between retries (thus default total retry time is 10min) + def __init__(self, - manifest_url=None, - manifest_branch="master", - manifest_file="default.xml", + manifestURL=None, + manifestBranch="master", + manifestFile="default.xml", tarball=None, jobs=None, - sync_all_branches=default_sync_all_branches, - update_tarball=default_update_tarball, + syncAllBranches=False, + updateTarballAge=7*24.0*3600.0, + manifestOverrideUrl=None, + repoDownloads=None, **kwargs): """ - @type manifest_url: string - @param manifest_url: The URL which points at the repo manifests repository. + @type manifestURL: string + @param manifestURL: The URL which points at the repo manifests repository. - @type manifest_branch: string - @param manifest_branch: The manifest branch to check out by default. + @type manifestBranch: string + @param manifestBranch: The manifest branch to check out by default. - @type manifest_file: string - @param manifest_file: The manifest to use for sync. + @type manifestFile: string + @param manifestFile: The manifest to use for sync. - @type sync_all_branches: lambda properties: bool. - @param sync_all_branches: returns the boolean we must synchronize all branches. + @type syncAllBranches: bool. + @param syncAllBranches: true, then we must slowly synchronize all branches. - @type update_tarball: lambda (properties,bool) : float - @param update_tarball: function to determine the update tarball policy, - given properties, and boolean indicating whether - the last repo sync was on all branches + @type updateTarballAge: float + @param updateTarballAge: renderable to determine the update tarball policy, + given properties Returns: max age of tarball in seconds, or None, if we want to skip tarball update + @type manifestOverrideUrl: string + @param manifestOverrideUrl: optional http URL for overriding the manifest + usually coming from Property setup by a ForceScheduler + + @type repoDownloads: list of strings + @param repoDownloads: optional repo download to perform after the repo sync + """ - self.manifest_url = manifest_url - self.manifest_branch = manifest_branch - self.manifest_file = manifest_file + self.manifestURL = manifestURL + self.manifestBranch = manifestBranch + self.manifestFile = manifestFile self.tarball = tarball self.jobs = jobs - def copy_callable(param_name,f): - if not callable(f): - raise ValueError("%s must be callable,but is of type %s"%(param_name,type(f))) - setattr(self, param_name, f) - copy_callable("sync_all_branches",sync_all_branches) - copy_callable("update_tarball",update_tarball) + self.syncAllBranches = syncAllBranches + self.updateTarballAge = updateTarballAge + self.manifestOverrideUrl = manifestOverrideUrl + if repoDownloads is None: + repoDownloads = [] + self.repoDownloads = repoDownloads Source.__init__(self, **kwargs) - assert self.manifest_url is not None + assert self.manifestURL is not None def computeSourceRevision(self, changes): if not changes: return None return changes[-1].revision - def parseDownloadProperty(self, s): - """ - lets try to be nice in the format we want - can support several instances of "repo download proj number/patch" (direct copy paste from gerrit web site) - or several instances of "proj number/patch" (simpler version) - This feature allows integrator to build with several pending interdependant changes. - returns list of repo downloads sent to the buildslave - """ - if s is None: - return [] - ret = [] - for cur_re in self.parse_download_re: - res = cur_re.search(s) - while res: - ret.append("%s %s" % (res.group(1), res.group(2))) - s = s[:res.start(0)] + s[res.end(0):] - res = cur_re.search(s) - return ret - - def buildDownloadList(self): - """taken the changesource and forcebuild property, - build the repo download command to send to the slave - making this a defereable allow config to tweak this - in order to e.g. manage dependancies - """ - downloads = self.build.getProperty("repo_downloads", []) - - # download patches based on GerritChangeSource events - for change in self.build.allChanges(): - if (change.properties.has_key("event.type") and - change.properties["event.type"] == "patchset-created"): - downloads.append("%s %s/%s"% (change.properties["event.change.project"], - change.properties["event.change.number"], - change.properties["event.patchSet.number"])) - - # download patches based on web site forced build properties: - # "repo_d", "repo_d0", .., "repo_d9" - # "repo_download", "repo_download0", .., "repo_download9" - for propName in ["repo_d"] + ["repo_d%d" % i for i in xrange(0,10)] + \ - ["repo_download"] + ["repo_download%d" % i for i in xrange(0,10)]: - s = self.build.getProperty(propName) - if s is not None: - downloads.extend(self.parseDownloadProperty(s)) - - self.repo_downloads = downloads - if downloads: - self.setProperty("repo_downloads", downloads, "repo step") - return defer.succeed(None) - def filterManifestPatches(self): """ Patches to manifest projects are a bit special. @@ -159,25 +172,25 @@ def filterManifestPatches(self): """ manifest_unrelated_downloads = [] manifest_related_downloads = [] - for download in self.repo_downloads: + for download in self.repoDownloads: project, ch_ps = download.split(" ")[-2:] - if ( self.manifest_url.endswith("/"+project) or - self.manifest_url.endswith("/"+project+".git")): + if (self.manifestURL.endswith("/"+project) or + self.manifestURL.endswith("/"+project+".git")): ch, ps = map(int, ch_ps.split("/")) - branch = "refs/changes/%02d/%d/%d"%(ch%100, ch, ps) + branch = "refs/changes/%02d/%d/%d" % (ch % 100, ch, ps) manifest_related_downloads.append( - ["git", "fetch", self.manifest_url, branch]) + ["git", "fetch", self.manifestURL, branch]) manifest_related_downloads.append( ["git", "cherry-pick", "FETCH_HEAD"]) else: manifest_unrelated_downloads.append(download) - self.repo_downloads = manifest_unrelated_downloads - self.manifest_downloads = manifest_related_downloads + self.repoDownloads = manifest_unrelated_downloads + self.manifestDownloads = manifest_related_downloads def _repoCmd(self, command, abandonOnFailure=True, **kwargs): return self._Cmd(["repo"]+command, abandonOnFailure=abandonOnFailure, **kwargs) - def _Cmd(self, command, abandonOnFailure=True,workdir=None, **kwargs): + def _Cmd(self, command, abandonOnFailure=True, workdir=None, **kwargs): if workdir is None: workdir = self.workdir self.cmd = cmd = buildstep.RemoteShellCommand(workdir, command, @@ -188,18 +201,21 @@ def _Cmd(self, command, abandonOnFailure=True,workdir=None, **kwargs): self.logEnviron = False cmd.useLog(self.stdio_log, False) self.stdio_log.addHeader("Starting command: %s\n" % (" ".join(command), )) - self.step_status.setText(["%s"%(" ".join(command[:2]))]) + self.step_status.setText(["%s" % (" ".join(command[:2]))]) d = self.runCommand(cmd) + def evaluateCommand(cmd): if abandonOnFailure and cmd.didFail(): - self.step_status.setText(["repo failed at: %s"%(" ".join(command[:2]))]) + self.step_status.setText(["repo failed at: %s" % (" ".join(command[:2]))]) self.stdio_log.addStderr("Source step failed while running command %s\n" % cmd) raise buildstep.BuildStepFailed() return cmd.rc d.addCallback(lambda _: evaluateCommand(cmd)) return d + def repoDir(self): return self.build.pathmodule.join(self.workdir, ".repo") + def sourcedirIsUpdateable(self): return self.pathExists(self.repoDir()) @@ -209,35 +225,25 @@ def startVC(self, branch, revision, patch): @defer.inlineCallbacks def doStartVC(self): - self.manifest_override_url = self.build.getProperty("manifest_override_url") self.stdio_log = self.addLogForRemoteCommands("stdio") - # run our setup callbacks from the start, we'll use the results later - properties = self.build.getProperties() - self.will_sync_all_branches = self.sync_all_branches(properties) - if self.update_tarball is not None: - time_to_update = self.update_tarball(properties, - self.will_sync_all_branches) - self.tarball_updating_age = time_to_update - - yield self.buildDownloadList() - self.filterManifestPatches() - if self.repo_downloads: - self.stdio_log.addHeader("will download:\n" + "repo download "+ "\nrepo download ".join(self.repo_downloads) + "\n") + if self.repoDownloads: + self.stdio_log.addHeader("will download:\n" + "repo download " + "\nrepo download ".join(self.repoDownloads) + "\n") self.willRetryInCaseOfFailure = True d = self.doRepoSync() + def maybeRetry(why): # in case the tree was corrupted somehow because of previous build # we clobber one time, and retry everything if why.check(buildstep.BuildStepFailed) and self.willRetryInCaseOfFailure: - self.stdio_log.addStderr("got issue at first try:\n" +str(why)+ + self.stdio_log.addStderr("got issue at first try:\n" + str(why) + "\nRetry after clobber...") return self.doRepoSync(forceClobber=True) - return why # propagate to self.failed + return why # propagate to self.failed d.addErrback(maybeRetry) yield d yield self.maybeUpdateTarball() @@ -262,32 +268,32 @@ def doRepoSync(self, forceClobber=False): yield self.doClobberStart() yield self.doCleanup() yield self._repoCmd(['init', - '-u', self.manifest_url, - '-b', self.manifest_branch, - '-m', self.manifest_file]) + '-u', self.manifestURL, + '-b', self.manifestBranch, + '-m', self.manifestFile]) - if self.manifest_override_url: - self.stdio_log.addHeader("overriding manifest with %s\n" %(self.manifest_override_url)) + if self.manifestOverrideUrl: + self.stdio_log.addHeader("overriding manifest with %s\n" % (self.manifestOverrideUrl)) local_file = yield self.pathExists(self.build.pathmodule.join(self.workdir, - self.manifest_override_url)) + self.manifestOverrideUrl)) if local_file: - yield self._Cmd(["cp", "-f", self.manifest_override_url, "manifest_override.xml"]) + yield self._Cmd(["cp", "-f", self.manifestOverrideUrl, "manifest_override.xml"]) else: - yield self._Cmd(["wget", self.manifest_override_url, "-O", "manifest_override.xml"]) + yield self._Cmd(["wget", self.manifestOverrideUrl, "-O", "manifest_override.xml"]) yield self._Cmd(["ln", "-sf", "../manifest_override.xml", "manifest.xml"], - workdir=self.build.pathmodule.join(self.workdir,".repo")) + workdir=self.build.pathmodule.join(self.workdir, ".repo")) - for command in self.manifest_downloads: - yield self._Cmd(command, workdir=self.build.pathmodule.join(self.workdir,".repo","manifests")) + for command in self.manifestDownloads: + yield self._Cmd(command, workdir=self.build.pathmodule.join(self.workdir, ".repo", "manifests")) command = ['sync'] if self.jobs: - command.append('-j' + str(self.jobs)) - if not self.will_sync_all_branches: + command.append('-j' + str(self.jobs)) + if not self.syncAllBranches: command.append('-c') self.step_status.setText(["repo sync"]) self.stdio_log.addHeader("synching manifest %s from branch %s from %s\n" - % (self.manifest_file, self.manifest_branch, self.manifest_url)) + % (self.manifestFile, self.manifestBranch, self.manifestURL)) yield self._repoCmd(command) command = ['manifest', '-r', '-o', 'manifest-original.xml'] @@ -300,45 +306,45 @@ def _findErrorMessages(self, error_re): if not hasattr(self.cmd, logname): continue msg = getattr(self.cmd, logname) - if not (re.search(error_re,msg) is None): + if not (re.search(error_re, msg) is None): return True return False def _sleep(self, delay): d = defer.Deferred() - reactor.callLater(delay,d.callback,1) + reactor.callLater(delay, d.callback, 1) return d @defer.inlineCallbacks def doRepoDownloads(self): self.repo_downloaded = "" - for download in self.repo_downloads: + for download in self.repoDownloads: command = ['download'] + download.split(' ') self.stdio_log.addHeader("downloading changeset %s\n" % (download)) retry = self.mirror_sync_retry + 1 while retry > 0: - yield self._repoCmd(command, abandonOnFailure = False, + yield self._repoCmd(command, abandonOnFailure=False, collectStdout=True, collectStderr=True) if not self._findErrorMessages(self.ref_not_found_re): break - retry -=1 - self.stdio_log.addStderr("failed downloading changeset %s\n"% (download)) + retry -= 1 + self.stdio_log.addStderr("failed downloading changeset %s\n" % (download)) self.stdio_log.addHeader("wait one minute for mirror sync\n") yield self._sleep(self.mirror_sync_sleep) if retry == 0: - self.step_status.setText(["repo: change %s does not exist"%download]) - self.step_status.setText2(["repo: change %s does not exist"%download]) + self.step_status.setText(["repo: change %s does not exist" % download]) + self.step_status.setText2(["repo: change %s does not exist" % download]) raise buildstep.BuildStepFailed() if self.cmd.didFail() or self._findErrorMessages(self.cherry_pick_error_re): # cherry pick error! We create a diff with status current workdir # in stdout, which reveals the merge errors and exit - command = ['forall','-c' ,'git' ,'diff', 'HEAD'] - yield self._repoCmd(command, abandonOnFailure = False) - self.step_status.setText(["download failed: %s"%download]) + command = ['forall', '-c', 'git', 'diff', 'HEAD'] + yield self._repoCmd(command, abandonOnFailure=False) + self.step_status.setText(["download failed: %s" % download]) raise buildstep.BuildStepFailed() if hasattr(self.cmd, 'stderr'): @@ -355,6 +361,7 @@ def doRepoDownloads(self): match2.group(1)) self.setProperty("repo_downloaded", self.repo_downloaded, "Source") + def computeTarballOptions(self): # Keep in mind that the compression part of tarball generation # can be non negligible @@ -372,31 +379,30 @@ def computeTarballOptions(self): @defer.inlineCallbacks def maybeExtractTarball(self): if self.tarball: - tar = self.computeTarballOptions() + [ '-xvf', self.tarball ] + tar = self.computeTarballOptions() + ['-xvf', self.tarball] res = yield self._Cmd(tar, abandonOnFailure=False) - if res: # error with tarball.. erase repo dir and tarball + if res: # error with tarball.. erase repo dir and tarball yield self._Cmd(["rm", "-f", self.tarball], abandonOnFailure=False) yield self.runRmdir(self.repoDir(), abandonOnFailure=False) @defer.inlineCallbacks def maybeUpdateTarball(self): - if not self.tarball or self.tarball_updating_age is None: + if not self.tarball or self.updateTarballAge is None: return # tarball path is absolute, so we cannot use slave's stat command # stat -c%Y gives mtime in second since epoch res = yield self._Cmd(["stat", "-c%Y", self.tarball], collectStdout=True, abandonOnFailure=False) if not res: tarball_mtime = int(self.cmd.stdout) - yield self._Cmd(["stat", "-c%Y", "." ],collectStdout=True) + yield self._Cmd(["stat", "-c%Y", "."], collectStdout=True) now_mtime = int(self.cmd.stdout) age = now_mtime - tarball_mtime - if res or age > self.tarball_updating_age: - tar = self.computeTarballOptions() + [ '-cvf', self.tarball,".repo"] + if res or age > self.updateTarballAge: + tar = self.computeTarballOptions() + ['-cvf', self.tarball, ".repo"] res = yield self._Cmd(tar, abandonOnFailure=False) - if res: # error with tarball.. erase tarball, but dont fail + if res: # error with tarball.. erase tarball, but dont fail yield self._Cmd(["rm", "-f", self.tarball], abandonOnFailure=False) - # a simple shell script to gather all cleanup tweaks... # doing them one by one just complicate the stuff # and messup the stdio log @@ -411,10 +417,10 @@ def _getCleanupCommand(self): cd .repo/manifests rm -f .git/index.lock git fetch origin - git reset --hard remotes/origin/%(manifest_branch)s - git config branch.default.merge %(manifest_branch)s + git reset --hard remotes/origin/%(manifestBranch)s + git config branch.default.merge %(manifestBranch)s cd .. - ln -sf manifests/%(manifest_file)s manifest.xml + ln -sf manifests/%(manifestFile)s manifest.xml cd .. fi repo forall -c rm -f .git/index.lock @@ -422,6 +428,7 @@ def _getCleanupCommand(self): repo forall -c git reset --hard HEAD 2>/dev/null rm -f %(workdir)s/.repo/project.list """) % self.__dict__ + def doCleanup(self): command = self._getCleanupCommand() - return self._Cmd(["bash", "-c", command],abandonOnFailure=False) + return self._Cmd(["bash", "-c", command], abandonOnFailure=False) diff --git a/master/buildbot/test/unit/test_changes_gerritchangesource.py b/master/buildbot/test/unit/test_changes_gerritchangesource.py index a0c5e509f3a..bcd8fe32512 100644 --- a/master/buildbot/test/unit/test_changes_gerritchangesource.py +++ b/master/buildbot/test/unit/test_changes_gerritchangesource.py @@ -20,7 +20,6 @@ class TestGerritChangeSource(changesource.ChangeSourceMixin, unittest.TestCase): - def setUp(self): return self.setUpChangeSource() @@ -40,6 +39,28 @@ def test_describe(self): # TODO: test the backoff algorithm + # this variable is reused in test_steps_source_repo + # to ensure correct integration between change source and repo step + expected_change = {'category': u'patchset-created', + 'files': ['unknown'], + 'repository': u'ssh://someuser@somehost:29418/pr', + 'author': u'Dustin ', + 'comments': u'fix 1234', + 'project': u'pr', + 'branch': u'br/4321', + 'revlink': u'http://buildbot.net', + 'properties': {u'event.change.owner.email': u'dustin@mozilla.com', + u'event.change.subject': u'fix 1234', + u'event.change.project': u'pr', + u'event.change.owner.name': u'Dustin', + u'event.change.number': u'4321', + u'event.change.url': u'http://buildbot.net', + u'event.change.branch': u'br', + u'event.type': u'patchset-created', + u'event.patchSet.revision': u'abcdef', + u'event.patchSet.number': u'12'}, + u'revision': u'abcdef'} + def test_lineReceived_patchset_created(self): s = self.newChangeSource('somehost', 'someuser') d = s.lineReceived(json.dumps(dict( @@ -52,20 +73,13 @@ def test_lineReceived_patchset_created(self): url="http://buildbot.net", subject="fix 1234" ), - patchSet=dict(revision="abcdef") + patchSet=dict(revision="abcdef", number="12") ))) def check(_): self.failUnlessEqual(len(self.changes_added), 1) c = self.changes_added[0] - self.assertEqual(c['author'], "Dustin ") - self.assertEqual(c['project'], "pr") - self.assertEqual(c['branch'], "br/4321") - self.assertEqual(c['revision'], "abcdef") - self.assertEqual(c['revlink'], "http://buildbot.net") - self.assertEqual(c['repository'], "ssh://someuser@somehost:29418/pr") - self.assertEqual(c['comments'], "fix 1234") - self.assertEqual(c['files'], [ 'unknown' ]) - self.assertEqual(c['properties']['event.change.subject'], 'fix 1234') + for k, v in c.items(): + self.assertEqual(self.expected_change[k], v) d.addCallback(check) return d diff --git a/master/buildbot/test/unit/test_steps_source_repo.py b/master/buildbot/test/unit/test_steps_source_repo.py index c0e62655fb1..9caf32c4b31 100644 --- a/master/buildbot/test/unit/test_steps_source_repo.py +++ b/master/buildbot/test/unit/test_steps_source_repo.py @@ -18,51 +18,79 @@ from buildbot.status.results import SUCCESS, FAILURE from buildbot.test.util import sourcesteps from buildbot.test.fake.remotecommand import ExpectShell, Expect +from buildbot.process.properties import Properties +from .test_changes_gerritchangesource import TestGerritChangeSource +from buildbot.changes.changes import Change import os.path + class RepoURL(unittest.TestCase): # testcases taken from old_source/Repo test + + def oneTest(self, props, expected): + p = Properties() + p.update(props, "test") + r = repo.RepoDownloadsFromProperties(props.keys()) + self.assertEqual(r.getRenderingFor(p), expected) + def test_parse1(self): - r = repo.Repo(manifest_url="a") - self.assertEqual(r.parseDownloadProperty("repo download test/bla 564/12"),["test/bla 564/12"]) + self.oneTest( + {'a': "repo download test/bla 564/12"}, ["test/bla 564/12"]) + def test_parse2(self): - r = repo.Repo(manifest_url="a") - self.assertEqual(r.parseDownloadProperty("repo download test/bla 564/12 repo download test/bla 564/2"),["test/bla 564/12","test/bla 564/2"]) + self.oneTest( + {'a': + "repo download test/bla 564/12 repo download test/bla 564/2"}, + ["test/bla 564/12", "test/bla 564/2"]) + self.oneTest({'a': "repo download test/bla 564/12", 'b': "repo download test/bla 564/2"}, [ + "test/bla 564/12", "test/bla 564/2"]) + def test_parse3(self): - r = repo.Repo(manifest_url="a") - self.assertEqual(r.parseDownloadProperty("repo download test/bla 564/12 repo download test/bla 564/2 test/foo 5/1"),["test/bla 564/12","test/bla 564/2","test/foo 5/1"]) - self.assertEqual(r.parseDownloadProperty("repo download test/bla 564/12"),["test/bla 564/12"]) + self.oneTest({'a': "repo download test/bla 564/12 repo download test/bla 564/2 test/foo 5/1"}, [ + "test/bla 564/12", "test/bla 564/2", "test/foo 5/1"]) + self.oneTest( + {'a': "repo download test/bla 564/12"}, ["test/bla 564/12"]) + class TestRepo(sourcesteps.SourceStepMixin, unittest.TestCase): def setUp(self): self.shouldRetry = False - self.logEnviron=True + self.logEnviron = True return self.setUpSourceStep() def tearDown(self): return self.tearDownSourceStep() + def shouldLogEnviron(self): r = self.logEnviron - self.logEnviron=False + self.logEnviron = False return r + def ExpectShell(self, **kw): - if not kw.has_key('workdir'): - kw['workdir']='wkdir' - if not kw.has_key('logEnviron'): - kw['logEnviron']=self.shouldLogEnviron() + if 'workdir' not in kw: + kw['workdir'] = 'wkdir' + if 'logEnviron' not in kw: + kw['logEnviron'] = self.shouldLogEnviron() return ExpectShell(**kw) - def mySetupStep(self,**kwargs): + + def mySetupStep(self, **kwargs): + if "repoDownloads" not in kwargs: + kwargs.update( + dict(repoDownloads=repo.RepoDownloadsFromProperties(["repo_download", "repo_download2"]))) self.setupStep( - repo.Repo(manifest_url='git://myrepo.com/manifest.git', - manifest_branch = "mb", - manifest_file = "mf", **kwargs)) + repo.Repo(manifestURL='git://myrepo.com/manifest.git', + manifestBranch="mb", + manifestFile="mf", + **kwargs)) self.build.allChanges = lambda x=None: [] + def myRunStep(self, result=SUCCESS, status_text=["update"]): self.expectOutcome(result=result, status_text=status_text) d = self.runStep() + def printlogs(res): - text= self.step.stdio_log.getTextWithHeaders() + text = self.step.stdio_log.getTextWithHeaders() if "Failure instance" in text and not self.shouldRetry: print text return res @@ -76,37 +104,42 @@ def expectClobber(self): logEnviron=self.logEnviron)) + 1, Expect('rmdir', dict(dir='wkdir', - logEnviron=self.logEnviron)) + logEnviron=self.logEnviron)) + 0, Expect('mkdir', dict(dir='wkdir', logEnviron=self.logEnviron)) + 0, ) + def expectnoClobber(self): # stat return 0, so nothing self.expectCommands( Expect('stat', dict(file=os.path.join('wkdir', '.repo'), logEnviron=self.logEnviron)) + 0, - ) - def expectRepoSync(self, which_fail=-1, breakatfail=False,syncoptions=["-c"], override_commands=[]): + ) + + def expectRepoSync(self, which_fail=-1, breakatfail=False, syncoptions=["-c"], override_commands=[]): commands = [ - self.ExpectShell(command=['bash', '-c', self.step._getCleanupCommand()]) - , - self.ExpectShell(command=['repo', 'init', '-u','git://myrepo.com/manifest.git', + self.ExpectShell( + command=[ + 'bash', '-c', self.step._getCleanupCommand()]), + self.ExpectShell( + command=['repo', 'init', '-u', 'git://myrepo.com/manifest.git', '-b', 'mb', '-m', 'mf']) - ]+ override_commands +[ - self.ExpectShell(command=['repo', 'sync']+syncoptions) - , - self.ExpectShell(command=['repo', 'manifest', '-r', '-o', 'manifest-original.xml']) - ] + ] + override_commands + [ + self.ExpectShell(command=['repo', 'sync'] + syncoptions), + self.ExpectShell( + command=['repo', 'manifest', '-r', '-o', 'manifest-original.xml']) + ] for i in xrange(len(commands)): - self.expectCommands(commands[i]+(which_fail==i and 1 or 0)) - if which_fail==i and breakatfail: + self.expectCommands(commands[i] + (which_fail == i and 1 or 0)) + if which_fail == i and breakatfail: break + def test_basic(self): """basic first time repo sync""" - self.mySetupStep() + self.mySetupStep(repoDownloads=None) self.expectClobber() self.expectRepoSync() return self.myRunStep(status_text=["update"]) @@ -122,12 +155,12 @@ def test_jobs(self): """basic first time repo sync with jobs""" self.mySetupStep(jobs=2) self.expectClobber() - self.expectRepoSync(syncoptions=["-j2","-c"]) + self.expectRepoSync(syncoptions=["-j2", "-c"]) return self.myRunStep(status_text=["update"]) def test_sync_all_branches(self): """basic first time repo sync with all branches""" - self.mySetupStep(sync_all_branches=lambda _:True) + self.mySetupStep(syncAllBranches=True) self.expectClobber() self.expectRepoSync(syncoptions=[]) return self.myRunStep(status_text=["update"]) @@ -136,20 +169,21 @@ def test_manifest_override(self): """repo sync with manifest_override_url property set download via wget """ - self.mySetupStep() - self.build.setProperty("manifest_override_url", - "http://u.rl/test.manifest", "test") + self.mySetupStep(manifestOverrideUrl= + "http://u.rl/test.manifest", + syncAllBranches=True) self.expectClobber() override_commands = [ - Expect('stat', dict(file=os.path.join('wkdir', 'http://u.rl/test.manifest'), - logEnviron=False)), - self.ExpectShell(logEnviron=False,command=['wget', + Expect( + 'stat', dict(file=os.path.join('wkdir', 'http://u.rl/test.manifest'), + logEnviron=False)), + self.ExpectShell(logEnviron=False, command=['wget', 'http://u.rl/test.manifest', - '-O', 'manifest_override.xml']) - , - self.ExpectShell(logEnviron=False,workdir=os.path.join('wkdir', '.repo'), - command=['ln', '-sf', '../manifest_override.xml', - 'manifest.xml']) + '-O', 'manifest_override.xml']), + self.ExpectShell( + logEnviron=False, workdir=os.path.join('wkdir', '.repo'), + command=['ln', '-sf', '../manifest_override.xml', + 'manifest.xml']) ] self.expectRepoSync(which_fail=2, syncoptions=[], override_commands=override_commands) @@ -159,21 +193,23 @@ def test_manifest_override_local(self): """repo sync with manifest_override_url property set copied from local FS """ - self.mySetupStep() - self.build.setProperty("manifest_override_url", - "test.manifest", "test") + self.mySetupStep(manifestOverrideUrl= + "test.manifest", + syncAllBranches=True) self.expectClobber() override_commands = [ Expect('stat', dict(file=os.path.join('wkdir', 'test.manifest'), logEnviron=False)), self.ExpectShell(logEnviron=False, - command=['cp', '-f', 'test.manifest', 'manifest_override.xml']), + command=[ + 'cp', '-f', 'test.manifest', 'manifest_override.xml']), self.ExpectShell(logEnviron=False, workdir=os.path.join('wkdir', '.repo'), command=['ln', '-sf', '../manifest_override.xml', 'manifest.xml']) ] - self.expectRepoSync(syncoptions=[], override_commands=override_commands) + self.expectRepoSync( + syncoptions=[], override_commands=override_commands) return self.myRunStep(status_text=["update"]) def test_tarball(self): @@ -181,13 +217,15 @@ def test_tarball(self): """ self.mySetupStep(tarball="/tarball.tar") self.expectClobber() - self.expectCommands(self.ExpectShell(command = ['tar', '-xvf', '/tarball.tar'])+0) + self.expectCommands( + self.ExpectShell(command=['tar', '-xvf', '/tarball.tar']) + 0) self.expectRepoSync() - self.expectCommands(self.ExpectShell(command = ['stat', '-c%Y', '/tarball.tar']) - + Expect.log('stdio',stdout=str(10000)) + self.expectCommands(self.ExpectShell(command=['stat', '-c%Y', '/tarball.tar']) + + Expect.log('stdio', stdout=str(10000)) + 0) - self.expectCommands(self.ExpectShell(command = ['stat', '-c%Y', '.']) - + Expect.log('stdio',stdout=str(10000+7*24*3600)) + self.expectCommands(self.ExpectShell(command=['stat', '-c%Y', '.']) + + Expect.log( + 'stdio', stdout=str(10000 + 7 * 24 * 3600)) + 0) return self.myRunStep(status_text=["update"]) @@ -197,95 +235,113 @@ def test_create_tarball(self): self.mySetupStep(tarball="/tarball.tgz") self.expectClobber() self.expectCommands( - self.ExpectShell(command = ['tar', '-z', '-xvf', '/tarball.tgz'])+1, - self.ExpectShell(command = ['rm', '-f', '/tarball.tgz'])+1, + self.ExpectShell( + command=['tar', '-z', '-xvf', '/tarball.tgz']) + 1, + self.ExpectShell(command=['rm', '-f', '/tarball.tgz']) + 1, Expect('rmdir', dict(dir=os.path.join('wkdir', '.repo'), logEnviron=False)) + 1) self.expectRepoSync() - self.expectCommands(self.ExpectShell(command = ['stat', '-c%Y', '/tarball.tgz']) - + Expect.log('stdio',stderr="file not found!") + self.expectCommands(self.ExpectShell(command=['stat', '-c%Y', '/tarball.tgz']) + + Expect.log('stdio', stderr="file not found!") + 1, - self.ExpectShell(command = ['tar', '-z', - '-cvf', '/tarball.tgz', '.repo']) + self.ExpectShell(command=['tar', '-z', + '-cvf', '/tarball.tgz', '.repo']) + 0) return self.myRunStep(status_text=["update"]) - def do_test_update_tarball(self,suffix,option): + def do_test_update_tarball(self, suffix, option): """repo sync update the tarball cache at the end (tarball older than a week) """ - self.mySetupStep(tarball="/tarball."+suffix) + self.mySetupStep(tarball="/tarball." + suffix) self.expectClobber() - self.expectCommands(self.ExpectShell(command = ['tar']+option+['-xvf', '/tarball.'+suffix])+0) + self.expectCommands( + self.ExpectShell(command=['tar'] + option + ['-xvf', '/tarball.' + suffix]) + 0) self.expectRepoSync() - self.expectCommands(self.ExpectShell(command = ['stat', '-c%Y', '/tarball.'+suffix]) - + Expect.log('stdio',stdout=str(10000)) + self.expectCommands(self.ExpectShell(command=['stat', '-c%Y', '/tarball.' + suffix]) + + Expect.log('stdio', stdout=str(10000)) + 0, - self.ExpectShell(command = ['stat', '-c%Y', '.']) - + Expect.log('stdio',stdout=str(10001+7*24*3600)) + self.ExpectShell(command=['stat', '-c%Y', '.']) + + Expect.log( + 'stdio', stdout=str(10001 + 7 * 24 * 3600)) + 0, - self.ExpectShell(command = ['tar']+option+ - ['-cvf', '/tarball.'+suffix, '.repo']) + self.ExpectShell(command=['tar'] + option + + ['-cvf', '/tarball.' + suffix, '.repo']) + 0) return self.myRunStep(status_text=["update"]) def test_update_tarball(self): self.do_test_update_tarball("tar", []) + def test_update_tarball_gz(self): """tarball compression variants""" self.do_test_update_tarball("tar.gz", ["-z"]) + def test_update_tarball_tgz(self): self.do_test_update_tarball("tgz", ["-z"]) + def test_update_tarball_bzip(self): self.do_test_update_tarball("tar.bz2", ["-j"]) + def test_update_tarball_lzma(self): self.do_test_update_tarball("tar.lzma", ["--lzma"]) + def test_update_tarball_lzop(self): self.do_test_update_tarball("tar.lzop", ["--lzop"]) - def test_update_tarball_fail1(self,suffix="tar",option=[]): + def test_update_tarball_fail1(self, suffix="tar", option=[]): """tarball extract fail -> remove the tarball + remove .repo dir """ - self.mySetupStep(tarball="/tarball."+suffix) + self.mySetupStep(tarball="/tarball." + suffix) self.expectClobber() - self.expectCommands(self.ExpectShell(command = ['tar']+option+['-xvf', '/tarball.'+suffix])+1, - self.ExpectShell(command = ['rm', '-f', '/tarball.tar'])+0, - Expect('rmdir', dict(dir=os.path.join('wkdir', '.repo'), - logEnviron=False)) - + 0) + self.expectCommands( + self.ExpectShell( + command=[ + 'tar'] + option + ['-xvf', '/tarball.' + suffix]) + 1, + self.ExpectShell( + command=['rm', '-f', '/tarball.tar']) + 0, + Expect( + 'rmdir', dict(dir=os.path.join('wkdir', '.repo'), + logEnviron=False)) + + 0) self.expectRepoSync() - self.expectCommands(self.ExpectShell(command = ['stat', '-c%Y', '/tarball.'+suffix]) - + Expect.log('stdio',stdout=str(10000)) + self.expectCommands(self.ExpectShell(command=['stat', '-c%Y', '/tarball.' + suffix]) + + Expect.log('stdio', stdout=str(10000)) + 0, - self.ExpectShell(command = ['stat', '-c%Y', '.']) - + Expect.log('stdio',stdout=str(10001+7*24*3600)) + self.ExpectShell(command=['stat', '-c%Y', '.']) + + Expect.log( + 'stdio', stdout=str(10001 + 7 * 24 * 3600)) + 0, - self.ExpectShell(command = ['tar']+option+ - ['-cvf', '/tarball.'+suffix, '.repo']) + self.ExpectShell(command=['tar'] + option + + ['-cvf', '/tarball.' + suffix, '.repo']) + 0) return self.myRunStep(status_text=["update"]) - def test_update_tarball_fail2(self,suffix="tar",option=[]): + def test_update_tarball_fail2(self, suffix="tar", option=[]): """tarball update fail -> remove the tarball + continue repo download """ - self.mySetupStep(tarball="/tarball."+suffix) + self.mySetupStep(tarball="/tarball." + suffix) self.build.setProperty("repo_download", "repo download test/bla 564/12", "test") self.expectClobber() - self.expectCommands(self.ExpectShell(command = ['tar']+option+['-xvf', '/tarball.'+suffix])+0) + self.expectCommands( + self.ExpectShell(command=['tar'] + option + ['-xvf', '/tarball.' + suffix]) + 0) self.expectRepoSync() - self.expectCommands(self.ExpectShell(command = ['stat', '-c%Y', '/tarball.'+suffix]) - + Expect.log('stdio',stdout=str(10000)) + self.expectCommands(self.ExpectShell(command=['stat', '-c%Y', '/tarball.' + suffix]) + + Expect.log('stdio', stdout=str(10000)) + 0, - self.ExpectShell(command = ['stat', '-c%Y', '.']) - + Expect.log('stdio',stdout=str(10001+7*24*3600)) + self.ExpectShell(command=['stat', '-c%Y', '.']) + + Expect.log( + 'stdio', stdout=str(10001 + 7 * 24 * 3600)) + 0, - self.ExpectShell(command = ['tar']+option+ - ['-cvf', '/tarball.'+suffix, '.repo']) + self.ExpectShell(command=['tar'] + option + + ['-cvf', '/tarball.' + suffix, '.repo']) + 1, - self.ExpectShell(command = ['rm', '-f', '/tarball.tar'])+0, - self.ExpectShell(command = ['repo', 'download', 'test/bla', '564/12']) - +0) + self.ExpectShell( + command=['rm', '-f', '/tarball.tar']) + 0, + self.ExpectShell( + command=['repo', 'download', 'test/bla', '564/12']) + + 0) return self.myRunStep(status_text=["update"]) def test_repo_downloads(self): @@ -296,11 +352,14 @@ def test_repo_downloads(self): self.expectnoClobber() self.expectRepoSync() self.expectCommands( - self.ExpectShell(command = ['repo', 'download', 'test/bla', '564/12']) - +0 - + Expect.log('stdio', stderr="test/bla refs/changes/64/564/12 -> FETCH_HEAD\n") + self.ExpectShell( + command=['repo', 'download', 'test/bla', '564/12']) + + 0 + + Expect.log( + 'stdio', stderr="test/bla refs/changes/64/564/12 -> FETCH_HEAD\n") + Expect.log('stdio', stderr="HEAD is now at 0123456789abcdef...\n")) - self.expectProperty("repo_downloaded", "564/12 0123456789abcdef ", "Source") + self.expectProperty( + "repo_downloaded", "564/12 0123456789abcdef ", "Source") return self.myRunStep(status_text=["update"]) def test_repo_downloads2(self): @@ -313,10 +372,12 @@ def test_repo_downloads2(self): self.expectnoClobber() self.expectRepoSync() self.expectCommands( - self.ExpectShell(command = ['repo', 'download', 'test/bla', '564/12']) - +0, - self.ExpectShell(command = ['repo', 'download', 'test/bla2', '565/12']) - +0) + self.ExpectShell( + command=['repo', 'download', 'test/bla', '564/12']) + + 0, + self.ExpectShell( + command=['repo', 'download', 'test/bla2', '565/12']) + + 0) return self.myRunStep(status_text=["update"]) def test_repo_download_manifest(self): @@ -328,60 +389,80 @@ def test_repo_download_manifest(self): "repo download manifest 565/12", "test") self.expectnoClobber() self.expectCommands( - self.ExpectShell(command=['bash', '-c', self.step._getCleanupCommand()]) - +0, - self.ExpectShell(command=['repo', 'init', '-u','git://myrepo.com/manifest.git', + self.ExpectShell( + command=['bash', '-c', self.step._getCleanupCommand()]) + + 0, + self.ExpectShell( + command=['repo', 'init', '-u', 'git://myrepo.com/manifest.git', '-b', 'mb', '-m', 'mf']) - +0, - self.ExpectShell(workdir=os.path.join('wkdir', '.repo', 'manifests'), - command=[ 'git','fetch','git://myrepo.com/manifest.git', - 'refs/changes/65/565/12']) - +0, - self.ExpectShell(workdir=os.path.join('wkdir', '.repo', 'manifests'), - command=['git','cherry-pick','FETCH_HEAD']) - +0, + + 0, + self.ExpectShell( + workdir=os.path.join('wkdir', '.repo', 'manifests'), + command=[ + 'git', 'fetch', 'git://myrepo.com/manifest.git', + 'refs/changes/65/565/12']) + + 0, + self.ExpectShell( + workdir=os.path.join('wkdir', '.repo', 'manifests'), + command=['git', 'cherry-pick', 'FETCH_HEAD']) + + 0, self.ExpectShell(command=['repo', 'sync', '-c']) - +0, - self.ExpectShell(command=['repo', 'manifest', '-r', '-o', 'manifest-original.xml']) - +0) + + 0, + self.ExpectShell( + command=['repo', 'manifest', '-r', '-o', 'manifest-original.xml']) + + 0) self.expectCommands( - self.ExpectShell(command = ['repo', 'download', 'test/bla', '564/12']) - +0) + self.ExpectShell( + command=['repo', 'download', 'test/bla', '564/12']) + + 0) return self.myRunStep(status_text=["update"]) def test_repo_downloads_mirror_sync(self): """repo downloads, with mirror synchronization issues""" self.mySetupStep() - self.step.mirror_sync_sleep = 0.001 # we dont really want the test to wait... + self.step.mirror_sync_sleep = 0.001 # we dont really want the test to wait... self.build.setProperty("repo_download", "repo download test/bla 564/12", "test") self.expectnoClobber() self.expectRepoSync() self.expectCommands( - self.ExpectShell(command = ['repo', 'download', 'test/bla', '564/12']) - +1 + Expect.log("stdio",stderr="fatal: Couldn't find remote ref \n"), - self.ExpectShell(command = ['repo', 'download', 'test/bla', '564/12']) - +1 + Expect.log("stdio",stderr="fatal: Couldn't find remote ref \n"), - self.ExpectShell(command = ['repo', 'download', 'test/bla', '564/12']) - +0) + self.ExpectShell( + command=['repo', 'download', 'test/bla', '564/12']) + + 1 + + Expect.log( + "stdio", stderr="fatal: Couldn't find remote ref \n"), + self.ExpectShell( + command=['repo', 'download', 'test/bla', '564/12']) + + 1 + + Expect.log( + "stdio", stderr="fatal: Couldn't find remote ref \n"), + self.ExpectShell( + command=['repo', 'download', 'test/bla', '564/12']) + + 0) return self.myRunStep(status_text=["update"]) def test_repo_downloads_change_missing(self): """repo downloads, with no actual mirror synchronization issues (still retries 2 times)""" self.mySetupStep() - self.step.mirror_sync_sleep = 0.001 # we dont really want the test to wait... - self.step.mirror_sync_retry = 1 # on retry once + self.step.mirror_sync_sleep = 0.001 # we dont really want the test to wait... + self.step.mirror_sync_retry = 1 # on retry once self.build.setProperty("repo_download", "repo download test/bla 564/12", "test") self.expectnoClobber() self.expectRepoSync() self.expectCommands( - self.ExpectShell(command = ['repo', 'download', 'test/bla', '564/12']) - +1 + Expect.log("stdio",stderr="fatal: Couldn't find remote ref \n"), - self.ExpectShell(command = ['repo', 'download', 'test/bla', '564/12']) - +1 + Expect.log("stdio",stderr="fatal: Couldn't find remote ref \n"), - ) - return self.myRunStep(result=FAILURE,status_text=["repo: change test/bla 564/12 does not exist"]) + self.ExpectShell( + command=['repo', 'download', 'test/bla', '564/12']) + + 1 + + Expect.log( + "stdio", stderr="fatal: Couldn't find remote ref \n"), + self.ExpectShell( + command=['repo', 'download', 'test/bla', '564/12']) + + 1 + + Expect.log( + "stdio", stderr="fatal: Couldn't find remote ref \n"), + ) + return self.myRunStep(result=FAILURE, status_text=["repo: change test/bla 564/12 does not exist"]) def test_repo_downloads_fail1(self): """repo downloads, cherry-pick returns 1""" @@ -391,12 +472,15 @@ def test_repo_downloads_fail1(self): self.expectnoClobber() self.expectRepoSync() self.expectCommands( - self.ExpectShell(command = ['repo', 'download', 'test/bla', '564/12']) - +1 + Expect.log("stdio",stderr="patch \n"), - self.ExpectShell(command = ['repo', 'forall', '-c', 'git', 'diff', 'HEAD']) - +0 - ) + self.ExpectShell( + command=['repo', 'download', 'test/bla', '564/12']) + + 1 + Expect.log("stdio", stderr="patch \n"), + self.ExpectShell( + command=['repo', 'forall', '-c', 'git', 'diff', 'HEAD']) + + 0 + ) return self.myRunStep(result=FAILURE, status_text=["download failed: test/bla 564/12"]) + def test_repo_downloads_fail2(self): """repo downloads, cherry-pick returns 0 but error in stderr""" self.mySetupStep() @@ -405,19 +489,61 @@ def test_repo_downloads_fail2(self): self.expectnoClobber() self.expectRepoSync() self.expectCommands( - self.ExpectShell(command = ['repo', 'download', 'test/bla', '564/12']) - +0 + Expect.log("stdio",stderr="Automatic cherry-pick failed \n"), - self.ExpectShell(command = ['repo', 'forall', '-c', 'git', 'diff', 'HEAD']) - +0 - ) + self.ExpectShell( + command=['repo', 'download', 'test/bla', '564/12']) + + 0 + + Expect.log("stdio", stderr="Automatic cherry-pick failed \n"), + self.ExpectShell( + command=['repo', 'forall', '-c', 'git', 'diff', 'HEAD']) + + 0 + ) return self.myRunStep(result=FAILURE, status_text=["download failed: test/bla 564/12"]) + def test_repo_downloads_from_change_source(self): + """basic repo download from change source, and check that repo_downloaded is updated""" + self.mySetupStep(repoDownloads=repo.RepoDownloadsFromChangeSource()) + chdict = TestGerritChangeSource.expected_change + change = Change(None, None, None, properties=chdict['properties']) + self.build.allChanges = lambda x=None: [change] + self.expectnoClobber() + self.expectRepoSync() + self.expectCommands( + self.ExpectShell(command=['repo', 'download', 'pr', '4321/12']) + + 0 + + Expect.log( + 'stdio', stderr="test/bla refs/changes/64/564/12 -> FETCH_HEAD\n") + + Expect.log('stdio', stderr="HEAD is now at 0123456789abcdef...\n")) + self.expectProperty( + "repo_downloaded", "564/12 0123456789abcdef ", "Source") + return self.myRunStep(status_text=["update"]) + + def test_repo_downloads_from_change_source_codebase(self): + """basic repo download from change source, and check that repo_downloaded is updated""" + self.mySetupStep(repoDownloads=repo.RepoDownloadsFromChangeSource("mycodebase")) + chdict = TestGerritChangeSource.expected_change + change = Change(None, None, None, properties=chdict['properties']) + # getSourceStamp is faked by SourceStepMixin + ss = self.build.getSourceStamp("") + ss.changes = [change] + self.expectnoClobber() + self.expectRepoSync() + self.expectCommands( + self.ExpectShell(command=['repo', 'download', 'pr', '4321/12']) + + 0 + + Expect.log( + 'stdio', stderr="test/bla refs/changes/64/564/12 -> FETCH_HEAD\n") + + Expect.log('stdio', stderr="HEAD is now at 0123456789abcdef...\n")) + self.expectProperty( + "repo_downloaded", "564/12 0123456789abcdef ", "Source") + return self.myRunStep(status_text=["update"]) + def test_update_fail1(self): """ fail at cleanup: ignored""" self.mySetupStep() self.expectnoClobber() self.expectRepoSync(which_fail=0, breakatfail=False) return self.myRunStep(status_text=["update"]) + def test_update_fail2(self): """fail at repo init: clobber""" self.mySetupStep() @@ -447,6 +573,7 @@ def test_update_fail4(self): self.expectRepoSync() self.shouldRetry = True return self.myRunStep(status_text=["update"]) + def test_update_doublefail(self): """fail at repo manifest: clobber but still fail""" self.mySetupStep() @@ -456,6 +583,7 @@ def test_update_doublefail(self): self.expectRepoSync(which_fail=3, breakatfail=True) self.shouldRetry = True return self.myRunStep(result=FAILURE, status_text=["repo failed at: repo manifest"]) + def test_update_doublefail2(self): """fail at repo sync: clobber but still fail""" self.mySetupStep() @@ -465,6 +593,7 @@ def test_update_doublefail2(self): self.expectRepoSync(which_fail=2, breakatfail=True) self.shouldRetry = True return self.myRunStep(result=FAILURE, status_text=["repo failed at: repo sync"]) + def test_update_doublefail3(self): """fail at repo init: clobber but still fail""" self.mySetupStep() diff --git a/master/docs/manual/cfg-buildsteps.rst b/master/docs/manual/cfg-buildsteps.rst index fa725987707..522d0f3ced6 100644 --- a/master/docs/manual/cfg-buildsteps.rst +++ b/master/docs/manual/cfg-buildsteps.rst @@ -788,15 +788,15 @@ for new and old projects. The Repo step takes the following arguments: -``manifest_url`` +``manifestURL`` (required): the URL at which the Repo's manifests source repository is available. -``manifest_branch`` +``manifestBranch`` (optional, defaults to ``master``): the manifest repository branch on which repo will take its manifest. Corresponds to the ``-b`` argument to the :command:`repo init` command. -``manifest_file`` +``manifestFile`` (optional, defaults to ``default.xml``): the manifest filename. Corresponds to the ``-m`` argument to the :command:`repo init` command. @@ -806,40 +806,77 @@ The Repo step takes the following arguments: fast bootstrap. If not present the tarball will be created automatically after first sync. It is a copy of the ``.repo`` directory which contains all the Git objects. This feature helps - to minimize network usage on very big projects. + to minimize network usage on very big projects with lots of slaves. ``jobs`` (optional, defaults to ``None``): Number of projects to fetch simultaneously while syncing. Passed to repo sync subcommand with "-j". -``sync_all_branches`` - (optional, defaults to if "manifest_override" property exists? -> True else -> False): - callback to control the policy of repo sync -c +``syncAllBranches`` + (optional, defaults to ``False``): renderable boolean to control whether ``repo`` + syncs all branches. i.e. ``repo sync -c`` -``update_tarball`` - (optional, defaults to "one week if we did not sync all branches"): - callback to control the policy of updating of the tarball - given properties, and boolean indicating whether - the last repo sync was on all branches - Returns: max age of tarball in seconds, or -1, if we +``updateTarballAge`` + (optional, defaults to "one week"): + renderable to control the policy of updating of the tarball + given properties + Returns: max age of tarball in seconds, or None, if we want to skip tarball update The default value should be good trade off on size of the tarball, and update frequency compared to cost of tarball creation -This Source step integrates with :bb:chsrc:`GerritChangeSource`, and will -automatically use the :command:`repo download` command of repo to -download the additionnal changes introduced by a pending changeset. +``repoDownloads`` + (optional, defaults to None): + list of ``repo download`` commands to perform at the end of the Repo step + each string in the list will be prefixed ``repo download``, and run as is. + This means you can include parameter in the string. e.g: -.. index:: double: Gerrit integration; Repo Build Step + - ``["-c project 1234/4"]`` will cherry-pick patchset 4 of patch 1234 in project ``project`` + + - ``["-f project 1234/4"]`` will enforce fast-forward on patchset 4 of patch 1234 in project ``project`` + +.. py:class:: buildbot.steps.source.repo.RepoDownloadsFromProperties + +``RepoDownloadsFromProperties`` can be used as a renderable of the ``repoDownload`` parameter +it will look in passed properties for string with following possible format: + + - ``repo download project change_number/patchset_number``. + + - ``project change_number/patchset_number``. + + - ``project/change_number/patchset_number``. -Gerrit integration can be also triggered using forced build with following properties: -``repo_d``, ``repo_d[0-9]``, ``repo_download``, ``repo_download[0-9]`` -with values in format: ``project/change_number/patchset_number``. All of these properties will be translated into a :command:`repo download`. This feature allows integrators to build with several pending interdependent changes, which at the moment cannot be described properly in Gerrit, and can only be described by humans. +.. py:class:: buildbot.steps.source.repo.RepoDownloadsFromChangeSource + +``RepoDownloadsFromChangeSource`` can be used as a renderable of the ``repoDownload`` parameter + +This rendereable integrates with :bb:chsrc:`GerritChangeSource`, and will +automatically use the :command:`repo download` command of repo to +download the additionnal changes introduced by a pending changeset. + +.. note:: you can use the two above Rendereable in conjuction by using the class ``buildbot.process.properties.FlattenList`` + +for example:: + + from buildbot.steps.source.repo import Repo, RepoDownloadsFromChangeSource, + from buildbot.steps.source.repo import RepoDownloadsFromProperties + from buildbot.process.properties import FlattenList + + factory.addStep(Repo(manifestUrl='git://mygerrit.org/manifest.git', + repoDownloads=FlattenList([RepoDownloadsFromChangeSource(), + RepoDownloadsFromProperties("repo_downloads") + ] + ) + )) + +.. index:: double: Gerrit integration; Repo Build Step + + .. _Source-Checkout-Slave-Side: Source Checkout (Slave-Side) diff --git a/master/docs/relnotes/index.rst b/master/docs/relnotes/index.rst index 4ffdd1eb923..5643453e3d2 100644 --- a/master/docs/relnotes/index.rst +++ b/master/docs/relnotes/index.rst @@ -73,6 +73,11 @@ Features * Master-side support for P4 is available, and provides a great deal more flexibility than the old slave-side step. See :bb:pull:`596`. +* Master-side support for Repo is available. + The step parameters changed to camelCase. + ``repo_downloads``, and ``manifest_override_url`` properties are no longer hardcoded, but instead consult as default values via renderables. + Renderable are used in favor of callables for ``syncAllBranches`` and ``updateTarball``. + * Builder configurations can now include a ``description``, which will appear in the web UI to help humans figure out what the builder does. * GNUAutoconf and other pre-defined factories now work correctly (:bb:bug:`2402`)