Skip to content

Commit

Permalink
Clean-up in Repo code.
Browse files Browse the repository at this point in the history 
Following parameters were renamed:
* repourl -> manifest_url,
* manifest -> manifest_file,
* repotarball -> tarball.
  • Loading branch information
@PiotrSikora
PiotrSikora committed Dec 13, 2010
1 parent 8ec6c59 commit 4f79328
Show file tree
Hide file tree
Showing 2 changed files with 85 additions and 92 deletions.
62 changes: 34 additions & 28 deletions master/buildbot/steps/source.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -866,43 +866,46 @@ def startVC(self, branch, revision, patch):
cmd = LoggedRemoteCommand("git", self.args)
self.startCommand(cmd)


class Repo(Source):
"""Check out a source tree from a repo repository 'repourl'."""
"""Check out a source tree from a repo repository described by manifest."""

name = "repo"

def __init__(self, repourl=None,
manifest_branch=None,
manifest=None,
repotarball=None,
def __init__(self,
manifest_url=None,
manifest_branch="master",
manifest_file="default.xml",
tarball=None,
**kwargs):
"""
@type repourl: string
@param repourl: the URL which points at the repo manifests repository
@type manifest_url: string
@param manifest_url: The URL which points at the repo manifests repository.
@type manifest_branch: string
@param manifest_branch: The manifest branch to check out by default.
@type manifest: string
@param manifest: The manifest to use for sync.
@type manifest_file: string
@param manifest_file: The manifest to use for sync.
"""
Source.__init__(self, **kwargs)
self.repourl = repourl
self.addFactoryArguments(repourl=repourl,
self.manifest_url = manifest_url
self.addFactoryArguments(manifest_url=manifest_url,
manifest_branch=manifest_branch,
manifest=manifest,
repotarball=repotarball,
manifest_file=manifest_file,
tarball=tarball,
)
self.args.update({'manifest_branch': manifest_branch,
'manifest': manifest,
'repotarball': repotarball,
'manifest_file': manifest_file,
'tarball': tarball,
})

def computeSourceRevision(self, changes):
if not changes:
return None
return changes[-1].revision

def parseDownloadProperty(self, s):
"""
lets try to be nice in the format we want
Expand All @@ -924,33 +927,36 @@ def parseDownloadProperty(self, s):
s = s[:res.start(0)] + s[res.end(0):]
res = cur_re.search(s)
return ret

def startVC(self, branch, revision, patch):
if branch is not None:
self.args['branch'] = branch
self.args['manifest_url'] = self.computeRepositoryURL(self.manifest_url)

self.args['repourl'] = self.computeRepositoryURL(self.repourl)
self.args['revision'] = revision
self.args['patch'] = patch
# only master has access to properties, so we must implement this here.
downloads = []
# download patches based on gerritchangesource events

# download patches based on GerritChangeSource events
for change in self.build.allChanges():
if (change.properties.has_key("event.type") and
change.properties["event.type"] == "patchset-created"):
downloads.append("%s %s/%s"% (change.properties["event.change.project"],
change.properties["event.change.number"],
change.properties["event.patchSet.number"]))

# download patches based on web site forced build properties:
# user can be lazy and just write: "d"
# or specify several properties: download, download1 ... download9
for propName in [ "d", "download"] + [ "download%d"%(i)for i in xrange(1,10)]:
# "repo_d", "repo_d0", .., "repo_d9"
# "repo_download", "repo_download0", .., "repo_download9"
for propName in ["repo_d"] + ["repo_d%d" % i for i in xrange(0,10)] + \
["repo_download"] + ["repo_download%d" % i for i in xrange(0,10)]:
try:
s = self.build.getProperty(propName)
downloads.extend(self.parseDownloadProperty(s))
except KeyError:
s = ""
downloads.extend(self.parseDownloadProperty(s))
self.args["downloads"] = downloads
self.setProperty("downloads",downloads)
pass

if downloads:
self.args["repo_downloads"] = downloads
self.setProperty("repo_downloads", downloads)

slavever = self.slaveVersion("repo")
if not slavever:
raise BuildSlaveTooOldError("slave is too old, does not know "
Expand Down
115 changes: 51 additions & 64 deletions slave/buildslave/commands/repo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,44 +27,35 @@ class Repo(SourceBaseCommand):
"""Repo specific VC operation. In addition to the arguments
handled by SourceBaseCommand, this command reads the following keys:
['repourl'] (required): the manifest GIT repository string
['repotarball'] (optional): the tarball base to accelerate the fetch
['manifest_branch'] (optional): which manifest repo version (i.e. branch or tag) to
retrieve. Default: "master".
['manifest'] (optional): Which manifest to use. Default: "default.xml".
['downloads'] (optional): optional repo downloads to do. computer by master counterpart
from gerrit changesource and forced build properties.
['manifest_url'] (required): The manifests repo repository.
['manifest_branch'] (optional): Which manifest repo version (i.e. branch or tag)
to retrieve. Default: "master".
['manifest_file'] (optional): Which manifest file to use. Default: "default.xml".
['tarball'] (optional): The tarball base to accelerate the fetch.
['repo_downloads'] (optional): Repo downloads to do. Computer from GerritChangeSource
and forced build properties.
"""

header = "repo operation"

def setup(self, args):
SourceBaseCommand.setup(self, args)
self.repourl = args['repourl']
self.manifest_url = args.get('manifest_url')
self.manifest_branch = args.get('manifest_branch')
if not self.manifest_branch:
self.manifest_branch = "master"
self.manifest = args.get('manifest')
self.repotarball = args.get('repotarball')
self.downloads = args.get('downloads')
if not self.manifest:
self.manifest = "default.xml"
self.sourcedata = "%s -b %s -m %s\n" % (self.repourl, self.manifest_branch, self.manifest)
self.manifest_file = args.get('manifest_file')
self.tarball = args.get('tarball')
self.repo_downloads = args.get('repo_downloads')
self.sourcedata = "%s %s %s" % (self.manifest_url, self.manifest_branch, self.manifest_file)

def _fullSrcdir(self):
return os.path.join(self.builder.basedir, self.srcdir)

def _commitSpec(self):
if self.revision:
return self.revision
return self.manifest_branch

def sourcedirIsUpdateable(self):
print os.path.join(self._fullSrcdir(), ".repo")
print os.path.isdir(os.path.join(self._fullSrcdir(), ".repo"))
return os.path.isdir(os.path.join(self._fullSrcdir(), ".repo"))

def _repoCmd(self, command, cb=None,abandonOnFailure=True, **kwargs):
def _repoCmd(self, command, cb=None, abandonOnFailure=True, **kwargs):
repo = self.getCommand("repo")
c = runprocess.RunProcess(self.builder, [repo] + command, self._fullSrcdir(),
sendRC=False, timeout=self.timeout,
Expand All @@ -76,82 +67,78 @@ def _repoCmd(self, command, cb=None,abandonOnFailure=True, **kwargs):
d.addCallback(self._abandonOnFailure)
d.addCallback(cb)
return d
def _tarCmd(self,cmds,callback):
cmd = ["tar"]+cmds
c = runprocess.RunProcess(self.builder, cmd, self._fullSrcdir(),
sendRC=False, timeout=self.timeout,
maxTime=self.maxTime, usePTY=False)
self.command = c
cmdexec = c.start()
cmdexec.addCallback(callback)
return cmdexec
def _gitCmd(self,subdir,cmds,callback):
cmd = ["git"]+cmds
c = runprocess.RunProcess(self.builder, cmd, os.path.join(self._fullSrcdir(), subdir),
sendRC=False, timeout=self.timeout,
maxTime=self.maxTime, usePTY=False)
self.command = c
cmdexec = c.start()
cmdexec.addCallback(callback)
return cmdexec

# If the repourl matches the sourcedata file, then
# we can say that the sourcedata matches. We can
# ignore branch changes, since Git can work with
# many branches fetched, and we deal with it properly
# in doVCUpdate.

def _tarCmd(self, cmds, callback):
cmd = ["tar"] + cmds
c = runprocess.RunProcess(self.builder, cmd, self._fullSrcdir(),
sendRC=False, timeout=self.timeout,
maxTime=self.maxTime, usePTY=False)
self.command = c
cmdexec = c.start()
cmdexec.addCallback(callback)
return cmdexec

def _gitCmd(self, subdir, cmds, callback):
cmd = ["git"] + cmds
c = runprocess.RunProcess(self.builder, cmd, os.path.join(self._fullSrcdir(), subdir),
sendRC=False, timeout=self.timeout,
maxTime=self.maxTime, usePTY=False)
self.command = c
cmdexec = c.start()
cmdexec.addCallback(callback)
return cmdexec

def sourcedataMatches(self):
try:
olddata = self.readSourcedata()
if not olddata.startswith(self.repourl+' '):
return False
return olddata == self.sourcedata
except IOError:
return False
return True


def doVCFull(self):
os.makedirs(self._fullSrcdir())
if self.repotarball and os.path.exists(self.repotarball):
return self._tarCmd(['-xvzf',self.repotarball], self._doInit)
if self.tarball and os.path.exists(self.tarball):
return self._tarCmd(['-xvzf', self.tarball], self._doInit)
else:
return self._doInit(None)

def _doInit(self,res):
# on fresh init, this file may confuse repo.
if os.path.exists(os.path.join(self._fullSrcdir(), ".repo/project.list")):
os.unlink(os.path.join(self._fullSrcdir(), ".repo/project.list"))
return self._repoCmd(['init', '-u',self.repourl,'-b',self.manifest_branch,'-m',self.manifest], self._didInit)
return self._repoCmd(['init', '-u', self.manifest_url, '-b', self.manifest_branch, '-m', self.manifest_file], self._didInit)

def _didInit(self, res):
return self.doVCUpdate()

def doVCUpdate(self):
command = ['forall', '-c', 'git', 'clean', '-f', '-d','-x']
command = ['forall', '-c', 'git', 'clean', '-f', '-d', '-x']
return self._repoCmd(command, self._doClean2, abandonOnFailure=False)

def _doClean2(self,dummy):
command = ['clean', '-f', '-d','-x']
command = ['clean', '-f', '-d', '-x']
return self._gitCmd(".repo/manifests",command, self._doSync)

def _doSync(self, dummy):
command = ['sync']
self.sendStatus({"header": "synching manifest%s from branch %s from %s\n"
% (self.manifest,self.manifest_branch, self.repourl)})
self.sendStatus({"header": "synching manifest %s from branch %s from %s\n"
% (self.manifest_file, self.manifest_branch, self.manifest_url)})
return self._repoCmd(command, self._didSync)

def _didSync(self, dummy):
if self.repotarball and not os.path.exists(self.repotarball):
return self._tarCmd(['-cvzf',self.repotarball,".repo"], self._doDownload)
if self.tarball and not os.path.exists(self.tarball):
return self._tarCmd(['-cvzf', self.tarball, ".repo"], self._doDownload)
else:
return self._doDownload(None)

def _doDownload(self, dummy):
if self.downloads:
if self.repo_downloads:
# download each changeset while the self.download variable is not empty
download = self.downloads.pop(0)
command = ['download']+download.split(' ')
download = self.repo_downloads.pop(0)
command = ['download'] + download.split(' ')
self.sendStatus({"header": "downloading changeset %s\n"
% (download)})
return self._repoCmd(command, self._doDownload) # call again
% (download)})
return self._repoCmd(command, self._doDownload, keepStderr=True) # call again
return defer.succeed(0)

def maybeNotDoVCFallback(self, res):
Expand Down

0 comments on commit 4f79328

Please sign in to comment.