diff --git a/buildbot/scripts/runner.py b/buildbot/scripts/runner.py index 9542bac69e1..246fe098d47 100644 --- a/buildbot/scripts/runner.py +++ b/buildbot/scripts/runner.py @@ -8,14 +8,91 @@ from buildbot.interfaces import BuildbotNotRunningError -# this is mostly just a front-end for mktap, twistd, and kill(1), but in the -# future it will also provide an interface to some developer tools that talk -# directly to a remote buildmaster (like 'try' and a status client) - # the create/start/stop commands should all be run as the same user, # preferably a separate 'buildbot' account. -class MakerBase(usage.Options): +# Note that the terms 'options' and 'config' are used intechangeably here - in +# fact, they are intercanged several times. Caveat legator. + +class OptionsWithOptionsFile(usage.Options): + # subclasses should set this to a list-of-lists in order to source the + # .buildbot/options file. + # buildbotOptions = [ [ 'optfile-name', 'option-name' ], .. ] + buildbotOptions = None + + def __init__(self, *args): + # for options in self.buildbotOptions, optParameters, and the options + # file, change the default in optParameters *before* calling through + # to the parent constructor + + if self.buildbotOptions: + optfile = loadOptionsFile() + for optfile_name, option_name in self.buildbotOptions: + for i in range(len(self.optParameters)): + if self.optParameters[i][0] == option_name and optfile_name in optfile: + self.optParameters[i][2] = optfile[optfile_name] + usage.Options.__init__(self, *args) + +def loadOptionsFile(filename="options", here=None, home=None): + """Find the .buildbot/FILENAME file. Crawl from the current directory up + towards the root, and also look in ~/.buildbot . The first directory + that's owned by the user and has the file we're looking for wins. Windows + skips the owned-by-user test. + + @rtype: dict + @return: a dictionary of names defined in the options file. If no options + file was found, return an empty dict. + """ + + if here is None: + here = os.getcwd() + here = os.path.abspath(here) + + if home is None: + if runtime.platformType == 'win32': + home = os.path.join(os.environ['APPDATA'], "buildbot") + else: + home = os.path.expanduser("~/.buildbot") + + searchpath = [] + toomany = 20 + while True: + searchpath.append(os.path.join(here, ".buildbot")) + next = os.path.dirname(here) + if next == here: + break # we've hit the root + here = next + toomany -= 1 # just in case + if toomany == 0: + raise ValueError("Hey, I seem to have wandered up into the " + "infinite glories of the heavens. Oops.") + searchpath.append(home) + + localDict = {} + + for d in searchpath: + if os.path.isdir(d): + if runtime.platformType != 'win32': + if os.stat(d)[stat.ST_UID] != os.getuid(): + print "skipping %s because you don't own it" % d + continue # security, skip other people's directories + optfile = os.path.join(d, filename) + if os.path.exists(optfile): + try: + f = open(optfile, "r") + options = f.read() + exec options in localDict + except: + print "error while reading %s" % optfile + raise + break + + for k in localDict.keys(): + if k.startswith("__"): + del localDict[k] + return localDict + +class MakerBase(OptionsWithOptionsFile): optFlags = [ ['help', 'h', "Display this message"], ["quiet", "q", "Do not emit the commands being run"], @@ -553,65 +630,6 @@ def restart(config): start(config) -def loadOptions(filename="options", here=None, home=None): - """Find the .buildbot/FILENAME file. Crawl from the current directory up - towards the root, and also look in ~/.buildbot . The first directory - that's owned by the user and has the file we're looking for wins. Windows - skips the owned-by-user test. - - @rtype: dict - @return: a dictionary of names defined in the options file. If no options - file was found, return an empty dict. - """ - - if here is None: - here = os.getcwd() - here = os.path.abspath(here) - - if home is None: - if runtime.platformType == 'win32': - home = os.path.join(os.environ['APPDATA'], "buildbot") - else: - home = os.path.expanduser("~/.buildbot") - - searchpath = [] - toomany = 20 - while True: - searchpath.append(os.path.join(here, ".buildbot")) - next = os.path.dirname(here) - if next == here: - break # we've hit the root - here = next - toomany -= 1 # just in case - if toomany == 0: - raise ValueError("Hey, I seem to have wandered up into the " - "infinite glories of the heavens. Oops.") - searchpath.append(home) - - localDict = {} - - for d in searchpath: - if os.path.isdir(d): - if runtime.platformType != 'win32': - if os.stat(d)[stat.ST_UID] != os.getuid(): - print "skipping %s because you don't own it" % d - continue # security, skip other people's directories - optfile = os.path.join(d, filename) - if os.path.exists(optfile): - try: - f = open(optfile, "r") - options = f.read() - exec options in localDict - except: - print "error while reading %s" % optfile - raise - break - - for k in localDict.keys(): - if k.startswith("__"): - del localDict[k] - return localDict - class StartOptions(MakerBase): optFlags = [ ['quiet', 'q', "Don't display startup log messages"], @@ -639,7 +657,7 @@ class RestartOptions(MakerBase): def getSynopsis(self): return "Usage: buildbot restart []" -class DebugClientOptions(usage.Options): +class DebugClientOptions(OptionsWithOptionsFile): optFlags = [ ['help', 'h', "Display this message"], ] @@ -647,6 +665,11 @@ class DebugClientOptions(usage.Options): ["master", "m", None, "Location of the buildmaster's slaveport (host:port)"], ["passwd", "p", None, "Debug password to use"], + ["myoption", "O", "DEF", "My Option!"], + ] + buildbotOptions = [ + [ 'debugMaster', 'passwd' ], + [ 'master', 'master' ], ] def parseArgs(self, *args): @@ -657,20 +680,19 @@ def parseArgs(self, *args): if len(args) > 2: raise usage.UsageError("I wasn't expecting so many arguments") + def postOptions(self): + print self['myoption'] + sys.exit(1) + def debugclient(config): from buildbot.clients import debug - opts = loadOptions() master = config.get('master') - if not master: - master = opts.get('master') if master is None: raise usage.UsageError("master must be specified: on the command " "line or in ~/.buildbot/options") passwd = config.get('passwd') - if not passwd: - passwd = opts.get('debugPassword') if passwd is None: raise usage.UsageError("passwd must be specified: on the command " "line or in ~/.buildbot/options") @@ -678,7 +700,7 @@ def debugclient(config): d = debug.DebugWidget(master, passwd) d.run() -class StatusClientOptions(usage.Options): +class StatusClientOptions(OptionsWithOptionsFile): optFlags = [ ['help', 'h', "Display this message"], ] @@ -686,6 +708,9 @@ class StatusClientOptions(usage.Options): ["master", "m", None, "Location of the buildmaster's status port (host:port)"], ] + buildbotOptions = [ + [ 'masterstatus', 'master' ], + ] def parseArgs(self, *args): if len(args) > 0: @@ -695,10 +720,7 @@ def parseArgs(self, *args): def statuslog(config): from buildbot.clients import base - opts = loadOptions() master = config.get('master') - if not master: - master = opts.get('masterstatus') if master is None: raise usage.UsageError("master must be specified: on the command " "line or in ~/.buildbot/options") @@ -707,19 +729,16 @@ def statuslog(config): def statusgui(config): from buildbot.clients import gtkPanes - opts = loadOptions() master = config.get('master') - if not master: - master = opts.get('masterstatus') if master is None: raise usage.UsageError("master must be specified: on the command " "line or in ~/.buildbot/options") c = gtkPanes.GtkClient(master) c.run() -class SendChangeOptions(usage.Options): +class SendChangeOptions(OptionsWithOptionsFile): def __init__(self): - usage.Options.__init__(self) + OptionsWithOptionsFile.__init__(self) self['properties'] = {} optParameters = [ @@ -738,6 +757,14 @@ def __init__(self): "Read the log messages from this file (- for stdin)"), ("when", "w", None, "timestamp to use as the change time"), ] + + buildbotOptions = [ + [ 'master', 'master' ], + [ 'username', 'username' ], + [ 'branch', 'branch' ], + [ 'category', 'category' ], + ] + def getSynopsis(self): return "Usage: buildbot sendchange [options] filenames.." def parseArgs(self, *args): @@ -752,11 +779,10 @@ def sendchange(config, runReactor=False): connection will be drpoped as soon as the Change has been sent.""" from buildbot.clients.sendchange import Sender - opts = loadOptions() - user = config.get('username', opts.get('username')) - master = config.get('master', opts.get('master')) - branch = config.get('branch', opts.get('branch')) - category = config.get('category', opts.get('category')) + user = config.get('username') + master = config.get('master') + branch = config.get('branch') + category = config.get('category') revision = config.get('revision') properties = config.get('properties', {}) if config.get('when'): @@ -794,7 +820,7 @@ def sendchange(config, runReactor=False): return d -class ForceOptions(usage.Options): +class ForceOptions(OptionsWithOptionsFile): optParameters = [ ["builder", None, None, "which Builder to start"], ["branch", None, None, "which branch to build"], @@ -814,7 +840,7 @@ def parseArgs(self, *args): self['reason'] = " ".join(args) -class TryOptions(usage.Options): +class TryOptions(OptionsWithOptionsFile): optParameters = [ ["connect", "c", None, "how to reach the buildmaster, either 'ssh' or 'pb'"], @@ -860,8 +886,26 @@ class TryOptions(usage.Options): ["dryrun", 'n', "Gather info, but don't actually submit."], ] + # here it is, the definitive, quirky mapping of .buildbot/options names to + # command-line options. Design by committee, anyone? + buildbotOptions = [ + [ 'try_connect', 'connect' ], + #[ 'try_builders', 'builders' ], <-- handled in postOptions + [ 'try_vc', 'vc' ], + [ 'try_branch', 'branch' ], + [ 'try_topdir', 'try-topdir' ], + [ 'try_topfile', 'try-topfile' ], + [ 'try_host', 'tryhost' ], + [ 'try_username', 'username' ], + [ 'try_dir', 'trydir' ], + [ 'try_password', 'passwd' ], + [ 'try_master', 'master' ], + #[ 'try_wait', 'wait' ], <-- handled in postOptions + [ 'masterstatus', 'master' ], + ] + def __init__(self): - super(TryOptions, self).__init__() + OptionsWithOptionsFile.__init__(self) self['builders'] = [] self['properties'] = {} @@ -884,12 +928,19 @@ def opt_patchlevel(self, option): def getSynopsis(self): return "Usage: buildbot try [options]" + def postOptions(self): + opts = loadOptionsFile() + if not self['builders']: + self['builders'] = opts.get('try_builders', []) + if opts.get('try_wait', False): + self['wait'] = True + def doTry(config): from buildbot.scripts import tryclient t = tryclient.Try(config) t.run() -class TryServerOptions(usage.Options): +class TryServerOptions(OptionsWithOptionsFile): optParameters = [ ["jobdir", None, None, "the jobdir (maildir) for submitting jobs"], ] @@ -912,7 +963,7 @@ def doTryServer(config): os.rename(tmpfile, newfile) -class CheckConfigOptions(usage.Options): +class CheckConfigOptions(OptionsWithOptionsFile): optFlags = [ ['quiet', 'q', "Don't display error messages or tracebacks"], ] diff --git a/buildbot/scripts/tryclient.py b/buildbot/scripts/tryclient.py index e8cdac27379..53ec1e3e9d1 100644 --- a/buildbot/scripts/tryclient.py +++ b/buildbot/scripts/tryclient.py @@ -376,15 +376,12 @@ class Try(pb.Referenceable): def __init__(self, config): self.config = config - self.opts = runner.loadOptions() - self.connect = self.getopt('connect', 'try_connect') + self.connect = self.getopt('connect') assert self.connect, "you must specify a connect style: ssh or pb" - self.builderNames = self.getopt('builders', 'try_builders') + self.builderNames = self.getopt('builders') - def getopt(self, config_name, options_name, default=None): + def getopt(self, config_name, default=None): value = self.config.get(config_name) - if value is None or value == []: - value = self.opts.get(options_name) if value is None or value == []: value = default return value @@ -392,14 +389,14 @@ def getopt(self, config_name, options_name, default=None): def createJob(self): # returns a Deferred which fires when the job parameters have been # created - opts = self.opts + # generate a random (unique) string. It would make sense to add a # hostname and process ID here, but a) I suspect that would cause # windows portability problems, and b) really this is good enough self.bsid = "%d-%s" % (time.time(), random.randint(0, 1000000)) # common options - branch = self.getopt("branch", "try_branch") + branch = self.getopt("branch") difffile = self.config.get("diff") if difffile: @@ -412,14 +409,14 @@ def createJob(self): ss = SourceStamp(branch, baserev, patch) d = defer.succeed(ss) else: - vc = self.getopt("vc", "try_vc") + vc = self.getopt("vc") if vc in ("cvs", "svn"): # we need to find the tree-top - topdir = self.getopt("try-topdir", "try_topdir") + topdir = self.getopt("try-topdir") if topdir: treedir = os.path.expanduser(topdir) else: - topfile = self.getopt("try-topfile", "try_topfile") + topfile = self.getopt("try-topfile") treedir = getTopdir(topfile) else: treedir = os.getcwd() @@ -453,12 +450,11 @@ def fakeDeliverJob(self): def deliverJob(self): # returns a Deferred that fires when the job has been delivered - opts = self.opts if self.connect == "ssh": - tryhost = self.getopt("tryhost", "try_host") - tryuser = self.getopt("username", "try_username") - trydir = self.getopt("trydir", "try_dir") + tryhost = self.getopt("tryhost") + tryuser = self.getopt("username") + trydir = self.getopt("trydir") argv = ["ssh", "-l", tryuser, tryhost, "buildbot", "tryserver", "--jobdir", trydir] @@ -469,9 +465,9 @@ def deliverJob(self): d = pp.d return d if self.connect == "pb": - user = self.getopt("username", "try_username") - passwd = self.getopt("passwd", "try_password") - master = self.getopt("master", "try_master") + user = self.getopt("username") + passwd = self.getopt("passwd") + master = self.getopt("master") tryhost, tryport = master.split(":") tryport = int(tryport) f = pb.PBClientFactory() @@ -514,7 +510,7 @@ def getStatus(self): self._getStatus_1() # contact the status port # we're probably using the ssh style - master = self.getopt("master", "masterstatus") + master = self.getopt("master") host, port = master.split(":") port = int(port) self.announce("contacting the status port at %s:%d" % (host, port)) diff --git a/buildbot/test/test_runner.py b/buildbot/test/test_runner.py index 51906689939..6457081ebbd 100644 --- a/buildbot/test/test_runner.py +++ b/buildbot/test/test_runner.py @@ -22,7 +22,7 @@ def make(self, d, key): def check(self, d, key): basedir = os.sep.join(d) - options = runner.loadOptions(self.optionsFile, here=basedir, + options = runner.loadOptionsFile(self.optionsFile, here=basedir, home=self.home) if key is None: self.failIf(options.has_key('key')) diff --git a/docs/buildbot.texinfo b/docs/buildbot.texinfo index ea434fd6883..b103679c09b 100644 --- a/docs/buildbot.texinfo +++ b/docs/buildbot.texinfo @@ -8612,10 +8612,10 @@ the @code{buildbot statusgui} command: buildbot statusgui --master @var{MASTERHOST}:@var{PORT} @end example -This command starts a simple Gtk+-based status client, which contains -a few boxes for each Builder that change color as events occur. It -uses the same @option{--master} argument as the @command{buildbot -statuslog} command (@pxref{statuslog}). +This command starts a simple Gtk+-based status client, which contains a few +boxes for each Builder that change color as events occur. It uses the same +@option{--master} argument and @code{masterstatus} option as the +@command{buildbot statuslog} command (@pxref{statuslog}). @node try, , statusgui, Developer Tools @subsection try @@ -9033,19 +9033,20 @@ VC server. It requires that you have a PBChangeSource buildbot sendchange --master @var{MASTERHOST}:@var{PORT} --username @var{USER} @var{FILENAMES..} @end example -There are other (optional) arguments which can influence the -@code{Change} that gets submitted: +The @code{master} and @code{username} arguments can also be given in the +options file (@pxref{.buildbot config directory}). There are other (optional) +arguments which can influence the @code{Change} that gets submitted: @table @code @item --branch -This provides the (string) branch specifier. If omitted, it defaults -to None, indicating the ``default branch''. All files included in this -Change must be on the same branch. +(or option @code{branch}) This provides the (string) branch specifier. If +omitted, it defaults to None, indicating the ``default branch''. All files +included in this Change must be on the same branch. @item --category -This provides the (string) category specifier. If omitted, it defaults -to None, indicating ``no category''. The category property is used -by Schedulers to filter what changes they listen to. +(or option @code{category}) This provides the (string) category specifier. If +omitted, it defaults to None, indicating ``no category''. The category property +is used by Schedulers to filter what changes they listen to. @item --revision_number This provides a (numeric) revision number for the change, used for VC systems @@ -9096,7 +9097,7 @@ value. @option{--master} can also be provided in @file{.debug/options} by the @code{master} key. @option{--passwd} can be provided by the -@code{debugPassword} key. +@code{debugPassword} key. @xref{.buildbot config directory}. The @code{Connect} button must be pressed before any of the other buttons will be active. This establishes the connection to the @@ -9170,63 +9171,79 @@ master = 'buildbot.example.org:18990' debugPassword = 'eiv7Po' @end example +Note carefully that the names in the @code{options} file usually do not match +the command-line option name. + @table @code @item masterstatus -Location of the @code{client.PBListener} status port, used by -@command{statuslog} and @command{statusgui}. +Equivalent to @code{--master} for @ref{statuslog} and @ref{statusgui}, this +gives the location of the @code{client.PBListener} status port. @item master -Location of the @code{debugPort} (for @command{debugclient}). Also the -location of the @code{pb.PBChangeSource} (for @command{sendchange}). -Usually shares the slaveport, but a future version may make it -possible to have these listen on a separate port number. +Equivalent to @code{--master} for @ref{debugclient} and @ref{sendchange}. +This option is used for two purposes. It is the location of the +@code{debugPort} for @command{debugclient} and the location of the +@code{pb.PBChangeSource} for @command{sendchange}. Generally these are the +same port. @item debugPassword -Must match the value of @code{c['debugPassword']}, used to protect the +Equivalent to @code{--passwd} for @ref{debugclient}. + +XXX Must match the value of @code{c['debugPassword']}, used to protect the debug port, for the @command{debugclient} command. @item username -Provides a default username for the @command{sendchange} command. - -@end table +Equivalent to @code{--username} for the @ref{sendchange} command. +@item branch +Equivalent to @code{--branch} for the @ref{sendchange} command. -The following options are used by the @code{buildbot try} command -(@pxref{try}): +@item category +Equivalent to @code{--category} for the @ref{sendchange} command. -@table @code @item try_connect -This specifies how the ``try'' command should deliver its request to -the buildmaster. The currently accepted values are ``ssh'' and ``pb''. +Equivalent to @code{--connect}, this specifies how the @ref{try} command should +deliver its request to the buildmaster. The currently accepted values are +``ssh'' and ``pb''. + @item try_builders -Which builders should be used for the ``try'' build. +Equivalent to @code{--builders}, specifies which builders should be used for +the @ref{try} build. + @item try_vc -This specifies the version control system being used. +Equivalent to @code{--vc} for @ref{try}, this specifies the version control +system being used. + @item try_branch -This indicates that the current tree is on a non-trunk branch. +Equivlanent to @code{--branch}, this indicates that the current tree is on a non-trunk branch. + @item try_topdir @item try_topfile -Use @code{try_topdir} to explicitly indicate the top of your working -tree, or @code{try_topfile} to name a file that will only be found in -that top-most directory. +Use @code{try_topdir}, equivalent to @code{--try-topdir}, to explicitly +indicate the top of your working tree, or @code{try_topfile}, equivalent to +@code{--try-topfile} to name a file that will only be found in that top-most +directory. @item try_host @item try_username @item try_dir -When try_connect is ``ssh'', the command will pay attention to -@code{try_host}, @code{try_username}, and @code{try_dir}. +When @code{try_connect} is ``ssh'', the command will use @code{try_host} for +@code{--tryhost}, @code{try_username} for @code{--username}, and @code{try_dir} +for @code{--trydir}. Apologies for the confusing presence and absence of +'try'. @item try_username @item try_password @item try_master -Instead, when @code{try_connect} is ``pb'', the command will pay -attention to @code{try_username}, @code{try_password}, and -@code{try_master}. +Similarly, when @code{try_connect} is ``pb'', the command will pay attention to +@code{try_username} for @code{--username}, @code{try_password} for +@code{--passwd}, and @code{try_master} for @code{--master}. @item try_wait @item masterstatus -@code{try_wait} and @code{masterstatus} are used to ask the ``try'' -command to wait for the requested build to complete. +@code{try_wait} and @code{masterstatus} (equivalent to @code{--wait} and +@code{master}, respectively) are used to ask the @ref{try} command to wait for +the requested build to complete. @end table