Skip to content

Commit

Permalink
refs buildbot#286, buildbot#492 - fix optionfile parsing for buildbot…
Browse files Browse the repository at this point in the history 
… subcommands, with improved docs
  • Loading branch information
Dustin J. Mitchell committed Dec 29, 2009
1 parent 9296935 commit 2b69157
Show file tree
Hide file tree
Showing 4 changed files with 214 additions and 150 deletions.
229 changes: 140 additions & 89 deletions buildbot/scripts/runner.py
View file
Expand Up @@ -8,14 +8,91 @@

from buildbot.interfaces import BuildbotNotRunningError

# this is mostly just a front-end for mktap, twistd, and kill(1), but in the
# future it will also provide an interface to some developer tools that talk
# directly to a remote buildmaster (like 'try' and a status client)

# the create/start/stop commands should all be run as the same user,
# preferably a separate 'buildbot' account.

class MakerBase(usage.Options):
# Note that the terms 'options' and 'config' are used intechangeably here - in
# fact, they are intercanged several times. Caveat legator.

class OptionsWithOptionsFile(usage.Options):
# subclasses should set this to a list-of-lists in order to source the
# .buildbot/options file.
# buildbotOptions = [ [ 'optfile-name', 'option-name' ], .. ]
buildbotOptions = None

def __init__(self, *args):
# for options in self.buildbotOptions, optParameters, and the options
# file, change the default in optParameters *before* calling through
# to the parent constructor

if self.buildbotOptions:
optfile = loadOptionsFile()
for optfile_name, option_name in self.buildbotOptions:
for i in range(len(self.optParameters)):
if self.optParameters[i][0] == option_name and optfile_name in optfile:
self.optParameters[i][2] = optfile[optfile_name]
usage.Options.__init__(self, *args)

def loadOptionsFile(filename="options", here=None, home=None):
"""Find the .buildbot/FILENAME file. Crawl from the current directory up
towards the root, and also look in ~/.buildbot . The first directory
that's owned by the user and has the file we're looking for wins. Windows
skips the owned-by-user test.
@rtype: dict
@return: a dictionary of names defined in the options file. If no options
file was found, return an empty dict.
"""

if here is None:
here = os.getcwd()
here = os.path.abspath(here)

if home is None:
if runtime.platformType == 'win32':
home = os.path.join(os.environ['APPDATA'], "buildbot")
else:
home = os.path.expanduser("~/.buildbot")

searchpath = []
toomany = 20
while True:
searchpath.append(os.path.join(here, ".buildbot"))
next = os.path.dirname(here)
if next == here:
break # we've hit the root
here = next
toomany -= 1 # just in case
if toomany == 0:
raise ValueError("Hey, I seem to have wandered up into the "
"infinite glories of the heavens. Oops.")
searchpath.append(home)

localDict = {}

for d in searchpath:
if os.path.isdir(d):
if runtime.platformType != 'win32':
if os.stat(d)[stat.ST_UID] != os.getuid():
print "skipping %s because you don't own it" % d
continue # security, skip other people's directories
optfile = os.path.join(d, filename)
if os.path.exists(optfile):
try:
f = open(optfile, "r")
options = f.read()
exec options in localDict
except:
print "error while reading %s" % optfile
raise
break

for k in localDict.keys():
if k.startswith("__"):
del localDict[k]
return localDict

class MakerBase(OptionsWithOptionsFile):
optFlags = [
['help', 'h', "Display this message"],
["quiet", "q", "Do not emit the commands being run"],
Expand Down Expand Up @@ -553,65 +630,6 @@ def restart(config):
start(config)


def loadOptions(filename="options", here=None, home=None):
"""Find the .buildbot/FILENAME file. Crawl from the current directory up
towards the root, and also look in ~/.buildbot . The first directory
that's owned by the user and has the file we're looking for wins. Windows
skips the owned-by-user test.
@rtype: dict
@return: a dictionary of names defined in the options file. If no options
file was found, return an empty dict.
"""

if here is None:
here = os.getcwd()
here = os.path.abspath(here)

if home is None:
if runtime.platformType == 'win32':
home = os.path.join(os.environ['APPDATA'], "buildbot")
else:
home = os.path.expanduser("~/.buildbot")

searchpath = []
toomany = 20
while True:
searchpath.append(os.path.join(here, ".buildbot"))
next = os.path.dirname(here)
if next == here:
break # we've hit the root
here = next
toomany -= 1 # just in case
if toomany == 0:
raise ValueError("Hey, I seem to have wandered up into the "
"infinite glories of the heavens. Oops.")
searchpath.append(home)

localDict = {}

for d in searchpath:
if os.path.isdir(d):
if runtime.platformType != 'win32':
if os.stat(d)[stat.ST_UID] != os.getuid():
print "skipping %s because you don't own it" % d
continue # security, skip other people's directories
optfile = os.path.join(d, filename)
if os.path.exists(optfile):
try:
f = open(optfile, "r")
options = f.read()
exec options in localDict
except:
print "error while reading %s" % optfile
raise
break

for k in localDict.keys():
if k.startswith("__"):
del localDict[k]
return localDict

class StartOptions(MakerBase):
optFlags = [
['quiet', 'q', "Don't display startup log messages"],
Expand Down Expand Up @@ -639,14 +657,19 @@ class RestartOptions(MakerBase):
def getSynopsis(self):
return "Usage: buildbot restart [<basedir>]"

class DebugClientOptions(usage.Options):
class DebugClientOptions(OptionsWithOptionsFile):
optFlags = [
['help', 'h', "Display this message"],
]
optParameters = [
["master", "m", None,
"Location of the buildmaster's slaveport (host:port)"],
["passwd", "p", None, "Debug password to use"],
["myoption", "O", "DEF", "My Option!"],
]
buildbotOptions = [
[ 'debugMaster', 'passwd' ],
[ 'master', 'master' ],
]

def parseArgs(self, *args):
Expand All @@ -657,35 +680,37 @@ def parseArgs(self, *args):
if len(args) > 2:
raise usage.UsageError("I wasn't expecting so many arguments")

def postOptions(self):

This comment has been minimized.

Sign in to view

Copy link
@krajaratnam

krajaratnam Feb 19, 2010

This appears to be some debug code. I've removed this in commit 9ab140a in order to fix buildbot#719.

Ok?
print self['myoption']
sys.exit(1)

def debugclient(config):
from buildbot.clients import debug
opts = loadOptions()

master = config.get('master')
if not master:
master = opts.get('master')
if master is None:
raise usage.UsageError("master must be specified: on the command "
"line or in ~/.buildbot/options")

passwd = config.get('passwd')
if not passwd:
passwd = opts.get('debugPassword')
if passwd is None:
raise usage.UsageError("passwd must be specified: on the command "
"line or in ~/.buildbot/options")

d = debug.DebugWidget(master, passwd)
d.run()

class StatusClientOptions(usage.Options):
class StatusClientOptions(OptionsWithOptionsFile):
optFlags = [
['help', 'h', "Display this message"],
]
optParameters = [
["master", "m", None,
"Location of the buildmaster's status port (host:port)"],
]
buildbotOptions = [
[ 'masterstatus', 'master' ],
]

def parseArgs(self, *args):
if len(args) > 0:
Expand All @@ -695,10 +720,7 @@ def parseArgs(self, *args):

def statuslog(config):
from buildbot.clients import base
opts = loadOptions()
master = config.get('master')
if not master:
master = opts.get('masterstatus')
if master is None:
raise usage.UsageError("master must be specified: on the command "
"line or in ~/.buildbot/options")
Expand All @@ -707,19 +729,16 @@ def statuslog(config):

def statusgui(config):
from buildbot.clients import gtkPanes
opts = loadOptions()
master = config.get('master')
if not master:
master = opts.get('masterstatus')
if master is None:
raise usage.UsageError("master must be specified: on the command "
"line or in ~/.buildbot/options")
c = gtkPanes.GtkClient(master)
c.run()

class SendChangeOptions(usage.Options):
class SendChangeOptions(OptionsWithOptionsFile):
def __init__(self):
usage.Options.__init__(self)
OptionsWithOptionsFile.__init__(self)
self['properties'] = {}

optParameters = [
Expand All @@ -738,6 +757,14 @@ def __init__(self):
"Read the log messages from this file (- for stdin)"),
("when", "w", None, "timestamp to use as the change time"),
]

buildbotOptions = [
[ 'master', 'master' ],
[ 'username', 'username' ],
[ 'branch', 'branch' ],
[ 'category', 'category' ],
]

def getSynopsis(self):
return "Usage: buildbot sendchange [options] filenames.."
def parseArgs(self, *args):
Expand All @@ -752,11 +779,10 @@ def sendchange(config, runReactor=False):
connection will be drpoped as soon as the Change has been sent."""
from buildbot.clients.sendchange import Sender

opts = loadOptions()
user = config.get('username', opts.get('username'))
master = config.get('master', opts.get('master'))
branch = config.get('branch', opts.get('branch'))
category = config.get('category', opts.get('category'))
user = config.get('username')
master = config.get('master')
branch = config.get('branch')
category = config.get('category')
revision = config.get('revision')
properties = config.get('properties', {})
if config.get('when'):
Expand Down Expand Up @@ -794,7 +820,7 @@ def sendchange(config, runReactor=False):
return d


class ForceOptions(usage.Options):
class ForceOptions(OptionsWithOptionsFile):
optParameters = [
["builder", None, None, "which Builder to start"],
["branch", None, None, "which branch to build"],
Expand All @@ -814,7 +840,7 @@ def parseArgs(self, *args):
self['reason'] = " ".join(args)


class TryOptions(usage.Options):
class TryOptions(OptionsWithOptionsFile):
optParameters = [
["connect", "c", None,
"how to reach the buildmaster, either 'ssh' or 'pb'"],
Expand Down Expand Up @@ -860,8 +886,26 @@ class TryOptions(usage.Options):
["dryrun", 'n', "Gather info, but don't actually submit."],
]

# here it is, the definitive, quirky mapping of .buildbot/options names to
# command-line options. Design by committee, anyone?
buildbotOptions = [
[ 'try_connect', 'connect' ],
#[ 'try_builders', 'builders' ], <-- handled in postOptions
[ 'try_vc', 'vc' ],
[ 'try_branch', 'branch' ],
[ 'try_topdir', 'try-topdir' ],
[ 'try_topfile', 'try-topfile' ],
[ 'try_host', 'tryhost' ],
[ 'try_username', 'username' ],
[ 'try_dir', 'trydir' ],
[ 'try_password', 'passwd' ],
[ 'try_master', 'master' ],
#[ 'try_wait', 'wait' ], <-- handled in postOptions
[ 'masterstatus', 'master' ],
]

def __init__(self):
super(TryOptions, self).__init__()
OptionsWithOptionsFile.__init__(self)
self['builders'] = []
self['properties'] = {}

Expand All @@ -884,12 +928,19 @@ def opt_patchlevel(self, option):
def getSynopsis(self):
return "Usage: buildbot try [options]"

def postOptions(self):
opts = loadOptionsFile()
if not self['builders']:
self['builders'] = opts.get('try_builders', [])
if opts.get('try_wait', False):
self['wait'] = True

def doTry(config):
from buildbot.scripts import tryclient
t = tryclient.Try(config)
t.run()

class TryServerOptions(usage.Options):
class TryServerOptions(OptionsWithOptionsFile):
optParameters = [
["jobdir", None, None, "the jobdir (maildir) for submitting jobs"],
]
Expand All @@ -912,7 +963,7 @@ def doTryServer(config):
os.rename(tmpfile, newfile)


class CheckConfigOptions(usage.Options):
class CheckConfigOptions(OptionsWithOptionsFile):
optFlags = [
['quiet', 'q', "Don't display error messages or tracebacks"],
]
Expand Down

1 comment on commit 2b69157

@djmitche
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes - thanks!

Please sign in to comment.