Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

merged

  • Loading branch information...
commit e21ee0f495a00520ce7b845f21b8c6576a2be815 2 parents ea00a18 + d0ef1bc
@ctb authored
Showing with 863 additions and 480 deletions.
  1. +21 −6 README.html
  2. +24 −6 README.txt
  3. +48 −0 client/build-nose.py
  4. +46 −0 client/build-quixote
  5. +321 −303 client/pony_client.py
  6. +4 −0 client/test_client/__init__.py
  7. +104 −10 client/test_client/test_git_client.py
  8. +9 −1 client/test_client/test_hg_client.py
  9. +163 −0 client/test_client/test_svn_client.py
  10. +1 −0  examples/change-receiver/github-notify.json
  11. +49 −0 examples/change-receiver/github-receiver.cgi
  12. +14 −0 examples/change-receiver/test-post-github-notify.py
  13. +4 −1 pony_build/coordinator.py
  14. +0 −35 pony_build/qx_web/util.py
  15. +0 −42 pony_build/server.py
  16. +0 −69 pony_build/tests/test_webhook_notify.py
  17. +4 −4 pony_build/tests/testutil.py
  18. 0  pony_build/{qx_web → web}/__init__.py
  19. +1 −1  pony_build/{qx_web → web}/run.py
  20. 0  pony_build/{qx_web → web}/templates/base.html
  21. 0  pony_build/{qx_web → web}/templates/empty.html
  22. 0  pony_build/{qx_web → web}/templates/feed_base.html
  23. 0  pony_build/{qx_web → web}/templates/feed_generic_index.html
  24. 0  pony_build/{qx_web → web}/templates/feed_generic_package_index.html
  25. 0  pony_build/{qx_web → web}/templates/feed_index.html
  26. 0  pony_build/{qx_web → web}/templates/forms.html
  27. 0  pony_build/{qx_web → web}/templates/img/background.png
  28. 0  pony_build/{qx_web → web}/templates/img/dark_line.png
  29. 0  pony_build/{qx_web → web}/templates/img/light_line.png
  30. 0  pony_build/{qx_web → web}/templates/package_all.html
  31. 0  pony_build/{qx_web → web}/templates/package_base.html
  32. 0  pony_build/{qx_web → web}/templates/package_summary.html
  33. 0  pony_build/{qx_web → web}/templates/results_base.html
  34. 0  pony_build/{qx_web → web}/templates/results_files_index.html
  35. 0  pony_build/{qx_web → web}/templates/results_index.html
  36. 0  pony_build/{qx_web → web}/templates/results_inspect.html
  37. 0  pony_build/{qx_web → web}/templates/style.css
  38. 0  pony_build/{qx_web → web}/templates/top_index.html
  39. 0  pony_build/{qx_web → web}/urls.py
  40. +48 −0 pony_build/web/util.py
  41. +2 −2 setup.py
View
27 README.html
@@ -327,13 +327,13 @@ <h1 class="title">pony-build</h1>
<h1><a id="pony-build-server" name="pony-build-server">pony-build server</a></h1>
<p>The command:</p>
<pre class="literal-block">
-python -m pony_build.qx_web.run &lt;shelve filename&gt; &lt;port&gt;
+python -m pony_build.web.run &lt;shelve filename&gt; -p &lt;port&gt;
</pre>
<p>will run the Quixote-based pony-build Web app on the given port,
-reading &amp; writing from the shelve database in 'filename'.</p>
+reading &amp; writing from the sqlite database in 'filename'.</p>
<p>For example,</p>
<pre class="literal-block">
-python -m pony_build.qx_web.run test.db 8080
+python -m pony_build.web.run test.db -p 8080
</pre>
<p>will run a server that can be accessed on <a class="reference" href="http://localhost:8080/">http://localhost:8080/</a>. This
server will report on whatever results are sent to it by the client (see
@@ -418,7 +418,7 @@ <h1 class="title">pony-build</h1>
someone else deal with translating those into e-mail alerts, etc.</p>
<p>All of the RSS feeds that pony-build makes available can be posted to
pubsubhubbub with the proper configuration (see -P and -S options to
-<tt class="docutils literal"><span class="pre">pony_build.qx_web.run</span></tt>). A simple example CGI callback script that
+<tt class="docutils literal"><span class="pre">pony_build.web.run</span></tt>). A simple example CGI callback script that
sends an e-mail is available in
<tt class="docutils literal"><span class="pre">examples/push-cgi/notifier/push-subscriber.cgi</span></tt> in the pony-build
source distribution.</p>
@@ -450,10 +450,15 @@ <h1 class="title">pony-build</h1>
<div class="section">
<h1><a id="development" name="development">Development</a></h1>
<p>pony-build is hosted on github, at: <a class="reference" href="http://github.com/ctb/pony-build">http://github.com/ctb/pony-build</a></p>
-<p>To run the tests:</p>
+<p>To run the server tests:</p>
<pre class="literal-block">
python -m pony_build.tests.run
</pre>
+<p>To run the client tests:</p>
+<pre class="literal-block">
+cd client
+nosetests
+</pre>
</div>
<div class="section">
<h1><a id="design-and-ideas-for-the-future" name="design-and-ideas-for-the-future">Design and Ideas for the Future</a></h1>
@@ -520,6 +525,12 @@ <h1 class="title">pony-build</h1>
</blockquote>
</div>
<div class="section">
+<h2><a id="contributors" name="contributors">Contributors</a></h2>
+<p>Jacob Kaplan-Moss, Max Laite, Jack Carlson, Fatima Cherkaoui, and Khushboo
+Shakya have all contributed code and ideas.</p>
+<p>(If I'm missing anyone, please drop me a note!)</p>
+</div>
+<div class="section">
<h2><a id="acks" name="acks">Acks</a></h2>
<p>Titus says: Jesse Noller, Doug Philips, and Josh Williams discussed
things with me and are, collectively, entirely responsible for any bad
@@ -533,12 +544,16 @@ <h1 class="title">pony-build</h1>
<a class="reference" href="http://lists.idyll.org/pipermail/testing-in-python/2009-March/001277.html">http://lists.idyll.org/pipermail/testing-in-python/2009-March/001277.html</a></blockquote>
<p>where Kumar suggests that I just use Hudson for chrissakes. He's
probably right.</p>
+<p>Eric Holscher and Jacob Kaplan-Moss took the pony-build idea and ran
+with it, producing a parallel universe of Django-based reporting
+servers and REST-ish clients that report via JSON. Check out
+devmason.com and 'pony_barn' to see their approach in action.</p>
</div>
<div class="section">
<h2><a id="references" name="references">References</a></h2>
<p>webhooks: <a class="reference" href="http://webhooks.pbworks.com/">http://webhooks.pbworks.com/</a></p>
<p>--</p>
-<p>CTB 8/24/09</p>
+<p>CTB 2/24/10</p>
</div>
</div>
</div>
View
30 README.txt
@@ -48,14 +48,14 @@ pony-build server
The command: ::
- python -m pony_build.qx_web.run <shelve filename> <port>
+ python -m pony_build.web.run <shelve filename> -p <port>
will run the Quixote-based pony-build Web app on the given port,
-reading & writing from the shelve database in 'filename'.
+reading & writing from the sqlite database in 'filename'.
For example, ::
- python -m pony_build.qx_web.run test.db 8080
+ python -m pony_build.web.run test.db -p 8080
will run a server that can be accessed on http://localhost:8080/. This
server will report on whatever results are sent to it by the client (see
@@ -155,7 +155,7 @@ someone else deal with translating those into e-mail alerts, etc.
All of the RSS feeds that pony-build makes available can be posted to
pubsubhubbub with the proper configuration (see -P and -S options to
-``pony_build.qx_web.run``). A simple example CGI callback script that
+``pony_build.web.run``). A simple example CGI callback script that
sends an e-mail is available in
``examples/push-cgi/notifier/push-subscriber.cgi`` in the pony-build
source distribution.
@@ -195,10 +195,15 @@ Development
pony-build is hosted on github, at: http://github.com/ctb/pony-build
-To run the tests::
+To run the server tests::
python -m pony_build.tests.run
+To run the client tests::
+
+ cd client
+ nosetests
+
Design and Ideas for the Future
===============================
@@ -270,6 +275,14 @@ soon let buildbot pick up the higher-end ideas if they're game, too.
send "final results, authenticate with update token"
receive "ack"
+Contributors
+------------
+
+Jacob Kaplan-Moss, Max Laite, Jack Carlson, Fatima Cherkaoui, and Khushboo
+Shakya have all contributed code and ideas.
+
+(If I'm missing anyone, please drop me a note!)
+
Acks
----
@@ -290,6 +303,11 @@ You can also read this discussion starting here,
where Kumar suggests that I just use Hudson for chrissakes. He's
probably right.
+Eric Holscher and Jacob Kaplan-Moss took the pony-build idea and ran
+with it, producing a parallel universe of Django-based reporting
+servers and REST-ish clients that report via JSON. Check out
+devmason.com and 'pony_barn' to see their approach in action.
+
References
----------
@@ -297,4 +315,4 @@ webhooks: http://webhooks.pbworks.com/
--
-CTB 8/24/09
+CTB 2/24/10
View
48 client/build-nose.py
@@ -0,0 +1,48 @@
+#! /usr/bin/env python
+import sys
+import pprint
+from pony_client import BuildCommand, TestCommand, do, send, \
+ TempDirectoryContext, SetupCommand, HgClone, check, parse_cmdline, \
+ PythonPackageEgg
+
+options, args = parse_cmdline()
+
+python_exe = 'python'
+if args:
+ python_exe = args[0]
+
+repo_url = 'http://bitbucket.org/jpellerin/nose/'
+
+tags = ['nose']
+name = 'build-nose'
+
+server_url = options.server_url
+
+if not options.force_build:
+ if not check(name, server_url, tags=tags):
+ print 'check build says no need to build; bye'
+ sys.exit(0)
+
+context = TempDirectoryContext()
+commands = [ HgClone(repo_url, name='checkout'),
+ BuildCommand([python_exe, 'setup.py', 'build_ext', '-i'],
+ name='compile'),
+ TestCommand([python_exe, 'setup.py', 'test'], name='run tests'),
+ PythonPackageEgg(python_exe)]
+
+results = do(name, commands, context=context)
+client_info, reslist, files = results
+
+if options.report:
+ print 'result: %s; sending' % (client_info['success'],)
+ send(server_url, results, tags=tags)
+else:
+ print 'build result:'
+ pprint.pprint(client_info)
+ pprint.pprint(reslist)
+
+ print '(NOT SENDING BUILD RESULT TO SERVER)'
+
+if not client_info['success']:
+ sys.exit(-1)
+
View
46 client/build-quixote
@@ -0,0 +1,46 @@
+#! /usr/bin/env python
+import sys
+import pprint
+from pony_client import BuildCommand, TestCommand, do, send, \
+ TempDirectoryContext, SetupCommand, GitClone, check, parse_cmdline, \
+ PythonPackageEgg, test_python_version
+
+options, _ = parse_cmdline()
+
+python_exe = options.python_executable
+
+if not test_python_version(python_exe):
+ print "Unable to find " + python_exe + ". Failing..."
+ sys.exit(1)
+
+repo_url = 'git://quixote.ca/quixote'
+
+tags = [python_exe] + options.tagset
+name = 'quixote'
+
+server_url = options.server_url
+
+if not options.force_build:
+ if not check(name, server_url, tags=tags):
+ print 'check build says no need to build; bye'
+ sys.exit(0)
+
+context = TempDirectoryContext()
+commands = [ GitClone(repo_url, name='checkout'),
+ BuildCommand([python_exe, 'setup.py', 'install'], name='compile')]
+
+results = do(name, commands, context=context)
+client_info, reslist, files = results
+
+if options.report:
+ print 'result: %s; sending' % (client_info['success'],)
+ send(server_url, results, tags=tags)
+else:
+ print 'build result:'
+ pprint.pprint(client_info)
+ pprint.pprint(reslist)
+
+ print '(NOT SENDING BUILD RESULT TO SERVER)'
+
+if not client_info['success']:
+ sys.exit(-1)
View
624 client/pony_client.py
@@ -26,7 +26,33 @@
###
+DEBUG_LEVEL = 5
+INFO_LEVEL = 3
+WARNING_LEVEL = 2
+CRITICAL_LEVEL = 1
+_log_level = WARNING_LEVEL
+
+def log_debug(*args):
+ log(DEBUG_LEVEL, *args)
+
+def log_info(*args):
+ log(INFO_LEVEL, *args)
+
+def log_warning(*args):
+ log(WARNING_LEVEL, *args)
+
+def log_critical(*args):
+ log(CRITICAL_LEVEL, *args)
+def log(level, *what):
+ if level <= _log_level:
+ sys.stdout.write(" ".join([ str(x) for x in what]) + "\n")
+
+def set_log_level(level):
+ global _log_level
+ _log_level = level
+
+###
DEFAULT_CACHE_DIR='~/.pony-build'
def guess_cache_dir(dirname):
@@ -35,7 +61,7 @@ def guess_cache_dir(dirname):
parent = os.path.expanduser(parent)
result = os.path.join(parent, dirname)
- return result
+ return (parent, result)
def create_cache_dir(cache_dir, dirname):
# trim the pkg name so we can create the main cache_dir and not the
@@ -49,13 +75,14 @@ def create_cache_dir(cache_dir, dirname):
cache_dir = cache_dir[:-pkglen]
if os.path.isdir(cache_dir):
- print 'cache_dir %s exists already!' % cache_dir
+ log_info('VCS cache_dir %s exists already!' % cache_dir)
else:
try:
- print 'Had to create a new cache_dir!'
+ log_info('created new VCS cache dir: %s' % cache_dir)
os.mkdir(cache_dir)
except OSError:
- raise Exception('Unable to create VCS cache_dir: %s' % cache_dir)
+ log_critical('Unable to create VCS cache_dir: %s' % cache_dir)
+ raise
###
@@ -64,8 +91,7 @@ def _replace_variables(cmd, variables_d):
cmd = variables_d[cmd[3:]]
return cmd
-def _run_command(command_list, cwd=None, variables=None, extra_kwargs={},
- verbose=False):
+def _run_command(command_list, cwd=None, variables=None, extra_kwargs={}):
if variables:
x = []
for cmd in command_list:
@@ -78,11 +104,10 @@ def _run_command(command_list, cwd=None, variables=None, extra_kwargs={},
if extra_kwargs:
default_kwargs.update(extra_kwargs)
- if verbose:
- print 'CWD', os.getcwd()
- print 'running in ->', cwd
- print 'command_list:', command_list
- print 'default kwargs:', default_kwargs
+ log_debug('_run_command cwd', os.getcwd())
+ log_debug('_run_command running in ->', cwd)
+ log_debug('_run_command command list:', command_list)
+ log_debug('_run_command default kwargs:', default_kwargs)
try:
p = subprocess.Popen(command_list, cwd=cwd, **default_kwargs)
@@ -94,18 +119,20 @@ def _run_command(command_list, cwd=None, variables=None, extra_kwargs={},
err = traceback.format_exc()
ret = -1
- if verbose:
- print 'status:', ret
+ log_debug('_run_command status', str(ret))
+ log_debug('_run_command stdout', out)
+ log_debug('_run_command stderr', err)
return (ret, out, err)
class FileToUpload(object):
def __init__(self, filename, location, description, visible):
"""
-filename - name to publish as
-location - full location on build system (not sent to server)
-description - brief description of file/arch for server
-"""
+ filename - name to publish as
+ location - full location on build system (not sent to server)
+ description - brief description of file/arch for server
+ """
+
self.data = open(location, 'rb').read()
self.filename = filename
self.description = description
@@ -152,7 +179,7 @@ def initialize(self):
self.tempdir = tempfile.mkdtemp()
self.cwd = os.getcwd()
- print 'changing to temp directory:', self.tempdir
+ log_info('changing to temp directory:', self.tempdir)
os.chdir(self.tempdir)
def finish(self):
@@ -161,7 +188,7 @@ def finish(self):
Context.finish(self)
finally:
if self.cleanup:
- print 'removing', self.tempdir
+ log_info('removing', self.tempdir)
shutil.rmtree(self.tempdir, ignore_errors=True)
def update_client_info(self, info):
@@ -170,15 +197,16 @@ def update_client_info(self, info):
class VirtualenvContext(Context):
"""
-A context that creates a new virtualenv and does everything within that
-environment.
+ A context that works within a new virtualenv.
-VirtualenvContext works by modifying the @@CTB
-"""
- def __init__(self, always_cleanup=True, dependencies=[], python='python'):
+ VirtualenvContext works by modifying the path to the Python executable.
+ """
+ def __init__(self, always_cleanup=True, dependencies=[], optional=[],
+ python='python'):
Context.__init__(self)
self.cleanup = always_cleanup
self.dependencies = dependencies
+ self.optional = optional # optional dependencies
self.python = python
# Create the virtualenv. Have to do this here so that commands can use
@@ -187,7 +215,7 @@ def __init__(self, always_cleanup=True, dependencies=[], python='python'):
self.tempdir = tempfile.mkdtemp()
- print 'creating virtualenv'
+ log_inf('creating virtualenv')
cmdlist = [python, '-m', 'virtualenv', '--no-site-packages',
self.tempdir]
(ret, out, err) = _run_command(cmdlist)
@@ -206,19 +234,35 @@ def __init__(self, always_cleanup=True, dependencies=[], python='python'):
def initialize(self):
Context.initialize(self)
- print 'changing to temp directory:', self.tempdir
+ log_info('changing to temp directory:', self.tempdir)
+
self.cwd = os.getcwd()
os.chdir(self.tempdir)
# install pip, then use it to install any packages desired
- print 'installing pip'
+ log_info('installing pip')
+
(ret, out, err) = _run_command([self.easy_install, '-U', 'pip'])
if ret != 0:
raise Exception("error in installing pip: %s, %s" % (out, err))
for dep in self.dependencies:
- print "installing", dep
- _run_command([self.pip, 'install', '-U', '-I'] + [dep])
+ log_info('installing dependency:', dep)
+ (ret, out, err) = _run_command([self.pip, 'install', '-U', '-I',
+ dep])
+
+ if ret != 0:
+ raise Exception("pip cannot install req dependency: %s" % dep)
+
+ for dep in self.optional:
+ log_info("installing optional dependency:", dep)
+ (ret, out, err) = _run_command([self.pip, 'install', '-U', '-I',
+ dep])
+
+ # @CTB should record failed installs of optional packages
+ # to client?
+ if ret != 0:
+ log_warning("pip cannot install optional dependency: %s" % dep)
def finish(self):
os.chdir(self.cwd)
@@ -226,7 +270,7 @@ def finish(self):
Context.finish(self)
finally:
if self.cleanup:
- print 'removing', self.tempdir
+ log_info("VirtualenvContext: removing", self.tempdir)
shutil.rmtree(self.tempdir, ignore_errors=True)
def update_client_info(self, info):
@@ -234,12 +278,15 @@ def update_client_info(self, info):
info['tempdir'] = self.tempdir
info['virtualenv'] = True
info['dependencies'] = self.dependencies
+ info['optional'] = self.optional
class UploadAFile(object):
"""
-@CTB add glob support
-"""
+ A build command that arranges to upload a specific file to the server.
+
+ @CTB add glob support!
+ """
def __init__(self, filepath, public_name, description, visible=True):
self.filepath = os.path.realpath(filepath)
self.public_name = public_name
@@ -268,8 +315,7 @@ def get_results(self):
class BaseCommand(object):
def __init__(self, command_list, name='', run_cwd=None,
- subprocess_kwargs=None, ignore_failure=False,
- verbose=False):
+ subprocess_kwargs=None, ignore_failure=False):
self.command_list = command_list
if name:
self.command_name = name
@@ -287,7 +333,6 @@ def __init__(self, command_list, name='', run_cwd=None,
self.subprocess_kwargs = dict(subprocess_kwargs)
self.ignore_failure = ignore_failure
- self.verbose = verbose
def __repr__(self):
return "%s (%s)" % (self.command_name, self.command_type)
@@ -299,8 +344,7 @@ def run(self, context):
start = time.time()
(ret, out, err) = _run_command(self.command_list, cwd=self.run_cwd,
variables=self.variables,
- extra_kwargs=self.subprocess_kwargs,
- verbose=self.verbose)
+ extra_kwargs=self.subprocess_kwargs)
self.status = ret
self.output = out
@@ -354,361 +398,307 @@ def run(self, context):
'an egg installation file',
visible=True)
+class _VersionControlClientBase(SetupCommand):
+ """
+ Base class for version control clients.
-class GitClone(SetupCommand):
- command_name = 'checkout'
+ Subclasses should define:
- def __init__(self, repository, branch='master', cache_dir=None,
- use_cache=True, **kwargs):
- SetupCommand.__init__(self, [], **kwargs)
- self.repository = repository
- self.branch = branch
+ - get_dirname()
+ - update_repository()
+ - create_repository(url, dirname, step='stepname')
+ - record_repository_info(dirname)
+ and optionally override 'get_results()'.
+
+ """
+
+ def __init__(self, use_cache=True, **kwargs):
+ SetupCommand.__init__(self, [], **kwargs)
self.use_cache = use_cache
- self.cache_dir = cache_dir
- if cache_dir:
- self.cache_dir = os.path.expanduser(cache_dir)
self.duration = -1
self.version_info = ''
-
self.results_dict = {}
def run(self, context):
- # first, guess the co dir name
- p = urlparse.urlparse(self.repository)
- path = p[2] # urlparse -> path
-
- dirname = path.rstrip('/').split('/')[-1]
- if dirname.endswith('.git'):
- dirname = dirname[:-4]
+ # dirname is the directory created by a succesful checkout.
+ dirname = self.get_dirname()
- print 'git checkout dirname guessed as: %s' % (dirname,)
-
- if self.use_cache:
- cache_dir = self.cache_dir
- if not cache_dir:
- # setup some variables for cache folder locations/create
- # cache_dir if it does not exist
- repo_dir = guess_cache_dir(dirname)
- create_cache_dir(repo_dir, dirname)
- # trim repo so we know where the users cache should be
- # so we can change to for git stuff later
- pkglength = len(dirname)
- cache_dir = repo_dir[:-pkglength]
- ##
+ # cwd is the directory we're going to ultimately put dirname under.
+ cwd = os.getcwd()
+ # NOTE: we flat out don't like the situation where the
+ # directory already exists. Force a clean checkout.
+ assert not os.path.exists(dirname)
+
if self.use_cache:
- cwd = os.getcwd()
- if os.path.exists(repo_dir):
+ # 'repo_dir' is the full cache directory containing the repo.
+ # this will be something like '~/.pony-build/<dirname>'.
+ #
+ # 'cache_dir' is the parent dir.
+
+ cache_dir, repo_dir = guess_cache_dir(dirname)
+
+ # does the repo already exist?
+ if os.path.exists(repo_dir): # YES
os.chdir(repo_dir)
- print 'changed to: ', repo_dir, 'to do fetch.'
+ log_info('changed to: ', repo_dir, 'to do fetch.')
+ self.update_repository()
+ else: # NO
+ # do a clone to create the repo dir
+ log_info('changing to: ' + cache_dir + ' to make new repo dir')
+ os.chdir(cache_dir)
- branchspec = '%s:%s' % (self.branch, self.branch)
- cmdlist = ['git', 'fetch', '-ufv', self.repository, branchspec]
- (ret, out, err) = _run_command(cmdlist)
+ self.create_repository(self.repository, dirname,
+ step='create cache')
+ assert os.path.isdir(repo_dir)
+
+ os.chdir(cwd)
- self.results_dict['cache_update'] = \
- dict(status=ret, output=out, errout=err,
- command=str(cmdlist))
+ log_info('Using the local cache at %s for cloning' % repo_dir)
+ location = repo_dir
+ else:
+ location = self.repository
- if ret != 0:
- raise Exception("cannot update cache: %s" % repo_dir)
+ self.create_repository(location, dirname, step='clone')
- cmdlist = ['git', 'checkout', '-f', self.branch]
- (ret, out, err) = _run_command(cmdlist)
+ if not os.path.exists(dirname) and os.path.isdir(dirname):
+ log_critical('wrong guess; %s does not exist. whoops' % (dirname,))
+ raise Exception
+
+ # get some info on what our repository version is
+ self.record_repository_info(dirname)
+ # record the build directory, too.
+ context.build_dir = os.path.join(os.getcwd(), dirname)
+ # signal success!
+ self.status = 0
- self.results_dict['cache_checkout_head'] = \
- dict(status=ret, output=out, errout=err,
- command=str(cmdlist))
+ def get_results(self):
+ self.results_dict['out'] = self.results_dict['errout'] = ''
+ self.results_dict['status'] = self.status
+ self.results_dict['type'] = self.command_type
+ self.results_dict['name'] = self.command_name
- if ret != 0:
- raise Exception("cannot reset cache: %s" % repo_dir)
+ return self.results_dict
- else: # need to create repo_dir
- # do a clone to create the repo dir
- print 'changing to: ' + cache_dir + ' to make new repo dir'
- os.chdir(cache_dir)
- cmdlist = ['git', 'clone', self.repository]
- (ret, out, err) = _run_command(cmdlist)
+class GitClone(_VersionControlClientBase):
+ """Check out and/or update a git repository."""
+
+ command_name = 'checkout'
- if ret != 0:
- raise Exception("cannot create cache in %s" % cache_dir)
-
- os.chdir(cwd)
- ##
-
- # now, do a clone, from either the parent OR the local cache
- location = self.repository
- if self.use_cache and os.path.isdir(repo_dir):
- location = repo_dir
- print 'Using the local cache for cloning'
+ def __init__(self, repository, branch='master', use_cache=True, **kwargs):
+ _VersionControlClientBase.__init__(self, use_cache=use_cache, **kwargs)
+
+ self.repository = repository
+ self.branch = branch
- cmdlist = ['git', 'clone', location]
+ def get_dirname(self):
+ "Calculate the directory name resulting from a successful checkout."
+ p = urlparse.urlparse(self.repository)
+ path = p[2] # urlparse -> path
+
+ dirname = path.rstrip('/').split('/')[-1]
+ if dirname.endswith('.git'):
+ dirname = dirname[:-4]
+
+ log_info('git checkout dirname guessed as: %s' % (dirname,))
+ return dirname
+
+ def update_repository(self):
+ branchspec = '%s:%s' % (self.branch, self.branch)
+ cmdlist = ['git', 'fetch', '-ufv', self.repository, branchspec]
+ print '***', cmdlist
(ret, out, err) = _run_command(cmdlist)
- self.results_dict['clone'] = \
- dict(status=ret, output=out, errout=err,
- command=str(cmdlist))
- if ret != 0:
- return
+ self.results_dict['cache_update'] = dict(status=ret, output=out,
+ errout=err,
+ command=str(cmdlist))
- print cmdlist, out
+ if ret != 0:
+ raise Exception("cannot update cache: %s" % repo_dir)
- if not os.path.exists(dirname) and os.path.isdir(dirname):
- print 'wrong guess; %s does not exist. whoops' % (dirname,)
- self.status = -1
- return
+ cmdlist = ['git', 'checkout', '-f', self.branch]
+ (ret, out, err) = _run_command(cmdlist)
- ##
+ self.results_dict['cache_checkout_head'] = dict(status=ret, output=out,
+ errout=err,
+ command=str(cmdlist))
- # check out the right branch
- if self.branch != 'master':
- cmdlist = ['git', 'checkout', 'origin/'+self.branch]
- (ret, out, err) = _run_command(cmdlist, dirname)
+ if ret != 0:
+ raise Exception("cannot reset cache: %s" % repo_dir)
- print cmdlist, out
+ def create_repository(self, url, dirname, step='clone'):
+ cmdlist = ['git', 'clone', url]
+ (ret, out, err) = _run_command(cmdlist)
- self.results_dict['checkout+origin'] = \
- dict(status=ret, output=out, errout=err,
- command=str(cmdlist), branch=self.branch)
- if ret != 0:
- return
+ self.results_dict[step] = dict(status=ret, output=out, errout=err,
+ command=str(cmdlist))
- cmdlist = ['git', 'checkout', '-b', self.branch]
+ if ret != 0:
+ cwd = os.getcwd()
+ raise Exception("cannot clone repository %s in %s" % (url, cwd))
- print cmdlist, out
+ if self.branch != 'master':
+ # fetch the right branch
+ branchspec = '%s:%s' % (self.branch, self.branch)
+ cmdlist = ['git', 'fetch', '-ufv', self.repository, branchspec]
+ (ret, out, err) = _run_command(cmdlist, dirname)
+ assert ret == 0, (out, err)
+ # check out the right branch
+ cmdlist = ['git', 'checkout', '-f', self.branch]
(ret, out, err) = _run_command(cmdlist, dirname)
- self.results_dict['checkout+-b'] = \
- dict(status=ret, output=out, errout=err,
- command=str(cmdlist), branch=self.branch)
- if ret != 0:
- return
+ assert ret == 0, (out, err)
- # get some info on what our HEAD is
+ def record_repository_info(self, repo_dir):
cmdlist = ['git', 'log', '-1', '--pretty=oneline']
- (ret, out, err) = _run_command(cmdlist, dirname)
+ (ret, out, err) = _run_command(cmdlist, repo_dir)
assert ret == 0, (cmdlist, ret, out, err)
self.version_info = out.strip()
- self.status = 0
-
- # set the build directory, too.
- context.build_dir = os.path.join(os.getcwd(),
- dirname)
-
def get_results(self):
- self.results_dict['out'] = self.results_dict['errout'] = ''
- self.results_dict['command'] = 'GitClone(%s, %s)' % (self.repository,
- self.branch)
- self.results_dict['status'] = self.status
- self.results_dict['type'] = self.command_type
- self.results_dict['name'] = self.command_name
-
+ # first, update basic
+ _VersionControlClientBase.get_results(self)
+
self.results_dict['version_type'] = 'git'
if self.version_info:
self.results_dict['version_info'] = self.version_info
+ self.results_dict['command'] = 'GitClone(%s, %s)' % (self.repository,
+ self.branch)
+
return self.results_dict
-class HgClone(SetupCommand):
+class HgClone(_VersionControlClientBase):
+ """Check out or update an Hg (Mercurial) repository."""
command_name = 'checkout'
- def __init__(self, repository, branch='default', cache_dir=None,
- use_cache=True, **kwargs):
- SetupCommand.__init__(self, [], **kwargs)
+ def __init__(self, repository, branch='default', use_cache=True, **kwargs):
+ _VersionControlClientBase.__init__(self, use_cache=use_cache, **kwargs)
self.repository = repository
self.branch = branch
assert branch == 'default'
- self.use_cache = use_cache
- self.cache_dir = cache_dir
- if cache_dir:
- self.cache_dir = os.path.expanduser(cache_dir)
-
- self.duration = -1
- self.version_info = ''
-
- self.results_dict = {}
-
- def run(self, context):
- # first, guess the checkout dir name
+ def get_dirname(self):
+ "Calculate the directory name resulting from a successful checkout."
p = urlparse.urlparse(self.repository)
- path = p[2] # urlparse -> path
+ path = p[2] # urlparse -> path
dirname = path.rstrip('/').split('/')[-1]
+ log_info('git checkout dirname guessed as: %s' % (dirname,))
+ return dirname
- print 'hg checkout dirname guessed as: %s' % (dirname,)
-
- if self.use_cache:
- cache_dir = self.cache_dir
- if not cache_dir:
- repo_dir = guess_cache_dir(dirname)
- #repo_dir= os.path.normpath(repo_dir)
- print "Here guessed repo_dir before creating "+ repo_dir
- create_cache_dir(repo_dir, dirname)
- pkglength = len(dirname)
- cache_dir = repo_dir[:-pkglength]
-
- ##
-
- if self.use_cache:
- cwd = os.getcwd()
-
- if os.path.exists(repo_dir):
- #if os.path.isdir(dirname):
- os.chdir(repo_dir)
- print 'changed to: ', repo_dir, 'to do fetch. '
- cmdlist = ['hg', 'pull', self.repository]
- (ret, out, err) = _run_command(cmdlist)
-
- self.results_dict['cache_pull'] = \
- dict(status=ret, output=out, errout=err,
- command=str(cmdlist))
-
- if ret != 0:
- return
+ def update_repository(self):
+ cmdlist = ['hg', 'pull', self.repository]
+ (ret, out, err) = _run_command(cmdlist)
- cmdlist = ['hg', 'update', '-C']
- (ret, out, err) = _run_command(cmdlist)
+ self.results_dict['cache_pull'] = dict(status=ret, output=out,
+ errout=err,
+ command=str(cmdlist))
- self.results_dict['cache_update'] = \
- dict(status=ret, output=out, errout=err,
- command=str(cmdlist))
+ if ret != 0:
+ raise Exception, "cannot pull from %s" % self.repository
- if ret != 0:
- return
- else:
- #do a clone to create repo_dir
- print 'changing to:' + cache_dir + ' to make new repo_dir'
- os.chdir(cache_dir)
- cmdlist = ['hg', 'clone', self.repository]
- (ret, out, err) = _run_command(cmdlist)
- print cmdlist, out
+ cmdlist = ['hg', 'update', '-C']
+ (ret, out, err) = _run_command(cmdlist)
- os.chdir(cwd)
+ self.results_dict['cache_update'] = \
+ dict(status=ret, output=out, errout=err,
+ command=str(cmdlist))
- ##
+ assert ret == 0, (out, err)
- # now, do a clone, from either the parent OR the local cache
- location = self.repository
- if self.use_cache and os.path.isdir(repo_dir):
- location = repo_dir
- print 'Using the local cache for cloning'
- cmdlist = ['hg', 'clone', location]
+ def create_repository(self, url, dirname, step='clone'):
+ cmdlist = ['hg', 'clone', url]
(ret, out, err) = _run_command(cmdlist)
- self.results_dict['clone'] = \
- dict(status=ret, output=out, errout=err,
- command=str(cmdlist))
- if ret != 0:
- return
-
- print cmdlist, out
+ self.results_dict[step] = dict(status=ret, output=out, errout=err,
+ command=str(cmdlist))
- if not os.path.exists(dirname) and os.path.isdir(dirname):
- print 'wrong guess; %s does not exist. whoops' % (dirname,)
- self.status = -1
- return
-
- ##
+ if ret != 0:
+ cwd = os.getcwd()
+ raise Exception("cannot clone repository %s in %s" % (url, cwd))
+ # @CTB branch stuff unimplemented
+
+ def record_repository_info(self, repo_dir):
# get some info on what our HEAD is
- cmdlist = ['hg', 'log', ]
- (ret, out, err) = _run_command(cmdlist, dirname)
-
+ cmdlist = ['hg', 'id', '-nib']
+ (ret, out, err) = _run_command(cmdlist, repo_dir)
assert ret == 0, (cmdlist, ret, out, err)
-
self.version_info = out.strip()
- self.status = 0
-
- # set the build directory, too.
- context.build_dir = os.path.join(os.getcwd(),
- dirname)
-
def get_results(self):
- self.results_dict['out'] = self.results_dict['errout'] = ''
+ # first, update basic
+ _VersionControlClientBase.get_results(self)
+
self.results_dict['command'] = 'HgCheckout(%s, %s)' % (self.repository,
self.branch)
- self.results_dict['status'] = self.status
- self.results_dict['type'] = self.command_type
- self.results_dict['name'] = self.command_name
-
self.results_dict['version_type'] = 'hg'
if self.version_info:
self.results_dict['version_info'] = self.version_info
return self.results_dict
-class SvnUpdate(SetupCommand):
+class SvnCheckout(_VersionControlClientBase):
+ """Check out or update a subversion repository."""
command_name = 'checkout'
- def __init__(self, dirname, repository, cache_dir=None, **kwargs):
- SetupCommand.__init__(self, [], **kwargs)
+ def __init__(self, dirname, repository, use_cache=True, **kwargs):
+ _VersionControlClientBase.__init__(self, use_cache=use_cache)
+
+ self.dirname = dirname
self.repository = repository
- self.cache_dir = None
- if cache_dir:
- self.cache_dir = os.path.expanduser(cache_dir)
- self.duration = -1
- self.dirname = dirname
+ def get_dirname(self):
+ return self.dirname
- def run(self, context):
- dirname = self.dirname
+ def update_repository(self):
+ cmdlist = ['svn', 'update', '--accept', 'theirs-full']
+ (ret, out, err) = _run_command(cmdlist)
- ##
+ self.results_dict['svn update'] = dict(status=ret, output=out,
+ errout=err,
+ command=str(cmdlist))
- if self.cache_dir:
- print 'updating cache dir:', self.cache_dir
- cwd = os.getcwd()
- os.chdir(self.cache_dir)
- cmdlist = ['svn', 'update']
+ if ret != 0:
+ log_critical("cannot svn update")
+ raise Exception, (cmdlist, ret, out, err)
+
+ def create_repository(self, url, dirname, step='clone'):
+ if os.path.isdir(url): # local dir? COPY.
+ shutil.copytree(url, dirname)
+ else: # remote repo? CO.
+ cmdlist = ['svn', 'co', url, dirname]
(ret, out, err) = _run_command(cmdlist)
- if ret != 0:
- self.command_list = cmdlist
- self.status = ret
- self.output = out
- self.errout = err
- return
-
- subdir = os.path.join(cwd, dirname)
- shutil.copytree(self.cache_dir, subdir)
- os.chdir(subdir)
- else:
- cmdlist = ['svn', 'co', self.repository, dirname]
+ self.results_dict[step] = dict(status=ret, output=out, errout=err,
+ command=str(cmdlist))
- (ret, out, err) = _run_command(cmdlist)
if ret != 0:
- self.command_list = cmdlist
- self.status = ret
- self.output = out
- self.errout = err
-
- return
-
- print cmdlist, out
+ log_critical("cannot svn checkout %s into %s" % (url, dirname))
+ raise Exception, "cannot svn checkout %s into %s" % (url, dirname)
- if not os.path.exists(dirname) and os.path.isdir(dirname):
- self.command_list = cmdlist
- self.status = -1
- self.output = ''
- self.errout = 'pony-build-client cannot find expected svn dir: %s' % (dirname,)
-
- print 'wrong guess; %s does not exist. whoops' % (dirname,)
- return
-
- os.chdir(dirname)
+ def record_repository_info(self, repo_dir):
+ cmdlist = ['svnversion']
+ (ret, out, err) = _run_command(cmdlist, repo_dir)
+ assert ret == 0, (cmdlist, ret, out, err)
+ self.version_info = out.strip()
- ##
+ def get_results(self):
+ # first, update basic
+ _VersionControlClientBase.get_results(self)
+
+ self.results_dict['command'] = 'SvnCheckout(%s, %s)' %(self.repository,
+ self.dirname)
+ self.results_dict['version_type'] = 'hg'
+ if self.version_info:
+ self.results_dict['version_info'] = self.version_info
- self.status = 0 # success
- self.output = ''
- self.errout = ''
+ return self.results_dict
###
@@ -723,7 +713,7 @@ def get_arch():
###
def _send(server, info, results):
- print 'connecting to', server
+ log_info('connecting to', server)
s = xmlrpclib.ServerProxy(server, allow_none=True)
(result_key, auth_key) = s.add_results(info, results)
return str(auth_key)
@@ -749,8 +739,8 @@ def _upload_file(server_url, fileobj, auth_key):
try:
http_result = urllib.urlopen(upload_url, fileobj.data)
except:
- print 'file upload failed:', fileobj
- print traceback.format_exc()
+ log_warning('file upload failed:', str(fileobj))
+ log_warning(traceback.format_exc())
def do(name, commands, context=None, arch=None, stop_if_failure=True):
reslist = []
@@ -759,7 +749,7 @@ def do(name, commands, context=None, arch=None, stop_if_failure=True):
context.initialize()
for c in commands:
- print 'running:', c
+ log_debug('running:', str(c))
if context:
context.start_command(c)
c.run(context)
@@ -804,12 +794,12 @@ def send(server_url, x, hostname=None, tags=()):
client_info['tags'] = tags
server_url = get_server_url(server_url)
- print 'using server URL:', server_url
+ log_info('using server URL:', server_url)
auth_key = _send(server_url, client_info, reslist)
if files_to_upload:
for fileobj in files_to_upload:
- print 'uploading', fileobj
+ log_debug('uploading', str(fileobj))
_upload_file(server_url, fileobj, auth_key)
def check(name, server_url, tags=(), hostname=None, arch=None, reserve_time=0):
@@ -862,17 +852,45 @@ def parse_cmdline(argv=[]):
cmdline.add_option('-v', '--verbose', dest='verbose',
action='store_true', default=False,
help='set verbose reporting')
+
+ cmdline.add_option('-e', '--python-executable', dest='python_executable',
+ action='store', default='python',
+ help='override the version of python used to build with')
+
+ cmdline.add_option('-t', '--tagset', dest='tagset',
+ action='store', default=[],
+ help='comma-delimited list of tags to be applied')
if not argv:
(options, args) = cmdline.parse_args()
else:
(options, args) = cmdline.parse_args(argv)
+
+ # parse the tagset
+ if options.tagset:
+ options.tagset = options.tagset.split(',')
+
+ # there should be nothing in args.
+ # if there is, print a warning, then crash and burn.
+ if args:
+ print "Error--unknown arguments detected. Failing..."
+ sys.exit(0)
return options, args
###
+
+def test_python_version(python_exe):
+ result = subprocess.Popen(python_exe + " -c \"print 'hello, world'\"", shell=True, \
+ stdout=subprocess.PIPE).communicate()
+ if result[0] != "hello, world\n":
+ return False
+ return True
+
+###
+
def get_python_config(options, args):
if not len(args):
python_ver = 'python2.5'
@@ -909,7 +927,7 @@ def get_python_config(options, args):
Python_package_egg
]),
'twill' : (get_python_config,
- [ SvnUpdate('twill', 'http://twill.googlecode.com/svn/branches/0.9.2-dev/twill', cache_dir='~/.pony-build/twill'),
+ [ SvnCheckout('twill', 'http://twill.googlecode.com/svn/branches/0.9.2-dev/twill', cache_dir='~/.pony-build/twill'),
PythonBuild,
PythonTest
]),
View
4 client/test_client/__init__.py
@@ -0,0 +1,4 @@
+import pony_client
+
+def setup():
+ pony_client.set_log_level(pony_client.DEBUG_LEVEL)
View
114 client/test_client/test_git_client.py
@@ -14,6 +14,14 @@
import pony_client
from pony_client import GitClone, TempDirectoryContext, _run_command
+_cwd = None
+def setup():
+ global _cwd
+ _cwd = os.getcwd()
+
+def teardown():
+ os.chdir(_cwd)
+
class Test_GitNonCachingCheckout(object):
repository_url = 'http://github.com/ctb/pony-build-git-test.git'
@@ -28,14 +36,46 @@ def teardown(self):
def test_basic(self):
"Run the GitClone command w/o caching and verify it."
command = GitClone(self.repository_url, use_cache=False)
- command.verbose = True
command.run(self.context)
- pprint.pprint(command.get_results()) # debugging output
+ # check version info
+ results_info = command.get_results()
+ pprint.pprint(results_info) # debugging output
+
+ assert results_info['version_info'] == """\
+c57591d8cc9ef3c293a2006416a0bb8b2ffed26d secondary commit"""
+ assert results_info['version_type'] == 'git'
+ # check files
os.chdir(self.context.tempdir)
assert os.path.exists(os.path.join('pony-build-git-test', 'test1'))
assert os.path.exists(os.path.join('pony-build-git-test', 'test2'))
+
+ def test_other_branch(self):
+ "Run the GitClone command for another branch."
+
+ command = GitClone(self.repository_url, branch='other',
+ use_cache=False)
+ command.run(self.context)
+
+ # check version info
+ results_info = command.get_results()
+ pprint.pprint(results_info) # debugging output
+
+ assert results_info['version_info'] == """\
+7f8a8e130a3cc631752e275ea57220a1b6e2dddb look ma, another branch\\!"""
+ assert results_info['version_type'] == 'git'
+
+ # check files
+ cwd = os.getcwd()
+ os.chdir(self.context.tempdir)
+ try:
+ assert os.path.exists(os.path.join('pony-build-git-test', 'test1'))
+ assert not os.path.exists(os.path.join('pony-build-git-test',
+ 'test2'))
+ assert os.path.exists(os.path.join('pony-build-git-test', 'test3'))
+ finally:
+ os.chdir(cwd)
def create_cache_location(repository_url):
@@ -50,7 +90,7 @@ def create_cache_location(repository_url):
print 'calculated repository dirname as:', repository_dirname
- repository_cache = pony_client.guess_cache_dir(repository_dirname)
+ (_, repository_cache) = pony_client.guess_cache_dir(repository_dirname)
assert repository_cache.startswith(temp_cache_location)
# this will create 'the_cache' directory that contains individual
@@ -81,10 +121,14 @@ def teardown(self):
def test_basic(self):
"Run the GitClone command and verify that it produces the right repo."
command = GitClone(self.repository_url)
- command.verbose = True
command.run(self.context)
- pprint.pprint(command.get_results()) # debugging output
+ results_info = command.get_results()
+ pprint.pprint(results_info) # debugging output
+
+ assert results_info['version_info'] == """\
+c57591d8cc9ef3c293a2006416a0bb8b2ffed26d secondary commit"""
+ assert results_info['version_type'] == 'git'
cwd = os.getcwd()
os.chdir(self.context.tempdir)
@@ -94,6 +138,31 @@ def test_basic(self):
finally:
os.chdir(cwd)
+ def test_other_branch(self):
+ "Run the GitClone command for another branch."
+
+ command = GitClone(self.repository_url, branch='other')
+ command.run(self.context)
+
+ # check version info
+ results_info = command.get_results()
+ pprint.pprint(results_info) # debugging output
+
+ assert results_info['version_info'] == """\
+7f8a8e130a3cc631752e275ea57220a1b6e2dddb look ma, another branch\\!"""
+ assert results_info['version_type'] == 'git'
+
+ # check files
+ cwd = os.getcwd()
+ os.chdir(self.context.tempdir)
+ try:
+ assert os.path.exists(os.path.join('pony-build-git-test', 'test1'))
+ assert not os.path.exists(os.path.join('pony-build-git-test',
+ 'test2'))
+ assert os.path.exists(os.path.join('pony-build-git-test', 'test3'))
+ finally:
+ os.chdir(cwd)
+
class Test_GitCachingUpdate(object):
repository_url = 'http://github.com/ctb/pony-build-git-test.git'
@@ -115,7 +184,7 @@ def setup(self):
os.chdir(cache_dir)
- # now, check out the test hg repository.
+ # now, check out the test git repository.
(ret, out, err) = _run_command(['git', 'clone', self.repository_url])
assert ret == 0, (out, err)
@@ -137,10 +206,14 @@ def teardown(self):
def test_basic(self):
"Run the GitClone command and verify that it produces an updated repo."
command = GitClone(self.repository_url)
- command.verbose = True
command.run(self.context)
- pprint.pprint(command.get_results()) # debugging output
+ results_info = command.get_results()
+ pprint.pprint(results_info) # debugging output
+
+ assert results_info['version_info'] == """\
+c57591d8cc9ef3c293a2006416a0bb8b2ffed26d secondary commit"""
+ assert results_info['version_type'] == 'git'
cwd = os.getcwd()
os.chdir(self.context.tempdir)
@@ -150,6 +223,27 @@ def test_basic(self):
finally:
os.chdir(cwd)
-
+ def test_other_branch(self):
+ "Run the GitClone command for another branch."
-
+ command = GitClone(self.repository_url, branch='other')
+ command.run(self.context)
+
+ # check version info
+ results_info = command.get_results()
+ pprint.pprint(results_info) # debugging output
+
+ assert results_info['version_info'] == """\
+7f8a8e130a3cc631752e275ea57220a1b6e2dddb look ma, another branch\\!"""
+ assert results_info['version_type'] == 'git'
+
+ # check files
+ cwd = os.getcwd()
+ os.chdir(self.context.tempdir)
+ try:
+ assert os.path.exists(os.path.join('pony-build-git-test', 'test1'))
+ assert not os.path.exists(os.path.join('pony-build-git-test',
+ 'test2'))
+ assert os.path.exists(os.path.join('pony-build-git-test', 'test3'))
+ finally:
+ os.chdir(cwd)
View
10 client/test_client/test_hg_client.py
@@ -8,6 +8,14 @@
import pony_client
from pony_client import HgClone, TempDirectoryContext, _run_command
+_cwd = None
+def setup():
+ global _cwd
+ _cwd = os.getcwd()
+
+def teardown():
+ os.chdir(_cwd)
+
class Test_MercurialNonCachingCheckout(object):
repository_url = 'http://bitbucket.org/ctb/pony-build-hg-test/'
@@ -44,7 +52,7 @@ def create_cache_location(repository_url):
print 'calculated repository dirname as:', repository_dirname
- repository_cache = pony_client.guess_cache_dir(repository_dirname)
+ (_, repository_cache) = pony_client.guess_cache_dir(repository_dirname)
assert repository_cache.startswith(temp_cache_location)
# this will create 'the_cache' directory that contains individual
View
163 client/test_client/test_svn_client.py
@@ -0,0 +1,163 @@
+"""
+svn VCS client tests.
+"""
+import sys
+import os, os.path
+import shutil
+import tempfile
+import pprint
+import urlparse
+
+import pony_client
+from pony_client import SvnCheckout, TempDirectoryContext, _run_command
+
+_cwd = None
+def setup():
+ global _cwd
+ _cwd = os.getcwd()
+
+def teardown():
+ os.chdir(_cwd)
+
+class Test_SvnNonCachingCheckout(object):
+ repository_url = 'http://pony-build.googlecode.com/svn/pony-build-svn-test'
+
+ def setup(self):
+ # create a context within which to run the SvnCheckout command
+ self.context = TempDirectoryContext()
+ self.context.initialize()
+
+ def teardown(self):
+ self.context.finish()
+
+ def test_basic(self):
+ "Run the SvnCheckout command w/o caching and verify it."
+ command = SvnCheckout('pony-build-svn-test', self.repository_url,
+ use_cache=False)
+ command.verbose = True
+ command.run(self.context)
+
+ pprint.pprint(command.get_results()) # debugging output
+
+ os.chdir(self.context.tempdir)
+ assert os.path.exists(os.path.join('pony-build-svn-test', 'test1'))
+ assert os.path.exists(os.path.join('pony-build-svn-test', 'test2'))
+
+
+def create_cache_location(repository_url):
+ # use os.environ to specify a new place for VCS cache stuff
+ temp_cache_parent = tempfile.mkdtemp()
+ temp_cache_location = os.path.join(temp_cache_parent, "the_cache")
+ os.environ['PONY_BUILD_CACHE'] = temp_cache_location
+
+ # figure out what the end checkout result should be
+ repository_path = urlparse.urlparse(repository_url)[2]
+ repository_dirname = repository_path.rstrip('/').split('/')[-1]
+
+ print 'calculated repository dirname as:', repository_dirname
+
+ (_, repository_cache) = pony_client.guess_cache_dir(repository_dirname)
+ assert repository_cache.startswith(temp_cache_location)
+
+ # this will create 'the_cache' directory that contains individual
+ # pkg caches.
+ pony_client.create_cache_dir(repository_cache, repository_dirname)
+ assert os.path.isdir(temp_cache_location)
+
+ return (temp_cache_parent, temp_cache_location)
+
+
+class Test_SvnCachingCheckout(object):
+ repository_url = 'http://pony-build.googlecode.com/svn/pony-build-svn-test'
+
+ def setup(self):
+ # create a context within which to run the SvnCheckout command
+ self.context = TempDirectoryContext()
+ self.context.initialize()
+
+ (cache_parent, cache_dir) = create_cache_location(self.repository_url)
+ self.cache_parent = cache_parent
+
+ def teardown(self):
+ self.context.finish()
+ del os.environ['PONY_BUILD_CACHE']
+
+ shutil.rmtree(self.cache_parent, ignore_errors=True)
+
+ def test_basic(self):
+ "Run the SvnCheckout command and verify that it works."
+ command = SvnCheckout('pony-build-svn-test', self.repository_url)
+ command.verbose = True
+ command.run(self.context)
+
+ pprint.pprint(command.get_results()) # debugging output
+
+ cwd = os.getcwd()
+ os.chdir(self.context.tempdir)
+ try:
+ assert os.path.exists(os.path.join('pony-build-svn-test', 'test1'))
+ assert os.path.exists(os.path.join('pony-build-svn-test', 'test2'))
+ finally:
+ os.chdir(cwd)
+
+
+class Test_SvnCachingUpdate(object):
+ repository_url = 'http://pony-build.googlecode.com/svn/pony-build-svn-test'
+
+ def setup(self):
+ # create a context within which to run the SvnCheckout command
+ self.context = TempDirectoryContext()
+ self.context.initialize()
+
+ (cache_parent, cache_dir) = create_cache_location(self.repository_url)
+ self.cache_parent = cache_parent
+
+ cwd = os.getcwd() # save current directory
+
+ #
+ # next, we want to set up the cached repository so that it contains an
+ # old checkout.
+ #
+
+ os.chdir(cache_dir)
+
+ # now, check out the test svn repository.
+ (ret, out, err) = _run_command(['svn', 'checkout',
+ self.repository_url,
+ 'pony-build-svn-test'])
+ assert ret == 0, (out, err)
+
+ # forcibly check out the first revision, instead of the second.
+ (ret, out, err) = _run_command(['svn', 'update', '-r2'],
+ cwd='pony-build-svn-test')
+ assert ret == 0, (out, err)
+ assert os.path.exists(os.path.join('pony-build-svn-test', 'test1'))
+ assert not os.path.exists(os.path.join('pony-build-svn-test', 'test2'))
+
+ os.chdir(cwd) # return to working dir.
+
+ def teardown(self):
+ self.context.finish()
+ del os.environ['PONY_BUILD_CACHE']
+
+ shutil.rmtree(self.cache_parent, ignore_errors=True)
+
+ def test_basic(self):
+ "Run the SvnCheckout command and verify that it updates right."
+ command = SvnCheckout('pony-build-svn-test', self.repository_url)
+ command.verbose = True
+ command.run(self.context)
+
+ pprint.pprint(command.get_results()) # debugging output
+
+ cwd = os.getcwd()
+ os.chdir(self.context.tempdir)
+ try:
+ assert os.path.exists(os.path.join('pony-build-svn-test', 'test1'))
+ assert os.path.exists(os.path.join('pony-build-svn-test', 'test2'))
+ finally:
+ os.chdir(cwd)
+
+
+
+
View
1  examples/change-receiver/github-notify.json
@@ -0,0 +1 @@
+{"commits":[{"modified":["IDEAS"],"url":"http://github.com/ctb/pony-build/commit/e95f152f6d4e50da340d361a41789e0f0b904d56","message":"test","author":{"email":"t@titus-browns-macbook-2.local","name":"Titus Brown"},"timestamp":"2009-12-04T17:03:04-08:00","removed":[],"id":"e95f152f6d4e50da340d361a41789e0f0b904d56","added":[]}],"repository":{"forks":4,"description": "CI for Python","url":"http://github.com/ctb/pony-build","fork":false,"watchers":31,"private":false,"homepage":"","owner":{"email":"titus@idyll.org","name":"ctb"},"name":"pony-build","open_issues":1},"ref":"refs/heads/test","before":"cd759759417253f630123e89684e9fd29a4d9225","after":"e95f152f6d4e50da340d361a41789e0f0b904d56"}
View
49 examples/change-receiver/github-receiver.cgi
@@ -0,0 +1,49 @@
+#! /u/t/dev/venv/bin/python
+"""
+A simple webhook receiver for github notification events.
+"""
+import sys
+
+import pprint
+
+import quixote
+from quixote.directory import Directory
+from quixote.server.cgi_server import run
+from quixote.publish import Publisher
+
+import json
+from cgi import parse_qs
+
+def process_notify_data(payload):
+ pprint.pprint(payload)
+
+ print 'branch info is:', payload['ref']
+ print 'repository name:', payload['repository']['name']
+ print 'repository url:', payload['repository']['url']
+
+ return
+
+class GithubSubscriber(Directory):
+ _q_exports = [ '' ]
+
+ def _q_index(self):
+ request = quixote.get_request()
+ form = request.form
+
+ if 'payload' in form:
+ payload = form['payload']
+ payload = json.loads(payload)
+
+ process_notify_data(payload)
+
+ response = request.response
+ response.set_status(204)
+ else:
+ return "ok, but got nothing?"
+
+def create_publisher():
+ # sets global Quixote publisher
+ return Publisher(GithubSubscriber(), display_exceptions='plain')
+
+if __name__ == '__main__':
+ run(create_publisher)
View
14 examples/change-receiver/test-post-github-notify.py
@@ -0,0 +1,14 @@
+#! /usr/bin/env python
+import sys
+import httplib
+from urlparse import urlparse
+import urllib
+
+url = sys.argv[1]
+
+package = open('github-notify.json').read()
+d = dict(payload=package)
+
+print urllib.urlencode(d)
+
+print urllib.urlopen(url, urllib.urlencode(d)).read()
View
5 pony_build/coordinator.py
@@ -51,7 +51,10 @@ def build_tagset(client_info, no_arch=False, no_host=False):
host = client_info['host']
package = client_info['package']
- tags = list(client_info['tags'])
+ if client_info['tags'] is not None:
+ tags = list(client_info['tags'])
+ else:
+ tags = []
tags.append('__package=' + package)
if not no_arch:
View
35 pony_build/qx_web/util.py
@@ -1,35 +0,0 @@
-import os.path
-import jinja2
-import datetime
-
-###
-
-# jinja2 prep
-
-thisdir = os.path.dirname(__file__)
-templatesdir = os.path.join(thisdir, 'templates')
-templatesdir = os.path.abspath(templatesdir)
-
-loader = jinja2.FileSystemLoader(templatesdir)
-env = jinja2.Environment(loader=loader)
-
-###
-
-day_diff = datetime.timedelta(1)
-hour_diff = datetime.timedelta(0, 3600)
-min_diff = datetime.timedelta(0, 60)
-
-def format_timestamp(t):
- dt = datetime.datetime.fromtimestamp(t)
- now = datetime.datetime.now()
-
- diff = now - dt
- if diff < min_diff:
- return dt.strftime("less than a minute ago (%I:%M %p)")
- elif diff < hour_diff:
- return dt.strftime("less than an hour ago (%I:%M %p)")
- elif diff < day_diff:
- return dt.strftime("less than a day ago (%I:%M %p)")
-
- return dt.strftime("%A, %d %B %Y, %I:%M %p")
-
View
42 pony_build/server.py
@@ -113,50 +113,11 @@ def _handle_upload(self):
message = 'upload attempt, but no upload content?!'
self._send_html_response(code, message)
-
- def _handle_notify(self):
- """Handle webhook notification, currently only from github."""
-
- data = ''
-
- content_length = self.headers.getheader('content-length')
- if content_length:
- content_length = int(content_length)
- data = self.rfile.read(content_length)
-
- qs = {}
- if '?' in self.path:
- url, qs = self.path.split('?', 1)
- qs = parse_qs(qs)
-
- format = 'unknown'
- package = ''
- try:
- format = qs.get('format')[0]
- package = qs.get('package')[0]
- except (TypeError, ValueError, KeyError):
- pass
-
- if not package:
- self._send_html_response(400, missing_package)
-
- if format == 'github': # @CTB hardcoded github webhook...!
- post_d = parse_qs(data)
- payload = post_d.get('payload')[0]
- payload = json.loads(payload)
-
- data = payload
-
- _coordinator.notify_of_changes(package, format, data)
-
- self._send_html_response(200, "received")
-
def handle(self):
"""
Handle:
/xmlrpc => SimpleXMLRPCServer
/upload => self._handle_upload
- /notify => self._handle_notify
all else => WSGI app for Web UI
"""
self.raw_requestline = self.rfile.readline()
@@ -182,9 +143,6 @@ def handle(self):
elif self.path.startswith('/upload?'):
return self._handle_upload()
- elif self.path.startswith('/notify'):
- return self._handle_notify()
-
## else:
handler = ServerHandler(
View
69 pony_build/tests/test_webhook_notify.py
@@ -1,69 +0,0 @@
-import os
-import shelve
-import time
-import json # req python 2.6
-import urllib
-
-import testutil
-from twill.commands import *
-
-from pony_build.coordinator import PonyBuildCoordinator
-
-###
-
-rpc_url = None
-DB_TEST_FILE=os.path.join(os.path.dirname(__file__), 'tests.db')
-
-github_data = dict(payload='{"commits":[{"modified":["IDEAS"],"url":"http://github.com/ctb/pony-build/commit/e95f152f6d4e50da340d361a41789e0f0b904d56","message":"test","author":{"email":"t@titus-browns-macbook-2.local","name":"Titus Brown"},"timestamp":"2009-12-04T17:03:04-08:00","removed":[],"id":"e95f152f6d4e50da340d361a41789e0f0b904d56","added":[]}],"repository":{"forks":4,"description": "CI for Python","url":"http://github.com/ctb/pony-build","fork":false,"watchers":31,"private":false,"homepage":"","owner":{"email":"titus@idyll.org","name":"ctb"},"name":"pony-build","open_issues":1},"ref":"refs/heads/test","before":"cd759759417253f630123e89684e9fd29a4d9225","after":"e95f152f6d4e50da340d361a41789e0f0b904d56"}')
-
-
-###
-
-def setup():
- testutil.run_server(DB_TEST_FILE)
- assert testutil._server_url
-
-def teardown():
- testutil.kill_server()
-
-def test_basic():
- go(testutil._server_url)
- go('./notify')
- code(400)
-
-def test_github_notify():
- fp = urllib.urlopen(testutil._server_url + \
- 'notify?format=github&package=pygr',
- urllib.urlencode(github_data))
-
- received = fp.read()
- assert received == 'received', (received,)
-
-def test_nopackage_notify():
- fp = urllib.urlopen(testutil._server_url + 'notify?format=github',
- urllib.urlencode(github_data))
-
- assert fp.getcode() == 400
- received = fp.read()
- assert received.startswith('missing'), received
-
-def test_coordinator_interface():
- pbc = PonyBuildCoordinator({})
-
-
- class ProcessChange(object):
- def __init__(self, package_name):
- self.got_change = False
- self.package_name = package_name
- def __call__(self, package, format, info):
- self.got_change = True
- assert self.package_name == package, (package, self.package_name,)
-
- package_name = 'thepackage'
- consumer = ProcessChange(package_name)
- pbc.add_change_consumer(package_name, consumer)
-
- # this should ultimately call 'consumer'
- pbc.notify_of_changes(package_name, 'generic', None)
- assert consumer.got_change
-
View
8 pony_build/tests/testutil.py
@@ -20,7 +20,7 @@ def run_server_wsgi_intercept(dbfilename):
port = 80
from pony_build import server, coordinator, dbsqlite
- from pony_build.qx_web import create_publisher, urls
+ from pony_build.web import create_publisher, urls
dbfile = dbsqlite.open_shelf(dbfilename)
dbfile = coordinator.IntDictWrapper(dbfile)
@@ -58,12 +58,12 @@ def run_server(DB_FILE, PORT=None):
if PORT is None:
PORT = int(os.environ.get('PB_TEST_PORT', DEFAULT_PORT))
- print 'STARTING:', sys.executable, 'pony_build.qx_web.run', os.getcwd()
+ print 'STARTING:', sys.executable, 'pony_build.web.run', os.getcwd()
print [sys.executable, '-u',
- '-m', 'pony_build.qx_web.run', DB_FILE,
+ '-m', 'pony_build.web.run', DB_FILE,
'-p', str(PORT)],
process = subprocess.Popen([sys.executable, '-u',
- '-m', 'pony_build.qx_web.run', DB_FILE,
+ '-m', 'pony_build.web.run', DB_FILE,
'-p', str(PORT)],
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE)
View
0  pony_build/qx_web/__init__.py → pony_build/web/__init__.py
File renamed without changes
View
2  pony_build/qx_web/run.py → pony_build/web/run.py
@@ -1,5 +1,5 @@
from optparse import OptionParser
-from .. import qx_web
+from .. import web as qx_web
if __name__ == '__main__':
# import figleaf
View
0  pony_build/qx_web/templates/base.html → pony_build/web/templates/base.html
File renamed without changes
View
0  pony_build/qx_web/templates/empty.html → pony_build/web/templates/empty.html
File renamed without changes
View
0  pony_build/qx_web/templates/feed_base.html → pony_build/web/templates/feed_base.html
File renamed without changes
View
0  pony_build/qx_web/templates/feed_generic_index.html → pony_build/web/templates/feed_generic_index.html
File renamed without changes
View
0  .../qx_web/templates/feed_generic_package_index.html → ...ild/web/templates/feed_generic_package_index.html
File renamed without changes
View
0  pony_build/qx_web/templates/feed_index.html → pony_build/web/templates/feed_index.html
File renamed without changes
View
0  pony_build/qx_web/templates/forms.html → pony_build/web/templates/forms.html
File renamed without changes
View
0  pony_build/qx_web/templates/img/background.png → pony_build/web/templates/img/background.png
File renamed without changes
View
0  pony_build/qx_web/templates/img/dark_line.png → pony_build/web/templates/img/dark_line.png
File renamed without changes
View
0  pony_build/qx_web/templates/img/light_line.png → pony_build/web/templates/img/light_line.png
File renamed without changes
View
0  pony_build/qx_web/templates/package_all.html → pony_build/web/templates/package_all.html
File renamed without changes
View
0  pony_build/qx_web/templates/package_base.html → pony_build/web/templates/package_base.html
File renamed without changes
View
0  pony_build/qx_web/templates/package_summary.html → pony_build/web/templates/package_summary.html
File renamed without changes
View
0  pony_build/qx_web/templates/results_base.html → pony_build/web/templates/results_base.html
File renamed without changes
View
0  pony_build/qx_web/templates/results_files_index.html → pony_build/web/templates/results_files_index.html
File renamed without changes
View
0  pony_build/qx_web/templates/results_index.html → pony_build/web/templates/results_index.html
File renamed without changes
View
0  pony_build/qx_web/templates/results_inspect.html → pony_build/web/templates/results_inspect.html
File renamed without changes
View
0  pony_build/qx_web/templates/style.css → pony_build/web/templates/style.css
File renamed without changes
View
0  pony_build/qx_web/templates/top_index.html → pony_build/web/templates/top_index.html
File renamed without changes
View
0  pony_build/qx_web/urls.py → pony_build/web/urls.py
File renamed without changes
View
48 pony_build/web/util.py
@@ -0,0 +1,48 @@
+import os.path
+import jinja2
+import datetime
+import math
+
+###
+
+# jinja2 prep
+
+thisdir = os.path.dirname(__file__)
+templatesdir = os.path.join(thisdir, 'templates')
+templatesdir = os.path.abspath(templatesdir)
+
+loader = jinja2.FileSystemLoader(templatesdir)
+env = jinja2.Environment(loader=loader)
+
+###
+
+day_diff = datetime.timedelta(1)
+hour_diff = datetime.timedelta(0, 3600)
+elevenmin_diff = datetime.timedelta(0, 660)
+min_diff = datetime.timedelta(0, 60)
+
+def format_timestamp(t):
+ dt = datetime.datetime.fromtimestamp(t)
+ now = datetime.datetime.now()
+
+ diff = now - dt
+ minutesSince = int(math.floor(diff.seconds / 60))
+
+ if diff > day_diff:
+ return dt.strftime("%A, %d %B %Y, %I:%M %p")
+ elif diff > hour_diff:
+ timeSince = (minutesSince / 60) + 1
+ if timeSince == 24:
+ timeSince = "a day"
+ else:
+ timeSince = str(timeSince) + " hours"
+ return "less than " + timeSince + " ago " + dt.strftime("(%I:%M %p)")
+ elif diff > elevenmin_diff:
+ timeSince = ((minutesSince / 10) + 1 ) * 10
+ if timeSince == 60:
+ timeSince = "an hour"
+ else:
+ timeSince = str(timeSince) + " minutes"
+ return "less than " + timeSince + " ago " + dt.strftime("(%I:%M %p)")
+
+ return str(minutesSince) + " minutes ago " + dt.strftime("(%I:%M %p)")
View
4 setup.py
@@ -15,7 +15,7 @@
root_dir = os.path.dirname(__file__)
if root_dir != '':
os.chdir(root_dir)
-templates_dir = os.path.join('pony_build', 'qx_web', 'templates')
+templates_dir = os.path.join('pony_build', 'web', 'templates')
data_files = []
for dirpath, dirnames, filenames in os.walk(templates_dir):
@@ -34,7 +34,7 @@
author_email = 't@idyll.org',
url = 'http://github.com/ctb/pony-build',
license = 'BSD',
- packages = ['pony_build', 'pony_build.qx_web'],
+ packages = ['pony_build', 'pony_build.web'],
data_files = data_files,
test_suite = 'nose.collector',
)
Please sign in to comment.
Something went wrong with that request. Please try again.