Skip to content

Commit

Permalink
Improved script for automatic extension testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@mitsuhiko
mitsuhiko committed Jul 25, 2010
1 parent a7d83a9 commit 3a80ecc
Show file tree
Hide file tree
Showing 2 changed files with 216 additions and 33 deletions.
5 changes: 4 additions & 1 deletion Makefile
View file
@@ -1,10 +1,13 @@
.PHONY: clean-pyc test upload-docs docs
.PHONY: clean-pyc ext-test test upload-docs docs

all: clean-pyc test

test:
python setup.py test

ext-test:
python tests/flaskext_test.py --browse

release:
python setup.py release sdist upload

Expand Down
244 changes: 212 additions & 32 deletions tests/flaskext_test.py
View file
Expand Up @@ -11,32 +11,151 @@

from __future__ import with_statement

import tempfile, subprocess, urllib2, os
import os
import sys
import shutil
import urllib2
import tempfile
import subprocess
import argparse
from cStringIO import StringIO

from flask import json

from setuptools.package_index import PackageIndex
from setuptools.archive_util import unpack_archive

flask_svc_url = 'http://flask.pocoo.org/extensions/'
tdir = tempfile.mkdtemp()

if sys.platform == 'darwin':
_tempdir = '/private/tmp'
else:
_tempdir = tempfile.gettempdir()
tdir = _tempdir + '/flaskext-test'
flaskdir = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..'))

def run_tests(checkout_dir):
cmd = ['tox']
return subprocess.call(cmd, cwd=checkout_dir,
stdout=open(os.path.join(tdir, 'tox.log'), 'w'),
stderr=subprocess.STDOUT)

RESULT_TEMPATE = u'''\
<!doctype html>
<title>Flask-Extension Test Results</title>
<style type=text/css>
body { font-family: 'Georgia', serif; font-size: 17px; color: #000; }
a { color: #004B6B; }
a:hover { color: #6D4100; }
h1, h2, h3 { font-family: 'Garamond', 'Georgia', serif; font-weight: normal; }
h1 { font-size: 30px; margin: 15px 0 5px 0; }
h2 { font-size: 24px; margin: 15px 0 5px 0; }
h3 { font-size: 19px; margin: 15px 0 5px 0; }
textarea, code,
pre { font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono',
'Bitstream Vera Sans Mono', monospace!important; font-size: 15px;
background: #eee; }
pre { padding: 7px 15px; line-height: 1.3; }
p { line-height: 1.4; }
table { border: 1px solid black; border-collapse: collapse;
margin: 15px 0; }
td, th { border: 1px solid black; padding: 4px 10px;
text-align: left; }
th { background: #eee; font-weight: normal; }
tr.success { background: #D3F5CC; }
tr.failed { background: #F5D2CB; }
</style>
<h1>Flask-Extension Test Results</h1>
<p>
This page contains the detailed test results for the test run of
all {{ 'approved' if approved }} Flask extensions.
<h2>Summary</h2>
<table class=results>
<thead>
<tr>
<th>Extension
<th>Version
<th>Author
<th>License
<th>Outcome
</tr>
</thead>
<tbody>
{%- for result in results %}
{% set outcome = 'success' if result.success else 'failed' %}
<tr class={{ outcome }}>
<th>{{ result.name }}
<td>{{ result.version }}
<td>{{ result.author }}
<td>{{ result.license }}
<td>{{ outcome }}
</tr>
{%- endfor %}
</tbody>
</table>
<h2>Test Logs</h2>
<p>Detailed test logs for all tests on all platforms:
{%- for result in results %}
{%- for iptr, log in result.logs|dictsort %}
<h3>{{ result.name }} - {{ result.version }} [{{ iptr }}]</h3>
<pre>{{ log }}</pre>
{%- endfor %}
{%- endfor %}
'''


def log(msg, *args):
print '[EXTTEST]', msg % args


class TestResult(object):

def __init__(self, name, folder, statuscode, interpreters):
intrptr = os.path.join(folder, '.tox/%s/bin/python'
% interpreters[0])
self.statuscode = statuscode
self.folder = folder
self.success = statuscode == 0

def fetch(field):
try:
c = subprocess.Popen([intrptr, 'setup.py',
'--' + field], cwd=folder,
stdout=subprocess.PIPE)
return c.communicate()[0].strip()
except OSError:
return '?'
self.name = name
self.license = fetch('license')
self.author = fetch('author')
self.version = fetch('version')

self.logs = {}
for interpreter in interpreters:
logfile = os.path.join(folder, '.tox/%s/log/test.log'
% interpreter)
if os.path.isfile(logfile):
self.logs[interpreter] = open(logfile).read()
else:
self.logs[interpreter] = ''


def create_tdir():
try:
shutil.rmtree(tdir)
except Exception:
pass
os.mkdir(tdir)


def package_flask():
distfolder = tdir + '/.flask-dist'
c = subprocess.Popen(['python', 'setup.py', 'sdist', '--formats=gztar',
'--dist', distfolder], cwd=flaskdir)
c.wait()
return os.path.join(distfolder, os.listdir(distfolder)[0])


def get_test_command(checkout_dir):
files = set(os.listdir(checkout_dir))
if 'Makefile' in files:
if os.path.isfile(checkout_dir + '/Makefile'):
return 'make test'
elif 'conftest.py' in files:
return 'py.test'
else:
return 'nosetests'
return 'python setup.py test'


def fetch_extensions_list():
Expand All @@ -47,50 +166,111 @@ def fetch_extensions_list():
yield ext


def checkout_extension(ext):
name = ext['name']
def checkout_extension(name):
log('Downloading extension %s to temporary folder', name)
root = os.path.join(tdir, name)
os.mkdir(root)
checkout_path = PackageIndex().download(ext['name'], root)
checkout_path = PackageIndex().download(name, root)

unpack_archive(checkout_path, root)
path = None
for fn in os.listdir(root):
path = os.path.join(root, fn)
if os.path.isdir(path):
break
log('Downloaded to %s', path)
return path


tox_template = """[tox]
envlist=py26
envlist=%(env)s
[testenv]
commands=
%s
downloadcache=
%s
deps=%(deps)s
commands=bash flaskext-runtest.sh {envlogdir}/test.log
downloadcache=%(cache)s
"""

def create_tox_ini(checkout_path):

def create_tox_ini(checkout_path, interpreters, flask_dep):
tox_path = os.path.join(checkout_path, 'tox.ini')
if not os.path.exists(tox_path):
with open(tox_path, 'w') as f:
f.write(tox_template % (get_test_command(checkout_path), tdir))
f.write(tox_template % {
'env': ','.join(interpreters),
'cache': tdir,
'deps': flask_dep
})

# XXX command line
only_approved = True

def test_all_extensions(only_approved=only_approved):
def iter_extensions(only_approved=True):
for ext in fetch_extensions_list():
if ext['approved'] or not only_approved:
checkout_path = checkout_extension(ext)
create_tox_ini(checkout_path)
ret = run_tests(checkout_path)
yield ext['name'], ret
yield ext['name']


def test_extension(name, interpreters, flask_dep):
checkout_path = checkout_extension(name)
log('Running tests with tox in %s', checkout_path)

# figure out the test command and write a wrapper script. We
# can't write that directly into the tox ini because tox does
# not invoke the command from the shell so we have no chance
# to pipe the output into a logfile
test_command = get_test_command(checkout_path)
log('Test command: %s', test_command)
f = open(checkout_path + '/flaskext-runtest.sh', 'w')
f.write(test_command + ' &> "$1"\n')
f.close()

create_tox_ini(checkout_path, interpreters, flask_dep)
rv = subprocess.call(['tox'], cwd=checkout_path)
return TestResult(name, checkout_path, rv, interpreters)


def run_tests(interpreters, only_approved=True):
results = {}
create_tdir()
log('Packaging Flask')
flask_dep = package_flask()
log('Running extension tests')
log('Temporary Environment: %s', tdir)
for name in iter_extensions(only_approved):
log('Testing %s', name)
result = test_extension(name, interpreters, flask_dep)
if result.success:
log('Extension test succeeded')
else:
log('Extension test failed')
results[name] = result
return results


def render_results(results, approved):
from jinja2 import Template
items = results.values()
items.sort(key=lambda x: x.name.lower())
rv = Template(RESULT_TEMPATE, autoescape=True).render(results=items,
approved=approved)
fd, filename = tempfile.mkstemp(suffix='.html')
os.fdopen(fd, 'w').write(rv.encode('utf-8') + '\n')
return filename


def main():
for name, ret in test_all_extensions():
print name, ret
parser = argparse.ArgumentParser(description='Runs Flask extension tests')
parser.add_argument('--all', dest='all', action='store_true',
help='run against all extensions, not just approved')
parser.add_argument('--browse', dest='browse', action='store_true',
help='show browser with the result summary')
args = parser.parse_args()

results = run_tests(['py26'], not args.all)
filename = render_results(results, not args.all)
if args.browse:
import webbrowser
webbrowser.open('file:///' + filename.lstrip('/'))
print 'Results written to', filename


if __name__ == '__main__':
Expand Down

0 comments on commit 3a80ecc

Please sign in to comment.