Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Improved script for automatic extension testing

  • Loading branch information...
commit 3a80ecc66057d97e3a0ceb40e99ea7bde035381b 1 parent a7d83a9
@mitsuhiko authored
Showing with 216 additions and 33 deletions.
  1. +4 −1 Makefile
  2. +212 −32 tests/flaskext_test.py
View
5 Makefile
@@ -1,10 +1,13 @@
-.PHONY: clean-pyc test upload-docs docs
+.PHONY: clean-pyc ext-test test upload-docs docs
all: clean-pyc test
test:
python setup.py test
+ext-test:
+ python tests/flaskext_test.py --browse
+
release:
python setup.py release sdist upload
View
244 tests/flaskext_test.py
@@ -11,7 +11,14 @@
from __future__ import with_statement
-import tempfile, subprocess, urllib2, os
+import os
+import sys
+import shutil
+import urllib2
+import tempfile
+import subprocess
+import argparse
+from cStringIO import StringIO
from flask import json
@@ -19,24 +26,136 @@
from setuptools.archive_util import unpack_archive
flask_svc_url = 'http://flask.pocoo.org/extensions/'
-tdir = tempfile.mkdtemp()
+if sys.platform == 'darwin':
+ _tempdir = '/private/tmp'
+else:
+ _tempdir = tempfile.gettempdir()
+tdir = _tempdir + '/flaskext-test'
+flaskdir = os.path.abspath(os.path.join(os.path.dirname(__file__),
+ '..'))
-def run_tests(checkout_dir):
- cmd = ['tox']
- return subprocess.call(cmd, cwd=checkout_dir,
- stdout=open(os.path.join(tdir, 'tox.log'), 'w'),
- stderr=subprocess.STDOUT)
+
+RESULT_TEMPATE = u'''\
+<!doctype html>
+<title>Flask-Extension Test Results</title>
+<style type=text/css>
+ body { font-family: 'Georgia', serif; font-size: 17px; color: #000; }
+ a { color: #004B6B; }
+ a:hover { color: #6D4100; }
+ h1, h2, h3 { font-family: 'Garamond', 'Georgia', serif; font-weight: normal; }
+ h1 { font-size: 30px; margin: 15px 0 5px 0; }
+ h2 { font-size: 24px; margin: 15px 0 5px 0; }
+ h3 { font-size: 19px; margin: 15px 0 5px 0; }
+ textarea, code,
+ pre { font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono',
+ 'Bitstream Vera Sans Mono', monospace!important; font-size: 15px;
+ background: #eee; }
+ pre { padding: 7px 15px; line-height: 1.3; }
+ p { line-height: 1.4; }
+ table { border: 1px solid black; border-collapse: collapse;
+ margin: 15px 0; }
+ td, th { border: 1px solid black; padding: 4px 10px;
+ text-align: left; }
+ th { background: #eee; font-weight: normal; }
+ tr.success { background: #D3F5CC; }
+ tr.failed { background: #F5D2CB; }
+</style>
+<h1>Flask-Extension Test Results</h1>
+<p>
+ This page contains the detailed test results for the test run of
+ all {{ 'approved' if approved }} Flask extensions.
+<h2>Summary</h2>
+<table class=results>
+ <thead>
+ <tr>
+ <th>Extension
+ <th>Version
+ <th>Author
+ <th>License
+ <th>Outcome
+ </tr>
+ </thead>
+ <tbody>
+ {%- for result in results %}
+ {% set outcome = 'success' if result.success else 'failed' %}
+ <tr class={{ outcome }}>
+ <th>{{ result.name }}
+ <td>{{ result.version }}
+ <td>{{ result.author }}
+ <td>{{ result.license }}
+ <td>{{ outcome }}
+ </tr>
+ {%- endfor %}
+ </tbody>
+</table>
+<h2>Test Logs</h2>
+<p>Detailed test logs for all tests on all platforms:
+{%- for result in results %}
+ {%- for iptr, log in result.logs|dictsort %}
+ <h3>{{ result.name }} - {{ result.version }} [{{ iptr }}]</h3>
+ <pre>{{ log }}</pre>
+ {%- endfor %}
+{%- endfor %}
+'''
+
+
+def log(msg, *args):
+ print '[EXTTEST]', msg % args
+
+
+class TestResult(object):
+
+ def __init__(self, name, folder, statuscode, interpreters):
+ intrptr = os.path.join(folder, '.tox/%s/bin/python'
+ % interpreters[0])
+ self.statuscode = statuscode
+ self.folder = folder
+ self.success = statuscode == 0
+
+ def fetch(field):
+ try:
+ c = subprocess.Popen([intrptr, 'setup.py',
+ '--' + field], cwd=folder,
+ stdout=subprocess.PIPE)
+ return c.communicate()[0].strip()
+ except OSError:
+ return '?'
+ self.name = name
+ self.license = fetch('license')
+ self.author = fetch('author')
+ self.version = fetch('version')
+
+ self.logs = {}
+ for interpreter in interpreters:
+ logfile = os.path.join(folder, '.tox/%s/log/test.log'
+ % interpreter)
+ if os.path.isfile(logfile):
+ self.logs[interpreter] = open(logfile).read()
+ else:
+ self.logs[interpreter] = ''
+
+
+def create_tdir():
+ try:
+ shutil.rmtree(tdir)
+ except Exception:
+ pass
+ os.mkdir(tdir)
+
+
+def package_flask():
+ distfolder = tdir + '/.flask-dist'
+ c = subprocess.Popen(['python', 'setup.py', 'sdist', '--formats=gztar',
+ '--dist', distfolder], cwd=flaskdir)
+ c.wait()
+ return os.path.join(distfolder, os.listdir(distfolder)[0])
def get_test_command(checkout_dir):
- files = set(os.listdir(checkout_dir))
- if 'Makefile' in files:
+ if os.path.isfile(checkout_dir + '/Makefile'):
return 'make test'
- elif 'conftest.py' in files:
- return 'py.test'
- else:
- return 'nosetests'
+ return 'python setup.py test'
def fetch_extensions_list():
@@ -47,50 +166,111 @@ def fetch_extensions_list():
yield ext
-def checkout_extension(ext):
- name = ext['name']
+def checkout_extension(name):
+ log('Downloading extension %s to temporary folder', name)
root = os.path.join(tdir, name)
os.mkdir(root)
- checkout_path = PackageIndex().download(ext['name'], root)
+ checkout_path = PackageIndex().download(name, root)
+
unpack_archive(checkout_path, root)
path = None
for fn in os.listdir(root):
path = os.path.join(root, fn)
if os.path.isdir(path):
break
+ log('Downloaded to %s', path)
return path
tox_template = """[tox]
-envlist=py26
+envlist=%(env)s
[testenv]
-commands=
-%s
-downloadcache=
-%s
+deps=%(deps)s
+commands=bash flaskext-runtest.sh {envlogdir}/test.log
+downloadcache=%(cache)s
"""
-def create_tox_ini(checkout_path):
+
+def create_tox_ini(checkout_path, interpreters, flask_dep):
tox_path = os.path.join(checkout_path, 'tox.ini')
if not os.path.exists(tox_path):
with open(tox_path, 'w') as f:
- f.write(tox_template % (get_test_command(checkout_path), tdir))
+ f.write(tox_template % {
+ 'env': ','.join(interpreters),
+ 'cache': tdir,
+ 'deps': flask_dep
+ })
-# XXX command line
-only_approved = True
-def test_all_extensions(only_approved=only_approved):
+def iter_extensions(only_approved=True):
for ext in fetch_extensions_list():
if ext['approved'] or not only_approved:
- checkout_path = checkout_extension(ext)
- create_tox_ini(checkout_path)
- ret = run_tests(checkout_path)
- yield ext['name'], ret
+ yield ext['name']
+
+
+def test_extension(name, interpreters, flask_dep):
+ checkout_path = checkout_extension(name)
+ log('Running tests with tox in %s', checkout_path)
+
+ # figure out the test command and write a wrapper script. We
+ # can't write that directly into the tox ini because tox does
+ # not invoke the command from the shell so we have no chance
+ # to pipe the output into a logfile
+ test_command = get_test_command(checkout_path)
+ log('Test command: %s', test_command)
+ f = open(checkout_path + '/flaskext-runtest.sh', 'w')
+ f.write(test_command + ' &> "$1"\n')
+ f.close()
+
+ create_tox_ini(checkout_path, interpreters, flask_dep)
+ rv = subprocess.call(['tox'], cwd=checkout_path)
+ return TestResult(name, checkout_path, rv, interpreters)
+
+
+def run_tests(interpreters, only_approved=True):
+ results = {}
+ create_tdir()
+ log('Packaging Flask')
+ flask_dep = package_flask()
+ log('Running extension tests')
+ log('Temporary Environment: %s', tdir)
+ for name in iter_extensions(only_approved):
+ log('Testing %s', name)
+ result = test_extension(name, interpreters, flask_dep)
+ if result.success:
+ log('Extension test succeeded')
+ else:
+ log('Extension test failed')
+ results[name] = result
+ return results
+
+
+def render_results(results, approved):
+ from jinja2 import Template
+ items = results.values()
+ items.sort(key=lambda x: x.name.lower())
+ rv = Template(RESULT_TEMPATE, autoescape=True).render(results=items,
+ approved=approved)
+ fd, filename = tempfile.mkstemp(suffix='.html')
+ os.fdopen(fd, 'w').write(rv.encode('utf-8') + '\n')
+ return filename
+
def main():
- for name, ret in test_all_extensions():
- print name, ret
+ parser = argparse.ArgumentParser(description='Runs Flask extension tests')
+ parser.add_argument('--all', dest='all', action='store_true',
+ help='run against all extensions, not just approved')
+ parser.add_argument('--browse', dest='browse', action='store_true',
+ help='show browser with the result summary')
+ args = parser.parse_args()
+
+ results = run_tests(['py26'], not args.all)
+ filename = render_results(results, not args.all)
+ if args.browse:
+ import webbrowser
+ webbrowser.open('file:///' + filename.lstrip('/'))
+ print 'Results written to', filename
if __name__ == '__main__':
Please sign in to comment.
Something went wrong with that request. Please try again.