From 3a80ecc66057d97e3a0ceb40e99ea7bde035381b Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sun, 25 Jul 2010 17:33:45 +0200 Subject: [PATCH] Improved script for automatic extension testing --- Makefile | 5 +- tests/flaskext_test.py | 244 +++++++++++++++++++++++++++++++++++------ 2 files changed, 216 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index ba898b9ed9..8055e8ef6a 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,13 @@ -.PHONY: clean-pyc test upload-docs docs +.PHONY: clean-pyc ext-test test upload-docs docs all: clean-pyc test test: python setup.py test +ext-test: + python tests/flaskext_test.py --browse + release: python setup.py release sdist upload diff --git a/tests/flaskext_test.py b/tests/flaskext_test.py index aa4b7461b0..3614ad5859 100644 --- a/tests/flaskext_test.py +++ b/tests/flaskext_test.py @@ -11,7 +11,14 @@ from __future__ import with_statement -import tempfile, subprocess, urllib2, os +import os +import sys +import shutil +import urllib2 +import tempfile +import subprocess +import argparse +from cStringIO import StringIO from flask import json @@ -19,24 +26,136 @@ from setuptools.archive_util import unpack_archive flask_svc_url = 'http://flask.pocoo.org/extensions/' -tdir = tempfile.mkdtemp() +if sys.platform == 'darwin': + _tempdir = '/private/tmp' +else: + _tempdir = tempfile.gettempdir() +tdir = _tempdir + '/flaskext-test' +flaskdir = os.path.abspath(os.path.join(os.path.dirname(__file__), + '..')) -def run_tests(checkout_dir): - cmd = ['tox'] - return subprocess.call(cmd, cwd=checkout_dir, - stdout=open(os.path.join(tdir, 'tox.log'), 'w'), - stderr=subprocess.STDOUT) + +RESULT_TEMPATE = u'''\ + +Flask-Extension Test Results + +

Flask-Extension Test Results

+

+ This page contains the detailed test results for the test run of + all {{ 'approved' if approved }} Flask extensions. +

Summary

+ + + + + + + {%- for result in results %} + {% set outcome = 'success' if result.success else 'failed' %} + + + {%- endfor %} + +
Extension + Version + Author + License + Outcome +
{{ result.name }} + {{ result.version }} + {{ result.author }} + {{ result.license }} + {{ outcome }} +
+

Test Logs

+

Detailed test logs for all tests on all platforms: +{%- for result in results %} + {%- for iptr, log in result.logs|dictsort %} +

{{ result.name }} - {{ result.version }} [{{ iptr }}]

+ 
{{ log }}
+ {%- endfor %} +{%- endfor %} +''' + + +def log(msg, *args): + print '[EXTTEST]', msg % args + + +class TestResult(object): + + def __init__(self, name, folder, statuscode, interpreters): + intrptr = os.path.join(folder, '.tox/%s/bin/python' + % interpreters[0]) + self.statuscode = statuscode + self.folder = folder + self.success = statuscode == 0 + + def fetch(field): + try: + c = subprocess.Popen([intrptr, 'setup.py', + '--' + field], cwd=folder, + stdout=subprocess.PIPE) + return c.communicate()[0].strip() + except OSError: + return '?' + self.name = name + self.license = fetch('license') + self.author = fetch('author') + self.version = fetch('version') + + self.logs = {} + for interpreter in interpreters: + logfile = os.path.join(folder, '.tox/%s/log/test.log' + % interpreter) + if os.path.isfile(logfile): + self.logs[interpreter] = open(logfile).read() + else: + self.logs[interpreter] = '' + + +def create_tdir(): + try: + shutil.rmtree(tdir) + except Exception: + pass + os.mkdir(tdir) + + +def package_flask(): + distfolder = tdir + '/.flask-dist' + c = subprocess.Popen(['python', 'setup.py', 'sdist', '--formats=gztar', + '--dist', distfolder], cwd=flaskdir) + c.wait() + return os.path.join(distfolder, os.listdir(distfolder)[0]) def get_test_command(checkout_dir): - files = set(os.listdir(checkout_dir)) - if 'Makefile' in files: + if os.path.isfile(checkout_dir + '/Makefile'): return 'make test' - elif 'conftest.py' in files: - return 'py.test' - else: - return 'nosetests' + return 'python setup.py test' def fetch_extensions_list(): @@ -47,50 +166,111 @@ def fetch_extensions_list(): yield ext -def checkout_extension(ext): - name = ext['name'] +def checkout_extension(name): + log('Downloading extension %s to temporary folder', name) root = os.path.join(tdir, name) os.mkdir(root) - checkout_path = PackageIndex().download(ext['name'], root) + checkout_path = PackageIndex().download(name, root) + unpack_archive(checkout_path, root) path = None for fn in os.listdir(root): path = os.path.join(root, fn) if os.path.isdir(path): break + log('Downloaded to %s', path) return path tox_template = """[tox] -envlist=py26 +envlist=%(env)s [testenv] -commands= -%s -downloadcache= -%s +deps=%(deps)s +commands=bash flaskext-runtest.sh {envlogdir}/test.log +downloadcache=%(cache)s """ -def create_tox_ini(checkout_path): + +def create_tox_ini(checkout_path, interpreters, flask_dep): tox_path = os.path.join(checkout_path, 'tox.ini') if not os.path.exists(tox_path): with open(tox_path, 'w') as f: - f.write(tox_template % (get_test_command(checkout_path), tdir)) + f.write(tox_template % { + 'env': ','.join(interpreters), + 'cache': tdir, + 'deps': flask_dep + }) -# XXX command line -only_approved = True -def test_all_extensions(only_approved=only_approved): +def iter_extensions(only_approved=True): for ext in fetch_extensions_list(): if ext['approved'] or not only_approved: - checkout_path = checkout_extension(ext) - create_tox_ini(checkout_path) - ret = run_tests(checkout_path) - yield ext['name'], ret + yield ext['name'] + + +def test_extension(name, interpreters, flask_dep): + checkout_path = checkout_extension(name) + log('Running tests with tox in %s', checkout_path) + + # figure out the test command and write a wrapper script. We + # can't write that directly into the tox ini because tox does + # not invoke the command from the shell so we have no chance + # to pipe the output into a logfile + test_command = get_test_command(checkout_path) + log('Test command: %s', test_command) + f = open(checkout_path + '/flaskext-runtest.sh', 'w') + f.write(test_command + ' &> "$1"\n') + f.close() + + create_tox_ini(checkout_path, interpreters, flask_dep) + rv = subprocess.call(['tox'], cwd=checkout_path) + return TestResult(name, checkout_path, rv, interpreters) + + +def run_tests(interpreters, only_approved=True): + results = {} + create_tdir() + log('Packaging Flask') + flask_dep = package_flask() + log('Running extension tests') + log('Temporary Environment: %s', tdir) + for name in iter_extensions(only_approved): + log('Testing %s', name) + result = test_extension(name, interpreters, flask_dep) + if result.success: + log('Extension test succeeded') + else: + log('Extension test failed') + results[name] = result + return results + + +def render_results(results, approved): + from jinja2 import Template + items = results.values() + items.sort(key=lambda x: x.name.lower()) + rv = Template(RESULT_TEMPATE, autoescape=True).render(results=items, + approved=approved) + fd, filename = tempfile.mkstemp(suffix='.html') + os.fdopen(fd, 'w').write(rv.encode('utf-8') + '\n') + return filename + def main(): - for name, ret in test_all_extensions(): - print name, ret + parser = argparse.ArgumentParser(description='Runs Flask extension tests') + parser.add_argument('--all', dest='all', action='store_true', + help='run against all extensions, not just approved') + parser.add_argument('--browse', dest='browse', action='store_true', + help='show browser with the result summary') + args = parser.parse_args() + + results = run_tests(['py26'], not args.all) + filename = render_results(results, not args.all) + if args.browse: + import webbrowser + webbrowser.open('file:///' + filename.lstrip('/')) + print 'Results written to', filename if __name__ == '__main__':