Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Improved script for automatic extension testing

  • Loading branch information...
commit 3a80ecc66057d97e3a0ceb40e99ea7bde035381b 1 parent a7d83a9
Armin Ronacher mitsuhiko authored

Showing 2 changed files with 216 additions and 33 deletions. Show diff stats Hide diff stats

  1. +4 1 Makefile
  2. +212 32 tests/flaskext_test.py
5 Makefile
... ... @@ -1,10 +1,13 @@
1   -.PHONY: clean-pyc test upload-docs docs
  1 +.PHONY: clean-pyc ext-test test upload-docs docs
2 2
3 3 all: clean-pyc test
4 4
5 5 test:
6 6 python setup.py test
7 7
  8 +ext-test:
  9 + python tests/flaskext_test.py --browse
  10 +
8 11 release:
9 12 python setup.py release sdist upload
10 13
244 tests/flaskext_test.py
@@ -11,7 +11,14 @@
11 11
12 12 from __future__ import with_statement
13 13
14   -import tempfile, subprocess, urllib2, os
  14 +import os
  15 +import sys
  16 +import shutil
  17 +import urllib2
  18 +import tempfile
  19 +import subprocess
  20 +import argparse
  21 +from cStringIO import StringIO
15 22
16 23 from flask import json
17 24
@@ -19,24 +26,136 @@
19 26 from setuptools.archive_util import unpack_archive
20 27
21 28 flask_svc_url = 'http://flask.pocoo.org/extensions/'
22   -tdir = tempfile.mkdtemp()
23 29
  30 +if sys.platform == 'darwin':
  31 + _tempdir = '/private/tmp'
  32 +else:
  33 + _tempdir = tempfile.gettempdir()
  34 +tdir = _tempdir + '/flaskext-test'
  35 +flaskdir = os.path.abspath(os.path.join(os.path.dirname(__file__),
  36 + '..'))
24 37
25   -def run_tests(checkout_dir):
26   - cmd = ['tox']
27   - return subprocess.call(cmd, cwd=checkout_dir,
28   - stdout=open(os.path.join(tdir, 'tox.log'), 'w'),
29   - stderr=subprocess.STDOUT)
  38 +
  39 +RESULT_TEMPATE = u'''\
  40 +<!doctype html>
  41 +<title>Flask-Extension Test Results</title>
  42 +<style type=text/css>
  43 + body { font-family: 'Georgia', serif; font-size: 17px; color: #000; }
  44 + a { color: #004B6B; }
  45 + a:hover { color: #6D4100; }
  46 + h1, h2, h3 { font-family: 'Garamond', 'Georgia', serif; font-weight: normal; }
  47 + h1 { font-size: 30px; margin: 15px 0 5px 0; }
  48 + h2 { font-size: 24px; margin: 15px 0 5px 0; }
  49 + h3 { font-size: 19px; margin: 15px 0 5px 0; }
  50 + textarea, code,
  51 + pre { font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono',
  52 + 'Bitstream Vera Sans Mono', monospace!important; font-size: 15px;
  53 + background: #eee; }
  54 + pre { padding: 7px 15px; line-height: 1.3; }
  55 + p { line-height: 1.4; }
  56 + table { border: 1px solid black; border-collapse: collapse;
  57 + margin: 15px 0; }
  58 + td, th { border: 1px solid black; padding: 4px 10px;
  59 + text-align: left; }
  60 + th { background: #eee; font-weight: normal; }
  61 + tr.success { background: #D3F5CC; }
  62 + tr.failed { background: #F5D2CB; }
  63 +</style>
  64 +<h1>Flask-Extension Test Results</h1>
  65 +<p>
  66 + This page contains the detailed test results for the test run of
  67 + all {{ 'approved' if approved }} Flask extensions.
  68 +<h2>Summary</h2>
  69 +<table class=results>
  70 + <thead>
  71 + <tr>
  72 + <th>Extension
  73 + <th>Version
  74 + <th>Author
  75 + <th>License
  76 + <th>Outcome
  77 + </tr>
  78 + </thead>
  79 + <tbody>
  80 + {%- for result in results %}
  81 + {% set outcome = 'success' if result.success else 'failed' %}
  82 + <tr class={{ outcome }}>
  83 + <th>{{ result.name }}
  84 + <td>{{ result.version }}
  85 + <td>{{ result.author }}
  86 + <td>{{ result.license }}
  87 + <td>{{ outcome }}
  88 + </tr>
  89 + {%- endfor %}
  90 + </tbody>
  91 +</table>
  92 +<h2>Test Logs</h2>
  93 +<p>Detailed test logs for all tests on all platforms:
  94 +{%- for result in results %}
  95 + {%- for iptr, log in result.logs|dictsort %}
  96 + <h3>{{ result.name }} - {{ result.version }} [{{ iptr }}]</h3>
  97 + <pre>{{ log }}</pre>
  98 + {%- endfor %}
  99 +{%- endfor %}
  100 +'''
  101 +
  102 +
  103 +def log(msg, *args):
  104 + print '[EXTTEST]', msg % args
  105 +
  106 +
  107 +class TestResult(object):
  108 +
  109 + def __init__(self, name, folder, statuscode, interpreters):
  110 + intrptr = os.path.join(folder, '.tox/%s/bin/python'
  111 + % interpreters[0])
  112 + self.statuscode = statuscode
  113 + self.folder = folder
  114 + self.success = statuscode == 0
  115 +
  116 + def fetch(field):
  117 + try:
  118 + c = subprocess.Popen([intrptr, 'setup.py',
  119 + '--' + field], cwd=folder,
  120 + stdout=subprocess.PIPE)
  121 + return c.communicate()[0].strip()
  122 + except OSError:
  123 + return '?'
  124 + self.name = name
  125 + self.license = fetch('license')
  126 + self.author = fetch('author')
  127 + self.version = fetch('version')
  128 +
  129 + self.logs = {}
  130 + for interpreter in interpreters:
  131 + logfile = os.path.join(folder, '.tox/%s/log/test.log'
  132 + % interpreter)
  133 + if os.path.isfile(logfile):
  134 + self.logs[interpreter] = open(logfile).read()
  135 + else:
  136 + self.logs[interpreter] = ''
  137 +
  138 +
  139 +def create_tdir():
  140 + try:
  141 + shutil.rmtree(tdir)
  142 + except Exception:
  143 + pass
  144 + os.mkdir(tdir)
  145 +
  146 +
  147 +def package_flask():
  148 + distfolder = tdir + '/.flask-dist'
  149 + c = subprocess.Popen(['python', 'setup.py', 'sdist', '--formats=gztar',
  150 + '--dist', distfolder], cwd=flaskdir)
  151 + c.wait()
  152 + return os.path.join(distfolder, os.listdir(distfolder)[0])
30 153
31 154
32 155 def get_test_command(checkout_dir):
33   - files = set(os.listdir(checkout_dir))
34   - if 'Makefile' in files:
  156 + if os.path.isfile(checkout_dir + '/Makefile'):
35 157 return 'make test'
36   - elif 'conftest.py' in files:
37   - return 'py.test'
38   - else:
39   - return 'nosetests'
  158 + return 'python setup.py test'
40 159
41 160
42 161 def fetch_extensions_list():
@@ -47,50 +166,111 @@ def fetch_extensions_list():
47 166 yield ext
48 167
49 168
50   -def checkout_extension(ext):
51   - name = ext['name']
  169 +def checkout_extension(name):
  170 + log('Downloading extension %s to temporary folder', name)
52 171 root = os.path.join(tdir, name)
53 172 os.mkdir(root)
54   - checkout_path = PackageIndex().download(ext['name'], root)
  173 + checkout_path = PackageIndex().download(name, root)
  174 +
55 175 unpack_archive(checkout_path, root)
56 176 path = None
57 177 for fn in os.listdir(root):
58 178 path = os.path.join(root, fn)
59 179 if os.path.isdir(path):
60 180 break
  181 + log('Downloaded to %s', path)
61 182 return path
62 183
63 184
64 185 tox_template = """[tox]
65   -envlist=py26
  186 +envlist=%(env)s
66 187
67 188 [testenv]
68   -commands=
69   -%s
70   -downloadcache=
71   -%s
  189 +deps=%(deps)s
  190 +commands=bash flaskext-runtest.sh {envlogdir}/test.log
  191 +downloadcache=%(cache)s
72 192 """
73 193
74   -def create_tox_ini(checkout_path):
  194 +
  195 +def create_tox_ini(checkout_path, interpreters, flask_dep):
75 196 tox_path = os.path.join(checkout_path, 'tox.ini')
76 197 if not os.path.exists(tox_path):
77 198 with open(tox_path, 'w') as f:
78   - f.write(tox_template % (get_test_command(checkout_path), tdir))
  199 + f.write(tox_template % {
  200 + 'env': ','.join(interpreters),
  201 + 'cache': tdir,
  202 + 'deps': flask_dep
  203 + })
79 204
80   -# XXX command line
81   -only_approved = True
82 205
83   -def test_all_extensions(only_approved=only_approved):
  206 +def iter_extensions(only_approved=True):
84 207 for ext in fetch_extensions_list():
85 208 if ext['approved'] or not only_approved:
86   - checkout_path = checkout_extension(ext)
87   - create_tox_ini(checkout_path)
88   - ret = run_tests(checkout_path)
89   - yield ext['name'], ret
  209 + yield ext['name']
  210 +
  211 +
  212 +def test_extension(name, interpreters, flask_dep):
  213 + checkout_path = checkout_extension(name)
  214 + log('Running tests with tox in %s', checkout_path)
  215 +
  216 + # figure out the test command and write a wrapper script. We
  217 + # can't write that directly into the tox ini because tox does
  218 + # not invoke the command from the shell so we have no chance
  219 + # to pipe the output into a logfile
  220 + test_command = get_test_command(checkout_path)
  221 + log('Test command: %s', test_command)
  222 + f = open(checkout_path + '/flaskext-runtest.sh', 'w')
  223 + f.write(test_command + ' &> "$1"\n')
  224 + f.close()
  225 +
  226 + create_tox_ini(checkout_path, interpreters, flask_dep)
  227 + rv = subprocess.call(['tox'], cwd=checkout_path)
  228 + return TestResult(name, checkout_path, rv, interpreters)
  229 +
  230 +
  231 +def run_tests(interpreters, only_approved=True):
  232 + results = {}
  233 + create_tdir()
  234 + log('Packaging Flask')
  235 + flask_dep = package_flask()
  236 + log('Running extension tests')
  237 + log('Temporary Environment: %s', tdir)
  238 + for name in iter_extensions(only_approved):
  239 + log('Testing %s', name)
  240 + result = test_extension(name, interpreters, flask_dep)
  241 + if result.success:
  242 + log('Extension test succeeded')
  243 + else:
  244 + log('Extension test failed')
  245 + results[name] = result
  246 + return results
  247 +
  248 +
  249 +def render_results(results, approved):
  250 + from jinja2 import Template
  251 + items = results.values()
  252 + items.sort(key=lambda x: x.name.lower())
  253 + rv = Template(RESULT_TEMPATE, autoescape=True).render(results=items,
  254 + approved=approved)
  255 + fd, filename = tempfile.mkstemp(suffix='.html')
  256 + os.fdopen(fd, 'w').write(rv.encode('utf-8') + '\n')
  257 + return filename
  258 +
90 259
91 260 def main():
92   - for name, ret in test_all_extensions():
93   - print name, ret
  261 + parser = argparse.ArgumentParser(description='Runs Flask extension tests')
  262 + parser.add_argument('--all', dest='all', action='store_true',
  263 + help='run against all extensions, not just approved')
  264 + parser.add_argument('--browse', dest='browse', action='store_true',
  265 + help='show browser with the result summary')
  266 + args = parser.parse_args()
  267 +
  268 + results = run_tests(['py26'], not args.all)
  269 + filename = render_results(results, not args.all)
  270 + if args.browse:
  271 + import webbrowser
  272 + webbrowser.open('file:///' + filename.lstrip('/'))
  273 + print 'Results written to', filename
94 274
95 275
96 276 if __name__ == '__main__':

0 comments on commit 3a80ecc

Please sign in to comment.
Something went wrong with that request. Please try again.