Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

[1.2.X] Fixed #14427 -- Added --bisect and --pair flags to runtests.p…

…y, making it easier to find pairs of tests that fail when run together.

Backport of r14079 from trunk.

git-svn-id: http://code.djangoproject.com/svn/django/branches/releases/1.2.X@14081 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit 935f6873dd9bfa1582575c0309f0f8dcab0c0102 1 parent e8d5038
@freakboy3742 freakboy3742 authored
Showing with 134 additions and 21 deletions.
  1. +134 −21 tests/runtests.py
View
155 tests/runtests.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-import os, sys, traceback
+import os, subprocess, sys, traceback
import unittest
import django.contrib as contrib
@@ -85,16 +85,17 @@ def runTest(self):
self.assert_(not unexpected, "Unexpected Errors: " + '\n'.join(unexpected))
self.assert_(not missing, "Missing Errors: " + '\n'.join(missing))
-def django_tests(verbosity, interactive, failfast, test_labels):
+def setup(verbosity, test_labels):
from django.conf import settings
-
- old_installed_apps = settings.INSTALLED_APPS
- old_root_urlconf = getattr(settings, "ROOT_URLCONF", "")
- old_template_dirs = settings.TEMPLATE_DIRS
- old_use_i18n = settings.USE_I18N
- old_login_url = settings.LOGIN_URL
- old_language_code = settings.LANGUAGE_CODE
- old_middleware_classes = settings.MIDDLEWARE_CLASSES
+ state = {
+ 'INSTALLED_APPS': settings.INSTALLED_APPS,
+ 'ROOT_URLCONF': getattr(settings, "ROOT_URLCONF", ""),
+ 'TEMPLATE_DIRS': settings.TEMPLATE_DIRS,
+ 'USE_I18N': settings.USE_I18N,
+ 'LOGIN_URL': settings.LOGIN_URL,
+ 'LANGUAGE_CODE': settings.LANGUAGE_CODE,
+ 'MIDDLEWARE_CLASSES': settings.MIDDLEWARE_CLASSES,
+ }
# Redirect some settings for the duration of these tests.
settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS
@@ -136,6 +137,18 @@ def django_tests(verbosity, interactive, failfast, test_labels):
if model_label not in settings.INSTALLED_APPS:
settings.INSTALLED_APPS.append(model_label)
+ return state
+
+def teardown(state):
+ from django.conf import settings
+ # Restore the old settings.
+ for key, value in state.items():
+ setattr(settings, key, value)
+
+def django_tests(verbosity, interactive, failfast, test_labels):
+ from django.conf import settings
+ state = setup(verbosity, test_labels)
+
# Add tests for invalid models.
extra_tests = []
for model_dir, model_name in get_invalid_models():
@@ -169,17 +182,105 @@ def django_tests(verbosity, interactive, failfast, test_labels):
test_runner = TestRunner(verbosity=verbosity, interactive=interactive, failfast=failfast)
failures = test_runner.run_tests(test_labels, extra_tests=extra_tests)
- if failures:
- sys.exit(bool(failures))
+ teardown(state)
+ return failures
- # Restore the old settings.
- settings.INSTALLED_APPS = old_installed_apps
- settings.ROOT_URLCONF = old_root_urlconf
- settings.TEMPLATE_DIRS = old_template_dirs
- settings.USE_I18N = old_use_i18n
- settings.LANGUAGE_CODE = old_language_code
- settings.LOGIN_URL = old_login_url
- settings.MIDDLEWARE_CLASSES = old_middleware_classes
+
+def bisect_tests(bisection_label, options, test_labels):
+ state = setup(int(options.verbosity), test_labels)
+
+ if not test_labels:
+ # Get the full list of test labels to use for bisection
+ from django.db.models.loading import get_apps
+ test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
+
+ print '***** Bisecting test suite:',' '.join(test_labels)
+
+ # Make sure the bisection point isn't in the test list
+ # Also remove tests that need to be run in specific combinations
+ for label in [bisection_label, 'model_inheritance_same_model_name']:
+ try:
+ test_labels.remove(label)
+ except ValueError:
+ pass
+
+ subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
+ if options.failfast:
+ subprocess_args.append('--failfast')
+ if options.verbosity:
+ subprocess_args.append('--verbosity=%s' % options.verbosity)
+ if not options.interactive:
+ subprocess_args.append('--noinput')
+
+ iteration = 1
+ while len(test_labels) > 1:
+ midpoint = len(test_labels)/2
+ test_labels_a = test_labels[:midpoint] + [bisection_label]
+ test_labels_b = test_labels[midpoint:] + [bisection_label]
+ print '***** Pass %da: Running the first half of the test suite' % iteration
+ print '***** Test labels:',' '.join(test_labels_a)
+ failures_a = subprocess.call(subprocess_args + test_labels_a)
+
+ print '***** Pass %db: Running the second half of the test suite' % iteration
+ print '***** Test labels:',' '.join(test_labels_b)
+ print
+ failures_b = subprocess.call(subprocess_args + test_labels_b)
+
+ if failures_a and not failures_b:
+ print "***** Problem found in first half. Bisecting again..."
+ iteration = iteration + 1
+ test_labels = test_labels_a[:-1]
+ elif failures_b and not failures_a:
+ print "***** Problem found in second half. Bisecting again..."
+ iteration = iteration + 1
+ test_labels = test_labels_b[:-1]
+ elif failures_a and failures_b:
+ print "***** Multiple sources of failure found"
+ break
+ else:
+ print "***** No source of failure found... try pair execution (--pair)"
+ break
+
+ if len(test_labels) == 1:
+ print "***** Source of error:",test_labels[0]
+ teardown(state)
+
+def paired_tests(paired_test, options, test_labels):
+ state = setup(int(options.verbosity), test_labels)
+
+ if not test_labels:
+ print ""
+ # Get the full list of test labels to use for bisection
+ from django.db.models.loading import get_apps
+ test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
+
+ print '***** Trying paired execution'
+
+ # Make sure the bisection point isn't in the test list
+ # Also remove tests that need to be run in specific combinations
+ for label in [paired_test, 'model_inheritance_same_model_name']:
+ try:
+ test_labels.remove(label)
+ except ValueError:
+ pass
+
+ subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
+ if options.failfast:
+ subprocess_args.append('--failfast')
+ if options.verbosity:
+ subprocess_args.append('--verbosity=%s' % options.verbosity)
+ if not options.interactive:
+ subprocess_args.append('--noinput')
+
+ for i, label in enumerate(test_labels):
+ print '***** %d of %d: Check test pairing with %s' % (i+1, len(test_labels), label)
+ failures = subprocess.call(subprocess_args + [label, paired_test])
+ if failures:
+ print '***** Found problem pair with',label
+ return
+
+ print '***** No problem pair found'
+ teardown(state)
if __name__ == "__main__":
from optparse import OptionParser
@@ -194,10 +295,22 @@ def django_tests(verbosity, interactive, failfast, test_labels):
help='Tells Django to stop running the test suite after first failed test.')
parser.add_option('--settings',
help='Python path to settings module, e.g. "myproject.settings". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.')
+ parser.add_option('--bisect', action='store', dest='bisect', default=None,
+ help="Bisect the test suite to discover a test that causes a test failure when combined with the named test.")
+ parser.add_option('--pair', action='store', dest='pair', default=None,
+ help="Run the test suite in pairs with the named test to find problem pairs.")
options, args = parser.parse_args()
if options.settings:
os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
elif "DJANGO_SETTINGS_MODULE" not in os.environ:
parser.error("DJANGO_SETTINGS_MODULE is not set in the environment. "
"Set it or use --settings.")
- django_tests(int(options.verbosity), options.interactive, options.failfast, args)
+
+ if options.bisect:
+ bisect_tests(options.bisect, options, args)
+ elif options.pair:
+ paired_tests(options.pair, options, args)
+ else:
+ failures = django_tests(int(options.verbosity), options.interactive, options.failfast, args)
+ if failures:
+ sys.exit(bool(failures))
Please sign in to comment.
Something went wrong with that request. Please try again.