Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

[1.2.X] Fixed #14427 -- Added --bisect and --pair flags to runtests.p…

…y, making it easier to find pairs of tests that fail when run together.

Backport of r14079 from trunk.

git-svn-id: http://code.djangoproject.com/svn/django/branches/releases/1.2.X@14081 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit 935f6873dd9bfa1582575c0309f0f8dcab0c0102 1 parent e8d5038
Russell Keith-Magee authored October 09, 2010

Showing 1 changed file with 134 additions and 21 deletions. Show diff stats Hide diff stats

  1. 155  tests/runtests.py
155  tests/runtests.py
... ...
@@ -1,6 +1,6 @@
1 1
 #!/usr/bin/env python
2 2
 
3  
-import os, sys, traceback
  3
+import os, subprocess, sys, traceback
4 4
 import unittest
5 5
 
6 6
 import django.contrib as contrib
@@ -85,16 +85,17 @@ def runTest(self):
85 85
         self.assert_(not unexpected, "Unexpected Errors: " + '\n'.join(unexpected))
86 86
         self.assert_(not missing, "Missing Errors: " + '\n'.join(missing))
87 87
 
88  
-def django_tests(verbosity, interactive, failfast, test_labels):
  88
+def setup(verbosity, test_labels):
89 89
     from django.conf import settings
90  
-
91  
-    old_installed_apps = settings.INSTALLED_APPS
92  
-    old_root_urlconf = getattr(settings, "ROOT_URLCONF", "")
93  
-    old_template_dirs = settings.TEMPLATE_DIRS
94  
-    old_use_i18n = settings.USE_I18N
95  
-    old_login_url = settings.LOGIN_URL
96  
-    old_language_code = settings.LANGUAGE_CODE
97  
-    old_middleware_classes = settings.MIDDLEWARE_CLASSES
  90
+    state = {
  91
+        'INSTALLED_APPS': settings.INSTALLED_APPS,
  92
+        'ROOT_URLCONF': getattr(settings, "ROOT_URLCONF", ""),
  93
+        'TEMPLATE_DIRS': settings.TEMPLATE_DIRS,
  94
+        'USE_I18N': settings.USE_I18N,
  95
+        'LOGIN_URL': settings.LOGIN_URL,
  96
+        'LANGUAGE_CODE': settings.LANGUAGE_CODE,
  97
+        'MIDDLEWARE_CLASSES': settings.MIDDLEWARE_CLASSES,
  98
+    }
98 99
 
99 100
     # Redirect some settings for the duration of these tests.
100 101
     settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS
@@ -136,6 +137,18 @@ def django_tests(verbosity, interactive, failfast, test_labels):
136 137
                 if model_label not in settings.INSTALLED_APPS:
137 138
                     settings.INSTALLED_APPS.append(model_label)
138 139
 
  140
+    return state
  141
+
  142
+def teardown(state):
  143
+    from django.conf import settings
  144
+    # Restore the old settings.
  145
+    for key, value in state.items():
  146
+        setattr(settings, key, value)
  147
+
  148
+def django_tests(verbosity, interactive, failfast, test_labels):
  149
+    from django.conf import settings
  150
+    state = setup(verbosity, test_labels)
  151
+
139 152
     # Add tests for invalid models.
140 153
     extra_tests = []
141 154
     for model_dir, model_name in get_invalid_models():
@@ -169,17 +182,105 @@ def django_tests(verbosity, interactive, failfast, test_labels):
169 182
         test_runner = TestRunner(verbosity=verbosity, interactive=interactive, failfast=failfast)
170 183
         failures = test_runner.run_tests(test_labels, extra_tests=extra_tests)
171 184
 
172  
-    if failures:
173  
-        sys.exit(bool(failures))
  185
+    teardown(state)
  186
+    return failures
174 187
 
175  
-    # Restore the old settings.
176  
-    settings.INSTALLED_APPS = old_installed_apps
177  
-    settings.ROOT_URLCONF = old_root_urlconf
178  
-    settings.TEMPLATE_DIRS = old_template_dirs
179  
-    settings.USE_I18N = old_use_i18n
180  
-    settings.LANGUAGE_CODE = old_language_code
181  
-    settings.LOGIN_URL = old_login_url
182  
-    settings.MIDDLEWARE_CLASSES = old_middleware_classes
  188
+
  189
+def bisect_tests(bisection_label, options, test_labels):
  190
+    state = setup(int(options.verbosity), test_labels)
  191
+
  192
+    if not test_labels:
  193
+        # Get the full list of test labels to use for bisection
  194
+        from django.db.models.loading import get_apps
  195
+        test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
  196
+
  197
+    print '***** Bisecting test suite:',' '.join(test_labels)
  198
+
  199
+    # Make sure the bisection point isn't in the test list
  200
+    # Also remove tests that need to be run in specific combinations
  201
+    for label in [bisection_label, 'model_inheritance_same_model_name']:
  202
+        try:
  203
+            test_labels.remove(label)
  204
+        except ValueError:
  205
+            pass
  206
+
  207
+    subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
  208
+    if options.failfast:
  209
+        subprocess_args.append('--failfast')
  210
+    if options.verbosity:
  211
+        subprocess_args.append('--verbosity=%s' % options.verbosity)
  212
+    if not options.interactive:
  213
+        subprocess_args.append('--noinput')
  214
+
  215
+    iteration = 1
  216
+    while len(test_labels) > 1:
  217
+        midpoint = len(test_labels)/2
  218
+        test_labels_a = test_labels[:midpoint] + [bisection_label]
  219
+        test_labels_b = test_labels[midpoint:] + [bisection_label]
  220
+        print '***** Pass %da: Running the first half of the test suite' % iteration
  221
+        print '***** Test labels:',' '.join(test_labels_a)
  222
+        failures_a = subprocess.call(subprocess_args + test_labels_a)
  223
+
  224
+        print '***** Pass %db: Running the second half of the test suite' % iteration
  225
+        print '***** Test labels:',' '.join(test_labels_b)
  226
+        print
  227
+        failures_b = subprocess.call(subprocess_args + test_labels_b)
  228
+
  229
+        if failures_a and not failures_b:
  230
+            print "***** Problem found in first half. Bisecting again..."
  231
+            iteration = iteration + 1
  232
+            test_labels = test_labels_a[:-1]
  233
+        elif failures_b and not failures_a:
  234
+            print "***** Problem found in second half. Bisecting again..."
  235
+            iteration = iteration + 1
  236
+            test_labels = test_labels_b[:-1]
  237
+        elif failures_a and failures_b:
  238
+            print "***** Multiple sources of failure found"
  239
+            break
  240
+        else:
  241
+            print "***** No source of failure found... try pair execution (--pair)"
  242
+            break
  243
+
  244
+    if len(test_labels) == 1:
  245
+        print "***** Source of error:",test_labels[0]
  246
+    teardown(state)
  247
+
  248
+def paired_tests(paired_test, options, test_labels):
  249
+    state = setup(int(options.verbosity), test_labels)
  250
+
  251
+    if not test_labels:
  252
+        print ""
  253
+        # Get the full list of test labels to use for bisection
  254
+        from django.db.models.loading import get_apps
  255
+        test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
  256
+
  257
+    print '***** Trying paired execution'
  258
+
  259
+    # Make sure the bisection point isn't in the test list
  260
+    # Also remove tests that need to be run in specific combinations
  261
+    for label in [paired_test, 'model_inheritance_same_model_name']:
  262
+        try:
  263
+            test_labels.remove(label)
  264
+        except ValueError:
  265
+            pass
  266
+
  267
+    subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
  268
+    if options.failfast:
  269
+        subprocess_args.append('--failfast')
  270
+    if options.verbosity:
  271
+        subprocess_args.append('--verbosity=%s' % options.verbosity)
  272
+    if not options.interactive:
  273
+        subprocess_args.append('--noinput')
  274
+
  275
+    for i, label in enumerate(test_labels):
  276
+        print '***** %d of %d: Check test pairing with %s' % (i+1, len(test_labels), label)
  277
+        failures = subprocess.call(subprocess_args + [label, paired_test])
  278
+        if failures:
  279
+            print '***** Found problem pair with',label
  280
+            return
  281
+
  282
+    print '***** No problem pair found'
  283
+    teardown(state)
183 284
 
184 285
 if __name__ == "__main__":
185 286
     from optparse import OptionParser
@@ -194,10 +295,22 @@ def django_tests(verbosity, interactive, failfast, test_labels):
194 295
         help='Tells Django to stop running the test suite after first failed test.')
195 296
     parser.add_option('--settings',
196 297
         help='Python path to settings module, e.g. "myproject.settings". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.')
  298
+    parser.add_option('--bisect', action='store', dest='bisect', default=None,
  299
+        help="Bisect the test suite to discover a test that causes a test failure when combined with the named test.")
  300
+    parser.add_option('--pair', action='store', dest='pair', default=None,
  301
+        help="Run the test suite in pairs with the named test to find problem pairs.")
197 302
     options, args = parser.parse_args()
198 303
     if options.settings:
199 304
         os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
200 305
     elif "DJANGO_SETTINGS_MODULE" not in os.environ:
201 306
         parser.error("DJANGO_SETTINGS_MODULE is not set in the environment. "
202 307
                       "Set it or use --settings.")
203  
-    django_tests(int(options.verbosity), options.interactive, options.failfast, args)
  308
+
  309
+    if options.bisect:
  310
+        bisect_tests(options.bisect, options, args)
  311
+    elif options.pair:
  312
+        paired_tests(options.pair, options, args)
  313
+    else:
  314
+        failures = django_tests(int(options.verbosity), options.interactive, options.failfast, args)
  315
+        if failures:
  316
+            sys.exit(bool(failures))

0 notes on commit 935f687

Please sign in to comment.
Something went wrong with that request. Please try again.