Fixed #14427 -- Added --bisect and --pair flags to runtests.py, making it easier to find pairs of tests that fail when run together.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@14079 bcc190cf-cafb-0310-a4f2-bffc1f526a37

tests/runtests.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os, sys, traceback
+import os, subprocess, sys, traceback
 import unittest
 
 import django.contrib as contrib
@@ -85,16 +85,17 @@ class InvalidModelTestCase(unittest.TestCase):
         self.assert_(not unexpected, "Unexpected Errors: " + '\n'.join(unexpected))
         self.assert_(not missing, "Missing Errors: " + '\n'.join(missing))
 
-def django_tests(verbosity, interactive, failfast, test_labels):
+def setup(verbosity, test_labels):
     from django.conf import settings
-
+    state = {
-    old_installed_apps = settings.INSTALLED_APPS
+        'INSTALLED_APPS': settings.INSTALLED_APPS,
-    old_root_urlconf = getattr(settings, "ROOT_URLCONF", "")
+        'ROOT_URLCONF': getattr(settings, "ROOT_URLCONF", ""),
-    old_template_dirs = settings.TEMPLATE_DIRS
+        'TEMPLATE_DIRS': settings.TEMPLATE_DIRS,
-    old_use_i18n = settings.USE_I18N
+        'USE_I18N': settings.USE_I18N,
-    old_login_url = settings.LOGIN_URL
+        'LOGIN_URL': settings.LOGIN_URL,
-    old_language_code = settings.LANGUAGE_CODE
+        'LANGUAGE_CODE': settings.LANGUAGE_CODE,
-    old_middleware_classes = settings.MIDDLEWARE_CLASSES
+        'MIDDLEWARE_CLASSES': settings.MIDDLEWARE_CLASSES,
+    }
 
     # Redirect some settings for the duration of these tests.
     settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS
@@ -136,6 +137,18 @@ def django_tests(verbosity, interactive, failfast, test_labels):
                 if model_label not in settings.INSTALLED_APPS:
                     settings.INSTALLED_APPS.append(model_label)
 
+    return state
+
+def teardown(state):
+    from django.conf import settings
+    # Restore the old settings.
+    for key, value in state.items():
+        setattr(settings, key, value)
+
+def django_tests(verbosity, interactive, failfast, test_labels):
+    from django.conf import settings
+    state = setup(verbosity, test_labels)
+
     # Add tests for invalid models.
     extra_tests = []
     for model_dir, model_name in get_invalid_models():
@@ -169,17 +182,105 @@ def django_tests(verbosity, interactive, failfast, test_labels):
         test_runner = TestRunner(verbosity=verbosity, interactive=interactive, failfast=failfast)
         failures = test_runner.run_tests(test_labels, extra_tests=extra_tests)
 
-    if failures:
+    teardown(state)
-        sys.exit(bool(failures))
+    return failures
 
-    # Restore the old settings.
+
-    settings.INSTALLED_APPS = old_installed_apps
+def bisect_tests(bisection_label, options, test_labels):
-    settings.ROOT_URLCONF = old_root_urlconf
+    state = setup(int(options.verbosity), test_labels)
-    settings.TEMPLATE_DIRS = old_template_dirs
+
-    settings.USE_I18N = old_use_i18n
+    if not test_labels:
-    settings.LANGUAGE_CODE = old_language_code
+        # Get the full list of test labels to use for bisection
-    settings.LOGIN_URL = old_login_url
+        from django.db.models.loading import get_apps
-    settings.MIDDLEWARE_CLASSES = old_middleware_classes
+        test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
+
+    print '***** Bisecting test suite:',' '.join(test_labels)
+
+    # Make sure the bisection point isn't in the test list
+    # Also remove tests that need to be run in specific combinations
+    for label in [bisection_label, 'model_inheritance_same_model_name']:
+        try:
+            test_labels.remove(label)
+        except ValueError:
+            pass
+
+    subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
+    if options.failfast:
+        subprocess_args.append('--failfast')
+    if options.verbosity:
+        subprocess_args.append('--verbosity=%s' % options.verbosity)
+    if not options.interactive:
+        subprocess_args.append('--noinput')
+
+    iteration = 1
+    while len(test_labels) > 1:
+        midpoint = len(test_labels)/2
+        test_labels_a = test_labels[:midpoint] + [bisection_label]
+        test_labels_b = test_labels[midpoint:] + [bisection_label]
+        print '***** Pass %da: Running the first half of the test suite' % iteration
+        print '***** Test labels:',' '.join(test_labels_a)
+        failures_a = subprocess.call(subprocess_args + test_labels_a)
+
+        print '***** Pass %db: Running the second half of the test suite' % iteration
+        print '***** Test labels:',' '.join(test_labels_b)
+        print
+        failures_b = subprocess.call(subprocess_args + test_labels_b)
+
+        if failures_a and not failures_b:
+            print "***** Problem found in first half. Bisecting again..."
+            iteration = iteration + 1
+            test_labels = test_labels_a[:-1]
+        elif failures_b and not failures_a:
+            print "***** Problem found in second half. Bisecting again..."
+            iteration = iteration + 1
+            test_labels = test_labels_b[:-1]
+        elif failures_a and failures_b:
+            print "***** Multiple sources of failure found"
+            break
+        else:
+            print "***** No source of failure found... try pair execution (--pair)"
+            break
+
+    if len(test_labels) == 1:
+        print "***** Source of error:",test_labels[0]
+    teardown(state)
+
+def paired_tests(paired_test, options, test_labels):
+    state = setup(int(options.verbosity), test_labels)
+
+    if not test_labels:
+        print ""
+        # Get the full list of test labels to use for bisection
+        from django.db.models.loading import get_apps
+        test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
+
+    print '***** Trying paired execution'
+
+    # Make sure the bisection point isn't in the test list
+    # Also remove tests that need to be run in specific combinations
+    for label in [paired_test, 'model_inheritance_same_model_name']:
+        try:
+            test_labels.remove(label)
+        except ValueError:
+            pass
+
+    subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
+    if options.failfast:
+        subprocess_args.append('--failfast')
+    if options.verbosity:
+        subprocess_args.append('--verbosity=%s' % options.verbosity)
+    if not options.interactive:
+        subprocess_args.append('--noinput')
+
+    for i, label in enumerate(test_labels):
+        print '***** %d of %d: Check test pairing with %s' % (i+1, len(test_labels), label)
+        failures = subprocess.call(subprocess_args + [label, paired_test])
+        if failures:
+            print '***** Found problem pair with',label
+            return
+
+    print '***** No problem pair found'
+    teardown(state)
 
 if __name__ == "__main__":
     from optparse import OptionParser
@@ -194,10 +295,22 @@ if __name__ == "__main__":
         help='Tells Django to stop running the test suite after first failed test.')
     parser.add_option('--settings',
         help='Python path to settings module, e.g. "myproject.settings". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.')
+    parser.add_option('--bisect', action='store', dest='bisect', default=None,
+        help="Bisect the test suite to discover a test that causes a test failure when combined with the named test.")
+    parser.add_option('--pair', action='store', dest='pair', default=None,
+        help="Run the test suite in pairs with the named test to find problem pairs.")
     options, args = parser.parse_args()
     if options.settings:
         os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
     elif "DJANGO_SETTINGS_MODULE" not in os.environ:
         parser.error("DJANGO_SETTINGS_MODULE is not set in the environment. "
                       "Set it or use --settings.")
-    django_tests(int(options.verbosity), options.interactive, options.failfast, args)
+
+    if options.bisect:
+        bisect_tests(options.bisect, options, args)
+    elif options.pair:
+        paired_tests(options.pair, options, args)
+    else:
+        failures = django_tests(int(options.verbosity), options.interactive, options.failfast, args)
+        if failures:
+            sys.exit(bool(failures))