mirror of
https://github.com/django/django.git
synced 2024-12-23 09:36:06 +00:00
Fixed #14427 -- Added --bisect and --pair flags to runtests.py, making it easier to find pairs of tests that fail when run together.
git-svn-id: http://code.djangoproject.com/svn/django/trunk@14079 bcc190cf-cafb-0310-a4f2-bffc1f526a37
This commit is contained in:
parent
03e7d64de4
commit
18c3ea5546
@ -1,6 +1,6 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
|
||||||
import os, sys, traceback
|
import os, subprocess, sys, traceback
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
import django.contrib as contrib
|
import django.contrib as contrib
|
||||||
@ -85,16 +85,17 @@ class InvalidModelTestCase(unittest.TestCase):
|
|||||||
self.assert_(not unexpected, "Unexpected Errors: " + '\n'.join(unexpected))
|
self.assert_(not unexpected, "Unexpected Errors: " + '\n'.join(unexpected))
|
||||||
self.assert_(not missing, "Missing Errors: " + '\n'.join(missing))
|
self.assert_(not missing, "Missing Errors: " + '\n'.join(missing))
|
||||||
|
|
||||||
def django_tests(verbosity, interactive, failfast, test_labels):
|
def setup(verbosity, test_labels):
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
|
state = {
|
||||||
old_installed_apps = settings.INSTALLED_APPS
|
'INSTALLED_APPS': settings.INSTALLED_APPS,
|
||||||
old_root_urlconf = getattr(settings, "ROOT_URLCONF", "")
|
'ROOT_URLCONF': getattr(settings, "ROOT_URLCONF", ""),
|
||||||
old_template_dirs = settings.TEMPLATE_DIRS
|
'TEMPLATE_DIRS': settings.TEMPLATE_DIRS,
|
||||||
old_use_i18n = settings.USE_I18N
|
'USE_I18N': settings.USE_I18N,
|
||||||
old_login_url = settings.LOGIN_URL
|
'LOGIN_URL': settings.LOGIN_URL,
|
||||||
old_language_code = settings.LANGUAGE_CODE
|
'LANGUAGE_CODE': settings.LANGUAGE_CODE,
|
||||||
old_middleware_classes = settings.MIDDLEWARE_CLASSES
|
'MIDDLEWARE_CLASSES': settings.MIDDLEWARE_CLASSES,
|
||||||
|
}
|
||||||
|
|
||||||
# Redirect some settings for the duration of these tests.
|
# Redirect some settings for the duration of these tests.
|
||||||
settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS
|
settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS
|
||||||
@ -136,6 +137,18 @@ def django_tests(verbosity, interactive, failfast, test_labels):
|
|||||||
if model_label not in settings.INSTALLED_APPS:
|
if model_label not in settings.INSTALLED_APPS:
|
||||||
settings.INSTALLED_APPS.append(model_label)
|
settings.INSTALLED_APPS.append(model_label)
|
||||||
|
|
||||||
|
return state
|
||||||
|
|
||||||
|
def teardown(state):
|
||||||
|
from django.conf import settings
|
||||||
|
# Restore the old settings.
|
||||||
|
for key, value in state.items():
|
||||||
|
setattr(settings, key, value)
|
||||||
|
|
||||||
|
def django_tests(verbosity, interactive, failfast, test_labels):
|
||||||
|
from django.conf import settings
|
||||||
|
state = setup(verbosity, test_labels)
|
||||||
|
|
||||||
# Add tests for invalid models.
|
# Add tests for invalid models.
|
||||||
extra_tests = []
|
extra_tests = []
|
||||||
for model_dir, model_name in get_invalid_models():
|
for model_dir, model_name in get_invalid_models():
|
||||||
@ -169,17 +182,105 @@ def django_tests(verbosity, interactive, failfast, test_labels):
|
|||||||
test_runner = TestRunner(verbosity=verbosity, interactive=interactive, failfast=failfast)
|
test_runner = TestRunner(verbosity=verbosity, interactive=interactive, failfast=failfast)
|
||||||
failures = test_runner.run_tests(test_labels, extra_tests=extra_tests)
|
failures = test_runner.run_tests(test_labels, extra_tests=extra_tests)
|
||||||
|
|
||||||
if failures:
|
teardown(state)
|
||||||
sys.exit(bool(failures))
|
return failures
|
||||||
|
|
||||||
# Restore the old settings.
|
|
||||||
settings.INSTALLED_APPS = old_installed_apps
|
def bisect_tests(bisection_label, options, test_labels):
|
||||||
settings.ROOT_URLCONF = old_root_urlconf
|
state = setup(int(options.verbosity), test_labels)
|
||||||
settings.TEMPLATE_DIRS = old_template_dirs
|
|
||||||
settings.USE_I18N = old_use_i18n
|
if not test_labels:
|
||||||
settings.LANGUAGE_CODE = old_language_code
|
# Get the full list of test labels to use for bisection
|
||||||
settings.LOGIN_URL = old_login_url
|
from django.db.models.loading import get_apps
|
||||||
settings.MIDDLEWARE_CLASSES = old_middleware_classes
|
test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
|
||||||
|
|
||||||
|
print '***** Bisecting test suite:',' '.join(test_labels)
|
||||||
|
|
||||||
|
# Make sure the bisection point isn't in the test list
|
||||||
|
# Also remove tests that need to be run in specific combinations
|
||||||
|
for label in [bisection_label, 'model_inheritance_same_model_name']:
|
||||||
|
try:
|
||||||
|
test_labels.remove(label)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
|
||||||
|
if options.failfast:
|
||||||
|
subprocess_args.append('--failfast')
|
||||||
|
if options.verbosity:
|
||||||
|
subprocess_args.append('--verbosity=%s' % options.verbosity)
|
||||||
|
if not options.interactive:
|
||||||
|
subprocess_args.append('--noinput')
|
||||||
|
|
||||||
|
iteration = 1
|
||||||
|
while len(test_labels) > 1:
|
||||||
|
midpoint = len(test_labels)/2
|
||||||
|
test_labels_a = test_labels[:midpoint] + [bisection_label]
|
||||||
|
test_labels_b = test_labels[midpoint:] + [bisection_label]
|
||||||
|
print '***** Pass %da: Running the first half of the test suite' % iteration
|
||||||
|
print '***** Test labels:',' '.join(test_labels_a)
|
||||||
|
failures_a = subprocess.call(subprocess_args + test_labels_a)
|
||||||
|
|
||||||
|
print '***** Pass %db: Running the second half of the test suite' % iteration
|
||||||
|
print '***** Test labels:',' '.join(test_labels_b)
|
||||||
|
print
|
||||||
|
failures_b = subprocess.call(subprocess_args + test_labels_b)
|
||||||
|
|
||||||
|
if failures_a and not failures_b:
|
||||||
|
print "***** Problem found in first half. Bisecting again..."
|
||||||
|
iteration = iteration + 1
|
||||||
|
test_labels = test_labels_a[:-1]
|
||||||
|
elif failures_b and not failures_a:
|
||||||
|
print "***** Problem found in second half. Bisecting again..."
|
||||||
|
iteration = iteration + 1
|
||||||
|
test_labels = test_labels_b[:-1]
|
||||||
|
elif failures_a and failures_b:
|
||||||
|
print "***** Multiple sources of failure found"
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
print "***** No source of failure found... try pair execution (--pair)"
|
||||||
|
break
|
||||||
|
|
||||||
|
if len(test_labels) == 1:
|
||||||
|
print "***** Source of error:",test_labels[0]
|
||||||
|
teardown(state)
|
||||||
|
|
||||||
|
def paired_tests(paired_test, options, test_labels):
|
||||||
|
state = setup(int(options.verbosity), test_labels)
|
||||||
|
|
||||||
|
if not test_labels:
|
||||||
|
print ""
|
||||||
|
# Get the full list of test labels to use for bisection
|
||||||
|
from django.db.models.loading import get_apps
|
||||||
|
test_labels = [app.__name__.split('.')[-2] for app in get_apps()]
|
||||||
|
|
||||||
|
print '***** Trying paired execution'
|
||||||
|
|
||||||
|
# Make sure the bisection point isn't in the test list
|
||||||
|
# Also remove tests that need to be run in specific combinations
|
||||||
|
for label in [paired_test, 'model_inheritance_same_model_name']:
|
||||||
|
try:
|
||||||
|
test_labels.remove(label)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings]
|
||||||
|
if options.failfast:
|
||||||
|
subprocess_args.append('--failfast')
|
||||||
|
if options.verbosity:
|
||||||
|
subprocess_args.append('--verbosity=%s' % options.verbosity)
|
||||||
|
if not options.interactive:
|
||||||
|
subprocess_args.append('--noinput')
|
||||||
|
|
||||||
|
for i, label in enumerate(test_labels):
|
||||||
|
print '***** %d of %d: Check test pairing with %s' % (i+1, len(test_labels), label)
|
||||||
|
failures = subprocess.call(subprocess_args + [label, paired_test])
|
||||||
|
if failures:
|
||||||
|
print '***** Found problem pair with',label
|
||||||
|
return
|
||||||
|
|
||||||
|
print '***** No problem pair found'
|
||||||
|
teardown(state)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
from optparse import OptionParser
|
from optparse import OptionParser
|
||||||
@ -194,10 +295,22 @@ if __name__ == "__main__":
|
|||||||
help='Tells Django to stop running the test suite after first failed test.')
|
help='Tells Django to stop running the test suite after first failed test.')
|
||||||
parser.add_option('--settings',
|
parser.add_option('--settings',
|
||||||
help='Python path to settings module, e.g. "myproject.settings". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.')
|
help='Python path to settings module, e.g. "myproject.settings". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.')
|
||||||
|
parser.add_option('--bisect', action='store', dest='bisect', default=None,
|
||||||
|
help="Bisect the test suite to discover a test that causes a test failure when combined with the named test.")
|
||||||
|
parser.add_option('--pair', action='store', dest='pair', default=None,
|
||||||
|
help="Run the test suite in pairs with the named test to find problem pairs.")
|
||||||
options, args = parser.parse_args()
|
options, args = parser.parse_args()
|
||||||
if options.settings:
|
if options.settings:
|
||||||
os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
|
os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
|
||||||
elif "DJANGO_SETTINGS_MODULE" not in os.environ:
|
elif "DJANGO_SETTINGS_MODULE" not in os.environ:
|
||||||
parser.error("DJANGO_SETTINGS_MODULE is not set in the environment. "
|
parser.error("DJANGO_SETTINGS_MODULE is not set in the environment. "
|
||||||
"Set it or use --settings.")
|
"Set it or use --settings.")
|
||||||
django_tests(int(options.verbosity), options.interactive, options.failfast, args)
|
|
||||||
|
if options.bisect:
|
||||||
|
bisect_tests(options.bisect, options, args)
|
||||||
|
elif options.pair:
|
||||||
|
paired_tests(options.pair, options, args)
|
||||||
|
else:
|
||||||
|
failures = django_tests(int(options.verbosity), options.interactive, options.failfast, args)
|
||||||
|
if failures:
|
||||||
|
sys.exit(bool(failures))
|
||||||
|
Loading…
Reference in New Issue
Block a user