newforms-admin: Merged from trunk up to [7928].

git-svn-id: http://code.djangoproject.com/svn/django/branches/newforms-admin@7937 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/core/handlers/base.py

@@ -60,7 +60,6 @@ class BaseHandler(object):
     def get_response(self, request):
         "Returns an HttpResponse object for the given HttpRequest"
         from django.core import exceptions, urlresolvers
-        from django.core.mail import mail_admins
         from django.conf import settings
 
         # Apply request middleware
@@ -122,10 +121,25 @@ class BaseHandler(object):
 
             if settings.DEBUG_PROPAGATE_EXCEPTIONS:
                 raise
-            elif settings.DEBUG:
+            return self.handle_uncaught_exception(request, resolver, exc_info)
+
+    def handle_uncaught_exception(self, request, resolver, exc_info):
+        """
+        Processing for any otherwise uncaught exceptions (those that will
+        generate HTTP 500 responses). Can be overridden by subclasses who want
+        customised 500 handling.
+
+        Be *very* careful when overriding this because the error could be
+        caused by anything, so assuming something like the database is always
+        available would be an error.
+        """
+        from django.conf import settings
+        from django.core.mail import mail_admins
+
+        if settings.DEBUG:
             from django.views import debug
             return debug.technical_500_response(request, *exc_info)
-            else:
+
         # When DEBUG is False, send an error message to the admins.
         subject = 'Error (%s IP): %s' % ((request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS and 'internal' or 'EXTERNAL'), request.path)
         try:

django/db/backends/__init__.py

@@ -53,6 +53,7 @@ class BaseDatabaseFeatures(object):
     time_field_needs_date = False
     interprets_empty_strings_as_nulls = False
     date_field_supports_time_value = True
+    can_use_chunked_reads = True
 
 class BaseDatabaseOperations(object):
     """

django/db/backends/sqlite3/base.py

@@ -40,6 +40,11 @@ Database.register_adapter(decimal.Decimal, util.rev_typecast_decimal)
 
 class DatabaseFeatures(BaseDatabaseFeatures):
     supports_constraints = False
+    # SQLite cannot handle us only partially reading from a cursor's result set
+    # and then writing the same rows to the database in another cursor. This
+    # setting ensures we always read result sets fully into memory all in one
+    # go.
+    can_use_chunked_reads = False
 
 class DatabaseOperations(BaseDatabaseOperations):
     def date_extract_sql(self, lookup_type, field_name):

django/db/models/sql/query.py

@@ -1616,10 +1616,16 @@ class Query(object):
 
         # The MULTI case.
         if self.ordering_aliases:
-            return order_modified_iter(cursor, len(self.ordering_aliases),
+            result = order_modified_iter(cursor, len(self.ordering_aliases),
                     self.connection.features.empty_fetchmany_value)
-        return iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)),
+        result = iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)),
                 self.connection.features.empty_fetchmany_value)
+        if not self.connection.features.can_use_chunked_reads:
+            # If we are using non-chunked reads, we return the same data
+            # structure as normally, but ensure it is all read into memory
+            # before going any further.
+            return list(result)
+        return result
 
 # Use the backend's custom Query class if it defines one. Otherwise, use the
 # default.

django/views/debug.py

@@ -19,8 +19,101 @@ def linebreak_iter(template_source):
         p = template_source.find('\n', p+1)
     yield len(template_source) + 1
 
-def get_template_exception_info(exc_type, exc_value, tb):
+def get_safe_settings():
-    origin, (start, end) = exc_value.source
+    "Returns a dictionary of the settings module, with sensitive settings blurred out."
+    settings_dict = {}
+    for k in dir(settings):
+        if k.isupper():
+            if HIDDEN_SETTINGS.search(k):
+                settings_dict[k] = '********************'
+            else:
+                settings_dict[k] = getattr(settings, k)
+    return settings_dict
+
+def technical_500_response(request, exc_type, exc_value, tb):
+    """
+    Create a technical server error response. The last three arguments are
+    the values returned from sys.exc_info() and friends.
+    """
+    reporter = ExceptionReporter(request, exc_type, exc_value, tb)
+    html = reporter.get_traceback_html()
+    return HttpResponseServerError(html, mimetype='text/html')
+
+class ExceptionReporter:
+    """
+    A class to organize and coordinate reporting on exceptions.
+    """
+    def __init__(self, request, exc_type, exc_value, tb):
+        self.request = request
+        self.exc_type = exc_type
+        self.exc_value = exc_value
+        self.tb = tb
+
+        self.template_info = None
+        self.template_does_not_exist = False
+        self.loader_debug_info = None
+
+        # Handle deprecated string exceptions
+        if isinstance(self.exc_type, basestring):
+            self.exc_value = Exception('Deprecated String Exception: %r' % self.exc_type)
+            self.exc_type = type(self.exc_value)
+
+    def get_traceback_html(self):
+        "Return HTML code for traceback."
+
+        if issubclass(self.exc_type, TemplateDoesNotExist):
+            from django.template.loader import template_source_loaders
+            self.template_does_not_exist = True
+            self.loader_debug_info = []
+            for loader in template_source_loaders:
+                try:
+                    source_list_func = getattr(__import__(loader.__module__, {}, {}, ['get_template_sources']), 'get_template_sources')
+                    # NOTE: This assumes exc_value is the name of the template that
+                    # the loader attempted to load.
+                    template_list = [{'name': t, 'exists': os.path.exists(t)} \
+                        for t in source_list_func(str(self.exc_value))]
+                except (ImportError, AttributeError):
+                    template_list = []
+                self.loader_debug_info.append({
+                    'loader': loader.__module__ + '.' + loader.__name__,
+                    'templates': template_list,
+                })
+        if settings.TEMPLATE_DEBUG and hasattr(self.exc_value, 'source'):
+            self.get_template_exception_info()
+
+        frames = self.get_traceback_frames()
+
+        unicode_hint = ''
+        if issubclass(self.exc_type, UnicodeError):
+            start = getattr(self.exc_value, 'start', None)
+            end = getattr(self.exc_value, 'end', None)
+            if start is not None and end is not None:
+                unicode_str = self.exc_value.args[1]
+                unicode_hint = smart_unicode(unicode_str[max(start-5, 0):min(end+5, len(unicode_str))], 'ascii', errors='replace')
+        from django import get_version
+        t = Template(TECHNICAL_500_TEMPLATE, name='Technical 500 template')
+        c = Context({
+            'exception_type': self.exc_type.__name__,
+            'exception_value': smart_unicode(self.exc_value, errors='replace'),
+            'unicode_hint': unicode_hint,
+            'frames': frames,
+            'lastframe': frames[-1],
+            'request': self.request,
+            'request_protocol': self.request.is_secure() and "https" or "http",
+            'settings': get_safe_settings(),
+            'sys_executable': sys.executable,
+            'sys_version_info': '%d.%d.%d' % sys.version_info[0:3],
+            'server_time': datetime.datetime.now(),
+            'django_version_info': get_version(),
+            'sys_path' : sys.path,
+            'template_info': self.template_info,
+            'template_does_not_exist': self.template_does_not_exist,
+            'loader_debug_info': self.loader_debug_info,
+        })
+        return t.render(c)
+
+    def get_template_exception_info(self):
+        origin, (start, end) = self.exc_value.source
         template_source = origin.reload()
         context_lines = 10
         line = 0
@@ -40,8 +133,8 @@ def get_template_exception_info(exc_type, exc_value, tb):
         top = max(1, line - context_lines)
         bottom = min(total, line + 1 + context_lines)
 
-    template_info = {
+        self.template_info = {
-        'message': exc_value.args[0],
+            'message': self.exc_value.args[0],
             'source_lines': source_lines[top:bottom],
             'before': before,
             'during': during,
@@ -52,154 +145,10 @@ def get_template_exception_info(exc_type, exc_value, tb):
             'line': line,
             'name': origin.name,
         }
-    exc_info = hasattr(exc_value, 'exc_info') and exc_value.exc_info or (exc_type, exc_value, tb)
+        if hasattr(self.exc_value, 'exc_info') and self.exc_value.exc_info:
-    return exc_info + (template_info,)
+            exc_type, exc_value, tb = self.exc_value.exc_info
 
-def get_safe_settings():
+    def _get_lines_from_file(self, filename, lineno, context_lines, loader=None, module_name=None):
-    "Returns a dictionary of the settings module, with sensitive settings blurred out."
-    settings_dict = {}
-    for k in dir(settings):
-        if k.isupper():
-            if HIDDEN_SETTINGS.search(k):
-                settings_dict[k] = '********************'
-            else:
-                settings_dict[k] = getattr(settings, k)
-    return settings_dict
-
-def technical_500_response(request, exc_type, exc_value, tb):
-    """
-    Create a technical server error response. The last three arguments are
-    the values returned from sys.exc_info() and friends.
-    """
-    html = get_traceback_html(request, exc_type, exc_value, tb)
-    return HttpResponseServerError(html, mimetype='text/html')
-
-def get_traceback_html(request, exc_type, exc_value, tb):
-    "Return HTML code for traceback."
-    template_info = None
-    template_does_not_exist = False
-    loader_debug_info = None
-
-    # Handle deprecated string exceptions
-    if isinstance(exc_type, basestring):
-        exc_value = Exception('Deprecated String Exception: %r' % exc_type)
-        exc_type = type(exc_value)
-
-    if issubclass(exc_type, TemplateDoesNotExist):
-        from django.template.loader import template_source_loaders
-        template_does_not_exist = True
-        loader_debug_info = []
-        for loader in template_source_loaders:
-            try:
-                source_list_func = getattr(__import__(loader.__module__, {}, {}, ['get_template_sources']), 'get_template_sources')
-                # NOTE: This assumes exc_value is the name of the template that
-                # the loader attempted to load.
-                template_list = [{'name': t, 'exists': os.path.exists(t)} \
-                    for t in source_list_func(str(exc_value))]
-            except (ImportError, AttributeError):
-                template_list = []
-            loader_debug_info.append({
-                'loader': loader.__module__ + '.' + loader.__name__,
-                'templates': template_list,
-            })
-    if settings.TEMPLATE_DEBUG and hasattr(exc_value, 'source'):
-        exc_type, exc_value, tb, template_info = get_template_exception_info(exc_type, exc_value, tb)
-    frames = []
-    while tb is not None:
-        # support for __traceback_hide__ which is used by a few libraries
-        # to hide internal frames.
-        if tb.tb_frame.f_locals.get('__traceback_hide__'):
-            tb = tb.tb_next
-            continue
-        filename = tb.tb_frame.f_code.co_filename
-        function = tb.tb_frame.f_code.co_name
-        lineno = tb.tb_lineno - 1
-        loader = tb.tb_frame.f_globals.get('__loader__')
-        module_name = tb.tb_frame.f_globals.get('__name__')
-        pre_context_lineno, pre_context, context_line, post_context = _get_lines_from_file(filename, lineno, 7, loader, module_name)
-        if pre_context_lineno is not None:
-            frames.append({
-                'tb': tb,
-                'filename': filename,
-                'function': function,
-                'lineno': lineno + 1,
-                'vars': tb.tb_frame.f_locals.items(),
-                'id': id(tb),
-                'pre_context': pre_context,
-                'context_line': context_line,
-                'post_context': post_context,
-                'pre_context_lineno': pre_context_lineno + 1,
-            })
-        tb = tb.tb_next
-
-    if not frames:
-        frames = [{
-            'filename': '<unknown>',
-            'function': '?',
-            'lineno': '?',
-        }]
-
-    unicode_hint = ''
-    if issubclass(exc_type, UnicodeError):
-        start = getattr(exc_value, 'start', None)
-        end = getattr(exc_value, 'end', None)
-        if start is not None and end is not None:
-            unicode_str = exc_value.args[1]
-            unicode_hint = smart_unicode(unicode_str[max(start-5, 0):min(end+5, len(unicode_str))], 'ascii', errors='replace')
-    from django import get_version
-    t = Template(TECHNICAL_500_TEMPLATE, name='Technical 500 template')
-    c = Context({
-        'exception_type': exc_type.__name__,
-        'exception_value': smart_unicode(exc_value, errors='replace'),
-        'unicode_hint': unicode_hint,
-        'frames': frames,
-        'lastframe': frames[-1],
-        'request': request,
-        'request_protocol': request.is_secure() and "https" or "http",
-        'settings': get_safe_settings(),
-        'sys_executable': sys.executable,
-        'sys_version_info': '%d.%d.%d' % sys.version_info[0:3],
-        'server_time': datetime.datetime.now(),
-        'django_version_info': get_version(),
-        'sys_path' : sys.path,
-        'template_info': template_info,
-        'template_does_not_exist': template_does_not_exist,
-        'loader_debug_info': loader_debug_info,
-    })
-    return t.render(c)
-
-def technical_404_response(request, exception):
-    "Create a technical 404 error response. The exception should be the Http404."
-    try:
-        tried = exception.args[0]['tried']
-    except (IndexError, TypeError):
-        tried = []
-    else:
-        if not tried:
-            # tried exists but is an empty list. The URLconf must've been empty.
-            return empty_urlconf(request)
-
-    t = Template(TECHNICAL_404_TEMPLATE, name='Technical 404 template')
-    c = Context({
-        'root_urlconf': settings.ROOT_URLCONF,
-        'request_path': request.path[1:], # Trim leading slash
-        'urlpatterns': tried,
-        'reason': str(exception),
-        'request': request,
-        'request_protocol': request.is_secure() and "https" or "http",
-        'settings': get_safe_settings(),
-    })
-    return HttpResponseNotFound(t.render(c), mimetype='text/html')
-
-def empty_urlconf(request):
-    "Create an empty URLconf 404 error response."
-    t = Template(EMPTY_URLCONF_TEMPLATE, name='Empty URLConf template')
-    c = Context({
-        'project_name': settings.SETTINGS_MODULE.split('.')[0]
-    })
-    return HttpResponse(t.render(c), mimetype='text/html')
-
-def _get_lines_from_file(filename, lineno, context_lines, loader=None, module_name=None):
         """
         Returns context_lines before and after lineno from file.
         Returns (pre_context_lineno, pre_context, context_line, post_context).
@@ -240,6 +189,90 @@ def _get_lines_from_file(filename, lineno, context_lines, loader=None, module_na
 
         return lower_bound, pre_context, context_line, post_context
 
+    def get_traceback_frames(self):
+        frames = []
+        tb = self.tb
+        while tb is not None:
+            # support for __traceback_hide__ which is used by a few libraries
+            # to hide internal frames.
+            if tb.tb_frame.f_locals.get('__traceback_hide__'):
+                tb = tb.tb_next
+                continue
+            filename = tb.tb_frame.f_code.co_filename
+            function = tb.tb_frame.f_code.co_name
+            lineno = tb.tb_lineno - 1
+            loader = tb.tb_frame.f_globals.get('__loader__')
+            module_name = tb.tb_frame.f_globals.get('__name__')
+            pre_context_lineno, pre_context, context_line, post_context = self._get_lines_from_file(filename, lineno, 7, loader, module_name)
+            if pre_context_lineno is not None:
+                frames.append({
+                    'tb': tb,
+                    'filename': filename,
+                    'function': function,
+                    'lineno': lineno + 1,
+                    'vars': tb.tb_frame.f_locals.items(),
+                    'id': id(tb),
+                    'pre_context': pre_context,
+                    'context_line': context_line,
+                    'post_context': post_context,
+                    'pre_context_lineno': pre_context_lineno + 1,
+                })
+            tb = tb.tb_next
+
+        if not frames:
+            frames = [{
+                'filename': '<unknown>',
+                'function': '?',
+                'lineno': '?',
+                'context_line': '???',
+            }]
+
+        return frames
+
+    def format_exception(self):
+        """
+        Return the same data as from traceback.format_exception.
+        """
+        import traceback
+        frames = self.get_traceback_frames()
+        tb = [ (f['filename'], f['lineno'], f['function'], f['context_line']) for f in frames ]
+        list = ['Traceback (most recent call last):\n']
+        list += traceback.format_list(tb)
+        list += traceback.format_exception_only(self.exc_type, self.exc_value)
+        return list
+
+
+def technical_404_response(request, exception):
+    "Create a technical 404 error response. The exception should be the Http404."
+    try:
+        tried = exception.args[0]['tried']
+    except (IndexError, TypeError):
+        tried = []
+    else:
+        if not tried:
+            # tried exists but is an empty list. The URLconf must've been empty.
+            return empty_urlconf(request)
+
+    t = Template(TECHNICAL_404_TEMPLATE, name='Technical 404 template')
+    c = Context({
+        'root_urlconf': settings.ROOT_URLCONF,
+        'request_path': request.path[1:], # Trim leading slash
+        'urlpatterns': tried,
+        'reason': str(exception),
+        'request': request,
+        'request_protocol': request.is_secure() and "https" or "http",
+        'settings': get_safe_settings(),
+    })
+    return HttpResponseNotFound(t.render(c), mimetype='text/html')
+
+def empty_urlconf(request):
+    "Create an empty URLconf 404 error response."
+    t = Template(EMPTY_URLCONF_TEMPLATE, name='Empty URLConf template')
+    c = Context({
+        'project_name': settings.SETTINGS_MODULE.split('.')[0]
+    })
+    return HttpResponse(t.render(c), mimetype='text/html')
+
 #
 # Templates are embedded in the file so that we know the error handler will
 # always work even if the template loader is broken.

tests/regressiontests/queries/models.py

@@ -6,7 +6,7 @@ import datetime
 import pickle
 
 from django.db import models
-from django.db.models.query import Q
+from django.db.models.query import Q, ITER_CHUNK_SIZE
 
 # Python 2.3 doesn't have sorted()
 try:
@@ -820,5 +820,15 @@ Bug #7698 -- People like to slice with '0' as the high-water mark.
 >>> Item.objects.all()[0:0]
 []
 
+Bug #7411 - saving to db must work even with partially read result set in
+another cursor.
+
+>>> for num in range(2 * ITER_CHUNK_SIZE + 1):
+...     _ = Number.objects.create(num=num)
+
+>>> for i, obj in enumerate(Number.objects.all()):
+...     obj.save()
+...     if i > 10: break
+
 """}