newforms-admin: Merged from trunk up to [7928].

git-svn-id: http://code.djangoproject.com/svn/django/branches/newforms-admin@7937 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/core/handlers/base.py

@@ -60,7 +60,6 @@ class BaseHandler(object):
     def get_response(self, request):
         "Returns an HttpResponse object for the given HttpRequest"
         from django.core import exceptions, urlresolvers
-        from django.core.mail import mail_admins
         from django.conf import settings
 
         # Apply request middleware
@@ -122,21 +121,36 @@ class BaseHandler(object):
 
             if settings.DEBUG_PROPAGATE_EXCEPTIONS:
                 raise
-            elif settings.DEBUG:
+            return self.handle_uncaught_exception(request, resolver, exc_info)
-                from django.views import debug
+
-                return debug.technical_500_response(request, *exc_info)
+    def handle_uncaught_exception(self, request, resolver, exc_info):
-            else:
+        """
-                # When DEBUG is False, send an error message to the admins.
+        Processing for any otherwise uncaught exceptions (those that will
-                subject = 'Error (%s IP): %s' % ((request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS and 'internal' or 'EXTERNAL'), request.path)
+        generate HTTP 500 responses). Can be overridden by subclasses who want
-                try:
+        customised 500 handling.
-                    request_repr = repr(request)
+
-                except:
+        Be *very* careful when overriding this because the error could be
-                    request_repr = "Request repr() unavailable"
+        caused by anything, so assuming something like the database is always
-                message = "%s\n\n%s" % (self._get_traceback(exc_info), request_repr)
+        available would be an error.
-                mail_admins(subject, message, fail_silently=True)
+        """
-                # Return an HttpResponse that displays a friendly error message.
+        from django.conf import settings
-                callback, param_dict = resolver.resolve500()
+        from django.core.mail import mail_admins
-                return callback(request, **param_dict)
+
+        if settings.DEBUG:
+            from django.views import debug
+            return debug.technical_500_response(request, *exc_info)
+
+        # When DEBUG is False, send an error message to the admins.
+        subject = 'Error (%s IP): %s' % ((request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS and 'internal' or 'EXTERNAL'), request.path)
+        try:
+            request_repr = repr(request)
+        except:
+            request_repr = "Request repr() unavailable"
+        message = "%s\n\n%s" % (self._get_traceback(exc_info), request_repr)
+        mail_admins(subject, message, fail_silently=True)
+        # Return an HttpResponse that displays a friendly error message.
+        callback, param_dict = resolver.resolve500()
+        return callback(request, **param_dict)
 
     def _get_traceback(self, exc_info=None):
         "Helper function to return the traceback as a string"

django/db/backends/__init__.py

@@ -53,6 +53,7 @@ class BaseDatabaseFeatures(object):
     time_field_needs_date = False
     interprets_empty_strings_as_nulls = False
     date_field_supports_time_value = True
+    can_use_chunked_reads = True
 
 class BaseDatabaseOperations(object):
     """

django/db/backends/sqlite3/base.py

@@ -40,6 +40,11 @@ Database.register_adapter(decimal.Decimal, util.rev_typecast_decimal)
 
 class DatabaseFeatures(BaseDatabaseFeatures):
     supports_constraints = False
+    # SQLite cannot handle us only partially reading from a cursor's result set
+    # and then writing the same rows to the database in another cursor. This
+    # setting ensures we always read result sets fully into memory all in one
+    # go.
+    can_use_chunked_reads = False
 
 class DatabaseOperations(BaseDatabaseOperations):
     def date_extract_sql(self, lookup_type, field_name):

django/db/models/sql/query.py

@@ -1616,10 +1616,16 @@ class Query(object):
 
         # The MULTI case.
         if self.ordering_aliases:
-            return order_modified_iter(cursor, len(self.ordering_aliases),
+            result = order_modified_iter(cursor, len(self.ordering_aliases),
                     self.connection.features.empty_fetchmany_value)
-        return iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)),
+        result = iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)),
                 self.connection.features.empty_fetchmany_value)
+        if not self.connection.features.can_use_chunked_reads:
+            # If we are using non-chunked reads, we return the same data
+            # structure as normally, but ensure it is all read into memory
+            # before going any further.
+            return list(result)
+        return result
 
 # Use the backend's custom Query class if it defines one. Otherwise, use the
 # default.

django/views/debug.py

@@ -19,42 +19,6 @@ def linebreak_iter(template_source):
         p = template_source.find('\n', p+1)
     yield len(template_source) + 1
 
-def get_template_exception_info(exc_type, exc_value, tb):
-    origin, (start, end) = exc_value.source
-    template_source = origin.reload()
-    context_lines = 10
-    line = 0
-    upto = 0
-    source_lines = []
-    before = during = after = ""
-    for num, next in enumerate(linebreak_iter(template_source)):
-        if start >= upto and end <= next:
-            line = num
-            before = escape(template_source[upto:start])
-            during = escape(template_source[start:end])
-            after = escape(template_source[end:next])
-        source_lines.append( (num, escape(template_source[upto:next])) )
-        upto = next
-    total = len(source_lines)
-
-    top = max(1, line - context_lines)
-    bottom = min(total, line + 1 + context_lines)
-
-    template_info = {
-        'message': exc_value.args[0],
-        'source_lines': source_lines[top:bottom],
-        'before': before,
-        'during': during,
-        'after': after,
-        'top': top,
-        'bottom': bottom,
-        'total': total,
-        'line': line,
-        'name': origin.name,
-    }
-    exc_info = hasattr(exc_value, 'exc_info') and exc_value.exc_info or (exc_type, exc_value, tb)
-    return exc_info + (template_info,)
-
 def get_safe_settings():
     "Returns a dictionary of the settings module, with sensitive settings blurred out."
     settings_dict = {}
@@ -71,102 +35,212 @@ def technical_500_response(request, exc_type, exc_value, tb):
     Create a technical server error response. The last three arguments are
     the values returned from sys.exc_info() and friends.
     """
-    html = get_traceback_html(request, exc_type, exc_value, tb)
+    reporter = ExceptionReporter(request, exc_type, exc_value, tb)
+    html = reporter.get_traceback_html()
     return HttpResponseServerError(html, mimetype='text/html')
 
-def get_traceback_html(request, exc_type, exc_value, tb):
+class ExceptionReporter:
-    "Return HTML code for traceback."
+    """
-    template_info = None
+    A class to organize and coordinate reporting on exceptions.
-    template_does_not_exist = False
+    """
-    loader_debug_info = None
+    def __init__(self, request, exc_type, exc_value, tb):
+        self.request = request
+        self.exc_type = exc_type
+        self.exc_value = exc_value
+        self.tb = tb
 
-    # Handle deprecated string exceptions
+        self.template_info = None
-    if isinstance(exc_type, basestring):
+        self.template_does_not_exist = False
-        exc_value = Exception('Deprecated String Exception: %r' % exc_type)
+        self.loader_debug_info = None
-        exc_type = type(exc_value)
 
-    if issubclass(exc_type, TemplateDoesNotExist):
+        # Handle deprecated string exceptions
-        from django.template.loader import template_source_loaders
+        if isinstance(self.exc_type, basestring):
-        template_does_not_exist = True
+            self.exc_value = Exception('Deprecated String Exception: %r' % self.exc_type)
-        loader_debug_info = []
+            self.exc_type = type(self.exc_value)
-        for loader in template_source_loaders:
+
+    def get_traceback_html(self):
+        "Return HTML code for traceback."
+
+        if issubclass(self.exc_type, TemplateDoesNotExist):
+            from django.template.loader import template_source_loaders
+            self.template_does_not_exist = True
+            self.loader_debug_info = []
+            for loader in template_source_loaders:
+                try:
+                    source_list_func = getattr(__import__(loader.__module__, {}, {}, ['get_template_sources']), 'get_template_sources')
+                    # NOTE: This assumes exc_value is the name of the template that
+                    # the loader attempted to load.
+                    template_list = [{'name': t, 'exists': os.path.exists(t)} \
+                        for t in source_list_func(str(self.exc_value))]
+                except (ImportError, AttributeError):
+                    template_list = []
+                self.loader_debug_info.append({
+                    'loader': loader.__module__ + '.' + loader.__name__,
+                    'templates': template_list,
+                })
+        if settings.TEMPLATE_DEBUG and hasattr(self.exc_value, 'source'):
+            self.get_template_exception_info()
+
+        frames = self.get_traceback_frames()
+
+        unicode_hint = ''
+        if issubclass(self.exc_type, UnicodeError):
+            start = getattr(self.exc_value, 'start', None)
+            end = getattr(self.exc_value, 'end', None)
+            if start is not None and end is not None:
+                unicode_str = self.exc_value.args[1]
+                unicode_hint = smart_unicode(unicode_str[max(start-5, 0):min(end+5, len(unicode_str))], 'ascii', errors='replace')
+        from django import get_version
+        t = Template(TECHNICAL_500_TEMPLATE, name='Technical 500 template')
+        c = Context({
+            'exception_type': self.exc_type.__name__,
+            'exception_value': smart_unicode(self.exc_value, errors='replace'),
+            'unicode_hint': unicode_hint,
+            'frames': frames,
+            'lastframe': frames[-1],
+            'request': self.request,
+            'request_protocol': self.request.is_secure() and "https" or "http",
+            'settings': get_safe_settings(),
+            'sys_executable': sys.executable,
+            'sys_version_info': '%d.%d.%d' % sys.version_info[0:3],
+            'server_time': datetime.datetime.now(),
+            'django_version_info': get_version(),
+            'sys_path' : sys.path,
+            'template_info': self.template_info,
+            'template_does_not_exist': self.template_does_not_exist,
+            'loader_debug_info': self.loader_debug_info,
+        })
+        return t.render(c)
+
+    def get_template_exception_info(self):
+        origin, (start, end) = self.exc_value.source
+        template_source = origin.reload()
+        context_lines = 10
+        line = 0
+        upto = 0
+        source_lines = []
+        before = during = after = ""
+        for num, next in enumerate(linebreak_iter(template_source)):
+            if start >= upto and end <= next:
+                line = num
+                before = escape(template_source[upto:start])
+                during = escape(template_source[start:end])
+                after = escape(template_source[end:next])
+            source_lines.append( (num, escape(template_source[upto:next])) )
+            upto = next
+        total = len(source_lines)
+
+        top = max(1, line - context_lines)
+        bottom = min(total, line + 1 + context_lines)
+
+        self.template_info = {
+            'message': self.exc_value.args[0],
+            'source_lines': source_lines[top:bottom],
+            'before': before,
+            'during': during,
+            'after': after,
+            'top': top,
+            'bottom': bottom,
+            'total': total,
+            'line': line,
+            'name': origin.name,
+        }
+        if hasattr(self.exc_value, 'exc_info') and self.exc_value.exc_info:
+            exc_type, exc_value, tb = self.exc_value.exc_info
+
+    def _get_lines_from_file(self, filename, lineno, context_lines, loader=None, module_name=None):
+        """
+        Returns context_lines before and after lineno from file.
+        Returns (pre_context_lineno, pre_context, context_line, post_context).
+        """
+        source = None
+        if loader is not None and hasattr(loader, "get_source"):
+            source = loader.get_source(module_name)
+            if source is not None:
+                source = source.splitlines()
+        if source is None:
             try:
-                source_list_func = getattr(__import__(loader.__module__, {}, {}, ['get_template_sources']), 'get_template_sources')
+                f = open(filename)
-                # NOTE: This assumes exc_value is the name of the template that
+                try:
-                # the loader attempted to load.
+                    source = f.readlines()
-                template_list = [{'name': t, 'exists': os.path.exists(t)} \
+                finally:
-                    for t in source_list_func(str(exc_value))]
+                    f.close()
-            except (ImportError, AttributeError):
+            except (OSError, IOError):
-                template_list = []
+                pass
-            loader_debug_info.append({
+        if source is None:
-                'loader': loader.__module__ + '.' + loader.__name__,
+            return None, [], None, []
-                'templates': template_list,
+
-            })
+        encoding = 'ascii'
-    if settings.TEMPLATE_DEBUG and hasattr(exc_value, 'source'):
+        for line in source[:2]:
-        exc_type, exc_value, tb, template_info = get_template_exception_info(exc_type, exc_value, tb)
+            # File coding may be specified. Match pattern from PEP-263
-    frames = []
+            # (http://www.python.org/dev/peps/pep-0263/)
-    while tb is not None:
+            match = re.search(r'coding[:=]\s*([-\w.]+)', line)
-        # support for __traceback_hide__ which is used by a few libraries
+            if match:
-        # to hide internal frames.
+                encoding = match.group(1)
-        if tb.tb_frame.f_locals.get('__traceback_hide__'):
+                break
+        source = [unicode(sline, encoding, 'replace') for sline in source]
+
+        lower_bound = max(0, lineno - context_lines)
+        upper_bound = lineno + context_lines
+
+        pre_context = [line.strip('\n') for line in source[lower_bound:lineno]]
+        context_line = source[lineno].strip('\n')
+        post_context = [line.strip('\n') for line in source[lineno+1:upper_bound]]
+
+        return lower_bound, pre_context, context_line, post_context
+
+    def get_traceback_frames(self):
+        frames = []
+        tb = self.tb
+        while tb is not None:
+            # support for __traceback_hide__ which is used by a few libraries
+            # to hide internal frames.
+            if tb.tb_frame.f_locals.get('__traceback_hide__'):
+                tb = tb.tb_next
+                continue
+            filename = tb.tb_frame.f_code.co_filename
+            function = tb.tb_frame.f_code.co_name
+            lineno = tb.tb_lineno - 1
+            loader = tb.tb_frame.f_globals.get('__loader__')
+            module_name = tb.tb_frame.f_globals.get('__name__')
+            pre_context_lineno, pre_context, context_line, post_context = self._get_lines_from_file(filename, lineno, 7, loader, module_name)
+            if pre_context_lineno is not None:
+                frames.append({
+                    'tb': tb,
+                    'filename': filename,
+                    'function': function,
+                    'lineno': lineno + 1,
+                    'vars': tb.tb_frame.f_locals.items(),
+                    'id': id(tb),
+                    'pre_context': pre_context,
+                    'context_line': context_line,
+                    'post_context': post_context,
+                    'pre_context_lineno': pre_context_lineno + 1,
+                })
             tb = tb.tb_next
-            continue
-        filename = tb.tb_frame.f_code.co_filename
-        function = tb.tb_frame.f_code.co_name
-        lineno = tb.tb_lineno - 1
-        loader = tb.tb_frame.f_globals.get('__loader__')
-        module_name = tb.tb_frame.f_globals.get('__name__')
-        pre_context_lineno, pre_context, context_line, post_context = _get_lines_from_file(filename, lineno, 7, loader, module_name)
-        if pre_context_lineno is not None:
-            frames.append({
-                'tb': tb,
-                'filename': filename,
-                'function': function,
-                'lineno': lineno + 1,
-                'vars': tb.tb_frame.f_locals.items(),
-                'id': id(tb),
-                'pre_context': pre_context,
-                'context_line': context_line,
-                'post_context': post_context,
-                'pre_context_lineno': pre_context_lineno + 1,
-            })
-        tb = tb.tb_next
 
-    if not frames:
+        if not frames:
-        frames = [{
+            frames = [{
-            'filename': '&lt;unknown&gt;',
+                'filename': '&lt;unknown&gt;',
-            'function': '?',
+                'function': '?',
-            'lineno': '?',
+                'lineno': '?',
-        }]
+                'context_line': '???',
+            }]
+
+        return frames
+
+    def format_exception(self):
+        """
+        Return the same data as from traceback.format_exception.
+        """
+        import traceback
+        frames = self.get_traceback_frames()
+        tb = [ (f['filename'], f['lineno'], f['function'], f['context_line']) for f in frames ]
+        list = ['Traceback (most recent call last):\n']
+        list += traceback.format_list(tb)
+        list += traceback.format_exception_only(self.exc_type, self.exc_value)
+        return list
 
-    unicode_hint = ''
-    if issubclass(exc_type, UnicodeError):
-        start = getattr(exc_value, 'start', None)
-        end = getattr(exc_value, 'end', None)
-        if start is not None and end is not None:
-            unicode_str = exc_value.args[1]
-            unicode_hint = smart_unicode(unicode_str[max(start-5, 0):min(end+5, len(unicode_str))], 'ascii', errors='replace')
-    from django import get_version
-    t = Template(TECHNICAL_500_TEMPLATE, name='Technical 500 template')
-    c = Context({
-        'exception_type': exc_type.__name__,
-        'exception_value': smart_unicode(exc_value, errors='replace'),
-        'unicode_hint': unicode_hint,
-        'frames': frames,
-        'lastframe': frames[-1],
-        'request': request,
-        'request_protocol': request.is_secure() and "https" or "http",
-        'settings': get_safe_settings(),
-        'sys_executable': sys.executable,
-        'sys_version_info': '%d.%d.%d' % sys.version_info[0:3],
-        'server_time': datetime.datetime.now(),
-        'django_version_info': get_version(),
-        'sys_path' : sys.path,
-        'template_info': template_info,
-        'template_does_not_exist': template_does_not_exist,
-        'loader_debug_info': loader_debug_info,
-    })
-    return t.render(c)
 
 def technical_404_response(request, exception):
     "Create a technical 404 error response. The exception should be the Http404."
@@ -199,47 +273,6 @@ def empty_urlconf(request):
     })
     return HttpResponse(t.render(c), mimetype='text/html')
 
-def _get_lines_from_file(filename, lineno, context_lines, loader=None, module_name=None):
-    """
-    Returns context_lines before and after lineno from file.
-    Returns (pre_context_lineno, pre_context, context_line, post_context).
-    """
-    source = None
-    if loader is not None and hasattr(loader, "get_source"):
-        source = loader.get_source(module_name)
-        if source is not None:
-            source = source.splitlines()
-    if source is None:
-        try:
-            f = open(filename)
-            try:
-                source = f.readlines()
-            finally:
-                f.close()
-        except (OSError, IOError):
-            pass
-    if source is None:
-        return None, [], None, []
-
-    encoding = 'ascii'
-    for line in source[:2]:
-        # File coding may be specified. Match pattern from PEP-263
-        # (http://www.python.org/dev/peps/pep-0263/)
-        match = re.search(r'coding[:=]\s*([-\w.]+)', line)
-        if match:
-            encoding = match.group(1)
-            break
-    source = [unicode(sline, encoding, 'replace') for sline in source]
-
-    lower_bound = max(0, lineno - context_lines)
-    upper_bound = lineno + context_lines
-
-    pre_context = [line.strip('\n') for line in source[lower_bound:lineno]]
-    context_line = source[lineno].strip('\n')
-    post_context = [line.strip('\n') for line in source[lineno+1:upper_bound]]
-
-    return lower_bound, pre_context, context_line, post_context
-
 #
 # Templates are embedded in the file so that we know the error handler will
 # always work even if the template loader is broken.

tests/regressiontests/queries/models.py

@@ -6,14 +6,14 @@ import datetime
 import pickle
 
 from django.db import models
-from django.db.models.query import Q
+from django.db.models.query import Q, ITER_CHUNK_SIZE
 
 # Python 2.3 doesn't have sorted()
 try:
     sorted
 except NameError:
     from django.utils.itercompat import sorted
-                
+
 class Tag(models.Model):
     name = models.CharField(max_length=10)
     parent = models.ForeignKey('self', blank=True, null=True,
@@ -820,5 +820,15 @@ Bug #7698 -- People like to slice with '0' as the high-water mark.
 >>> Item.objects.all()[0:0]
 []
 
+Bug #7411 - saving to db must work even with partially read result set in
+another cursor.
+
+>>> for num in range(2 * ITER_CHUNK_SIZE + 1):
+...     _ = Number.objects.create(num=num)
+
+>>> for i, obj in enumerate(Number.objects.all()):
+...     obj.save()
+...     if i > 10: break
+
 """}