boulder-oracle-sprint: Merged to [5490]

git-svn-id: http://code.djangoproject.com/svn/django/branches/boulder-oracle-sprint@5491 bcc190cf-cafb-0310-a4f2-bffc1f526a37

AUTHORS

@@ -221,6 +221,7 @@ answer newbie questions, and generally made Django that much better:
     Aaron Swartz <http://www.aaronsw.com/>
     Ville Säävuori <http://www.unessa.net/>
     Tyson Tate <tyson@fallingbullets.com>
+    Frank Tegtmeyer <fte@fte.to>
     thebjorn <bp@datakortet.no>
     Zach Thompson <zthompson47@gmail.com>
     Tom Tobin

django/conf/locale/sv/LC_MESSAGES/djangojs.po

@@ -2,9 +2,6 @@
 # Copyright (C) 2005
 # This file is distributed under the same license as the Django package.
 #
-#
-# Robin Sonefors <ozamosi@blinkenlights.se>, 2005.
-# Mikko Hellsing <mikko@sorl.net>, 2007.
 msgid ""
 msgstr ""
 "Project-Id-Version: djangojs\n"

django/contrib/admin/templatetags/admin_modify.py

@@ -94,15 +94,15 @@ class FieldWidgetNode(template.Node):
             return cls.nodelists[klass]
     get_nodelist = classmethod(get_nodelist)
 
-    def render(self, context):
+    def iter_render(self, context):
         bound_field = template.resolve_variable(self.bound_field_var, context)
 
         context.push()
         context['bound_field'] = bound_field
 
-        output = self.get_nodelist(bound_field.field.__class__).render(context)
+        for chunk in self.get_nodelist(bound_field.field.__class__).iter_render(context):
+            yield chunk
         context.pop()
-        return output
 
 class FieldWrapper(object):
     def __init__(self, field ):
@@ -157,7 +157,7 @@ class EditInlineNode(template.Node):
     def __init__(self, rel_var):
         self.rel_var = rel_var
 
-    def render(self, context):
+    def iter_render(self, context):
         relation = template.resolve_variable(self.rel_var, context)
         context.push()
         if relation.field.rel.edit_inline == models.TABULAR:
@@ -169,10 +169,9 @@ class EditInlineNode(template.Node):
         original = context.get('original', None)
         bound_related_object = relation.bind(context['form'], original, bound_related_object_class)
         context['bound_related_object'] = bound_related_object
-        t = loader.get_template(bound_related_object.template_name())
+        for chunk in loader.get_template(bound_related_object.template_name()).iter_render(context):
-        output = t.render(context)
+            yield chunk
         context.pop()
-        return output
 
 def output_all(form_fields):
     return ''.join([str(f) for f in form_fields])

django/contrib/admin/templatetags/adminapplist.py

@@ -7,7 +7,7 @@ class AdminApplistNode(template.Node):
     def __init__(self, varname):
         self.varname = varname
 
-    def render(self, context):
+    def iter_render(self, context):
         from django.db import models
         from django.utils.text import capfirst
         app_list = []
@@ -54,7 +54,7 @@ class AdminApplistNode(template.Node):
                         'models': model_list,
                     })
         context[self.varname] = app_list
-        return ''
+        return ()
 
 def get_admin_app_list(parser, token):
     """

django/contrib/admin/templatetags/log.py

@@ -10,14 +10,14 @@ class AdminLogNode(template.Node):
     def __repr__(self):
         return "<GetAdminLog Node>"
 
-    def render(self, context):
+    def iter_render(self, context):
         if self.user is None:
             context[self.varname] = LogEntry.objects.all().select_related()[:self.limit]
         else:
             if not self.user.isdigit():
                 self.user = context[self.user].id
             context[self.varname] = LogEntry.objects.filter(user__id__exact=self.user).select_related()[:self.limit]
-        return ''
+        return ()
 
 class DoGetAdminLog:
     """

django/contrib/auth/__init__.py

@@ -53,6 +53,8 @@ def login(request, user):
     user.save()
     request.session[SESSION_KEY] = user.id
     request.session[BACKEND_SESSION_KEY] = user.backend
+    if hasattr(request, 'user'):
+        request.user = user
 
 def logout(request):
     """
@@ -66,6 +68,9 @@ def logout(request):
         del request.session[BACKEND_SESSION_KEY]
     except KeyError:
         pass
+    if hasattr(request, 'user'):
+        from django.contrib.auth.models import AnonymousUser
+        request.user = AnonymousUser()
 
 def get_user(request):
     from django.contrib.auth.models import AnonymousUser

django/contrib/comments/templatetags/comments.py

@@ -24,7 +24,7 @@ class CommentFormNode(template.Node):
         self.photo_options, self.rating_options = photo_options, rating_options
         self.is_public = is_public
 
-    def render(self, context):
+    def iter_render(self, context):
         from django.conf import settings
         from django.utils.text import normalize_newlines
         import base64
@@ -33,7 +33,7 @@ class CommentFormNode(template.Node):
             try:
                 self.obj_id = template.resolve_variable(self.obj_id_lookup_var, context)
             except template.VariableDoesNotExist:
-                return ''
+                return
             # Validate that this object ID is valid for this content-type.
             # We only have to do this validation if obj_id_lookup_var is provided,
             # because do_comment_form() validates hard-coded object IDs.
@@ -67,9 +67,9 @@ class CommentFormNode(template.Node):
             context['hash'] = Comment.objects.get_security_hash(context['options'], context['photo_options'], context['rating_options'], context['target'])
             context['logout_url'] = settings.LOGOUT_URL
             default_form = loader.get_template(COMMENT_FORM)
-        output = default_form.render(context)
+        for chunk in default_form.iter_render(context):
+            yield chunk
         context.pop()
-        return output
 
 class CommentCountNode(template.Node):
     def __init__(self, package, module, context_var_name, obj_id, var_name, free):
@@ -77,7 +77,7 @@ class CommentCountNode(template.Node):
         self.context_var_name, self.obj_id = context_var_name, obj_id
         self.var_name, self.free = var_name, free
 
-    def render(self, context):
+    def iter_render(self, context):
         from django.conf import settings
         manager = self.free and FreeComment.objects or Comment.objects
         if self.context_var_name is not None:
@@ -86,7 +86,7 @@ class CommentCountNode(template.Node):
             content_type__app_label__exact=self.package,
             content_type__model__exact=self.module, site__id__exact=settings.SITE_ID).count()
         context[self.var_name] = comment_count
-        return ''
+        return ()
 
 class CommentListNode(template.Node):
     def __init__(self, package, module, context_var_name, obj_id, var_name, free, ordering, extra_kwargs=None):
@@ -96,14 +96,14 @@ class CommentListNode(template.Node):
         self.ordering = ordering
         self.extra_kwargs = extra_kwargs or {}
 
-    def render(self, context):
+    def iter_render(self, context):
         from django.conf import settings
         get_list_function = self.free and FreeComment.objects.filter or Comment.objects.get_list_with_karma
         if self.context_var_name is not None:
             try:
                 self.obj_id = template.resolve_variable(self.context_var_name, context)
             except template.VariableDoesNotExist:
-                return ''
+                return ()
         kwargs = {
             'object_id__exact': self.obj_id,
             'content_type__app_label__exact': self.package,
@@ -127,7 +127,7 @@ class CommentListNode(template.Node):
                 comment_list = [c for c in comment_list if not c.is_hidden or (user_id == c.user_id)]
 
         context[self.var_name] = comment_list
-        return ''
+        return ()
 
 class DoCommentForm:
     """

django/contrib/sessions/models.py

@@ -1,4 +1,4 @@
-import base64, md5, random, sys, datetime
+import base64, md5, random, sys, datetime, os, time
 import cPickle as pickle
 from django.db import models
 from django.utils.translation import gettext_lazy as _
@@ -14,9 +14,9 @@ class SessionManager(models.Manager):
     def get_new_session_key(self):
         "Returns session key that isn't being used."
         # The random module is seeded when this Apache child is created.
-        # Use person_id and SECRET_KEY as added salt.
+        # Use SECRET_KEY as added salt.
         while 1:
-            session_key = md5.new(str(random.randint(0, sys.maxint - 1)) + str(random.randint(0, sys.maxint - 1)) + settings.SECRET_KEY).hexdigest()
+            session_key = md5.new("%s%s%s%s" % (random.randint(0, sys.maxint - 1), os.getpid(), time.time(), settings.SECRET_KEY)).hexdigest()
             try:
                 self.get(session_key=session_key)
             except self.model.DoesNotExist:

django/core/servers/basehttp.py

@@ -309,7 +309,7 @@ class ServerHandler(object):
         """
         if not self.result_is_file() and not self.sendfile():
             for data in self.result:
-                self.write(data)
+                self.write(data, False)
             self.finish_content()
         self.close()
 
@@ -377,7 +377,7 @@ class ServerHandler(object):
         else:
             self._write('Status: %s\r\n' % self.status)
 
-    def write(self, data):
+    def write(self, data, flush=True):
         """'write()' callable as specified by PEP 333"""
 
         assert type(data) is StringType,"write() argument must be string"
@@ -394,7 +394,8 @@ class ServerHandler(object):
 
         # XXX check Content-Length and truncate if too many bytes written?
         self._write(data)
-        self._flush()
+        if flush:
+            self._flush()
 
     def sendfile(self):
         """Platform-specific file transmission
@@ -421,8 +422,6 @@ class ServerHandler(object):
         if not self.headers_sent:
             self.headers['Content-Length'] = "0"
             self.send_headers()
-        else:
-            pass # XXX check if content-length was too short?
 
     def close(self):
         try:

django/http/__init__.py

@@ -222,6 +222,12 @@ class HttpResponse(object):
         content = ''.join(self._container)
         if isinstance(content, unicode):
             content = content.encode(self._charset)
+
+        # If self._container was an iterator, we have just exhausted it, so we
+        # need to save the results for anything else that needs access
+        if not self._is_string:
+            self._container = [content]
+            self._is_string = True
         return content
 
     def _set_content(self, value):
@@ -231,14 +237,10 @@ class HttpResponse(object):
     content = property(_get_content, _set_content)
 
     def __iter__(self):
-        self._iterator = self._container.__iter__()
+        for chunk in self._container:
-        return self
+            if isinstance(chunk, unicode):
-
+                chunk = chunk.encode(self._charset)
-    def next(self):
+            yield chunk
-        chunk = self._iterator.next()
-        if isinstance(chunk, unicode):
-            chunk = chunk.encode(self._charset)
-        return chunk
 
     def close(self):
         if hasattr(self._container, 'close'):

django/middleware/common.py

@@ -11,7 +11,8 @@ class CommonMiddleware(object):
         - Forbids access to User-Agents in settings.DISALLOWED_USER_AGENTS
 
         - URL rewriting: Based on the APPEND_SLASH and PREPEND_WWW settings,
-          this middleware appends missing slashes and/or prepends missing "www."s.
+          this middleware appends missing slashes and/or prepends missing
+          "www."s.
 
         - ETags: If the USE_ETAGS setting is set, ETags will be calculated from
           the entire page content and Not Modified responses will be returned
@@ -74,7 +75,10 @@ class CommonMiddleware(object):
 
         # Use ETags, if requested.
         if settings.USE_ETAGS:
-            etag = md5.new(response.content).hexdigest()
+            if response.has_header('ETag'):
+                etag = response['ETag']
+            else:
+                etag = md5.new(response.content).hexdigest()
             if response.status_code >= 200 and response.status_code < 300 and request.META.get('HTTP_IF_NONE_MATCH') == etag:
                 response = http.HttpResponseNotModified()
             else:

django/newforms/models.py

@@ -19,9 +19,8 @@ def save_instance(form, instance, fields=None, fail_message='saved', commit=True
     """
     Saves bound Form ``form``'s cleaned_data into model instance ``instance``.
 
-    Assumes ``form`` has a field for every non-AutoField database field in
+    If commit=True, then the changes to ``instance`` will be saved to the
-    ``instance``. If commit=True, then the changes to ``instance`` will be
+    database. Returns ``instance``.
-    saved to the database. Returns ``instance``.
     """
     from django.db import models
     opts = instance.__class__._meta

django/oldforms/__init__.py

@@ -309,6 +309,10 @@ class FormField(object):
         return data
     html2python = staticmethod(html2python)
 
+    def iter_render(self, data):
+        # this even needed?
+        return (self.render(data),)
+
     def render(self, data):
         raise NotImplementedError
 

django/shortcuts/__init__.py

@@ -7,7 +7,7 @@ from django.http import HttpResponse, Http404
 from django.db.models.manager import Manager
 
 def render_to_response(*args, **kwargs):
-    return HttpResponse(loader.render_to_string(*args, **kwargs))
+    return HttpResponse(loader.render_to_iter(*args, **kwargs))
 load_and_render = render_to_response # For backwards compatibility.
 
 def get_object_or_404(klass, *args, **kwargs):

django/template/__init__.py

@@ -55,6 +55,7 @@ times with multiple contexts)
 '\n<html>\n\n</html>\n'
 """
 import re
+import types
 from inspect import getargspec
 from django.conf import settings
 from django.template.context import Context, RequestContext, ContextPopException
@@ -167,9 +168,12 @@ class Template(object):
             for subnode in node:
                 yield subnode
 
-    def render(self, context):
+    def iter_render(self, context):
         "Display stage -- can be called many times"
-        return self.nodelist.render(context)
+        return self.nodelist.iter_render(context)
+
+    def render(self, context):
+        return ''.join(self.iter_render(context))
 
 def compile_string(template_string, origin):
     "Compiles template_string into NodeList ready for rendering"
@@ -488,9 +492,6 @@ class TokenParser(object):
             self.pointer = i
             return s
 
-
-
-
 filter_raw_string = r"""
 ^%(i18n_open)s"(?P<i18n_constant>%(str)s)"%(i18n_close)s|
 ^"(?P<constant>%(str)s)"|
@@ -698,10 +699,26 @@ def resolve_variable(path, context):
             del bits[0]
     return current
 
+class NodeBase(type):
+    def __new__(cls, name, bases, attrs):
+        """
+        Ensures that either a 'render' or 'render_iter' method is defined on
+        any Node sub-class. This avoids potential infinite loops at runtime.
+        """
+        if not (isinstance(attrs.get('render'), types.FunctionType) or
+                isinstance(attrs.get('iter_render'), types.FunctionType)):
+            raise TypeError('Unable to create Node subclass without either "render" or "iter_render" method.')
+        return type.__new__(cls, name, bases, attrs)
+
 class Node(object):
+    __metaclass__ = NodeBase
+
+    def iter_render(self, context):
+        return (self.render(context),)
+
     def render(self, context):
         "Return the node rendered as a string"
-        pass
+        return ''.join(self.iter_render(context))
 
     def __iter__(self):
         yield self
@@ -717,13 +734,12 @@ class Node(object):
 
 class NodeList(list):
     def render(self, context):
-        bits = []
+        return ''.join(self.iter_render(context))
+
+    def iter_render(self, context):
         for node in self:
-            if isinstance(node, Node):
+            for chunk in node.iter_render(context):
-                bits.append(self.render_node(node, context))
+                yield chunk
-            else:
-                bits.append(node)
-        return ''.join(bits)
 
     def get_nodes_by_type(self, nodetype):
         "Return a list of all nodes of the given type"
@@ -732,24 +748,25 @@ class NodeList(list):
             nodes.extend(node.get_nodes_by_type(nodetype))
         return nodes
 
-    def render_node(self, node, context):
-        return(node.render(context))
-
 class DebugNodeList(NodeList):
-    def render_node(self, node, context):
+    def iter_render(self, context):
-        try:
+        for node in self:
-            result = node.render(context)
+            if not isinstance(node, Node):
-        except TemplateSyntaxError, e:
+                yield node
-            if not hasattr(e, 'source'):
+                continue
-                e.source = node.source
+            try:
-            raise
+                for chunk in node.iter_render(context):
-        except Exception, e:
+                    yield chunk
-            from sys import exc_info
+            except TemplateSyntaxError, e:
-            wrapped = TemplateSyntaxError('Caught an exception while rendering: %s' % e)
+                if not hasattr(e, 'source'):
-            wrapped.source = node.source
+                    e.source = node.source
-            wrapped.exc_info = exc_info()
+                raise
-            raise wrapped
+            except Exception, e:
-        return result
+                from sys import exc_info
+                wrapped = TemplateSyntaxError('Caught an exception while rendering: %s' % e)
+                wrapped.source = node.source
+                wrapped.exc_info = exc_info()
+                raise wrapped
 
 class TextNode(Node):
     def __init__(self, s):
@@ -758,6 +775,9 @@ class TextNode(Node):
     def __repr__(self):
         return "<Text Node: '%s'>" % self.s[:25]
 
+    def iter_render(self, context):
+        return (self.s,)
+
     def render(self, context):
         return self.s
 
@@ -781,6 +801,9 @@ class VariableNode(Node):
         else:
             return output
 
+    def iter_render(self, context):
+        return (self.render(context),)
+
     def render(self, context):
         output = self.filter_expression.resolve(context)
         return self.encode_output(output)
@@ -869,6 +892,9 @@ class Library(object):
             def __init__(self, vars_to_resolve):
                 self.vars_to_resolve = vars_to_resolve
 
+            #def iter_render(self, context):
+            #    return (self.render(context),)
+
             def render(self, context):
                 resolved_vars = [resolve_variable(var, context) for var in self.vars_to_resolve]
                 return func(*resolved_vars)
@@ -891,7 +917,7 @@ class Library(object):
                 def __init__(self, vars_to_resolve):
                     self.vars_to_resolve = vars_to_resolve
 
-                def render(self, context):
+                def iter_render(self, context):
                     resolved_vars = [resolve_variable(var, context) for var in self.vars_to_resolve]
                     if takes_context:
                         args = [context] + resolved_vars
@@ -907,7 +933,7 @@ class Library(object):
                         else:
                             t = get_template(file_name)
                         self.nodelist = t.nodelist
-                    return self.nodelist.render(context_class(dict))
+                    return self.nodelist.iter_render(context_class(dict))
 
             compile_func = curry(generic_tag_compiler, params, defaults, getattr(func, "_decorated_function", func).__name__, InclusionNode)
             compile_func.__doc__ = func.__doc__

django/template/defaulttags.py

@@ -4,6 +4,7 @@ from django.template import Node, NodeList, Template, Context, resolve_variable
 from django.template import TemplateSyntaxError, VariableDoesNotExist, BLOCK_TAG_START, BLOCK_TAG_END, VARIABLE_TAG_START, VARIABLE_TAG_END, SINGLE_BRACE_START, SINGLE_BRACE_END, COMMENT_TAG_START, COMMENT_TAG_END
 from django.template import get_library, Library, InvalidTemplateLibrary
 from django.conf import settings
+from django.utils.itercompat import groupby
 import sys
 import re
 
@@ -14,12 +15,11 @@ if not hasattr(__builtins__, 'reversed'):
         for index in xrange(len(data)-1, -1, -1):
             yield data[index]
 
-
 register = Library()
 
 class CommentNode(Node):
-    def render(self, context):
+    def iter_render(self, context):
-        return ''
+        return ()
 
 class CycleNode(Node):
     def __init__(self, cyclevars, variable_name=None):
@@ -28,6 +28,9 @@ class CycleNode(Node):
         self.counter = -1
         self.variable_name = variable_name
 
+    def iter_render(self, context):
+        return (self.render(context),)
+
     def render(self, context):
         self.counter += 1
         value = self.cyclevars[self.counter % self.cyclevars_len]
@@ -36,29 +39,32 @@ class CycleNode(Node):
         return value
 
 class DebugNode(Node):
-    def render(self, context):
+    def iter_render(self, context):
         from pprint import pformat
-        output = [pformat(val) for val in context]
+        for val in context:
-        output.append('\n\n')
+            yield pformat(val)
-        output.append(pformat(sys.modules))
+        yield "\n\n"
-        return ''.join(output)
+        yield pformat(sys.modules)
 
 class FilterNode(Node):
     def __init__(self, filter_expr, nodelist):
         self.filter_expr, self.nodelist = filter_expr, nodelist
 
-    def render(self, context):
+    def iter_render(self, context):
         output = self.nodelist.render(context)
         # apply filters
         context.update({'var': output})
         filtered = self.filter_expr.resolve(context)
         context.pop()
-        return filtered
+        return (filtered,)
 
 class FirstOfNode(Node):
     def __init__(self, vars):
         self.vars = vars
 
+    def iter_render(self, context):
+        return (self.render(context),)
+
     def render(self, context):
         for var in self.vars:
             try:
@@ -94,8 +100,7 @@ class ForNode(Node):
         nodes.extend(self.nodelist_loop.get_nodes_by_type(nodetype))
         return nodes
 
-    def render(self, context):
+    def iter_render(self, context):
-        nodelist = NodeList()
         if 'forloop' in context:
             parentloop = context['forloop']
         else:
@@ -103,12 +108,12 @@ class ForNode(Node):
         context.push()
         try:
             values = self.sequence.resolve(context, True)
+            if values is None:
+                values = ()
+            elif not hasattr(values, '__len__'):
+                values = list(values)
         except VariableDoesNotExist:
-            values = []
+            values = ()
-        if values is None:
-            values = []
-        if not hasattr(values, '__len__'):
-            values = list(values)
         len_values = len(values)
         if self.reversed:
             values = reversed(values)
@@ -127,12 +132,17 @@ class ForNode(Node):
                 'parentloop': parentloop,
             }
             if unpack:
-                # If there are multiple loop variables, unpack the item into them.
+                # If there are multiple loop variables, unpack the item into
+                # them.
                 context.update(dict(zip(self.loopvars, item)))
             else:
                 context[self.loopvars[0]] = item
+
+            # We inline this to avoid the overhead since ForNode is pretty
+            # common.
             for node in self.nodelist_loop:
-                nodelist.append(node.render(context))
+                for chunk in node.iter_render(context):
+                    yield chunk
             if unpack:
                 # The loop variables were pushed on to the context so pop them
                 # off again. This is necessary because the tag lets the length
@@ -141,7 +151,6 @@ class ForNode(Node):
                 # context.
                 context.pop()
         context.pop()
-        return nodelist.render(context)
 
 class IfChangedNode(Node):
     def __init__(self, nodelist, *varlist):
@@ -149,7 +158,7 @@ class IfChangedNode(Node):
         self._last_seen = None
         self._varlist = varlist
 
-    def render(self, context):
+    def iter_render(self, context):
         if 'forloop' in context and context['forloop']['first']:
             self._last_seen = None
         try:
@@ -167,11 +176,9 @@ class IfChangedNode(Node):
             self._last_seen = compare_to
             context.push()
             context['ifchanged'] = {'firstloop': firstloop}
-            content = self.nodelist.render(context)
+            for chunk in self.nodelist.iter_render(context):
+                yield chunk
             context.pop()
-            return content
-        else:
-            return ''
 
 class IfEqualNode(Node):
     def __init__(self, var1, var2, nodelist_true, nodelist_false, negate):
@@ -182,7 +189,7 @@ class IfEqualNode(Node):
     def __repr__(self):
         return "<IfEqualNode>"
 
-    def render(self, context):
+    def iter_render(self, context):
         try:
             val1 = resolve_variable(self.var1, context)
         except VariableDoesNotExist:
@@ -192,8 +199,8 @@ class IfEqualNode(Node):
         except VariableDoesNotExist:
             val2 = None
         if (self.negate and val1 != val2) or (not self.negate and val1 == val2):
-            return self.nodelist_true.render(context)
+            return self.nodelist_true.iter_render(context)
-        return self.nodelist_false.render(context)
+        return self.nodelist_false.iter_render(context)
 
 class IfNode(Node):
     def __init__(self, bool_exprs, nodelist_true, nodelist_false, link_type):
@@ -218,7 +225,7 @@ class IfNode(Node):
         nodes.extend(self.nodelist_false.get_nodes_by_type(nodetype))
         return nodes
 
-    def render(self, context):
+    def iter_render(self, context):
         if self.link_type == IfNode.LinkTypes.or_:
             for ifnot, bool_expr in self.bool_exprs:
                 try:
@@ -226,8 +233,8 @@ class IfNode(Node):
                 except VariableDoesNotExist:
                     value = None
                 if (value and not ifnot) or (ifnot and not value):
-                    return self.nodelist_true.render(context)
+                    return self.nodelist_true.iter_render(context)
-            return self.nodelist_false.render(context)
+            return self.nodelist_false.iter_render(context)
         else:
             for ifnot, bool_expr in self.bool_exprs:
                 try:
@@ -235,8 +242,8 @@ class IfNode(Node):
                 except VariableDoesNotExist:
                     value = None
                 if not ((value and not ifnot) or (ifnot and not value)):
-                    return self.nodelist_false.render(context)
+                    return self.nodelist_false.iter_render(context)
-            return self.nodelist_true.render(context)
+            return self.nodelist_true.iter_render(context)
 
     class LinkTypes:
         and_ = 0,
@@ -247,21 +254,16 @@ class RegroupNode(Node):
         self.target, self.expression = target, expression
         self.var_name = var_name
 
-    def render(self, context):
+    def iter_render(self, context):
         obj_list = self.target.resolve(context, True)
         if obj_list == None: # target_var wasn't found in context; fail silently
             context[self.var_name] = []
-            return ''
+            return ()
-        output = [] # list of dictionaries in the format {'grouper': 'key', 'list': [list of contents]}
+        # List of dictionaries in the format
-        for obj in obj_list:
+        # {'grouper': 'key', 'list': [list of contents]}.
-            grouper = self.expression.resolve(obj, True)
+        context[self.var_name] = [{'grouper':key, 'list':list(val)} for key, val in
-            # TODO: Is this a sensible way to determine equality?
+            groupby(obj_list, lambda v, f=self.expression.resolve: f(v, True))]
-            if output and repr(output[-1]['grouper']) == repr(grouper):
+        return ()
-                output[-1]['list'].append(obj)
-            else:
-                output.append({'grouper': grouper, 'list': [obj]})
-        context[self.var_name] = output
-        return ''
 
 def include_is_allowed(filepath):
     for root in settings.ALLOWED_INCLUDE_ROOTS:
@@ -273,10 +275,10 @@ class SsiNode(Node):
     def __init__(self, filepath, parsed):
         self.filepath, self.parsed = filepath, parsed
 
-    def render(self, context):
+    def iter_render(self, context):
         if not include_is_allowed(self.filepath):
             if settings.DEBUG:
-                return "[Didn't have permission to include file]"
+                return ("[Didn't have permission to include file]",)
             else:
                 return '' # Fail silently for invalid includes.
         try:
@@ -287,23 +289,25 @@ class SsiNode(Node):
             output = ''
         if self.parsed:
             try:
-                t = Template(output, name=self.filepath)
+                return Template(output, name=self.filepath).iter_render(context)
-                return t.render(context)
             except TemplateSyntaxError, e:
                 if settings.DEBUG:
                     return "[Included template had syntax error: %s]" % e
                 else:
                     return '' # Fail silently for invalid included templates.
-        return output
+        return (output,)
 
 class LoadNode(Node):
-    def render(self, context):
+    def iter_render(self, context):
-        return ''
+        return ()
 
 class NowNode(Node):
     def __init__(self, format_string):
         self.format_string = format_string
 
+    def iter_render(self, context):
+        return (self.render(context),)
+
     def render(self, context):
         from datetime import datetime
         from django.utils.dateformat import DateFormat
@@ -332,6 +336,9 @@ class TemplateTagNode(Node):
     def __init__(self, tagtype):
         self.tagtype = tagtype
 
+    def iter_render(self, context):
+        return (self.render(context),)
+
     def render(self, context):
         return self.mapping.get(self.tagtype, '')
 
@@ -341,18 +348,18 @@ class URLNode(Node):
         self.args = args
         self.kwargs = kwargs
 
-    def render(self, context):
+    def iter_render(self, context):
         from django.core.urlresolvers import reverse, NoReverseMatch
         args = [arg.resolve(context) for arg in self.args]
         kwargs = dict([(k, v.resolve(context)) for k, v in self.kwargs.items()])
         try:
-            return reverse(self.view_name, args=args, kwargs=kwargs)
+            return (reverse(self.view_name, args=args, kwargs=kwargs),)
         except NoReverseMatch:
             try:
                 project_name = settings.SETTINGS_MODULE.split('.')[0]
                 return reverse(project_name + '.' + self.view_name, args=args, kwargs=kwargs)
             except NoReverseMatch:
-                return ''
+                return ()
 
 class WidthRatioNode(Node):
     def __init__(self, val_expr, max_expr, max_width):
@@ -360,6 +367,9 @@ class WidthRatioNode(Node):
         self.max_expr = max_expr
         self.max_width = max_width
 
+    def iter_render(self, context):
+        return (self.render(context),)
+
     def render(self, context):
         try:
             value = self.val_expr.resolve(context)
@@ -383,13 +393,13 @@ class WithNode(Node):
     def __repr__(self):
         return "<WithNode>"
 
-    def render(self, context):
+    def iter_render(self, context):
         val = self.var.resolve(context)
         context.push()
         context[self.name] = val
-        output = self.nodelist.render(context)
+        for chunk in self.nodelist.iter_render(context):
+            yield chunk
         context.pop()
-        return output
 
 #@register.tag
 def comment(parser, token):

django/template/loader.py

@@ -87,14 +87,12 @@ def get_template_from_string(source, origin=None, name=None):
     """
     return Template(source, origin, name)
 
-def render_to_string(template_name, dictionary=None, context_instance=None):
+def _render_setup(template_name, dictionary=None, context_instance=None):
     """
-    Loads the given template_name and renders it with the given dictionary as
+    Common setup code for render_to_string and render_to_iter.
-    context. The template_name may be a string to load a single template using
-    get_template, or it may be a tuple to use select_template to find one of
-    the templates in the list. Returns a string.
     """
-    dictionary = dictionary or {}
+    if dictionary is None:
+        dictionary = {}
     if isinstance(template_name, (list, tuple)):
         t = select_template(template_name)
     else:
@@ -103,7 +101,28 @@ def render_to_string(template_name, dictionary=None, context_instance=None):
         context_instance.update(dictionary)
     else:
         context_instance = Context(dictionary)
-    return t.render(context_instance)
+    return t, context_instance
+
+def render_to_string(template_name, dictionary=None, context_instance=None):
+    """
+    Loads the given template_name and renders it with the given dictionary as
+    context. The template_name may be a string to load a single template using
+    get_template, or it may be a tuple to use select_template to find one of
+    the templates in the list. Returns a string.
+    """
+    t, c = _render_setup(template_name, dictionary=dictionary, context_instance=context_instance)
+    return t.render(c)
+
+def render_to_iter(template_name, dictionary=None, context_instance=None):
+    """
+    Loads the given template_name and renders it with the given dictionary as
+    context. The template_name may be a string to load a single template using
+    get_template, or it may be a tuple to use select_template to find one of
+    the templates in the list. Returns a string.
+    """
+    t, c = _render_setup(template_name, dictionary=dictionary, context_instance=context_instance)
+    return t.iter_render(c)
+
 
 def select_template(template_name_list):
     "Given a list of template names, returns the first that can be loaded."

django/template/loader_tags.py

@@ -15,14 +15,14 @@ class BlockNode(Node):
     def __repr__(self):
         return "<Block Node: %s. Contents: %r>" % (self.name, self.nodelist)
 
-    def render(self, context):
+    def iter_render(self, context):
         context.push()
         # Save context in case of block.super().
         self.context = context
         context['block'] = self
-        result = self.nodelist.render(context)
+        for chunk in self.nodelist.iter_render(context):
+            yield chunk
         context.pop()
-        return result
 
     def super(self):
         if self.parent:
@@ -59,7 +59,7 @@ class ExtendsNode(Node):
         else:
             return get_template_from_string(source, origin, parent)
 
-    def render(self, context):
+    def iter_render(self, context):
         compiled_parent = self.get_parent(context)
         parent_is_child = isinstance(compiled_parent.nodelist[0], ExtendsNode)
         parent_blocks = dict([(n.name, n) for n in compiled_parent.nodelist.get_nodes_by_type(BlockNode)])
@@ -79,7 +79,7 @@ class ExtendsNode(Node):
                 parent_block.parent = block_node.parent
                 parent_block.add_parent(parent_block.nodelist)
                 parent_block.nodelist = block_node.nodelist
-        return compiled_parent.render(context)
+        return compiled_parent.iter_render(context)
 
 class ConstantIncludeNode(Node):
     def __init__(self, template_path):
@@ -91,27 +91,26 @@ class ConstantIncludeNode(Node):
                 raise
             self.template = None
 
-    def render(self, context):
+    def iter_render(self, context):
         if self.template:
-            return self.template.render(context)
+            return self.template.iter_render(context)
-        else:
+        return ()
-            return ''
 
 class IncludeNode(Node):
     def __init__(self, template_name):
         self.template_name = template_name
 
-    def render(self, context):
+    def iter_render(self, context):
         try:
             template_name = resolve_variable(self.template_name, context)
             t = get_template(template_name)
-            return t.render(context)
+            return t.iter_render(context)
         except TemplateSyntaxError, e:
             if settings.TEMPLATE_DEBUG:
                 raise
-            return ''
+            return ()
         except:
-            return '' # Fail silently for invalid included templates.
+            return () # Fail silently for invalid included templates.
 
 def do_block(parser, token):
     """

django/test/utils.py

@@ -11,12 +11,21 @@ from django.template import Template
 TEST_DATABASE_PREFIX = 'test_'
 
 def instrumented_test_render(self, context):
-    """An instrumented Template render method, providing a signal 
+    """
-    that can be intercepted by the test system Client
+    An instrumented Template render method, providing a signal that can be
-    
+    intercepted by the test system Client.
     """
     dispatcher.send(signal=signals.template_rendered, sender=self, template=self, context=context)
     return self.nodelist.render(context)
+
+def instrumented_test_iter_render(self, context):
+    """
+    An instrumented Template iter_render method, providing a signal that can be
+    intercepted by the test system Client.
+    """
+    for chunk in self.nodelist.iter_render(context):
+        yield chunk
+    dispatcher.send(signal=signals.template_rendered, sender=self, template=self, context=context)
     
 class TestSMTPConnection(object):
     """A substitute SMTP connection for use during test sessions.
@@ -44,7 +53,9 @@ def setup_test_environment():
         
     """
     Template.original_render = Template.render
+    Template.original_iter_render = Template.iter_render
     Template.render = instrumented_test_render
+    Template.iter_render = instrumented_test_render
     
     mail.original_SMTPConnection = mail.SMTPConnection
     mail.SMTPConnection = TestSMTPConnection
@@ -59,7 +70,8 @@ def teardown_test_environment():
         
     """
     Template.render = Template.original_render
-    del Template.original_render
+    Template.iter_render = Template.original_iter_render
+    del Template.original_render, Template.original_iter_render
     
     mail.SMTPConnection = mail.original_SMTPConnection
     del mail.original_SMTPConnection

django/utils/itercompat.py

@@ -7,7 +7,8 @@ these implementations if necessary.
 import itertools
 
 def compat_tee(iterable):
-    """Return two independent iterators from a single iterable.
+    """
+    Return two independent iterators from a single iterable.
 
     Based on http://www.python.org/doc/2.3.5/lib/itertools-example.html
     """
@@ -25,7 +26,28 @@ def compat_tee(iterable):
     next = iter(iterable).next
     return gen(next), gen(next)
 
+def groupby(iterable, keyfunc=None):
+    """
+    Taken from http://docs.python.org/lib/itertools-functions.html
+    """
+    if keyfunc is None:
+        keyfunc = lambda x:x
+    iterable = iter(iterable)
+    l = [iterable.next()]
+    lastkey = keyfunc(l)
+    for item in iterable:
+        key = keyfunc(item)
+        if key != lastkey:
+            yield lastkey, l
+            lastkey = key
+            l = [item]
+        else:
+            l.append(item)
+    yield lastkey, l
+
 if hasattr(itertools, 'tee'):
     tee = itertools.tee
 else:
     tee = compat_tee
+if hasattr(itertools, 'groupby'):
+    groupby = itertools.groupby

django/views/debug.py

@@ -137,7 +137,7 @@ def technical_500_response(request, exc_type, exc_value, tb):
         'template_does_not_exist': template_does_not_exist,
         'loader_debug_info': loader_debug_info,
     })
-    return HttpResponseServerError(t.render(c), mimetype='text/html')
+    return HttpResponseServerError(t.iter_render(c), mimetype='text/html')
 
 def technical_404_response(request, exception):
     "Create a technical 404 error response. The exception should be the Http404."
@@ -160,7 +160,7 @@ def technical_404_response(request, exception):
         'request_protocol': request.is_secure() and "https" or "http",
         'settings': get_safe_settings(),
     })
-    return HttpResponseNotFound(t.render(c), mimetype='text/html')
+    return HttpResponseNotFound(t.iter_render(c), mimetype='text/html')
 
 def empty_urlconf(request):
     "Create an empty URLconf 404 error response."
@@ -168,7 +168,7 @@ def empty_urlconf(request):
     c = Context({
         'project_name': settings.SETTINGS_MODULE.split('.')[0]
     })
-    return HttpResponseNotFound(t.render(c), mimetype='text/html')
+    return HttpResponseNotFound(t.iter_render(c), mimetype='text/html')
 
 def _get_lines_from_file(filename, lineno, context_lines, loader=None, module_name=None):
     """

django/views/defaults.py

@@ -76,7 +76,7 @@ def page_not_found(request, template_name='404.html'):
             The path of the requested URL (e.g., '/app/pages/bad_page/')
     """
     t = loader.get_template(template_name) # You need to create a 404.html template.
-    return http.HttpResponseNotFound(t.render(RequestContext(request, {'request_path': request.path})))
+    return http.HttpResponseNotFound(t.iter_render(RequestContext(request, {'request_path': request.path})))
 
 def server_error(request, template_name='500.html'):
     """
@@ -86,4 +86,4 @@ def server_error(request, template_name='500.html'):
     Context: None
     """
     t = loader.get_template(template_name) # You need to create a 500.html template.
-    return http.HttpResponseServerError(t.render(Context({})))
+    return http.HttpResponseServerError(t.iter_render(Context({})))

django/views/generic/create_update.py

@@ -68,7 +68,7 @@ def create_object(request, model, template_name=None,
             c[key] = value()
         else:
             c[key] = value
-    return HttpResponse(t.render(c))
+    return HttpResponse(t.iter_render(c))
 
 def update_object(request, model, object_id=None, slug=None,
         slug_field=None, template_name=None, template_loader=loader,
@@ -141,7 +141,7 @@ def update_object(request, model, object_id=None, slug=None,
             c[key] = value()
         else:
             c[key] = value
-    response = HttpResponse(t.render(c))
+    response = HttpResponse(t.iter_render(c))
     populate_xheaders(request, response, model, getattr(object, object._meta.pk.attname))
     return response
 
@@ -195,6 +195,6 @@ def delete_object(request, model, post_delete_redirect,
                 c[key] = value()
             else:
                 c[key] = value
-        response = HttpResponse(t.render(c))
+        response = HttpResponse(t.iter_render(c))
         populate_xheaders(request, response, model, getattr(object, object._meta.pk.attname))
         return response

django/views/generic/date_based.py

@@ -44,7 +44,7 @@ def archive_index(request, queryset, date_field, num_latest=15,
             c[key] = value()
         else:
             c[key] = value
-    return HttpResponse(t.render(c), mimetype=mimetype)
+    return HttpResponse(t.iter_render(c), mimetype=mimetype)
 
 def archive_year(request, year, queryset, date_field, template_name=None,
         template_loader=loader, extra_context=None, allow_empty=False,
@@ -92,7 +92,7 @@ def archive_year(request, year, queryset, date_field, template_name=None,
             c[key] = value()
         else:
             c[key] = value
-    return HttpResponse(t.render(c), mimetype=mimetype)
+    return HttpResponse(t.iter_render(c), mimetype=mimetype)
 
 def archive_month(request, year, month, queryset, date_field,
         month_format='%b', template_name=None, template_loader=loader,
@@ -158,7 +158,7 @@ def archive_month(request, year, month, queryset, date_field,
             c[key] = value()
         else:
             c[key] = value
-    return HttpResponse(t.render(c), mimetype=mimetype)
+    return HttpResponse(t.iter_render(c), mimetype=mimetype)
 
 def archive_week(request, year, week, queryset, date_field,
         template_name=None, template_loader=loader,
@@ -206,7 +206,7 @@ def archive_week(request, year, week, queryset, date_field,
             c[key] = value()
         else:
             c[key] = value
-    return HttpResponse(t.render(c), mimetype=mimetype)
+    return HttpResponse(t.iter_render(c), mimetype=mimetype)
 
 def archive_day(request, year, month, day, queryset, date_field,
         month_format='%b', day_format='%d', template_name=None,
@@ -270,7 +270,7 @@ def archive_day(request, year, month, day, queryset, date_field,
             c[key] = value()
         else:
             c[key] = value
-    return HttpResponse(t.render(c), mimetype=mimetype)
+    return HttpResponse(t.iter_render(c), mimetype=mimetype)
 
 def archive_today(request, **kwargs):
     """
@@ -339,6 +339,6 @@ def object_detail(request, year, month, day, queryset, date_field,
             c[key] = value()
         else:
             c[key] = value
-    response = HttpResponse(t.render(c), mimetype=mimetype)
+    response = HttpResponse(t.iter_render(c), mimetype=mimetype)
     populate_xheaders(request, response, model, getattr(obj, obj._meta.pk.name))
     return response

django/views/generic/list_detail.py

@@ -84,7 +84,7 @@ def object_list(request, queryset, paginate_by=None, page=None,
         model = queryset.model
         template_name = "%s/%s_list.html" % (model._meta.app_label, model._meta.object_name.lower())
     t = template_loader.get_template(template_name)
-    return HttpResponse(t.render(c), mimetype=mimetype)
+    return HttpResponse(t.iter_render(c), mimetype=mimetype)
 
 def object_detail(request, queryset, object_id=None, slug=None,
         slug_field=None, template_name=None, template_name_field=None,
@@ -126,6 +126,6 @@ def object_detail(request, queryset, object_id=None, slug=None,
             c[key] = value()
         else:
             c[key] = value
-    response = HttpResponse(t.render(c), mimetype=mimetype)
+    response = HttpResponse(t.iter_render(c), mimetype=mimetype)
     populate_xheaders(request, response, model, getattr(obj, obj._meta.pk.name))
     return response

django/views/generic/simple.py

@@ -15,7 +15,7 @@ def direct_to_template(request, template, extra_context={}, mimetype=None, **kwa
             dictionary[key] = value
     c = RequestContext(request, dictionary)
     t = loader.get_template(template)
-    return HttpResponse(t.render(c), mimetype=mimetype)
+    return HttpResponse(t.iter_render(c), mimetype=mimetype)
 
 def redirect_to(request, url, **kwargs):
     """

django/views/static.py

@@ -92,7 +92,7 @@ def directory_index(path, fullpath):
         'directory' : path + '/',
         'file_list' : files,
     })
-    return HttpResponse(t.render(c))
+    return HttpResponse(t.iter_render(c))
 
 def was_modified_since(header=None, mtime=0, size=0):
     """

docs/authentication.txt

@@ -161,8 +161,8 @@ The ``User`` model has a custom manager that has the following helper functions:
     * ``make_random_password(length=10, allowed_chars='abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789')``
       Returns a random password with the given length and given string of
       allowed characters. (Note that the default value of ``allowed_chars``
-      doesn't contain ``"I"`` or letters that look like it, to avoid user
+      doesn't contain letters that can cause user confusion, including
-      confusion.
+      ``1``, ``I`` and ``0``).
 
 Basic usage
 -----------

docs/newforms.txt

@@ -1224,7 +1224,7 @@ Form validation happens when the data is cleaned. If you want to customise
 this process, there are various places you can change, each one serving a
 different purpose. Thee types of cleaning methods are run during form
 processing. These are normally executed when you call the ``is_valid()``
-method on a form. There are other things that can kick of cleaning and
+method on a form. There are other things that can trigger cleaning and
 validation (accessing the ``errors`` attribute or calling ``full_clean()``
 directly), but normally they won't be needed.
 
@@ -1234,7 +1234,7 @@ the ``ValidationError`` constructor. If no ``ValidationError`` is raised, the
 method should return the cleaned (normalised) data as a Python object.
 
 If you detect multiple errors during a cleaning method and wish to signal all
-of them to the form submittor, it is possible to pass a list of errors to the
+of them to the form submitter, it is possible to pass a list of errors to the
 ``ValidationError`` constructor.
 
 The three types of cleaning methods are:
@@ -1293,7 +1293,7 @@ dictionary.
 The previous paragraph means that if you are overriding ``Form.clean()``, you
 should iterate through ``self.cleaned_data.items()``, possibly considering the
 ``_errors`` dictionary attribute on the form as well. In this way, you will
-already know which fields have passed thei individual validation requirements.
+already know which fields have passed their individual validation requirements.
 
 A simple example
 ~~~~~~~~~~~~~~~~

docs/templates_python.txt

@@ -219,13 +219,13 @@ be replaced with the name of the invalid variable.
 
     While ``TEMPLATE_STRING_IF_INVALID`` can be a useful debugging tool,
     it is a bad idea to turn it on as a 'development default'.
-    
+
     Many templates, including those in the Admin site, rely upon the
     silence of the template system when a non-existent variable is
     encountered. If you assign a value other than ``''`` to
     ``TEMPLATE_STRING_IF_INVALID``, you will experience rendering
     problems with these templates and sites.
-    
+
     Generally, ``TEMPLATE_STRING_IF_INVALID`` should only be enabled
     in order to debug a specific template problem, then cleared
     once debugging is complete.
@@ -693,14 +693,15 @@ how the compilation works and how the rendering works.
 
 When Django compiles a template, it splits the raw template text into
 ''nodes''. Each node is an instance of ``django.template.Node`` and has
-a ``render()`` method. A compiled template is, simply, a list of ``Node``
+either a ``render()`` or ``iter_render()`` method. A compiled template is,
-objects. When you call ``render()`` on a compiled template object, the template
+simply, a list of ``Node`` objects. When you call ``render()`` on a compiled
-calls ``render()`` on each ``Node`` in its node list, with the given context.
+template object, the template calls ``render()`` on each ``Node`` in its node
-The results are all concatenated together to form the output of the template.
+list, with the given context. The results are all concatenated together to
+form the output of the template.
 
 Thus, to define a custom template tag, you specify how the raw template tag is
 converted into a ``Node`` (the compilation function), and what the node's
-``render()`` method does.
+``render()`` or ``iter_render()`` method does.
 
 Writing the compilation function
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -770,7 +771,8 @@ Writing the renderer
 ~~~~~~~~~~~~~~~~~~~~
 
 The second step in writing custom tags is to define a ``Node`` subclass that
-has a ``render()`` method.
+has a ``render()`` method (we will discuss the ``iter_render()`` alternative
+in `Improving rendering speed`_, below).
 
 Continuing the above example, we need to define ``CurrentTimeNode``::
 
@@ -874,7 +876,7 @@ current context, available in the ``render`` method::
         def __init__(self, date_to_be_formatted, format_string):
             self.date_to_be_formatted = date_to_be_formatted
             self.format_string = format_string
-        
+
         def render(self, context):
             try:
                 actual_date = resolve_variable(self.date_to_be_formatted, context)
@@ -1175,6 +1177,48 @@ For more examples of complex rendering, see the source code for ``{% if %}``,
 
 .. _configuration:
 
+Improving rendering speed
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For most practical purposes, the ``render()`` method on a ``Node`` will be
+sufficient and the simplest way to implement a new tag. However, if your
+template tag is expected to produce large strings via ``render()``, you can
+speed up the rendering process (and reduce memory usage) using iterative
+rendering via the ``iter_render()`` method.
+
+The ``iter_render()`` method should either be an iterator that yields string
+chunks, one at a time, or a method that returns a sequence of string chunks.
+The template renderer will join the successive chunks together when creating
+the final output. The improvement over the ``render()`` method here is that
+you do not need to create one large string containing all the output of the
+``Node``, instead you can produce the output in smaller chunks.
+
+By way of example, here's a trivial ``Node`` subclass that simply returns the
+contents of a file it is given::
+
+    class FileNode(Node):
+        def __init__(self, filename):
+            self.filename = filename
+
+        def iter_render(self):
+            for line in file(self.filename):
+                yield line
+
+For very large files, the full file contents will never be read entirely into
+memory when this tag is used, which is a useful optimisation.
+
+If you define an ``iter_render()`` method on your ``Node`` subclass, you do
+not need to define a ``render()`` method. The reverse is true as well: the
+default ``Node.iter_render()`` method will call your ``render()`` method if
+necessary. A useful side-effect of this is that you can develop a new tag
+using ``render()`` and producing all the output at once, which is easy to
+debug. Then you can rewrite the method as an iterator, rename it to
+``iter_render()`` and everything will still work.
+
+It is compulsory, however, to define *either* ``render()`` or ``iter_render()``
+in your subclass. If you omit them both, a ``TypeError`` will be raised when
+the code is imported.
+
 Configuring the template system in standalone mode
 ==================================================
 
@@ -1206,3 +1250,4 @@ is of obvious interest.
 
 .. _settings file: ../settings/#using-settings-without-the-django-settings-module-environment-variable
 .. _settings documentation: ../settings/
+