Fixed #4565 -- Changed template rendering to use iterators, rather than

creating large strings, as much as possible. This is all backwards compatible. Thanks, Brian Harring. git-svn-id: http://code.djangoproject.com/svn/django/trunk@5482 bcc190cf-cafb-0310-a4f2-bffc1f526a37
2025-10-31 09:41:08 +00:00 · 2007-06-17 07:11:37 +00:00
parent 44dd91ec6d
commit bccb8897e6
21 changed files with 284 additions and 161 deletions
--- a/django/template/init.py
+++ b/django/template/init.py
@@ -55,6 +55,7 @@ times with multiple contexts)
 '\n<html>\n\n</html>\n'
 """
 import re
+import types
 from inspect import getargspec
 from django.conf import settings
 from django.template.context import Context, RequestContext, ContextPopException
@@ -167,9 +168,12 @@ class Template(object):
            for subnode in node:
                yield subnode

-    def render(self, context):
+    def iter_render(self, context):
        "Display stage -- can be called many times"
-        return self.nodelist.render(context)
+        return self.nodelist.iter_render(context)
+
+    def render(self, context):
+        return ''.join(self.iter_render(context))

 def compile_string(template_string, origin):
    "Compiles template_string into NodeList ready for rendering"
@@ -698,10 +702,26 @@ def resolve_variable(path, context):
            del bits[0]
    return current

+class NodeBase(type):
+    def __new__(cls, name, bases, attrs):
+        """
+        Ensures that either a 'render' or 'render_iter' method is defined on
+        any Node sub-class. This avoids potential infinite loops at runtime.
+        """
+        if not (isinstance(attrs.get('render'), types.FunctionType) or
+                isinstance(attrs.get('iter_render'), types.FunctionType)):
+            raise TypeError('Unable to create Node subclass without either "render" or "iter_render" method.')
+        return type.__new__(cls, name, bases, attrs)
+
 class Node(object):
+    __metaclass__ = NodeBase
+
+    def iter_render(self, context):
+        return (self.render(context),)
+
    def render(self, context):
        "Return the node rendered as a string"
-        pass
+        return ''.join(self.iter_render(context))

    def __iter__(self):
        yield self
@@ -717,13 +737,12 @@ class Node(object):

 class NodeList(list):
    def render(self, context):
-        bits = []
+        return ''.join(self.iter_render(context))
+
+    def iter_render(self, context):
        for node in self:
-            if isinstance(node, Node):
-                bits.append(self.render_node(node, context))
-            else:
-                bits.append(node)
-        return ''.join(bits)
+            for chunk in node.iter_render(context):
+                yield chunk

    def get_nodes_by_type(self, nodetype):
        "Return a list of all nodes of the given type"
@@ -732,24 +751,26 @@ class NodeList(list):
            nodes.extend(node.get_nodes_by_type(nodetype))
        return nodes

-    def render_node(self, node, context):
-        return(node.render(context))

 class DebugNodeList(NodeList):
-    def render_node(self, node, context):
-        try:
-            result = node.render(context)
-        except TemplateSyntaxError, e:
-            if not hasattr(e, 'source'):
-                e.source = node.source
-            raise
-        except Exception, e:
-            from sys import exc_info
-            wrapped = TemplateSyntaxError('Caught an exception while rendering: %s' % e)
-            wrapped.source = node.source
-            wrapped.exc_info = exc_info()
-            raise wrapped
-        return result
+    def iter_render(self, context):
+        for node in self:
+            if not isinstance(node, Node):
+                yield node
+                continue
+            try:
+                for chunk in node.iter_render(context):
+                    yield chunk
+            except TemplateSyntaxError, e:
+                if not hasattr(e, 'source'):
+                    e.source = node.source
+                raise
+            except Exception, e:
+                from sys import exc_info
+                wrapped = TemplateSyntaxError('Caught an exception while rendering: %s' % e)
+                wrapped.source = node.source
+                wrapped.exc_info = exc_info()
+                raise wrapped

 class TextNode(Node):
    def __init__(self, s):
@@ -758,6 +779,9 @@ class TextNode(Node):
    def __repr__(self):
        return "<Text Node: '%s'>" % self.s[:25]

+    def iter_render(self, context):
+        return (self.s,)
+
    def render(self, context):
        return self.s

@@ -781,6 +805,9 @@ class VariableNode(Node):
        else:
            return output

+    def iter_render(self, context):
+        return (self.render(context),)
+
    def render(self, context):
        output = self.filter_expression.resolve(context)
        return self.encode_output(output)
@@ -869,6 +896,9 @@ class Library(object):
            def __init__(self, vars_to_resolve):
                self.vars_to_resolve = vars_to_resolve

+            #def iter_render(self, context):
+            #    return (self.render(context),)
+
            def render(self, context):
                resolved_vars = [resolve_variable(var, context) for var in self.vars_to_resolve]
                return func(*resolved_vars)
@@ -891,7 +921,7 @@ class Library(object):
                def __init__(self, vars_to_resolve):
                    self.vars_to_resolve = vars_to_resolve

-                def render(self, context):
+                def iter_render(self, context):
                    resolved_vars = [resolve_variable(var, context) for var in self.vars_to_resolve]
                    if takes_context:
                        args = [context] + resolved_vars
@@ -907,7 +937,7 @@ class Library(object):
                        else:
                            t = get_template(file_name)
                        self.nodelist = t.nodelist
-                    return self.nodelist.render(context_class(dict))
+                    return self.nodelist.iter_render(context_class(dict))

            compile_func = curry(generic_tag_compiler, params, defaults, getattr(func, "_decorated_function", func).__name__, InclusionNode)
            compile_func.__doc__ = func.__doc__
--- a/django/template/defaulttags.py
+++ b/django/template/defaulttags.py
@@ -14,12 +14,11 @@ if not hasattr(__builtins__, 'reversed'):
        for index in xrange(len(data)-1, -1, -1):
            yield data[index]

-
 register = Library()

 class CommentNode(Node):
-    def render(self, context):
-        return ''
+    def iter_render(self, context):
+        return ()

 class CycleNode(Node):
    def __init__(self, cyclevars, variable_name=None):
@@ -28,6 +27,9 @@ class CycleNode(Node):
        self.counter = -1
        self.variable_name = variable_name

+    def iter_render(self, context):
+        return (self.render(context),)
+
    def render(self, context):
        self.counter += 1
        value = self.cyclevars[self.counter % self.cyclevars_len]
@@ -36,29 +38,32 @@ class CycleNode(Node):
        return value

 class DebugNode(Node):
-    def render(self, context):
+    def iter_render(self, context):
        from pprint import pformat
-        output = [pformat(val) for val in context]
-        output.append('\n\n')
-        output.append(pformat(sys.modules))
-        return ''.join(output)
+        for val in context:
+            yield pformat(val)
+        yield "\n\n"
+        yield pformat(sys.modules)

 class FilterNode(Node):
    def __init__(self, filter_expr, nodelist):
        self.filter_expr, self.nodelist = filter_expr, nodelist

-    def render(self, context):
+    def iter_render(self, context):
        output = self.nodelist.render(context)
        # apply filters
        context.update({'var': output})
        filtered = self.filter_expr.resolve(context)
        context.pop()
-        return filtered
+        return (filtered,)

 class FirstOfNode(Node):
    def __init__(self, vars):
        self.vars = vars

+    def iter_render(self, context):
+        return (self.render(context),)
+
    def render(self, context):
        for var in self.vars:
            try:
@@ -94,8 +99,7 @@ class ForNode(Node):
        nodes.extend(self.nodelist_loop.get_nodes_by_type(nodetype))
        return nodes

-    def render(self, context):
-        nodelist = NodeList()
+    def iter_render(self, context):
        if 'forloop' in context:
            parentloop = context['forloop']
        else:
@@ -103,12 +107,12 @@ class ForNode(Node):
        context.push()
        try:
            values = self.sequence.resolve(context, True)
+            if values is None:
+                values = ()
+            elif not hasattr(values, '__len__'):
+                values = list(values)
        except VariableDoesNotExist:
-            values = []
-        if values is None:
-            values = []
-        if not hasattr(values, '__len__'):
-            values = list(values)
+            values = ()
        len_values = len(values)
        if self.reversed:
            values = reversed(values)
@@ -127,12 +131,17 @@ class ForNode(Node):
                'parentloop': parentloop,
            }
            if unpack:
-                # If there are multiple loop variables, unpack the item into them.
+                # If there are multiple loop variables, unpack the item into
+                # them.
                context.update(dict(zip(self.loopvars, item)))
            else:
                context[self.loopvars[0]] = item
+
+            # We inline this to avoid the overhead since ForNode is pretty
+            # common.
            for node in self.nodelist_loop:
-                nodelist.append(node.render(context))
+                for chunk in node.iter_render(context):
+                    yield chunk
            if unpack:
                # The loop variables were pushed on to the context so pop them
                # off again. This is necessary because the tag lets the length
@@ -141,7 +150,6 @@ class ForNode(Node):
                # context.
                context.pop()
        context.pop()
-        return nodelist.render(context)

 class IfChangedNode(Node):
    def __init__(self, nodelist, *varlist):
@@ -149,7 +157,7 @@ class IfChangedNode(Node):
        self._last_seen = None
        self._varlist = varlist

-    def render(self, context):
+    def iter_render(self, context):
        if 'forloop' in context and context['forloop']['first']:
            self._last_seen = None
        try:
@@ -167,11 +175,9 @@ class IfChangedNode(Node):
            self._last_seen = compare_to
            context.push()
            context['ifchanged'] = {'firstloop': firstloop}
-            content = self.nodelist.render(context)
+            for chunk in self.nodelist.iter_render(context):
+                yield chunk
            context.pop()
-            return content
-        else:
-            return ''

 class IfEqualNode(Node):
    def __init__(self, var1, var2, nodelist_true, nodelist_false, negate):
@@ -182,7 +188,7 @@ class IfEqualNode(Node):
    def __repr__(self):
        return "<IfEqualNode>"

-    def render(self, context):
+    def iter_render(self, context):
        try:
            val1 = resolve_variable(self.var1, context)
        except VariableDoesNotExist:
@@ -192,8 +198,8 @@ class IfEqualNode(Node):
        except VariableDoesNotExist:
            val2 = None
        if (self.negate and val1 != val2) or (not self.negate and val1 == val2):
-            return self.nodelist_true.render(context)
-        return self.nodelist_false.render(context)
+            return self.nodelist_true.iter_render(context)
+        return self.nodelist_false.iter_render(context)

 class IfNode(Node):
    def __init__(self, bool_exprs, nodelist_true, nodelist_false, link_type):
@@ -218,7 +224,7 @@ class IfNode(Node):
        nodes.extend(self.nodelist_false.get_nodes_by_type(nodetype))
        return nodes

-    def render(self, context):
+    def iter_render(self, context):
        if self.link_type == IfNode.LinkTypes.or_:
            for ifnot, bool_expr in self.bool_exprs:
                try:
@@ -226,8 +232,8 @@ class IfNode(Node):
                except VariableDoesNotExist:
                    value = None
                if (value and not ifnot) or (ifnot and not value):
-                    return self.nodelist_true.render(context)
-            return self.nodelist_false.render(context)
+                    return self.nodelist_true.iter_render(context)
+            return self.nodelist_false.iter_render(context)
        else:
            for ifnot, bool_expr in self.bool_exprs:
                try:
@@ -235,8 +241,8 @@ class IfNode(Node):
                except VariableDoesNotExist:
                    value = None
                if not ((value and not ifnot) or (ifnot and not value)):
-                    return self.nodelist_false.render(context)
-            return self.nodelist_true.render(context)
+                    return self.nodelist_false.iter_render(context)
+            return self.nodelist_true.iter_render(context)

    class LinkTypes:
        and_ = 0,
@@ -247,11 +253,11 @@ class RegroupNode(Node):
        self.target, self.expression = target, expression
        self.var_name = var_name

-    def render(self, context):
+    def iter_render(self, context):
        obj_list = self.target.resolve(context, True)
        if obj_list == None: # target_var wasn't found in context; fail silently
            context[self.var_name] = []
-            return ''
+            return ()
        output = [] # list of dictionaries in the format {'grouper': 'key', 'list': [list of contents]}
        for obj in obj_list:
            grouper = self.expression.resolve(obj, True)
@@ -261,7 +267,7 @@ class RegroupNode(Node):
            else:
                output.append({'grouper': grouper, 'list': [obj]})
        context[self.var_name] = output
-        return ''
+        return ()

 def include_is_allowed(filepath):
    for root in settings.ALLOWED_INCLUDE_ROOTS:
@@ -273,10 +279,10 @@ class SsiNode(Node):
    def __init__(self, filepath, parsed):
        self.filepath, self.parsed = filepath, parsed

-    def render(self, context):
+    def iter_render(self, context):
        if not include_is_allowed(self.filepath):
            if settings.DEBUG:
-                return "[Didn't have permission to include file]"
+                return ("[Didn't have permission to include file]",)
            else:
                return '' # Fail silently for invalid includes.
        try:
@@ -287,23 +293,25 @@ class SsiNode(Node):
            output = ''
        if self.parsed:
            try:
-                t = Template(output, name=self.filepath)
-                return t.render(context)
+                return Template(output, name=self.filepath).iter_render(context)
            except TemplateSyntaxError, e:
                if settings.DEBUG:
                    return "[Included template had syntax error: %s]" % e
                else:
                    return '' # Fail silently for invalid included templates.
-        return output
+        return (output,)

 class LoadNode(Node):
-    def render(self, context):
-        return ''
+    def iter_render(self, context):
+        return ()

 class NowNode(Node):
    def __init__(self, format_string):
        self.format_string = format_string

+    def iter_render(self, context):
+        return (self.render(context),)
+
    def render(self, context):
        from datetime import datetime
        from django.utils.dateformat import DateFormat
@@ -332,6 +340,9 @@ class TemplateTagNode(Node):
    def __init__(self, tagtype):
        self.tagtype = tagtype

+    def iter_render(self, context):
+        return (self.render(context),)
+
    def render(self, context):
        return self.mapping.get(self.tagtype, '')

@@ -341,18 +352,18 @@ class URLNode(Node):
        self.args = args
        self.kwargs = kwargs

-    def render(self, context):
+    def iter_render(self, context):
        from django.core.urlresolvers import reverse, NoReverseMatch
        args = [arg.resolve(context) for arg in self.args]
        kwargs = dict([(k, v.resolve(context)) for k, v in self.kwargs.items()])
        try:
-            return reverse(self.view_name, args=args, kwargs=kwargs)
+            return (reverse(self.view_name, args=args, kwargs=kwargs),)
        except NoReverseMatch:
            try:
                project_name = settings.SETTINGS_MODULE.split('.')[0]
                return reverse(project_name + '.' + self.view_name, args=args, kwargs=kwargs)
            except NoReverseMatch:
-                return ''
+                return ()

 class WidthRatioNode(Node):
    def __init__(self, val_expr, max_expr, max_width):
@@ -360,6 +371,9 @@ class WidthRatioNode(Node):
        self.max_expr = max_expr
        self.max_width = max_width

+    def iter_render(self, context):
+        return (self.render(context),)
+
    def render(self, context):
        try:
            value = self.val_expr.resolve(context)
@@ -383,13 +397,13 @@ class WithNode(Node):
    def __repr__(self):
        return "<WithNode>"

-    def render(self, context):
+    def iter_render(self, context):
        val = self.var.resolve(context)
        context.push()
        context[self.name] = val
-        output = self.nodelist.render(context)
+        for chunk in self.nodelist.iter_render(context):
+            yield chunk
        context.pop()
-        return output

 #@register.tag
 def comment(parser, token):
--- a/django/template/loader.py
+++ b/django/template/loader.py
@@ -87,14 +87,12 @@ def get_template_from_string(source, origin=None, name=None):
    """
    return Template(source, origin, name)

-def render_to_string(template_name, dictionary=None, context_instance=None):
+def _render_setup(template_name, dictionary=None, context_instance=None):
    """
-    Loads the given template_name and renders it with the given dictionary as
-    context. The template_name may be a string to load a single template using
-    get_template, or it may be a tuple to use select_template to find one of
-    the templates in the list. Returns a string.
+    Common setup code for render_to_string and render_to_iter.
    """
-    dictionary = dictionary or {}
+    if dictionary is None:
+        dictionary = {}
    if isinstance(template_name, (list, tuple)):
        t = select_template(template_name)
    else:
@@ -103,7 +101,28 @@ def render_to_string(template_name, dictionary=None, context_instance=None):
        context_instance.update(dictionary)
    else:
        context_instance = Context(dictionary)
-    return t.render(context_instance)
+    return t, context_instance
+
+def render_to_string(template_name, dictionary=None, context_instance=None):
+    """
+    Loads the given template_name and renders it with the given dictionary as
+    context. The template_name may be a string to load a single template using
+    get_template, or it may be a tuple to use select_template to find one of
+    the templates in the list. Returns a string.
+    """
+    t, c = _render_setup(template_name, dictionary=dictionary, context_instance=context_instance)
+    return t.render(c)
+
+def render_to_iter(template_name, dictionary=None, context_instance=None):
+    """
+    Loads the given template_name and renders it with the given dictionary as
+    context. The template_name may be a string to load a single template using
+    get_template, or it may be a tuple to use select_template to find one of
+    the templates in the list. Returns a string.
+    """
+    t, c = _render_setup(template_name, dictionary=dictionary, context_instance=context_instance)
+    return t.iter_render(c)
+

 def select_template(template_name_list):
    "Given a list of template names, returns the first that can be loaded."
--- a/django/template/loader_tags.py
+++ b/django/template/loader_tags.py
@@ -15,14 +15,14 @@ class BlockNode(Node):
    def __repr__(self):
        return "<Block Node: %s. Contents: %r>" % (self.name, self.nodelist)

-    def render(self, context):
+    def iter_render(self, context):
        context.push()
        # Save context in case of block.super().
        self.context = context
        context['block'] = self
-        result = self.nodelist.render(context)
+        for chunk in self.nodelist.iter_render(context):
+            yield chunk
        context.pop()
-        return result

    def super(self):
        if self.parent:
@@ -59,7 +59,7 @@ class ExtendsNode(Node):
        else:
            return get_template_from_string(source, origin, parent)

-    def render(self, context):
+    def iter_render(self, context):
        compiled_parent = self.get_parent(context)
        parent_is_child = isinstance(compiled_parent.nodelist[0], ExtendsNode)
        parent_blocks = dict([(n.name, n) for n in compiled_parent.nodelist.get_nodes_by_type(BlockNode)])
@@ -79,7 +79,7 @@ class ExtendsNode(Node):
                parent_block.parent = block_node.parent
                parent_block.add_parent(parent_block.nodelist)
                parent_block.nodelist = block_node.nodelist
-        return compiled_parent.render(context)
+        return compiled_parent.iter_render(context)

 class ConstantIncludeNode(Node):
    def __init__(self, template_path):
@@ -91,27 +91,26 @@ class ConstantIncludeNode(Node):
                raise
            self.template = None

-    def render(self, context):
+    def iter_render(self, context):
        if self.template:
-            return self.template.render(context)
-        else:
-            return ''
+            return self.template.iter_render(context)
+        return ()

 class IncludeNode(Node):
    def __init__(self, template_name):
        self.template_name = template_name

-    def render(self, context):
+    def iter_render(self, context):
        try:
            template_name = resolve_variable(self.template_name, context)
            t = get_template(template_name)
-            return t.render(context)
+            return t.iter_render(context)
        except TemplateSyntaxError, e:
            if settings.TEMPLATE_DEBUG:
                raise
-            return ''
+            return ()
        except:
-            return '' # Fail silently for invalid included templates.
+            return () # Fail silently for invalid included templates.

 def do_block(parser, token):
    """