Fixed #15053 -- Enabled recursive template loading.

django/template/__init__.py

@@ -59,8 +59,8 @@ from .base import (TemplateDoesNotExist, TemplateSyntaxError,           # NOQA
 from .context import ContextPopException                                # NOQA
 
 # Template parts
-from .base import (Context, Node, NodeList, RequestContext,             # NOQA
-                   StringOrigin, Template, Variable)
+from .base import (Context, Node, NodeList, Origin, RequestContext,     # NOQA
+                   Template, Variable)
 
 # Deprecated in Django 1.8, will be removed in Django 2.0.
 from .base import resolve_variable                                      # NOQA

django/template/base.py

@@ -134,7 +134,15 @@ class TemplateSyntaxError(Exception):
 
 
 class TemplateDoesNotExist(Exception):
-    pass
+    """
+    This exception is used when template loaders are unable to find a
+    template. The tried argument is an optional list of tuples containing
+    (origin, status), where origin is an Origin object and status is a string
+    with the reason the template wasn't found.
+    """
+    def __init__(self, msg, tried=None):
+        self.tried = tried or []
+        super(TemplateDoesNotExist, self).__init__(msg)
 
 
 class TemplateEncodingError(Exception):
@@ -157,23 +165,29 @@ class InvalidTemplateLibrary(Exception):
 
 
 class Origin(object):
-    def __init__(self, name):
+    def __init__(self, name, template_name=None, loader=None):
         self.name = name
-
-    def reload(self):
-        raise NotImplementedError('subclasses of Origin must provide a reload() method')
+        self.template_name = template_name
+        self.loader = loader
 
     def __str__(self):
         return self.name
 
+    def __eq__(self, other):
+        if not isinstance(other, Origin):
+            return False
 
-class StringOrigin(Origin):
-    def __init__(self, source):
-        super(StringOrigin, self).__init__(UNKNOWN_SOURCE)
-        self.source = source
+        return (
+            self.name == other.name and
+            self.loader == other.loader
+        )
 
-    def reload(self):
-        return self.source
+    @property
+    def loader_name(self):
+        if self.loader:
+            return '%s.%s' % (
+                self.loader.__module__, self.loader.__class__.__name__,
+            )
 
 
 class Template(object):
@@ -191,7 +205,7 @@ class Template(object):
             from .engine import Engine
             engine = Engine.get_default()
         if origin is None:
-            origin = StringOrigin(template_string)
+            origin = Origin(UNKNOWN_SOURCE)
         self.name = name
         self.origin = origin
         self.engine = engine

django/template/engine.py

@@ -124,15 +124,25 @@ class Engine(object):
             raise ImproperlyConfigured(
                 "Invalid value in template loaders configuration: %r" % loader)
 
-    def find_template(self, name, dirs=None):
+    def find_template(self, name, dirs=None, skip=None):
+        tried = []
         for loader in self.template_loaders:
-            try:
-                source, display_name = loader(name, dirs)
-                origin = self.make_origin(display_name, loader, name, dirs)
-                return source, origin
-            except TemplateDoesNotExist:
-                pass
-        raise TemplateDoesNotExist(name)
+            if loader.supports_recursion:
+                try:
+                    template = loader.get_template(
+                        name, template_dirs=dirs, skip=skip,
+                    )
+                    return template, template.origin
+                except TemplateDoesNotExist as e:
+                    tried.extend(e.tried)
+            else:
+                # RemovedInDjango21Warning: Use old api for non-recursive
+                # loaders.
+                try:
+                    return loader(name, dirs)
+                except TemplateDoesNotExist:
+                    pass
+        raise TemplateDoesNotExist(name, tried=tried)
 
     def from_string(self, template_code):
         """
@@ -234,11 +244,3 @@ class Engine(object):
                 continue
         # If we get here, none of the templates could be loaded
         raise TemplateDoesNotExist(', '.join(not_found))
-
-    def make_origin(self, display_name, loader, name, dirs):
-        if self.debug and display_name:
-            # Inner import to avoid circular dependency
-            from .loader import LoaderOrigin
-            return LoaderOrigin(display_name, loader, name, dirs)
-        else:
-            return None

django/template/loader.py

@@ -4,25 +4,20 @@ from django.utils.deprecation import RemovedInDjango20Warning
 
 from . import engines
 from .backends.django import DjangoTemplates
-from .base import Origin, TemplateDoesNotExist
+from .base import TemplateDoesNotExist
 from .engine import (
     _context_instance_undefined, _dictionary_undefined, _dirs_undefined,
 )
 from .loaders import base
 
 
-class LoaderOrigin(Origin):
-    def __init__(self, display_name, loader, name, dirs):
-        super(LoaderOrigin, self).__init__(display_name)
-        self.loader, self.loadname, self.dirs = loader, name, dirs
-
-
 def get_template(template_name, dirs=_dirs_undefined, using=None):
     """
     Loads and returns a template for the given name.
 
     Raises TemplateDoesNotExist if no such template exists.
     """
+    tried = []
     engines = _engine_list(using)
     for engine in engines:
         try:
@@ -37,10 +32,10 @@ def get_template(template_name, dirs=_dirs_undefined, using=None):
                     stacklevel=2)
             else:
                 return engine.get_template(template_name)
-        except TemplateDoesNotExist:
-            pass
+        except TemplateDoesNotExist as e:
+            tried.extend(e.tried)
 
-    raise TemplateDoesNotExist(template_name)
+    raise TemplateDoesNotExist(template_name, tried=tried)
 
 
 def select_template(template_name_list, dirs=_dirs_undefined, using=None):
@@ -51,6 +46,7 @@ def select_template(template_name_list, dirs=_dirs_undefined, using=None):
 
     Raises TemplateDoesNotExist if no such template exists.
     """
+    tried = []
     engines = _engine_list(using)
     for template_name in template_name_list:
         for engine in engines:
@@ -66,11 +62,11 @@ def select_template(template_name_list, dirs=_dirs_undefined, using=None):
                         stacklevel=2)
                 else:
                     return engine.get_template(template_name)
-            except TemplateDoesNotExist:
-                pass
+            except TemplateDoesNotExist as e:
+                tried.extend(e.tried)
 
     if template_name_list:
-        raise TemplateDoesNotExist(', '.join(template_name_list))
+        raise TemplateDoesNotExist(', '.join(template_name_list), tried=tried)
     else:
         raise TemplateDoesNotExist("No template names provided")
 
@@ -96,6 +92,7 @@ def render_to_string(template_name, context=None,
         return template.render(context, request)
 
     else:
+        tried = []
         # Some deprecated arguments were passed - use the legacy code path
         for engine in _engine_list(using):
             try:
@@ -126,13 +123,14 @@ def render_to_string(template_name, context=None,
                         "Skipping template backend %s because its render_to_string "
                         "method doesn't support the dictionary argument." %
                         engine.name, stacklevel=2)
-            except TemplateDoesNotExist:
+            except TemplateDoesNotExist as e:
+                tried.extend(e.tried)
                 continue
 
         if template_name:
             if isinstance(template_name, (list, tuple)):
                 template_name = ', '.join(template_name)
-            raise TemplateDoesNotExist(template_name)
+            raise TemplateDoesNotExist(template_name, tried=tried)
         else:
             raise TemplateDoesNotExist("No template names provided")
 

django/template/loader_tags.py

@@ -82,6 +82,7 @@ class BlockNode(Node):
 
 class ExtendsNode(Node):
     must_be_first = True
+    context_key = 'extends_context'
 
     def __init__(self, nodelist, parent_name, template_dirs=None):
         self.nodelist = nodelist
@@ -92,6 +93,39 @@ class ExtendsNode(Node):
     def __repr__(self):
         return '<ExtendsNode: extends %s>' % self.parent_name.token
 
+    def find_template(self, template_name, context):
+        """
+        This is a wrapper around engine.find_template(). A history is kept in
+        the render_context attribute between successive extends calls and
+        passed as the skip argument. This enables extends to work recursively
+        without extending the same template twice.
+        """
+        # RemovedInDjango21Warning: If any non-recursive loaders are installed
+        # do a direct template lookup. If the same template name appears twice,
+        # raise an exception to avoid system recursion.
+        for loader in context.template.engine.template_loaders:
+            if not loader.supports_recursion:
+                history = context.render_context.setdefault(
+                    self.context_key, [context.template.origin.template_name],
+                )
+                if template_name in history:
+                    raise ExtendsError(
+                        "Cannot extend templates recursively when using "
+                        "non-recursive template loaders",
+                    )
+                template = context.template.engine.get_template(template_name)
+                history.append(template_name)
+                return template
+
+        history = context.render_context.setdefault(
+            self.context_key, [context.template.origin],
+        )
+        template, origin = context.template.engine.find_template(
+            template_name, skip=history,
+        )
+        history.append(origin)
+        return template
+
     def get_parent(self, context):
         parent = self.parent_name.resolve(context)
         if not parent:
@@ -107,7 +141,7 @@ class ExtendsNode(Node):
         if isinstance(getattr(parent, 'template', None), Template):
             # parent is a django.template.backends.django.Template
             return parent.template
-        return context.template.engine.get_template(parent)
+        return self.find_template(parent, context)
 
     def render(self, context):
         compiled_parent = self.get_parent(context)

django/template/loaders/base.py

@@ -1,4 +1,8 @@
-from django.template.base import Template, TemplateDoesNotExist
+import warnings
+from inspect import getargspec
+
+from django.template.base import Origin, Template, TemplateDoesNotExist
+from django.utils.deprecation import RemovedInDjango21Warning
 
 
 class Loader(object):
@@ -9,15 +13,54 @@ class Loader(object):
         self.engine = engine
 
     def __call__(self, template_name, template_dirs=None):
+        # RemovedInDjango21Warning: Allow loaders to be called like functions.
         return self.load_template(template_name, template_dirs)
 
-    def load_template(self, template_name, template_dirs=None):
-        source, display_name = self.load_template_source(
-            template_name, template_dirs)
-        origin = self.engine.make_origin(
-            display_name, self.load_template_source,
-            template_name, template_dirs)
+    def get_template(self, template_name, template_dirs=None, skip=None):
+        """
+        Calls self.get_template_sources() and returns a Template object for
+        the first template matching template_name. If skip is provided,
+        template origins in skip are ignored. This is used to avoid recursion
+        during template extending.
+        """
+        tried = []
 
+        args = [template_name]
+        # RemovedInDjango21Warning: Add template_dirs for compatibility with
+        # old loaders
+        if 'template_dirs' in getargspec(self.get_template_sources)[0]:
+            args.append(template_dirs)
+
+        for origin in self.get_template_sources(*args):
+            if skip is not None and origin in skip:
+                tried.append((origin, 'Skipped'))
+                continue
+
+            try:
+                contents = self.get_contents(origin)
+            except TemplateDoesNotExist:
+                tried.append((origin, 'Source does not exist'))
+                continue
+            else:
+                return Template(
+                    contents, origin, origin.template_name, self.engine,
+                )
+
+        raise TemplateDoesNotExist(template_name, tried=tried)
+
+    def load_template(self, template_name, template_dirs=None):
+        warnings.warn(
+            'The load_template() method is deprecated. Use get_template() '
+            'instead.', RemovedInDjango21Warning,
+        )
+        source, display_name = self.load_template_source(
+            template_name, template_dirs,
+        )
+        origin = Origin(
+            name=display_name,
+            template_name=template_name,
+            loader=self,
+        )
         try:
             template = Template(source, origin, template_name, self.engine)
         except TemplateDoesNotExist:
@@ -29,14 +72,23 @@ class Loader(object):
         else:
             return template, None
 
-    def load_template_source(self, template_name, template_dirs=None):
+    def get_template_sources(self, template_name):
         """
-        Returns a tuple containing the source and origin for the given
+        An iterator that yields possible matching template paths for a
         template name.
         """
         raise NotImplementedError(
-            "subclasses of Loader must provide "
-            "a load_template_source() method")
+            'subclasses of Loader must provide a get_template_sources() method'
+        )
+
+    def load_template_source(self, template_name, template_dirs=None):
+        """
+        RemovedInDjango21Warning: Returns a tuple containing the source and
+        origin for the given template name.
+        """
+        raise NotImplementedError(
+            'subclasses of Loader must provide a load_template_source() method'
+        )
 
     def reset(self):
         """
@@ -44,3 +96,11 @@ class Loader(object):
         templates or cached loader modules).
         """
         pass
+
+    @property
+    def supports_recursion(self):
+        """
+        RemovedInDjango21Warning: This is an internal property used by the
+        ExtendsNode during the deprecation of non-recursive loaders.
+        """
+        return hasattr(self, 'get_contents')

django/template/loaders/cached.py

@@ -4,8 +4,11 @@ to load templates from them in order, caching the result.
 """
 
 import hashlib
+import warnings
+from inspect import getargspec
 
-from django.template.base import Template, TemplateDoesNotExist
+from django.template.base import Origin, Template, TemplateDoesNotExist
+from django.utils.deprecation import RemovedInDjango21Warning
 from django.utils.encoding import force_bytes
 
 from .base import Loader as BaseLoader
@@ -15,20 +18,84 @@ class Loader(BaseLoader):
 
     def __init__(self, engine, loaders):
         self.template_cache = {}
-        self.find_template_cache = {}
+        self.find_template_cache = {}  # RemovedInDjango21Warning
+        self.get_template_cache = {}
         self.loaders = engine.get_template_loaders(loaders)
         super(Loader, self).__init__(engine)
 
-    def cache_key(self, template_name, template_dirs):
-        if template_dirs:
-            # If template directories were specified, use a hash to differentiate
-            return '-'.join([template_name, hashlib.sha1(force_bytes('|'.join(template_dirs))).hexdigest()])
+    def get_contents(self, origin):
+        return origin.loader.get_contents(origin)
+
+    def get_template(self, template_name, template_dirs=None, skip=None):
+        key = self.cache_key(template_name, template_dirs, skip)
+        cached = self.get_template_cache.get(key)
+        if cached:
+            if isinstance(cached, TemplateDoesNotExist):
+                raise cached
+            return cached
+
+        try:
+            template = super(Loader, self).get_template(
+                template_name, template_dirs, skip,
+            )
+        except TemplateDoesNotExist as e:
+            self.get_template_cache[key] = e
+            raise
         else:
-            return template_name
+            self.get_template_cache[key] = template
+
+        return template
+
+    def get_template_sources(self, template_name, template_dirs=None):
+        for loader in self.loaders:
+            args = [template_name]
+            # RemovedInDjango21Warning: Add template_dirs for compatibility
+            # with old loaders
+            if 'template_dirs' in getargspec(loader.get_template_sources)[0]:
+                args.append(template_dirs)
+            for origin in loader.get_template_sources(*args):
+                yield origin
+
+    def cache_key(self, template_name, template_dirs, skip=None):
+        """
+        Generate a cache key for the template name, dirs, and skip.
+
+        If skip is provided, only origins that match template_name are included
+        in the cache key. This ensures each template is only parsed and cached
+        once if contained in different extend chains like:
+
+            x -> a -> a
+            y -> a -> a
+            z -> a -> a
+        """
+        dirs_prefix = ''
+        skip_prefix = ''
+
+        if skip:
+            matching = [origin.name for origin in skip if origin.template_name == template_name]
+            if matching:
+                skip_prefix = self.generate_hash(matching)
+
+        if template_dirs:
+            dirs_prefix = self.generate_hash(template_dirs)
+
+        return ("%s-%s-%s" % (template_name, skip_prefix, dirs_prefix)).strip('-')
+
+    def generate_hash(self, values):
+        return hashlib.sha1(force_bytes('|'.join(values))).hexdigest()
+
+    @property
+    def supports_recursion(self):
+        """
+        RemovedInDjango21Warning: This is an internal property used by the
+        ExtendsNode during the deprecation of non-recursive loaders.
+        """
+        return all(hasattr(loader, 'get_contents') for loader in self.loaders)
 
     def find_template(self, name, dirs=None):
         """
-        Helper method. Lookup the template :param name: in all the configured loaders
+        RemovedInDjango21Warning: An internal method to lookup the template
+        name in all the configured loaders.
         """
         key = self.cache_key(name, dirs)
         try:
@@ -41,7 +108,11 @@ class Loader(BaseLoader):
                 except TemplateDoesNotExist:
                     pass
                 else:
-                    origin = self.engine.make_origin(display_name, loader, name, dirs)
+                    origin = Origin(
+                        name=display_name,
+                        template_name=name,
+                        loader=loader,
+                    )
                     result = template, origin
                     break
         self.find_template_cache[key] = result
@@ -52,6 +123,10 @@ class Loader(BaseLoader):
             raise TemplateDoesNotExist(name)
 
     def load_template(self, template_name, template_dirs=None):
+        warnings.warn(
+            'The load_template() method is deprecated. Use get_template() '
+            'instead.', RemovedInDjango21Warning,
+        )
         key = self.cache_key(template_name, template_dirs)
         template_tuple = self.template_cache.get(key)
         # A cached previous failure:
@@ -74,4 +149,5 @@ class Loader(BaseLoader):
     def reset(self):
         "Empty the template cache."
         self.template_cache.clear()
-        self.find_template_cache.clear()
+        self.find_template_cache.clear()  # RemovedInDjango21Warning
+        self.get_template_cache.clear()

django/template/loaders/eggs.py

@@ -1,9 +1,12 @@
 # Wrapper for loading templates from eggs via pkg_resources.resource_string.
 from __future__ import unicode_literals
 
+import warnings
+
 from django.apps import apps
-from django.template.base import TemplateDoesNotExist
+from django.template.base import Origin, TemplateDoesNotExist
 from django.utils import six
+from django.utils.deprecation import RemovedInDjango21Warning
 
 from .base import Loader as BaseLoader
 
@@ -13,6 +16,14 @@ except ImportError:
     resource_string = None
 
 
+class EggOrigin(Origin):
+
+    def __init__(self, app_name, pkg_name, *args, **kwargs):
+        self.app_name = app_name
+        self.pkg_name = pkg_name
+        return super(EggOrigin, self).__init__(*args, **kwargs)
+
+
 class Loader(BaseLoader):
 
     def __init__(self, engine):
@@ -20,19 +31,42 @@ class Loader(BaseLoader):
             raise RuntimeError("Setuptools must be installed to use the egg loader")
         super(Loader, self).__init__(engine)
 
+    def get_contents(self, origin):
+        try:
+            source = resource_string(origin.app_name, origin.pkg_name)
+        except:
+            raise TemplateDoesNotExist(origin)
+
+        if six.PY2:
+            source = source.decode(self.engine.file_charset)
+
+        return source
+
+    def get_template_sources(self, template_name):
+        pkg_name = 'templates/' + template_name
+        for app_config in apps.get_app_configs():
+            yield EggOrigin(
+                app_name=app_config.name,
+                pkg_name=pkg_name,
+                name="egg:%s:%s" % (app_config.name, pkg_name),
+                template_name=template_name,
+                loader=self,
+            )
+
     def load_template_source(self, template_name, template_dirs=None):
         """
         Loads templates from Python eggs via pkg_resource.resource_string.
 
         For every installed app, it tries to get the resource (app, template_name).
         """
-        pkg_name = 'templates/' + template_name
-        for app_config in apps.get_app_configs():
+        warnings.warn(
+            'The load_template_sources() method is deprecated. Use '
+            'get_template() or get_contents() instead.',
+            RemovedInDjango21Warning,
+        )
+        for origin in self.get_template_sources(template_name):
             try:
-                resource = resource_string(app_config.name, pkg_name)
-            except Exception:
-                continue
-            if six.PY2:
-                resource = resource.decode(self.engine.file_charset)
-            return (resource, 'egg:%s:%s' % (app_config.name, pkg_name))
+                return self.get_contents(origin), origin.name
+            except TemplateDoesNotExist:
+                pass
         raise TemplateDoesNotExist(template_name)

django/template/loaders/filesystem.py

@@ -4,10 +4,12 @@ Wrapper for loading templates from the filesystem.
 
 import errno
 import io
+import warnings
 
 from django.core.exceptions import SuspiciousFileOperation
-from django.template.base import TemplateDoesNotExist
+from django.template.base import Origin, TemplateDoesNotExist
 from django.utils._os import safe_join
+from django.utils.deprecation import RemovedInDjango21Warning
 
 from .base import Loader as BaseLoader
 
@@ -17,28 +19,46 @@ class Loader(BaseLoader):
     def get_dirs(self):
         return self.engine.dirs
 
+    def get_contents(self, origin):
+        try:
+            with io.open(origin.name, encoding=self.engine.file_charset) as fp:
+                return fp.read()
+        except IOError as e:
+            if e.errno == errno.ENOENT:
+                raise TemplateDoesNotExist(origin)
+            raise
+
     def get_template_sources(self, template_name, template_dirs=None):
         """
-        Returns the absolute paths to "template_name", when appended to each
-        directory in "template_dirs". Any paths that don't lie inside one of the
-        template dirs are excluded from the result set, for security reasons.
+        Return an Origin object pointing to an absolute path in each directory
+        in template_dirs. For security reasons, if a path doesn't lie inside
+        one of the template_dirs it is excluded from the result set.
         """
         if not template_dirs:
             template_dirs = self.get_dirs()
         for template_dir in template_dirs:
             try:
-                yield safe_join(template_dir, template_name)
+                name = safe_join(template_dir, template_name)
             except SuspiciousFileOperation:
                 # The joined path was located outside of this template_dir
                 # (it might be inside another one, so this isn't fatal).
-                pass
+                continue
+
+            yield Origin(
+                name=name,
+                template_name=template_name,
+                loader=self,
+            )
 
     def load_template_source(self, template_name, template_dirs=None):
-        for filepath in self.get_template_sources(template_name, template_dirs):
+        warnings.warn(
+            'The load_template_sources() method is deprecated. Use '
+            'get_template() or get_contents() instead.',
+            RemovedInDjango21Warning,
+        )
+        for origin in self.get_template_sources(template_name, template_dirs):
             try:
-                with io.open(filepath, encoding=self.engine.file_charset) as fp:
-                    return fp.read(), filepath
-            except IOError as e:
-                if e.errno != errno.ENOENT:
-                    raise
+                return self.get_contents(origin), origin.name
+            except TemplateDoesNotExist:
+                pass
         raise TemplateDoesNotExist(template_name)

django/template/loaders/locmem.py

@@ -2,7 +2,10 @@
 Wrapper for loading templates from a plain Python dict.
 """
 
-from django.template.base import TemplateDoesNotExist
+import warnings
+
+from django.template.base import Origin, TemplateDoesNotExist
+from django.utils.deprecation import RemovedInDjango21Warning
 
 from .base import Loader as BaseLoader
 
@@ -13,7 +16,25 @@ class Loader(BaseLoader):
         self.templates_dict = templates_dict
         super(Loader, self).__init__(engine)
 
+    def get_contents(self, origin):
+        try:
+            return self.templates_dict[origin.name]
+        except KeyError:
+            raise TemplateDoesNotExist(origin)
+
+    def get_template_sources(self, template_name):
+        yield Origin(
+            name=template_name,
+            template_name=template_name,
+            loader=self,
+        )
+
     def load_template_source(self, template_name, template_dirs=None):
+        warnings.warn(
+            'The load_template_sources() method is deprecated. Use '
+            'get_template() or get_contents() instead.',
+            RemovedInDjango21Warning,
+        )
         try:
             return self.templates_dict[template_name], template_name
         except KeyError: