[soc2009/multidb] Merged up to trunk r11864.

git-svn-id: http://code.djangoproject.com/svn/django/branches/soc2009/multidb@11866 bcc190cf-cafb-0310-a4f2-bffc1f526a37

AUTHORS

@@ -288,6 +288,7 @@ answer newbie questions, and generally made Django that much better:
     Martin Mahner <http://www.mahner.org/>
     Matt McClanahan <http://mmcc.cx/>
     Frantisek Malina <vizualbod@vizualbod.com>
+    Mike Malone <mjmalone@gmail.com>
     Martin Maney <http://www.chipy.org/Martin_Maney>
     masonsimon+django@gmail.com
     Manuzhai

django/conf/global_settings.py

@@ -161,9 +161,9 @@ TEMPLATE_DIRS = ()
 # See the comments in django/core/template/loader.py for interface
 # documentation.
 TEMPLATE_LOADERS = (
-    'django.template.loaders.filesystem.load_template_source',
+    'django.template.loaders.filesystem.Loader',
-    'django.template.loaders.app_directories.load_template_source',
+    'django.template.loaders.app_directories.Loader',
-#     'django.template.loaders.eggs.load_template_source',
+#     'django.template.loaders.eggs.Loader',
 )
 
 # List of processors used by RequestContext to populate the context.

django/conf/project_template/settings.py

@@ -56,9 +56,9 @@ SECRET_KEY = ''
 
 # List of callables that know how to import templates from various sources.
 TEMPLATE_LOADERS = (
-    'django.template.loaders.filesystem.load_template_source',
+    'django.template.loaders.filesystem.Loader',
-    'django.template.loaders.app_directories.load_template_source',
+    'django.template.loaders.app_directories.Loader',
-#     'django.template.loaders.eggs.load_template_source',
+#     'django.template.loaders.eggs.Loader',
 )
 
 MIDDLEWARE_CLASSES = (

django/contrib/admin/options.py

@@ -693,8 +693,8 @@ class ModelAdmin(BaseModelAdmin):
             selected = request.POST.getlist(helpers.ACTION_CHECKBOX_NAME)
             if not selected:
                 # Reminder that something needs to be selected or nothing will happen
-                msg = "Items must be selected in order to perform actions on them. No items have been changed."
+                msg = _("Items must be selected in order to perform actions on them. No items have been changed.")
-                self.message_user(request, _(msg))
+                self.message_user(request, msg)
                 return None
 
             response = func(self, request, queryset.filter(pk__in=selected))
@@ -707,8 +707,8 @@ class ModelAdmin(BaseModelAdmin):
             else:
                 return HttpResponseRedirect(".")
         else:
-            msg = "No action selected."
+            msg = _("No action selected.")
-            self.message_user(request, _(msg))
+            self.message_user(request, msg)
 
     @csrf_protect
     @transaction.commit_on_success

django/contrib/auth/models.py

@@ -47,6 +47,13 @@ def check_password(raw_password, enc_password):
 class SiteProfileNotAvailable(Exception):
     pass
 
+class PermissionManager(models.Manager):
+    def get_by_natural_key(self, codename, app_label, model):
+        return self.get(
+            codename=codename,
+            content_type=ContentType.objects.get_by_natural_key(app_label, model)
+        )
+
 class Permission(models.Model):
     """The permissions system provides a way to assign permissions to specific users and groups of users.
 
@@ -63,6 +70,7 @@ class Permission(models.Model):
     name = models.CharField(_('name'), max_length=50)
     content_type = models.ForeignKey(ContentType)
     codename = models.CharField(_('codename'), max_length=100)
+    objects = PermissionManager()
 
     class Meta:
         verbose_name = _('permission')
@@ -76,6 +84,10 @@ class Permission(models.Model):
             unicode(self.content_type),
             unicode(self.name))
 
+    def natural_key(self):
+        return (self.codename,) + self.content_type.natural_key()
+    natural_key.dependencies = ['contenttypes.contenttype']
+
 class Group(models.Model):
     """Groups are a generic way of categorizing users to apply permissions, or some other label, to those users. A user can belong to any number of groups.
 

django/contrib/contenttypes/models.py

@@ -8,6 +8,13 @@ class ContentTypeManager(models.Manager):
     # This cache is shared by all the get_for_* methods.
     _cache = {}
 
+    def get_by_natural_key(self, app_label, model):
+        try:
+            ct = self.__class__._cache[(app_label, model)]
+        except KeyError:
+            ct = self.get(app_label=app_label, model=model)
+        return ct
+
     def get_for_model(self, model):
         """
         Returns the ContentType object for a given model, creating the
@@ -93,3 +100,6 @@ class ContentType(models.Model):
         so code that calls this method should catch it.
         """
         return self.model_class()._default_manager.get(**kwargs)
+
+    def natural_key(self):
+        return (self.app_label, self.model)

django/core/management/commands/dumpdata.py

@@ -17,6 +17,8 @@ class Command(BaseCommand):
                 'fixtures into. Defaults to the "default" database.'),
         make_option('-e', '--exclude', dest='exclude',action='append', default=[],
             help='App to exclude (use multiple --exclude to exclude multiple apps).'),
+        make_option('-n', '--natural', action='store_true', dest='use_natural_keys', default=False,
+            help='Use natural keys if they are available.'),
     )
     help = 'Output the contents of the database as a fixture of the given format.'
     args = '[appname ...]'
@@ -30,6 +32,7 @@ class Command(BaseCommand):
         connection = connections[using]
         exclude = options.get('exclude',[])
         show_traceback = options.get('traceback', False)
+        use_natural_keys = options.get('use_natural_keys', False)
 
         excluded_apps = set(get_app(app_label) for app_label in exclude)
 
@@ -73,22 +76,86 @@ class Command(BaseCommand):
         except KeyError:
             raise CommandError("Unknown serialization format: %s" % format)
 
-        # Get a list of synchronized tables
+        # Now collate the objects to be serialized.
-        tables = connection.introspection.table_names()
-
         objects = []
-        for app, model_list in app_list.items():
+        for model in sort_dependencies(app_list.items()):
-            if model_list is None:
+            if not model._meta.proxy:
-                model_list = get_models(app)
+                objects.extend(model._default_manager.using(using).all())
-
-            for model in model_list:
-                # Don't serialize proxy models, or models that haven't been synchronized
-                if not model._meta.proxy and model._meta.db_table in tables:
-                    objects.extend(model._default_manager.using(using).all())
 
         try:
-            return serializers.serialize(format, objects, indent=indent)
+            return serializers.serialize(format, objects, indent=indent,
+                        use_natural_keys=use_natural_keys)
         except Exception, e:
             if show_traceback:
                 raise
             raise CommandError("Unable to serialize database: %s" % e)
+
+def sort_dependencies(app_list):
+    """Sort a list of app,modellist pairs into a single list of models.
+
+    The single list of models is sorted so that any model with a natural key
+    is serialized before a normal model, and any model with a natural key
+    dependency has it's dependencies serialized first.
+    """
+    from django.db.models import get_model, get_models
+    # Process the list of models, and get the list of dependencies
+    model_dependencies = []
+    models = set()
+    for app, model_list in app_list:
+        if model_list is None:
+            model_list = get_models(app)
+
+        for model in model_list:
+            models.add(model)
+            # Add any explicitly defined dependencies
+            if hasattr(model, 'natural_key'):
+                deps = getattr(model.natural_key, 'dependencies', [])
+                if deps:
+                    deps = [get_model(*d.split('.')) for d in deps]
+            else:
+                deps = []
+
+            # Now add a dependency for any FK or M2M relation with
+            # a model that defines a natural key
+            for field in model._meta.fields:
+                if hasattr(field.rel, 'to'):
+                    rel_model = field.rel.to
+                    if hasattr(rel_model, 'natural_key'):
+                        deps.append(rel_model)
+            for field in model._meta.many_to_many:
+                rel_model = field.rel.to
+                if hasattr(rel_model, 'natural_key'):
+                    deps.append(rel_model)
+            model_dependencies.append((model, deps))
+
+    model_dependencies.reverse()
+    # Now sort the models to ensure that dependencies are met. This
+    # is done by repeatedly iterating over the input list of models.
+    # If all the dependencies of a given model are in the final list,
+    # that model is promoted to the end of the final list. This process
+    # continues until the input list is empty, or we do a full iteration
+    # over the input models without promoting a model to the final list.
+    # If we do a full iteration without a promotion, that means there are
+    # circular dependencies in the list.
+    model_list = []
+    while model_dependencies:
+        skipped = []
+        changed = False
+        while model_dependencies:
+            model, deps = model_dependencies.pop()
+            if all((d not in models or d in model_list) for d in deps):
+                # If all of the models in the dependency list are either already
+                # on the final model list, or not on the original serialization list,
+                # then we've found another model with all it's dependencies satisfied.
+                model_list.append(model)
+                changed = True
+            else:
+                skipped.append((model, deps))
+        if not changed:
+            raise CommandError("Can't resolve dependencies for %s in serialized app list." %
+                ', '.join('%s.%s' % (model._meta.app_label, model._meta.object_name)
+                for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__))
+            )
+        model_dependencies = skipped
+
+    return model_list

django/core/serializers/base.py

@@ -33,6 +33,7 @@ class Serializer(object):
 
         self.stream = options.get("stream", StringIO())
         self.selected_fields = options.get("fields")
+        self.use_natural_keys = options.get("use_natural_keys", False)
 
         self.start_serialization()
         for obj in queryset:

django/core/serializers/json.py

@@ -24,6 +24,7 @@ class Serializer(PythonSerializer):
     def end_serialization(self):
         self.options.pop('stream', None)
         self.options.pop('fields', None)
+        self.options.pop('use_natural_keys', None)
         simplejson.dump(self.objects, self.stream, cls=DjangoJSONEncoder, **self.options)
 
     def getvalue(self):

django/core/serializers/python.py

@@ -47,17 +47,24 @@ class Serializer(base.Serializer):
     def handle_fk_field(self, obj, field):
         related = getattr(obj, field.name)
         if related is not None:
-            if field.rel.field_name == related._meta.pk.name:
+            if self.use_natural_keys and hasattr(related, 'natural_key'):
-                # Related to remote object via primary key
+                related = related.natural_key()
-                related = related._get_pk_val()
             else:
-                # Related to remote object via other field
+                if field.rel.field_name == related._meta.pk.name:
-                related = getattr(related, field.rel.field_name)
+                    # Related to remote object via primary key
-        self._current[field.name] = smart_unicode(related, strings_only=True)
+                    related = related._get_pk_val()
+                else:
+                    # Related to remote object via other field
+                    related = smart_unicode(getattr(related, field.rel.field_name), strings_only=True)
+        self._current[field.name] = related
 
     def handle_m2m_field(self, obj, field):
         if field.rel.through._meta.auto_created:
-            self._current[field.name] = [smart_unicode(related._get_pk_val(), strings_only=True)
+            if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
+                m2m_value = lambda value: value.natural_key()
+            else:
+                m2m_value = lambda value: smart_unicode(value._get_pk_val(), strings_only=True)
+            self._current[field.name] = [m2m_value(related)
                                for related in getattr(obj, field.name).iterator()]
 
     def getvalue(self):
@@ -86,13 +93,28 @@ def Deserializer(object_list, **options):
 
             # Handle M2M relations
             if field.rel and isinstance(field.rel, models.ManyToManyRel):
-                m2m_convert = field.rel.to._meta.pk.to_python
+                if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
-                m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value]
+                    def m2m_convert(value):
+                        if hasattr(value, '__iter__'):
+                            return field.rel.to._default_manager.get_by_natural_key(*value).pk
+                        else:
+                            return smart_unicode(field.rel.to._meta.pk.to_python(value))
+                else:
+                    m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v))
+                m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
 
             # Handle FK fields
             elif field.rel and isinstance(field.rel, models.ManyToOneRel):
                 if field_value is not None:
-                    data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
+                    if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
+                        if hasattr(field_value, '__iter__'):
+                            obj = field.rel.to._default_manager.get_by_natural_key(*field_value)
+                            value = getattr(obj, field.rel.field_name)
+                        else:
+                            value = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
+                        data[field.attname] = value
+                    else:
+                        data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
                 else:
                     data[field.attname] = None
 

django/core/serializers/pyyaml.py

@@ -26,9 +26,9 @@ class Serializer(PythonSerializer):
     """
     Convert a queryset to YAML.
     """
-    
+
     internal_use_only = False
-    
+
     def handle_field(self, obj, field):
         # A nasty special case: base YAML doesn't support serialization of time
         # types (as opposed to dates or datetimes, which it does support). Since
@@ -40,10 +40,11 @@ class Serializer(PythonSerializer):
             self._current[field.name] = str(getattr(obj, field.name))
         else:
             super(Serializer, self).handle_field(obj, field)
-    
+
     def end_serialization(self):
         self.options.pop('stream', None)
         self.options.pop('fields', None)
+        self.options.pop('use_natural_keys', None)
         yaml.dump(self.objects, self.stream, Dumper=DjangoSafeDumper, **self.options)
 
     def getvalue(self):

django/core/serializers/xml_serializer.py

@@ -81,13 +81,22 @@ class Serializer(base.Serializer):
         self._start_relational_field(field)
         related = getattr(obj, field.name)
         if related is not None:
-            if field.rel.field_name == related._meta.pk.name:
+            if self.use_natural_keys and hasattr(related, 'natural_key'):
-                # Related to remote object via primary key
+                # If related object has a natural key, use it
-                related = related._get_pk_val()
+                related = related.natural_key()
+                # Iterable natural keys are rolled out as subelements
+                for key_value in related:
+                    self.xml.startElement("natural", {})
+                    self.xml.characters(smart_unicode(key_value))
+                    self.xml.endElement("natural")
             else:
-                # Related to remote object via other field
+                if field.rel.field_name == related._meta.pk.name:
-                related = getattr(related, field.rel.field_name)
+                    # Related to remote object via primary key
-            self.xml.characters(smart_unicode(related))
+                    related = related._get_pk_val()
+                else:
+                    # Related to remote object via other field
+                    related = getattr(related, field.rel.field_name)
+                self.xml.characters(smart_unicode(related))
         else:
             self.xml.addQuickElement("None")
         self.xml.endElement("field")
@@ -100,8 +109,25 @@ class Serializer(base.Serializer):
         """
         if field.rel.through._meta.auto_created:
             self._start_relational_field(field)
+            if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
+                # If the objects in the m2m have a natural key, use it
+                def handle_m2m(value):
+                    natural = value.natural_key()
+                    # Iterable natural keys are rolled out as subelements
+                    self.xml.startElement("object", {})
+                    for key_value in natural:
+                        self.xml.startElement("natural", {})
+                        self.xml.characters(smart_unicode(key_value))
+                        self.xml.endElement("natural")
+                    self.xml.endElement("object")
+            else:
+                def handle_m2m(value):
+                    self.xml.addQuickElement("object", attrs={
+                        'pk' : smart_unicode(value._get_pk_val())
+                    })
             for relobj in getattr(obj, field.name).iterator():
-                self.xml.addQuickElement("object", attrs={"pk" : smart_unicode(relobj._get_pk_val())})
+                handle_m2m(relobj)
+
             self.xml.endElement("field")
 
     def _start_relational_field(self, field):
@@ -187,16 +213,40 @@ class Deserializer(base.Deserializer):
         if node.getElementsByTagName('None'):
             return None
         else:
-            return field.rel.to._meta.get_field(field.rel.field_name).to_python(
+            if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
-                       getInnerText(node).strip())
+                keys = node.getElementsByTagName('natural')
+                if keys:
+                    # If there are 'natural' subelements, it must be a natural key
+                    field_value = [getInnerText(k).strip() for k in keys]
+                    obj = field.rel.to._default_manager.get_by_natural_key(*field_value)
+                    obj_pk = getattr(obj, field.rel.field_name)
+                else:
+                    # Otherwise, treat like a normal PK
+                    field_value = getInnerText(node).strip()
+                    obj_pk = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
+                return obj_pk
+            else:
+                field_value = getInnerText(node).strip()
+                return field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
 
     def _handle_m2m_field_node(self, node, field):
         """
         Handle a <field> node for a ManyToManyField.
         """
-        return [field.rel.to._meta.pk.to_python(
+        if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
-                    c.getAttribute("pk"))
+            def m2m_convert(n):
-                    for c in node.getElementsByTagName("object")]
+                keys = n.getElementsByTagName('natural')
+                if keys:
+                    # If there are 'natural' subelements, it must be a natural key
+                    field_value = [getInnerText(k).strip() for k in keys]
+                    obj_pk = field.rel.to._default_manager.get_by_natural_key(*field_value).pk
+                else:
+                    # Otherwise, treat like a normal PK value.
+                    obj_pk = field.rel.to._meta.pk.to_python(n.getAttribute('pk'))
+                return obj_pk
+        else:
+            m2m_convert = lambda n: field.rel.to._meta.pk.to_python(n.getAttribute('pk'))
+        return [m2m_convert(c) for c in node.getElementsByTagName("object")]
 
     def _get_model_from_node(self, node, attr):
         """

django/template/__init__.py

@@ -173,9 +173,16 @@ class Template(object):
             for subnode in node:
                 yield subnode
 
+    def _render(self, context):
+        return self.nodelist.render(context)
+
     def render(self, context):
         "Display stage -- can be called many times"
-        return self.nodelist.render(context)
+        context.render_context.push()
+        try:
+            return self._render(context)
+        finally:
+            context.render_context.pop()
 
 def compile_string(template_string, origin):
     "Compiles template_string into NodeList ready for rendering"

django/template/context.py

@@ -12,45 +12,42 @@ class ContextPopException(Exception):
     "pop() has been called more times than push()"
     pass
 
-class Context(object):
+class BaseContext(object):
-    "A stack container for variable context"
+    def __init__(self, dict_=None):
-    def __init__(self, dict_=None, autoescape=True, current_app=None):
         dict_ = dict_ or {}
         self.dicts = [dict_]
-        self.autoescape = autoescape
-        self.current_app = current_app
 
     def __repr__(self):
         return repr(self.dicts)
 
     def __iter__(self):
-        for d in self.dicts:
+        for d in reversed(self.dicts):
             yield d
 
     def push(self):
         d = {}
-        self.dicts = [d] + self.dicts
+        self.dicts.append(d)
         return d
 
     def pop(self):
         if len(self.dicts) == 1:
             raise ContextPopException
-        return self.dicts.pop(0)
+        return self.dicts.pop()
 
     def __setitem__(self, key, value):
         "Set a variable in the current context"
-        self.dicts[0][key] = value
+        self.dicts[-1][key] = value
 
     def __getitem__(self, key):
         "Get a variable's value, starting at the current context and going upward"
-        for d in self.dicts:
+        for d in reversed(self.dicts):
             if key in d:
                 return d[key]
         raise KeyError(key)
 
     def __delitem__(self, key):
         "Delete a variable from the current context"
-        del self.dicts[0][key]
+        del self.dicts[-1][key]
 
     def has_key(self, key):
         for d in self.dicts:
@@ -58,21 +55,58 @@ class Context(object):
                 return True
         return False
 
-    __contains__ = has_key
+    def __contains__(self, key):
+        return self.has_key(key)
 
     def get(self, key, otherwise=None):
-        for d in self.dicts:
+        for d in reversed(self.dicts):
             if key in d:
                 return d[key]
         return otherwise
 
+class Context(BaseContext):
+    "A stack container for variable context"
+    def __init__(self, dict_=None, autoescape=True, current_app=None):
+        self.autoescape = autoescape
+        self.current_app = current_app
+        self.render_context = RenderContext()
+        super(Context, self).__init__(dict_)
+
     def update(self, other_dict):
         "Like dict.update(). Pushes an entire dictionary's keys and values onto the context."
         if not hasattr(other_dict, '__getitem__'):
             raise TypeError('other_dict must be a mapping (dictionary-like) object.')
-        self.dicts = [other_dict] + self.dicts
+        self.dicts.append(other_dict)
         return other_dict
 
+class RenderContext(BaseContext):
+    """
+    A stack container for storing Template state.
+
+    RenderContext simplifies the implementation of template Nodes by providing a
+    safe place to store state between invocations of a node's `render` method.
+
+    The RenderContext also provides scoping rules that are more sensible for
+    'template local' variables. The render context stack is pushed before each
+    template is rendered, creating a fresh scope with nothing in it. Name
+    resolution fails if a variable is not found at the top of the RequestContext
+    stack. Thus, variables are local to a specific template and don't affect the
+    rendering of other templates as they would if they were stored in the normal
+    template context.
+    """
+    def __iter__(self):
+        for d in self.dicts[-1]:
+            yield d
+
+    def has_key(self, key):
+        return key in self.dicts[-1]
+
+    def get(self, key, otherwise=None):
+        d = self.dicts[-1]
+        if key in d:
+            return d[key]
+        return otherwise
+
 # This is a function rather than module-level procedural code because we only
 # want it to execute if somebody uses RequestContext.
 def get_standard_processors():

django/template/defaulttags.py

@@ -57,11 +57,14 @@ class CsrfTokenNode(Node):
 
 class CycleNode(Node):
     def __init__(self, cyclevars, variable_name=None):
-        self.cycle_iter = itertools_cycle(cyclevars)
+        self.cyclevars = cyclevars
         self.variable_name = variable_name
 
     def render(self, context):
-        value = self.cycle_iter.next().resolve(context)
+        if self not in context.render_context:
+            context.render_context[self] = itertools_cycle(self.cyclevars)
+        cycle_iter = context.render_context[self]
+        value = cycle_iter.next().resolve(context)
         if self.variable_name:
             context[self.variable_name] = value
         return value

django/template/loader.py

@@ -27,6 +27,36 @@ from django.conf import settings
 
 template_source_loaders = None
 
+class BaseLoader(object):
+    is_usable = False
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def __call__(self, template_name, template_dirs=None):
+        return self.load_template(template_name, template_dirs)
+
+    def load_template(self, template_name, template_dirs=None):
+        source, origin = self.load_template_source(template_name, template_dirs)
+        template = get_template_from_string(source, name=template_name)
+        return template, origin
+
+    def load_template_source(self, template_name, template_dirs=None):
+        """
+        Returns a tuple containing the source and origin for the given template
+        name.
+
+        """
+        raise NotImplementedError
+
+    def reset(self):
+        """
+        Resets any state maintained by the loader instance (e.g., cached
+        templates or cached loader modules).
+
+        """
+        pass
+
 class LoaderOrigin(Origin):
     def __init__(self, display_name, loader, name, dirs):
         super(LoaderOrigin, self).__init__(display_name)
@@ -41,29 +71,50 @@ def make_origin(display_name, loader, name, dirs):
     else:
         return None
 
-def find_template_source(name, dirs=None):
+def find_template_loader(loader):
+    if hasattr(loader, '__iter__'):
+        loader, args = loader[0], loader[1:]
+    else:
+        args = []
+    if isinstance(loader, basestring):
+        module, attr = loader.rsplit('.', 1)
+        try:
+            mod = import_module(module)
+        except ImportError:
+            raise ImproperlyConfigured('Error importing template source loader %s: "%s"' % (loader, e))
+        try:
+            TemplateLoader = getattr(mod, attr)
+        except AttributeError, e:
+            raise ImproperlyConfigured('Error importing template source loader %s: "%s"' % (loader, e))
+
+        if hasattr(TemplateLoader, 'load_template_source'):
+            func = TemplateLoader(*args)
+        else:
+            # Try loading module the old way - string is full path to callable
+            if args:
+                raise ImproperlyConfigured("Error importing template source loader %s - can't pass arguments to function-based loader." % loader)
+            func = TemplateLoader
+
+        if not func.is_usable:
+            import warnings
+            warnings.warn("Your TEMPLATE_LOADERS setting includes %r, but your Python installation doesn't support that type of template loading. Consider removing that line from TEMPLATE_LOADERS." % loader)
+            return None
+        else:
+            return func
+    else:
+        raise ImproperlyConfigured('Loader does not define a "load_template" callable template source loader')
+
+def find_template(name, dirs=None):
     # Calculate template_source_loaders the first time the function is executed
     # because putting this logic in the module-level namespace may cause
     # circular import errors. See Django ticket #1292.
     global template_source_loaders
     if template_source_loaders is None:
         loaders = []
-        for path in settings.TEMPLATE_LOADERS:
+        for loader_name in settings.TEMPLATE_LOADERS:
-            i = path.rfind('.')
+            loader = find_template_loader(loader_name)
-            module, attr = path[:i], path[i+1:]
+            if loader is not None:
-            try:
+                loaders.append(loader)
-                mod = import_module(module)
-            except ImportError, e:
-                raise ImproperlyConfigured, 'Error importing template source loader %s: "%s"' % (module, e)
-            try:
-                func = getattr(mod, attr)
-            except AttributeError:
-                raise ImproperlyConfigured, 'Module "%s" does not define a "%s" callable template source loader' % (module, attr)
-            if not func.is_usable:
-                import warnings
-                warnings.warn("Your TEMPLATE_LOADERS setting includes %r, but your Python installation doesn't support that type of template loading. Consider removing that line from TEMPLATE_LOADERS." % path)
-            else:
-                loaders.append(func)
         template_source_loaders = tuple(loaders)
     for loader in template_source_loaders:
         try:
@@ -73,13 +124,27 @@ def find_template_source(name, dirs=None):
             pass
     raise TemplateDoesNotExist, name
 
+def find_template_source(name, dirs=None):
+    # For backward compatibility
+    import warnings
+    warnings.warn(
+        "`django.template.loaders.find_template_source` is deprecated; use `django.template.loaders.find_template` instead.",
+        PendingDeprecationWarning
+    )
+    template, origin = find_template(name, dirs)
+    if hasattr(template, 'render'):
+        raise Exception("Found a compiled template that is incompatible with the deprecated `django.template.loaders.find_template_source` function.")
+    return template, origin
+
 def get_template(template_name):
     """
     Returns a compiled Template object for the given template name,
     handling template inheritance recursively.
     """
-    source, origin = find_template_source(template_name)
+    template, origin = find_template(template_name)
-    template = get_template_from_string(source, origin, template_name)
+    if not hasattr(template, 'render'):
+        # template needs to be compiled
+        template = get_template_from_string(template, origin, template_name)
     return template
 
 def get_template_from_string(source, origin=None, name=None):

django/template/loader_tags.py

@@ -1,14 +1,43 @@
 from django.template import TemplateSyntaxError, TemplateDoesNotExist, Variable
 from django.template import Library, Node, TextNode
-from django.template.loader import get_template, get_template_from_string, find_template_source
+from django.template.loader import get_template
 from django.conf import settings
 from django.utils.safestring import mark_safe
 
 register = Library()
 
+BLOCK_CONTEXT_KEY = 'block_context'
+
 class ExtendsError(Exception):
     pass
 
+class BlockContext(object):
+    def __init__(self):
+        # Dictionary of FIFO queues.
+        self.blocks = {}
+
+    def add_blocks(self, blocks):
+        for name, block in blocks.iteritems():
+            if name in self.blocks:
+                self.blocks[name].insert(0, block)
+            else:
+                self.blocks[name] = [block]
+
+    def pop(self, name):
+        try:
+            return self.blocks[name].pop()
+        except (IndexError, KeyError):
+            return None
+
+    def push(self, name, block):
+        self.blocks[name].append(block)
+
+    def get_block(self, name):
+        try:
+            return self.blocks[name][-1]
+        except (IndexError, KeyError):
+            return None
+
 class BlockNode(Node):
     def __init__(self, name, nodelist, parent=None):
         self.name, self.nodelist, self.parent = name, nodelist, parent
@@ -17,25 +46,32 @@ class BlockNode(Node):
         return "<Block Node: %s. Contents: %r>" % (self.name, self.nodelist)
 
     def render(self, context):
+        block_context = context.render_context.get(BLOCK_CONTEXT_KEY)
         context.push()
-        # Save context in case of block.super().
+        if block_context is None:
-        self.context = context
+            context['block'] = self
-        context['block'] = self
+            result = self.nodelist.render(context)
-        result = self.nodelist.render(context)
+        else:
+            push = block = block_context.pop(self.name)
+            if block is None:
+                block = self
+            # Create new block so we can store context without thread-safety issues.
+            block = BlockNode(block.name, block.nodelist)
+            block.context = context
+            context['block'] = block
+            result = block.nodelist.render(context)
+            if push is not None:
+                block_context.push(self.name, push)
         context.pop()
         return result
 
     def super(self):
-        if self.parent:
+        render_context = self.context.render_context
-            return mark_safe(self.parent.render(self.context))
+        if (BLOCK_CONTEXT_KEY in render_context and
+            render_context[BLOCK_CONTEXT_KEY].get_block(self.name) is not None):
+            return mark_safe(self.render(self.context))
         return ''
 
-    def add_parent(self, nodelist):
-        if self.parent:
-            self.parent.add_parent(nodelist)
-        else:
-            self.parent = BlockNode(self.name, nodelist)
-
 class ExtendsNode(Node):
     must_be_first = True
 
@@ -43,6 +79,7 @@ class ExtendsNode(Node):
         self.nodelist = nodelist
         self.parent_name, self.parent_name_expr = parent_name, parent_name_expr
         self.template_dirs = template_dirs
+        self.blocks = dict([(n.name, n) for n in nodelist.get_nodes_by_type(BlockNode)])
 
     def __repr__(self):
         if self.parent_name_expr:
@@ -61,40 +98,34 @@ class ExtendsNode(Node):
         if hasattr(parent, 'render'):
             return parent # parent is a Template object
         try:
-            source, origin = find_template_source(parent, self.template_dirs)
+            return get_template(parent)
         except TemplateDoesNotExist:
             raise TemplateSyntaxError, "Template %r cannot be extended, because it doesn't exist" % parent
-        else:
-            return get_template_from_string(source, origin, parent)
 
     def render(self, context):
         compiled_parent = self.get_parent(context)
-        parent_blocks = dict([(n.name, n) for n in compiled_parent.nodelist.get_nodes_by_type(BlockNode)])
-        for block_node in self.nodelist.get_nodes_by_type(BlockNode):
-            # Check for a BlockNode with this node's name, and replace it if found.
-            try:
-                parent_block = parent_blocks[block_node.name]
-            except KeyError:
-                # This BlockNode wasn't found in the parent template, but the
-                # parent block might be defined in the parent's *parent*, so we
-                # add this BlockNode to the parent's ExtendsNode nodelist, so
-                # it'll be checked when the parent node's render() is called.
 
-                # Find out if the parent template has a parent itself
+        if BLOCK_CONTEXT_KEY not in context.render_context:
-                for node in compiled_parent.nodelist:
+            context.render_context[BLOCK_CONTEXT_KEY] = BlockContext()
-                    if not isinstance(node, TextNode):
+        block_context = context.render_context[BLOCK_CONTEXT_KEY]
-                        # If the first non-text node is an extends, handle it.
+
-                        if isinstance(node, ExtendsNode):
+        # Add the block nodes from this node to the block context
-                            node.nodelist.append(block_node)
+        block_context.add_blocks(self.blocks)
-                        # Extends must be the first non-text node, so once you find
+
-                        # the first non-text node you can stop looking. 
+        # If this block's parent doesn't have an extends node it is the root,
-                        break
+        # and its block nodes also need to be added to the block context.
-            else:
+        for node in compiled_parent.nodelist:
-                # Keep any existing parents and add a new one. Used by BlockNode.
+            # The ExtendsNode has to be the first non-text node.
-                parent_block.parent = block_node.parent
+            if not isinstance(node, TextNode):
-                parent_block.add_parent(parent_block.nodelist)
+                if not isinstance(node, ExtendsNode):
-                parent_block.nodelist = block_node.nodelist
+                    blocks = dict([(n.name, n) for n in
-        return compiled_parent.render(context)
+                                   compiled_parent.nodelist.get_nodes_by_type(BlockNode)])
+                    block_context.add_blocks(blocks)
+                break
+
+        # Call Template._render explicitly so the parser context stays
+        # the same.
+        return compiled_parent._render(context)
 
 class ConstantIncludeNode(Node):
     def __init__(self, template_path):

django/template/loaders/app_directories.py

@@ -9,6 +9,7 @@ import sys
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.template import TemplateDoesNotExist
+from django.template.loader import BaseLoader
 from django.utils._os import safe_join
 from django.utils.importlib import import_module
 
@@ -27,29 +28,47 @@ for app in settings.INSTALLED_APPS:
 # It won't change, so convert it to a tuple to save memory.
 app_template_dirs = tuple(app_template_dirs)
 
-def get_template_sources(template_name, template_dirs=None):
+class Loader(BaseLoader):
-    """
+    is_usable = True
-    Returns the absolute paths to "template_name", when appended to each
+
-    directory in "template_dirs". Any paths that don't lie inside one of the
+    def get_template_sources(self, template_name, template_dirs=None):
-    template dirs are excluded from the result set, for security reasons.
+        """
-    """
+        Returns the absolute paths to "template_name", when appended to each
-    if not template_dirs:
+        directory in "template_dirs". Any paths that don't lie inside one of the
-        template_dirs = app_template_dirs
+        template dirs are excluded from the result set, for security reasons.
-    for template_dir in template_dirs:
+        """
-        try:
+        if not template_dirs:
-            yield safe_join(template_dir, template_name)
+            template_dirs = app_template_dirs
-        except UnicodeDecodeError:
+        for template_dir in template_dirs:
-            # The template dir name was a bytestring that wasn't valid UTF-8.
+            try:
-            raise
+                yield safe_join(template_dir, template_name)
-        except ValueError:
+            except UnicodeDecodeError:
-            # The joined path was located outside of template_dir.
+                # The template dir name was a bytestring that wasn't valid UTF-8.
-            pass
+                raise
+            except ValueError:
+                # The joined path was located outside of template_dir.
+                pass
+
+    def load_template_source(self, template_name, template_dirs=None):
+        for filepath in self.get_template_sources(template_name, template_dirs):
+            try:
+                file = open(filepath)
+                try:
+                    return (file.read().decode(settings.FILE_CHARSET), filepath)
+                finally:
+                    file.close()
+            except IOError:
+                pass
+        raise TemplateDoesNotExist, template_name
+
+_loader = Loader()
 
 def load_template_source(template_name, template_dirs=None):
-    for filepath in get_template_sources(template_name, template_dirs):
+    # For backwards compatibility
-        try:
+    import warnings
-            return (open(filepath).read().decode(settings.FILE_CHARSET), filepath)
+    warnings.warn(
-        except IOError:
+        "'django.template.loaders.app_directories.load_template_source' is deprecated; use 'django.template.loaders.app_directories.Loader' instead.",
-            pass
+        PendingDeprecationWarning
-    raise TemplateDoesNotExist, template_name
+    )
+    return _loader.load_template_source(template_name, template_dirs)
 load_template_source.is_usable = True

django/template/loaders/cached.py

@@ -0,0 +1,46 @@
+"""
+Wrapper class that takes a list of template loaders as an argument and attempts
+to load templates from them in order, caching the result.
+"""
+
+from django.template import TemplateDoesNotExist
+from django.template.loader import BaseLoader, get_template_from_string, find_template_loader, make_origin
+from django.utils.importlib import import_module
+from django.core.exceptions import ImproperlyConfigured
+
+class Loader(BaseLoader):
+    is_usable = True
+
+    def __init__(self, loaders):
+        self.template_cache = {}
+        self._loaders = loaders
+        self._cached_loaders = []
+
+    @property
+    def loaders(self):
+        # Resolve loaders on demand to avoid circular imports
+        if not self._cached_loaders:
+            for loader in self._loaders:
+                self._cached_loaders.append(find_template_loader(loader))
+        return self._cached_loaders
+
+    def find_template(self, name, dirs=None):
+        for loader in self.loaders:
+            try:
+                template, display_name = loader(name, dirs)
+                return (template, make_origin(display_name, loader, name, dirs))
+            except TemplateDoesNotExist:
+                pass
+        raise TemplateDoesNotExist, name
+
+    def load_template(self, template_name, template_dirs=None):
+        if template_name not in self.template_cache:
+            template, origin = self.find_template(template_name, template_dirs)
+            if not hasattr(template, 'render'):
+                template = get_template_from_string(template, origin, template_name)
+            self.template_cache[template_name] = (template, origin)
+        return self.template_cache[template_name]
+
+    def reset(self):
+        "Empty the template cache."
+        self.template_cache.clear()

django/template/loaders/eggs.py

@@ -6,20 +6,34 @@ except ImportError:
     resource_string = None
 
 from django.template import TemplateDoesNotExist
+from django.template.loader import BaseLoader
 from django.conf import settings
 
-def load_template_source(template_name, template_dirs=None):
+class Loader(BaseLoader):
-    """
+    is_usable = resource_string is not None
-    Loads templates from Python eggs via pkg_resource.resource_string.
 
-    For every installed app, it tries to get the resource (app, template_name).
+    def load_template_source(self, template_name, template_dirs=None):
-    """
+        """
-    if resource_string is not None:
+        Loads templates from Python eggs via pkg_resource.resource_string.
-        pkg_name = 'templates/' + template_name
+
-        for app in settings.INSTALLED_APPS:
+        For every installed app, it tries to get the resource (app, template_name).
-            try:
+        """
-                return (resource_string(app, pkg_name).decode(settings.FILE_CHARSET), 'egg:%s:%s' % (app, pkg_name))
+        if resource_string is not None:
-            except:
+            pkg_name = 'templates/' + template_name
-                pass
+            for app in settings.INSTALLED_APPS:
-    raise TemplateDoesNotExist, template_name
+                try:
+                    return (resource_string(app, pkg_name).decode(settings.FILE_CHARSET), 'egg:%s:%s' % (app, pkg_name))
+                except:
+                    pass
+        raise TemplateDoesNotExist, template_name
+
+_loader = Loader()
+
+def load_template_source(template_name, template_dirs=None):
+    import warnings
+    warnings.warn(
+        "'django.template.loaders.eggs.load_template_source' is deprecated; use 'django.template.loaders.eggs.Loader' instead.",
+        PendingDeprecationWarning
+    )
+    return _loader.load_template_source(template_name, template_dirs)
 load_template_source.is_usable = resource_string is not None

django/template/loaders/filesystem.py

@@ -4,38 +4,58 @@ Wrapper for loading templates from the filesystem.
 
 from django.conf import settings
 from django.template import TemplateDoesNotExist
+from django.template.loader import BaseLoader
 from django.utils._os import safe_join
 
-def get_template_sources(template_name, template_dirs=None):
+class Loader(BaseLoader):
-    """
+    is_usable = True
-    Returns the absolute paths to "template_name", when appended to each
+
-    directory in "template_dirs". Any paths that don't lie inside one of the
+    def get_template_sources(self, template_name, template_dirs=None):
-    template dirs are excluded from the result set, for security reasons.
+        """
-    """
+        Returns the absolute paths to "template_name", when appended to each
-    if not template_dirs:
+        directory in "template_dirs". Any paths that don't lie inside one of the
-        template_dirs = settings.TEMPLATE_DIRS
+        template dirs are excluded from the result set, for security reasons.
-    for template_dir in template_dirs:
+        """
-        try:
+        if not template_dirs:
-            yield safe_join(template_dir, template_name)
+            template_dirs = settings.TEMPLATE_DIRS
-        except UnicodeDecodeError:
+        for template_dir in template_dirs:
-            # The template dir name was a bytestring that wasn't valid UTF-8.
+            try:
-            raise
+                yield safe_join(template_dir, template_name)
-        except ValueError:
+            except UnicodeDecodeError:
-            # The joined path was located outside of this particular
+                # The template dir name was a bytestring that wasn't valid UTF-8.
-            # template_dir (it might be inside another one, so this isn't
+                raise
-            # fatal).
+            except ValueError:
-            pass
+                # The joined path was located outside of this particular
+                # template_dir (it might be inside another one, so this isn't
+                # fatal).
+                pass
+
+    def load_template_source(self, template_name, template_dirs=None):
+        tried = []
+        for filepath in self.get_template_sources(template_name, template_dirs):
+            try:
+                file = open(filepath)
+                try:
+                    return (file.read().decode(settings.FILE_CHARSET), filepath)
+                finally:
+                    file.close()
+            except IOError:
+                tried.append(filepath)
+        if tried:
+            error_msg = "Tried %s" % tried
+        else:
+            error_msg = "Your TEMPLATE_DIRS setting is empty. Change it to point to at least one template directory."
+        raise TemplateDoesNotExist, error_msg
+    load_template_source.is_usable = True
+
+_loader = Loader()
 
 def load_template_source(template_name, template_dirs=None):
-    tried = []
+    # For backwards compatibility
-    for filepath in get_template_sources(template_name, template_dirs):
+    import warnings
-        try:
+    warnings.warn(
-            return (open(filepath).read().decode(settings.FILE_CHARSET), filepath)
+        "'django.template.loaders.filesystem.load_template_source' is deprecated; use 'django.template.loaders.filesystem.Loader' instead.",
-        except IOError:
+        PendingDeprecationWarning
-            tried.append(filepath)
+    )
-    if tried:
+    return _loader.load_template_source(template_name, template_dirs)
-        error_msg = "Tried %s" % tried
-    else:
-        error_msg = "Your TEMPLATE_DIRS setting is empty. Change it to point to at least one template directory."
-    raise TemplateDoesNotExist, error_msg
 load_template_source.is_usable = True

django/test/utils.py

@@ -36,8 +36,8 @@ def setup_test_environment():
         - Set the email backend to the locmem email backend.
         - Setting the active locale to match the LANGUAGE_CODE setting.
     """
-    Template.original_render = Template.render
+    Template.original_render = Template._render
-    Template.render = instrumented_test_render
+    Template._render = instrumented_test_render
 
     mail.original_SMTPConnection = mail.SMTPConnection
     mail.SMTPConnection = locmem.EmailBackend
@@ -56,7 +56,7 @@ def teardown_test_environment():
         - Restoring the email sending functions
 
     """
-    Template.render = Template.original_render
+    Template._render = Template.original_render
     del Template.original_render
 
     mail.SMTPConnection = mail.original_SMTPConnection

django/views/debug.py

@@ -76,8 +76,12 @@ class ExceptionReporter:
                         for t in source_list_func(str(self.exc_value))]
                 except (ImportError, AttributeError):
                     template_list = []
+                if hasattr(loader, '__class__'):
+                    loader_name = loader.__module__ + '.' + loader.__class__.__name__
+                else:
+                    loader_name = loader.__module__ + '.' + loader.__name__
                 self.loader_debug_info.append({
-                    'loader': loader.__module__ + '.' + loader.__name__,
+                    'loader': loader_name,
                     'templates': template_list,
                 })
         if settings.TEMPLATE_DEBUG and hasattr(self.exc_value, 'source'):

docs/howto/custom-template-tags.txt

@@ -463,6 +463,85 @@ new ``Context`` in this example, the results would have *always* been
 automatically escaped, which may not be the desired behavior if the template
 tag is used inside a ``{% autoescape off %}`` block.
 
+.. _template_tag_thread_safety:
+
+Thread-safety considerations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.2
+
+Once a node is parsed, its ``render`` method may be called any number of times.
+Since Django is sometimes run in multi-threaded environments, a single node may
+be simultaneously rendering with different contexts in response to two separate
+requests. Therefore, it's important to make sure your template tags are thread
+safe.
+
+To make sure your template tags are thread safe, you should never store state
+information on the node itself. For example, Django provides a builtin ``cycle``
+template tag that cycles among a list of given strings each time it's rendered::
+
+    {% for o in some_list %}
+        <tr class="{% cycle 'row1' 'row2' %}>
+            ...
+        </tr>
+    {% endfor %}
+
+A naive implementation of ``CycleNode`` might look something like this::
+
+    class CycleNode(Node):
+        def __init__(self, cyclevars):
+            self.cycle_iter = itertools.cycle(cyclevars)
+        def render(self, context):
+            return self.cycle_iter.next()
+
+But, suppose we have two templates rendering the template snippet from above at
+the same time:
+
+    1. Thread 1 performs its first loop iteration, ``CycleNode.render()``
+       returns 'row1'
+    2. Thread 2 performs its first loop iteration, ``CycleNode.render()``
+       returns 'row2'
+    3. Thread 1 performs its second loop iteration, ``CycleNode.render()``
+       returns 'row1'
+    4. Thread 2 performs its second loop iteration, ``CycleNode.render()``
+       returns 'row2'
+
+The CycleNode is iterating, but it's iterating globally. As far as Thread 1
+and Thread 2 are concerned, it's always returning the same value. This is
+obviously not what we want!
+
+To address this problem, Django provides a ``render_context`` that's associated
+with the ``context`` of the template that is currently being rendered. The
+``render_context`` behaves like a Python dictionary, and should be used to store
+``Node`` state between invocations of the ``render`` method.
+
+Let's refactor our ``CycleNode`` implementation to use the ``render_context``::
+
+    class CycleNode(Node):
+        def __init__(self, cyclevars):
+            self.cyclevars = cyclevars
+        def render(self, context):
+            if self not in context.render_context:
+                context.render_context[self] = itertools.cycle(self.cyclevars)
+            cycle_iter = context.render_context[self]
+            return cycle_iter.next()
+
+Note that it's perfectly safe to store global information that will not change
+throughout the life of the ``Node`` as an attribute. In the case of
+``CycleNode``, the ``cyclevars`` argument doesn't change after the ``Node`` is
+instantiated, so we don't need to put it in the ``render_context``. But state
+information that is specific to the template that is currently being rendered,
+like the current iteration of the ``CycleNode``, should be stored in the
+``render_context``.
+
+.. note::
+    Notice how we used ``self`` to scope the ``CycleNode`` specific information
+    within the ``render_context``. There may be multiple ``CycleNodes`` in a
+    given template, so we need to be careful not to clobber another node's state
+    information. The easiest way to do this is to always use ``self`` as the key
+    into ``render_context``. If you're keeping track of several state variables,
+    make ``render_context[self]`` a dictionary.
+
 Registering the tag
 ~~~~~~~~~~~~~~~~~~~
 

docs/internals/deprecation.txt

@@ -48,14 +48,20 @@ their deprecation, as per the :ref:`Django deprecation policy
           manager in the ``User`` model (``user.message_set``), and the
           associated methods (``user.message_set.create()`` and
           ``user.get_and_delete_messages()``), which have
-          been deprecated since the 1.2 release, will be removed.  The 
+          been deprecated since the 1.2 release, will be removed.  The
-          :ref:`messages framework <ref-contrib-messages>` should be used 
+          :ref:`messages framework <ref-contrib-messages>` should be used
           instead.
 
         * Authentication backends need to support the ``obj`` parameter for
           permission checking. The ``supports_object_permissions`` variable
           is not checked any longer and can be removed.
 
+        * The ability to specify a callable template loader rather than a
+          ``Loader`` class will be removed, as will the ``load_template_source``
+          functions that are included with the built in template loaders for
+          backwards compatibility. These have been deprecated since the 1.2
+          release.
+
     * 2.0
         * ``django.views.defaults.shortcut()``. This function has been moved
           to ``django.contrib.contenttypes.views.shortcut()`` as part of the

docs/ref/contrib/sitemaps.txt

@@ -36,7 +36,7 @@ To install the sitemap app, follow these steps:
     1. Add ``'django.contrib.sitemaps'`` to your :setting:`INSTALLED_APPS`
        setting.
        
-    2. Make sure ``'django.template.loaders.app_directories.load_template_source'``
+    2. Make sure ``'django.template.loaders.app_directories.Loader'``
        is in your :setting:`TEMPLATE_LOADERS` setting. It's in there by default,
        so you'll only need to change this if you've changed that setting.
 
@@ -45,7 +45,7 @@ To install the sitemap app, follow these steps:
 
 (Note: The sitemap application doesn't install any database tables. The only
 reason it needs to go into :setting:`INSTALLED_APPS` is so that the
-:func:`~django.template.loaders.app_directories.load_template_source` template
+:func:`~django.template.loaders.app_directories.Loader` template
 loader can find the default templates.)
 
 Initialization

docs/ref/django-admin.txt

@@ -240,6 +240,16 @@ model names.
 The :djadminopt:`--database` option can be used to specify the database
 onto which the data will be loaded.
 
+.. django-admin-option:: --natural
+
+.. versionadded:: 1.2
+
+Use :ref:`natural keys <topics-serialization-natural-keys>` to represent
+any foreign key and many-to-many relationship with a model that provides
+a natural key definition. If you are dumping ``contrib.auth`` ``Permission``
+objects or ``contrib.contenttypes`` ``ContentType`` objects, you should
+probably be using this flag.
+
 flush
 -----
 
@@ -807,7 +817,7 @@ information.
 
 .. versionadded:: 1.2
 
-Use the ``--failfast`` option to stop running tests and report the failure 
+Use the ``--failfast`` option to stop running tests and report the failure
 immediately after a test fails.
 
 testserver <fixture fixture ...>

docs/ref/settings.txt

@@ -903,7 +903,7 @@ MESSAGE_LEVEL
 
 Default: `messages.INFO`
 
-Sets the minimum message level that will be recorded by the messages 
+Sets the minimum message level that will be recorded by the messages
 framework. See the :ref:`messages documentation <ref-contrib-messages>` for
 more details.
 
@@ -1246,11 +1246,14 @@ TEMPLATE_LOADERS
 
 Default::
 
-     ('django.template.loaders.filesystem.load_template_source',
+     ('django.template.loaders.filesystem.Loader',
-      'django.template.loaders.app_directories.load_template_source')
+      'django.template.loaders.app_directories.Loader')
 
-A tuple of callables (as strings) that know how to import templates from
+A tuple of template loader classes, specified as strings. Each ``Loader`` class
-various sources. See :ref:`ref-templates-api`.
+knows how to import templates from a particular sources. Optionally, a tuple can be
+used instead of a string. The first item in the tuple should be the ``Loader``'s
+module, subsequent items are passed to the ``Loader`` during initialization. See
+:ref:`ref-templates-api`.
 
 .. setting:: TEMPLATE_STRING_IF_INVALID
 

docs/ref/templates/api.txt

@@ -322,7 +322,7 @@ and return a dictionary of items to be merged into the context. By default,
    cannot be turned off by the :setting:`TEMPLATE_CONTEXT_PROCESSORS` setting.
 
 .. versionadded:: 1.2
-   The ``'messages'`` context processor was added.  For more information, see 
+   The ``'messages'`` context processor was added.  For more information, see
    the :ref:`messages documentation <ref-contrib-messages>`.
 
 Each processor is applied in order. That means, if one processor adds a
@@ -379,7 +379,7 @@ If :setting:`TEMPLATE_CONTEXT_PROCESSORS` contains this processor, every
 
 .. versionchanged:: 1.2
    Prior to version 1.2, the ``messages`` variable was a lazy accessor for
-   ``user.get_and_delete_messages()``. It has been changed to include any 
+   ``user.get_and_delete_messages()``. It has been changed to include any
    messages added via the :ref:`messages framework <ref-contrib-messages`.
 
 django.core.context_processors.debug
@@ -448,7 +448,7 @@ If :setting:`TEMPLATE_CONTEXT_PROCESSORS` contains this processor, every
    context processor.  For backwards compatibility the ``'auth'`` context
    processor will continue to supply the ``messages`` variable until Django
    1.4.  If you use the ``messages`` variable, your project will work with
-   either (or both) context processors, but it is recommended to add 
+   either (or both) context processors, but it is recommended to add
    ``django.contrib.messages.context_processors.messages`` so your project
    will be prepared for the future upgrade.
 
@@ -571,11 +571,11 @@ by editing your :setting:`TEMPLATE_LOADERS` setting. :setting:`TEMPLATE_LOADERS`
 should be a tuple of strings, where each string represents a template loader.
 Here are the template loaders that come with Django:
 
-``django.template.loaders.filesystem.load_template_source``
+``django.template.loaders.filesystem.Loader``
     Loads templates from the filesystem, according to :setting:`TEMPLATE_DIRS`.
     This loader is enabled by default.
 
-``django.template.loaders.app_directories.load_template_source``
+``django.template.loaders.app_directories.Loader``
     Loads templates from Django apps on the filesystem. For each app in
     :setting:`INSTALLED_APPS`, the loader looks for a ``templates``
     subdirectory. If the directory exists, Django looks for templates in there.
@@ -599,12 +599,43 @@ Here are the template loaders that come with Django:
 
     This loader is enabled by default.
 
-``django.template.loaders.eggs.load_template_source``
+``django.template.loaders.eggs.Loader``
     Just like ``app_directories`` above, but it loads templates from Python
     eggs rather than from the filesystem.
 
     This loader is disabled by default.
 
+``django.template.loaders.cached.Loader``
+    By default, the templating system will read and compile your templates every
+    time they need to be rendered. While the Django templating system is quite
+    fast, the overhead from reading and compiling templates can add up.
+
+    The cached template loader is a class-based loader that you configure with
+    a list of other loaders that it should wrap. The wrapped loaders are used to
+    locate unknown templates when they are first encountered. The cached loader
+    then stores the compiled ``Template`` in memory. The cached ``Template``
+    instance is returned for subsequent requests to load the same template.
+
+    For example, to enable template caching with the ``filesystem`` and
+    ``app_directories`` template loaders you might use the following settings::
+
+        TEMPLATE_LOADERS = (
+            ('django.template.loaders.cached.Loader', (
+                'django.template.loaders.filesystem.Loader',
+                'django.template.loaders.app_directories.Loader',
+            )),
+        )
+
+    .. note::
+        All of the built-in Django template tags are safe to use with the cached
+        loader, but if you're using custom template tags that come from third
+        party packages, or that you wrote yourself, you should ensure that the
+        ``Node`` implementation for each tag is thread-safe. For more
+        information, see
+        :ref:`template tag thread safety considerations<template_tag_thread_safety>`.
+
+    This loader is disabled by default.
+
 Django uses the template loaders in order according to the
 :setting:`TEMPLATE_LOADERS` setting. It uses each loader until a loader finds a
 match.
@@ -667,3 +698,68 @@ settings you wish to specify. You might want to consider setting at least
 and :setting:`TEMPLATE_DEBUG`. All available settings are described in the
 :ref:`settings documentation <ref-settings>`, and any setting starting with
 ``TEMPLATE_`` is of obvious interest.
+
+.. _topic-template-alternate-language:
+
+Using an alternative template language
+======================================
+
+.. versionadded 1.2
+
+The Django ``Template`` and ``Loader`` classes implement a simple API for
+loading and rendering templates. By providing some simple wrapper classes that
+implement this API we can use third party template systems like `Jinja2
+<http://jinja.pocoo.org/2/>`_ or `Cheetah <http://www.cheetahtemplate.org/>`_. This
+allows us to use third-party template libraries without giving up useful Django
+features like the Django ``Context`` object and handy shortcuts like
+``render_to_response()``.
+
+The core component of the Django templating system is the ``Template`` class.
+This class has a very simple interface: it has a constructor that takes a single
+positional argument specifying the template string, and a ``render()`` method
+that takes a ``django.template.context.Context`` object and returns a string
+containing the rendered response.
+
+Suppose we're using a template language that defines a ``Template`` object with
+a ``render()`` method that takes a dictionary rather than a ``Context`` object.
+We can write a simple wrapper that implements the Django ``Template`` interface::
+
+    import some_template_language
+    class Template(some_template_language.Template):
+        def render(self, context):
+            # flatten the Django Context into a single dictionary.
+            context_dict = {}
+            for d in context.dicts:
+                context_dict.update(d)
+            return super(Template, self).render(context_dict)
+
+That's all that's required to make our fictional ``Template`` class compatible
+with the Django loading and rendering system!
+
+The next step is to write a ``Loader`` class that returns instances of our custom
+template class instead of the default ``django.template.Template``. Custom ``Loader``
+classes should inherit from ``django.template.loader.BaseLoader`` and override
+the ``load_template_source()`` method, which takes a ``template_name`` argument,
+loads the template from disk (or elsewhere), and returns a tuple:
+``(template_string, template_origin)``.
+
+The ``load_template()`` method of the ``Loader`` class retrieves the template
+string by calling ``load_template_source()``, instantiates a ``Template`` from
+the template source, and returns a tuple: ``(template, template_origin)``. Since
+this is the method that actually instantiates the ``Template``, we'll need to
+override it to use our custom template class instead. We can inherit from the
+builtin ``django.template.loaders.app_directories.Loader`` to take advantage of
+the ``load_template_source()`` method implemented there::
+
+    from django.template.loaders import app_directories
+    class Loader(app_directories.Loader):
+        is_usable = True
+
+        def load_template(self, template_name, template_dirs=None):
+            source, origin = self.load_template_source(template_name, template_dirs)
+            template = Template(source)
+            return template, origin
+
+Finally, we need to modify our project settings, telling Django to use our custom
+loader. Now we can write all of our templates in our alternative template
+language while continuing to use the rest of the Django templating system.

docs/releases/1.2.txt

@@ -218,6 +218,20 @@ database specific conversions, then you will need to provide an
 implementation ``get_db_prep_*`` that uses the ``connection``
 argument to resolve database-specific values.
 
+Stateful template tags
+----------------------
+
+Template tags that store rendering state on the node itself may experience
+problems if they are used with the new :ref:`cached
+template loader<template-loaders>`.
+
+All of the built-in Django template tags are safe to use with the cached
+loader, but if you're using custom template tags that come from third
+party packages, or that you wrote yourself, you should ensure that the
+``Node`` implementation for each tag is thread-safe. For more
+information, see
+:ref:`template tag thread safety considerations<template_tag_thread_safety>`.
+
 .. _deprecated-features-1.2:
 
 Features deprecated in 1.2
@@ -271,11 +285,11 @@ additional arguments, those arguments can be passed to the
 :meth:`~django.core.mail.get_connection()` call::
 
     connection = get_connection('django.core.mail.backends.smtp', hostname='localhost', port=1234)
-    
+
 User Messages API
 -----------------
 
-The API for storing messages in the user ``Message`` model (via 
+The API for storing messages in the user ``Message`` model (via
 ``user.message_set.create``) is now deprecated and will be removed in Django
 1.4 according to the standard :ref:`release process <internals-release-process>`.
 
@@ -288,20 +302,20 @@ with the following::
     from django.contrib import messages
     messages.add_message(request, messages.INFO, 'a message')
 
-Additionally, if you make use of the method, you need to replace the 
+Additionally, if you make use of the method, you need to replace the
 following::
 
     for message in user.get_and_delete_messages():
         ...
-    
+
 with::
 
     from django.contrib import messages
     for message in messages.get_messages(request):
         ...
-    
+
-For more information, see the full 
+For more information, see the full
-:ref:`messages documentation <ref-contrib-messages>`. You should begin to 
+:ref:`messages documentation <ref-contrib-messages>`. You should begin to
 update your code to use the new API immediately.
 
 What's new in Django 1.2
@@ -389,3 +403,28 @@ Also, filters may now be used in the ``if`` expression. For example:
           class="highlight"
         {% endif %}
       >{{ message }}</div>
+
+Template caching
+----------------
+
+In previous versions of Django, every time you rendered a template it
+would be reloaded from disk. In Django 1.2, you can use a :ref:`cached
+template loader <template-loaders>` to load templates once, then use a
+cached the result for every subsequent render. This can lead to a
+significant performance improvement if your templates are broken into
+lots of smaller subtemplates (using the ``{% extends %}`` or ``{%
+include %}`` tags).
+
+As a side effect, it is now much easier to support non-Django template
+languages. For more details, see the :ref:`notes on supporting
+non-Django template languages<topic-template-alternate-language>`.
+
+Natural keys in fixtures
+------------------------
+
+Fixtures can refer to remote objects using
+:ref:`topics-serialization-natural-keys`. This lookup scheme is an
+alternative to the normal primary-key based object references in a
+fixture, improving readability, and resolving problems referring to
+objects whose primary key value may not be predictable or known.
+

docs/topics/serialization.txt

@@ -154,10 +154,10 @@ to install third-party Python modules:
 .. _PyYAML: http://www.pyyaml.org/
 
 Notes for specific serialization formats
-----------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 json
-~~~~
+^^^^
 
 If you're using UTF-8 (or any other non-ASCII encoding) data with the JSON
 serializer, you must pass ``ensure_ascii=False`` as a parameter to the
@@ -191,3 +191,191 @@ them. Something like this will work::
 
 .. _special encoder: http://svn.red-bean.com/bob/simplejson/tags/simplejson-1.7/docs/index.html
 
+.. _topics-serialization-natural-keys:
+
+Natural keys
+------------
+
+The default serialization strategy for foreign keys and many-to-many
+relations is to serialize the value of the primary key(s) of the
+objects in the relation. This strategy works well for most types of
+object, but it can cause difficulty in some circumstances.
+
+Consider the case of a list of objects that have foreign key on
+:class:`ContentType`. If you're going to serialize an object that
+refers to a content type, you need to have a way to refer to that
+content type. Content Types are automatically created by Django as
+part of the database synchronization process, so you don't need to
+include content types in a fixture or other serialized data. As a
+result, the primary key of any given content type isn't easy to
+predict - it will depend on how and when :djadmin:`syncdb` was
+executed to create the content types.
+
+There is also the matter of convenience. An integer id isn't always
+the most convenient way to refer to an object; sometimes, a
+more natural reference would be helpful.
+
+Deserialization of natural keys
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is for these reasons that Django provides `natural keys`. A natural
+key is a tuple of values that can be used to uniquely identify an
+object instance without using the primary key value.
+
+Consider the following two models::
+
+    from django.db import models
+
+    class Person(models.Model):
+        first_name = models.CharField(max_length=100)
+        last_name = models.CharField(max_length=100)
+
+        birthdate = models.DateField()
+
+    class Book(models.Model):
+        name = models.CharField(max_length=100)
+        author = models.ForeignKey(Person)
+
+Ordinarily, serialized data for ``Book`` would use an integer to refer to
+the author. For example, in JSON, a Book might be serialized as::
+
+    ...
+    {
+        "pk": 1,
+        "model": "store.book",
+        "fields": {
+            "name": "Mostly Harmless",
+            "author": 42
+        }
+    }
+    ...
+
+This isn't a particularly natural way to refer to an author. It
+requires that you know the primary key value for the author; it also
+requires that this primary key value is stable and predictable.
+
+However, if we add natural key handling to Person, the fixture becomes
+much more humane. To add natural key handling, you define a default
+Manager for Person with a ``get_by_natural_key()`` method. In the case
+of a Person, a good natural key might be the pair of first and last
+name::
+
+    from django.db import models
+
+    class PersonManager(models.Manager):
+        def get_by_natural_key(self, first_name, last_name):
+            return self.filter(first_name=first_name, last_name=last_name)
+
+    class Person(models.Model):
+        objects = PersonManager()
+
+        first_name = models.CharField(max_length=100)
+        last_name = models.CharField(max_length=100)
+
+        birthdate = models.DateField()
+
+Now books can use that natural key to refer to ``Person`` objects::
+
+    ...
+    {
+        "pk": 1,
+        "model": "store.book",
+        "fields": {
+            "name": "Mostly Harmless",
+            "author": ["Douglas", "Adams"]
+        }
+    }
+    ...
+
+When you try to load this serialized data, Django will use the
+``get_by_natural_key()`` method to resolve ``["Douglas", "Adams"]``
+into the primary key of an actual ``Person`` object.
+
+Serialization of natural keys
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+So how do you get Django to emit a natural key when serializing an object?
+Firstly, you need to add another method -- this time to the model itself::
+
+    class Person(models.Model):
+        objects = PersonManager()
+
+        first_name = models.CharField(max_length=100)
+        last_name = models.CharField(max_length=100)
+
+        birthdate = models.DateField()
+
+        def natural_key(self):
+            return (self.first_name, self.last_name)
+
+Then, when you call ``serializers.serialize()``, you provide a
+``use_natural_keys=True`` argument::
+
+    >>> serializers.serialize([book1, book2], format='json', indent=2, use_natural_keys=True)
+
+When ``use_natural_keys=True`` is specified, Django will use the
+``natural_key()`` method to serialize any reference to objects of the
+type that defines the method.
+
+If you are using :djadmin:`dumpdata` to generate serialized data, you
+use the `--natural` command line flag to generate natural keys.
+
+.. note::
+
+    You don't need to define both ``natural_key()`` and
+    ``get_by_natural_key()``. If you don't want Django to output
+    natural keys during serialization, but you want to retain the
+    ability to load natural keys, then you can opt to not implement
+    the ``natural_key()`` method.
+
+    Conversely, if (for some strange reason) you want Django to output
+    natural keys during serialization, but *not* be able to load those
+    key values, just don't define the ``get_by_natural_key()`` method.
+
+Dependencies during serialization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since natural keys rely on database lookups to resolve references, it
+is important that data exists before it is referenced. You can't make
+a `forward reference` with natural keys - the data you are referencing
+must exist before you include a natural key reference to that data.
+
+To accommodate this limitation, calls to :djadmin:`dumpdata` that use
+the :djadminopt:`--natural` optionwill serialize any model with a
+``natural_key()`` method before it serializes normal key objects.
+
+However, this may not always be enough. If your natural key refers to
+another object (by using a foreign key or natural key to another object
+as part of a natural key), then you need to be able to ensure that
+the objects on which a natural key depends occur in the serialized data
+before the natural key requires them.
+
+To control this ordering, you can define dependencies on your
+``natural_key()`` methods. You do this by setting a ``dependencies``
+attribute on the ``natural_key()`` method itself.
+
+For example, consider the ``Permission`` model in ``contrib.auth``.
+The following is a simplified version of the ``Permission`` model::
+
+    class Permission(models.Model):
+        name = models.CharField(max_length=50)
+        content_type = models.ForeignKey(ContentType)
+        codename = models.CharField(max_length=100)
+        # ...
+        def natural_key(self):
+            return (self.codename,) + self.content_type.natural_key()
+
+The natural key for a ``Permission`` is a combination of the codename for the
+``Permission``, and the ``ContentType`` to which the ``Permission`` applies. This means
+that ``ContentType`` must be serialized before ``Permission``. To define this
+dependency, we add one extra line::
+
+    class Permission(models.Model):
+        # ...
+        def natural_key(self):
+            return (self.codename,) + self.content_type.natural_key()
+        natural_key.dependencies = ['contenttypes.contenttype']
+
+This definition ensures that ``ContentType`` models are serialized before
+``Permission`` models. In turn, any object referencing ``Permission`` will
+be serialized after both ``ContentType`` and ``Permission``.

tests/modeltests/fixtures/fixtures/fixture6.json

@@ -0,0 +1,41 @@
+[
+    {
+        "pk": "1",
+        "model": "fixtures.tag",
+        "fields": {
+            "name": "copyright",
+            "tagged_type": ["fixtures", "article"],
+            "tagged_id": "3"
+        }
+    },
+    {
+        "pk": "2",
+        "model": "fixtures.tag",
+        "fields": {
+            "name": "law",
+            "tagged_type": ["fixtures", "article"],
+            "tagged_id": "3"
+        }
+    },
+    {
+    	"pk": "1",
+    	"model": "fixtures.person",
+    	"fields": {
+    	    "name": "Django Reinhardt"
+    	}
+    },
+    {
+    	"pk": "2",
+    	"model": "fixtures.person",
+    	"fields": {
+    	    "name": "Stephane Grappelli"
+    	}
+    },
+        {
+    	"pk": "3",
+    	"model": "fixtures.person",
+    	"fields": {
+    	    "name": "Prince"
+    	}
+    }
+]

tests/modeltests/fixtures/fixtures/fixture7.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+    <object pk="2" model="fixtures.tag">
+        <field type="CharField" name="name">legal</field>
+        <field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel">
+            <natural>fixtures</natural>
+            <natural>article</natural>
+        </field>
+        <field type="PositiveIntegerField" name="tagged_id">3</field>
+    </object>
+    <object pk="3" model="fixtures.tag">
+        <field type="CharField" name="name">django</field>
+        <field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel">
+            <natural>fixtures</natural>
+            <natural>article</natural>
+        </field>
+        <field type="PositiveIntegerField" name="tagged_id">4</field>
+    </object>
+    <object pk="4" model="fixtures.tag">
+        <field type="CharField" name="name">world domination</field>
+        <field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel">
+            <natural>fixtures</natural>
+            <natural>article</natural>
+        </field>
+        <field type="PositiveIntegerField" name="tagged_id">4</field>
+    </object>
+</django-objects>

tests/modeltests/fixtures/fixtures/fixture8.json

@@ -0,0 +1,32 @@
+[
+    {
+    	"pk": "1",
+    	"model": "fixtures.visa",
+    	"fields": {
+    	    "person": ["Django Reinhardt"],
+    	    "permissions": [
+    	        ["add_user", "auth", "user"],
+    	        ["change_user", "auth", "user"],
+    	        ["delete_user", "auth", "user"]
+    	    ]
+    	}
+    },
+    {
+    	"pk": "2",
+    	"model": "fixtures.visa",
+    	"fields": {
+    	    "person": ["Stephane Grappelli"],
+    	    "permissions": [
+    	        ["add_user", "auth", "user"]
+    	    ]
+    	}
+    },
+        {
+    	"pk": "3",
+    	"model": "fixtures.visa",
+    	"fields": {
+    	    "person": ["Prince"],
+    	    "permissions": []
+    	}
+    }
+]

tests/modeltests/fixtures/fixtures/fixture9.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+    <object pk="2" model="fixtures.visa">
+        <field type="CharField" name="person">
+            <natural>Stephane Grappelli</natural>
+        </field>
+        <field to="auth.permission" name="permissions" rel="ManyToManyRel">
+            <object>
+                <natural>add_user</natural>
+                <natural>auth</natural>
+                <natural>user</natural>
+            </object>
+            <object>
+                <natural>delete_user</natural>
+                <natural>auth</natural>
+                <natural>user</natural>
+            </object>
+        </field>
+    </object>
+    <object pk="3" model="fixtures.person">
+        <field type="CharField" name="name">
+            <natural>Artist formerly known as &quot;Prince&quot;</natural>
+        </field>
+    </object>
+    <object pk="3" model="fixtures.visa">
+        <field type="CharField" name="person">
+            <natural>Artist formerly known as &quot;Prince&quot;</natural>
+        </field>
+        <field to="auth.permission" name="permissions" rel="ManyToManyRel">
+            <object>
+                <natural>change_user</natural>
+                <natural>auth</natural>
+                <natural>user</natural>
+            </object>
+        </field>
+    </object>
+    <object pk="1" model="fixtures.book">
+        <field type="CharField" name="name">Music for all ages</field>
+        <field to="fixtures.person" name="authors" rel="ManyToManyRel">
+            <object>
+                <natural>Django Reinhardt</natural>
+            </object>
+            <object>
+                <natural>Artist formerly known as &quot;Prince&quot;</natural>
+            </object>
+        </field>
+    </object>
+</django-objects>

tests/modeltests/fixtures/models.py

@@ -8,9 +8,13 @@ in the application directory, on in one of the directories named in the
 ``FIXTURE_DIRS`` setting.
 """
 
+from django.contrib.auth.models import Permission
+from django.contrib.contenttypes import generic
+from django.contrib.contenttypes.models import ContentType
 from django.db import models, DEFAULT_DB_ALIAS
 from django.conf import settings
 
+
 class Category(models.Model):
     title = models.CharField(max_length=100)
     description = models.TextField()
@@ -31,6 +35,62 @@ class Article(models.Model):
     class Meta:
         ordering = ('-pub_date', 'headline')
 
+class Blog(models.Model):
+    name = models.CharField(max_length=100)
+    featured = models.ForeignKey(Article, related_name='fixtures_featured_set')
+    articles = models.ManyToManyField(Article, blank=True,
+                                      related_name='fixtures_articles_set')
+
+    def __unicode__(self):
+        return self.name
+
+
+class Tag(models.Model):
+    name = models.CharField(max_length=100)
+    tagged_type = models.ForeignKey(ContentType, related_name="fixtures_tag_set")
+    tagged_id = models.PositiveIntegerField(default=0)
+    tagged = generic.GenericForeignKey(ct_field='tagged_type',
+                                       fk_field='tagged_id')
+
+    def __unicode__(self):
+        return '<%s: %s> tagged "%s"' % (self.tagged.__class__.__name__,
+                                         self.tagged, self.name)
+
+class PersonManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class Person(models.Model):
+    objects = PersonManager()
+    name = models.CharField(max_length=100)
+    def __unicode__(self):
+        return self.name
+
+    class Meta:
+        ordering = ('name',)
+
+    def natural_key(self):
+        return (self.name,)
+
+class Visa(models.Model):
+    person = models.ForeignKey(Person)
+    permissions = models.ManyToManyField(Permission, blank=True)
+
+    def __unicode__(self):
+        return '%s %s' % (self.person.name,
+                          ', '.join(p.name for p in self.permissions.all()))
+
+class Book(models.Model):
+    name = models.CharField(max_length=100)
+    authors = models.ManyToManyField(Person)
+
+    def __unicode__(self):
+        return '%s by %s' % (self.name,
+                          ' and '.join(a.name for a in self.authors.all()))
+
+    class Meta:
+        ordering = ('name',)
+
 __test__ = {'API_TESTS': """
 >>> from django.core import management
 >>> from django.db.models import get_app
@@ -90,12 +150,53 @@ __test__ = {'API_TESTS': """
 >>> Article.objects.all()
 [<Article: XML identified as leading cause of cancer>, <Article: Django conquers world!>, <Article: Copyright is fine the way it is>, <Article: Poker on TV is great!>, <Article: Python program becomes self aware>]
 
+# Load fixture 6, JSON file with dynamic ContentType fields. Testing ManyToOne.
+>>> management.call_command('loaddata', 'fixture6.json', verbosity=0)
+>>> Tag.objects.all()
+[<Tag: <Article: Copyright is fine the way it is> tagged "copyright">, <Tag: <Article: Copyright is fine the way it is> tagged "law">]
+
+# Load fixture 7, XML file with dynamic ContentType fields. Testing ManyToOne.
+>>> management.call_command('loaddata', 'fixture7.xml', verbosity=0)
+>>> Tag.objects.all()
+[<Tag: <Article: Copyright is fine the way it is> tagged "copyright">, <Tag: <Article: Copyright is fine the way it is> tagged "legal">, <Tag: <Article: Django conquers world!> tagged "django">, <Tag: <Article: Django conquers world!> tagged "world domination">]
+
+# Load fixture 8, JSON file with dynamic Permission fields. Testing ManyToMany.
+>>> management.call_command('loaddata', 'fixture8.json', verbosity=0)
+>>> Visa.objects.all()
+[<Visa: Django Reinhardt Can add user, Can change user, Can delete user>, <Visa: Stephane Grappelli Can add user>, <Visa: Prince >]
+
+# Load fixture 9, XML file with dynamic Permission fields. Testing ManyToMany.
+>>> management.call_command('loaddata', 'fixture9.xml', verbosity=0)
+>>> Visa.objects.all()
+[<Visa: Django Reinhardt Can add user, Can change user, Can delete user>, <Visa: Stephane Grappelli Can add user, Can delete user>, <Visa: Artist formerly known as "Prince" Can change user>]
+
+>>> Book.objects.all()
+[<Book: Music for all ages by Artist formerly known as "Prince" and Django Reinhardt>]
+
 # Load a fixture that doesn't exist
 >>> management.call_command('loaddata', 'unknown.json', verbosity=0)
 
 # object list is unaffected
 >>> Article.objects.all()
 [<Article: XML identified as leading cause of cancer>, <Article: Django conquers world!>, <Article: Copyright is fine the way it is>, <Article: Poker on TV is great!>, <Article: Python program becomes self aware>]
+
+# By default, you get raw keys on dumpdata
+>>> management.call_command('dumpdata', 'fixtures.book', format='json')
+[{"pk": 1, "model": "fixtures.book", "fields": {"name": "Music for all ages", "authors": [3, 1]}}]
+
+# But you can get natural keys if you ask for them and they are available
+>>> management.call_command('dumpdata', 'fixtures.book', format='json', use_natural_keys=True)
+[{"pk": 1, "model": "fixtures.book", "fields": {"name": "Music for all ages", "authors": [["Artist formerly known as \\"Prince\\""], ["Django Reinhardt"]]}}]
+
+# Dump the current contents of the database as a JSON fixture
+>>> management.call_command('dumpdata', 'fixtures', format='json', use_natural_keys=True)
+[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": "News Stories"}}, {"pk": 5, "model": "fixtures.article", "fields": {"headline": "XML identified as leading cause of cancer", "pub_date": "2006-06-16 16:00:00"}}, {"pk": 4, "model": "fixtures.article", "fields": {"headline": "Django conquers world!", "pub_date": "2006-06-16 15:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": {"headline": "Copyright is fine the way it is", "pub_date": "2006-06-16 14:00:00"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker on TV is great!", "pub_date": "2006-06-16 11:00:00"}}, {"pk": 1, "model": "fixtures.article", "fields": {"headline": "Python program becomes self aware", "pub_date": "2006-06-16 11:00:00"}}, {"pk": 1, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "copyright", "tagged_id": 3}}, {"pk": 2, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "legal", "tagged_id": 3}}, {"pk": 3, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "django", "tagged_id": 4}}, {"pk": 4, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "world domination", "tagged_id": 4}}, {"pk": 3, "model": "fixtures.person", "fields": {"name": "Artist formerly known as \\"Prince\\""}}, {"pk": 1, "model": "fixtures.person", "fields": {"name": "Django Reinhardt"}}, {"pk": 2, "model": "fixtures.person", "fields": {"name": "Stephane Grappelli"}}, {"pk": 1, "model": "fixtures.visa", "fields": {"person": ["Django Reinhardt"], "permissions": [["add_user", "auth", "user"], ["change_user", "auth", "user"], ["delete_user", "auth", "user"]]}}, {"pk": 2, "model": "fixtures.visa", "fields": {"person": ["Stephane Grappelli"], "permissions": [["add_user", "auth", "user"], ["delete_user", "auth", "user"]]}}, {"pk": 3, "model": "fixtures.visa", "fields": {"person": ["Artist formerly known as \\"Prince\\""], "permissions": [["change_user", "auth", "user"]]}}, {"pk": 1, "model": "fixtures.book", "fields": {"name": "Music for all ages", "authors": [["Artist formerly known as \\"Prince\\""], ["Django Reinhardt"]]}}]
+
+# Dump the current contents of the database as an XML fixture
+>>> management.call_command('dumpdata', 'fixtures', format='xml', use_natural_keys=True)
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0"><object pk="1" model="fixtures.category"><field type="CharField" name="title">News Stories</field><field type="TextField" name="description">Latest news stories</field></object><object pk="5" model="fixtures.article"><field type="CharField" name="headline">XML identified as leading cause of cancer</field><field type="DateTimeField" name="pub_date">2006-06-16 16:00:00</field></object><object pk="4" model="fixtures.article"><field type="CharField" name="headline">Django conquers world!</field><field type="DateTimeField" name="pub_date">2006-06-16 15:00:00</field></object><object pk="3" model="fixtures.article"><field type="CharField" name="headline">Copyright is fine the way it is</field><field type="DateTimeField" name="pub_date">2006-06-16 14:00:00</field></object><object pk="2" model="fixtures.article"><field type="CharField" name="headline">Poker on TV is great!</field><field type="DateTimeField" name="pub_date">2006-06-16 11:00:00</field></object><object pk="1" model="fixtures.article"><field type="CharField" name="headline">Python program becomes self aware</field><field type="DateTimeField" name="pub_date">2006-06-16 11:00:00</field></object><object pk="1" model="fixtures.tag"><field type="CharField" name="name">copyright</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><natural>fixtures</natural><natural>article</natural></field><field type="PositiveIntegerField" name="tagged_id">3</field></object><object pk="2" model="fixtures.tag"><field type="CharField" name="name">legal</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><natural>fixtures</natural><natural>article</natural></field><field type="PositiveIntegerField" name="tagged_id">3</field></object><object pk="3" model="fixtures.tag"><field type="CharField" name="name">django</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><natural>fixtures</natural><natural>article</natural></field><field type="PositiveIntegerField" name="tagged_id">4</field></object><object pk="4" model="fixtures.tag"><field type="CharField" name="name">world domination</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><natural>fixtures</natural><natural>article</natural></field><field type="PositiveIntegerField" name="tagged_id">4</field></object><object pk="3" model="fixtures.person"><field type="CharField" name="name">Artist formerly known as "Prince"</field></object><object pk="1" model="fixtures.person"><field type="CharField" name="name">Django Reinhardt</field></object><object pk="2" model="fixtures.person"><field type="CharField" name="name">Stephane Grappelli</field></object><object pk="1" model="fixtures.visa"><field to="fixtures.person" name="person" rel="ManyToOneRel"><natural>Django Reinhardt</natural></field><field to="auth.permission" name="permissions" rel="ManyToManyRel"><object><natural>add_user</natural><natural>auth</natural><natural>user</natural></object><object><natural>change_user</natural><natural>auth</natural><natural>user</natural></object><object><natural>delete_user</natural><natural>auth</natural><natural>user</natural></object></field></object><object pk="2" model="fixtures.visa"><field to="fixtures.person" name="person" rel="ManyToOneRel"><natural>Stephane Grappelli</natural></field><field to="auth.permission" name="permissions" rel="ManyToManyRel"><object><natural>add_user</natural><natural>auth</natural><natural>user</natural></object><object><natural>delete_user</natural><natural>auth</natural><natural>user</natural></object></field></object><object pk="3" model="fixtures.visa"><field to="fixtures.person" name="person" rel="ManyToOneRel"><natural>Artist formerly known as "Prince"</natural></field><field to="auth.permission" name="permissions" rel="ManyToManyRel"><object><natural>change_user</natural><natural>auth</natural><natural>user</natural></object></field></object><object pk="1" model="fixtures.book"><field type="CharField" name="name">Music for all ages</field><field to="fixtures.person" name="authors" rel="ManyToManyRel"><object><natural>Artist formerly known as "Prince"</natural></object><object><natural>Django Reinhardt</natural></object></field></object></django-objects>
+
 """}
 
 # Database flushing does not work on MySQL with the default storage engine
@@ -196,6 +297,23 @@ Multiple fixtures named 'fixture5' in '...fixtures'. Aborting.
 >>> Category.objects.all()
 []
 
+# Load back in fixture 1, we need the articles from it
+>>> management.call_command('loaddata', 'fixture1', verbosity=0)
+
+# Try to load fixture 6 using format discovery
+>>> management.call_command('loaddata', 'fixture6', verbosity=0)
+>>> Tag.objects.all()
+[<Tag: <Article: Time to reform copyright> tagged "copyright">, <Tag: <Article: Time to reform copyright> tagged "law">]
+
+# Dump the current contents of the database as a JSON fixture
+>>> management.call_command('dumpdata', 'fixtures', format='json', use_natural_keys=True)
+[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": "News Stories"}}, {"pk": 3, "model": "fixtures.article", "fields": {"headline": "Time to reform copyright", "pub_date": "2006-06-16 13:00:00"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place on ESPN", "pub_date": "2006-06-16 12:00:00"}}, {"pk": 1, "model": "fixtures.article", "fields": {"headline": "Python program becomes self aware", "pub_date": "2006-06-16 11:00:00"}}, {"pk": 1, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "copyright", "tagged_id": 3}}, {"pk": 2, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "law", "tagged_id": 3}}, {"pk": 1, "model": "fixtures.person", "fields": {"name": "Django Reinhardt"}}, {"pk": 3, "model": "fixtures.person", "fields": {"name": "Prince"}}, {"pk": 2, "model": "fixtures.person", "fields": {"name": "Stephane Grappelli"}}]
+
+# Dump the current contents of the database as an XML fixture
+>>> management.call_command('dumpdata', 'fixtures', format='xml', use_natural_keys=True)
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0"><object pk="1" model="fixtures.category"><field type="CharField" name="title">News Stories</field><field type="TextField" name="description">Latest news stories</field></object><object pk="3" model="fixtures.article"><field type="CharField" name="headline">Time to reform copyright</field><field type="DateTimeField" name="pub_date">2006-06-16 13:00:00</field></object><object pk="2" model="fixtures.article"><field type="CharField" name="headline">Poker has no place on ESPN</field><field type="DateTimeField" name="pub_date">2006-06-16 12:00:00</field></object><object pk="1" model="fixtures.article"><field type="CharField" name="headline">Python program becomes self aware</field><field type="DateTimeField" name="pub_date">2006-06-16 11:00:00</field></object><object pk="1" model="fixtures.tag"><field type="CharField" name="name">copyright</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><natural>fixtures</natural><natural>article</natural></field><field type="PositiveIntegerField" name="tagged_id">3</field></object><object pk="2" model="fixtures.tag"><field type="CharField" name="name">law</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><natural>fixtures</natural><natural>article</natural></field><field type="PositiveIntegerField" name="tagged_id">3</field></object><object pk="1" model="fixtures.person"><field type="CharField" name="name">Django Reinhardt</field></object><object pk="3" model="fixtures.person"><field type="CharField" name="name">Prince</field></object><object pk="2" model="fixtures.person"><field type="CharField" name="name">Stephane Grappelli</field></object></django-objects>
+
 """
 
 from django.test import TestCase

tests/regressiontests/admin_widgets/urls2.py

@@ -3,5 +3,5 @@ from django.conf.urls.defaults import *
 import widgetadmin
 
 urlpatterns = patterns('',
-    (r'^deep/down/admin/(.*)', widgetadmin.site.root),
+    (r'^deep/down/admin/', include(widgetadmin.site.urls)),
 )

tests/regressiontests/fixtures_regress/fixtures/forward_ref_lookup.json

@@ -0,0 +1,32 @@
+[
+    {
+        "pk": "4",
+        "model": "fixtures_regress.person",
+        "fields": {
+            "name": "Neal Stephenson"
+        }
+    },
+    {
+        "pk": "2",
+        "model": "fixtures_regress.store",
+        "fields": {
+            "name": "Amazon"
+        }
+    },
+    {
+        "pk": "3",
+        "model": "fixtures_regress.store",
+        "fields": {
+            "name": "Borders"
+        }
+    },
+    {
+        "pk": 1,
+        "model": "fixtures_regress.book",
+        "fields": {
+            "name": "Cryptonomicon",
+            "author": ["Neal Stephenson"],
+            "stores": [["Amazon"], ["Borders"]]
+        }
+    }
+]

tests/regressiontests/fixtures_regress/fixtures/non_natural_1.json

@@ -0,0 +1,25 @@
+[
+    {
+        "pk": 12,
+        "model": "fixtures_regress.person",
+        "fields": {
+            "name": "Greg Egan"
+        }
+    },
+    {
+        "pk": 11,
+        "model": "fixtures_regress.store",
+        "fields": {
+            "name": "Angus and Robertson"
+        }
+    },
+    {
+        "pk": 10,
+        "model": "fixtures_regress.book",
+        "fields": {
+            "name": "Permutation City",
+            "author": 12,
+            "stores": [11]
+        }
+    }
+]

tests/regressiontests/fixtures_regress/fixtures/non_natural_2.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+    <object pk="22" model="fixtures_regress.person">
+        <field type="CharField" name="name">Orson Scott Card</field>
+    </object>
+    <object pk="21" model="fixtures_regress.store">
+        <field type="CharField" name="name">Collins Bookstore</field>
+    </object>
+    <object pk="20" model="fixtures_regress.book">
+        <field type="CharField" name="name">Ender's Game</field>
+        <field to="fixtures_regress.person" name="author" rel="ManyToOneRel">22</field>
+        <field to="fixtures_regress.store" name="stores" rel="ManyToManyRel">
+            <object pk="21"/>
+        </field>
+    </object>
+</django-objects>

tests/regressiontests/fixtures_regress/models.py

@@ -13,7 +13,7 @@ class Animal(models.Model):
     specimens = models.Manager()
 
     def __unicode__(self):
-        return self.common_name
+        return self.name
 
 def animal_pre_save_check(signal, sender, instance, **kwargs):
     "A signal that is used to check the type of data loaded from fixtures"
@@ -70,10 +70,66 @@ class Article(models.Model):
 class Widget(models.Model):
     name = models.CharField(max_length=255)
 
+    class Meta:
+        ordering = ('name',)
+
+    def __unicode__(self):
+        return self.name
+
 class WidgetProxy(Widget):
     class Meta:
         proxy = True
 
+# Check for forward references in FKs and M2Ms with natural keys
+
+class TestManager(models.Manager):
+    def get_by_natural_key(self, key):
+        return self.get(name=key)
+
+class Store(models.Model):
+    objects = TestManager()
+    name = models.CharField(max_length=255)
+
+    class Meta:
+        ordering = ('name',)
+
+    def __unicode__(self):
+        return self.name
+
+    def natural_key(self):
+        return (self.name,)
+
+class Person(models.Model):
+    objects = TestManager()
+    name = models.CharField(max_length=255)
+
+    class Meta:
+        ordering = ('name',)
+
+    def __unicode__(self):
+        return self.name
+
+    # Person doesn't actually have a dependency on store, but we need to define
+    # one to test the behaviour of the dependency resolution algorithm.
+    def natural_key(self):
+        return (self.name,)
+    natural_key.dependencies = ['fixtures_regress.store']
+
+class Book(models.Model):
+    name = models.CharField(max_length=255)
+    author = models.ForeignKey(Person)
+    stores = models.ManyToManyField(Store)
+
+    class Meta:
+        ordering = ('name',)
+
+    def __unicode__(self):
+        return u'%s by %s (available at %s)' % (
+            self.name,
+            self.author.name,
+            ', '.join(s.name for s in self.stores.all())
+        )
+
 __test__ = {'API_TESTS':"""
 >>> from django.core import management
 
@@ -193,4 +249,121 @@ Weight = 1.2 (<type 'float'>)
 >>> management.call_command('dumpdata', 'fixtures_regress', format='json')
 [{"pk": 1, "model": "fixtures_regress.widget", "fields": {"name": "grommet"}}]
 
+###############################################
+# Check that natural key requirements are taken into account
+# when serializing models
+>>> management.call_command('loaddata', 'forward_ref_lookup.json', verbosity=0)
+
+>>> management.call_command('dumpdata', 'fixtures_regress.book', 'fixtures_regress.person', 'fixtures_regress.store', verbosity=0, use_natural_keys=True)
+[{"pk": 2, "model": "fixtures_regress.store", "fields": {"name": "Amazon"}}, {"pk": 3, "model": "fixtures_regress.store", "fields": {"name": "Borders"}}, {"pk": 4, "model": "fixtures_regress.person", "fields": {"name": "Neal Stephenson"}}, {"pk": 1, "model": "fixtures_regress.book", "fields": {"stores": [["Amazon"], ["Borders"]], "name": "Cryptonomicon", "author": ["Neal Stephenson"]}}]
+
+# Now lets check the dependency sorting explicitly
+
+# First Some models with pathological circular dependencies
+>>> class Circle1(models.Model):
+...     name = models.CharField(max_length=255)
+...     def natural_key(self):
+...         return self.name
+...     natural_key.dependencies = ['fixtures_regress.circle2']
+
+>>> class Circle2(models.Model):
+...     name = models.CharField(max_length=255)
+...     def natural_key(self):
+...         return self.name
+...     natural_key.dependencies = ['fixtures_regress.circle1']
+
+>>> class Circle3(models.Model):
+...     name = models.CharField(max_length=255)
+...     def natural_key(self):
+...         return self.name
+...     natural_key.dependencies = ['fixtures_regress.circle3']
+
+>>> class Circle4(models.Model):
+...     name = models.CharField(max_length=255)
+...     def natural_key(self):
+...         return self.name
+...     natural_key.dependencies = ['fixtures_regress.circle5']
+
+>>> class Circle5(models.Model):
+...     name = models.CharField(max_length=255)
+...     def natural_key(self):
+...         return self.name
+...     natural_key.dependencies = ['fixtures_regress.circle6']
+
+>>> class Circle6(models.Model):
+...     name = models.CharField(max_length=255)
+...     def natural_key(self):
+...         return self.name
+...     natural_key.dependencies = ['fixtures_regress.circle4']
+
+>>> class ExternalDependency(models.Model):
+...     name = models.CharField(max_length=255)
+...     def natural_key(self):
+...         return self.name
+...     natural_key.dependencies = ['fixtures_regress.book']
+
+# It doesn't matter what order you mention the models
+# Store *must* be serialized before then Person, and both
+# must be serialized before Book.
+>>> from django.core.management.commands.dumpdata import sort_dependencies
+>>> sort_dependencies([('fixtures_regress', [Book, Person, Store])])
+[<class 'regressiontests.fixtures_regress.models.Store'>, <class 'regressiontests.fixtures_regress.models.Person'>, <class 'regressiontests.fixtures_regress.models.Book'>]
+
+>>> sort_dependencies([('fixtures_regress', [Book, Store, Person])])
+[<class 'regressiontests.fixtures_regress.models.Store'>, <class 'regressiontests.fixtures_regress.models.Person'>, <class 'regressiontests.fixtures_regress.models.Book'>]
+
+>>> sort_dependencies([('fixtures_regress', [Store, Book, Person])])
+[<class 'regressiontests.fixtures_regress.models.Store'>, <class 'regressiontests.fixtures_regress.models.Person'>, <class 'regressiontests.fixtures_regress.models.Book'>]
+
+>>> sort_dependencies([('fixtures_regress', [Store, Person, Book])])
+[<class 'regressiontests.fixtures_regress.models.Store'>, <class 'regressiontests.fixtures_regress.models.Person'>, <class 'regressiontests.fixtures_regress.models.Book'>]
+
+>>> sort_dependencies([('fixtures_regress', [Person, Book, Store])])
+[<class 'regressiontests.fixtures_regress.models.Store'>, <class 'regressiontests.fixtures_regress.models.Person'>, <class 'regressiontests.fixtures_regress.models.Book'>]
+
+>>> sort_dependencies([('fixtures_regress', [Person, Store, Book])])
+[<class 'regressiontests.fixtures_regress.models.Store'>, <class 'regressiontests.fixtures_regress.models.Person'>, <class 'regressiontests.fixtures_regress.models.Book'>]
+
+# A dangling dependency - assume the user knows what they are doing.
+>>> sort_dependencies([('fixtures_regress', [Person, Circle1, Store, Book])])
+[<class 'regressiontests.fixtures_regress.models.Circle1'>, <class 'regressiontests.fixtures_regress.models.Store'>, <class 'regressiontests.fixtures_regress.models.Person'>, <class 'regressiontests.fixtures_regress.models.Book'>]
+
+# A tight circular dependency
+>>> sort_dependencies([('fixtures_regress', [Person, Circle2, Circle1, Store, Book])])
+Traceback (most recent call last):
+...
+CommandError: Can't resolve dependencies for fixtures_regress.Circle1, fixtures_regress.Circle2 in serialized app list.
+
+>>> sort_dependencies([('fixtures_regress', [Circle1, Book, Circle2])])
+Traceback (most recent call last):
+...
+CommandError: Can't resolve dependencies for fixtures_regress.Circle1, fixtures_regress.Circle2 in serialized app list.
+
+# A self referential dependency
+>>> sort_dependencies([('fixtures_regress', [Book, Circle3])])
+Traceback (most recent call last):
+...
+CommandError: Can't resolve dependencies for fixtures_regress.Circle3 in serialized app list.
+
+# A long circular dependency
+>>> sort_dependencies([('fixtures_regress', [Person, Circle2, Circle1, Circle3, Store, Book])])
+Traceback (most recent call last):
+...
+CommandError: Can't resolve dependencies for fixtures_regress.Circle1, fixtures_regress.Circle2, fixtures_regress.Circle3 in serialized app list.
+
+# A dependency on a normal, non-natural-key model
+>>> sort_dependencies([('fixtures_regress', [Person, ExternalDependency, Book])])
+[<class 'regressiontests.fixtures_regress.models.Person'>, <class 'regressiontests.fixtures_regress.models.Book'>, <class 'regressiontests.fixtures_regress.models.ExternalDependency'>]
+
+###############################################
+# Check that normal primary keys still work
+# on a model with natural key capabilities
+
+>>> management.call_command('loaddata', 'non_natural_1.json', verbosity=0)
+>>> management.call_command('loaddata', 'non_natural_2.xml', verbosity=0)
+
+>>> Book.objects.all()
+[<Book: Cryptonomicon by Neal Stephenson (available at Amazon, Borders)>, <Book: Ender's Game by Orson Scott Card (available at Collins Bookstore)>, <Book: Permutation City by Greg Egan (available at Angus and Robertson)>]
+
 """}
+

tests/regressiontests/generic_inline_admin/urls.py

@@ -2,5 +2,5 @@ from django.conf.urls.defaults import *
 from django.contrib import admin
 
 urlpatterns = patterns('',
-    (r'^admin/(.*)', admin.site.root),
+    (r'^admin/', include(admin.site.urls)),
 )

tests/regressiontests/templates/context.py

@@ -10,9 +10,13 @@ context_tests = r"""
 >>> c['a'] = 2
 >>> c['a']
 2
+>>> c.get('a')
+2
 >>> c.pop()
 {'a': 2}
 >>> c['a']
 1
+>>> c.get('foo', 42)
+42
 """
 

tests/regressiontests/templates/tests.py

@@ -15,7 +15,7 @@ import unittest
 from django import template
 from django.core import urlresolvers
 from django.template import loader
-from django.template.loaders import app_directories, filesystem
+from django.template.loaders import app_directories, filesystem, cached
 from django.utils.translation import activate, deactivate, ugettext as _
 from django.utils.safestring import mark_safe
 from django.utils.tzinfo import LocalTimezone
@@ -101,13 +101,15 @@ class UTF8Class:
 
 class Templates(unittest.TestCase):
     def test_loaders_security(self):
+        ad_loader = app_directories.Loader()
+        fs_loader = filesystem.Loader()
         def test_template_sources(path, template_dirs, expected_sources):
             if isinstance(expected_sources, list):
                 # Fix expected sources so they are normcased and abspathed
                 expected_sources = [os.path.normcase(os.path.abspath(s)) for s in expected_sources]
             # Test the two loaders (app_directores and filesystem).
-            func1 = lambda p, t: list(app_directories.get_template_sources(p, t))
+            func1 = lambda p, t: list(ad_loader.get_template_sources(p, t))
-            func2 = lambda p, t: list(filesystem.get_template_sources(p, t))
+            func2 = lambda p, t: list(fs_loader.get_template_sources(p, t))
             for func in (func1, func2):
                 if isinstance(expected_sources, list):
                     self.assertEqual(func(path, template_dirs), expected_sources)
@@ -198,8 +200,11 @@ class Templates(unittest.TestCase):
             except KeyError:
                 raise template.TemplateDoesNotExist, template_name
 
+        cache_loader = cached.Loader(('test_template_loader',))
+        cache_loader._cached_loaders = (test_template_loader,)
+
         old_template_loaders = loader.template_source_loaders
-        loader.template_source_loaders = [test_template_loader]
+        loader.template_source_loaders = [cache_loader]
 
         failures = []
         tests = template_tests.items()
@@ -232,20 +237,22 @@ class Templates(unittest.TestCase):
             for invalid_str, result in [('', normal_string_result),
                                         (expected_invalid_str, invalid_string_result)]:
                 settings.TEMPLATE_STRING_IF_INVALID = invalid_str
-                try:
+                for is_cached in (False, True):
-                    test_template = loader.get_template(name)
+                    try:
-                    output = self.render(test_template, vals)
+                        test_template = loader.get_template(name)
-                except ContextStackException:
+                        output = self.render(test_template, vals)
-                    failures.append("Template test (TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Context stack was left imbalanced" % (invalid_str, name))
+                    except ContextStackException:
-                    continue
+                        failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Context stack was left imbalanced" % (is_cached, invalid_str, name))
-                except Exception:
+                        continue
-                    exc_type, exc_value, exc_tb = sys.exc_info()
+                    except Exception:
-                    if exc_type != result:
+                        exc_type, exc_value, exc_tb = sys.exc_info()
-                        tb = '\n'.join(traceback.format_exception(exc_type, exc_value, exc_tb))
+                        if exc_type != result:
-                        failures.append("Template test (TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Got %s, exception: %s\n%s" % (invalid_str, name, exc_type, exc_value, tb))
+                            tb = '\n'.join(traceback.format_exception(exc_type, exc_value, exc_tb))
-                    continue
+                            failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Got %s, exception: %s\n%s" % (is_cached, invalid_str, name, exc_type, exc_value, tb))
-                if output != result:
+                        continue
-                    failures.append("Template test (TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Expected %r, got %r" % (invalid_str, name, result, output))
+                    if output != result:
+                        failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Expected %r, got %r" % (is_cached, invalid_str, name, result, output))
+                cache_loader.reset()
 
             if 'LANGUAGE_CODE' in vals[1]:
                 deactivate()

tests/regressiontests/test_client_regress/models.py

@@ -10,6 +10,7 @@ from django.test.utils import ContextList
 from django.core.urlresolvers import reverse
 from django.core.exceptions import SuspiciousOperation
 from django.template import TemplateDoesNotExist, TemplateSyntaxError, Context
+from django.template import loader
 
 class AssertContainsTests(TestCase):
     def setUp(self):
@@ -436,6 +437,11 @@ class ExceptionTests(TestCase):
 
 class TemplateExceptionTests(TestCase):
     def setUp(self):
+        # Reset the loaders so they don't try to render cached templates.
+        if loader.template_source_loaders is not None:
+            for template_loader in loader.template_source_loaders:
+                if hasattr(template_loader, 'reset'):
+                    template_loader.reset()
         self.old_templates = settings.TEMPLATE_DIRS
         settings.TEMPLATE_DIRS = ()