Fixed #7052 -- Added support for natural keys in serialization.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@11863 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/core/management/commands/dumpdata.py

@@ -13,6 +13,8 @@ class Command(BaseCommand):
             help='Specifies the indent level to use when pretty-printing output'),
         make_option('-e', '--exclude', dest='exclude',action='append', default=[],
             help='App to exclude (use multiple --exclude to exclude multiple apps).'),
+        make_option('-n', '--natural', action='store_true', dest='use_natural_keys', default=False,
+            help='Use natural keys if they are available.'),
     )
     help = 'Output the contents of the database as a fixture of the given format.'
     args = '[appname ...]'
@@ -24,6 +26,7 @@ class Command(BaseCommand):
         indent = options.get('indent',None)
         exclude = options.get('exclude',[])
         show_traceback = options.get('traceback', False)
+        use_natural_keys = options.get('use_natural_keys', False)
 
         excluded_apps = [get_app(app_label) for app_label in exclude]
 
@@ -67,18 +70,86 @@ class Command(BaseCommand):
         except KeyError:
             raise CommandError("Unknown serialization format: %s" % format)
 
+        # Now collate the objects to be serialized.
         objects = []
-        for app, model_list in app_list.items():
-            if model_list is None:
-                model_list = get_models(app)
-
-            for model in model_list:
-                if not model._meta.proxy:
-                    objects.extend(model._default_manager.all())
+        for model in sort_dependencies(app_list.items()):
+            if not model._meta.proxy:
+                objects.extend(model._default_manager.all())
 
         try:
-            return serializers.serialize(format, objects, indent=indent)
+            return serializers.serialize(format, objects, indent=indent,
+                        use_natural_keys=use_natural_keys)
         except Exception, e:
             if show_traceback:
                 raise
             raise CommandError("Unable to serialize database: %s" % e)
+
+def sort_dependencies(app_list):
+    """Sort a list of app,modellist pairs into a single list of models.
+
+    The single list of models is sorted so that any model with a natural key
+    is serialized before a normal model, and any model with a natural key
+    dependency has it's dependencies serialized first.
+    """
+    from django.db.models import get_model, get_models
+    # Process the list of models, and get the list of dependencies
+    model_dependencies = []
+    models = set()
+    for app, model_list in app_list:
+        if model_list is None:
+            model_list = get_models(app)
+
+        for model in model_list:
+            models.add(model)
+            # Add any explicitly defined dependencies
+            if hasattr(model, 'natural_key'):
+                deps = getattr(model.natural_key, 'dependencies', [])
+                if deps:
+                    deps = [get_model(*d.split('.')) for d in deps]
+            else:
+                deps = []
+
+            # Now add a dependency for any FK or M2M relation with
+            # a model that defines a natural key
+            for field in model._meta.fields:
+                if hasattr(field.rel, 'to'):
+                    rel_model = field.rel.to
+                    if hasattr(rel_model, 'natural_key'):
+                        deps.append(rel_model)
+            for field in model._meta.many_to_many:
+                rel_model = field.rel.to
+                if hasattr(rel_model, 'natural_key'):
+                    deps.append(rel_model)
+            model_dependencies.append((model, deps))
+
+    model_dependencies.reverse()
+    # Now sort the models to ensure that dependencies are met. This
+    # is done by repeatedly iterating over the input list of models.
+    # If all the dependencies of a given model are in the final list,
+    # that model is promoted to the end of the final list. This process
+    # continues until the input list is empty, or we do a full iteration
+    # over the input models without promoting a model to the final list.
+    # If we do a full iteration without a promotion, that means there are
+    # circular dependencies in the list.
+    model_list = []
+    while model_dependencies:
+        skipped = []
+        changed = False
+        while model_dependencies:
+            model, deps = model_dependencies.pop()
+            if all((d not in models or d in model_list) for d in deps):
+                # If all of the models in the dependency list are either already
+                # on the final model list, or not on the original serialization list,
+                # then we've found another model with all it's dependencies satisfied.
+                model_list.append(model)
+                changed = True
+            else:
+                skipped.append((model, deps))
+        if not changed:
+            raise CommandError("Can't resolve dependencies for %s in serialized app list." %
+                ', '.join('%s.%s' % (model._meta.app_label, model._meta.object_name)
+                for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__))
+            )
+        model_dependencies = skipped
+
+    return model_list

django/core/serializers/base.py

@@ -33,6 +33,7 @@ class Serializer(object):
 
         self.stream = options.get("stream", StringIO())
         self.selected_fields = options.get("fields")
+        self.use_natural_keys = options.get("use_natural_keys", False)
 
         self.start_serialization()
         for obj in queryset:

django/core/serializers/json.py

@@ -24,6 +24,7 @@ class Serializer(PythonSerializer):
     def end_serialization(self):
         self.options.pop('stream', None)
         self.options.pop('fields', None)
+        self.options.pop('use_natural_keys', None)
         simplejson.dump(self.objects, self.stream, cls=DjangoJSONEncoder, **self.options)
 
     def getvalue(self):

django/core/serializers/python.py

@@ -47,17 +47,24 @@ class Serializer(base.Serializer):
     def handle_fk_field(self, obj, field):
         related = getattr(obj, field.name)
         if related is not None:
-            if field.rel.field_name == related._meta.pk.name:
-                # Related to remote object via primary key
-                related = related._get_pk_val()
+            if self.use_natural_keys and hasattr(related, 'natural_key'):
+                related = related.natural_key()
             else:
-                # Related to remote object via other field
-                related = getattr(related, field.rel.field_name)
-        self._current[field.name] = smart_unicode(related, strings_only=True)
+                if field.rel.field_name == related._meta.pk.name:
+                    # Related to remote object via primary key
+                    related = related._get_pk_val()
+                else:
+                    # Related to remote object via other field
+                    related = smart_unicode(getattr(related, field.rel.field_name), strings_only=True)
+        self._current[field.name] = related
 
     def handle_m2m_field(self, obj, field):
         if field.rel.through._meta.auto_created:
-            self._current[field.name] = [smart_unicode(related._get_pk_val(), strings_only=True)
+            if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
+                m2m_value = lambda value: value.natural_key()
+            else:
+                m2m_value = lambda value: smart_unicode(value._get_pk_val(), strings_only=True)
+            self._current[field.name] = [m2m_value(related)
                                for related in getattr(obj, field.name).iterator()]
 
     def getvalue(self):
@@ -86,13 +93,28 @@ def Deserializer(object_list, **options):
 
             # Handle M2M relations
             if field.rel and isinstance(field.rel, models.ManyToManyRel):
-                m2m_convert = field.rel.to._meta.pk.to_python
-                m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value]
+                if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
+                    def m2m_convert(value):
+                        if hasattr(value, '__iter__'):
+                            return field.rel.to._default_manager.get_by_natural_key(*value).pk
+                        else:
+                            return smart_unicode(field.rel.to._meta.pk.to_python(value))
+                else:
+                    m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v))
+                m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
 
             # Handle FK fields
             elif field.rel and isinstance(field.rel, models.ManyToOneRel):
                 if field_value is not None:
-                    data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
+                    if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
+                        if hasattr(field_value, '__iter__'):
+                            obj = field.rel.to._default_manager.get_by_natural_key(*field_value)
+                            value = getattr(obj, field.rel.field_name)
+                        else:
+                            value = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
+                        data[field.attname] = value
+                    else:
+                        data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
                 else:
                     data[field.attname] = None
 

django/core/serializers/pyyaml.py

@@ -26,9 +26,9 @@ class Serializer(PythonSerializer):
     """
     Convert a queryset to YAML.
     """
-    
+
     internal_use_only = False
-    
+
     def handle_field(self, obj, field):
         # A nasty special case: base YAML doesn't support serialization of time
         # types (as opposed to dates or datetimes, which it does support). Since
@@ -40,10 +40,11 @@ class Serializer(PythonSerializer):
             self._current[field.name] = str(getattr(obj, field.name))
         else:
             super(Serializer, self).handle_field(obj, field)
-    
+
     def end_serialization(self):
         self.options.pop('stream', None)
         self.options.pop('fields', None)
+        self.options.pop('use_natural_keys', None)
         yaml.dump(self.objects, self.stream, Dumper=DjangoSafeDumper, **self.options)
 
     def getvalue(self):

django/core/serializers/xml_serializer.py

@@ -81,13 +81,22 @@ class Serializer(base.Serializer):
         self._start_relational_field(field)
         related = getattr(obj, field.name)
         if related is not None:
-            if field.rel.field_name == related._meta.pk.name:
-                # Related to remote object via primary key
-                related = related._get_pk_val()
+            if self.use_natural_keys and hasattr(related, 'natural_key'):
+                # If related object has a natural key, use it
+                related = related.natural_key()
+                # Iterable natural keys are rolled out as subelements
+                for key_value in related:
+                    self.xml.startElement("natural", {})
+                    self.xml.characters(smart_unicode(key_value))
+                    self.xml.endElement("natural")
             else:
-                # Related to remote object via other field
-                related = getattr(related, field.rel.field_name)
-            self.xml.characters(smart_unicode(related))
+                if field.rel.field_name == related._meta.pk.name:
+                    # Related to remote object via primary key
+                    related = related._get_pk_val()
+                else:
+                    # Related to remote object via other field
+                    related = getattr(related, field.rel.field_name)
+                self.xml.characters(smart_unicode(related))
         else:
             self.xml.addQuickElement("None")
         self.xml.endElement("field")
@@ -100,8 +109,25 @@ class Serializer(base.Serializer):
         """
         if field.rel.through._meta.auto_created:
             self._start_relational_field(field)
+            if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
+                # If the objects in the m2m have a natural key, use it
+                def handle_m2m(value):
+                    natural = value.natural_key()
+                    # Iterable natural keys are rolled out as subelements
+                    self.xml.startElement("object", {})
+                    for key_value in natural:
+                        self.xml.startElement("natural", {})
+                        self.xml.characters(smart_unicode(key_value))
+                        self.xml.endElement("natural")
+                    self.xml.endElement("object")
+            else:
+                def handle_m2m(value):
+                    self.xml.addQuickElement("object", attrs={
+                        'pk' : smart_unicode(value._get_pk_val())
+                    })
             for relobj in getattr(obj, field.name).iterator():
-                self.xml.addQuickElement("object", attrs={"pk" : smart_unicode(relobj._get_pk_val())})
+                handle_m2m(relobj)
+
             self.xml.endElement("field")
 
     def _start_relational_field(self, field):
@@ -187,16 +213,40 @@ class Deserializer(base.Deserializer):
         if node.getElementsByTagName('None'):
             return None
         else:
-            return field.rel.to._meta.get_field(field.rel.field_name).to_python(
-                       getInnerText(node).strip())
+            if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
+                keys = node.getElementsByTagName('natural')
+                if keys:
+                    # If there are 'natural' subelements, it must be a natural key
+                    field_value = [getInnerText(k).strip() for k in keys]
+                    obj = field.rel.to._default_manager.get_by_natural_key(*field_value)
+                    obj_pk = getattr(obj, field.rel.field_name)
+                else:
+                    # Otherwise, treat like a normal PK
+                    field_value = getInnerText(node).strip()
+                    obj_pk = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
+                return obj_pk
+            else:
+                field_value = getInnerText(node).strip()
+                return field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
 
     def _handle_m2m_field_node(self, node, field):
         """
         Handle a <field> node for a ManyToManyField.
         """
-        return [field.rel.to._meta.pk.to_python(
-                    c.getAttribute("pk"))
-                    for c in node.getElementsByTagName("object")]
+        if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
+            def m2m_convert(n):
+                keys = n.getElementsByTagName('natural')
+                if keys:
+                    # If there are 'natural' subelements, it must be a natural key
+                    field_value = [getInnerText(k).strip() for k in keys]
+                    obj_pk = field.rel.to._default_manager.get_by_natural_key(*field_value).pk
+                else:
+                    # Otherwise, treat like a normal PK value.
+                    obj_pk = field.rel.to._meta.pk.to_python(n.getAttribute('pk'))
+                return obj_pk
+        else:
+            m2m_convert = lambda n: field.rel.to._meta.pk.to_python(n.getAttribute('pk'))
+        return [m2m_convert(c) for c in node.getElementsByTagName("object")]
 
     def _get_model_from_node(self, node, attr):
         """