Fixed #24558 -- Made dumpdata mapping ordering deterministic.

Thanks to gfairchild for the report and Claude for the review.

django/core/serializers/python.py

@@ -5,6 +5,8 @@ other serializers.
 """
 from __future__ import unicode_literals
 
+from collections import OrderedDict
+
 from django.apps import apps
 from django.conf import settings
 from django.core.serializers import base
@@ -28,20 +30,17 @@ class Serializer(base.Serializer):
         pass
 
     def start_object(self, obj):
-        self._current = {}
+        self._current = OrderedDict()
 
     def end_object(self, obj):
         self.objects.append(self.get_dump_object(obj))
         self._current = None
 
     def get_dump_object(self, obj):
-        data = {
+        data = OrderedDict([('model', force_text(obj._meta))])
-            "model": force_text(obj._meta),
-            "fields": self._current,
-        }
         if not self.use_natural_primary_keys or not hasattr(obj, 'natural_key'):
             data["pk"] = force_text(obj._get_pk_val(), strings_only=True)
-
+        data['fields'] = self._current
         return data
 
     def handle_field(self, obj, field):

django/core/serializers/pyyaml.py

@@ -4,6 +4,7 @@ YAML serializer.
 Requires PyYaml (http://pyyaml.org/), but that's checked for in __init__.
 """
 
+import collections
 import decimal
 import sys
 from io import StringIO
@@ -29,7 +30,11 @@ class DjangoSafeDumper(SafeDumper):
     def represent_decimal(self, data):
         return self.represent_scalar('tag:yaml.org,2002:str', str(data))
 
+    def represent_ordered_dict(self, data):
+        return self.represent_mapping('tag:yaml.org,2002:map', data.items())
+
 DjangoSafeDumper.add_representer(decimal.Decimal, DjangoSafeDumper.represent_decimal)
+DjangoSafeDumper.add_representer(collections.OrderedDict, DjangoSafeDumper.represent_ordered_dict)
 
 
 class Serializer(PythonSerializer):

django/core/serializers/xml_serializer.py

@@ -4,6 +4,7 @@ XML serializer.
 
 from __future__ import unicode_literals
 
+from collections import OrderedDict
 from xml.dom import pulldom
 from xml.sax import handler
 from xml.sax.expatreader import ExpatParser as _ExpatParser
@@ -49,7 +50,7 @@ class Serializer(base.Serializer):
             raise base.SerializationError("Non-model object (%s) encountered during serialization" % type(obj))
 
         self.indent(1)
-        attrs = {"model": smart_text(obj._meta)}
+        attrs = OrderedDict([("model", smart_text(obj._meta))])
         if not self.use_natural_primary_keys or not hasattr(obj, 'natural_key'):
             obj_pk = obj._get_pk_val()
             if obj_pk is not None:
@@ -70,10 +71,10 @@ class Serializer(base.Serializer):
         ManyToManyFields)
         """
         self.indent(2)
-        self.xml.startElement("field", {
+        self.xml.startElement("field", OrderedDict([
-            "name": field.name,
+            ("name", field.name),
-            "type": field.get_internal_type()
+            ("type", field.get_internal_type()),
-        })
+        ]))
 
         # Get a "string version" of the object's data.
         if getattr(obj, field.name) is not None:
@@ -140,11 +141,11 @@ class Serializer(base.Serializer):
         Helper to output the <field> element for relational fields
         """
         self.indent(2)
-        self.xml.startElement("field", {
+        self.xml.startElement("field", OrderedDict([
-            "name": field.name,
+            ("name", field.name),
-            "rel": field.remote_field.__class__.__name__,
+            ("rel", field.remote_field.__class__.__name__),
-            "to": smart_text(field.remote_field.model._meta),
+            ("to", smart_text(field.remote_field.model._meta)),
-        })
+        ]))
 
 
 class Deserializer(base.Deserializer):

docs/releases/1.9.txt

@@ -164,6 +164,8 @@ Management Commands
   :djadmin:`sqlmigrate`, the SQL code generated for each migration operation is
   preceded by the operation's description.
 
+* The :djadmin:`dumpdata` command output is now deterministically ordered.
+
 Models
 ^^^^^^
 

tests/serializers/tests.py

@@ -266,6 +266,17 @@ class SerializersTestBase(object):
             obj.save()
         self.assertEqual(Category.objects.all().count(), 5)
 
+    def test_deterministic_mapping_ordering(self):
+        """Mapping such as fields should be deterministically ordered. (#24558)"""
+        output = serializers.serialize(self.serializer_name, [self.a1], indent=2)
+        categories = self.a1.categories.values_list('pk', flat=True)
+        self.assertEqual(output, self.mapping_ordering_str % {
+            'article_pk': self.a1.pk,
+            'author_pk': self.a1.author_id,
+            'first_category_pk': categories[0],
+            'second_category_pk': categories[1],
+        })
+
 
 class SerializersTransactionTestBase(object):
 
@@ -303,6 +314,15 @@ class XmlSerializerTestCase(SerializersTestBase, TestCase):
         <field type="CharField" name="name">Non-fiction</field>
     </object>
 </django-objects>"""
+    mapping_ordering_str = """<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+  <object model="serializers.article" pk="%(article_pk)s">
+    <field name="author" rel="ManyToOneRel" to="serializers.author">%(author_pk)s</field>
+    <field name="headline" type="CharField">Poker has no place on ESPN</field>
+    <field name="pub_date" type="DateTimeField">2006-06-16T11:00:00</field>
+    <field name="categories" rel="ManyToManyRel" to="serializers.category"><object pk="%(first_category_pk)s"></object><object pk="%(second_category_pk)s"></object></field>
+  </object>
+</django-objects>"""
 
     @staticmethod
     def _comparison_value(value):
@@ -373,6 +393,22 @@ class JsonSerializerTestCase(SerializersTestBase, TestCase):
         "model": "serializers.category",
         "fields": {"name": "Non-fiction"}
     }]"""
+    mapping_ordering_str = """[
+{
+  "model": "serializers.article",
+  "pk": %(article_pk)s,
+  "fields": {
+    "author": %(author_pk)s,
+    "headline": "Poker has no place on ESPN",
+    "pub_date": "2006-06-16T11:00:00",
+    "categories": [
+      %(first_category_pk)s,
+      %(second_category_pk)s
+    ]
+  }
+}
+]
+"""
 
     @staticmethod
     def _validate_output(serial_str):
@@ -533,6 +569,15 @@ class YamlSerializerTestCase(SerializersTestBase, TestCase):
     name: Non-fiction
   model: serializers.category"""
 
+    mapping_ordering_str = """- model: serializers.article
+  pk: %(article_pk)s
+  fields:
+    author: %(author_pk)s
+    headline: Poker has no place on ESPN
+    pub_date: 2006-06-16 11:00:00
+    categories: [%(first_category_pk)s, %(second_category_pk)s]
+"""
+
     @staticmethod
     def _validate_output(serial_str):
         try:

tests/timezones/tests.py

@@ -594,7 +594,7 @@ class SerializationTests(TestCase):
     def assert_yaml_contains_datetime(self, yaml, dt):
         # Depending on the yaml dumper, '!timestamp' might be absent
         six.assertRegex(self, yaml,
-            r"- fields: {dt: !(!timestamp)? '%s'}" % re.escape(dt))
+            r"\n  fields: {dt: !(!timestamp)? '%s'}" % re.escape(dt))
 
     def test_naive_datetime(self):
         dt = datetime.datetime(2011, 9, 1, 13, 20, 30)