[1.7.x] Fixed #23938 -- Added migration support for m2m to concrete fields and vice versa

Thanks to Michael D. Hoyle for the report and Tim Graham for the review.

Backport of 623ccdd598625591d1a12fc1564cf3ef9a87581f from master

django/db/migrations/autodetector.py

@@ -778,6 +778,9 @@ class MigrationAutodetector(object):
         Fields that have been added
         """
         for app_label, model_name, field_name in sorted(self.new_field_keys - self.old_field_keys):
+            self._generate_added_field(app_label, model_name, field_name)
+
+    def _generate_added_field(self, app_label, model_name, field_name):
         field = self.new_apps.get_model(app_label, model_name)._meta.get_field_by_name(field_name)[0]
         # Fields that are foreignkeys/m2ms depend on stuff
         dependencies = []
@@ -796,32 +799,24 @@ class MigrationAutodetector(object):
                     field.rel.through._meta.app_label,
                     field.rel.through._meta.object_name,
                     None,
-                        True
+                    True,
                 ))
         # You can't just add NOT NULL fields with no default or fields
         # which don't allow empty strings as default.
+        preserve_default = True
         if (not field.null and not field.has_default() and
                 not isinstance(field, models.ManyToManyField) and
                 not (field.blank and field.empty_strings_allowed)):
             field = field.clone()
             field.default = self.questioner.ask_not_null_addition(field_name, model_name)
+            preserve_default = False
         self.add_operation(
             app_label,
             operations.AddField(
                 model_name=model_name,
                 name=field_name,
                 field=field,
-                        preserve_default=False,
-                    ),
-                    dependencies=dependencies,
-                )
-            else:
-                self.add_operation(
-                    app_label,
-                    operations.AddField(
-                        model_name=model_name,
-                        name=field_name,
-                        field=field,
+                preserve_default=preserve_default,
             ),
             dependencies=dependencies,
         )
@@ -831,6 +826,9 @@ class MigrationAutodetector(object):
         Fields that have been removed.
         """
         for app_label, model_name, field_name in sorted(self.old_field_keys - self.new_field_keys):
+            self._generate_removed_field(app_label, model_name, field_name)
+
+    def _generate_removed_field(self, app_label, model_name, field_name):
         self.add_operation(
             app_label,
             operations.RemoveField(
@@ -868,6 +866,8 @@ class MigrationAutodetector(object):
             old_field_dec = self.deep_deconstruct(old_field)
             new_field_dec = self.deep_deconstruct(new_field)
             if old_field_dec != new_field_dec:
+                if (not isinstance(old_field, models.ManyToManyField) and
+                        not isinstance(new_field, models.ManyToManyField)):
                     preserve_default = True
                     if (old_field.null and not new_field.null and not new_field.has_default() and
                             not isinstance(new_field, models.ManyToManyField)):
@@ -887,6 +887,9 @@ class MigrationAutodetector(object):
                             preserve_default=preserve_default,
                         )
                     )
+                else:
+                    self._generate_removed_field(app_label, model_name, field_name)
+                    self._generate_added_field(app_label, model_name, field_name)
 
     def _generate_altered_foo_together(self, operation):
         option_name = operation.option_name

docs/releases/1.7.2.txt

@@ -44,7 +44,7 @@ Bugfixes
 * Fixed a migration crash that prevented changing a nullable field with a
   default to non-nullable with the same default (:ticket:`23738`).
 
-* Fixed a migrations crash when adding ``GeometryField``\s with ``blank=True``
+* Fixed a migration crash when adding ``GeometryField``\s with ``blank=True``
   on PostGIS (:ticket:`23731`).
 
 * Allowed usage of ``DateTimeField()`` as ``Transform.output_field``
@@ -144,7 +144,7 @@ Bugfixes
 * ``makemigrations`` no longer prompts for a default value when adding
   ``TextField()`` or ``CharField()`` without a ``default`` (:ticket:`23405`).
 
-* Fixed migration crash when adding ``order_with_respect_to`` to a table
+* Fixed a migration crash when adding ``order_with_respect_to`` to a table
   with existing rows (:ticket:`23983`).
 
 * Restored the ``pre_migrate`` signal if all apps have migrations
@@ -181,3 +181,6 @@ Bugfixes
 
 * Supported strings escaped by third-party libraries with the ``__html__``
   convention in the template engine (:ticket:`23831`).
+
+* Fixed a migration crash when changing a ``ManyToManyField`` into a concrete
+  field and vice versa (:ticket:`23938`).

tests/migrations/test_autodetector.py

@@ -116,6 +116,10 @@ class AutodetectorTests(TestCase):
         ("id", models.AutoField(primary_key=True)),
         ("publishers", models.ManyToManyField("testapp.Publisher", through="testapp.Contract")),
     ])
+    author_with_former_m2m = ModelState("testapp", "Author", [
+        ("id", models.AutoField(primary_key=True)),
+        ("publishers", models.CharField(max_length=100)),
+    ])
     author_with_options = ModelState("testapp", "Author", [
         ("id", models.AutoField(primary_key=True)),
     ], {
@@ -1274,6 +1278,39 @@ class AutodetectorTests(TestCase):
         self.assertOperationAttributes(changes, "testapp", 0, 3, name="publisher", model_name='contract')
         self.assertOperationAttributes(changes, "testapp", 0, 4, name="Contract")
 
+    def test_concrete_field_changed_to_many_to_many(self):
+        """
+        #23938 - Tests that changing a concrete field into a ManyToManyField
+        first removes the concrete field and then adds the m2m field.
+        """
+        before = self.make_project_state([self.author_with_former_m2m])
+        after = self.make_project_state([self.author_with_m2m, self.publisher])
+        autodetector = MigrationAutodetector(before, after)
+        changes = autodetector._detect_changes()
+        # Right number/type of migrations?
+        self.assertNumberMigrations(changes, "testapp", 1)
+        self.assertOperationTypes(changes, "testapp", 0, ["CreateModel", "RemoveField", "AddField"])
+        self.assertOperationAttributes(changes, 'testapp', 0, 0, name='Publisher')
+        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="publishers", model_name='author')
+        self.assertOperationAttributes(changes, 'testapp', 0, 2, name="publishers", model_name='author')
+
+    def test_many_to_many_changed_to_concrete_field(self):
+        """
+        #23938 - Tests that changing a ManyToManyField into a concrete field
+        first removes the m2m field and then adds the concrete field.
+        """
+        before = self.make_project_state([self.author_with_m2m, self.publisher])
+        after = self.make_project_state([self.author_with_former_m2m])
+        autodetector = MigrationAutodetector(before, after)
+        changes = autodetector._detect_changes()
+        # Right number/type of migrations?
+        self.assertNumberMigrations(changes, "testapp", 1)
+        self.assertOperationTypes(changes, "testapp", 0, ["RemoveField", "AddField", "DeleteModel"])
+        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="publishers", model_name='author')
+        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="publishers", model_name='author')
+        self.assertOperationAttributes(changes, 'testapp', 0, 2, name='Publisher')
+        self.assertOperationFieldAttributes(changes, 'testapp', 0, 1, max_length=100)
+
     def test_non_circular_foreignkey_dependency_removal(self):
         """
         If two models with a ForeignKey from one to the other are removed at the