diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py
index f0e85f4e19..09346b4d27 100644
--- a/django/core/management/commands/makemessages.py
+++ b/django/core/management/commands/makemessages.py
@@ -581,6 +581,8 @@ class Command(BaseCommand):
 
         if msgs:
             if locale_dir is NO_LOCALE_DIR:
+                for build_file in build_files:
+                    build_file.cleanup()
                 file_path = os.path.normpath(build_files[0].path)
                 raise CommandError(
                     "Unable to find a locale path to store translations for "
diff --git a/tests/i18n/project_dir/app_no_locale/test.html b/tests/i18n/project_dir/app_no_locale/test.html
new file mode 100644
index 0000000000..1caebc9e15
--- /dev/null
+++ b/tests/i18n/project_dir/app_no_locale/test.html
@@ -0,0 +1,4 @@
+A non-Python file will be classed as translatable.
+
+The temporary file created from this should be removed from the file system if
+an error is raised.
diff --git a/tests/i18n/test_extraction.py b/tests/i18n/test_extraction.py
index d6f0e861b0..3ba371f66c 100644
--- a/tests/i18n/test_extraction.py
+++ b/tests/i18n/test_extraction.py
@@ -765,6 +765,8 @@ class CustomLayoutExtractionTests(ExtractorTests):
         )
         with self.assertRaisesMessage(management.CommandError, msg):
             management.call_command('makemessages', locale=[LOCALE], verbosity=0)
+        # Working files are cleaned up on an error.
+        self.assertFalse(os.path.exists('./app_no_locale/test.html.py'))
 
     def test_project_locale_paths(self):
         self._test_project_locale_paths(os.path.join(self.test_dir, 'project_locale'))