Fixed #20354 -- makemessages no longer crashes with UnicodeDecodeError

Handle the `UnicodeDecodeError` exception, send a warning to `stdout` with the
file name and location, and continue processing other files.

django/core/management/commands/makemessages.py

@@ -294,7 +294,10 @@ class Command(NoArgsCommand):
             os.unlink(potfile)
 
         for f in file_list:
-            f.process(self, potfile, self.domain, self.keep_pot)
+            try:
+                f.process(self, potfile, self.domain, self.keep_pot)
+            except UnicodeDecodeError:
+                self.stdout.write("UnicodeDecodeError: skipped file %s in %s" % (f.file, f.dirpath))
         return potfile
 
     def find_files(self, root):

tests/i18n/commands/extraction.py

@@ -30,6 +30,10 @@ class ExtractorTests(SimpleTestCase):
             return
         shutil.rmtree(dname)
 
+    def rmfile(self, filepath):
+        if os.path.exists(filepath):
+            os.remove(filepath)
+
     def tearDown(self):
         os.chdir(self.test_dir)
         try:
@@ -126,18 +130,22 @@ class BasicExtractorTests(ExtractorTests):
         # Check that the temporary file was cleaned up
         self.assertFalse(os.path.exists('./templates/template_with_error.tpl.py'))
 
+    def test_unicode_decode_error(self):
+        os.chdir(self.test_dir)
+        shutil.copyfile('./not_utf8.sample', './not_utf8.txt')
+        self.addCleanup(self.rmfile, os.path.join(self.test_dir, 'not_utf8.txt'))
+        stdout = StringIO()
+        management.call_command('makemessages', locale=LOCALE, stdout=stdout)
+        self.assertIn("UnicodeDecodeError: skipped file not_utf8.txt in .",
+                      force_text(stdout.getvalue()))
+
     def test_extraction_warning(self):
         """test xgettext warning about multiple bare interpolation placeholders"""
         os.chdir(self.test_dir)
         shutil.copyfile('./code.sample', './code_sample.py')
+        self.addCleanup(self.rmfile, os.path.join(self.test_dir, 'code_sample.py'))
         stdout = StringIO()
-        try:
-            management.call_command('makemessages', locale=LOCALE, stdout=stdout)
-        finally:
-            try:
-                os.remove('./code_sample.py')
-            except OSError:
-                pass
+        management.call_command('makemessages', locale=LOCALE, stdout=stdout)
         self.assertIn("code_sample.py:4", force_text(stdout.getvalue()))
 
     def test_template_message_context_extractor(self):

tests/i18n/commands/not_utf8.sample

@@ -0,0 +1 @@
+Copyright (c) 2009 Øyvind Sean Kinsey, oyvind@kinsey.no