unicode: Added FILE_CHARSET setting and use it to decode files read from disk.

Based on a patch from Ivan Sagalaev. Fixed #4021.


git-svn-id: http://code.djangoproject.com/svn/django/branches/unicode@5058 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/conf/global_settings.py

@@ -95,6 +95,9 @@ MANAGERS = ADMINS
 DEFAULT_CONTENT_TYPE = 'text/html'
 DEFAULT_CHARSET = 'utf-8'
 
+# Encoding of files read from disk (template and initial SQL files).
+FILE_CHARSET = 'utf-8'
+
 # E-mail address that error messages come from.
 SERVER_EMAIL = 'root@localhost'
 

django/core/management.py

@@ -379,11 +379,11 @@ def get_custom_sql_for_model(model):
     for sql_file in sql_files:
         if os.path.exists(sql_file):
             fp = open(sql_file, 'U')
-            for statement in statements.split(fp.read()):
+            for statement in statements.split(fp.read().decode(settings.FILE_CHARSET)):
                 # Remove any comments from the file
-                statement = re.sub(r"--.*[\n\Z]", "", statement)
+                statement = re.sub(ur"--.*[\n\Z]", "", statement)
                 if statement.strip():
-                    output.append(statement + ";")
+                    output.append(statement + u";")
             fp.close()
 
     return output

django/template/loaders/app_directories.py

@@ -34,7 +34,7 @@ def get_template_sources(template_name, template_dirs=None):
 def load_template_source(template_name, template_dirs=None):
     for filepath in get_template_sources(template_name, template_dirs):
         try:
-            return (open(filepath).read(), filepath)
+            return (open(filepath).read().decode(settings.FILE_CHARSET), filepath)
         except IOError:
             pass
     raise TemplateDoesNotExist, template_name

django/template/loaders/eggs.py

@@ -18,7 +18,7 @@ def load_template_source(template_name, template_dirs=None):
         pkg_name = 'templates/' + template_name
         for app in settings.INSTALLED_APPS:
             try:
-                return (resource_string(app, pkg_name), 'egg:%s:%s ' % (app, pkg_name))
+                return (resource_string(app, pkg_name), 'egg:%s:%s ' % (app, pkg_name)).decode(settings.FILE_CHARSET)
             except:
                 pass
     raise TemplateDoesNotExist, template_name

django/template/loaders/filesystem.py

@@ -14,7 +14,7 @@ def load_template_source(template_name, template_dirs=None):
     tried = []
     for filepath in get_template_sources(template_name, template_dirs):
         try:
-            return (open(filepath).read(), filepath)
+            return (open(filepath).read().decode(settings.FILE_CHARSET), filepath)
         except IOError:
             tried.append(filepath)
     if tried:

docs/settings.txt

@@ -426,6 +426,14 @@ Subject-line prefix for e-mail messages sent with ``django.core.mail.mail_admins
 or ``django.core.mail.mail_managers``. You'll probably want to include the
 trailing space.
 
+FILE_CHARSET
+------------
+
+Default: ``'utf-8'``
+
+The character encoding used to decode any files read from disk. This includes
+template files and initial SQL data files.
+
 FIXTURE_DIRS
 -------------