Merged Unicode branch into trunk (r4952:5608). This should be fully

backwards compatible for all practical purposes.

Fixed #2391, #2489, #2996, #3322, #3344, #3370, #3406, #3432, #3454, #3492, #3582, #3690, #3878, #3891, #3937, #4039, #4141, #4227, #4286, #4291, #4300, #4452, #4702


git-svn-id: http://code.djangoproject.com/svn/django/trunk@5609 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/core/handlers/modpython.py

@@ -49,7 +49,7 @@ class ModPythonRequest(http.HttpRequest):
         if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
         else:
-            self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
+            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
 
     def _get_request(self):
         if not hasattr(self, '_request'):
@@ -58,7 +58,7 @@ class ModPythonRequest(http.HttpRequest):
 
     def _get_get(self):
         if not hasattr(self, '_get'):
-            self._get = http.QueryDict(self._req.args)
+            self._get = http.QueryDict(self._req.args, encoding=self._encoding)
         return self._get
 
     def _set_get(self, get):
@@ -160,7 +160,7 @@ class ModPythonHandler(BaseHandler):
         req.content_type = response['Content-Type']
         for key, value in response.headers.items():
             if key != 'Content-Type':
-                req.headers_out[key] = value
+                req.headers_out[str(key)] = str(value)
         for c in response.cookies.values():
             req.headers_out.add('Set-Cookie', c.output(header=''))
         req.status = response.status_code

django/core/handlers/wsgi.py

@@ -113,9 +113,9 @@ class WSGIRequest(http.HttpRequest):
                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
             else:
-                self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
+                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
         else:
-            self._post, self._files = http.QueryDict(''), datastructures.MultiValueDict()
+            self._post, self._files = http.QueryDict('', encoding=self._encoding), datastructures.MultiValueDict()
 
     def _get_request(self):
         if not hasattr(self, '_request'):
@@ -125,7 +125,7 @@ class WSGIRequest(http.HttpRequest):
     def _get_get(self):
         if not hasattr(self, '_get'):
             # The WSGI spec says 'QUERY_STRING' may be absent.
-            self._get = http.QueryDict(self.environ.get('QUERY_STRING', ''))
+            self._get = http.QueryDict(self.environ.get('QUERY_STRING', ''), encoding=self._encoding)
         return self._get
 
     def _set_get(self, get):
@@ -200,8 +200,8 @@ class WSGIHandler(BaseHandler):
         except KeyError:
             status_text = 'UNKNOWN STATUS CODE'
         status = '%s %s' % (response.status_code, status_text)
-        response_headers = response.headers.items()
+        response_headers = [(str(k), str(v)) for k, v in response.headers.items()]
         for c in response.cookies.values():
-            response_headers.append(('Set-Cookie', c.output(header='')))
+            response_headers.append(('Set-Cookie', str(c.output(header=''))))
         start_response(status, response_headers)
         return response

django/core/mail.py

@@ -3,12 +3,13 @@ Tools for sending email.
 """
 
 from django.conf import settings
+from django.utils.encoding import smart_str, force_unicode
 from email import Charset, Encoders
 from email.MIMEText import MIMEText
 from email.MIMEMultipart import MIMEMultipart
 from email.MIMEBase import MIMEBase
 from email.Header import Header
-from email.Utils import formatdate
+from email.Utils import formatdate, parseaddr, formataddr
 import mimetypes
 import os
 import smtplib
@@ -67,8 +68,18 @@ class SafeMIMEText(MIMEText):
         "Forbids multi-line headers, to prevent header injection."
         if '\n' in val or '\r' in val:
             raise BadHeaderError, "Header values can't contain newlines (got %r for header %r)" % (val, name)
-        if name == "Subject":
-            val = Header(val, settings.DEFAULT_CHARSET)
+        try:
+            val = str(force_unicode(val))
+        except UnicodeEncodeError:
+            if name.lower() in ('to', 'from', 'cc'):
+                result = []
+                for item in val.split(', '):
+                    nm, addr = parseaddr(item)
+                    nm = str(Header(nm, settings.DEFAULT_CHARSET))
+                    result.append(formataddr((nm, str(addr))))
+                val = ', '.join(result)
+            else:
+                val = Header(force_unicode(val), settings.DEFAULT_CHARSET)
         MIMEText.__setitem__(self, name, val)
 
 class SafeMIMEMultipart(MIMEMultipart):
@@ -76,8 +87,18 @@ class SafeMIMEMultipart(MIMEMultipart):
         "Forbids multi-line headers, to prevent header injection."
         if '\n' in val or '\r' in val:
             raise BadHeaderError, "Header values can't contain newlines (got %r for header %r)" % (val, name)
-        if name == "Subject":
-            val = Header(val, settings.DEFAULT_CHARSET)
+        try:
+            val = str(force_unicode(val))
+        except UnicodeEncodeError:
+            if name.lower() in ('to', 'from', 'cc'):
+                result = []
+                for item in val.split(', '):
+                    nm, addr = parseaddr(item)
+                    nm = str(Header(nm, settings.DEFAULT_CHARSET))
+                    result.append(formataddr((nm, str(addr))))
+                val = ', '.join(result)
+            else:
+                val = Header(force_unicode(val), settings.DEFAULT_CHARSET)
         MIMEMultipart.__setitem__(self, name, val)
 
 class SMTPConnection(object):
@@ -176,6 +197,14 @@ class EmailMessage(object):
 
     def __init__(self, subject='', body='', from_email=None, to=None, bcc=None,
             connection=None, attachments=None, headers=None):
+        """
+        Initialise a single email message (which can be sent to multiple
+        recipients).
+
+        All strings used to create the message can be unicode strings (or UTF-8
+        bytestrings). The SafeMIMEText class will handle any necessary encoding
+        conversions.
+        """
         self.to = to or []
         self.bcc = bcc or []
         self.from_email = from_email or settings.DEFAULT_FROM_EMAIL
@@ -192,7 +221,7 @@ class EmailMessage(object):
 
     def message(self):
         encoding = self.encoding or settings.DEFAULT_CHARSET
-        msg = SafeMIMEText(self.body, self.content_subtype, encoding)
+        msg = SafeMIMEText(smart_str(self.body, settings.DEFAULT_CHARSET), self.content_subtype, encoding)
         if self.attachments:
             body_msg = msg
             msg = SafeMIMEMultipart(_subtype=self.multipart_subtype)

django/core/management.py

@@ -427,11 +427,11 @@ def get_custom_sql_for_model(model):
     for sql_file in sql_files:
         if os.path.exists(sql_file):
             fp = open(sql_file, 'U')
-            for statement in statements.split(fp.read()):
+            for statement in statements.split(fp.read().decode(settings.FILE_CHARSET)):
                 # Remove any comments from the file
-                statement = re.sub(r"--.*[\n\Z]", "", statement)
+                statement = re.sub(ur"--.*[\n\Z]", "", statement)
                 if statement.strip():
-                    output.append(statement + ";")
+                    output.append(statement + u";")
             fp.close()
 
     return output

django/core/serializers/__init__.py

@@ -6,7 +6,7 @@ Usage::
     >>> from django.core import serializers
     >>> json = serializers.serialize("json", some_query_set)
     >>> objects = list(serializers.deserialize("json", json))
-        
+
 To add your own serializers, use the SERIALIZATION_MODULES setting::
 
     SERIALIZATION_MODULES = {
@@ -30,19 +30,19 @@ try:
     import yaml
     BUILTIN_SERIALIZERS["yaml"] = "django.core.serializers.pyyaml"
 except ImportError:
-    pass    
+    pass
 
 _serializers = {}
-        
+
 def register_serializer(format, serializer_module):
     """Register a new serializer by passing in a module name."""
     module = __import__(serializer_module, {}, {}, [''])
     _serializers[format] = module
-    
+
 def unregister_serializer(format):
     """Unregister a given serializer"""
     del _serializers[format]
-    
+
 def get_serializer(format):
     if not _serializers:
         _load_serializers()
@@ -52,12 +52,12 @@ def get_serializer_formats():
     if not _serializers:
         _load_serializers()
     return _serializers.keys()
-    
+
 def get_deserializer(format):
     if not _serializers:
         _load_serializers()
     return _serializers[format].Deserializer
-    
+
 def serialize(format, queryset, **options):
     """
     Serialize a queryset (or any iterator that returns database objects) using
@@ -87,4 +87,4 @@ def _load_serializers():
         register_serializer(format, BUILTIN_SERIALIZERS[format])
     if hasattr(settings, "SERIALIZATION_MODULES"):
         for format in settings.SERIALIZATION_MODULES:
-            register_serializer(format, settings.SERIALIZATION_MODULES[format])
+            register_serializer(format, settings.SERIALIZATION_MODULES[format])

django/core/serializers/base.py

@@ -7,6 +7,7 @@ try:
 except ImportError:
     from StringIO import StringIO
 from django.db import models
+from django.utils.encoding import smart_str, smart_unicode
 
 class SerializationError(Exception):
     """Something bad happened during serialization."""
@@ -59,7 +60,7 @@ class Serializer(object):
             value = getattr(obj, "get_%s_url" % field.name, lambda: None)()
         else:
             value = field.flatten_data(follow=None, obj=obj).get(field.name, "")
-        return str(value)
+        return smart_unicode(value)
 
     def start_serialization(self):
         """
@@ -154,7 +155,7 @@ class DeserializedObject(object):
         self.m2m_data = m2m_data
 
     def __repr__(self):
-        return "<DeserializedObject: %s>" % str(self.object)
+        return "<DeserializedObject: %s>" % smart_str(self.object)
 
     def save(self, save_m2m=True):
         self.object.save()

django/core/serializers/python.py

@@ -7,33 +7,34 @@ other serializers.
 from django.conf import settings
 from django.core.serializers import base
 from django.db import models
+from django.utils.encoding import smart_unicode
 
 class Serializer(base.Serializer):
     """
     Serializes a QuerySet to basic Python objects.
     """
-    
+
     def start_serialization(self):
         self._current = None
         self.objects = []
-        
+
     def end_serialization(self):
         pass
-        
+
     def start_object(self, obj):
         self._current = {}
-        
+
     def end_object(self, obj):
         self.objects.append({
-            "model"  : str(obj._meta),
-            "pk"     : str(obj._get_pk_val()),
+            "model"  : smart_unicode(obj._meta),
+            "pk"     : smart_unicode(obj._get_pk_val()),
             "fields" : self._current
         })
         self._current = None
-        
+
     def handle_field(self, obj, field):
         self._current[field.name] = getattr(obj, field.name)
-        
+
     def handle_fk_field(self, obj, field):
         related = getattr(obj, field.name)
         if related is not None:
@@ -44,17 +45,17 @@ class Serializer(base.Serializer):
                 # Related to remote object via other field
                 related = getattr(related, field.rel.field_name)
         self._current[field.name] = related
-    
+
     def handle_m2m_field(self, obj, field):
         self._current[field.name] = [related._get_pk_val() for related in getattr(obj, field.name).iterator()]
-    
+
     def getvalue(self):
         return self.objects
 
 def Deserializer(object_list, **options):
     """
     Deserialize simple Python objects back into Django ORM instances.
-    
+
     It's expected that you pass the Python objects themselves (instead of a
     stream or a string) to the constructor
     """
@@ -64,36 +65,30 @@ def Deserializer(object_list, **options):
         Model = _get_model(d["model"])
         data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])}
         m2m_data = {}
-        
+
         # Handle each field
         for (field_name, field_value) in d["fields"].iteritems():
-            if isinstance(field_value, unicode):
-                field_value = field_value.encode(options.get("encoding", settings.DEFAULT_CHARSET))
-                
+            if isinstance(field_value, str):
+                field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
+
             field = Model._meta.get_field(field_name)
-            
+
             # Handle M2M relations
             if field.rel and isinstance(field.rel, models.ManyToManyRel):
-                pks = []
                 m2m_convert = field.rel.to._meta.pk.to_python
-                for pk in field_value:
-                    if isinstance(pk, unicode):
-                        pks.append(m2m_convert(pk.encode(options.get("encoding", settings.DEFAULT_CHARSET))))
-                    else:
-                        pks.append(m2m_convert(pk))
-                m2m_data[field.name] = pks
-                
+                m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value]
+
             # Handle FK fields
             elif field.rel and isinstance(field.rel, models.ManyToOneRel):
                 if field_value:
                     data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
                 else:
                     data[field.attname] = None
-                    
+
             # Handle all other fields
             else:
                 data[field.name] = field.to_python(field_value)
-                
+
         yield base.DeserializedObject(Model(**data), m2m_data)
 
 def _get_model(model_identifier):
@@ -105,5 +100,5 @@ def _get_model(model_identifier):
     except TypeError:
         Model = None
     if Model is None:
-        raise base.DeserializationError("Invalid model identifier: '%s'" % model_identifier)
+        raise base.DeserializationError(u"Invalid model identifier: '%s'" % model_identifier)
     return Model

django/core/serializers/pyyaml.py

@@ -21,7 +21,7 @@ class Serializer(PythonSerializer):
         self.options.pop('stream', None)
         self.options.pop('fields', None)
         yaml.dump(self.objects, self.stream, **self.options)
-        
+
     def getvalue(self):
         return self.stream.getvalue()
 
@@ -35,4 +35,4 @@ def Deserializer(stream_or_string, **options):
         stream = stream_or_string
     for obj in PythonDeserializer(yaml.load(stream)):
         yield obj
-        
+

django/core/serializers/xml_serializer.py

@@ -6,13 +6,14 @@ from django.conf import settings
 from django.core.serializers import base
 from django.db import models
 from django.utils.xmlutils import SimplerXMLGenerator
+from django.utils.encoding import smart_unicode
 from xml.dom import pulldom
 
 class Serializer(base.Serializer):
     """
     Serializes a QuerySet to XML.
     """
-    
+
     def indent(self, level):
         if self.options.get('indent', None) is not None:
             self.xml.ignorableWhitespace('\n' + ' ' * self.options.get('indent', None) * level)
@@ -24,7 +25,7 @@ class Serializer(base.Serializer):
         self.xml = SimplerXMLGenerator(self.stream, self.options.get("encoding", settings.DEFAULT_CHARSET))
         self.xml.startDocument()
         self.xml.startElement("django-objects", {"version" : "1.0"})
-        
+
     def end_serialization(self):
         """
         End serialization -- end the document.
@@ -32,27 +33,27 @@ class Serializer(base.Serializer):
         self.indent(0)
         self.xml.endElement("django-objects")
         self.xml.endDocument()
-        
+
     def start_object(self, obj):
         """
         Called as each object is handled.
         """
         if not hasattr(obj, "_meta"):
             raise base.SerializationError("Non-model object (%s) encountered during serialization" % type(obj))
-            
+
         self.indent(1)
         self.xml.startElement("object", {
-            "pk"    : str(obj._get_pk_val()),
-            "model" : str(obj._meta),
+            "pk"    : smart_unicode(obj._get_pk_val()),
+            "model" : smart_unicode(obj._meta),
         })
-        
+
     def end_object(self, obj):
         """
         Called after handling all fields for an object.
         """
         self.indent(1)
         self.xml.endElement("object")
-        
+
     def handle_field(self, obj, field):
         """
         Called to handle each field on an object (except for ForeignKeys and
@@ -63,17 +64,17 @@ class Serializer(base.Serializer):
             "name" : field.name,
             "type" : field.get_internal_type()
         })
-        
+
         # Get a "string version" of the object's data (this is handled by the
-        # serializer base class). 
+        # serializer base class).
         if getattr(obj, field.name) is not None:
             value = self.get_string_value(obj, field)
-            self.xml.characters(str(value))
+            self.xml.characters(smart_unicode(value))
         else:
             self.xml.addQuickElement("None")
 
         self.xml.endElement("field")
-        
+
     def handle_fk_field(self, obj, field):
         """
         Called to handle a ForeignKey (we need to treat them slightly
@@ -88,11 +89,11 @@ class Serializer(base.Serializer):
             else:
                 # Related to remote object via other field
                 related = getattr(related, field.rel.field_name)
-            self.xml.characters(str(related))
+            self.xml.characters(smart_unicode(related))
         else:
             self.xml.addQuickElement("None")
         self.xml.endElement("field")
-    
+
     def handle_m2m_field(self, obj, field):
         """
         Called to handle a ManyToManyField. Related objects are only
@@ -101,9 +102,9 @@ class Serializer(base.Serializer):
         """
         self._start_relational_field(field)
         for relobj in getattr(obj, field.name).iterator():
-            self.xml.addQuickElement("object", attrs={"pk" : str(relobj._get_pk_val())})
+            self.xml.addQuickElement("object", attrs={"pk" : smart_unicode(relobj._get_pk_val())})
         self.xml.endElement("field")
-        
+
     def _start_relational_field(self, field):
         """
         Helper to output the <field> element for relational fields
@@ -112,33 +113,33 @@ class Serializer(base.Serializer):
         self.xml.startElement("field", {
             "name" : field.name,
             "rel"  : field.rel.__class__.__name__,
-            "to"   : str(field.rel.to._meta),
+            "to"   : smart_unicode(field.rel.to._meta),
         })
-        
+
 class Deserializer(base.Deserializer):
     """
     Deserialize XML.
     """
-    
+
     def __init__(self, stream_or_string, **options):
         super(Deserializer, self).__init__(stream_or_string, **options)
-        self.encoding = self.options.get("encoding", settings.DEFAULT_CHARSET)
-        self.event_stream = pulldom.parse(self.stream) 
-    
+        self.event_stream = pulldom.parse(self.stream)
+
     def next(self):
         for event, node in self.event_stream:
             if event == "START_ELEMENT" and node.nodeName == "object":
                 self.event_stream.expandNode(node)
                 return self._handle_object(node)
         raise StopIteration
-        
+
     def _handle_object(self, node):
         """
         Convert an <object> node to a DeserializedObject.
         """
-        # Look up the model using the model loading mechanism. If this fails, bail.
+        # Look up the model using the model loading mechanism. If this fails,
+        # bail.
         Model = self._get_model_from_node(node, "model")
-        
+
         # Start building a data dictionary from the object.  If the node is
         # missing the pk attribute, bail.
         pk = node.getAttribute("pk")
@@ -146,11 +147,11 @@ class Deserializer(base.Deserializer):
             raise base.DeserializationError("<object> node is missing the 'pk' attribute")
 
         data = {Model._meta.pk.attname : Model._meta.pk.to_python(pk)}
-        
+
         # Also start building a dict of m2m data (this is saved as
         # {m2m_accessor_attribute : [list_of_related_objects]})
         m2m_data = {}
-        
+
         # Deseralize each field.
         for field_node in node.getElementsByTagName("field"):
             # If the field is missing the name attribute, bail (are you
@@ -158,12 +159,12 @@ class Deserializer(base.Deserializer):
             field_name = field_node.getAttribute("name")
             if not field_name:
                 raise base.DeserializationError("<field> node is missing the 'name' attribute")
-            
+
             # Get the field from the Model. This will raise a
             # FieldDoesNotExist if, well, the field doesn't exist, which will
             # be propagated correctly.
             field = Model._meta.get_field(field_name)
-            
+
             # As is usually the case, relation fields get the special treatment.
             if field.rel and isinstance(field.rel, models.ManyToManyRel):
                 m2m_data[field.name] = self._handle_m2m_field_node(field_node, field)
@@ -173,12 +174,12 @@ class Deserializer(base.Deserializer):
                 if len(field_node.childNodes) == 1 and field_node.childNodes[0].nodeName == 'None':
                     value = None
                 else:
-                    value = field.to_python(getInnerText(field_node).strip().encode(self.encoding))
+                    value = field.to_python(getInnerText(field_node).strip())
                 data[field.name] = value
-        
+
         # Return a DeserializedObject so that the m2m data has a place to live.
         return base.DeserializedObject(Model(**data), m2m_data)
-        
+
     def _handle_fk_field_node(self, node, field):
         """
         Handle a <field> node for a ForeignKey
@@ -188,16 +189,16 @@ class Deserializer(base.Deserializer):
             return None
         else:
             return field.rel.to._meta.get_field(field.rel.field_name).to_python(
-                       getInnerText(node).strip().encode(self.encoding))
-        
+                       getInnerText(node).strip())
+
     def _handle_m2m_field_node(self, node, field):
         """
-        Handle a <field> node for a ManyToManyField
+        Handle a <field> node for a ManyToManyField.
         """
         return [field.rel.to._meta.pk.to_python(
-                    c.getAttribute("pk").encode(self.encoding)) 
+                    c.getAttribute("pk"))
                     for c in node.getElementsByTagName("object")]
-    
+
     def _get_model_from_node(self, node, attr):
         """
         Helper to look up a model from a <object model=...> or a <field
@@ -217,8 +218,8 @@ class Deserializer(base.Deserializer):
                 "<%s> node has invalid model identifier: '%s'" % \
                     (node.nodeName, model_identifier))
         return Model
-        
-    
+
+
 def getInnerText(node):
     """
     Get all the inner text of a DOM node (recursively).
@@ -232,4 +233,5 @@ def getInnerText(node):
             inner_text.extend(getInnerText(child))
         else:
            pass
-    return "".join(inner_text)
+    return u"".join(inner_text)
+

django/core/urlresolvers.py

@@ -9,6 +9,7 @@ a string) and returns a tuple in this format:
 
 from django.http import Http404
 from django.core.exceptions import ImproperlyConfigured, ViewDoesNotExist
+from django.utils.encoding import iri_to_uri
 from django.utils.functional import memoize
 import re
 
@@ -37,14 +38,20 @@ def get_callable(lookup_view, can_fail=False):
     If can_fail is True, lookup_view might be a URL pattern label, so errors
     during the import fail and the string is returned.
     """
-    if not callable(lookup_view):
-        mod_name, func_name = get_mod_func(lookup_view)
-        try:
-            if func_name != '':
-                lookup_view = getattr(__import__(mod_name, {}, {}, ['']), func_name)
-        except (ImportError, AttributeError):
-            if not can_fail:
-                raise
+    try:
+        # Bail out early if lookup_view is not ASCII. This can't be a function.
+        lookup_view = lookup_view.encode('ascii')
+
+        if not callable(lookup_view):
+            mod_name, func_name = get_mod_func(lookup_view)
+            try:
+                if func_name != '':
+                    lookup_view = getattr(__import__(mod_name, {}, {}, ['']), func_name)
+            except (ImportError, AttributeError):
+                if not can_fail:
+                    raise
+    except UnicodeEncodeError:
+        pass
     return lookup_view
 get_callable = memoize(get_callable, _callable_cache)
 
@@ -265,7 +272,7 @@ class RegexURLResolver(object):
         except (ImportError, AttributeError):
             raise NoReverseMatch
         if lookup_view in self.reverse_dict:
-            return ''.join([reverse_helper(part.regex, *args, **kwargs) for part in self.reverse_dict[lookup_view]])
+            return u''.join([reverse_helper(part.regex, *args, **kwargs) for part in self.reverse_dict[lookup_view]])
         raise NoReverseMatch
 
     def reverse_helper(self, lookup_view, *args, **kwargs):
@@ -279,5 +286,5 @@ def resolve(path, urlconf=None):
 def reverse(viewname, urlconf=None, args=None, kwargs=None):
     args = args or []
     kwargs = kwargs or {}
-    return '/' + get_resolver(urlconf).reverse(viewname, *args, **kwargs)
+    return iri_to_uri(u'/' + get_resolver(urlconf).reverse(viewname, *args, **kwargs))
 

django/core/validators.py

@@ -10,8 +10,9 @@ form field is required.
 
 import urllib2
 from django.conf import settings
-from django.utils.translation import gettext, gettext_lazy, ngettext
+from django.utils.translation import ugettext as _, ugettext_lazy, ungettext
 from django.utils.functional import Promise, lazy
+from django.utils.encoding import force_unicode
 import re
 
 _datere = r'\d{4}-\d{1,2}-\d{1,2}'
@@ -32,16 +33,17 @@ phone_re = re.compile(r'^[A-PR-Y0-9]{3}-[A-PR-Y0-9]{3}-[A-PR-Y0-9]{4}$', re.IGNO
 slug_re = re.compile(r'^[-\w]+$')
 url_re = re.compile(r'^https?://\S+$')
 
-lazy_inter = lazy(lambda a,b: str(a) % b, str)
+lazy_inter = lazy(lambda a,b: force_unicode(a) % b, unicode)
 
 class ValidationError(Exception):
     def __init__(self, message):
         "ValidationError can be passed a string or a list."
         if isinstance(message, list):
-            self.messages = message
+            self.messages = [force_unicode(msg) for msg in message]
         else:
             assert isinstance(message, (basestring, Promise)), ("%s should be a string" % repr(message))
-            self.messages = [message]
+            self.messages = [force_unicode(message)]
+
     def __str__(self):
         # This is needed because, without a __str__(), printing an exception
         # instance would result in this:
@@ -53,39 +55,40 @@ class CriticalValidationError(Exception):
     def __init__(self, message):
         "ValidationError can be passed a string or a list."
         if isinstance(message, list):
-            self.messages = message
+            self.messages = [force_unicode(msg) for msg in message]
         else:
             assert isinstance(message, (basestring, Promise)), ("'%s' should be a string" % message)
-            self.messages = [message]
+            self.messages = [force_unicode(message)]
+
     def __str__(self):
         return str(self.messages)
 
 def isAlphaNumeric(field_data, all_data):
     if not alnum_re.search(field_data):
-        raise ValidationError, gettext("This value must contain only letters, numbers and underscores.")
+        raise ValidationError, _("This value must contain only letters, numbers and underscores.")
 
 def isAlphaNumericURL(field_data, all_data):
     if not alnumurl_re.search(field_data):
-        raise ValidationError, gettext("This value must contain only letters, numbers, underscores, dashes or slashes.")
+        raise ValidationError, _("This value must contain only letters, numbers, underscores, dashes or slashes.")
 
 def isSlug(field_data, all_data):
     if not slug_re.search(field_data):
-        raise ValidationError, gettext("This value must contain only letters, numbers, underscores or hyphens.")
+        raise ValidationError, _("This value must contain only letters, numbers, underscores or hyphens.")
 
 def isLowerCase(field_data, all_data):
     if field_data.lower() != field_data:
-        raise ValidationError, gettext("Uppercase letters are not allowed here.")
+        raise ValidationError, _("Uppercase letters are not allowed here.")
 
 def isUpperCase(field_data, all_data):
     if field_data.upper() != field_data:
-        raise ValidationError, gettext("Lowercase letters are not allowed here.")
+        raise ValidationError, _("Lowercase letters are not allowed here.")
 
 def isCommaSeparatedIntegerList(field_data, all_data):
     for supposed_int in field_data.split(','):
         try:
             int(supposed_int)
         except ValueError:
-            raise ValidationError, gettext("Enter only digits separated by commas.")
+            raise ValidationError, _("Enter only digits separated by commas.")
 
 def isCommaSeparatedEmailList(field_data, all_data):
     """
@@ -97,32 +100,32 @@ def isCommaSeparatedEmailList(field_data, all_data):
         try:
             isValidEmail(supposed_email.strip(), '')
         except ValidationError:
-            raise ValidationError, gettext("Enter valid e-mail addresses separated by commas.")
+            raise ValidationError, _("Enter valid e-mail addresses separated by commas.")
 
 def isValidIPAddress4(field_data, all_data):
     if not ip4_re.search(field_data):
-        raise ValidationError, gettext("Please enter a valid IP address.")
+        raise ValidationError, _("Please enter a valid IP address.")
 
 def isNotEmpty(field_data, all_data):
     if field_data.strip() == '':
-        raise ValidationError, gettext("Empty values are not allowed here.")
+        raise ValidationError, _("Empty values are not allowed here.")
 
 def isOnlyDigits(field_data, all_data):
     if not field_data.isdigit():
-        raise ValidationError, gettext("Non-numeric characters aren't allowed here.")
+        raise ValidationError, _("Non-numeric characters aren't allowed here.")
 
 def isNotOnlyDigits(field_data, all_data):
     if field_data.isdigit():
-        raise ValidationError, gettext("This value can't be comprised solely of digits.")
+        raise ValidationError, _("This value can't be comprised solely of digits.")
 
 def isInteger(field_data, all_data):
     # This differs from isOnlyDigits because this accepts the negative sign
     if not integer_re.search(field_data):
-        raise ValidationError, gettext("Enter a whole number.")
+        raise ValidationError, _("Enter a whole number.")
 
 def isOnlyLetters(field_data, all_data):
     if not field_data.isalpha():
-        raise ValidationError, gettext("Only alphabetical characters are allowed here.")
+        raise ValidationError, _("Only alphabetical characters are allowed here.")
 
 def _isValidDate(date_string):
     """
@@ -137,30 +140,30 @@ def _isValidDate(date_string):
     # This check is needed because strftime is used when saving the date
     # value to the database, and strftime requires that the year be >=1900.
     if year < 1900:
-        raise ValidationError, gettext('Year must be 1900 or later.')
+        raise ValidationError, _('Year must be 1900 or later.')
     try:
         date(year, month, day)
     except ValueError, e:
-        msg = gettext('Invalid date: %s') % gettext(str(e))
+        msg = _('Invalid date: %s') % _(str(e))
         raise ValidationError, msg    
 
 def isValidANSIDate(field_data, all_data):
     if not ansi_date_re.search(field_data):
-        raise ValidationError, gettext('Enter a valid date in YYYY-MM-DD format.')
+        raise ValidationError, _('Enter a valid date in YYYY-MM-DD format.')
     _isValidDate(field_data)
 
 def isValidANSITime(field_data, all_data):
     if not ansi_time_re.search(field_data):
-        raise ValidationError, gettext('Enter a valid time in HH:MM format.')
+        raise ValidationError, _('Enter a valid time in HH:MM format.')
 
 def isValidANSIDatetime(field_data, all_data):
     if not ansi_datetime_re.search(field_data):
-        raise ValidationError, gettext('Enter a valid date/time in YYYY-MM-DD HH:MM format.')
+        raise ValidationError, _('Enter a valid date/time in YYYY-MM-DD HH:MM format.')
     _isValidDate(field_data.split()[0])
 
 def isValidEmail(field_data, all_data):
     if not email_re.search(field_data):
-        raise ValidationError, gettext('Enter a valid e-mail address.')
+        raise ValidationError, _('Enter a valid e-mail address.')
 
 def isValidImage(field_data, all_data):
     """
@@ -172,22 +175,22 @@ def isValidImage(field_data, all_data):
     try:
         content = field_data['content']
     except TypeError:
-        raise ValidationError, gettext("No file was submitted. Check the encoding type on the form.")
+        raise ValidationError, _("No file was submitted. Check the encoding type on the form.")
     try:
         Image.open(StringIO(content))
     except IOError: # Python Imaging Library doesn't recognize it as an image
-        raise ValidationError, gettext("Upload a valid image. The file you uploaded was either not an image or a corrupted image.")
+        raise ValidationError, _("Upload a valid image. The file you uploaded was either not an image or a corrupted image.")
 
 def isValidImageURL(field_data, all_data):
     uc = URLMimeTypeCheck(('image/jpeg', 'image/gif', 'image/png'))
     try:
         uc(field_data, all_data)
     except URLMimeTypeCheck.InvalidContentType:
-        raise ValidationError, gettext("The URL %s does not point to a valid image.") % field_data
+        raise ValidationError, _("The URL %s does not point to a valid image.") % field_data
 
 def isValidPhone(field_data, all_data):
     if not phone_re.search(field_data):
-        raise ValidationError, gettext('Phone numbers must be in XXX-XXX-XXXX format. "%s" is invalid.') % field_data
+        raise ValidationError, _('Phone numbers must be in XXX-XXX-XXXX format. "%s" is invalid.') % field_data
 
 def isValidQuicktimeVideoURL(field_data, all_data):
     "Checks that the given URL is a video that can be played by QuickTime (qt, mpeg)"
@@ -195,11 +198,11 @@ def isValidQuicktimeVideoURL(field_data, all_data):
     try:
         uc(field_data, all_data)
     except URLMimeTypeCheck.InvalidContentType:
-        raise ValidationError, gettext("The URL %s does not point to a valid QuickTime video.") % field_data
+        raise ValidationError, _("The URL %s does not point to a valid QuickTime video.") % field_data
 
 def isValidURL(field_data, all_data):
     if not url_re.search(field_data):
-        raise ValidationError, gettext("A valid URL is required.")
+        raise ValidationError, _("A valid URL is required.")
 
 def isValidHTML(field_data, all_data):
     import urllib, urllib2
@@ -213,14 +216,14 @@ def isValidHTML(field_data, all_data):
         return
     from xml.dom.minidom import parseString
     error_messages = [e.firstChild.wholeText for e in parseString(u.read()).getElementsByTagName('messages')[0].getElementsByTagName('msg')]
-    raise ValidationError, gettext("Valid HTML is required. Specific errors are:\n%s") % "\n".join(error_messages)
+    raise ValidationError, _("Valid HTML is required. Specific errors are:\n%s") % "\n".join(error_messages)
 
 def isWellFormedXml(field_data, all_data):
     from xml.dom.minidom import parseString
     try:
         parseString(field_data)
     except Exception, e: # Naked except because we're not sure what will be thrown
-        raise ValidationError, gettext("Badly formed XML: %s") % str(e)
+        raise ValidationError, _("Badly formed XML: %s") % str(e)
 
 def isWellFormedXmlFragment(field_data, all_data):
     isWellFormedXml('<root>%s</root>' % field_data, all_data)
@@ -250,7 +253,7 @@ def isValidUSState(field_data, all_data):
     "Checks that the given string is a valid two-letter U.S. state abbreviation"
     states = ['AA', 'AE', 'AK', 'AL', 'AP', 'AR', 'AS', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'FM', 'GA', 'GU', 'HI', 'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME', 'MH', 'MI', 'MN', 'MO', 'MP', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ', 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'PR', 'PW', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VA', 'VI', 'VT', 'WA', 'WI', 'WV', 'WY']
     if field_data.upper() not in states:
-        raise ValidationError, gettext("Enter a valid U.S. state abbreviation.")
+        raise ValidationError, _("Enter a valid U.S. state abbreviation.")
 
 def hasNoProfanities(field_data, all_data):
     """
@@ -264,14 +267,14 @@ def hasNoProfanities(field_data, all_data):
     if words_seen:
         from django.utils.text import get_text_list
         plural = len(words_seen) > 1
-        raise ValidationError, ngettext("Watch your mouth! The word %s is not allowed here.",
+        raise ValidationError, ungettext("Watch your mouth! The word %s is not allowed here.",
             "Watch your mouth! The words %s are not allowed here.", plural) % \
             get_text_list(['"%s%s%s"' % (i[0], '-'*(len(i)-2), i[-1]) for i in words_seen], 'and')
 
 class AlwaysMatchesOtherField(object):
     def __init__(self, other_field_name, error_message=None):
         self.other = other_field_name
-        self.error_message = error_message or lazy_inter(gettext_lazy("This field must match the '%s' field."), self.other)
+        self.error_message = error_message or lazy_inter(ugettext_lazy("This field must match the '%s' field."), self.other)
         self.always_test = True
 
     def __call__(self, field_data, all_data):
@@ -290,7 +293,7 @@ class ValidateIfOtherFieldEquals(object):
                 v(field_data, all_data)
 
 class RequiredIfOtherFieldNotGiven(object):
-    def __init__(self, other_field_name, error_message=gettext_lazy("Please enter something for at least one field.")):
+    def __init__(self, other_field_name, error_message=ugettext_lazy("Please enter something for at least one field.")):
         self.other, self.error_message = other_field_name, error_message
         self.always_test = True
 
@@ -299,7 +302,7 @@ class RequiredIfOtherFieldNotGiven(object):
             raise ValidationError, self.error_message
 
 class RequiredIfOtherFieldsGiven(object):
-    def __init__(self, other_field_names, error_message=gettext_lazy("Please enter both fields or leave them both empty.")):
+    def __init__(self, other_field_names, error_message=ugettext_lazy("Please enter both fields or leave them both empty.")):
         self.other, self.error_message = other_field_names, error_message
         self.always_test = True
 
@@ -310,7 +313,7 @@ class RequiredIfOtherFieldsGiven(object):
 
 class RequiredIfOtherFieldGiven(RequiredIfOtherFieldsGiven):
     "Like RequiredIfOtherFieldsGiven, but takes a single field name instead of a list."
-    def __init__(self, other_field_name, error_message=gettext_lazy("Please enter both fields or leave them both empty.")):
+    def __init__(self, other_field_name, error_message=ugettext_lazy("Please enter both fields or leave them both empty.")):
         RequiredIfOtherFieldsGiven.__init__(self, [other_field_name], error_message)
 
 class RequiredIfOtherFieldEquals(object):
@@ -318,7 +321,7 @@ class RequiredIfOtherFieldEquals(object):
         self.other_field = other_field
         self.other_value = other_value
         other_label = other_label or other_value
-        self.error_message = error_message or lazy_inter(gettext_lazy("This field must be given if %(field)s is %(value)s"), {
+        self.error_message = error_message or lazy_inter(ugettext_lazy("This field must be given if %(field)s is %(value)s"), {
             'field': other_field, 'value': other_label})
         self.always_test = True
 
@@ -331,7 +334,7 @@ class RequiredIfOtherFieldDoesNotEqual(object):
         self.other_field = other_field
         self.other_value = other_value
         other_label = other_label or other_value
-        self.error_message = error_message or lazy_inter(gettext_lazy("This field must be given if %(field)s is not %(value)s"), {
+        self.error_message = error_message or lazy_inter(ugettext_lazy("This field must be given if %(field)s is not %(value)s"), {
             'field': other_field, 'value': other_label})
         self.always_test = True
 
@@ -350,7 +353,7 @@ class IsLessThanOtherField(object):
 class UniqueAmongstFieldsWithPrefix(object):
     def __init__(self, field_name, prefix, error_message):
         self.field_name, self.prefix = field_name, prefix
-        self.error_message = error_message or gettext_lazy("Duplicate values are not allowed.")
+        self.error_message = error_message or ugettext_lazy("Duplicate values are not allowed.")
 
     def __call__(self, field_data, all_data):
         for field_name, value in all_data.items():
@@ -365,11 +368,11 @@ class NumberIsInRange(object):
         self.lower, self.upper = lower, upper
         if not error_message:
             if lower and upper:
-                 self.error_message = gettext("This value must be between %(lower)s and %(upper)s.") % {'lower': lower, 'upper': upper}
+                 self.error_message = _("This value must be between %(lower)s and %(upper)s.") % {'lower': lower, 'upper': upper}
             elif lower:
-                self.error_message = gettext("This value must be at least %s.") % lower
+                self.error_message = _("This value must be at least %s.") % lower
             elif upper:
-                self.error_message = gettext("This value must be no more than %s.") % upper
+                self.error_message = _("This value must be no more than %s.") % upper
         else:
             self.error_message = error_message
 
@@ -405,7 +408,7 @@ class IsAPowerOf(object):
         from math import log
         val = log(int(field_data)) / log(self.power_of)
         if val != int(val):
-            raise ValidationError, gettext("This value must be a power of %s.") % self.power_of
+            raise ValidationError, _("This value must be a power of %s.") % self.power_of
 
 class IsValidDecimal(object):
     def __init__(self, max_digits, decimal_places):
@@ -414,19 +417,19 @@ class IsValidDecimal(object):
     def __call__(self, field_data, all_data):
         match = decimal_re.search(str(field_data))
         if not match:
-            raise ValidationError, gettext("Please enter a valid decimal number.")
+            raise ValidationError, _("Please enter a valid decimal number.")
         
         digits = len(match.group('digits') or '')
         decimals = len(match.group('decimals') or '')
         
         if digits + decimals > self.max_digits:
-            raise ValidationError, ngettext("Please enter a valid decimal number with at most %s total digit.",
+            raise ValidationError, ungettext("Please enter a valid decimal number with at most %s total digit.",
                 "Please enter a valid decimal number with at most %s total digits.", self.max_digits) % self.max_digits
         if digits > (self.max_digits - self.decimal_places):
-            raise ValidationError, ngettext( "Please enter a valid decimal number with a whole part of at most %s digit.",
+            raise ValidationError, ungettext( "Please enter a valid decimal number with a whole part of at most %s digit.",
                 "Please enter a valid decimal number with a whole part of at most %s digits.", str(self.max_digits-self.decimal_places)) % str(self.max_digits-self.decimal_places)
         if decimals > self.decimal_places:
-            raise ValidationError, ngettext("Please enter a valid decimal number with at most %s decimal place.",
+            raise ValidationError, ungettext("Please enter a valid decimal number with at most %s decimal place.",
                 "Please enter a valid decimal number with at most %s decimal places.", self.decimal_places) % self.decimal_places
 
 def isValidFloat(field_data, all_data):
@@ -434,7 +437,7 @@ def isValidFloat(field_data, all_data):
     try:
         float(data)
     except ValueError:
-        raise ValidationError, gettext("Please enter a valid floating point number.")
+        raise ValidationError, ugettext("Please enter a valid floating point number.")
 
 class HasAllowableSize(object):
     """
@@ -443,14 +446,14 @@ class HasAllowableSize(object):
     """
     def __init__(self, min_size=None, max_size=None, min_error_message=None, max_error_message=None):
         self.min_size, self.max_size = min_size, max_size
-        self.min_error_message = min_error_message or lazy_inter(gettext_lazy("Make sure your uploaded file is at least %s bytes big."), min_size)
-        self.max_error_message = max_error_message or lazy_inter(gettext_lazy("Make sure your uploaded file is at most %s bytes big."), max_size)
+        self.min_error_message = min_error_message or lazy_inter(ugettext_lazy("Make sure your uploaded file is at least %s bytes big."), min_size)
+        self.max_error_message = max_error_message or lazy_inter(ugettext_lazy("Make sure your uploaded file is at most %s bytes big."), max_size)
 
     def __call__(self, field_data, all_data):
         try:
             content = field_data['content']
         except TypeError:
-            raise ValidationError, gettext_lazy("No file was submitted. Check the encoding type on the form.")
+            raise ValidationError, ugettext_lazy("No file was submitted. Check the encoding type on the form.")
         if self.min_size is not None and len(content) < self.min_size:
             raise ValidationError, self.min_error_message
         if self.max_size is not None and len(content) > self.max_size:
@@ -461,7 +464,7 @@ class MatchesRegularExpression(object):
     Checks that the field matches the given regular-expression. The regex
     should be in string format, not already compiled.
     """
-    def __init__(self, regexp, error_message=gettext_lazy("The format for this field is wrong.")):
+    def __init__(self, regexp, error_message=ugettext_lazy("The format for this field is wrong.")):
         self.regexp = re.compile(regexp)
         self.error_message = error_message
 
@@ -476,7 +479,7 @@ class AnyValidator(object):
     as a validation error. The message is rather unspecific, so it's best to
     specify one on instantiation.
     """
-    def __init__(self, validator_list=None, error_message=gettext_lazy("This field is invalid.")):
+    def __init__(self, validator_list=None, error_message=ugettext_lazy("This field is invalid.")):
         if validator_list is None: validator_list = []
         self.validator_list = validator_list
         self.error_message = error_message
@@ -512,10 +515,10 @@ class URLMimeTypeCheck(object):
         try:
             info = urllib2.urlopen(field_data).info()
         except (urllib2.HTTPError, urllib2.URLError):
-            raise URLMimeTypeCheck.CouldNotRetrieve, gettext("Could not retrieve anything from %s.") % field_data
+            raise URLMimeTypeCheck.CouldNotRetrieve, _("Could not retrieve anything from %s.") % field_data
         content_type = info['content-type']
         if content_type not in self.mime_type_list:
-            raise URLMimeTypeCheck.InvalidContentType, gettext("The URL %(url)s returned the invalid Content-Type header '%(contenttype)s'.") % {
+            raise URLMimeTypeCheck.InvalidContentType, _("The URL %(url)s returned the invalid Content-Type header '%(contenttype)s'.") % {
                 'url': field_data, 'contenttype': content_type}
 
 class RelaxNGCompact(object):