unicode: Reverted [5388] and fixed the problem in a different way. Checked

every occurrence of smart_unicode() and force_unicode() that was not previously
a str() call, so hopefully the problems will not reoccur. Fixed #4447. Refs #4435, #4430.


git-svn-id: http://code.djangoproject.com/svn/django/branches/unicode@5400 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/contrib/admin/views/main.py

@@ -142,7 +142,7 @@ class AdminBoundField(object):
             return self._display
         except AttributeError:
             if isinstance(self.field.rel, models.ManyToOneRel):
-                self._display = force_unicode(getattr(self.original, self.field.name))
+                self._display = force_unicode(getattr(self.original, self.field.name), strings_only=True)
             elif isinstance(self.field.rel, models.ManyToManyRel):
                 self._display = u", ".join([force_unicode(obj) for obj in getattr(self.original, self.field.name).all()])
             return self._display

django/core/mail.py

@@ -3,7 +3,7 @@ Tools for sending email.
 """
 
 from django.conf import settings
-from django.utils.encoding import smart_str, smart_unicode
+from django.utils.encoding import smart_str, force_unicode
 from email.MIMEText import MIMEText
 from email.Header import Header
 from email.Utils import formatdate, parseaddr, formataddr
@@ -62,7 +62,7 @@ class SafeMIMEText(MIMEText):
         if '\n' in val or '\r' in val:
             raise BadHeaderError, "Header values can't contain newlines (got %r for header %r)" % (val, name)
         try:
-            val = str(smart_unicode(val))
+            val = str(force_unicode(val))
         except UnicodeEncodeError:
             if name.lower() in ('to', 'from', 'cc'):
                 result = []
@@ -72,7 +72,7 @@ class SafeMIMEText(MIMEText):
                     result.append(formataddr((nm, str(addr))))
                 val = ', '.join(result)
             else:
-                val = Header(smart_unicode(val), settings.DEFAULT_CHARSET)
+                val = Header(force_unicode(val), settings.DEFAULT_CHARSET)
         MIMEText.__setitem__(self, name, val)
 
 class SMTPConnection(object):

django/core/serializers/python.py

@@ -69,7 +69,7 @@ def Deserializer(object_list, **options):
         # Handle each field
         for (field_name, field_value) in d["fields"].iteritems():
             if isinstance(field_value, str):
-                field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET))
+                field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
 
             field = Model._meta.get_field(field_name)
 

django/db/backends/util.py

@@ -1,6 +1,6 @@
 import datetime
 from time import time
-from django.utils.encoding import smart_unicode
+from django.utils.encoding import smart_unicode, force_unicode
 
 try:
     import decimal
@@ -44,10 +44,11 @@ def convert_args(args):
     """
     Convert sequence or dictionary to contain unicode values.
     """
+    to_unicode = lambda s: force_unicode(s, strings_only=True)
     if isinstance(args, (list, tuple)):
-        return tuple([smart_unicode(val) for val in args])
+        return tuple([to_unicode(val) for val in args])
     else:
-        return dict([(smart_unicode(k), smart_unicode(v)) for k, v in args.items()])
+        return dict([(to_unicode(k), to_unicode(v)) for k, v in args.items()])
 
 ###############################################
 # Converters from database (string) to Python #

django/db/models/base.py

@@ -320,7 +320,7 @@ class Model(object):
 
     def _get_FIELD_display(self, field):
         value = getattr(self, field.attname)
-        return force_unicode(dict(field.choices).get(value, value))
+        return force_unicode(dict(field.choices).get(value, value), strings_only=True)
 
     def _get_next_or_previous_by_FIELD(self, field, is_next, **kwargs):
         op = is_next and '>' or '<'

django/oldforms/__init__.py

@@ -479,7 +479,7 @@ class SelectField(FormField):
     def __init__(self, field_name, choices=None, size=1, is_required=False, validator_list=None, member_name=None):
         if validator_list is None: validator_list = []
         if choices is None: choices = []
-        choices = [(k, smart_unicode(v)) for k, v in choices]
+        choices = [(k, smart_unicode(v, strings_only=True)) for k, v in choices]
         self.field_name = field_name
         # choices is a list of (value, human-readable key) tuples because order matters
         self.choices, self.size, self.is_required = choices, size, is_required

django/utils/encoding.py

@@ -12,22 +12,26 @@ class StrAndUnicode(object):
     def __str__(self):
         return self.__unicode__().encode('utf-8')
 
-def smart_unicode(s, encoding='utf-8', errors='strict'):
+def smart_unicode(s, encoding='utf-8', strings_only=False, errors='strict'):
     """
     Returns a unicode object representing 's'. Treats bytestrings using the
     'encoding' codec.
+
+    If strings_only is True, don't convert (some) non-string-like objects.
     """
     if isinstance(s, Promise):
         # The input is the result of a gettext_lazy() call.
         return s
-    return force_unicode(s, encoding, errors)
+    return force_unicode(s, encoding, strings_only, errors)
 
-def force_unicode(s, encoding='utf-8', errors='strict'):
+def force_unicode(s, encoding='utf-8', strings_only=False, errors='strict'):
     """
     Similar to smart_unicode, except that lazy instances are resolved to
     strings, rather than kept as lazy objects.
+
+    If strings_only is True, don't convert (some) non-string-like objects.
     """
-    if s is None:
+    if strings_only and isinstance(s, (types.NoneType, int)):
         return s
     if not isinstance(s, basestring,):
         if hasattr(s, '__unicode__'):

django/utils/feedgenerator.py

@@ -42,20 +42,21 @@ class SyndicationFeed(object):
     def __init__(self, title, link, description, language=None, author_email=None,
             author_name=None, author_link=None, subtitle=None, categories=None,
             feed_url=None, feed_copyright=None):
+        to_unicode = lambda s: force_unicode(s, strings_only=True)
         if categories:
             categories = [force_unicode(c) for c in categories]
         self.feed = {
-            'title': force_unicode(title),
+            'title': to_unicode(title),
             'link': iri_to_uri(link),
-            'description': force_unicode(description),
+            'description': to_unicode(description),
             'language': force_unicode(language),
-            'author_email': force_unicode(author_email),
+            'author_email': to_unicode(author_email),
-            'author_name': force_unicode(author_name),
+            'author_name': to_unicode(author_name),
             'author_link': iri_to_uri(author_link),
-            'subtitle': force_unicode(subtitle),
+            'subtitle': to_unicode(subtitle),
             'categories': categories or (),
             'feed_url': iri_to_uri(feed_url),
-            'feed_copyright': force_unicode(feed_copyright),
+            'feed_copyright': to_unicode(feed_copyright),
         }
         self.items = []
 
@@ -67,21 +68,22 @@ class SyndicationFeed(object):
         objects except pubdate, which is a datetime.datetime object, and
         enclosure, which is an instance of the Enclosure class.
         """
+        to_unicode = lambda s: force_unicode(s, strings_only=True)
         if categories:
-            categories = [force_unicode(c) for c in categories]
+            categories = [to_unicode(c) for c in categories]
         self.items.append({
-            'title': force_unicode(title),
+            'title': to_unicode(title),
             'link': iri_to_uri(link),
-            'description': force_unicode(description),
+            'description': to_unicode(description),
-            'author_email': force_unicode(author_email),
+            'author_email': to_unicode(author_email),
-            'author_name': force_unicode(author_name),
+            'author_name': to_unicode(author_name),
             'author_link': iri_to_uri(author_link),
             'pubdate': pubdate,
-            'comments': force_unicode(comments),
+            'comments': to_unicode(comments),
-            'unique_id': force_unicode(unique_id),
+            'unique_id': to_unicode(unique_id),
             'enclosure': enclosure,
             'categories': categories or (),
-            'item_copyright': force_unicode(item_copyright),
+            'item_copyright': to_unicode(item_copyright),
         })
 
     def num_items(self):

tests/regressiontests/model_regress/models.py

@@ -1,9 +1,15 @@
 # coding: utf-8
 from django.db import models
 
+CHOICES = (
+    (1, 'first'),
+    (2, 'second'),
+)
+
 class Article(models.Model):
     headline = models.CharField(maxlength=100, default='Default headline')
     pub_date = models.DateTimeField()
+    status = models.IntegerField(blank=True, null=True, choices=CHOICES)
 
     class Meta:
         ordering = ('pub_date','headline')
@@ -13,4 +19,16 @@ class Article(models.Model):
     def __unicode__(self):
         return self.headline
 
-__test__ = {'API_TESTS': "" }
+__test__ = {'API_TESTS': """
+(NOTE: Part of the regression test here is merely parsing the model
+declaration. The verbose_name, in particular, did not always work.)
+
+An empty choice field should return None for the display name.
+
+>>> from datetime import datetime
+>>> a = Article(headline="Look at me!", pub_date=datetime.now())
+>>> a.save()
+>>> a.get_status_display() is None
+True
+"""
+}