unicode: Changed the way re-encoding of form field submission works so that

file uploads are no longer completely broken. Added tests for this as well.


git-svn-id: http://code.djangoproject.com/svn/django/branches/unicode@5464 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/http/__init__.py

@@ -3,7 +3,7 @@ from Cookie import SimpleCookie
 from pprint import pformat
 from urllib import urlencode
 from django.utils.datastructures import MultiValueDict
-from django.utils.encoding import smart_str, iri_to_uri
+from django.utils.encoding import smart_str, iri_to_uri, force_unicode
 
 RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
 
@@ -49,13 +49,15 @@ class HttpRequest(object):
 
     def _set_encoding(self, val):
         """
-        Sets the encoding used for GET/POST accesses.
+        Sets the encoding used for GET/POST accesses. If the GET or POST
+        dictionary has already been created it is removed and recreated on the
+        next access (so that it is decoded correctly).
         """
         self._encoding = val
         if hasattr(self, '_get'):
-            self.GET.encoding = val
+            del self._get
         if hasattr(self, '_post'):
-            self.POST.encoding = val
+            del self._post
 
     def _get_encoding(self):
         return self._encoding
@@ -113,27 +115,23 @@ class QueryDict(MultiValueDict):
             self.encoding = encoding
         self._mutable = True
         for key, value in parse_qsl((query_string or ''), True): # keep_blank_values=True
-            self.appendlist(key, value)
+            self.appendlist(force_unicode(key, errors='replace'), force_unicode(value, errors='replace'))
         self._mutable = mutable
 
     def _assert_mutable(self):
         if not self._mutable:
             raise AttributeError, "This QueryDict instance is immutable"
 
-    def __getitem__(self, key):
-        return str_to_unicode(MultiValueDict.__getitem__(self, key), self.encoding)
-
     def __setitem__(self, key, value):
         self._assert_mutable()
+        key = str_to_unicode(key, self.encoding)
+        value = str_to_unicode(value, self.encoding)
         MultiValueDict.__setitem__(self, key, value)
 
     def __delitem__(self, key):
         self._assert_mutable()
         super(QueryDict, self).__delitem__(key)
 
-    def get(self, key, default=None):
-        return str_to_unicode(MultiValueDict.get(self, key, default), self.encoding)
-
     def __copy__(self):
         result = self.__class__('', mutable=True)
         for key, value in dict.items(self):
@@ -148,16 +146,10 @@ class QueryDict(MultiValueDict):
             dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo))
         return result
 
-    def getlist(self, key):
-        """
-        Returns a copy of the list associated with "key". This isn't a
-        reference to the original list because this method converts all the
-        values to unicode (without changing the original).
-        """
-        return [str_to_unicode(v, self.encoding) for v in MultiValueDict.getlist(self, key)]
-
     def setlist(self, key, list_):
         self._assert_mutable()
+        key = str_to_unicode(key, self.encoding)
+        list_ = [str_to_unicode(elt, self.encoding) for elt in list_]
         MultiValueDict.setlist(self, key, list_)
 
     def setlistdefault(self, key, default_list=()):
@@ -168,43 +160,33 @@ class QueryDict(MultiValueDict):
 
     def appendlist(self, key, value):
         self._assert_mutable()
+        key = str_to_unicode(key, self.encoding)
+        value = str_to_unicode(value, self.encoding)
         MultiValueDict.appendlist(self, key, value)
 
     def update(self, other_dict):
         self._assert_mutable()
-        MultiValueDict.update(self, other_dict)
+        f = lambda s: str_to_unicode(s, self.encoding)
+        d = dict([(f(k), f(v)) for k, v in other_dict.items()])
+        MultiValueDict.update(self, d)
 
     def pop(self, key, *args):
         self._assert_mutable()
-        val = MultiValueDict.pop(self, key, *args)
-        if isinstance(val, list):
-            return [str_to_unicode(v, self.encoding) for v in val]
-        return str_to_unicode(val, self.encoding)
+        return MultiValueDict.pop(self, key, *args)
 
     def popitem(self):
         self._assert_mutable()
-        key, values = MultiValueDict.popitem(self)
-        return str_to_unicode(key, self.encoding), [str_to_unicode(v, self.encoding) for v in values]
-
-    def keys(self):
-        return [str_to_unicode(k, self.encoding) for k in MultiValueDict.keys(self)]
-
-    def values(self):
-        return [str_to_unicode(v, self.encoding) for v in MultiValueDict.values(self)]
-
-    def items(self):
-        return [(str_to_unicode(k, self.encoding), str_to_unicode(v, self.encoding)) for k, v in MultiValueDict.items(self)]
-
-    def lists(self):
-        return [(str_to_unicode(k, self.encoding), [str_to_unicode(v, self.encoding) for v in v_list]) for k, v_list in MultiValueDict.lists(self)]
+        return MultiValueDict.popitem(self)
 
     def clear(self):
         self._assert_mutable()
         MultiValueDict.clear(self)
 
-    def setdefault(self, *args):
+    def setdefault(self, key, default=None):
         self._assert_mutable()
-        return MultiValueDict.setdefault(self, *args)
+        key = str_to_unicode(key, self.encoding)
+        default = str_to_unicode(default, self.encoding)
+        return MultiValueDict.setdefault(self, key, default)
 
     def copy(self):
         "Returns a mutable copy of this object."

django/test/client.py

@@ -68,10 +68,7 @@ def encode_multipart(boundary, data):
         if isinstance(value, file):
             lines.extend([
                 '--' + boundary,
-                'Content-Disposition: form-data; name="%s"' % to_str(key),
-                '',
-                '--' + boundary,
-                'Content-Disposition: form-data; name="%s_file"; filename="%s"' % (to_str(key), to_str(value.name)),
+                'Content-Disposition: form-data; name="%s"; filename="%s"' % (to_str(key), to_str(value.name)),
                 'Content-Type: application/octet-stream',
                 '',
                 value.read()

tests/regressiontests/httpwrappers/tests.py

@@ -16,7 +16,7 @@ Traceback (most recent call last):
 AttributeError: This QueryDict instance is immutable
 
 >>> q.get('foo', 'default')
-u'default'
+'default'
 
 >>> q.getlist('foo')
 []
@@ -103,7 +103,7 @@ False
 >>> q['name'] = 'john'
 
 >>> q.get('foo', 'default')
-u'default'
+'default'
 
 >>> q.get('name', 'default')
 u'john'
@@ -167,10 +167,10 @@ u'hello'
 [u'bar', u'baz', u'another', u'hello']
 
 >>> q.pop('foo', 'not there')
-u'not there'
+'not there'
 
 >>> q.get('foo', 'not there')
-u'not there'
+'not there'
 
 >>> q.setdefault('foo', 'bar')
 u'bar'
@@ -201,7 +201,7 @@ u'bar'
 >>> q['bar']
 Traceback (most recent call last):
 ...
-MultiValueDictKeyError: "Key 'bar' not found in <MultiValueDict: {'foo': ['bar']}>"
+MultiValueDictKeyError: "Key 'bar' not found in <MultiValueDict: {u'foo': [u'bar']}>"
 
 >>> q['something'] = 'bar'
 Traceback (most recent call last):
@@ -212,7 +212,7 @@ AttributeError: This QueryDict instance is immutable
 u'bar'
 
 >>> q.get('bar', 'default')
-u'default'
+'default'
 
 >>> q.getlist('foo')
 [u'bar']
@@ -303,7 +303,7 @@ AttributeError: This QueryDict instance is immutable
 u'no'
 
 >>> q.get('foo', 'default')
-u'default'
+'default'
 
 >>> q.getlist('vote')
 [u'yes', u'no']

tests/regressiontests/test_client_regress/models.py

@@ -4,6 +4,7 @@ Regression tests for the Test Client, especially the customized assertions.
 """
 from django.test import Client, TestCase
 from django.core import mail
+import os
 
 class AssertTemplateUsedTests(TestCase):
     fixtures = ['testdata.json']
@@ -162,3 +163,12 @@ class AssertFormErrorTests(TestCase):
         except AssertionError, e:
             self.assertEqual(str(e), "The field 'email' on form 'form' in context 0 does not contain the error 'Some error.' (actual errors: [u'Enter a valid e-mail address.'])")
 
+class AssertFileUploadTests(TestCase):
+    def test_simple_upload(self):
+        fd = open(os.path.join(os.path.dirname(__file__), "views.py"))
+        post_data = {
+            'name': 'Ringo',
+            'file_field': fd,
+        }
+        response = self.client.post('/test_client_regress/file_upload/', post_data)
+        self.assertEqual(response.status_code, 200)

tests/regressiontests/test_client_regress/urls.py

@@ -1,7 +1,7 @@
 from django.conf.urls.defaults import *
-from django.views.generic.simple import redirect_to
 import views
 
 urlpatterns = patterns('',
     (r'^no_template_view/$', views.no_template_view),
+    (r'^file_upload/$', views.file_upload_view),
 )

tests/regressiontests/test_client_regress/views.py

@@ -1,8 +1,20 @@
 from django.core.mail import EmailMessage, SMTPConnection
-from django.http import HttpResponse
+from django.http import HttpResponse, HttpResponseServerError
 from django.shortcuts import render_to_response
 
 def no_template_view(request):
     "A simple view that expects a GET request, and returns a rendered template"
     return HttpResponse("No template used")
 
+def file_upload_view(request):
+    """
+    Check that a file upload can be updated into the POST dictionary without
+    going pear-shaped.
+    """
+    form_data = request.POST.copy()
+    form_data.update(request.FILES)
+    if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode):
+        return HttpResponse('')
+    else:
+        return HttpResponseServerError()
+