[soc2009/http-wsgi-improvements] Adds http.HttpResponseStreaming, with docs, tests, and support in four built-in middleware classes. Refs #7581.

git-svn-id: http://code.djangoproject.com/svn/django/branches/soc2009/http-wsgi-improvements@11449 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/contrib/csrf/middleware.py

@@ -64,6 +64,8 @@ class CsrfResponseMiddleware(object):
     csrfmiddlewaretoken if the response/request have an active
     session.
     """
+    streaming_safe = True
+
     def process_response(self, request, response):
         if getattr(response, 'csrf_exempt', False):
             return response
@@ -102,6 +104,11 @@ class CsrfResponseMiddleware(object):
 
             # Modify any POST forms
             response.content = _POST_FORM_RE.sub(add_csrf_field, response.content)
+            # Handle streaming responses
+            if getattr(response, "content_generator", False):
+                response.content = (_POST_FORM_RE.sub(add_csrf_field, chunk) for chunk in response.content_generator)
+            else:
+                response.content = _POST_FORM_RE.sub(add_csrf_field, response.content)
         return response
 
 class CsrfMiddleware(CsrfViewMiddleware, CsrfResponseMiddleware):

django/core/handlers/base.py

@@ -74,9 +74,13 @@ class BaseHandler(object):
                 response = self.get_response(request)
 
                 # Apply response middleware
+                streaming = getattr(response, "content_generator", False)
+                streaming_safe = lambda x: getattr(x.im_self, "streaming_safe", False)
                 if not isinstance(response, http.HttpResponseSendFile):
                     for middleware_method in self._response_middleware:
-                        response = middleware_method(request, response)
+                        if not streaming or streaming_safe(middleware_method):
+                            print middleware_method
+                            response = middleware_method(request, response)
                     response = self.apply_response_fixes(request, response)
         finally:
             signals.request_finished.send(sender=self.__class__)

django/http/__init__.py

@@ -436,6 +436,47 @@ class HttpResponse(object):
     def tell(self):
         return sum([len(chunk) for chunk in self._container])
 
+class HttpResponseStreaming(HttpResponse):
+    """
+        This class behaves the same as HttpResponse, except that the content
+        attribute is an unconsumed generator or iterator.
+    """
+    def __init__(self, content='', mimetype=None, status=None,
+                 content_type=None, request=None):
+        super(HttpResponseStreaming, self).__init__('', mimetype,
+            status, content_type, request)
+
+        self._container = content
+        self._is_string = False
+
+    def _consume_content(self):
+        if not self._is_string:
+            content = self._container
+            self._container = [''.join(content)]
+            if hasattr(content, 'close'):
+                content.close()
+            self._is_string = True
+
+    def _get_content(self):
+        self._consume_content()
+        return super(HttpResponseStreaming, self)._get_content()
+
+    def _set_content(self, value):
+        if not isinstance(value, basestring) and hasattr(value, "__iter__"):
+            self._container = value
+            self._is_string = False
+        else:
+            self._container = [value]
+            self._is_string = True
+
+    content = property(_get_content, _set_content)
+
+    def _get_content_generator(self):
+        if not self._is_string:
+            return self._container
+
+    content_generator = property(_get_content_generator)
+
 class HttpResponseSendFile(HttpResponse): 
     sendfile_fh = None
 

django/middleware/common.py

@@ -27,6 +27,7 @@ class CommonMiddleware(object):
           the entire page content and Not Modified responses will be returned
           appropriately.
     """
+    streaming_safe = True
 
     def process_request(self, request):
         """
@@ -100,14 +101,15 @@ class CommonMiddleware(object):
         if settings.USE_ETAGS:
             if response.has_header('ETag'):
                 etag = response['ETag']
-            else:
+            # Do not consume the content of HttpResponseStreaming
+            elif not getattr(response, "content_generator", False):
                 etag = '"%s"' % md5_constructor(response.content).hexdigest()
-            if response.status_code >= 200 and response.status_code < 300 and request.META.get('HTTP_IF_NONE_MATCH') == etag:
-                cookies = response.cookies
-                response = http.HttpResponseNotModified()
-                response.cookies = cookies
-            else:
-                response['ETag'] = etag
+                if response.status_code >= 200 and response.status_code < 300 and request.META.get('HTTP_IF_NONE_MATCH') == etag:
+                    cookies = response.cookies
+                    response = http.HttpResponseNotModified()
+                    response.cookies = cookies
+                else:
+                    response['ETag'] = etag
 
         return response
 

django/middleware/gzip.py

@@ -1,6 +1,6 @@
 import re
 
-from django.utils.text import compress_string
+from django.utils.text import compress_sequence, compress_string
 from django.utils.cache import patch_vary_headers
 
 re_accepts_gzip = re.compile(r'\bgzip\b')
@@ -11,9 +11,15 @@ class GZipMiddleware(object):
     It sets the Vary header accordingly, so that caches will base their storage
     on the Accept-Encoding header.
     """
-    def process_response(self, request, response):
+    streaming_safe = True
+
+    def process_response(self, request, response):         
+        # Do not consume the content of HttpResponseStreaming responses just to
+        # check content length
+        streaming = getattr(response, "content_generator", False)
+
         # It's not worth compressing non-OK or really short responses.
-        if response.status_code != 200 or len(response.content) < 200:
+        if response.status_code != 200 or (not streaming and len(response.content) < 200):
             return response
 
         patch_vary_headers(response, ('Accept-Encoding',))
@@ -32,7 +38,11 @@ class GZipMiddleware(object):
         if not re_accepts_gzip.search(ae):
             return response
 
-        response.content = compress_string(response.content)
+        if streaming:
+            response.content = compress_sequence(response.content_generator)
+            del response['Content-Length']
+        else:
+            response.content = compress_string(response.content)
+            response['Content-Length'] = str(len(response.content))
         response['Content-Encoding'] = 'gzip'
-        response['Content-Length'] = str(len(response.content))
         return response

django/middleware/http.py

@@ -8,9 +8,12 @@ class ConditionalGetMiddleware(object):
 
     Also sets the Date and Content-Length response-headers.
     """
+    streaming_safe = True
+
     def process_response(self, request, response):
         response['Date'] = http_date()
-        if not response.has_header('Content-Length'):
+        streaming = getattr(response, "content_generator", False)
+        if not response.has_header('Content-Length') and not streaming:
             response['Content-Length'] = str(len(response.content))
 
         if response.has_header('ETag'):

django/utils/text.py

@@ -176,6 +176,24 @@ def compress_string(s):
     zfile.close()
     return zbuf.getvalue()
 
+# WARNING - be aware that compress_sequence does not achieve the same
+# level of compression as compress_string
+def compress_sequence(sequence):
+    import cStringIO, gzip
+    zbuf = cStringIO.StringIO()
+    zfile = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
+    yield zbuf.getvalue()
+    for item in sequence:
+        position = zbuf.tell()
+        zfile.write(item)
+        zfile.flush()
+        zbuf.seek(position)
+        yield zbuf.read()
+    position = zbuf.tell()
+    zfile.close()
+    zbuf.seek(position)
+    yield zbuf.read()
+
 ustring_re = re.compile(u"([\u0080-\uffff])")
 
 def javascript_quote(s, quote_double_quotes=False):

docs/ref/request-response.txt

@@ -597,6 +597,35 @@ live in :mod:`django.http`.
 
     **Note:** Response middleware is bypassed by HttpResponseSendFile.
 
+.. class:: HttpResponseStreaming
+
+    .. versionadded:: 1.1
+
+    A special response class that does not consume generators before returning
+    the response. To do this, it bypasses middleware that is not useful for
+    chunked responses, and is treated specially by middleware that is useful.
+
+    It is primarily useful for sending large responses that would cause
+    timeouts if sent with a normal HttpResponse.
+
+    **Note:** Of the built-in response middleware, this class works correctly with:
+
+    * :class:`django.middleware.common.CommonMiddleware`
+
+    * :class:`django.middleware.gzip.GZipMiddleware`
+
+    * :class:`django.middleware.http.ConditionalGetMiddleware`
+
+    * :class:`django.contrib.csrf.middleware.CsrfMiddleware`
+
+    Developers of third-party middleware who wish to make it work with this class
+    should note that any time they access :class:`HttpResponseStreaming.content`, it will
+    break the functionality of this class. Instead, replace :attr:`HttpResponseStreaming.content`
+    by wrapping the value of :attr:`HttpResponseStreaming.content_generator`. :class:`django.middleware.gzip.GZipMiddleware`
+    is a good example to follow. To inform the handler to send :class:`HttpResponseStreaming`
+    responses through your middleware, add the class attribute ``streaming_safe = True``
+    to your middleware class.
+
 .. class:: HttpResponseRedirect
 
     The constructor takes a single argument -- the path to redirect to. This

tests/regressiontests/response_streaming/tests.py

@@ -0,0 +1,32 @@
+import urllib, os
+
+from django.test import TestCase
+from django.conf import settings
+from django.core.files import temp as tempfile
+
+def x():
+    for i in range(0, 10):
+        yield unicode(i) + u'\n'
+
+class ResponseStreamingTests(TestCase):
+    def test_streaming(self):
+        response = self.client.get('/streaming/stream_file/')
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response['Content-Disposition'],
+                'attachment; filename=test.csv')
+        self.assertEqual(response['Content-Type'], 'text/csv')
+        self.assertTrue(not response._is_string)
+        self.assertEqual("".join(iter(response)), "".join(x()))
+        self.assertTrue(not response._is_string)
+
+    def test_bad_streaming(self):
+        response = self.client.get('/streaming/stream_file/')
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response['Content-Disposition'],
+                'attachment; filename=test.csv')
+        self.assertEqual(response['Content-Type'], 'text/csv')
+        self.assertTrue(not response._is_string)
+        self.assertEqual(response.content, "".join(x()))
+        self.assertTrue(response._is_string)

tests/regressiontests/response_streaming/urls.py

@@ -0,0 +1,7 @@
+from django.conf.urls.defaults import patterns
+
+import views
+
+urlpatterns = patterns('',
+    (r'^stream_file/$', views.test_streaming),
+)

tests/regressiontests/response_streaming/views.py

@@ -0,0 +1,13 @@
+import urllib
+
+from django.http import HttpResponseStreaming
+from time import sleep
+
+def x():
+    for i in range(0, 10):
+        yield unicode(i) + u'\n'
+
+def test_streaming(request):
+    response = HttpResponseStreaming(content=x(), mimetype='text/csv')
+    response['Content-Disposition'] = 'attachment; filename=test.csv'
+    return response

tests/urls.py

@@ -36,6 +36,9 @@ urlpatterns = patterns('',
     # HttpResponseSendfile tests 
  	(r'^sendfile/', include('regressiontests.sendfile.urls')), 
 
+    # HttpResponseStreaming tests
+    (r'^streaming/', include('regressiontests.response_streaming.urls')),
+
     # conditional get views
     (r'condition/', include('regressiontests.conditional_processing.urls')),
 )