diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 3d81f580fb..c651543ff9 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -126,6 +126,8 @@ JING_PATH = "/usr/bin/jing" # response phase the middleware will be applied in reverse order. MIDDLEWARE_CLASSES = ( "django.middleware.sessions.SessionMiddleware", +# "django.middleware.http.ConditionalGetMiddleware", +# "django.middleware.gzip.GZipMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.doc.XViewMiddleware", ) @@ -145,6 +147,7 @@ SESSION_COOKIE_DOMAIN = None # A string like ".lawrence.com", or No # The cache backend to use. See the docstring in django.core.cache for the # possible values. CACHE_BACKEND = 'simple://' +CACHE_MIDDLEWARE_KEY_PREFIX = '' #################### # COMMENTS # diff --git a/django/core/db/backends/mysql.py b/django/core/db/backends/mysql.py index e678740b33..2e77adbc43 100644 --- a/django/core/db/backends/mysql.py +++ b/django/core/db/backends/mysql.py @@ -21,6 +21,32 @@ django_conversions.update({ FIELD_TYPE.TIME: typecasts.typecast_time, }) +# This is an extra debug layer over MySQL queries, to display warnings. +# It's only used when DEBUG=True. +class MysqlDebugWrapper: + def __init__(self, cursor): + self.cursor = cursor + + def execute(self, sql, params=()): + try: + return self.cursor.execute(sql, params) + except Database.Warning, w: + self.cursor.execute("SHOW WARNINGS") + raise Database.Warning, "%s: %s" % (w, self.cursor.fetchall()) + + def executemany(self, sql, param_list): + try: + return self.cursor.executemany(sql, param_list) + except Database.Warning: + self.cursor.execute("SHOW WARNINGS") + raise Database.Warning, "%s: %s" % (w, self.cursor.fetchall()) + + def __getattr__(self, attr): + if self.__dict__.has_key(attr): + return self.__dict__[attr] + else: + return getattr(self.cursor, attr) + class DatabaseWrapper: def __init__(self): self.connection = None @@ -32,7 +58,7 @@ class DatabaseWrapper: self.connection = Database.connect(user=DATABASE_USER, db=DATABASE_NAME, passwd=DATABASE_PASSWORD, host=DATABASE_HOST, conv=django_conversions) if DEBUG: - return base.CursorDebugWrapper(self.connection.cursor(), self) + return base.CursorDebugWrapper(MysqlDebugWrapper(self.connection.cursor()), self) return self.connection.cursor() def commit(self): diff --git a/django/middleware/cache.py b/django/middleware/cache.py index 7f4057eec7..8216c40ae1 100644 --- a/django/middleware/cache.py +++ b/django/middleware/cache.py @@ -1,88 +1,70 @@ +import copy from django.conf import settings from django.core.cache import cache +from django.utils.cache import get_cache_key, learn_cache_key, patch_response_headers from django.utils.httpwrappers import HttpResponseNotModified -from django.utils.text import compress_string -import datetime, md5 class CacheMiddleware: """ Cache middleware. If this is enabled, each Django-powered page will be - cached for CACHE_MIDDLEWARE_SECONDS seconds. Cache is based on URLs. Pages - with GET or POST parameters are not cached. + cached for CACHE_MIDDLEWARE_SECONDS seconds. Cache is based on URLs. - If the cache is shared across multiple sites using the same Django - installation, set the CACHE_MIDDLEWARE_KEY_PREFIX to the name of the site, - or some other string that is unique to this Django instance, to prevent key - collisions. + Only parameter-less GET or HEAD-requests with status code 200 are cached. - This middleware will also make the following optimizations: + This middleware expects that a HEAD request is answered with a response + exactly like the corresponding GET request. - * If the CACHE_MIDDLEWARE_GZIP setting is True, the content will be - gzipped. + When a hit occurs, a shallow copy of the original response object is + returned from process_request. - * ETags will be added, using a simple MD5 hash of the page's content. + Pages will be cached based on the contents of the request headers + listed in the response's "Vary" header. This means that pages shouldn't + change their "Vary" header. + + This middleware also sets ETag, Last-Modified, Expires and Cache-Control + headers on the response object. """ + def __init__(self, cache_timeout=None, key_prefix=None): + self.cache_timeout = cache_timeout + if cache_timeout is None: + self.cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS + self.key_prefix = key_prefix + if key_prefix is None: + self.key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX + def process_request(self, request): - """ - Checks whether the page is already cached. If it is, returns the cached - version. Also handles ETag stuff. - """ - if request.GET or request.POST: - request._cache_middleware_set_cache = False + "Checks whether the page is already cached and returns the cached version if available." + if not request.META['REQUEST_METHOD'] in ('GET', 'HEAD') or request.GET: + request._cache_update_cache = False return None # Don't bother checking the cache. - accept_encoding = '' - if settings.CACHE_MIDDLEWARE_GZIP: - try: - accept_encoding = request.META['HTTP_ACCEPT_ENCODING'] - except KeyError: - pass - accepts_gzip = 'gzip' in accept_encoding - request._cache_middleware_accepts_gzip = accepts_gzip - - # This uses the same cache_key as views.decorators.cache.cache_page, - # so the cache can be shared. - cache_key = 'views.decorators.cache.cache_page.%s.%s.%s' % \ - (settings.CACHE_MIDDLEWARE_KEY_PREFIX, request.path, accepts_gzip) - request._cache_middleware_key = cache_key + cache_key = get_cache_key(request, self.key_prefix) + if cache_key is None: + request._cache_update_cache = True + return None # No cache information available, need to rebuild. response = cache.get(cache_key, None) if response is None: - request._cache_middleware_set_cache = True - return None - else: - request._cache_middleware_set_cache = False - # Logic is from http://simon.incutio.com/archive/2003/04/23/conditionalGet - try: - if_none_match = request.META['HTTP_IF_NONE_MATCH'] - except KeyError: - if_none_match = None - try: - if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE'] - except KeyError: - if_modified_since = None - if if_none_match is None and if_modified_since is None: - pass - elif if_none_match is not None and response['ETag'] != if_none_match: - pass - elif if_modified_since is not None and response['Last-Modified'] != if_modified_since: - pass - else: - return HttpResponseNotModified() - return response + request._cache_update_cache = True + return None # No cache information available, need to rebuild. + + request._cache_update_cache = False + return copy.copy(response) def process_response(self, request, response): - """ - Sets the cache, if needed. - """ - if request._cache_middleware_set_cache: - content = response.get_content_as_string(settings.DEFAULT_CHARSET) - if request._cache_middleware_accepts_gzip: - content = compress_string(content) - response.content = content - response['Content-Encoding'] = 'gzip' - response['ETag'] = md5.new(content).hexdigest() - response['Content-Length'] = '%d' % len(content) - response['Last-Modified'] = datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') - cache.set(request._cache_middleware_key, response, settings.CACHE_MIDDLEWARE_SECONDS) + "Sets the cache, if needed." + if not request._cache_update_cache: + # We don't need to update the cache, just return. + return response + if not request.META['REQUEST_METHOD'] == 'GET': + # This is a stronger requirement than above. It is needed + # because of interactions between this middleware and the + # HTTPMiddleware, which throws the body of a HEAD-request + # away before this middleware gets a chance to cache it. + return response + if not response.status_code == 200: + return response + patch_response_headers(response, self.cache_timeout) + cache_key = learn_cache_key(request, response, self.cache_timeout, self.key_prefix) + cache.set(cache_key, response, self.cache_timeout) return response diff --git a/django/middleware/gzip.py b/django/middleware/gzip.py new file mode 100644 index 0000000000..201bec2000 --- /dev/null +++ b/django/middleware/gzip.py @@ -0,0 +1,24 @@ +import re +from django.utils.text import compress_string +from django.utils.cache import patch_vary_headers + +re_accepts_gzip = re.compile(r'\bgzip\b') + +class GZipMiddleware: + """ + This middleware compresses content if the browser allows gzip compression. + It sets the Vary header accordingly, so that caches will base their storage + on the Accept-Encoding header. + """ + def process_response(self, request, response): + patch_vary_headers(response, ('Accept-Encoding',)) + if response.has_header('Content-Encoding'): + return response + + ae = request.META.get('HTTP_ACCEPT_ENCODING', '') + if not re_accepts_gzip.search(ae): + return response + + response.content = compress_string(response.content) + response['Content-Encoding'] = 'gzip' + return response diff --git a/django/middleware/http.py b/django/middleware/http.py new file mode 100644 index 0000000000..2bccd60903 --- /dev/null +++ b/django/middleware/http.py @@ -0,0 +1,37 @@ +import datetime + +class ConditionalGetMiddleware: + """ + Handles conditional GET operations. If the response has a ETag or + Last-Modified header, and the request has If-None-Match or + If-Modified-Since, the response is replaced by an HttpNotModified. + + Removes the content from any response to a HEAD request. + + Also sets the Date and Content-Length response-headers. + """ + def process_response(self, request, response): + now = datetime.datetime.utcnow() + response['Date'] = now.strftime('%a, %d %b %Y %H:%M:%S GMT') + if not response.has_header('Content-Length'): + response['Content-Length'] = str(len(response.content)) + + if response.has_header('ETag'): + if_none_match = request.META.get('HTTP_IF_NONE_MATCH', None) + if if_none_match == response['ETag']: + response.status_code = 304 + response.content = '' + response['Content-Length'] = '0' + + if response.has_header('Last-Modified'): + last_mod = response['Last-Modified'] + if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE', None) + if if_modified_since == response['Last-Modified']: + response.status_code = 304 + response.content = '' + response['Content-Length'] = '0' + + if request.META['REQUEST_METHOD'] == 'HEAD': + response.content = '' + + return response diff --git a/django/middleware/sessions.py b/django/middleware/sessions.py index a588e3e95b..42b2118410 100644 --- a/django/middleware/sessions.py +++ b/django/middleware/sessions.py @@ -1,5 +1,6 @@ from django.conf.settings import SESSION_COOKIE_NAME, SESSION_COOKIE_AGE, SESSION_COOKIE_DOMAIN from django.models.core import sessions +from django.utils.cache import patch_vary_headers import datetime TEST_COOKIE_NAME = 'testcookie' @@ -61,6 +62,7 @@ class SessionMiddleware: def process_response(self, request, response): # If request.session was modified, or if response.session was set, save # those changes and set a session cookie. + patch_vary_headers(response, ('Cookie',)) try: modified = request.session.modified except AttributeError: diff --git a/django/utils/cache.py b/django/utils/cache.py new file mode 100644 index 0000000000..fcd0825a22 --- /dev/null +++ b/django/utils/cache.py @@ -0,0 +1,121 @@ +""" +This module contains helper functions for controlling caching. It does so by +managing the "Vary" header of responses. It includes functions to patch the +header of response objects directly and decorators that change functions to do +that header-patching themselves. + +For information on the Vary header, see: + + http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.44 + +Essentially, the "Vary" HTTP header defines which headers a cache should take +into account when building its cache key. Requests with the same path but +different header content for headers named in "Vary" need to get different +cache keys to prevent delivery of wrong content. + +A example: i18n middleware would need to distinguish caches by the +"Accept-language" header. +""" + +import datetime, md5, re +from django.conf import settings +from django.core.cache import cache + +vary_delim_re = re.compile(r',\s*') + +def patch_response_headers(response, cache_timeout=None): + """ + Adds some useful headers to the given HttpResponse object: + ETag, Last-Modified, Expires and Cache-Control + + Each header is only added if it isn't already set. + + cache_timeout is in seconds. The CACHE_MIDDLEWARE_SECONDS setting is used + by default. + """ + if cache_timeout is None: + cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS + now = datetime.datetime.utcnow() + expires = now + datetime.timedelta(0, cache_timeout) + if not response.has_header('ETag'): + response['ETag'] = md5.new(response.content).hexdigest() + if not response.has_header('Last-Modified'): + response['Last-Modified'] = now.strftime('%a, %d %b %Y %H:%M:%S GMT') + if not response.has_header('Expires'): + response['Expires'] = expires.strftime('%a, %d %b %Y %H:%M:%S GMT') + if not response.has_header('Cache-Control'): + response['Cache-Control'] = 'max-age=%d' % cache_timeout + +def patch_vary_headers(response, newheaders): + """ + Adds (or updates) the "Vary" header in the given HttpResponse object. + newheaders is a list of header names that should be in "Vary". Existing + headers in "Vary" aren't removed. + """ + # Note that we need to keep the original order intact, because cache + # implementations may rely on the order of the Vary contents in, say, + # computing an MD5 hash. + vary = [] + if response.has_header('Vary'): + vary = vary_delim_re.split(response['Vary']) + oldheaders = dict([(el.lower(), 1) for el in vary]) + for newheader in newheaders: + if not newheader.lower() in oldheaders: + vary.append(newheader) + response['Vary'] = ', '.join(vary) + +def _generate_cache_key(request, headerlist, key_prefix): + "Returns a cache key from the headers given in the header list." + ctx = md5.new() + for header in headerlist: + value = request.META.get(header, None) + if value is not None: + ctx.update(value) + return 'views.decorators.cache.cache_page.%s.%s.%s' % (key_prefix, request.path, ctx.hexdigest()) + +def get_cache_key(request, key_prefix=None): + """ + Returns a cache key based on the request path. It can be used in the + request phase because it pulls the list of headers to take into account + from the global path registry and uses those to build a cache key to check + against. + + If there is no headerlist stored, the page needs to be rebuilt, so this + function returns None. + """ + if key_prefix is None: + key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX + cache_key = 'views.decorators.cache.cache_header.%s.%s' % (key_prefix, request.path) + headerlist = cache.get(cache_key, None) + if headerlist is not None: + return _generate_cache_key(request, headerlist, key_prefix) + else: + return None + +def learn_cache_key(request, response, cache_timeout=None, key_prefix=None): + """ + Learns what headers to take into account for some request path from the + response object. It stores those headers in a global path registry so that + later access to that path will know what headers to take into account + without building the response object itself. The headers are named in the + Vary header of the response, but we want to prevent response generation. + + The list of headers to use for cache key generation is stored in the same + cache as the pages themselves. If the cache ages some data out of the + cache, this just means that we have to build the response once to get at + the Vary header and so at the list of headers to use for the cache key. + """ + if key_prefix is None: + key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX + if cache_timeout is None: + cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS + cache_key = 'views.decorators.cache.cache_header.%s.%s' % (key_prefix, request.path) + if response.has_header('Vary'): + headerlist = ['HTTP_'+header.upper().replace('-', '_') for header in vary_delim_re.split(response['Vary'])] + cache.set(cache_key, headerlist, cache_timeout) + return _generate_cache_key(request, headerlist, key_prefix) + else: + # if there is no Vary header, we still need a cache key + # for the request.path + cache.set(cache_key, [], cache_timeout) + return _generate_cache_key(request, [], key_prefix) diff --git a/django/utils/decorators.py b/django/utils/decorators.py new file mode 100644 index 0000000000..b21a4e4248 --- /dev/null +++ b/django/utils/decorators.py @@ -0,0 +1,22 @@ +"Functions that help with dynamically creating decorators for views." + +def decorator_from_middleware(middleware_class): + """ + Given a middleware class (not an instance), returns a view decorator. This + lets you use middleware functionality on a per-view basis. + """ + def _decorator_from_middleware(view_func, *args, **kwargs): + middleware = middleware_class(*args, **kwargs) + def _wrapped_view(request, *args, **kwargs): + if hasattr(middleware, 'process_request'): + result = middleware.process_request(request) + if result is not None: + return result + response = view_func(request, *args, **kwargs) + if hasattr(middleware, 'process_response'): + result = middleware.process_response(request, response) + if result is not None: + return result + return response + return _wrapped_view + return _decorator_from_middleware diff --git a/django/views/admin/main.py b/django/views/admin/main.py index 8632c2a626..fc24b590ef 100644 --- a/django/views/admin/main.py +++ b/django/views/admin/main.py @@ -252,7 +252,7 @@ def change_list(request, app_label, module_name): lookup_val = request.GET.get(lookup_kwarg, None) lookup_val2 = request.GET.get(lookup_kwarg2, None) filter_template.append('

By %s: