mirrors/django
1
0
Fork 0
mirror of https://github.com/django/django.git synced 2025-11-07 07:15:35 +00:00
Code Issues 32 Releases Wiki Activity

Fixed #13795 -- Added a site-wide cache prefix and cache versioning. Thanks to bruth for the patch.

Browse Source 
git-svn-id: http://code.djangoproject.com/svn/django/trunk@14623 bcc190cf-cafb-0310-a4f2-bffc1f526a37
This commit is contained in:
Russell Keith-Magee
2010-11-19 15:39:35 +00:00
parent 261aee26c1
commit 99d247f4cb
12 changed files with 694 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
django/core/cache/__init__.py vendored
View File

@@ -67,18 +67,30 @@ def parse_backend_uri(backend_uri):
return scheme, host, params
def get_cache(backend_uri):
def get_cache(backend_uri, key_prefix=None, version=None, key_func=None):
if key_prefix is None:
key_prefix = settings.CACHE_KEY_PREFIX
if version is None:
version = settings.CACHE_VERSION
if key_func is None:
key_func = settings.CACHE_KEY_FUNCTION
if key_func is not None and not callable(key_func):
key_func_module_path, key_func_name = key_func.rsplit('.', 1)
key_func_module = importlib.import_module(key_func_module_path)
key_func = getattr(key_func_module, key_func_name)
scheme, host, params = parse_backend_uri(backend_uri)
if scheme in BACKENDS:
name = 'django.core.cache.backends.%s' % BACKENDS[scheme]
else:
name = scheme
module = importlib.import_module(name)
return module.CacheClass(host, params)
return module.CacheClass(host, params, key_prefix=key_prefix, version=version, key_func=key_func)
cache = get_cache(settings.CACHE_BACKEND)
# Some caches -- pythont-memcached in particular -- need to do a cleanup at the
# Some caches -- python-memcached in particular -- need to do a cleanup at the
# end of a request cycle. If the cache provides a close() method, wire it up
# here.
if hasattr(cache, 'close'):

87
django/core/cache/backends/base.py vendored
View File

@@ -3,6 +3,7 @@
import warnings
from django.core.exceptions import ImproperlyConfigured, DjangoRuntimeWarning
from django.utils.encoding import smart_str
class InvalidCacheBackendError(ImproperlyConfigured):
pass
@@ -13,8 +14,17 @@ class CacheKeyWarning(DjangoRuntimeWarning):
# Memcached does not accept keys longer than this.
MEMCACHE_MAX_KEY_LENGTH = 250
def default_key_func(key, key_prefix, version):
"""Default function to generate keys.
Constructs the key used by all other methods. By default it prepends
the `key_prefix'. CACHE_KEY_FUNCTION can be used to specify an alternate
function with custom key making behavior.
"""
return ':'.join([key_prefix, str(version), smart_str(key)])
class BaseCache(object):
def __init__(self, params):
def __init__(self, params, key_prefix='', version=1, key_func=None):
timeout = params.get('timeout', 300)
try:
timeout = int(timeout)
@@ -34,7 +44,25 @@ class BaseCache(object):
except (ValueError, TypeError):
self._cull_frequency = 3
def add(self, key, value, timeout=None):
self.key_prefix = smart_str(key_prefix)
self.version = version
self.key_func = key_func or default_key_func
def make_key(self, key, version=None):
"""Constructs the key used by all other methods. By default it
uses the key_func to generate a key (which, by default,
prepends the `key_prefix' and 'version'). An different key
function can be provided at the time of cache construction;
alternatively, you can subclass the cache backend to provide
custom key making behavior.
"""
if version is None:
version = self.version
new_key = self.key_func(key, self.key_prefix, version)
return new_key
def add(self, key, value, timeout=None, version=None):
"""
Set a value in the cache if the key does not already exist. If
timeout is given, that timeout will be used for the key; otherwise
@@ -44,27 +72,27 @@ class BaseCache(object):
"""
raise NotImplementedError
def get(self, key, default=None):
def get(self, key, default=None, version=None):
"""
Fetch a given key from the cache. If the key does not exist, return
default, which itself defaults to None.
"""
raise NotImplementedError
def set(self, key, value, timeout=None):
def set(self, key, value, timeout=None, version=None):
"""
Set a value in the cache. If timeout is given, that timeout will be
used for the key; otherwise the default cache timeout will be used.
"""
raise NotImplementedError
def delete(self, key):
def delete(self, key, version=None):
"""
Delete a key from the cache, failing silently.
"""
raise NotImplementedError
def get_many(self, keys):
def get_many(self, keys, version=None):
"""
Fetch a bunch of keys from the cache. For certain backends (memcached,
pgsql) this can be *much* faster when fetching multiple values.
@@ -74,34 +102,35 @@ class BaseCache(object):
"""
d = {}
for k in keys:
val = self.get(k)
val = self.get(k, version=version)
if val is not None:
d[k] = val
return d
def has_key(self, key):
def has_key(self, key, version=None):
"""
Returns True if the key is in the cache and has not expired.
"""
return self.get(key) is not None
return self.get(key, version=version) is not None
def incr(self, key, delta=1):
def incr(self, key, delta=1, version=None):
"""
Add delta to value in the cache. If the key does not exist, raise a
ValueError exception.
"""
if key not in self:
value = self.get(key, version=version)
if value is None:
raise ValueError("Key '%s' not found" % key)
new_value = self.get(key) + delta
self.set(key, new_value)
new_value = value + delta
self.set(key, new_value, version=version)
return new_value
def decr(self, key, delta=1):
def decr(self, key, delta=1, version=None):
"""
Subtract delta from value in the cache. If the key does not exist, raise
a ValueError exception.
"""
return self.incr(key, -delta)
return self.incr(key, -delta, version=version)
def __contains__(self, key):
"""
@@ -112,7 +141,7 @@ class BaseCache(object):
# if a subclass overrides it.
return self.has_key(key)
def set_many(self, data, timeout=None):
def set_many(self, data, timeout=None, version=None):
"""
Set a bunch of values in the cache at once from a dict of key/value
pairs. For certain backends (memcached), this is much more efficient
@@ -122,16 +151,16 @@ class BaseCache(object):
the default cache timeout will be used.
"""
for key, value in data.items():
self.set(key, value, timeout)
self.set(key, value, timeout=timeout, version=version)
def delete_many(self, keys):
def delete_many(self, keys, version=None):
"""
Set a bunch of values in the cache at once. For certain backends
(memcached), this is much more efficient than calling delete() multiple
times.
"""
for key in keys:
self.delete(key)
self.delete(key, version=version)
def clear(self):
"""Remove *all* values from the cache at once."""
@@ -154,3 +183,23 @@ class BaseCache(object):
'errors if used with memcached: %r' % key,
CacheKeyWarning)
def incr_version(self, key, delta=1, version=None):
"""Adds delta to the cache version for the supplied key. Returns the
new version.
"""
if version is None:
version = self.version
value = self.get(key, version=version)
if value is None:
raise ValueError("Key '%s' not found" % key)
self.set(key, value, version=version+delta)
self.delete(key, version=version)
return version+delta
def decr_version(self, key, delta=1, version=None):
"""Substracts delta from the cache version for the supplied key. Returns
the new version.
"""
return self.incr_version(key, -delta, version)

21
django/core/cache/backends/db.py vendored
View File

@@ -26,8 +26,8 @@ class Options(object):
self.proxy = False
class BaseDatabaseCacheClass(BaseCache):
def __init__(self, table, params):
BaseCache.__init__(self, params)
def __init__(self, table, params, key_prefix='', version=1, key_func=None):
BaseCache.__init__(self, params, key_prefix, version, key_func)
self._table = table
class CacheEntry(object):
@@ -35,7 +35,8 @@ class BaseDatabaseCacheClass(BaseCache):
self.cache_model_class = CacheEntry
class CacheClass(BaseDatabaseCacheClass):
def get(self, key, default=None):
def get(self, key, default=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
db = router.db_for_read(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
@@ -55,11 +56,13 @@ class CacheClass(BaseDatabaseCacheClass):
value = connections[db].ops.process_clob(row[1])
return pickle.loads(base64.decodestring(value))
def set(self, key, value, timeout=None):
def set(self, key, value, timeout=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
self._base_set('set', key, value, timeout)
def add(self, key, value, timeout=None):
def add(self, key, value, timeout=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
return self._base_set('add', key, value, timeout)
@@ -95,8 +98,10 @@ class CacheClass(BaseDatabaseCacheClass):
transaction.commit_unless_managed(using=db)
return True
def delete(self, key):
def delete(self, key, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()
@@ -104,8 +109,10 @@ class CacheClass(BaseDatabaseCacheClass):
cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % table, [key])
transaction.commit_unless_managed(using=db)
def has_key(self, key):
def has_key(self, key, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
db = router.db_for_read(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()

25
django/core/cache/backends/dummy.py vendored
View File

@@ -3,34 +3,39 @@
from django.core.cache.backends.base import BaseCache
class CacheClass(BaseCache):
def __init__(self, *args, **kwargs):
pass
def __init__(self, host, *args, **kwargs):
BaseCache.__init__(self, *args, **kwargs)
def add(self, key, *args, **kwargs):
def add(self, key, value, timeout=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
return True
def get(self, key, default=None):
def get(self, key, default=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
return default
def set(self, key, *args, **kwargs):
def set(self, key, value, timeout=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
def delete(self, key, *args, **kwargs):
def delete(self, key, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
def get_many(self, *args, **kwargs):
def get_many(self, keys, version=None):
return {}
def has_key(self, key, *args, **kwargs):
def has_key(self, key, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
return False
def set_many(self, *args, **kwargs):
def set_many(self, data, version=None):
pass
def delete_many(self, *args, **kwargs):
def delete_many(self, keys, version=None):
pass
def clear(self):

27
django/core/cache/backends/filebased.py vendored
View File

@@ -12,22 +12,23 @@ from django.core.cache.backends.base import BaseCache
from django.utils.hashcompat import md5_constructor
class CacheClass(BaseCache):
def __init__(self, dir, params):
BaseCache.__init__(self, params)
def __init__(self, dir, params, key_prefix='', version=1, key_func=None):
BaseCache.__init__(self, params, key_prefix, version, key_func)
self._dir = dir
if not os.path.exists(self._dir):
self._createdir()
def add(self, key, value, timeout=None):
self.validate_key(key)
if self.has_key(key):
def add(self, key, value, timeout=None, version=None):
if self.has_key(key, version=version):
return False
self.set(key, value, timeout)
self.set(key, value, timeout, version=version)
return True
def get(self, key, default=None):
def get(self, key, default=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
fname = self._key_to_file(key)
try:
f = open(fname, 'rb')
@@ -44,8 +45,10 @@ class CacheClass(BaseCache):
pass
return default
def set(self, key, value, timeout=None):
def set(self, key, value, timeout=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
fname = self._key_to_file(key)
dirname = os.path.dirname(fname)
@@ -68,7 +71,8 @@ class CacheClass(BaseCache):
except (IOError, OSError):
pass
def delete(self, key):
def delete(self, key, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
try:
self._delete(self._key_to_file(key))
@@ -85,7 +89,8 @@ class CacheClass(BaseCache):
except (IOError, OSError):
pass
def has_key(self, key):
def has_key(self, key, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
fname = self._key_to_file(key)
try:
@@ -140,7 +145,7 @@ class CacheClass(BaseCache):
Thus, a cache key of "foo" gets turnned into a file named
``{cache-dir}ac/bd/18db4cc2f85cedef654fccc4a4d8``.
"""
path = md5_constructor(key.encode('utf-8')).hexdigest()
path = md5_constructor(key).hexdigest()
path = os.path.join(path[:2], path[2:4], path[4:])
return os.path.join(self._dir, path)

19
django/core/cache/backends/locmem.py vendored
View File

@@ -10,13 +10,14 @@ from django.core.cache.backends.base import BaseCache
from django.utils.synch import RWLock
class CacheClass(BaseCache):
def __init__(self, _, params):
BaseCache.__init__(self, params)
def __init__(self, _, params, key_prefix='', version=1, key_func=None):
BaseCache.__init__(self, params, key_prefix, version, key_func)
self._cache = {}
self._expire_info = {}
self._lock = RWLock()
def add(self, key, value, timeout=None):
def add(self, key, value, timeout=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
self._lock.writer_enters()
try:
@@ -31,7 +32,8 @@ class CacheClass(BaseCache):
finally:
self._lock.writer_leaves()
def get(self, key, default=None):
def get(self, key, default=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
self._lock.reader_enters()
try:
@@ -64,7 +66,8 @@ class CacheClass(BaseCache):
self._cache[key] = value
self._expire_info[key] = time.time() + timeout
def set(self, key, value, timeout=None):
def set(self, key, value, timeout=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
self._lock.writer_enters()
# Python 2.4 doesn't allow combined try-except-finally blocks.
@@ -76,7 +79,8 @@ class CacheClass(BaseCache):
finally:
self._lock.writer_leaves()
def has_key(self, key):
def has_key(self, key, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
self._lock.reader_enters()
try:
@@ -117,7 +121,8 @@ class CacheClass(BaseCache):
except KeyError:
pass
def delete(self, key):
def delete(self, key, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
self._lock.writer_enters()
try:

53
django/core/cache/backends/memcached.py vendored
View File

@@ -3,7 +3,6 @@
import time
from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
from django.utils.encoding import smart_unicode, smart_str
try:
import cmemcache as memcache
@@ -19,8 +18,8 @@ except ImportError:
raise InvalidCacheBackendError("Memcached cache backend requires either the 'memcache' or 'cmemcache' library")
class CacheClass(BaseCache):
def __init__(self, server, params):
BaseCache.__init__(self, params)
def __init__(self, server, params, key_prefix='', version=1, key_func=None):
BaseCache.__init__(self, params, key_prefix, version, key_func)
self._cache = memcache.Client(server.split(';'))
def _get_memcache_timeout(self, timeout):
@@ -39,30 +38,43 @@ class CacheClass(BaseCache):
timeout += int(time.time())
return timeout
def add(self, key, value, timeout=0):
def add(self, key, value, timeout=0, version=None):
key = self.make_key(key, version=version)
if isinstance(value, unicode):
value = value.encode('utf-8')
return self._cache.add(smart_str(key), value, self._get_memcache_timeout(timeout))
return self._cache.add(key, value, self._get_memcache_timeout(timeout))
def get(self, key, default=None):
val = self._cache.get(smart_str(key))
def get(self, key, default=None, version=None):
key = self.make_key(key, version=version)
val = self._cache.get(key)
if val is None:
return default
return val
def set(self, key, value, timeout=0):
self._cache.set(smart_str(key), value, self._get_memcache_timeout(timeout))
def set(self, key, value, timeout=0, version=None):
key = self.make_key(key, version=version)
self._cache.set(key, value, self._get_memcache_timeout(timeout))
def delete(self, key):
self._cache.delete(smart_str(key))
def delete(self, key, version=None):
key = self.make_key(key, version=version)
self._cache.delete(key)
def get_many(self, keys):
return self._cache.get_multi(map(smart_str,keys))
def get_many(self, keys, version=None):
new_keys = map(lambda x: self.make_key(x, version=version), keys)
ret = self._cache.get_multi(new_keys)
if ret:
_ = {}
m = dict(zip(new_keys, keys))
for k, v in ret.items():
_[m[k]] = v
ret = _
return ret
def close(self, **kwargs):
self._cache.disconnect_all()
def incr(self, key, delta=1):
def incr(self, key, delta=1, version=None):
key = self.make_key(key, version=version)
try:
val = self._cache.incr(key, delta)
@@ -76,7 +88,8 @@ class CacheClass(BaseCache):
return val
def decr(self, key, delta=1):
def decr(self, key, delta=1, version=None):
key = self.make_key(key, version=version)
try:
val = self._cache.decr(key, delta)
@@ -89,16 +102,18 @@ class CacheClass(BaseCache):
raise ValueError("Key '%s' not found" % key)
return val
def set_many(self, data, timeout=0):
def set_many(self, data, timeout=0, version=None):
safe_data = {}
for key, value in data.items():
key = self.make_key(key, version=version)
if isinstance(value, unicode):
value = value.encode('utf-8')
safe_data[smart_str(key)] = value
safe_data[key] = value
self._cache.set_multi(safe_data, self._get_memcache_timeout(timeout))
def delete_many(self, keys):
self._cache.delete_multi(map(smart_str, keys))
def delete_many(self, keys, version=None):
l = lambda x: self.make_key(x, version=version)
self._cache.delete_multi(map(l, keys))
def clear(self):
self._cache.flush_all()