Fixed #11675 -- Added support for the PyLibMC cache library. In order to support this, and clean up some other 1.3 caching additions, this patch also includes some changes to the way caches are defined. This means you can now have multiple caches, in the same way you have multiple databases. A huge thanks to Jacob Burch for the work on the PyLibMC backend, and to Jannis for his work on the cache definition changes.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@15005 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/core/cache/__init__.py

@@ -12,8 +12,13 @@ get_cache() function made available here. get_cache() takes a backend URI
 (e.g. "memcached://127.0.0.1:11211/") and returns an instance of a backend
 cache class.
 
-See docs/cache.txt for information on the public API.
+See docs/topics/cache.txt for information on the public API.
 """
+from django.conf import settings
+from django.core import signals
+from django.core.cache.backends.base import (
+    InvalidCacheBackendError, CacheKeyWarning, BaseCache)
+from django.utils import importlib
 
 try:
     # The mod_python version is more efficient, so try importing it first.
@@ -27,10 +32,9 @@ except ImportError:
         # PendingDeprecationWarning
         from cgi import parse_qsl
 
-from django.conf import settings
-from django.core import signals
-from django.core.cache.backends.base import InvalidCacheBackendError, CacheKeyWarning
-from django.utils import importlib
+__all__ = [
+    'get_cache', 'cache', 'DEFAULT_CACHE_ALIAS'
+]
 
 # Name for use in settings file --> name of module in "backends" directory.
 # Any backend scheme that is not in this dictionary is treated as a Python
@@ -43,6 +47,8 @@ BACKENDS = {
     'dummy': 'dummy',
 }
 
+DEFAULT_CACHE_ALIAS = 'default'
+
 def parse_backend_uri(backend_uri):
     """
     Converts the "backend_uri" into a cache scheme ('db', 'memcached', etc), a
@@ -67,32 +73,102 @@ def parse_backend_uri(backend_uri):
 
     return scheme, host, params
 
-def get_cache(backend_uri, key_prefix=None, version=None, key_func=None):
-    if key_prefix is None:
-        key_prefix = settings.CACHE_KEY_PREFIX
-    if version is None:
-        version = settings.CACHE_VERSION
-    if key_func is None:
-        key_func = settings.CACHE_KEY_FUNCTION
+if not settings.CACHES:
+    import warnings
+    warnings.warn(
+        "settings.CACHE_* is deprecated; use settings.CACHES instead.",
+        PendingDeprecationWarning
+    )
+    # Mapping for new-style cache backend api
+    backend_classes = {
+        'memcached': 'memcached.CacheClass',
+        'locmem': 'locmem.LocMemCache',
+        'file': 'filebased.FileBasedCache',
+        'db': 'db.DatabaseCache',
+        'dummy': 'dummy.DummyCache',
+    }
+    engine, host, params = parse_backend_uri(settings.CACHE_BACKEND)
+    if engine in backend_classes:
+        engine = 'django.core.cache.backends.%s' % backend_classes[engine]
+    defaults = {
+        'BACKEND': engine,
+        'LOCATION': host,
+    }
+    defaults.update(params)
+    settings.CACHES[DEFAULT_CACHE_ALIAS] = defaults
 
-    if key_func is not None and not callable(key_func):
-        key_func_module_path, key_func_name = key_func.rsplit('.', 1)
-        key_func_module = importlib.import_module(key_func_module_path)
-        key_func = getattr(key_func_module, key_func_name)
+if DEFAULT_CACHE_ALIAS not in settings.CACHES:
+    raise ImproperlyConfigured("You must define a '%s' cache" % DEFAULT_CACHE_ALIAS)
 
-    scheme, host, params = parse_backend_uri(backend_uri)
-    if scheme in BACKENDS:
-        name = 'django.core.cache.backends.%s' % BACKENDS[scheme]
+def parse_backend_conf(backend, **kwargs):
+    """
+    Helper function to parse the backend configuration
+    that doesn't use the URI notation.
+    """
+    # Try to get the CACHES entry for the given backend name first
+    conf = settings.CACHES.get(backend, None)
+    if conf is not None:
+        args = conf.copy()
+        backend = args.pop('BACKEND')
+        location = args.pop('LOCATION', '')
+        return backend, location, args
     else:
-        name = scheme
-    module = importlib.import_module(name)
-    return module.CacheClass(host, params, key_prefix=key_prefix, version=version, key_func=key_func)
+        # Trying to import the given backend, in case it's a dotted path
+        mod_path, cls_name = backend.rsplit('.', 1)
+        try:
+            mod = importlib.import_module(mod_path)
+            backend_cls = getattr(mod, cls_name)
+        except (AttributeError, ImportError):
+            raise InvalidCacheBackendError("Could not find backend '%s'" % backend)
+        location = kwargs.pop('LOCATION', '')
+        return backend, location, kwargs
+    raise InvalidCacheBackendError(
+        "Couldn't find a cache backend named '%s'" % backend)
 
-cache = get_cache(settings.CACHE_BACKEND)
+def get_cache(backend, **kwargs):
+    """
+    Function to load a cache backend dynamically. This is flexible by design
+    to allow different use cases:
+
+    To load a backend with the old URI-based notation::
+
+        cache = get_cache('locmem://')
+
+    To load a backend that is pre-defined in the settings::
+
+        cache = get_cache('default')
+
+    To load a backend with its dotted import path,
+    including arbitrary options::
+
+        cache = get_cache('django.core.cache.backends.memcached.MemcachedCache', **{
+            'LOCATION': '127.0.0.1:11211', 'TIMEOUT': 30,
+        })
+
+    """
+    try:
+        if '://' in backend:
+            # for backwards compatibility
+            backend, location, params = parse_backend_uri(backend)
+            if backend in BACKENDS:
+                backend = 'django.core.cache.backends.%s' % BACKENDS[backend]
+            params.update(kwargs)
+            mod = importlib.import_module(backend)
+            backend_cls = mod.CacheClass
+        else:
+            backend, location, params = parse_backend_conf(backend, **kwargs)
+            mod_path, cls_name = backend.rsplit('.', 1)
+            mod = importlib.import_module(mod_path)
+            backend_cls = getattr(mod, cls_name)
+    except (AttributeError, ImportError), e:
+        raise InvalidCacheBackendError(
+            "Could not find backend '%s': %s" % (backend, e))
+    return backend_cls(location, params)
+
+cache = get_cache(DEFAULT_CACHE_ALIAS)
 
 # Some caches -- python-memcached in particular -- need to do a cleanup at the
 # end of a request cycle. If the cache provides a close() method, wire it up
 # here.
 if hasattr(cache, 'close'):
     signals.request_finished.connect(cache.close)
-

django/core/cache/backends/base.py

@@ -2,8 +2,10 @@
 
 import warnings
 
+from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured, DjangoRuntimeWarning
 from django.utils.encoding import smart_str
+from django.utils.importlib import import_module
 
 class InvalidCacheBackendError(ImproperlyConfigured):
     pass
@@ -15,38 +17,55 @@ class CacheKeyWarning(DjangoRuntimeWarning):
 MEMCACHE_MAX_KEY_LENGTH = 250
 
 def default_key_func(key, key_prefix, version):
-    """Default function to generate keys.
+    """
+    Default function to generate keys.
 
     Constructs the key used by all other methods. By default it prepends
-    the `key_prefix'. CACHE_KEY_FUNCTION can be used to specify an alternate
+    the `key_prefix'. KEY_FUNCTION can be used to specify an alternate
     function with custom key making behavior.
     """
     return ':'.join([key_prefix, str(version), smart_str(key)])
 
+def get_key_func(key_func):
+    """
+    Function to decide which key function to use.
+
+    Defaults to ``default_key_func``.
+    """
+    if key_func is not None:
+        if callable(key_func):
+            return key_func
+        else:
+            key_func_module_path, key_func_name = key_func.rsplit('.', 1)
+            key_func_module = import_module(key_func_module_path)
+            return getattr(key_func_module, key_func_name)
+    return default_key_func
+
 class BaseCache(object):
-    def __init__(self, params, key_prefix='', version=1, key_func=None):
-        timeout = params.get('timeout', 300)
+    def __init__(self, params):
+        timeout = params.get('timeout', params.get('TIMEOUT', 300))
         try:
             timeout = int(timeout)
         except (ValueError, TypeError):
             timeout = 300
         self.default_timeout = timeout
 
-        max_entries = params.get('max_entries', 300)
+        options = params.get('OPTIONS', {})
+        max_entries = params.get('max_entries', options.get('MAX_ENTRIES', 300))
         try:
             self._max_entries = int(max_entries)
         except (ValueError, TypeError):
             self._max_entries = 300
 
-        cull_frequency = params.get('cull_frequency', 3)
+        cull_frequency = params.get('cull_frequency', options.get('CULL_FREQUENCY', 3))
         try:
             self._cull_frequency = int(cull_frequency)
         except (ValueError, TypeError):
             self._cull_frequency = 3
 
-        self.key_prefix = smart_str(key_prefix)
-        self.version = version
-        self.key_func = key_func or default_key_func
+        self.key_prefix = smart_str(params.get('KEY_PREFIX', ''))
+        self.version = params.get('VERSION', 1)
+        self.key_func = get_key_func(params.get('KEY_FUNCTION', None))
 
     def make_key(self, key, version=None):
         """Constructs the key used by all other methods. By default it

django/core/cache/backends/db.py

@@ -25,16 +25,16 @@ class Options(object):
         self.managed = True
         self.proxy = False
 
-class BaseDatabaseCacheClass(BaseCache):
-    def __init__(self, table, params, key_prefix='', version=1, key_func=None):
-        BaseCache.__init__(self, params, key_prefix, version, key_func)
+class BaseDatabaseCache(BaseCache):
+    def __init__(self, table, params):
+        BaseCache.__init__(self, params)
         self._table = table
 
         class CacheEntry(object):
             _meta = Options(table)
         self.cache_model_class = CacheEntry
 
-class CacheClass(BaseDatabaseCacheClass):
+class DatabaseCache(BaseDatabaseCache):
     def get(self, key, default=None, version=None):
         key = self.make_key(key, version=version)
         self.validate_key(key)
@@ -140,3 +140,7 @@ class CacheClass(BaseDatabaseCacheClass):
         table = connections[db].ops.quote_name(self._table)
         cursor = connections[db].cursor()
         cursor.execute('DELETE FROM %s' % table)
+
+# For backwards compatibility
+class CacheClass(DatabaseCache):
+    pass

django/core/cache/backends/dummy.py

@@ -2,7 +2,7 @@
 
 from django.core.cache.backends.base import BaseCache
 
-class CacheClass(BaseCache):
+class DummyCache(BaseCache):
     def __init__(self, host, *args, **kwargs):
         BaseCache.__init__(self, *args, **kwargs)
 
@@ -40,3 +40,7 @@ class CacheClass(BaseCache):
 
     def clear(self):
         pass
+
+# For backwards compatibility
+class CacheClass(DummyCache):
+    pass

django/core/cache/backends/filebased.py

@@ -11,9 +11,9 @@ except ImportError:
 from django.core.cache.backends.base import BaseCache
 from django.utils.hashcompat import md5_constructor
 
-class CacheClass(BaseCache):
-    def __init__(self, dir, params, key_prefix='', version=1, key_func=None):
-        BaseCache.__init__(self, params, key_prefix, version, key_func)
+class FileBasedCache(BaseCache):
+    def __init__(self, dir, params):
+        BaseCache.__init__(self, params)
         self._dir = dir
         if not os.path.exists(self._dir):
             self._createdir()
@@ -161,3 +161,7 @@ class CacheClass(BaseCache):
             shutil.rmtree(self._dir)
         except (IOError, OSError):
             pass
+
+# For backwards compatibility
+class CacheClass(FileBasedCache):
+    pass

django/core/cache/backends/locmem.py

@@ -9,12 +9,19 @@ except ImportError:
 from django.core.cache.backends.base import BaseCache
 from django.utils.synch import RWLock
 
-class CacheClass(BaseCache):
-    def __init__(self, _, params, key_prefix='', version=1, key_func=None):
-        BaseCache.__init__(self, params, key_prefix, version, key_func)
-        self._cache = {}
-        self._expire_info = {}
-        self._lock = RWLock()
+# Global in-memory store of cache data. Keyed by name, to provide
+# multiple named local memory caches.
+_caches = {}
+_expire_info = {}
+_locks = {}
+
+class LocMemCache(BaseCache):
+    def __init__(self, name, params):
+        BaseCache.__init__(self, params)
+        global _caches, _expire_info, _locks
+        self._cache = _caches.setdefault(name, {})
+        self._expire_info = _expire_info.setdefault(name, {})
+        self._lock = _locks.setdefault(name, RWLock())
 
     def add(self, key, value, timeout=None, version=None):
         key = self.make_key(key, version=version)
@@ -133,3 +140,7 @@ class CacheClass(BaseCache):
     def clear(self):
         self._cache.clear()
         self._expire_info.clear()
+
+# For backwards compatibility
+class CacheClass(LocMemCache):
+    pass

django/core/cache/backends/memcached.py

@@ -1,26 +1,34 @@
 "Memcached cache backend"
 
 import time
+from threading import local
 
 from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
+from django.utils import importlib
 
-try:
-    import cmemcache as memcache
-    import warnings
-    warnings.warn(
-        "Support for the 'cmemcache' library has been deprecated. Please use python-memcached instead.",
-        DeprecationWarning
-    )
-except ImportError:
-    try:
-        import memcache
-    except:
-        raise InvalidCacheBackendError("Memcached cache backend requires either the 'memcache' or 'cmemcache' library")
+class BaseMemcachedCache(BaseCache):
+    def __init__(self, server, params, library, value_not_found_exception):
+        super(BaseMemcachedCache, self).__init__(params)
+        if isinstance(server, basestring):
+            self._servers = server.split(';')
+        else:
+            self._servers = server
 
-class CacheClass(BaseCache):
-    def __init__(self, server, params, key_prefix='', version=1, key_func=None):
-        BaseCache.__init__(self, params, key_prefix, version, key_func)
-        self._cache = memcache.Client(server.split(';'))
+        # The exception type to catch from the underlying library for a key
+        # that was not found. This is a ValueError for python-memcache,
+        # pylibmc.NotFound for pylibmc, and cmemcache will return None without
+        # raising an exception.
+        self.LibraryValueNotFoundException = value_not_found_exception
+
+        self._lib = library
+        self._options = params.get('OPTIONS', None)
+
+    @property
+    def _cache(self):
+        """
+        Implements transparent thread-safe access to a memcached client.
+        """
+        return self._lib.Client(self._servers)
 
     def _get_memcache_timeout(self, timeout):
         """
@@ -79,13 +87,13 @@ class CacheClass(BaseCache):
             val = self._cache.incr(key, delta)
 
         # python-memcache responds to incr on non-existent keys by
-        # raising a ValueError. Cmemcache returns None. In both
-        # cases, we should raise a ValueError though.
-        except ValueError:
+        # raising a ValueError, pylibmc by raising a pylibmc.NotFound
+        # and Cmemcache returns None. In all cases,
+        # we should raise a ValueError though.
+        except self.LibraryValueNotFoundException:
             val = None
         if val is None:
             raise ValueError("Key '%s' not found" % key)
-
         return val
 
     def decr(self, key, delta=1, version=None):
@@ -93,10 +101,11 @@ class CacheClass(BaseCache):
         try:
             val = self._cache.decr(key, delta)
 
-        # python-memcache responds to decr on non-existent keys by
-        # raising a ValueError. Cmemcache returns None. In both
-        # cases, we should raise a ValueError though.
-        except ValueError:
+        # python-memcache responds to incr on non-existent keys by
+        # raising a ValueError, pylibmc by raising a pylibmc.NotFound
+        # and Cmemcache returns None. In all cases,
+        # we should raise a ValueError though.
+        except self.LibraryValueNotFoundException:
             val = None
         if val is None:
             raise ValueError("Key '%s' not found" % key)
@@ -117,3 +126,59 @@ class CacheClass(BaseCache):
 
     def clear(self):
         self._cache.flush_all()
+
+# For backwards compatibility -- the default cache class tries a
+# cascading lookup of cmemcache, then memcache.
+class CacheClass(BaseMemcachedCache):
+    def __init__(self, server, params):
+        try:
+            import cmemcache as memcache
+            import warnings
+            warnings.warn(
+                "Support for the 'cmemcache' library has been deprecated. Please use python-memcached or pyblimc instead.",
+                DeprecationWarning
+            )
+        except ImportError:
+            try:
+                import memcache
+            except:
+                raise InvalidCacheBackendError(
+                    "Memcached cache backend requires either the 'memcache' or 'cmemcache' library"
+                    )
+        super(CacheClass, self).__init__(server, params,
+                                         library=memcache,
+                                         value_not_found_exception=ValueError)
+
+class MemcachedCache(BaseMemcachedCache):
+    "An implementation of a cache binding using python-memcached"
+    def __init__(self, server, params):
+        import memcache
+        super(MemcachedCache, self).__init__(server, params,
+                                             library=memcache,
+                                             value_not_found_exception=ValueError)
+
+class PyLibMCCache(BaseMemcachedCache):
+    "An implementation of a cache binding using pylibmc"
+    def __init__(self, server, params):
+        import pylibmc
+        self._local = local()
+        super(PyLibMCCache, self).__init__(server, params,
+                                           library=pylibmc,
+                                           value_not_found_exception=pylibmc.NotFound)
+
+    @property
+    def _cache(self):
+        # PylibMC uses cache options as the 'behaviors' attribute.
+        # It also needs to use threadlocals, because some versions of
+        # PylibMC don't play well with the GIL.
+        client = getattr(self._local, 'client', None)
+        if client:
+            return client
+
+        client = self._lib.Client(self._servers)
+        if self._options:
+            client.behaviors = self._options
+
+        self._local.client = client
+
+        return client