From 272eab5cd8ebafc68af3bca65f4f0d217c49ce53 Mon Sep 17 00:00:00 2001 From: Jacob Kaplan-Moss Date: Sun, 25 Sep 2005 20:01:35 +0000 Subject: [PATCH] Added "locmem" and "file" cache backends. "locmem" is a thread-safe local-memory cache, and "file" is a file-based cache. This refs #515; much thanks to Eugene Lazutkin! git-svn-id: http://code.djangoproject.com/svn/django/trunk@686 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/core/cache.py | 156 +++++++++++++++++++++++++++++++++++++++--- django/utils/synch.py | 85 +++++++++++++++++++++++ 2 files changed, 230 insertions(+), 11 deletions(-) create mode 100644 django/utils/synch.py diff --git a/django/core/cache.py b/django/core/cache.py index 46ace6c085..18ff75f19f 100644 --- a/django/core/cache.py +++ b/django/core/cache.py @@ -15,16 +15,21 @@ The CACHE_BACKEND setting is a quasi-URI; examples are: memcached://127.0.0.1:11211/ A memcached backend; the server is running on localhost port 11211. - pgsql://tablename/ A pgsql backend (the pgsql backend uses - the same database/username as the rest of - the CMS, so only a table name is needed.) + sql://tablename/ A SQL backend. If you use this backend, + you must have django.contrib.cache in + INSTALLED_APPS, and you must have installed + the tables for django.contrib.cache. - file:///var/tmp/django.cache/ A file-based cache at /var/tmp/django.cache + file:///var/tmp/django_cache/ A file-based cache stored in the directory + /var/tmp/django_cache/. simple:/// A simple single-process memory cache; you probably don't want to use this except for testing. Note that this cache backend is NOT threadsafe! + + locmem:/// A more sophisticaed local memory cache; + this is multi-process- and thread-safe. All caches may take arguments; these are given in query-string style. Valid arguments are: @@ -50,13 +55,10 @@ arguments are: For example: memcached://127.0.0.1:11211/?timeout=60 - pgsql://tablename/?timeout=120&max_entries=500&cull_percentage=4 + sql://tablename/?timeout=120&max_entries=500&cull_percentage=4 Invalid arguments are silently ignored, as are invalid values of known arguments. - -So far, only the memcached and simple backend have been implemented; backends -using postgres, and file-system storage are planned. """ ############## @@ -181,13 +183,15 @@ class _SimpleCache(_Cache): def get(self, key, default=None): now = time.time() - exp = self._expire_info.get(key, now) - if exp is not None and exp < now: + exp = self._expire_info.get(key) + if exp is None: + return default + elif exp < now: del self._cache[key] del self._expire_info[key] return default else: - return self._cache.get(key, default) + return self._cache[key] def set(self, key, value, timeout=None): if len(self._cache) >= self._max_entries: @@ -219,6 +223,134 @@ class _SimpleCache(_Cache): for k in doomed: self.delete(k) +############################### +# Thread-safe in-memory cache # +############################### + +try: + import cPickle as pickle +except ImportError: + import pickle +from django.utils.synch import RWLock + +class _LocMemCache(_SimpleCache): + """Thread-safe in-memory cache""" + + def __init__(self, host, params): + _SimpleCache.__init__(self, host, params) + self._lock = RWLock() + + def get(self, key, default=None): + should_delete = False + self._lock.reader_enters() + try: + now = time.time() + exp = self._expire_info.get(key) + if exp is None: + return default + elif exp < now: + should_delete = True + else: + return self._cache[key] + finally: + self._lock.reader_leaves() + if should_delete: + self._lock.writer_enters() + try: + del self._cache[key] + del self._expire_info[key] + return default + finally: + self._lock.writer_leaves() + + def set(self, key, value, timeout=None): + self._lock.writer_enters() + try: + _SimpleCache.set(self, key, value, timeout) + finally: + self._lock.writer_leaves() + + def delete(self, key): + self._lock.writer_enters() + try: + _SimpleCache.delete(self, key) + finally: + self._lock.writer_leaves() + +#################### +# File-based cache # +#################### + +import os +import urllib + +class _FileCache(_SimpleCache): + """File-based cache""" + + def __init__(self, dir, params): + self._dir = dir + if not os.path.exists(self._dir): + try: + os.makedirs(self._dir) + except OSError: + raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir + _SimpleCache.__init__(self, dir, params) + del self._cache + del self._expire_info + + def get(self, key, default=None): + fname = self._key_to_file(key) + try: + f = open(fname, 'rb') + exp = pickle.load(f) + now = time.time() + if exp < now: + f.close() + os.remove(fname) + else: + return pickle.load(f) + except (IOError, pickle.PickleError): + pass + return default + + def set(self, key, value, timeout=None): + fname = self._key_to_file(key) + if timeout is None: + timeout = self.default_timeout + filelist = os.listdir(self._dir) + if len(filelist) > self._max_entries: + self._cull(filelist) + try: + f = open(fname, 'wb') + now = time.time() + pickle.dump(now + timeout, f, 2) + pickle.dump(value, f, 2) + except (IOError, OSError): + raise + + def delete(self, key): + try: + os.remove(self._key_to_file(key)) + except (IOError, OSError): + pass + + def has_key(self, key): + return os.path.exists(self._key_to_file(key)) + + def _cull(self, filelist): + if self.cull_frequency == 0: + doomed = filelist + else: + doomed = [k for (i, k) in enumerate(filelist) if i % self._cull_frequency == 0] + for fname in doomed: + try: + os.remove(os.path.join(self._dir, fname)) + except (IOError, OSError): + pass + + def _key_to_file(self, key): + return os.path.join(self._dir, urllib.quote_plus(key)) + ########################################## # Read settings and load a cache backend # ########################################## @@ -228,6 +360,8 @@ from cgi import parse_qsl _BACKENDS = { 'memcached' : _MemcachedCache, 'simple' : _SimpleCache, + 'locmem' : _LocMemCache, + 'file' : _FileCache, } def get_cache(backend_uri): diff --git a/django/utils/synch.py b/django/utils/synch.py new file mode 100644 index 0000000000..1e6b546a78 --- /dev/null +++ b/django/utils/synch.py @@ -0,0 +1,85 @@ +""" +Synchronization primitives: + + - reader-writer lock (preference to writers) + +(Contributed to Django by eugene@lazutkin.com) +""" + +import threading + +class RWLock: + """ + Classic implementation of reader-writer lock with preference to writers. + + Readers can access a resource simultaneously. + Writers get an exclusive access. + + API is self-descriptive: + reader_enters() + reader_leaves() + writer_enters() + writer_leaves() + """ + + def __init__(self): + self.mutex = threading.RLock() + self.can_read = threading.Semaphore(0) + self.can_write = threading.Semaphore(0) + self.active_readers = 0 + self.active_writers = 0 + self.waiting_readers = 0 + self.waiting_writers = 0 + + def reader_enters(self): + self.mutex.acquire() + try: + if self.active_writers == 0 and self.waiting_writers == 0: + self.active_readers += 1 + self.can_read.release() + else: + self.waiting_readers += 1 + finally: + self.mutex.release() + self.can_read.acquire() + + def reader_leaves(self): + self.mutex.acquire() + try: + self.active_readers -= 1 + if self.active_readers == 0 and self.waiting_writers != 0: + self.active_writers += 1 + self.waiting_writers -= 1 + self.can_write.release() + finally: + self.mutex.release() + + def writer_enters(self): + self.mutex.acquire() + try: + if self.active_writers == 0 and self.waiting_writers == 0 and self.active_readers == 0: + self.active_writers += 1 + self.can_write.release() + else: + self.waiting_writers += 1 + finally: + self.mutex.release() + self.can_write.acquire() + + def writer_leaves(self): + self.mutex.acquire() + try: + self.active_writers -= 1 + if self.waiting_writers != 0: + self.active_writers += 1 + self.waiting_writers -= 1 + self.can_write.release() + elif self.waiting_readers != 0: + t = self.waiting_readers + self.waiting_readers = 0 + self.active_readers += t + while t > 0: + self.can_read.release() + t -= 1 + finally: + self.mutex.release()