forked from s434650/CatOrNot
914 lines
31 KiB
Python
914 lines
31 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
"""
|
||
|
|
werkzeug.contrib.cache
|
||
|
|
~~~~~~~~~~~~~~~~~~~~~~
|
||
|
|
|
||
|
|
The main problem with dynamic Web sites is, well, they're dynamic. Each
|
||
|
|
time a user requests a page, the webserver executes a lot of code, queries
|
||
|
|
the database, renders templates until the visitor gets the page he sees.
|
||
|
|
|
||
|
|
This is a lot more expensive than just loading a file from the file system
|
||
|
|
and sending it to the visitor.
|
||
|
|
|
||
|
|
For most Web applications, this overhead isn't a big deal but once it
|
||
|
|
becomes, you will be glad to have a cache system in place.
|
||
|
|
|
||
|
|
How Caching Works
|
||
|
|
=================
|
||
|
|
|
||
|
|
Caching is pretty simple. Basically you have a cache object lurking around
|
||
|
|
somewhere that is connected to a remote cache or the file system or
|
||
|
|
something else. When the request comes in you check if the current page
|
||
|
|
is already in the cache and if so, you're returning it from the cache.
|
||
|
|
Otherwise you generate the page and put it into the cache. (Or a fragment
|
||
|
|
of the page, you don't have to cache the full thing)
|
||
|
|
|
||
|
|
Here is a simple example of how to cache a sidebar for 5 minutes::
|
||
|
|
|
||
|
|
def get_sidebar(user):
|
||
|
|
identifier = 'sidebar_for/user%d' % user.id
|
||
|
|
value = cache.get(identifier)
|
||
|
|
if value is not None:
|
||
|
|
return value
|
||
|
|
value = generate_sidebar_for(user=user)
|
||
|
|
cache.set(identifier, value, timeout=60 * 5)
|
||
|
|
return value
|
||
|
|
|
||
|
|
Creating a Cache Object
|
||
|
|
=======================
|
||
|
|
|
||
|
|
To create a cache object you just import the cache system of your choice
|
||
|
|
from the cache module and instantiate it. Then you can start working
|
||
|
|
with that object:
|
||
|
|
|
||
|
|
>>> from werkzeug.contrib.cache import SimpleCache
|
||
|
|
>>> c = SimpleCache()
|
||
|
|
>>> c.set("foo", "value")
|
||
|
|
>>> c.get("foo")
|
||
|
|
'value'
|
||
|
|
>>> c.get("missing") is None
|
||
|
|
True
|
||
|
|
|
||
|
|
Please keep in mind that you have to create the cache and put it somewhere
|
||
|
|
you have access to it (either as a module global you can import or you just
|
||
|
|
put it into your WSGI application).
|
||
|
|
|
||
|
|
:copyright: (c) 2014 by the Werkzeug Team, see AUTHORS for more details.
|
||
|
|
:license: BSD, see LICENSE for more details.
|
||
|
|
"""
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import errno
|
||
|
|
import tempfile
|
||
|
|
import platform
|
||
|
|
from hashlib import md5
|
||
|
|
from time import time
|
||
|
|
try:
|
||
|
|
import cPickle as pickle
|
||
|
|
except ImportError: # pragma: no cover
|
||
|
|
import pickle
|
||
|
|
|
||
|
|
from werkzeug._compat import iteritems, string_types, text_type, \
|
||
|
|
integer_types, to_native
|
||
|
|
from werkzeug.posixemulation import rename
|
||
|
|
|
||
|
|
|
||
|
|
def _items(mappingorseq):
|
||
|
|
"""Wrapper for efficient iteration over mappings represented by dicts
|
||
|
|
or sequences::
|
||
|
|
|
||
|
|
>>> for k, v in _items((i, i*i) for i in xrange(5)):
|
||
|
|
... assert k*k == v
|
||
|
|
|
||
|
|
>>> for k, v in _items(dict((i, i*i) for i in xrange(5))):
|
||
|
|
... assert k*k == v
|
||
|
|
|
||
|
|
"""
|
||
|
|
if hasattr(mappingorseq, 'items'):
|
||
|
|
return iteritems(mappingorseq)
|
||
|
|
return mappingorseq
|
||
|
|
|
||
|
|
|
||
|
|
class BaseCache(object):
|
||
|
|
|
||
|
|
"""Baseclass for the cache systems. All the cache systems implement this
|
||
|
|
API or a superset of it.
|
||
|
|
|
||
|
|
:param default_timeout: the default timeout (in seconds) that is used if
|
||
|
|
no timeout is specified on :meth:`set`. A timeout
|
||
|
|
of 0 indicates that the cache never expires.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, default_timeout=300):
|
||
|
|
self.default_timeout = default_timeout
|
||
|
|
|
||
|
|
def _normalize_timeout(self, timeout):
|
||
|
|
if timeout is None:
|
||
|
|
timeout = self.default_timeout
|
||
|
|
return timeout
|
||
|
|
|
||
|
|
def get(self, key):
|
||
|
|
"""Look up key in the cache and return the value for it.
|
||
|
|
|
||
|
|
:param key: the key to be looked up.
|
||
|
|
:returns: The value if it exists and is readable, else ``None``.
|
||
|
|
"""
|
||
|
|
return None
|
||
|
|
|
||
|
|
def delete(self, key):
|
||
|
|
"""Delete `key` from the cache.
|
||
|
|
|
||
|
|
:param key: the key to delete.
|
||
|
|
:returns: Whether the key existed and has been deleted.
|
||
|
|
:rtype: boolean
|
||
|
|
"""
|
||
|
|
return True
|
||
|
|
|
||
|
|
def get_many(self, *keys):
|
||
|
|
"""Returns a list of values for the given keys.
|
||
|
|
For each key an item in the list is created::
|
||
|
|
|
||
|
|
foo, bar = cache.get_many("foo", "bar")
|
||
|
|
|
||
|
|
Has the same error handling as :meth:`get`.
|
||
|
|
|
||
|
|
:param keys: The function accepts multiple keys as positional
|
||
|
|
arguments.
|
||
|
|
"""
|
||
|
|
return [self.get(k) for k in keys]
|
||
|
|
|
||
|
|
def get_dict(self, *keys):
|
||
|
|
"""Like :meth:`get_many` but return a dict::
|
||
|
|
|
||
|
|
d = cache.get_dict("foo", "bar")
|
||
|
|
foo = d["foo"]
|
||
|
|
bar = d["bar"]
|
||
|
|
|
||
|
|
:param keys: The function accepts multiple keys as positional
|
||
|
|
arguments.
|
||
|
|
"""
|
||
|
|
return dict(zip(keys, self.get_many(*keys)))
|
||
|
|
|
||
|
|
def set(self, key, value, timeout=None):
|
||
|
|
"""Add a new key/value to the cache (overwrites value, if key already
|
||
|
|
exists in the cache).
|
||
|
|
|
||
|
|
:param key: the key to set
|
||
|
|
:param value: the value for the key
|
||
|
|
:param timeout: the cache timeout for the key in seconds (if not
|
||
|
|
specified, it uses the default timeout). A timeout of
|
||
|
|
0 idicates that the cache never expires.
|
||
|
|
:returns: ``True`` if key has been updated, ``False`` for backend
|
||
|
|
errors. Pickling errors, however, will raise a subclass of
|
||
|
|
``pickle.PickleError``.
|
||
|
|
:rtype: boolean
|
||
|
|
"""
|
||
|
|
return True
|
||
|
|
|
||
|
|
def add(self, key, value, timeout=None):
|
||
|
|
"""Works like :meth:`set` but does not overwrite the values of already
|
||
|
|
existing keys.
|
||
|
|
|
||
|
|
:param key: the key to set
|
||
|
|
:param value: the value for the key
|
||
|
|
:param timeout: the cache timeout for the key in seconds (if not
|
||
|
|
specified, it uses the default timeout). A timeout of
|
||
|
|
0 idicates that the cache never expires.
|
||
|
|
:returns: Same as :meth:`set`, but also ``False`` for already
|
||
|
|
existing keys.
|
||
|
|
:rtype: boolean
|
||
|
|
"""
|
||
|
|
return True
|
||
|
|
|
||
|
|
def set_many(self, mapping, timeout=None):
|
||
|
|
"""Sets multiple keys and values from a mapping.
|
||
|
|
|
||
|
|
:param mapping: a mapping with the keys/values to set.
|
||
|
|
:param timeout: the cache timeout for the key in seconds (if not
|
||
|
|
specified, it uses the default timeout). A timeout of
|
||
|
|
0 idicates that the cache never expires.
|
||
|
|
:returns: Whether all given keys have been set.
|
||
|
|
:rtype: boolean
|
||
|
|
"""
|
||
|
|
rv = True
|
||
|
|
for key, value in _items(mapping):
|
||
|
|
if not self.set(key, value, timeout):
|
||
|
|
rv = False
|
||
|
|
return rv
|
||
|
|
|
||
|
|
def delete_many(self, *keys):
|
||
|
|
"""Deletes multiple keys at once.
|
||
|
|
|
||
|
|
:param keys: The function accepts multiple keys as positional
|
||
|
|
arguments.
|
||
|
|
:returns: Whether all given keys have been deleted.
|
||
|
|
:rtype: boolean
|
||
|
|
"""
|
||
|
|
return all(self.delete(key) for key in keys)
|
||
|
|
|
||
|
|
def has(self, key):
|
||
|
|
"""Checks if a key exists in the cache without returning it. This is a
|
||
|
|
cheap operation that bypasses loading the actual data on the backend.
|
||
|
|
|
||
|
|
This method is optional and may not be implemented on all caches.
|
||
|
|
|
||
|
|
:param key: the key to check
|
||
|
|
"""
|
||
|
|
raise NotImplementedError(
|
||
|
|
'%s doesn\'t have an efficient implementation of `has`. That '
|
||
|
|
'means it is impossible to check whether a key exists without '
|
||
|
|
'fully loading the key\'s data. Consider using `self.get` '
|
||
|
|
'explicitly if you don\'t care about performance.'
|
||
|
|
)
|
||
|
|
|
||
|
|
def clear(self):
|
||
|
|
"""Clears the cache. Keep in mind that not all caches support
|
||
|
|
completely clearing the cache.
|
||
|
|
|
||
|
|
:returns: Whether the cache has been cleared.
|
||
|
|
:rtype: boolean
|
||
|
|
"""
|
||
|
|
return True
|
||
|
|
|
||
|
|
def inc(self, key, delta=1):
|
||
|
|
"""Increments the value of a key by `delta`. If the key does
|
||
|
|
not yet exist it is initialized with `delta`.
|
||
|
|
|
||
|
|
For supporting caches this is an atomic operation.
|
||
|
|
|
||
|
|
:param key: the key to increment.
|
||
|
|
:param delta: the delta to add.
|
||
|
|
:returns: The new value or ``None`` for backend errors.
|
||
|
|
"""
|
||
|
|
value = (self.get(key) or 0) + delta
|
||
|
|
return value if self.set(key, value) else None
|
||
|
|
|
||
|
|
def dec(self, key, delta=1):
|
||
|
|
"""Decrements the value of a key by `delta`. If the key does
|
||
|
|
not yet exist it is initialized with `-delta`.
|
||
|
|
|
||
|
|
For supporting caches this is an atomic operation.
|
||
|
|
|
||
|
|
:param key: the key to increment.
|
||
|
|
:param delta: the delta to subtract.
|
||
|
|
:returns: The new value or `None` for backend errors.
|
||
|
|
"""
|
||
|
|
value = (self.get(key) or 0) - delta
|
||
|
|
return value if self.set(key, value) else None
|
||
|
|
|
||
|
|
|
||
|
|
class NullCache(BaseCache):
|
||
|
|
|
||
|
|
"""A cache that doesn't cache. This can be useful for unit testing.
|
||
|
|
|
||
|
|
:param default_timeout: a dummy parameter that is ignored but exists
|
||
|
|
for API compatibility with other caches.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def has(self, key):
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
class SimpleCache(BaseCache):
|
||
|
|
|
||
|
|
"""Simple memory cache for single process environments. This class exists
|
||
|
|
mainly for the development server and is not 100% thread safe. It tries
|
||
|
|
to use as many atomic operations as possible and no locks for simplicity
|
||
|
|
but it could happen under heavy load that keys are added multiple times.
|
||
|
|
|
||
|
|
:param threshold: the maximum number of items the cache stores before
|
||
|
|
it starts deleting some.
|
||
|
|
:param default_timeout: the default timeout that is used if no timeout is
|
||
|
|
specified on :meth:`~BaseCache.set`. A timeout of
|
||
|
|
0 indicates that the cache never expires.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, threshold=500, default_timeout=300):
|
||
|
|
BaseCache.__init__(self, default_timeout)
|
||
|
|
self._cache = {}
|
||
|
|
self.clear = self._cache.clear
|
||
|
|
self._threshold = threshold
|
||
|
|
|
||
|
|
def _prune(self):
|
||
|
|
if len(self._cache) > self._threshold:
|
||
|
|
now = time()
|
||
|
|
toremove = []
|
||
|
|
for idx, (key, (expires, _)) in enumerate(self._cache.items()):
|
||
|
|
if (expires != 0 and expires <= now) or idx % 3 == 0:
|
||
|
|
toremove.append(key)
|
||
|
|
for key in toremove:
|
||
|
|
self._cache.pop(key, None)
|
||
|
|
|
||
|
|
def _normalize_timeout(self, timeout):
|
||
|
|
timeout = BaseCache._normalize_timeout(self, timeout)
|
||
|
|
if timeout > 0:
|
||
|
|
timeout = time() + timeout
|
||
|
|
return timeout
|
||
|
|
|
||
|
|
def get(self, key):
|
||
|
|
try:
|
||
|
|
expires, value = self._cache[key]
|
||
|
|
if expires == 0 or expires > time():
|
||
|
|
return pickle.loads(value)
|
||
|
|
except (KeyError, pickle.PickleError):
|
||
|
|
return None
|
||
|
|
|
||
|
|
def set(self, key, value, timeout=None):
|
||
|
|
expires = self._normalize_timeout(timeout)
|
||
|
|
self._prune()
|
||
|
|
self._cache[key] = (expires, pickle.dumps(value,
|
||
|
|
pickle.HIGHEST_PROTOCOL))
|
||
|
|
return True
|
||
|
|
|
||
|
|
def add(self, key, value, timeout=None):
|
||
|
|
expires = self._normalize_timeout(timeout)
|
||
|
|
self._prune()
|
||
|
|
item = (expires, pickle.dumps(value,
|
||
|
|
pickle.HIGHEST_PROTOCOL))
|
||
|
|
if key in self._cache:
|
||
|
|
return False
|
||
|
|
self._cache.setdefault(key, item)
|
||
|
|
return True
|
||
|
|
|
||
|
|
def delete(self, key):
|
||
|
|
return self._cache.pop(key, None) is not None
|
||
|
|
|
||
|
|
def has(self, key):
|
||
|
|
try:
|
||
|
|
expires, value = self._cache[key]
|
||
|
|
return expires == 0 or expires > time()
|
||
|
|
except KeyError:
|
||
|
|
return False
|
||
|
|
|
||
|
|
_test_memcached_key = re.compile(r'[^\x00-\x21\xff]{1,250}$').match
|
||
|
|
|
||
|
|
|
||
|
|
class MemcachedCache(BaseCache):
|
||
|
|
|
||
|
|
"""A cache that uses memcached as backend.
|
||
|
|
|
||
|
|
The first argument can either be an object that resembles the API of a
|
||
|
|
:class:`memcache.Client` or a tuple/list of server addresses. In the
|
||
|
|
event that a tuple/list is passed, Werkzeug tries to import the best
|
||
|
|
available memcache library.
|
||
|
|
|
||
|
|
This cache looks into the following packages/modules to find bindings for
|
||
|
|
memcached:
|
||
|
|
|
||
|
|
- ``pylibmc``
|
||
|
|
- ``google.appengine.api.memcached``
|
||
|
|
- ``memcached``
|
||
|
|
- ``libmc``
|
||
|
|
|
||
|
|
Implementation notes: This cache backend works around some limitations in
|
||
|
|
memcached to simplify the interface. For example unicode keys are encoded
|
||
|
|
to utf-8 on the fly. Methods such as :meth:`~BaseCache.get_dict` return
|
||
|
|
the keys in the same format as passed. Furthermore all get methods
|
||
|
|
silently ignore key errors to not cause problems when untrusted user data
|
||
|
|
is passed to the get methods which is often the case in web applications.
|
||
|
|
|
||
|
|
:param servers: a list or tuple of server addresses or alternatively
|
||
|
|
a :class:`memcache.Client` or a compatible client.
|
||
|
|
:param default_timeout: the default timeout that is used if no timeout is
|
||
|
|
specified on :meth:`~BaseCache.set`. A timeout of
|
||
|
|
0 indicates that the cache never expires.
|
||
|
|
:param key_prefix: a prefix that is added before all keys. This makes it
|
||
|
|
possible to use the same memcached server for different
|
||
|
|
applications. Keep in mind that
|
||
|
|
:meth:`~BaseCache.clear` will also clear keys with a
|
||
|
|
different prefix.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, servers=None, default_timeout=300, key_prefix=None):
|
||
|
|
BaseCache.__init__(self, default_timeout)
|
||
|
|
if servers is None or isinstance(servers, (list, tuple)):
|
||
|
|
if servers is None:
|
||
|
|
servers = ['127.0.0.1:11211']
|
||
|
|
self._client = self.import_preferred_memcache_lib(servers)
|
||
|
|
if self._client is None:
|
||
|
|
raise RuntimeError('no memcache module found')
|
||
|
|
else:
|
||
|
|
# NOTE: servers is actually an already initialized memcache
|
||
|
|
# client.
|
||
|
|
self._client = servers
|
||
|
|
|
||
|
|
self.key_prefix = to_native(key_prefix)
|
||
|
|
|
||
|
|
def _normalize_key(self, key):
|
||
|
|
key = to_native(key, 'utf-8')
|
||
|
|
if self.key_prefix:
|
||
|
|
key = self.key_prefix + key
|
||
|
|
return key
|
||
|
|
|
||
|
|
def _normalize_timeout(self, timeout):
|
||
|
|
timeout = BaseCache._normalize_timeout(self, timeout)
|
||
|
|
if timeout > 0:
|
||
|
|
timeout = int(time()) + timeout
|
||
|
|
return timeout
|
||
|
|
|
||
|
|
def get(self, key):
|
||
|
|
key = self._normalize_key(key)
|
||
|
|
# memcached doesn't support keys longer than that. Because often
|
||
|
|
# checks for so long keys can occur because it's tested from user
|
||
|
|
# submitted data etc we fail silently for getting.
|
||
|
|
if _test_memcached_key(key):
|
||
|
|
return self._client.get(key)
|
||
|
|
|
||
|
|
def get_dict(self, *keys):
|
||
|
|
key_mapping = {}
|
||
|
|
have_encoded_keys = False
|
||
|
|
for key in keys:
|
||
|
|
encoded_key = self._normalize_key(key)
|
||
|
|
if not isinstance(key, str):
|
||
|
|
have_encoded_keys = True
|
||
|
|
if _test_memcached_key(key):
|
||
|
|
key_mapping[encoded_key] = key
|
||
|
|
_keys = list(key_mapping)
|
||
|
|
d = rv = self._client.get_multi(_keys)
|
||
|
|
if have_encoded_keys or self.key_prefix:
|
||
|
|
rv = {}
|
||
|
|
for key, value in iteritems(d):
|
||
|
|
rv[key_mapping[key]] = value
|
||
|
|
if len(rv) < len(keys):
|
||
|
|
for key in keys:
|
||
|
|
if key not in rv:
|
||
|
|
rv[key] = None
|
||
|
|
return rv
|
||
|
|
|
||
|
|
def add(self, key, value, timeout=None):
|
||
|
|
key = self._normalize_key(key)
|
||
|
|
timeout = self._normalize_timeout(timeout)
|
||
|
|
return self._client.add(key, value, timeout)
|
||
|
|
|
||
|
|
def set(self, key, value, timeout=None):
|
||
|
|
key = self._normalize_key(key)
|
||
|
|
timeout = self._normalize_timeout(timeout)
|
||
|
|
return self._client.set(key, value, timeout)
|
||
|
|
|
||
|
|
def get_many(self, *keys):
|
||
|
|
d = self.get_dict(*keys)
|
||
|
|
return [d[key] for key in keys]
|
||
|
|
|
||
|
|
def set_many(self, mapping, timeout=None):
|
||
|
|
new_mapping = {}
|
||
|
|
for key, value in _items(mapping):
|
||
|
|
key = self._normalize_key(key)
|
||
|
|
new_mapping[key] = value
|
||
|
|
|
||
|
|
timeout = self._normalize_timeout(timeout)
|
||
|
|
failed_keys = self._client.set_multi(new_mapping, timeout)
|
||
|
|
return not failed_keys
|
||
|
|
|
||
|
|
def delete(self, key):
|
||
|
|
key = self._normalize_key(key)
|
||
|
|
if _test_memcached_key(key):
|
||
|
|
return self._client.delete(key)
|
||
|
|
|
||
|
|
def delete_many(self, *keys):
|
||
|
|
new_keys = []
|
||
|
|
for key in keys:
|
||
|
|
key = self._normalize_key(key)
|
||
|
|
if _test_memcached_key(key):
|
||
|
|
new_keys.append(key)
|
||
|
|
return self._client.delete_multi(new_keys)
|
||
|
|
|
||
|
|
def has(self, key):
|
||
|
|
key = self._normalize_key(key)
|
||
|
|
if _test_memcached_key(key):
|
||
|
|
return self._client.append(key, '')
|
||
|
|
return False
|
||
|
|
|
||
|
|
def clear(self):
|
||
|
|
return self._client.flush_all()
|
||
|
|
|
||
|
|
def inc(self, key, delta=1):
|
||
|
|
key = self._normalize_key(key)
|
||
|
|
return self._client.incr(key, delta)
|
||
|
|
|
||
|
|
def dec(self, key, delta=1):
|
||
|
|
key = self._normalize_key(key)
|
||
|
|
return self._client.decr(key, delta)
|
||
|
|
|
||
|
|
def import_preferred_memcache_lib(self, servers):
|
||
|
|
"""Returns an initialized memcache client. Used by the constructor."""
|
||
|
|
try:
|
||
|
|
import pylibmc
|
||
|
|
except ImportError:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
return pylibmc.Client(servers)
|
||
|
|
|
||
|
|
try:
|
||
|
|
from google.appengine.api import memcache
|
||
|
|
except ImportError:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
return memcache.Client()
|
||
|
|
|
||
|
|
try:
|
||
|
|
import memcache
|
||
|
|
except ImportError:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
return memcache.Client(servers)
|
||
|
|
|
||
|
|
try:
|
||
|
|
import libmc
|
||
|
|
except ImportError:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
return libmc.Client(servers)
|
||
|
|
|
||
|
|
|
||
|
|
# backwards compatibility
|
||
|
|
GAEMemcachedCache = MemcachedCache
|
||
|
|
|
||
|
|
|
||
|
|
class RedisCache(BaseCache):
|
||
|
|
|
||
|
|
"""Uses the Redis key-value store as a cache backend.
|
||
|
|
|
||
|
|
The first argument can be either a string denoting address of the Redis
|
||
|
|
server or an object resembling an instance of a redis.Redis class.
|
||
|
|
|
||
|
|
Note: Python Redis API already takes care of encoding unicode strings on
|
||
|
|
the fly.
|
||
|
|
|
||
|
|
.. versionadded:: 0.7
|
||
|
|
|
||
|
|
.. versionadded:: 0.8
|
||
|
|
`key_prefix` was added.
|
||
|
|
|
||
|
|
.. versionchanged:: 0.8
|
||
|
|
This cache backend now properly serializes objects.
|
||
|
|
|
||
|
|
.. versionchanged:: 0.8.3
|
||
|
|
This cache backend now supports password authentication.
|
||
|
|
|
||
|
|
.. versionchanged:: 0.10
|
||
|
|
``**kwargs`` is now passed to the redis object.
|
||
|
|
|
||
|
|
:param host: address of the Redis server or an object which API is
|
||
|
|
compatible with the official Python Redis client (redis-py).
|
||
|
|
:param port: port number on which Redis server listens for connections.
|
||
|
|
:param password: password authentication for the Redis server.
|
||
|
|
:param db: db (zero-based numeric index) on Redis Server to connect.
|
||
|
|
:param default_timeout: the default timeout that is used if no timeout is
|
||
|
|
specified on :meth:`~BaseCache.set`. A timeout of
|
||
|
|
0 indicates that the cache never expires.
|
||
|
|
:param key_prefix: A prefix that should be added to all keys.
|
||
|
|
|
||
|
|
Any additional keyword arguments will be passed to ``redis.Redis``.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, host='localhost', port=6379, password=None,
|
||
|
|
db=0, default_timeout=300, key_prefix=None, **kwargs):
|
||
|
|
BaseCache.__init__(self, default_timeout)
|
||
|
|
if host is None:
|
||
|
|
raise ValueError('RedisCache host parameter may not be None')
|
||
|
|
if isinstance(host, string_types):
|
||
|
|
try:
|
||
|
|
import redis
|
||
|
|
except ImportError:
|
||
|
|
raise RuntimeError('no redis module found')
|
||
|
|
if kwargs.get('decode_responses', None):
|
||
|
|
raise ValueError('decode_responses is not supported by '
|
||
|
|
'RedisCache.')
|
||
|
|
self._client = redis.Redis(host=host, port=port, password=password,
|
||
|
|
db=db, **kwargs)
|
||
|
|
else:
|
||
|
|
self._client = host
|
||
|
|
self.key_prefix = key_prefix or ''
|
||
|
|
|
||
|
|
def _normalize_timeout(self, timeout):
|
||
|
|
timeout = BaseCache._normalize_timeout(self, timeout)
|
||
|
|
if timeout == 0:
|
||
|
|
timeout = -1
|
||
|
|
return timeout
|
||
|
|
|
||
|
|
def dump_object(self, value):
|
||
|
|
"""Dumps an object into a string for redis. By default it serializes
|
||
|
|
integers as regular string and pickle dumps everything else.
|
||
|
|
"""
|
||
|
|
t = type(value)
|
||
|
|
if t in integer_types:
|
||
|
|
return str(value).encode('ascii')
|
||
|
|
return b'!' + pickle.dumps(value)
|
||
|
|
|
||
|
|
def load_object(self, value):
|
||
|
|
"""The reversal of :meth:`dump_object`. This might be called with
|
||
|
|
None.
|
||
|
|
"""
|
||
|
|
if value is None:
|
||
|
|
return None
|
||
|
|
if value.startswith(b'!'):
|
||
|
|
try:
|
||
|
|
return pickle.loads(value[1:])
|
||
|
|
except pickle.PickleError:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
return int(value)
|
||
|
|
except ValueError:
|
||
|
|
# before 0.8 we did not have serialization. Still support that.
|
||
|
|
return value
|
||
|
|
|
||
|
|
def get(self, key):
|
||
|
|
return self.load_object(self._client.get(self.key_prefix + key))
|
||
|
|
|
||
|
|
def get_many(self, *keys):
|
||
|
|
if self.key_prefix:
|
||
|
|
keys = [self.key_prefix + key for key in keys]
|
||
|
|
return [self.load_object(x) for x in self._client.mget(keys)]
|
||
|
|
|
||
|
|
def set(self, key, value, timeout=None):
|
||
|
|
timeout = self._normalize_timeout(timeout)
|
||
|
|
dump = self.dump_object(value)
|
||
|
|
if timeout == -1:
|
||
|
|
result = self._client.set(name=self.key_prefix + key,
|
||
|
|
value=dump)
|
||
|
|
else:
|
||
|
|
result = self._client.setex(name=self.key_prefix + key,
|
||
|
|
value=dump, time=timeout)
|
||
|
|
return result
|
||
|
|
|
||
|
|
def add(self, key, value, timeout=None):
|
||
|
|
timeout = self._normalize_timeout(timeout)
|
||
|
|
dump = self.dump_object(value)
|
||
|
|
return (
|
||
|
|
self._client.setnx(name=self.key_prefix + key, value=dump) and
|
||
|
|
self._client.expire(name=self.key_prefix + key, time=timeout)
|
||
|
|
)
|
||
|
|
|
||
|
|
def set_many(self, mapping, timeout=None):
|
||
|
|
timeout = self._normalize_timeout(timeout)
|
||
|
|
# Use transaction=False to batch without calling redis MULTI
|
||
|
|
# which is not supported by twemproxy
|
||
|
|
pipe = self._client.pipeline(transaction=False)
|
||
|
|
|
||
|
|
for key, value in _items(mapping):
|
||
|
|
dump = self.dump_object(value)
|
||
|
|
if timeout == -1:
|
||
|
|
pipe.set(name=self.key_prefix + key, value=dump)
|
||
|
|
else:
|
||
|
|
pipe.setex(name=self.key_prefix + key, value=dump,
|
||
|
|
time=timeout)
|
||
|
|
return pipe.execute()
|
||
|
|
|
||
|
|
def delete(self, key):
|
||
|
|
return self._client.delete(self.key_prefix + key)
|
||
|
|
|
||
|
|
def delete_many(self, *keys):
|
||
|
|
if not keys:
|
||
|
|
return
|
||
|
|
if self.key_prefix:
|
||
|
|
keys = [self.key_prefix + key for key in keys]
|
||
|
|
return self._client.delete(*keys)
|
||
|
|
|
||
|
|
def has(self, key):
|
||
|
|
return self._client.exists(self.key_prefix + key)
|
||
|
|
|
||
|
|
def clear(self):
|
||
|
|
status = False
|
||
|
|
if self.key_prefix:
|
||
|
|
keys = self._client.keys(self.key_prefix + '*')
|
||
|
|
if keys:
|
||
|
|
status = self._client.delete(*keys)
|
||
|
|
else:
|
||
|
|
status = self._client.flushdb()
|
||
|
|
return status
|
||
|
|
|
||
|
|
def inc(self, key, delta=1):
|
||
|
|
return self._client.incr(name=self.key_prefix + key, amount=delta)
|
||
|
|
|
||
|
|
def dec(self, key, delta=1):
|
||
|
|
return self._client.decr(name=self.key_prefix + key, amount=delta)
|
||
|
|
|
||
|
|
|
||
|
|
class FileSystemCache(BaseCache):
|
||
|
|
|
||
|
|
"""A cache that stores the items on the file system. This cache depends
|
||
|
|
on being the only user of the `cache_dir`. Make absolutely sure that
|
||
|
|
nobody but this cache stores files there or otherwise the cache will
|
||
|
|
randomly delete files therein.
|
||
|
|
|
||
|
|
:param cache_dir: the directory where cache files are stored.
|
||
|
|
:param threshold: the maximum number of items the cache stores before
|
||
|
|
it starts deleting some. A threshold value of 0
|
||
|
|
indicates no threshold.
|
||
|
|
:param default_timeout: the default timeout that is used if no timeout is
|
||
|
|
specified on :meth:`~BaseCache.set`. A timeout of
|
||
|
|
0 indicates that the cache never expires.
|
||
|
|
:param mode: the file mode wanted for the cache files, default 0600
|
||
|
|
"""
|
||
|
|
|
||
|
|
#: used for temporary files by the FileSystemCache
|
||
|
|
_fs_transaction_suffix = '.__wz_cache'
|
||
|
|
#: keep amount of files in a cache element
|
||
|
|
_fs_count_file = '__wz_cache_count'
|
||
|
|
|
||
|
|
def __init__(self, cache_dir, threshold=500, default_timeout=300,
|
||
|
|
mode=0o600):
|
||
|
|
BaseCache.__init__(self, default_timeout)
|
||
|
|
self._path = cache_dir
|
||
|
|
self._threshold = threshold
|
||
|
|
self._mode = mode
|
||
|
|
|
||
|
|
try:
|
||
|
|
os.makedirs(self._path)
|
||
|
|
except OSError as ex:
|
||
|
|
if ex.errno != errno.EEXIST:
|
||
|
|
raise
|
||
|
|
|
||
|
|
self._update_count(value=len(self._list_dir()))
|
||
|
|
|
||
|
|
@property
|
||
|
|
def _file_count(self):
|
||
|
|
return self.get(self._fs_count_file) or 0
|
||
|
|
|
||
|
|
def _update_count(self, delta=None, value=None):
|
||
|
|
# If we have no threshold, don't count files
|
||
|
|
if self._threshold == 0:
|
||
|
|
return
|
||
|
|
|
||
|
|
if delta:
|
||
|
|
new_count = self._file_count + delta
|
||
|
|
else:
|
||
|
|
new_count = value or 0
|
||
|
|
self.set(self._fs_count_file, new_count, mgmt_element=True)
|
||
|
|
|
||
|
|
def _normalize_timeout(self, timeout):
|
||
|
|
timeout = BaseCache._normalize_timeout(self, timeout)
|
||
|
|
if timeout != 0:
|
||
|
|
timeout = time() + timeout
|
||
|
|
return int(timeout)
|
||
|
|
|
||
|
|
def _list_dir(self):
|
||
|
|
"""return a list of (fully qualified) cache filenames
|
||
|
|
"""
|
||
|
|
mgmt_files = [self._get_filename(name).split('/')[-1]
|
||
|
|
for name in (self._fs_count_file,)]
|
||
|
|
return [os.path.join(self._path, fn) for fn in os.listdir(self._path)
|
||
|
|
if not fn.endswith(self._fs_transaction_suffix)
|
||
|
|
and fn not in mgmt_files]
|
||
|
|
|
||
|
|
def _prune(self):
|
||
|
|
if self._threshold == 0 or not self._file_count > self._threshold:
|
||
|
|
return
|
||
|
|
|
||
|
|
entries = self._list_dir()
|
||
|
|
now = time()
|
||
|
|
for idx, fname in enumerate(entries):
|
||
|
|
try:
|
||
|
|
remove = False
|
||
|
|
with open(fname, 'rb') as f:
|
||
|
|
expires = pickle.load(f)
|
||
|
|
remove = (expires != 0 and expires <= now) or idx % 3 == 0
|
||
|
|
|
||
|
|
if remove:
|
||
|
|
os.remove(fname)
|
||
|
|
except (IOError, OSError):
|
||
|
|
pass
|
||
|
|
self._update_count(value=len(self._list_dir()))
|
||
|
|
|
||
|
|
def clear(self):
|
||
|
|
for fname in self._list_dir():
|
||
|
|
try:
|
||
|
|
os.remove(fname)
|
||
|
|
except (IOError, OSError):
|
||
|
|
self._update_count(value=len(self._list_dir()))
|
||
|
|
return False
|
||
|
|
self._update_count(value=0)
|
||
|
|
return True
|
||
|
|
|
||
|
|
def _get_filename(self, key):
|
||
|
|
if isinstance(key, text_type):
|
||
|
|
key = key.encode('utf-8') # XXX unicode review
|
||
|
|
hash = md5(key).hexdigest()
|
||
|
|
return os.path.join(self._path, hash)
|
||
|
|
|
||
|
|
def get(self, key):
|
||
|
|
filename = self._get_filename(key)
|
||
|
|
try:
|
||
|
|
with open(filename, 'rb') as f:
|
||
|
|
pickle_time = pickle.load(f)
|
||
|
|
if pickle_time == 0 or pickle_time >= time():
|
||
|
|
return pickle.load(f)
|
||
|
|
else:
|
||
|
|
os.remove(filename)
|
||
|
|
return None
|
||
|
|
except (IOError, OSError, pickle.PickleError):
|
||
|
|
return None
|
||
|
|
|
||
|
|
def add(self, key, value, timeout=None):
|
||
|
|
filename = self._get_filename(key)
|
||
|
|
if not os.path.exists(filename):
|
||
|
|
return self.set(key, value, timeout)
|
||
|
|
return False
|
||
|
|
|
||
|
|
def set(self, key, value, timeout=None, mgmt_element=False):
|
||
|
|
# Management elements have no timeout
|
||
|
|
if mgmt_element:
|
||
|
|
timeout = 0
|
||
|
|
|
||
|
|
# Don't prune on management element update, to avoid loop
|
||
|
|
else:
|
||
|
|
self._prune()
|
||
|
|
|
||
|
|
timeout = self._normalize_timeout(timeout)
|
||
|
|
filename = self._get_filename(key)
|
||
|
|
try:
|
||
|
|
fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix,
|
||
|
|
dir=self._path)
|
||
|
|
with os.fdopen(fd, 'wb') as f:
|
||
|
|
pickle.dump(timeout, f, 1)
|
||
|
|
pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)
|
||
|
|
rename(tmp, filename)
|
||
|
|
os.chmod(filename, self._mode)
|
||
|
|
except (IOError, OSError):
|
||
|
|
return False
|
||
|
|
else:
|
||
|
|
# Management elements should not count towards threshold
|
||
|
|
if not mgmt_element:
|
||
|
|
self._update_count(delta=1)
|
||
|
|
return True
|
||
|
|
|
||
|
|
def delete(self, key, mgmt_element=False):
|
||
|
|
try:
|
||
|
|
os.remove(self._get_filename(key))
|
||
|
|
except (IOError, OSError):
|
||
|
|
return False
|
||
|
|
else:
|
||
|
|
# Management elements should not count towards threshold
|
||
|
|
if not mgmt_element:
|
||
|
|
self._update_count(delta=-1)
|
||
|
|
return True
|
||
|
|
|
||
|
|
def has(self, key):
|
||
|
|
filename = self._get_filename(key)
|
||
|
|
try:
|
||
|
|
with open(filename, 'rb') as f:
|
||
|
|
pickle_time = pickle.load(f)
|
||
|
|
if pickle_time == 0 or pickle_time >= time():
|
||
|
|
return True
|
||
|
|
else:
|
||
|
|
os.remove(filename)
|
||
|
|
return False
|
||
|
|
except (IOError, OSError, pickle.PickleError):
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
class UWSGICache(BaseCache):
|
||
|
|
""" Implements the cache using uWSGI's caching framework.
|
||
|
|
|
||
|
|
.. note::
|
||
|
|
This class cannot be used when running under PyPy, because the uWSGI
|
||
|
|
API implementation for PyPy is lacking the needed functionality.
|
||
|
|
|
||
|
|
:param default_timeout: The default timeout in seconds.
|
||
|
|
:param cache: The name of the caching instance to connect to, for
|
||
|
|
example: mycache@localhost:3031, defaults to an empty string, which
|
||
|
|
means uWSGI will cache in the local instance. If the cache is in the
|
||
|
|
same instance as the werkzeug app, you only have to provide the name of
|
||
|
|
the cache.
|
||
|
|
"""
|
||
|
|
def __init__(self, default_timeout=300, cache=''):
|
||
|
|
BaseCache.__init__(self, default_timeout)
|
||
|
|
|
||
|
|
if platform.python_implementation() == 'PyPy':
|
||
|
|
raise RuntimeError("uWSGI caching does not work under PyPy, see "
|
||
|
|
"the docs for more details.")
|
||
|
|
|
||
|
|
try:
|
||
|
|
import uwsgi
|
||
|
|
self._uwsgi = uwsgi
|
||
|
|
except ImportError:
|
||
|
|
raise RuntimeError("uWSGI could not be imported, are you "
|
||
|
|
"running under uWSGI?")
|
||
|
|
|
||
|
|
self.cache = cache
|
||
|
|
|
||
|
|
def get(self, key):
|
||
|
|
rv = self._uwsgi.cache_get(key, self.cache)
|
||
|
|
if rv is None:
|
||
|
|
return
|
||
|
|
return pickle.loads(rv)
|
||
|
|
|
||
|
|
def delete(self, key):
|
||
|
|
return self._uwsgi.cache_del(key, self.cache)
|
||
|
|
|
||
|
|
def set(self, key, value, timeout=None):
|
||
|
|
return self._uwsgi.cache_update(key, pickle.dumps(value),
|
||
|
|
self._normalize_timeout(timeout),
|
||
|
|
self.cache)
|
||
|
|
|
||
|
|
def add(self, key, value, timeout=None):
|
||
|
|
return self._uwsgi.cache_set(key, pickle.dumps(value),
|
||
|
|
self._normalize_timeout(timeout),
|
||
|
|
self.cache)
|
||
|
|
|
||
|
|
def clear(self):
|
||
|
|
return self._uwsgi.cache_clear(self.cache)
|
||
|
|
|
||
|
|
def has(self, key):
|
||
|
|
return self._uwsgi.cache_exists(key, self.cache) is not None
|