[Zope3-checkins] CVS: Zope3/src/zope/app/cache - __init__.py:1.2 annotationcacheable.py:1.2 caching.py:1.2 configure.zcml:1.2 ram.py:1.2
Jim Fulton
jim@zope.com
Wed, 25 Dec 2002 09:13:45 -0500
Update of /cvs-repository/Zope3/src/zope/app/cache
In directory cvs.zope.org:/tmp/cvs-serv15352/src/zope/app/cache
Added Files:
__init__.py annotationcacheable.py caching.py configure.zcml
ram.py
Log Message:
Grand renaming:
- Renamed most files (especially python modules) to lower case.
- Moved views and interfaces into separate hierarchies within each
project, where each top-level directory under the zope package
is a separate project.
- Moved everything to src from lib/python.
lib/python will eventually go away. I need access to the cvs
repository to make this happen, however.
There are probably some bits that are broken. All tests pass
and zope runs, but I haven't tried everything. There are a number
of cleanups I'll work on tomorrow.
=== Zope3/src/zope/app/cache/__init__.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/cache/__init__.py Wed Dec 25 09:12:44 2002
@@ -0,0 +1,2 @@
+#
+# This file is necessary to make this directory a package.
=== Zope3/src/zope/app/cache/annotationcacheable.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/cache/annotationcacheable.py Wed Dec 25 09:12:44 2002
@@ -0,0 +1,44 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""An adapter of annotatable objects."""
+
+from zope.component import getAdapter, getService
+from zope.app.interfaces.annotation import IAnnotations
+from zope.app.interfaces.cache.cache import ICacheable
+
+annotation_key = 'zope.app.cache.CacheManager'
+
+class AnnotationCacheable:
+ """Stores cache information in object's annotations."""
+
+ __implements__ = ICacheable
+
+ def __init__(self, context):
+ self._context = context
+
+ def getCacheId(self):
+ annotations = getAdapter(self._context, IAnnotations)
+ return annotations.get(annotation_key, None)
+
+ def setCacheId(self, id):
+ # Remove object from old cache
+ old_cache_id = self.getCacheId()
+ if old_cache_id and old_cache_id != id:
+ service = getService(self._context, "Caching")
+ cache = service.getCache(old_cache_id)
+ cache.invalidate(self._context)
+ annotations = getAdapter(self._context, IAnnotations)
+ annotations[annotation_key] = id
+
+ cacheId = property(getCacheId, setCacheId, None, "Associated cache name")
=== Zope3/src/zope/app/cache/caching.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/cache/caching.py Wed Dec 25 09:12:44 2002
@@ -0,0 +1,37 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Helpers for caching."""
+
+from zope.component import getAdapter, getService
+from zope.component import ComponentLookupError
+from zope.app.interfaces.cache.cache import ICacheable
+from zope.app.interfaces.traversing.physicallylocatable import IPhysicallyLocatable
+
+
+def getCacheForObj(obj):
+ """Returns the cache associated with obj or None."""
+ adapter = getAdapter(obj, ICacheable)
+ cache_id = adapter.getCacheId()
+ if not cache_id:
+ return None
+ service = getService(obj, "Caching")
+ return service.getCache(cache_id)
+
+def getLocationForCache(obj):
+ """Returns the location to be used for caching the object or None."""
+ try:
+ locatable = getAdapter(obj, IPhysicallyLocatable)
+ return "/".join(locatable.getPhysicalPath())
+ except (ComponentLookupError, TypeError):
+ return None
=== Zope3/src/zope/app/cache/configure.zcml 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/cache/configure.zcml Wed Dec 25 09:12:44 2002
@@ -0,0 +1,23 @@
+<zopeConfigure
+ xmlns='http://namespaces.zope.org/zope'
+ xmlns:browser='http://namespaces.zope.org/browser'
+ xmlns:event='http://namespaces.zope.org/event'
+ package="zope.app.cache"
+>
+
+ <serviceType id="Caching" interface="zope.app.interfaces.cache.cache.ICachingService" />
+
+ <adapter factory="zope.app.cache.annotationcacheable.AnnotationCacheable"
+ provides="zope.app.interfaces.cache.cache.ICacheable"
+ for="zope.app.interfaces.annotation.IAnnotatable" />
+
+
+ <content class="zope.app.cache.ram.RAMCache">
+ <factory id="zope.app.caching.RAMCache"
+ permission="zope.Public" />
+ <require permission="zope.Public"
+ interface="zope.app.interfaces.cache.ram.IRAMCache" />
+ <implements interface="zope.app.interfaces.annotation.IAttributeAnnotatable" />
+ </content>
+
+</zopeConfigure>
=== Zope3/src/zope/app/cache/ram.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/cache/ram.py Wed Dec 25 09:12:44 2002
@@ -0,0 +1,365 @@
+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id$
+"""
+from time import time
+from thread import allocate_lock
+from pickle import dumps
+from persistence import Persistent
+from zope.app.interfaces.cache.ram import IRAMCache
+from zope.component import getAdapter
+from zope.component.exceptions import ComponentLookupError
+from zope.app.interfaces.traversing.physicallylocatable import IPhysicallyLocatable
+from zope.app.interfaces.event import IObjectModifiedEvent
+
+# A global caches dictionary shared between threads
+caches = {}
+
+# A writelock for caches dictionary
+writelock = allocate_lock()
+
+# A counter for cache ids and its lock
+cache_id_counter = 0
+cache_id_writelock = allocate_lock()
+
+class RAMCache(Persistent):
+ """RAM Cache
+
+ The design of this class is heavily based on RAMCacheManager in
+ Zope2.
+
+ The idea behind the RAMCache is that it should be shared between
+ threads, so that the same objects are not cached in each thread.
+ This is achieved by storing the cache data structure itself as a
+ module level variable (RAMCache.caches). This, of course,
+ requires locking on modifications of that data structure.
+
+ RAMCache is a persistent object. The actual data storage is a
+ volatile object, which can be acquired/created by calling
+ _getStorage(). Storage objects are shared between threads and
+ handle their blocking internally.
+ """
+
+ __implements__ = IRAMCache
+
+ def __init__(self):
+
+ # A timestamp and a counter are used here because using just a
+ # timestamp and an id (address) produced unit test failures on
+ # Windows (where ticks are 55ms long). If we want to use just
+ # the counter, we need to make it persistent, because the
+ # RAMCaches are persistent.
+
+ cache_id_writelock.acquire()
+ try:
+ global cache_id_counter
+ cache_id_counter +=1
+ self._cacheId = "%s_%f_%d" % (id(self), time(), cache_id_counter)
+ finally:
+ cache_id_writelock.release()
+
+ self.requestVars = ()
+ self.maxEntries = 1000
+ self.maxAge = 3600
+ self.cleanupInterval = 300
+
+ def getStatistics(self):
+ s = self._getStorage()
+ return s.getStatistics()
+
+ def update(self, maxEntries=None, maxAge=None, cleanupInterval=None):
+
+ if maxEntries is not None:
+ self.maxEntries = maxEntries
+
+ if maxAge is not None:
+ self.maxAge = maxAge
+
+ if cleanupInterval is not None:
+ self.cleanupInterval = cleanupInterval
+
+ self._getStorage().update(maxEntries, maxAge, cleanupInterval)
+
+
+ def invalidate(self, ob, key=None):
+ s = self._getStorage()
+ if key:
+ key = self._buildKey(key)
+ s.invalidate(ob, key)
+ else:
+ s.invalidate(ob)
+
+
+ def invalidateAll(self):
+ s = self._getStorage()
+ s.invalidateAll()
+
+
+ def query(self, ob, key=None, default=None):
+ s = self._getStorage()
+ key = self._buildKey(key)
+ try:
+ return s.getEntry(ob, key)
+ except KeyError:
+ return default
+
+ def set(self, data, ob, key=None):
+ s = self._getStorage()
+ key = self._buildKey(key)
+ s.setEntry(ob, key, data)
+
+ def _getStorage(self):
+ "Finds or creates a storage object."
+
+ global caches
+ global writelock
+ cacheId = self._cacheId
+ writelock.acquire()
+ try:
+ if not caches.has_key(cacheId):
+ caches[cacheId] = Storage(self.maxEntries, self.maxAge,
+ self.cleanupInterval)
+ self._v_storage = caches[cacheId]
+ finally:
+ writelock.release()
+ return self._v_storage
+
+ def _buildKey(kw):
+ "Build a tuple which can be used as an index for a cached value"
+
+ if kw:
+ items = kw.items()
+ items.sort()
+ return tuple(items)
+
+ return ()
+ _buildKey = staticmethod(_buildKey)
+
+ def notify(self, event):
+ """See ISubscriber
+
+ This method receives ObjectModified events and invalidates
+ cached entries for the objects that raise them.
+ """
+
+ if IObjectModifiedEvent.isImplementedBy(event):
+ self._getStorage().invalidate(event.location)
+
+
+class Storage:
+ """Storage.
+
+ Storage keeps the count and does the aging and cleanup of cached
+ entries.
+
+ This object is shared between threads. It corresponds to a single
+ persistent RAMCache object. Storage does the locking necessary
+ for thread safety.
+
+ """
+
+ def __init__(self, maxEntries=1000, maxAge=3600, cleanupInterval=300):
+ self._data = {}
+ self._misses = {}
+ self._invalidate_queue = []
+ self.maxEntries = maxEntries
+ self.maxAge = maxAge
+ self.cleanupInterval = cleanupInterval
+ self.writelock = allocate_lock()
+ self.lastCleanup = time()
+
+ def update(self, maxEntries=None, maxAge=None, cleanupInterval=None):
+ """Set the configuration options.
+
+ None values are ignored.
+ """
+ if maxEntries is not None:
+ self.maxEntries = maxEntries
+
+ if maxAge is not None:
+ self.maxAge = maxAge
+
+ if cleanupInterval is not None:
+ self.cleanupInterval = cleanupInterval
+
+ def getEntry(self, ob, key):
+ try:
+ data = self._data[ob][key]
+ except KeyError:
+ if ob not in self._misses:
+ self._misses[ob] = 0
+ self._misses[ob] += 1
+ raise
+ else:
+ data[2] += 1 # increment access count
+ return data[0]
+
+
+ def setEntry(self, ob, key, value):
+ """Stores a value for the object. Creates the necessary
+ dictionaries."""
+
+ if self.lastCleanup <= time() - self.cleanupInterval:
+ self.cleanup()
+
+ self.writelock.acquire()
+ try:
+ if ob not in self._data:
+ self._data[ob] = {}
+
+ timestamp = time()
+ # [data, ctime, access count]
+ self._data[ob][key] = [value, timestamp, 0]
+ finally:
+ self.writelock.release()
+ self._invalidate_queued()
+
+ def _do_invalidate(self, ob, key=None):
+ """This does the actual invalidation, but does not handle the locking.
+
+ This method is supposed to be called from invalidate()
+ """
+ try:
+ if key is None:
+ del self._data[ob]
+ self._misses[ob] = 0
+ else:
+ del self._data[ob][key]
+ if len(self._data[ob]) < 1:
+ del self._data[ob]
+ except KeyError:
+ pass
+
+ def _invalidate_queued(self):
+ """This method should be called after each writelock release."""
+
+ while len(self._invalidate_queue):
+ obj, key = self._invalidate_queue.pop()
+ self.invalidate(obj, key)
+
+ def invalidate(self, ob, key=None):
+ """Drop the cached values.
+
+ Drop all the values for an object if no key is provided or
+ just one entry if the key is provided.
+
+ """
+ if self.writelock.acquire(0):
+ try:
+ self._do_invalidate(ob, key)
+ finally:
+ self.writelock.release()
+ # self._invalidate_queued() not called to avoid a recursion
+ else:
+ self._invalidate_queue.append((ob,key))
+
+
+ def invalidateAll(self):
+ """Drop all the cached values.
+ """
+ self.writelock.acquire()
+ try:
+ self._data = {}
+ self._misses = {}
+ self._invalidate_queue = []
+ finally:
+ self.writelock.release()
+
+
+ def removeStaleEntries(self):
+ "Remove the entries older than maxAge"
+
+ if self.maxAge > 0:
+ punchline = time() - self.maxAge
+ self.writelock.acquire()
+ try:
+ for object, dict in self._data.items():
+ for key, val in self._data[object].items():
+ if self._data[object][key][1] < punchline:
+ del self._data[object][key]
+ if len(self._data[object]) < 1:
+ del self._data[object]
+ finally:
+ self.writelock.release()
+ self._invalidate_queued()
+
+ def cleanup(self):
+ "Cleanup the data"
+ self.removeStaleEntries()
+ self.removeLeastAccessed()
+
+ def removeLeastAccessed(self):
+ ""
+
+ self.writelock.acquire()
+ try:
+ keys = []
+ for ob in self._data:
+ for key in self._data[ob]:
+ keys.append((ob, key))
+
+ if len(keys) > self.maxEntries:
+ def cmpByCount(x,y):
+ ob1, key1 = x
+ ob2, key2 = y
+ return cmp(self._data[ob1][key1],
+ self._data[ob2][key2])
+ keys.sort(cmpByCount)
+
+ ob, key = keys[self.maxEntries]
+ maxDropCount = self._data[ob][key][2]
+
+ keys.reverse()
+
+ for ob, key in keys:
+ if self._data[ob][key][2] <= maxDropCount:
+ del self._data[ob][key]
+ if len(self._data[ob]) < 1:
+ del self._data[ob]
+
+ self._clearAccessCounters()
+ finally:
+ self.writelock.release()
+ self._invalidate_queued()
+
+ def _clearAccessCounters(self):
+ for ob in self._data:
+ for key in self._data[ob]:
+ self._data[ob][key][2] = 0
+ for k in self._misses:
+ self._misses[k] = 0
+
+ def getKeys(self, object):
+ return self._data[object].keys()
+
+ def getStatistics(self):
+ "Basically see IRAMCache"
+ objects = self._data.keys()
+ objects.sort()
+ result = []
+
+ for ob in objects:
+ size = len(dumps(self._data[ob]))
+ hits = 0
+ for entry in self._data[ob].values():
+ hits += entry[2]
+ result.append({'path': ob,
+ 'hits': hits,
+ 'misses': self._misses[ob],
+ 'size': size,
+ 'entries': len(self._data[ob])})
+ return tuple(result)
+
+__doc__ = RAMCache.__doc__ + __doc__