[Zodb-checkins] SVN: ZODB/branches/jim-gc/src/ Trying to deal with https://bugs.launchpad.net/zodb/+bug/641481
Jim Fulton
jim at zope.com
Thu Sep 23 18:10:30 EDT 2010
Log message for revision 116780:
Trying to deal with https://bugs.launchpad.net/zodb/+bug/641481
I'm fairly sure that the problem has to do with the fact that ZODB'
object caches implementation has its own weakref implementation.
The approach in this branch is to basically stop using GC for
persistent objects that are in object caches. This is a band aid. :(
Once consequence is that there will significant leakage if caches
aren't explicitly cleared when they're no-longer used. In normal
application code, this shouldn't be a problem, since caches typically
are used for the life of a program. It could be a problem for test
code that open databases and don't close them.
I'm still chasing memory leaks. It's possible that these are all due
to tests not closing databases (or clearing caches if they are managed
directly.)
Changed:
U ZODB/branches/jim-gc/src/ZODB/Connection.py
U ZODB/branches/jim-gc/src/ZODB/DB.py
U ZODB/branches/jim-gc/src/persistent/cPickleCache.c
U ZODB/branches/jim-gc/src/persistent/tests/test_overriding_attrs.py
U ZODB/branches/jim-gc/src/persistent/tests/utils.py
-=-
Modified: ZODB/branches/jim-gc/src/ZODB/Connection.py
===================================================================
--- ZODB/branches/jim-gc/src/ZODB/Connection.py 2010-09-23 22:09:36 UTC (rev 116779)
+++ ZODB/branches/jim-gc/src/ZODB/Connection.py 2010-09-23 22:10:29 UTC (rev 116780)
@@ -1050,6 +1050,12 @@
if connection is not self:
connection.open(transaction_manager, False)
+ def _close_permanently(self):
+ if self._mvcc_storage:
+ self._storage.release()
+ self._storage = self._normal_storage = None
+ self._cache.clear()
+
def _resetCache(self):
"""Creates a new cache, discarding the old one.
@@ -1064,11 +1070,6 @@
if getattr(self, '_reader', None) is not None:
self._reader._cache = cache
- def _releaseStorage(self):
- """Tell the storage to release resources it's using"""
- if self._mvcc_storage:
- self._storage.release()
-
##########################################################################
# Python protocol
Modified: ZODB/branches/jim-gc/src/ZODB/DB.py
===================================================================
--- ZODB/branches/jim-gc/src/ZODB/DB.py 2010-09-23 22:09:36 UTC (rev 116779)
+++ ZODB/branches/jim-gc/src/ZODB/DB.py 2010-09-23 22:10:29 UTC (rev 116780)
@@ -195,8 +195,7 @@
# strong reference to `c` now, breaking the cycle would not
# reclaim `c` now, and `c` would be left in a user-visible
# crazy state.
- c._resetCache()
- c._releaseStorage()
+ c._close_permanently()
def reduce_size(self):
self._reduce_size()
@@ -641,7 +640,7 @@
def _(c):
c.transaction_manager.abort()
c.afterCompletion = c.newTransaction = c.close = noop
- c._storage = c._normal_storage = None
+ c._close_permanently()
self.storage.close()
del self.storage
Modified: ZODB/branches/jim-gc/src/persistent/cPickleCache.c
===================================================================
--- ZODB/branches/jim-gc/src/persistent/cPickleCache.c 2010-09-23 22:09:36 UTC (rev 116779)
+++ ZODB/branches/jim-gc/src/persistent/cPickleCache.c 2010-09-23 22:10:29 UTC (rev 116780)
@@ -782,6 +782,7 @@
if (PyDict_SetItem(self->data, key, v) < 0)
return NULL;
/* the dict should have a borrowed reference */
+ PyObject_GC_UnTrack(v);
Py_DECREF(v);
Py_INCREF(self);
@@ -796,6 +797,28 @@
Py_RETURN_NONE;
}
+static int cc_del_item(ccobject *self, PyObject *key);
+
+static PyObject*
+cc_clear_method(ccobject *self)
+{
+ /* Cowardly mostly use other methods to clear the cache. */
+ while (1)
+ {
+ Py_ssize_t pos = 0;
+ PyObject *k, *v;
+ if (! PyDict_Next(self->data, &pos, &k, &v))
+ break;
+ Py_INCREF(k);
+ pos = cc_del_item(self, k);
+ Py_DECREF(k);
+ if (pos < 0)
+ return NULL;
+ }
+
+ Py_RETURN_NONE;
+}
+
static struct PyMethodDef cc_methods[] = {
{"items", (PyCFunction)cc_items, METH_NOARGS,
"Return list of oid, object pairs for all items in cache."},
@@ -830,6 +853,8 @@
"(if this is known to the cache)."},
{"new_ghost", (PyCFunction)cc_new_ghost, METH_VARARGS,
"new_ghost() -- Initialize a ghost and add it to the cache."},
+ {"clear", (PyCFunction)cc_clear_method, METH_NOARGS,
+ "Remove all items from the cache."},
{NULL, NULL} /* sentinel */
};
@@ -915,6 +940,7 @@
if (o->cache)
{
Py_INCREF(o); /* account for uncounted reference */
+ PyObject_GC_Track(o);
if (PyDict_DelItem(self->data, o->oid) < 0)
return -1;
}
@@ -928,16 +954,21 @@
}
Py_XDECREF(self->jar);
+ self->jar = NULL;
while (PyDict_Next(self->data, &pos, &k, &v))
{
- Py_INCREF(v);
+ if (! PyType_Check(v))
+ {
+ Py_INCREF(v);
+ PyObject_GC_Track(v);
+ }
if (PyDict_SetItem(self->data, k, Py_None) < 0)
return -1;
}
Py_XDECREF(self->data);
self->data = NULL;
- self->jar = NULL;
+
return 0;
}
@@ -1123,6 +1154,7 @@
if (PyDict_SetItem(self->data, key, v) < 0)
return -1;
/* the dict should have a borrowed reference */
+ PyObject_GC_UnTrack(v);
Py_DECREF(v);
p = (cPersistentObject *)v;
@@ -1180,6 +1212,7 @@
Py_DECREF((PyObject *)p->cache);
p->cache = NULL;
+ PyObject_GC_Track(p);
}
if (PyDict_DelItem(self->data, key) < 0)
Modified: ZODB/branches/jim-gc/src/persistent/tests/test_overriding_attrs.py
===================================================================
--- ZODB/branches/jim-gc/src/persistent/tests/test_overriding_attrs.py 2010-09-23 22:09:36 UTC (rev 116779)
+++ ZODB/branches/jim-gc/src/persistent/tests/test_overriding_attrs.py 2010-09-23 22:10:29 UTC (rev 116780)
@@ -68,6 +68,8 @@
And we see that the object was activated before calling the
__getattr__ method.
+
+ >>> jar.close() # cleanup
"""
# Don't pretend we have any special attributes.
if name.startswith("__") and name.endswrith("__"):
@@ -145,6 +147,8 @@
0
See the very important note in the comment below!
+
+ >>> jar.close() # cleanup
"""
#################################################################
@@ -253,6 +257,8 @@
0
>>> o.tmp_foo
3
+
+ >>> jar.close() # cleanup
"""
#################################################################
@@ -354,6 +360,9 @@
...
AttributeError: tmp_z
+
+ >>> jar.close() # cleanup
+
"""
#################################################################
Modified: ZODB/branches/jim-gc/src/persistent/tests/utils.py
===================================================================
--- ZODB/branches/jim-gc/src/persistent/tests/utils.py 2010-09-23 22:09:36 UTC (rev 116779)
+++ ZODB/branches/jim-gc/src/persistent/tests/utils.py 2010-09-23 22:10:29 UTC (rev 116780)
@@ -16,7 +16,7 @@
self.cache[obj._p_oid] = obj
def close(self):
- pass
+ self.cache.clear()
# the following methods must be implemented to be a jar
@@ -54,7 +54,7 @@
self.remembered = obj.__getstate__()
def close(self):
- pass
+ self.cache.clear()
def fake_commit(self):
self.remembered = self.obj.__getstate__()
More information about the Zodb-checkins
mailing list