[Zope3-checkins] CVS: ZODB4/src/zodb/storage/tests - base.py:1.9 basic.py:1.8 mt.py:1.6 packable.py:1.5 readonly.py:1.6 recovery.py:1.5 revision.py:1.6 synchronization.py:1.5 test_autopack.py:1.10 test_file.py:1.7 undo.py:1.7 undoversion.py:1.4 version.py:1.4

Barry Warsaw barry@wooz.org
Thu, 13 Mar 2003 16:33:01 -0500


Update of /cvs-repository/ZODB4/src/zodb/storage/tests
In directory cvs.zope.org:/tmp/cvs-serv27419/src/zodb/storage/tests

Modified Files:
	base.py basic.py mt.py packable.py readonly.py recovery.py 
	revision.py synchronization.py test_autopack.py test_file.py 
	undo.py undoversion.py version.py 
Log Message:
> I believe we're ready to merge back to the head.

merging the opaque-pickles-branch back into the head



=== ZODB4/src/zodb/storage/tests/base.py 1.8 => 1.9 ===
--- ZODB4/src/zodb/storage/tests/base.py:1.8	Tue Feb 25 17:57:52 2003
+++ ZODB4/src/zodb/storage/tests/base.py	Thu Mar 13 16:32:29 2003
@@ -23,14 +23,14 @@
 import os
 import sys
 import errno
-import types
 import shutil
 import unittest
 
+from persistence import Persistent
 from transaction import get_transaction
 
 from zodb.db import DB
-from zodb.serialize import ConnectionObjectReader, ObjectWriter
+from zodb.serialize import ConnectionObjectReader, ObjectWriter, findrefs
 from zodb.ztransaction import Transaction
 from zodb.storage.tests.minpo import MinPO
 from zodb.storage.base import ZERO, BerkeleyConfig
@@ -62,13 +62,13 @@
     """
     d = {}
     for arg in args:
-        if isinstance(arg, types.StringType):
+        if isinstance(arg, str):
             d[oid] = arg
         elif arg is None:
             pass
         else:
             for oid, serial in arg:
-                if not isinstance(serial, types.StringType):
+                if not isinstance(serial, str):
                     raise serial # error from ZEO server
                 d[oid] = serial
     return d
@@ -94,6 +94,20 @@
             if err[0] != errno.ENOENT:
                 raise
 
+class C(Persistent):
+    pass
+
+
+class Jar:
+    def __init__(self, storage):
+        self._storage = storage
+        
+    def newObjectId(self):
+        return self._storage.newObjectId()
+
+    def register(self, obj):
+        obj._p_oid = self.newObjectId()
+
 
 class StorageTestBase(unittest.TestCase):
 
@@ -114,7 +128,7 @@
         self._close()
 
     def _dostore(self, oid=None, revid=None, data=None, version=None,
-                 already_pickled=0, user=None, description=None):
+                 already_pickled=False, user=None, description=None):
         """Do a complete storage transaction.  The defaults are:
 
          - oid=None, ask the storage for a new oid
@@ -128,12 +142,17 @@
             oid = self._storage.newObjectId()
         if revid is None:
             revid = ZERO
+        refs = []
         if data is None:
             data = MinPO(7)
-        if type(data) == types.IntType:
+        if isinstance(data, int):
             data = MinPO(data)
         if not already_pickled:
-            data = zodb_pickle(data)
+            data, refs = zodb_pickle(data)
+        elif isinstance(data, tuple):
+            data, refs = data
+        else:
+            refs = findrefs(data)
         if version is None:
             version = ''
         # Begin the transaction
@@ -145,7 +164,7 @@
         try:
             self._storage.tpcBegin(t)
             # Store an object
-            r1 = self._storage.store(oid, revid, data, version, t)
+            r1 = self._storage.store(oid, revid, data, refs, version, t)
             # Finish the transaction
             r2 = self._storage.tpcVote(t)
             revid = handle_serials(oid, r1, r2)
@@ -157,7 +176,7 @@
 
     def _dostoreNP(self, oid=None, revid=None, data=None, version=None,
                    user=None, description=None):
-        return self._dostore(oid, revid, data, version, already_pickled=1,
+        return self._dostore(oid, revid, data, version, already_pickled=True,
                              user=user, description=description)
     # The following methods depend on optional storage features.
 
@@ -193,6 +212,45 @@
         self._storage.tpcFinish(t)
         return oids
 
+    # some helper functions for setting up a valid root so that
+    # the storage can be packed
+
+    def _initroot(self):
+        self._jar = jar = Jar(self._storage)
+        self._reader = ConnectionObjectReader(jar, {})
+        try:
+            self._storage.load(ZERO, '')
+        except KeyError:
+            root = self._newobj()
+            t = Transaction()
+            t.note("initial database creation")
+            self._storage.tpcBegin(t)
+            data, refs = zodb_pickle(root)
+            self._storage.store(ZERO, None, data, refs, '', t)
+            self._storage.tpcVote(t)
+            self._storage.tpcFinish(t)
+
+    def _newobj(self):
+        obj = C()
+        obj._p_jar = self._jar
+        return obj
+
+    def _linked_newobj(self):
+        # Create a new object and make sure the root has a reference to it.
+        # Returns the object, the root, and the revid of the root.
+        data, revid0 = self._storage.load(ZERO, '')
+        root = self._reader.getObject(data)
+        obj = self._newobj()
+        # Link the root object to the persistent object, in order to keep the
+        # persistent object alive.  XXX Order here is important: an attribute
+        # on the root object must be set first, so that it gets oid 0, /then/
+        # the attribute on the obj can be set.
+        root.obj = obj
+        obj.value = 0
+        root.value = 0
+        root._p_jar = self._jar
+        revid0 = self._dostore(ZERO, revid=revid0, data=root)
+        return obj, root, revid0
 
 
 class BerkeleyTestBase(StorageTestBase):


=== ZODB4/src/zodb/storage/tests/basic.py 1.7 => 1.8 ===
--- ZODB4/src/zodb/storage/tests/basic.py:1.7	Tue Feb 11 10:59:28 2003
+++ ZODB4/src/zodb/storage/tests/basic.py	Thu Mar 13 16:32:29 2003
@@ -35,7 +35,7 @@
 
     def checkIStorage(self):
         self.assert_(IStorage.isImplementedBy(self._storage))
-    
+
     def checkDatabaseVersion(self):
         version = "abcd"
         self._storage.setVersion(version)
@@ -59,7 +59,7 @@
         # different Transaction object than the one we've begun on.
         self._storage.tpcBegin(t)
         self.assertRaises(StorageTransactionError, self._storage.store,
-                          0, 0, 0, 0, Transaction())
+                          0, 0, 0, 0, 0, Transaction())
 
         try:
             self._storage.abortVersion('dummy', Transaction())
@@ -76,7 +76,7 @@
             assert 0, "Should have failed, invalid transaction."
 
         self.assertRaises(StorageTransactionError, self._storage.store,
-                          0, 1, 2, 3, Transaction())
+                          0, 1, 2, 3, 4, Transaction())
         self._storage.tpcAbort(t)
 
     def checkSerialIsNoneForInitialRevision(self):
@@ -86,8 +86,8 @@
         self._storage.tpcBegin(txn)
         # Use None for serial.  Don't use _dostore() here because that coerces
         # serial=None to serial=ZERO.
-        r1 = self._storage.store(oid, None, zodb_pickle(MinPO(11)),
-                                       '', txn)
+        data, refs = zodb_pickle(MinPO(11))
+        r1 = self._storage.store(oid, None, data, refs, '', txn)
         r2 = self._storage.tpcVote(txn)
         self._storage.tpcFinish(txn)
         newrevid = handle_serials(oid, r1, r2)
@@ -133,7 +133,8 @@
         oid = self._storage.newObjectId()
         t = Transaction()
         self._storage.tpcBegin(t)
-        self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
+        data, refs = zodb_pickle(MinPO(5))
+        self._storage.store(oid, ZERO, data, refs, '', t)
         # Now abort this transaction
         self._storage.tpcAbort(t)
         # Now start all over again
@@ -146,7 +147,8 @@
         oid = self._storage.newObjectId()
         t = Transaction()
         self._storage.tpcBegin(t)
-        self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
+        data, refs = zodb_pickle(MinPO(5))
+        self._storage.store(oid, ZERO, data, refs, '', t)
         # Now abort this transaction
         self._storage.tpcVote(t)
         self._storage.tpcAbort(t)
@@ -191,7 +193,7 @@
         tid = chr(42) * 8
         self._storage.tpcBegin(t, tid)
         oid = self._storage.newObjectId()
-        data = zodb_pickle(MinPO(8))
-        self._storage.store(oid, None, data, '', t)
+        data, refs = zodb_pickle(MinPO(8))
+        self._storage.store(oid, None, data, refs, '', t)
         self._storage.tpcVote(t)
         self._storage.tpcFinish(t)


=== ZODB4/src/zodb/storage/tests/mt.py 1.5 => 1.6 ===
--- ZODB4/src/zodb/storage/tests/mt.py:1.5	Thu Mar 13 15:48:50 2003
+++ ZODB4/src/zodb/storage/tests/mt.py	Thu Mar 13 16:32:29 2003
@@ -143,7 +143,7 @@
         return oid
 
     def dostore(self, i):
-        data = zodb_pickle(MinPO((self.getName(), i)))
+        data, refs = zodb_pickle(MinPO((self.getName(), i)))
         t = Transaction()
         oid = self.oid()
         self.pause()
@@ -152,7 +152,7 @@
         self.pause()
 
         # Always create a new object, signified by None for revid
-        r1 = self.storage.store(oid, None, data, '', t)
+        r1 = self.storage.store(oid, None, data, refs, '', t)
         self.pause()
 
         r2 = self.storage.tpcVote(t)


=== ZODB4/src/zodb/storage/tests/packable.py 1.4 => 1.5 ===
--- ZODB4/src/zodb/storage/tests/packable.py:1.4	Wed Feb  5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/packable.py	Thu Mar 13 16:32:29 2003
@@ -14,127 +14,20 @@
 """Run some tests relevant for storages that support pack()."""
 
 import time
-from cStringIO import StringIO
 
-try:
-    import cPickle
-    pickle = cPickle
-    #import cPickle as pickle
-except ImportError:
-    import pickle
-
-from persistence import Persistent
 from transaction import get_transaction
 
 from zodb.db import DB
-from zodb.storage.base import ZERO
+from zodb.interfaces import ZERO
+from zodb.serialize import getDBRoot, ConnectionObjectReader
+from zodb.ztransaction import Transaction
 from zodb.storage.tests.minpo import MinPO
+from zodb.storage.tests.base import zodb_unpickle
+from zodb.storage.fsdump import dump
 
 
 
-# This class is for the root object.  It must not contain a getoid() method
-# (really, attribute).  The persistent pickling machinery -- in the dumps()
-# function below -- will pickle Root objects as normal, but any attributes
-# which reference persistent Object instances will get pickled as persistent
-# ids, not as the object's state.  This makes the referencesf stuff work,
-# because it pickle sniffs for persistent ids (so we have to get those
-# persistent ids into the root object's pickle).
-class Root:
-    pass
-
-
-# This is the persistent Object class.  Because it has a getoid() method, the
-# persistent pickling machinery -- in the dumps() function below -- will
-# pickle the oid string instead of the object's actual state.  Yee haw, this
-# stuff is deep. ;)
-class Object:
-    def __init__(self, oid):
-        self._oid = oid
-
-    def getoid(self):
-        return self._oid
-
-class C(Persistent):
-    pass
-
-
-# Here's where all the magic occurs.  Sadly, the pickle module is a bit
-# underdocumented, but here's what happens: by setting the persistent_id
-# attribute to getpersid() on the pickler, that function gets called for every
-# object being pickled.  By returning None when the object has no getoid
-# attribute, it signals pickle to serialize the object as normal.  That's how
-# the Root instance gets pickled correctly.  But, if the object has a getoid
-# attribute, then by returning that method's value, we tell pickle to
-# serialize the persistent id of the object instead of the object's state.
-# That sets the pickle up for proper sniffing by the referencesf machinery.
-# Fun, huh?
-def dumps(obj):
-    def getpersid(obj):
-        if hasattr(obj, 'getoid'):
-            return obj.getoid()
-        return None
-    s = StringIO()
-    p = pickle.Pickler(s)
-    p.persistent_id = getpersid
-    p.dump(obj)
-    return s.getvalue()
-
-
-
-class PackableStorageBase:
-    # We keep a cache of object ids to instances so that the unpickler can
-    # easily return any persistent object.
-    _cache = {}
-
-    def _newobj(self):
-        # This is a convenience method to create a new persistent Object
-        # instance.  It asks the storage for a new object id, creates the
-        # instance with the given oid, populates the cache and returns the
-        # object.
-        oid = self._storage.newObjectId()
-        obj = Object(oid)
-        self._cache[obj.getoid()] = obj
-        return obj
-
-    def _makeloader(self):
-        # This is the other side of the persistent pickling magic.  We need a
-        # custom unpickler to mirror our custom pickler above.  By setting the
-        # persistent_load function of the unpickler to self._cache.get(),
-        # whenever a persistent id is unpickled, it will actually return the
-        # Object instance out of the cache.  As far as returning a function
-        # with an argument bound to an instance attribute method, we do it
-        # this way because it makes the code in the tests more succinct.
-        #
-        # BUT!  Be careful in your use of loads() vs. pickle.loads().  loads()
-        # should only be used on the Root object's pickle since it's the only
-        # special one.  All the Object instances should use pickle.loads().
-        def loads(str, persfunc=self._cache.get):
-            fp = StringIO(str)
-            u = pickle.Unpickler(fp)
-            u.persistent_load = persfunc
-            return u.load()
-        return loads
-
-
-
-class PackableStorage(PackableStorageBase):
-    def _initroot(self):
-        try:
-            self._storage.load(ZERO, '')
-        except KeyError:
-            from persistence.dict import PersistentDict
-            from zodb.ztransaction import Transaction
-            file = StringIO()
-            p = cPickle.Pickler(file, 1)
-            p.dump((PersistentDict, None))
-            p.dump(PersistentDict().__getstate__())
-            t = Transaction()
-            t.note("initial database creation")
-            self._storage.tpcBegin(t)
-            self._storage.store(ZERO, None, file.getvalue(), '', t)
-            self._storage.tpcVote(t)
-            self._storage.tpcFinish(t)
-
+class PackableStorage:
     def checkPackEmptyStorage(self):
         self._storage.pack(time.time())
 
@@ -152,30 +45,29 @@
         raises = self.assertRaises
         # Create a `persistent' object
         obj = self._newobj()
-        oid = obj.getoid()
         obj.value = 1
+        oid = obj._p_oid
         # Commit three different revisions
-        revid1 = self._dostoreNP(oid, data=pickle.dumps(obj))
+        revid1 = self._dostore(oid, data=obj)
         obj.value = 2
-        revid2 = self._dostoreNP(oid, revid=revid1, data=pickle.dumps(obj))
+        revid2 = self._dostore(oid, revid=revid1, data=obj)
         obj.value = 3
-        revid3 = self._dostoreNP(oid, revid=revid2, data=pickle.dumps(obj))
+        revid3 = self._dostore(oid, revid=revid2, data=obj)
         # Now make sure all three revisions can be extracted
         data = self._storage.loadSerial(oid, revid1)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 1)
         data = self._storage.loadSerial(oid, revid2)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 2)
         data = self._storage.loadSerial(oid, revid3)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 3)
         # Now pack all transactions; need to sleep a second to make
         # sure that the pack time is greater than the last commit time.
-        time.sleep(1)
+        now = int(time.time())
+        while now >= int(time.time()):
+            time.sleep(0.1)
         self._storage.pack(time.time())
         # All revisions of the object should be gone, since there is no
         # reference from the root object to this object.
@@ -186,147 +78,132 @@
     def checkPackJustOldRevisions(self):
         eq = self.assertEqual
         raises = self.assertRaises
-        loads = self._makeloader()
-        # Create a root object.  This can't be an instance of Object,
-        # otherwise the pickling machinery will serialize it as a persistent
-        # id and not as an object that contains references (persistent ids) to
-        # other objects.
-        root = Root()
-        # Create a persistent object, with some initial state
-        obj = self._newobj()
-        oid = obj.getoid()
-        # Link the root object to the persistent object, in order to keep the
-        # persistent object alive.  Store the root object.
-        root.obj = obj
-        root.value = 0
-        revid0 = self._dostoreNP(ZERO, data=dumps(root))
+        # Create a root object
+        self._initroot()
+        obj, root, revid0 = self._linked_newobj()
         # Make sure the root can be retrieved
         data, revid = self._storage.load(ZERO, '')
         eq(revid, revid0)
-        eq(loads(data).value, 0)
+        eq(zodb_unpickle(data).value, 0)
         # Commit three different revisions of the other object
         obj.value = 1
-        revid1 = self._dostoreNP(oid, data=pickle.dumps(obj))
+        oid = obj._p_oid
+        revid1 = self._dostore(oid, data=obj)
         obj.value = 2
-        revid2 = self._dostoreNP(oid, revid=revid1, data=pickle.dumps(obj))
+        revid2 = self._dostore(oid, revid=revid1, data=obj)
         obj.value = 3
-        revid3 = self._dostoreNP(oid, revid=revid2, data=pickle.dumps(obj))
+        revid3 = self._dostore(oid, revid=revid2, data=obj)
         # Now make sure all three revisions can be extracted
         data = self._storage.loadSerial(oid, revid1)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 1)
         data = self._storage.loadSerial(oid, revid2)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 2)
         data = self._storage.loadSerial(oid, revid3)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 3)
         # Now pack just revisions 1 and 2.  The object's current revision
         # should stay alive because it's pointed to by the root.
-        time.sleep(1)
+        now = int(time.time())
+        while now >= int(time.time()):
+            time.sleep(0.1)
         self._storage.pack(time.time())
-        # Make sure the revisions are gone, but that object zero and revision
-        # 3 are still there and correct
+        # Make sure the revisions are gone, but that the root object and
+        # revision 3 are still there and correct
         data, revid = self._storage.load(ZERO, '')
         eq(revid, revid0)
-        eq(loads(data).value, 0)
+        eq(zodb_unpickle(data).value, 0)
         raises(KeyError, self._storage.loadSerial, oid, revid1)
         raises(KeyError, self._storage.loadSerial, oid, revid2)
         data = self._storage.loadSerial(oid, revid3)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 3)
         data, revid = self._storage.load(oid, '')
         eq(revid, revid3)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 3)
 
     def checkPackOnlyOneObject(self):
         eq = self.assertEqual
         raises = self.assertRaises
-        loads = self._makeloader()
-        # Create a root object.  This can't be an instance of Object,
-        # otherwise the pickling machinery will serialize it as a persistent
-        # id and not as an object that contains references (persistent ids) to
-        # other objects.
-        root = Root()
+        # Create a root object.
+        self._initroot()
+        data, revid0 = self._storage.load(ZERO, '')
+        root = self._reader.getObject(data)
+        root.value = -1
+        root._p_jar = self._jar
         # Create a persistent object, with some initial state
         obj1 = self._newobj()
-        oid1 = obj1.getoid()
+        obj1.value = -1
+        oid1 = obj1._p_oid
         # Create another persistent object, with some initial state.  Make
         # sure its oid is greater than the first object's oid.
         obj2 = self._newobj()
-        oid2 = obj2.getoid()
+        obj2.value = -1
+        oid2 = obj2._p_oid
         self.failUnless(oid2 > oid1)
         # Link the root object to the persistent objects, in order to keep
         # them alive.  Store the root object.
         root.obj1 = obj1
         root.obj2 = obj2
         root.value = 0
-        revid0 = self._dostoreNP(ZERO, data=dumps(root))
+        revid0 = self._dostore(ZERO, data=root, revid=revid0)
         # Make sure the root can be retrieved
         data, revid = self._storage.load(ZERO, '')
         eq(revid, revid0)
-        eq(loads(data).value, 0)
+        eq(zodb_unpickle(data).value, 0)
         # Commit three different revisions of the first object
         obj1.value = 1
-        revid1 = self._dostoreNP(oid1, data=pickle.dumps(obj1))
+        revid1 = self._dostore(oid1, data=obj1)
         obj1.value = 2
-        revid2 = self._dostoreNP(oid1, revid=revid1, data=pickle.dumps(obj1))
+        revid2 = self._dostore(oid1, revid=revid1, data=obj1)
         obj1.value = 3
-        revid3 = self._dostoreNP(oid1, revid=revid2, data=pickle.dumps(obj1))
+        revid3 = self._dostore(oid1, revid=revid2, data=obj1)
         # Now make sure all three revisions can be extracted
         data = self._storage.loadSerial(oid1, revid1)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid1)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 1)
         data = self._storage.loadSerial(oid1, revid2)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid1)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 2)
         data = self._storage.loadSerial(oid1, revid3)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid1)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 3)
         # Now commit a revision of the second object
         obj2.value = 11
-        revid4 = self._dostoreNP(oid2, data=pickle.dumps(obj2))
+        revid4 = self._dostore(oid2, data=obj2)
         # And make sure the revision can be extracted
         data = self._storage.loadSerial(oid2, revid4)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid2)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 11)
         # Now pack just revisions 1 and 2 of object1.  Object1's current
         # revision should stay alive because it's pointed to by the root, as
         # should Object2's current revision.
-        time.sleep(1)
+        now = time.time()
+        now = int(time.time())
+        while now >= int(time.time()):
+            time.sleep(0.1)
         self._storage.pack(time.time())
         # Make sure the revisions are gone, but that object zero, object2, and
         # revision 3 of object1 are still there and correct.
         data, revid = self._storage.load(ZERO, '')
         eq(revid, revid0)
-        eq(loads(data).value, 0)
+        eq(zodb_unpickle(data).value, 0)
         raises(KeyError, self._storage.loadSerial, oid1, revid1)
         raises(KeyError, self._storage.loadSerial, oid1, revid2)
         data = self._storage.loadSerial(oid1, revid3)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid1)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 3)
         data, revid = self._storage.load(oid1, '')
         eq(revid, revid3)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid1)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 3)
         data, revid = self._storage.load(oid2, '')
         eq(revid, revid4)
-        eq(loads(data).value, 11)
+        eq(zodb_unpickle(data).value, 11)
         data = self._storage.loadSerial(oid2, revid4)
-        pobj = pickle.loads(data)
-        eq(pobj.getoid(), oid2)
+        pobj = zodb_unpickle(data)
         eq(pobj.value, 11)
 
     def checkPackUnlinkedFromRoot(self):


=== ZODB4/src/zodb/storage/tests/readonly.py 1.5 => 1.6 ===
--- ZODB4/src/zodb/storage/tests/readonly.py:1.5	Tue Feb 25 13:55:03 2003
+++ ZODB4/src/zodb/storage/tests/readonly.py	Thu Mar 13 16:32:29 2003
@@ -52,7 +52,7 @@
         self.assertRaises(ReadOnlyError, self._storage.commitVersion,
                           '', '', t)
         self.assertRaises(ReadOnlyError, self._storage.store,
-                          ZERO, None, '', '', t)
+                          ZERO, None, '', '', '', t)
 
         if IUndoStorage.isImplementedBy(self._storage):
             self.assertRaises(ReadOnlyError, self._storage.undo,


=== ZODB4/src/zodb/storage/tests/recovery.py 1.4 => 1.5 ===
--- ZODB4/src/zodb/storage/tests/recovery.py:1.4	Wed Feb  5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/recovery.py	Thu Mar 13 16:32:29 2003
@@ -23,6 +23,7 @@
 from zodb.ztransaction import Transaction
 from zodb.storage.tests.iterator import IteratorDeepCompare
 from zodb.storage.tests.base import MinPO, zodb_unpickle
+from zodb.serialize import findrefs
 
 
 class RecoveryStorage(IteratorDeepCompare):
@@ -76,8 +77,19 @@
             self.assertRaises(StopIteration, txniter.next)
             self.assertEqual(data.oid, oid)
             self.assertEqual(data.data, None)
+            siter.close()
 
     def checkRecoverUndoInVersion(self):
+
+        def checkObj(oid, version, v_data, nv_data):
+            # Cause test to fail if object oid has wrong data.
+            # Last two args are MinPO value on version and non-version.
+            data, revid = self._storage.load(oid, "")
+            self.assertEqual(zodb_unpickle(data).value, nv_data)
+            data, revid = self._storage.load(oid, version)
+            self.assertEqual(zodb_unpickle(data).value, v_data)
+
+
         oid = self._storage.newObjectId()
         version = "aVersion"
         revid_a = self._dostore(oid, data=MinPO(91))
@@ -103,11 +115,7 @@
         self._undo(self._storage.undoInfo()[0]['id'], oid)
         self.assert_(not self._storage.versionEmpty(version))
 
-        # check the data is what we expect it to be
-        data, revid = self._storage.load(oid, version)
-        self.assertEqual(zodb_unpickle(data), MinPO(92))
-        data, revid = self._storage.load(oid, '')
-        self.assertEqual(zodb_unpickle(data), MinPO(91))
+        checkObj(oid, version, 92, 91)
 
         # and swap the storages
         tmp = self._storage
@@ -117,11 +125,7 @@
         self._undo(self._storage.undoInfo()[0]['id'], oid)
         self.assert_(not self._storage.versionEmpty(version))
 
-        # check the data is what we expect it to be
-        data, revid = self._storage.load(oid, version)
-        self.assertEqual(zodb_unpickle(data), MinPO(92))
-        data, revid = self._storage.load(oid, '')
-        self.assertEqual(zodb_unpickle(data), MinPO(91))
+        checkObj(oid, version, 92, 91)
 
         # swap them back
         self._storage = tmp
@@ -135,6 +139,7 @@
         self.compare(self._storage, self._dst)
 
     def checkRestoreAcrossPack(self):
+        eq = self.assertEqual
         db = DB(self._storage)
         c = db.open()
         r = c.root()
@@ -150,10 +155,46 @@
 
         # copy the final transaction manually.  even though there
         # was a pack, the restore() ought to succeed.
-        final = list(self._storage.iterator())[-1]
+        iter = self._storage.iterator()
+        final = list(iter)[-1]
         self._dst.tpcBegin(final, final.tid, final.status)
+        datarefs = []
         for r in final:
-            self._dst.restore(r.oid, r.serial, r.data, r.version, r.data_txn,
-                              final)
+            datarefs.append((r.data, r.refs))
+            self._dst.restore(r.oid, r.serial, r.data, r.refs, r.version,
+                              r.data_txn, final)
         self._dst.tpcVote(final)
         self._dst.tpcFinish(final)
+        iter.close()
+        data, refs = datarefs[0]
+        eq(findrefs(data), refs)
+        data, refs = datarefs[1]
+        eq(data, None)
+        eq(refs, None)
+
+    def checkPackWithGCOnDestinationAfterRestore(self):
+        raises = self.assertRaises
+        db = DB(self._storage)
+        conn = db.open()
+        root = conn.root()
+        root.obj = obj1 = MinPO(1)
+        txn = get_transaction()
+        txn.note('root -> obj')
+        txn.commit()
+        root.obj.obj = obj2 = MinPO(2)
+        txn = get_transaction()
+        txn.note('root -> obj -> obj')
+        txn.commit()
+        del root.obj
+        txn = get_transaction()
+        txn.note('root -X->')
+        txn.commit()
+        # Now copy the transactions to the destination
+        self._dst.copyTransactionsFrom(self._storage)
+        # Now pack the destination
+        self._dst.pack(time.time())
+        # And check to see that the root object exists, but not the other
+        # objects.
+        data, serial = self._dst.load(root._p_oid, '')
+        raises(KeyError, self._dst.load, obj1._p_oid, '')
+        raises(KeyError, self._dst.load, obj2._p_oid, '')


=== ZODB4/src/zodb/storage/tests/revision.py 1.5 => 1.6 ===
--- ZODB4/src/zodb/storage/tests/revision.py:1.5	Tue Feb 25 13:55:03 2003
+++ ZODB4/src/zodb/storage/tests/revision.py	Thu Mar 13 16:32:29 2003
@@ -17,7 +17,7 @@
 
 from zodb.storage.base import ZERO
 from zodb.storage.tests.minpo import MinPO
-from zodb.storage.tests.base import zodb_unpickle, zodb_pickle
+from zodb.storage.tests.base import zodb_unpickle
 
 class RevisionStorage:
 


=== ZODB4/src/zodb/storage/tests/synchronization.py 1.4 => 1.5 ===
--- ZODB4/src/zodb/storage/tests/synchronization.py:1.4	Wed Feb  5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/synchronization.py	Thu Mar 13 16:32:29 2003
@@ -103,11 +103,11 @@
 
     def checkStoreNotCommitting(self):
         self.verifyNotCommitting(self._storage.store,
-                                 OID, SERIALNO, "", "", Transaction())
+                                 OID, SERIALNO, "", "", "", Transaction())
 
     def checkStoreWrongTrans(self):
         self.verifyWrongTrans(self._storage.store,
-                              OID, SERIALNO, "", "", Transaction())
+                              OID, SERIALNO, "", "", "", Transaction())
 
 ##    def checkNewOidNotCommitting(self):
 ##        self.verifyNotCommitting(self._storage.new_oid)


=== ZODB4/src/zodb/storage/tests/test_autopack.py 1.9 => 1.10 ===
--- ZODB4/src/zodb/storage/tests/test_autopack.py:1.9	Wed Feb  5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/test_autopack.py	Thu Mar 13 16:32:29 2003
@@ -287,7 +287,8 @@
         obj2 = C()
         t = Transaction()
         storage.tpcBegin(t)
-        obj2sn = storage.store('\0'*7 + '\2', ZERO, zodb_pickle(obj2), '', t)
+        data, refs = zodb_pickle(obj2)
+        obj2sn = storage.store('\0'*7 + '\2', ZERO, data, refs, '', t)
         # Now, acquire the condvar lock and start a thread that will do a
         # pack, up to the _sweep call.  Wait for the _mark() call to
         # complete.
@@ -302,7 +303,8 @@
         # links the object to root.
         root.obj2 = obj2
         rootsn = storage.getSerial(ZERO)
-        rootsn = storage.store(ZERO, rootsn, zodb_pickle(root), '', t)
+        data, refs = zodb_pickle(root)
+        rootsn = storage.store(ZERO, rootsn, data, refs, '', t)
         storage.tpcVote(t)
         storage.tpcFinish(t)
         # And notify the pack thread that it can do the sweep and collect
@@ -349,12 +351,14 @@
         packthread = self._getPackThread(storage)
         packthread.start()
         self._cv.wait()
-        obj2sn = storage.store('\0'*7 + '\2', ZERO, zodb_pickle(obj2), '', t)
+        data, refs = zodb_pickle(obj2)
+        obj2sn = storage.store('\0'*7 + '\2', ZERO, data, refs, '', t)
         # Now that the _mark() has finished, complete the transaction, which
         # links the object to root.
         root.obj2 = obj2
         rootsn = storage.getSerial(ZERO)
-        rootsn = storage.store(ZERO, rootsn, zodb_pickle(root), '', t)
+        data, refs = zodb_pickle(root)
+        rootsn = storage.store(ZERO, rootsn, data, refs, '', t)
         storage.tpcVote(t)
         storage.tpcFinish(t)
         # And notify the pack thread that it can do the sweep and collect


=== ZODB4/src/zodb/storage/tests/test_file.py 1.6 => 1.7 ===
--- ZODB4/src/zodb/storage/tests/test_file.py:1.6	Wed Feb  5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/test_file.py	Thu Mar 13 16:32:29 2003
@@ -11,9 +11,12 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
-import zodb.storage.file
-import sys, os, unittest
+import os
+import sys
 import errno
+import unittest
+
+import zodb.storage.file
 from zodb.ztransaction import Transaction
 from zodb.storage.interfaces import *
 
@@ -23,6 +26,8 @@
 
 from zodb.storage.tests.base import MinPO, zodb_unpickle
 
+from transaction import get_transaction
+
 class FileStorageTests(
     base.StorageTestBase,
     basic.BasicStorage,
@@ -50,6 +55,7 @@
         self.open(create=1)
 
     def tearDown(self):
+        get_transaction().abort()
         self._test_read_index()
         self._storage.close()
         base.removefs("FileStorageTests.fs")
@@ -65,7 +71,7 @@
             from zodb.storage.fsdump import Dumper
             Dumper("FileStorageTests.fs").dump()
             raise
-        
+
         L1 = copy._index.items()
         L2 = self._storage._index.items()
         L1.sort(); L2.sort()


=== ZODB4/src/zodb/storage/tests/undo.py 1.6 => 1.7 ===
--- ZODB4/src/zodb/storage/tests/undo.py:1.6	Fri Feb 28 11:16:57 2003
+++ ZODB4/src/zodb/storage/tests/undo.py	Thu Mar 13 16:32:29 2003
@@ -4,11 +4,11 @@
 """
 
 import time
-import types
 
 from zodb import interfaces
+from zodb.interfaces import ZERO
 from zodb.ztransaction import Transaction
-from zodb.utils import u64, p64, z64
+from zodb.utils import u64, p64
 from zodb.db import DB
 
 from zodb.storage.tests.minpo import MinPO
@@ -25,10 +25,10 @@
     def _transaction_begin(self):
         self.__serials = {}
 
-    def _transaction_store(self, oid, rev, data, vers, trans):
-        r = self._storage.store(oid, rev, data, vers, trans)
+    def _transaction_store(self, oid, rev, (data, refs), vers, trans):
+        r = self._storage.store(oid, rev, data, refs, vers, trans)
         if r:
-            if type(r) == types.StringType:
+            if isinstance(r, str):
                 self.__serials[oid] = r
             else:
                 for oid, serial in r:
@@ -43,13 +43,16 @@
     def _transaction_newserial(self, oid):
         return self.__serials[oid]
 
-    def _multi_obj_transaction(self, objs):
+    def _multi_obj_transaction(self, objs, note=None):
         newrevs = {}
         t = Transaction()
+        if note:
+            t.note(note)
         self._storage.tpcBegin(t)
         self._transaction_begin()
-        for oid, rev, data in objs:
-            self._transaction_store(oid, rev, data, '', t)
+        # datarefs is a tuple of (data, refs)
+        for oid, rev, datarefs in objs:
+            self._transaction_store(oid, rev, datarefs, '', t)
             newrevs[oid] = None
         self._transaction_vote(t)
         self._storage.tpcFinish(t)
@@ -196,11 +199,10 @@
     def checkTwoObjectUndo(self):
         eq = self.assertEqual
         # Convenience
-        p31, p32, p51, p52 = map(zodb_pickle,
-                                 map(MinPO, (31, 32, 51, 52)))
+        p31, p32, p51, p52 = [zodb_pickle(MinPO(i)) for i in (31, 32, 51, 52)]
         oid1 = self._storage.newObjectId()
         oid2 = self._storage.newObjectId()
-        revid1 = revid2 = z64
+        revid1 = revid2 = ZERO
         # Store two objects in the same transaction
         t = Transaction()
         self._storage.tpcBegin(t)
@@ -251,26 +253,25 @@
         # Convenience
         eq = self.assertEqual
         unless = self.failUnless
-        p30, p31, p32, p50, p51, p52 = map(zodb_pickle,
-                                           map(MinPO,
-                                               (30, 31, 32, 50, 51, 52)))
+        p30, p31, p32, p50, p51, p52 = [zodb_pickle(MinPO(i))
+                                        for i in (30, 31, 32, 50, 51, 52)]
         oid1 = self._storage.newObjectId()
         oid2 = self._storage.newObjectId()
-        revid1 = revid2 = z64
+        revid1 = revid2 = ZERO
         # Store two objects in the same transaction
         d = self._multi_obj_transaction([(oid1, revid1, p30),
                                          (oid2, revid2, p50),
-                                         ])
+                                         ], "first update")
         eq(d[oid1], d[oid2])
         # Update those same two objects
         d = self._multi_obj_transaction([(oid1, d[oid1], p31),
                                          (oid2, d[oid2], p51),
-                                         ])
+                                         ], "second update")
         eq(d[oid1], d[oid2])
         # Update those same two objects
         d = self._multi_obj_transaction([(oid1, d[oid1], p32),
                                          (oid2, d[oid2], p52),
-                                         ])
+                                         ], "third update")
         eq(d[oid1], d[oid2])
         revid1 = self._transaction_newserial(oid1)
         revid2 = self._transaction_newserial(oid2)
@@ -285,14 +286,12 @@
         tid = info[0]['id']
         tid1 = info[1]['id']
         t = Transaction()
+        t.note("undo transaction containing two objects")
         self._storage.tpcBegin(t)
         oids = self._storage.undo(tid, t)
         oids1 = self._storage.undo(tid1, t)
         self._storage.tpcVote(t)
         self._storage.tpcFinish(t)
-        # We get the finalization stuff called an extra time:
-##        self._storage.tpc_vote(t)
-##        self._storage.tpc_finish(t)
         eq(len(oids), 2)
         eq(len(oids1), 2)
         unless(oid1 in oids)
@@ -305,6 +304,7 @@
         info = self._storage.undoInfo()
         tid = info[0]['id']
         t = Transaction()
+        t.note("undo the undo")
         self._storage.tpcBegin(t)
         oids = self._storage.undo(tid, t)
         self._storage.tpcVote(t)
@@ -320,15 +320,14 @@
 
     def checkTwoObjectUndoAgain(self):
         eq = self.assertEqual
-        p31, p32, p33, p51, p52, p53 = map(
-            zodb_pickle,
-            map(MinPO, (31, 32, 33, 51, 52, 53)))
+        p31, p32, p33, p51, p52, p53 = [zodb_pickle(MinPO(i))
+                                        for i in (31, 32, 33, 51, 52, 53)]
         # Like the above, but the first revision of the objects are stored in
         # different transactions.
         oid1 = self._storage.newObjectId()
         oid2 = self._storage.newObjectId()
-        revid1 = self._dostore(oid1, data=p31, already_pickled=1)
-        revid2 = self._dostore(oid2, data=p51, already_pickled=1)
+        revid1 = self._dostore(oid1, data=p31, already_pickled=True)
+        revid2 = self._dostore(oid2, data=p51, already_pickled=True)
         # Update those same two objects
         t = Transaction()
         self._storage.tpcBegin(t)
@@ -412,10 +411,8 @@
         oid1 = oid
         revid1 = revid_c
         oid2 = self._storage.newObjectId()
-        revid2 = z64
-        p81, p82, p91, p92 = map(zodb_pickle,
-                                 map(MinPO, (81, 82, 91, 92)))
-
+        revid2 = ZERO
+        p81, p82, p91, p92 = [zodb_pickle(MinPO(i)) for i in (81, 82, 91, 92)]
         t = Transaction()
         self._storage.tpcBegin(t)
         self._transaction_begin()
@@ -450,8 +447,10 @@
     def checkTransactionalUndoAfterPack(self):
         eq = self.assertEqual
         # Add a few object revisions
-        oid = self._storage.newObjectId()
-        revid1 = self._dostore(oid, data=MinPO(51))
+        self._initroot()
+        obj, root, revid0 = self._linked_newobj()
+        oid = obj._p_oid
+        revid1 = self._dostore(oid, data=MinPO(51), revid=obj._p_serial)
         # For the pack(), we need a timestamp greater than revid1's timestamp.
         # The semantics of pack()'s `t' argument is that all non-current
         # revisions with an earlier timestamp will be packed away.  If they
@@ -468,12 +467,14 @@
         revid3 = self._dostore(oid, revid=revid2, data=MinPO(53))
         # Now get the undo log
         info = self._storage.undoInfo()
-        eq(len(info), 3)
+        eq(len(info), 5)
         tid = info[0]['id']
+
         # Now pack just the initial revision of the object.  We need the
         # second revision otherwise we won't be able to undo the third
         # revision!
         self._storage.pack(packtime)
+
         # Make some basic assertions about the undo information now
         info2 = self._storage.undoInfo()
         eq(len(info2), 2)
@@ -564,7 +565,8 @@
             for j in range(OBJECTS):
                 oid = s.newObjectId()
                 obj = MinPO(i * OBJECTS + j)
-                revid = s.store(oid, None, zodb_pickle(obj), '', t)
+                data, refs = zodb_pickle(obj)
+                revid = s.store(oid, None, data, refs, '', t)
                 orig.append((tid, oid, revid))
             s.tpcVote(t)
             s.tpcFinish(t)


=== ZODB4/src/zodb/storage/tests/undoversion.py 1.3 => 1.4 ===
--- ZODB4/src/zodb/storage/tests/undoversion.py:1.3	Wed Feb  5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/undoversion.py	Thu Mar 13 16:32:29 2003
@@ -28,13 +28,16 @@
                                 version=version)
         revid_c = self._dostore(oid, revid=revid_b, data=MinPO(93),
                                 version=version)
+
         info=self._storage.undoInfo()
         tid=info[0]['id']
         t = Transaction()
+        t.note("undo last revision (93)")
         self._storage.tpcBegin(t)
         oids = self._storage.undo(tid, t)
         self._storage.tpcVote(t)
         self._storage.tpcFinish(t)
+
         assert len(oids) == 1
         assert oids[0] == oid
         data, revid = self._storage.load(oid, '')
@@ -45,10 +48,12 @@
         assert zodb_unpickle(data) == MinPO(92)
         # Now commit the version...
         t = Transaction()
+        t.note("commit version")
         self._storage.tpcBegin(t)
         oids = self._storage.commitVersion(version, '', t)
         self._storage.tpcVote(t)
         self._storage.tpcFinish(t)
+
         assert len(oids) == 1
         assert oids[0] == oid
 
@@ -60,16 +65,18 @@
         info=self._storage.undoInfo()
         tid=info[0]['id']
         t = Transaction()
+        t.note("undo commit version")
         self._storage.tpcBegin(t)
         oids = self._storage.undo(tid, t)
         self._storage.tpcVote(t)
         self._storage.tpcFinish(t)
+
         assert len(oids) == 1
         assert oids[0] == oid
         data, revid = self._storage.load(oid, version)
-        assert zodb_unpickle(data) == MinPO(92)
+        self.assertEqual(zodb_unpickle(data), MinPO(92))
         data, revid = self._storage.load(oid, '')
-        assert zodb_unpickle(data) == MinPO(91)
+        self.assertEqual(zodb_unpickle(data), MinPO(91))
         # Now abort the version
         t = Transaction()
         self._storage.tpcBegin(t)


=== ZODB4/src/zodb/storage/tests/version.py 1.3 => 1.4 ===
--- ZODB4/src/zodb/storage/tests/version.py:1.3	Wed Feb  5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/version.py	Thu Mar 13 16:32:29 2003
@@ -16,18 +16,23 @@
 Any storage that supports versions should be able to pass all these tests.
 """
 
+import time
+
 from zodb import interfaces
+from zodb.db import DB
 from zodb.ztransaction import Transaction
 from zodb.storage.tests.minpo import MinPO
 from zodb.storage.tests.base import zodb_unpickle
 from zodb.storage.interfaces import IUndoStorage, IVersionStorage
 
+from transaction import get_transaction
+
 
 class VersionStorage:
 
     def checkIVersionStorage(self):
         self.assert_(IVersionStorage.isImplementedBy(self._storage))
-    
+
     def checkVersionedStoreAndLoad(self):
         eq = self.assertEqual
         # Store a couple of non-version revisions of the object
@@ -117,10 +122,6 @@
         unless('one' in versions)
         unless('two' in versions)
         unless('three' in versions)
-        # Now flex the `max' argument
-        versions = self._storage.versions(1)
-        self.assertEqual(len(versions), 1)
-        unless('one' in versions or 'two' in versions or 'three' in versions)
 
     def _setup_version(self, version='test-version'):
         # Store some revisions in the non-version
@@ -332,3 +333,45 @@
                           self._storage.load, oid, '')
         self.assertRaises(KeyError,
                           self._storage.load, oid, 'two')
+
+    def checkPackVersions(self):
+        db = DB(self._storage)
+        cn = db.open(version="testversion")
+        root = cn.root()
+
+        obj = root["obj"] = MinPO("obj")
+        root["obj2"] = MinPO("obj2")
+        txn = get_transaction()
+        txn.note("create 2 objs in version")
+        txn.commit()
+
+        obj.value = "77"
+        txn = get_transaction()
+        txn.note("modify obj in version")
+        txn.commit()
+
+        # undo the modification to generate a mix of backpointers
+        # and versions for pack to chase
+        info = db.undoInfo()
+        db.undo(info[0]["id"])
+        txn = get_transaction()
+        txn.note("undo modification")
+        txn.commit()
+
+        self._storage.pack(time.time())
+
+        db.commitVersion("testversion")
+        txn = get_transaction()
+        txn.note("commit version")
+        txn.commit()
+
+        cn = db.open()
+        root = cn.root()
+        root["obj"] = "no version"
+
+        txn = get_transaction()
+        txn.note("modify obj")
+        txn.commit()
+
+        self._storage.pack(time.time())
+