[Zodb-checkins] CVS: ZODB4/src/zodb/storage - bdbminimal.py:1.6

Barry Warsaw barry@wooz.org
Wed, 22 Jan 2003 14:29:44 -0500


Update of /cvs-repository/ZODB4/src/zodb/storage
In directory cvs.zope.org:/tmp/cvs-serv10364

Modified Files:
	bdbminimal.py 
Log Message:
Forward port from ZODB 3.2

- Get db (bsddb) and ZERO from the zodb.storage.base module

- Generalize storage-specific metadata (i.e. packtime -> info)

- Add _packing flag and implement fix for pack race condition.


=== ZODB4/src/zodb/storage/bdbminimal.py 1.5 => 1.6 ===
--- ZODB4/src/zodb/storage/bdbminimal.py:1.5	Wed Jan 15 18:28:03 2003
+++ ZODB4/src/zodb/storage/bdbminimal.py	Wed Jan 22 14:29:41 2003
@@ -1,6 +1,6 @@
 ##############################################################################
 #
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# Copyright (c) 2001 Zope Corporation and Contributors.
 # All Rights Reserved.
 #
 # This software is subject to the provisions of the Zope Public License,
@@ -13,31 +13,19 @@
 ##############################################################################
 
 """Berkeley storage without undo or versioning.
-"""
-
-__version__ = '$Revision$'[-2:][0]
 
-# In Python 2.3, we can simply use the bsddb module, but for Python 2.2, we
-# need to use pybsddb3, a.k.a. bsddb3.
-try:
-    from bsddb import _db as db
-except ImportError:
-    from bsddb3 import db
+$Revision$
+"""
 
 from zodb import interfaces
 from zodb.utils import p64, u64
 from zodb.serialize import findrefs
 from zodb.conflict import ConflictResolvingStorage, ResolvedSerial
-
-# BerkeleyBase class provides some common functionality for BerkeleyDB-based
-# storages.  It in turn inherits from BaseStorage which itself provides some
-# common storage functionality.
-from zodb.storage.base import BerkeleyBase, PackStop, _WorkThread
+from zodb.storage.base import db, ZERO, BerkeleyBase, PackStop, _WorkThread
 
 ABORT = 'A'
 COMMIT = 'C'
 PRESENT = 'X'
-ZERO = '\0'*8
 
 
 
@@ -89,6 +77,13 @@
         #     no pending entry.  It is a database invariant that if the
         #     pending table is empty, the oids table must also be empty.
         #
+        # info -- {key -> value}
+        #     This table contains storage metadata information.  The keys and
+        #     values are simple strings of variable length.   Here are the
+        #     valid keys:
+        #
+        #         version - the version of the database (reserved for ZODB4)
+        #
         # packmark -- [oid]
         #     Every object reachable from the root during a classic pack
         #     operation will have its oid present in this table.
@@ -100,6 +95,8 @@
         #     references exist, such that the objects can be completely packed
         #     away.
         #
+        self._packing = False
+        self._info = self._setupDB('info')
         self._serials = self._setupDB('serials', db.DB_DUP)
         self._pickles = self._setupDB('pickles')
         self._refcounts = self._setupDB('refcounts')
@@ -181,6 +178,8 @@
                     if soid <> oid:
                         break
                     if stid <> tid:
+                        # This is the previous revision of the object, so
+                        # decref its referents and clean up its pickles.
                         cs.delete()
                         data = self._pickles.get(oid+stid, txn=txn)
                         assert data is not None
@@ -198,8 +197,16 @@
             if co: co.close()
             if cs: cs.close()
         # We're done with this table
-        self._oids.truncate(txn)
         self._pending.truncate(txn)
+        # If we're in the middle of a pack, we need to add to the packmark
+        # table any objects that were modified in this transaction.
+        # Otherwise, there's a race condition where mark might have happened,
+        # then the object is added, then sweep runs, deleting the object
+        # created in the interrim.
+        if self._packing:
+            for oid in self._oids.keys():
+                self._packmark.put(oid, PRESENT, txn=txn)
+        self._oids.truncate(txn)
         # Now, to finish up, we need apply the refcount deltas to the
         # refcounts table, and do recursive collection of all refcount == 0
         # objects.
@@ -353,6 +360,7 @@
         # A simple wrapper around the bulk of packing, but which acquires a
         # lock that prevents multiple packs from running at the same time.
         self._packlock.acquire()
+        self._packing = True
         try:
             # We don't wrap this in _withtxn() because we're going to do the
             # operation across several Berkeley transactions, which allows
@@ -363,20 +371,11 @@
             # collect object revisions
             self._dopack()
         finally:
+            self._packing = False
             self._packlock.release()
         self.log('classic pack finished')
 
     def _dopack(self):
-        # XXX There is a potential race condition here that we need to address
-        # so we don't lose objects.  Say we've just completed the mark phase
-        # and another thread comes along and stores an object.  Now we enter
-        # the sweep phase and notice that that other object isn't in the
-        # marked list.  So we add it to the collection list.  Full is less
-        # vulnerable to this hole (although it's still there) because it
-        # usually won't pack all the way to the current time.  I want to try
-        # to write a test case for this situation before I fix it, but that's
-        # difficult without instrumenting _dopack() for testing.
-        #
         # Do a mark and sweep for garbage collection.  Calculate the set of
         # objects reachable from the root.  Anything else is a candidate for
         # having all their revisions packed away.  The set of reachable
@@ -407,7 +406,6 @@
         # we'll save the mark data in the packmark table.  The oidqueue is a
         # BerkeleyDB Queue that holds the list of object ids to look at next,
         # and by using this we don't need to keep an in-memory dictionary.
-        assert len(self._packmark) == 0
         assert len(self._oidqueue) == 0
         # Quick exit for empty storages
         if not self._serials:
@@ -462,10 +460,6 @@
             if self._stop:
                 raise PackStop, 'stopped in _collect_objs()'
             oid = orec[1]
-            # Never pack away the root object.
-            if oid == ZERO:
-                orec = self._oidqueue.consume(txn)
-                continue
             # Delete the object from the serials table
             c = self._serials.cursor(txn)
             try: