[Zodb-checkins] CVS: ZODB3/BDBStorage - BDBFullStorage.py:1.44.4.3

Barry Warsaw barry@wooz.org
Tue, 21 Jan 2003 17:28:41 -0500


Update of /cvs-repository/ZODB3/BDBStorage
In directory cvs.zope.org:/tmp/cvs-serv26584

Modified Files:
      Tag: ZODB3-3_1-branch
	BDBFullStorage.py 
Log Message:
Backporting of various changes from the 3.2 branch.  Specifically:

- generalize the table for storage metadata, i.e. packtime -> info

- get ZERO from the package

- close the pack race condition


=== ZODB3/BDBStorage/BDBFullStorage.py 1.44.4.2 => 1.44.4.3 ===
--- ZODB3/BDBStorage/BDBFullStorage.py:1.44.4.2	Tue Jan  7 14:38:52 2003
+++ ZODB3/BDBStorage/BDBFullStorage.py	Tue Jan 21 17:28:36 2003
@@ -1,6 +1,6 @@
 ##############################################################################
 #
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# Copyright (c) 2001 Zope Corporation and Contributors.
 # All Rights Reserved.
 #
 # This software is subject to the provisions of the Zope Public License,
@@ -13,9 +13,9 @@
 ##############################################################################
 
 """Berkeley storage with full undo and versioning support.
-"""
 
-__version__ = '$Revision$'.split()[-2:][0]
+$Revision$
+"""
 
 import time
 import cPickle as pickle
@@ -27,13 +27,12 @@
 from ZODB.TimeStamp import TimeStamp
 from ZODB.ConflictResolution import ConflictResolvingStorage, ResolvedSerial
 
-from BDBStorage import db
+from BDBStorage import db, ZERO
 from BDBStorage.BerkeleyBase import BerkeleyBase, PackStop, _WorkThread
 
 ABORT = 'A'
 COMMIT = 'C'
 PRESENT = 'X'
-ZERO = '\0'*8
 
 # Special flag for uncreated objects (i.e. Does Not Exist)
 DNE = '\377'*8
@@ -178,9 +177,15 @@
         #     pending table is empty, the oids, pvids, and prevrevids tables
         #     must also be empty.
         #
-        # packtime -- tid
-        #     The time of the last pack.  It is illegal to undo to before the
-        #     last pack time.
+        # info -- {key -> value}
+        #     This table contains storage metadata information.  The keys and
+        #     values are simple strings of variable length.   Here are the
+        #     valid keys:
+        #
+        #         packtime - time of the last pack.  It is illegal to undo to
+        #         before the last pack time.
+        #
+        #         version - the version of the database (reserved for ZODB4)
         #
         # objrevs -- {newserial+oid -> oldserial}
         #     This table collects object revision information for packing
@@ -204,6 +209,8 @@
         #     This table is a Queue, not a BTree.  It is used during the mark
         #     phase of pack() and contains a list of oids for work to be done.
         #
+        self._packing = False
+        self._info = self._setupDB('info')
         self._serials = self._setupDB('serials', db.DB_DUP)
         self._pickles = self._setupDB('pickles')
         self._refcounts = self._setupDB('refcounts')
@@ -223,7 +230,6 @@
         # Tables to support packing.
         self._objrevs = self._setupDB('objrevs', db.DB_DUP)
         self._packmark = self._setupDB('packmark')
-        self._packtime = self._setupDB('packtime')
         self._oidqueue = self._setupDB('oidqueue', 0, db.DB_QUEUE, 8)
         self._delqueue = self._setupDB('delqueue', 0, db.DB_QUEUE, 8)
         # Do recovery and consistency checks
@@ -429,10 +435,18 @@
             refcount = self._refcounts.get(oid, ZERO, txn=txn)
             self._refcounts.put(oid, incr(refcount, delta), txn=txn)
         # Now clean up the temporary log tables
-        self._oids.truncate(txn)
         self._pvids.truncate(txn)
         self._prevrevids.truncate(txn)
         self._pending.truncate(txn)
+        # If we're in the middle of a pack, we need to add to the packmark
+        # table any objects that were modified in this transaction.
+        # Otherwise, there's a race condition where mark might have happened,
+        # then the object is added, then sweep runs, deleting the object
+        # created in the interrim.
+        if self._packing:
+            for oid in self._oids.keys():
+                self._packmark.put(oid, PRESENT, txn=txn)
+        self._oids.truncate(txn)
 
     def _dobegin(self, txn, tid, u, d, e):
         # When a transaction begins, we set the pending flag to ABORT,
@@ -1032,13 +1046,7 @@
             self._lock_release()
 
     def _last_packtime(self):
-        packtimes = self._packtime.keys()
-        if len(packtimes) == 1:
-            return packtimes[0]
-        elif len(packtimes) == 0:
-            return ZERO
-        else:
-            assert False, 'too many packtimes'
+        return self._info.get('packtime', ZERO)
 
     def lastTransaction(self):
         """Return transaction id for last committed transaction"""
@@ -1306,11 +1314,10 @@
         finally:
             self._lock_release()
 
-    #
     # Packing
     #
     # There are two types of pack operations, the classic pack and the
-    # autopack.  Autopack's sole job is to periodically delete non-current
+    # autopack.  Autopack's primary job is to periodically delete non-current
     # object revisions.  It runs in a thread and has an `autopack time' which
     # is essentially just a time in the past at which to autopack to.  For
     # example, you might set up autopack to run once per hour, packing away
@@ -1333,7 +1340,6 @@
     # acquisition as granularly as possible so that packing doesn't block
     # other operations for too long.  But remember we don't use Berkeley locks
     # so we have to be careful about our application level locks.
-    #
 
     # First, the public API for classic pack
     def pack(self, t, zreferencesf):
@@ -1346,6 +1352,7 @@
         # A simple wrapper around the bulk of packing, but which acquires a
         # lock that prevents multiple packs from running at the same time.
         self._packlock.acquire()
+        self._packing = True
         try:
             # We don't wrap this in _withtxn() because we're going to do the
             # operation across several Berkeley transactions, which allows
@@ -1353,6 +1360,7 @@
             # done.
             self._dopack(t)
         finally:
+            self._packing = False
             self._packlock.release()
         self.log('classic pack finished')
 
@@ -1417,6 +1425,7 @@
         # A simple wrapper around the bulk of packing, but which acquires a
         # lock that prevents multiple packs from running at the same time.
         self._packlock.acquire()
+        self._packing = True
         try:
             # We don't wrap this in _withtxn() because we're going to do the
             # operation across several Berkeley transactions, which allows
@@ -1424,6 +1433,7 @@
             # done.
             self._dopack(t, gc)
         finally:
+            self._packing = False
             self._packlock.release()
         self.log('autopack finished')
 
@@ -1471,10 +1481,9 @@
             if co: co.close()
             if ct: ct.close()
         # Note that before we commit this Berkeley transaction, we also need
-        # to update the packtime table, so we can't have the possibility of a
-        # race condition with undoLog().
-        self._packtime.truncate(txn)
-        self._packtime.put(packtid, PRESENT, txn=txn)
+        # to update the last packtime entry, so we can't have the possibility
+        # of a race condition with undoLog().
+        self._info.put('packtime', packtid, txn=txn)
 
     def _decrefPickle(self, oid, lrevid, txn):
         if lrevid == DNE:
@@ -1601,7 +1610,6 @@
         # oidqueue is a BerkeleyDB Queue that holds the list of object ids to
         # look at next, and by using this we don't need to keep an in-memory
         # dictionary.
-        assert len(self._packmark) == 0
         assert len(self._oidqueue) == 0
         # Quick exit for empty storages
         if not self._serials: