[Zodb-checkins] CVS: StandaloneZODB/bsddb3Storage/bsddb3Storage - Full.py:1.40.2.1

[Jim Fulton]

Update of /cvs-repository/StandaloneZODB/bsddb3Storage/bsddb3Storage
In directory cvs.zope.org:/tmp/cvs-serv24317

Modified Files:
      Tag: bsddb3Storage-picklelog-branch
	Full.py 
Log Message:
Begun work on adding a pickle log. The idea is as follows:

  In each stpre call (or other call that writes pickles), execute a
  BDB transaction that writes the pickle to the pickle table and
  writes the pickle key to the pickle log.

  On ZODB transaction tpc_finish, remove the keys from the pickle log.

  On ZODB tpc_abort or on startup, remove the pickles for the keys in
  the pickle log and clear the pickle log.

  This should pretty reasonably limit the size of a BDB transaction.
  It may make things go faster. It should make tpc_finish go much
  faster. By doing many more smaller BDB transactions, we shouldn't
  block for really long times.

The work isn't done, but is being checked in so Barry can hack on it.




=== StandaloneZODB/bsddb3Storage/bsddb3Storage/Full.py 1.40 => 1.40.2.1 ===
 import time
 
+from cPickle import loads, Pickler
+Pickler = Pickler()
+Pickler.fast = 1 # Don't use a memo
+fast_pickle_dumps = Pickler.dump
+del Pickler
+
 # This uses the Dunn/Kuchling PyBSDDB v3 extension module available from
 # http://pybsddb.sourceforge.net
 from bsddb3 import db
@@ -97,7 +103,15 @@
         # pickles -- {oid+revid -> pickle}
         #     Maps the concrete object referenced by oid+revid to that
         #     object's data pickle.
+
+        # Jim
         #
+        # picklelog -- {oid+revid}
+        #     Keeps a log of pickles that haven't been committed yet.
+        #     This allows us to write pickles as we get them in the
+        #     in separate BDB transactions.
+        
+        
         # These are used only by the Full implementation.
         #
         # vids -- {version_string -> vid}
@@ -167,6 +181,8 @@
         # Tables common to the base framework 
         self._serials = self._setupDB('serials')
         self._pickles = self._setupDB('pickles')
+        self._picklelog = self._setupDB('picklelog')
+        
         # These are specific to the full implementation
         self._vids            = self._setupDB('vids')
         self._versions        = self._setupDB('versions')
@@ -176,6 +192,7 @@
         self._txnoids         = self._setupDB('txnoids', db.DB_DUP)
         self._refcounts       = self._setupDB('refcounts')
         self._pickleRefcounts = self._setupDB('pickleRefcounts')
+        
         # Initialize our cache of the next available version id.
         record = self._versions.cursor().last()
         if record:
@@ -191,6 +208,7 @@
     def close(self):
         self._serials.close()
         self._pickles.close()
+        self._picklelog.close()
         self._vids.close()
         self._versions.close()
         self._currentVersions.close()
@@ -643,8 +661,10 @@
                         tuple(map(U64, (oid, ovid, vid))))
                 else:
                     nvrevid = onvrevid
-            # Record the update to this object in the commit log.
-            self._commitlog.write_object(oid, vid, nvrevid, data, oserial)
+
+            self._log_object(oid, vid, nvrevid, data, oserial)
+
+            
         finally:
             self._lock_release()
         # Return our cached serial number for the object.  If conflict
@@ -653,6 +673,33 @@
             return ResolvedSerial
         return self._serial
 
+    def _log_object(self, oid, vid, nvrevid, data, oserial):
+
+        # Save data for later commit. We do this by writing the pickle
+        # directly to BDB and saving the pickle key in the pickle log.
+        # We extract the references and save them in the transaction log.
+
+        # Get the references
+        refdoids = []
+        referencesf(pickle, refdoids)
+        refdoids = fast_pickle_dumps(refdoids)
+
+        # Record the update to this object in the commit log.
+        self._commitlog.write_object(oid, vid, nvrevid, refdoids, oserial)
+
+        # Save the pickle in the database:
+        txn = self._env.txn_begin()
+        try:
+            key = oid + self._serial
+            self._pickles.put(key, data, txn=txn)
+            self._picklelog.put(key, '', txn=txn)
+        except:
+            txn.abort()
+            raise
+        else:
+            txn.commit()
+        
+
     def transactionalUndo(self, tid, transaction):
         if transaction is not self._transaction:
             raise POSException.StorageTransactionError(self, transaction)
@@ -776,8 +823,7 @@
                 # see duplicate oids in this iteration.
                 oids[oid] = 1
             for oid, vid, nvrevid, data, prevrevid in newstates:
-                self._commitlog.write_object(oid, vid, nvrevid, data,
-                                             prevrevid)
+                self._log_object(oid, vid, nvrevid, data, prevrevid)
                 oids[oid] = 1
             return oids.keys()
         finally: