[Zodb-checkins] CVS: StandaloneZODB/bsddb3Storage/bsddb3Storage - Full.py:1.40.2.7
Barry Warsaw
barry@wooz.org
Thu, 22 Aug 2002 17:55:49 -0400
Update of /cvs-repository/StandaloneZODB/bsddb3Storage/bsddb3Storage
In directory cvs.zope.org:/tmp/cvs-serv14573/lib/python/bsddb3Storage
Modified Files:
Tag: bsddb3Storage-picklelog-branch
Full.py
Log Message:
Be more aggressive about optimistically writing transaction records,
so as to mitigate the potential worst case locking scenario. Some
specifics:
_log_object(): Put not only the pickle data, but also the metadata
records optimistically. This means in the normal case, we don't need
to write those in the loop at the end of _finish(). However, we do
have to remove these records if we abort the transaction.
_finish(): Added another opcode `x' which is just like `o' except that
it skips writing the metadata record, but does all the other
bookkeeping. `x' is used for store() calls, while `o' is used for
other calls that don't (yet?) use aggressive optimism <wink> such as
in commitVersion().
Also, added a bunch of commented out debug prints which can help to
analyze lock usage during each mini-loop.
Also, transform db.DBNoMemberError into the new
POSException.TransactionTooLargeError error.
Also, incorporate periodic checkpointing into _finish().
(Get rid of the duplicate _log_object() method!)
Whitespace normalization.
=== StandaloneZODB/bsddb3Storage/bsddb3Storage/Full.py 1.40.2.6 => 1.40.2.7 ===
--- StandaloneZODB/bsddb3Storage/bsddb3Storage/Full.py:1.40.2.6 Mon Jun 24 11:34:00 2002
+++ StandaloneZODB/bsddb3Storage/bsddb3Storage/Full.py Thu Aug 22 17:55:48 2002
@@ -2,14 +2,14 @@
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
-#
+#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
-#
+#
##############################################################################
"""Berkeley storage with full undo and versioning support.
@@ -184,7 +184,7 @@
# Maps the concrete object referenced by oid+tid to the reference
# count of its pickle.
#
- # Tables common to the base framework
+ # Tables common to the base framework
self._serials = self._setupDB('serials')
self._pickles = self._setupDB('pickles')
self._picklelog = self._setupDB('picklelog')
@@ -197,7 +197,7 @@
self._txnoids = self._setupDB('txnoids', db.DB_DUP)
self._refcounts = self._setupDB('refcounts')
self._pickleRefcounts = self._setupDB('pickleRefcounts')
-
+
# Initialize our cache of the next available version id.
record = self._versions.cursor().last()
if record:
@@ -213,7 +213,7 @@
# timestamps. Things like packing and undoing will break.
#self._nextserial = 0L
#self.profiler = hotshot.Profile('profile.dat', lineevents=1)
-
+
def close(self):
#self.profiler.close()
self._serials.close()
@@ -301,7 +301,7 @@
if rec is None:
break
op, data = rec
- if op == 'o':
+ if op in 'ox':
# This is a `versioned' object record. Information about
# this object must be stored in the pickle table, the
# object metadata table, the currentVersions tables , and
@@ -330,8 +330,10 @@
for roid in refdoids:
refcounts[roid] = refcounts.get(roid, 0) + 1
# Update the metadata table
- metadata.append((key,
- ''.join((vid,nvrevid,lrevid,prevrevid))))
+ if op == 'o':
+ # `x' opcode does an immediate write to metadata
+ metadata.append(
+ (key, ''.join((vid,nvrevid,lrevid,prevrevid))))
# If we're in a real version, update this table too. This
# ends up putting multiple copies of the vid/oid records
# in the table, but it's easier to weed those out later
@@ -359,31 +361,44 @@
rec = c.next_dup()
finally:
c.close()
- # It's actually faster to boogie through this list twice
+ # It's actually faster to boogie through this list twice
+ #print >> sys.stderr, 'start:', self._lockstats()
for oid, tid in serials:
self._txnoids.put(tid, oid, txn=txn)
+ #print >> sys.stderr, 'post-txnoids:', self._lockstats()
for oid, tid in serials:
self._serials.put(oid, tid, txn=txn)
+ #print >> sys.stderr, 'post-serials:', self._lockstats()
for key, data in metadata:
self._metadata.put(key, data, txn=txn)
+ #print >> sys.stderr, 'post-metadata:', self._lockstats()
for roid, delta in refcounts.items():
refcount = self._refcounts.get(roid, ZERO, txn=txn)
self._refcounts.put(roid, incr(refcount, delta), txn=txn)
+ #print >> sys.stderr, 'post-refcounts:', self._lockstats()
for key, delta in picklerefcounts.items():
refcount = self._pickleRefcounts.get(key, ZERO, txn=txn)
self._pickleRefcounts.put(key, incr(refcount, delta), txn=txn)
# We're done with the picklelog
self._picklelog.truncate(txn)
+ #print >> sys.stderr, 'loop-finish:', self._lockstats()
+ # Handle lock exhaustion differently
+ except db.DBNoMemoryError, e:
+ txn.abort()
+ self._docheckpoint()
+ raise POSException.TransactionTooLargeError, e
except:
# If any errors whatsoever occurred, abort the transaction with
# Berkeley, leave the commit log file in the PROMISED state (since
# its changes were never committed), and re-raise the exception.
txn.abort()
+ self._docheckpoint()
raise
else:
# Everything is hunky-dory. Commit the Berkeley transaction, and
# reset the commit log for the next transaction.
txn.commit()
+ self._docheckpoint()
self._closelog()
def _abort(self):
@@ -391,6 +406,7 @@
# pickle log, since we're abort this transaction.
for key in self._picklelog.keys():
del self._pickles[key]
+ del self._metadata[key]
# Done with the picklelog
self._picklelog.truncate()
BerkeleyBase._abort(self)
@@ -607,7 +623,7 @@
def loadSerial(self, oid, serial):
return self._loadSerialEx(oid, serial)[0]
-
+
def getSerial(self, oid):
# Return the revision id for the current revision of this object,
# irrespective of any versions.
@@ -637,8 +653,9 @@
def _log_object(self, oid, vid, nvrevid, data, oserial):
# Save data for later commit. We do this by writing the pickle
- # directly to the table and saving the pickle key in the pickle log.
- # We extract the references and save them in the transaction log.
+ # directly to the pickle table and saving the pickle key in the pickle
+ # log. We'll also save the metadata using the same technique. We
+ # extract the references and save them in the transaction log.
#
# Get the oids to the objects this pickle references
refdoids = []
@@ -649,7 +666,11 @@
txn = self._env.txn_begin()
try:
key = oid + self._serial
- self._pickles.put(key, data[:1000], txn=txn)
+ self._pickles.put(key, data, txn=txn)
+ self._metadata.put(
+ key,
+ ''.join((vid, nvrevid, self._serial, oserial)),
+ txn=txn)
self._picklelog.put(key, '', txn=txn)
except:
txn.abort()
@@ -731,29 +752,6 @@
return ResolvedSerial
return self._serial
- def _log_object(self, oid, vid, nvrevid, data, oserial):
- # Save data for later commit. We do this by writing the pickle
- # directly to BDB and saving the pickle key in the pickle log.
- # We extract the references and save them in the transaction log.
- #
- # Get the references
- refdoids = []
- referencesf(data, refdoids)
- # Record the update to this object in the commit log.
- self._commitlog.write_object(oid, vid, nvrevid, refdoids, oserial)
- # Save the pickle in the database
- txn = self._env.txn_begin()
- try:
- key = oid + self._serial
- self._pickles.put(key, data, txn=txn)
- self._picklelog.put(key, '', txn=txn)
- except:
- txn.abort()
- raise
- else:
- txn.commit()
-
-
def transactionalUndo(self, tid, transaction):
if transaction is not self._transaction:
raise POSException.StorageTransactionError(self, transaction)
@@ -1352,7 +1350,7 @@
def gcRefcount(oid):
"""Return the reference count of the specified object.
-
+
Raises KeyError if there is no object with oid. Both the oid argument
and the returned reference count are integers.
"""
@@ -1492,7 +1490,7 @@
class _RecordsIterator:
"""Provide transaction meta-data and forward iteration through the
transactions in a storage.
-
+
Items *must* be accessed sequentially (e.g. with a for loop).
"""