[Zope-Checkins] CVS: StandaloneZODB/ZODB - BaseStorage.py:1.18 FileStorage.py:1.79 POSException.py:1.10 fsdump.py:1.2
Guido van Rossum
guido@python.org
Thu, 24 Jan 2002 21:15:08 -0500
Update of /cvs-repository/StandaloneZODB/ZODB
In directory cvs.zope.org:/tmp/cvs-serv21356
Modified Files:
BaseStorage.py FileStorage.py POSException.py fsdump.py
Log Message:
Commit changes from the short-lived "Recovery" branch to the trunk.
Highlights:
BaseStorage.py:
Fix copyTransactionsFrom() when commitVersion(), abortVersion() or
transactionalUndo() is used.
FileStorage.py:
Add restore() method, which can store data records corresponding to
undo or version manipulations; add close() method to FileIterator
class; raise POSKeyError instead of KeyError for bad keys, to ensure
safe formatting of transaction ids (binary strings) in tracebacks.
POSException.py:
Add POSKeyError.
fsdump.py:
Deal with records indicating the undo or abort of a version doing
object creation.
tests/IteratorStorage.py:
New unittests for the iterator() method and interface of the
storage API; new unit tests for extended file iterators; new class,
IteratorDeepCompare; test of the iterator .close() method.
tests/testFileStorage.py:
Add class FileStorageRecoveryTest, which adds two simple tests for
copyTransactionsFrom(). This indirectly tests the new restore()
method.
=== StandaloneZODB/ZODB/BaseStorage.py 1.17 => 1.18 ===
_ts=None
ok=1
- preindex={}; preget=preindex.get # waaaa
+ preindex={};
+ preget=preindex.get # waaaa
+ # restore() is a new storage API method which has an identical
+ # signature to store() except that it does not return anything.
+ # Semantically, restore() is also identical to store() except that it
+ # doesn't do the ConflictError or VersionLockError consistency
+ # checks. The reason to use restore() over store() in this method is
+ # that store() cannot be used to copy transactions spanning a version
+ # commit or abort, or over transactional undos.
+ #
+ # We'll use restore() if it's available, otherwise we'll fall back to
+ # using store(). However, if we use store, then
+ # copyTransactionsFrom() may fail with VersionLockError or
+ # ConflictError.
+ if hasattr(self, 'restore'):
+ restoring = 1
+ else:
+ restoring = 0
for transaction in other.iterator():
tid=transaction.tid
@@ -252,9 +269,12 @@
for r in transaction:
oid=r.oid
if verbose: print `oid`, r.version, len(r.data)
- pre=preget(oid, None)
- s=self.store(oid, pre, r.data, r.version, transaction)
- preindex[oid]=s
+ if restoring:
+ self.restore(oid, r.serial, r.data, r.version, transaction)
+ else:
+ pre=preget(oid, None)
+ s=self.store(oid, pre, r.data, r.version, transaction)
+ preindex[oid]=s
self.tpc_vote(transaction)
self.tpc_finish(transaction)
=== StandaloneZODB/ZODB/FileStorage.py 1.78 => 1.79 ===
from struct import pack, unpack
import POSException
-from POSException import UndoError
+from POSException import UndoError, POSKeyError
from TimeStamp import TimeStamp
from lock_file import lock_file
from utils import t32, p64, U64, cp
@@ -574,7 +574,10 @@
file=self._file
seek=file.seek
read=file.read
- pos=_index[oid]
+ try:
+ pos=_index[oid]
+ except KeyError:
+ raise POSKeyError(oid)
while 1:
seek(pos)
h=read(42)
@@ -583,7 +586,8 @@
if dserial == serial: break # Yeee ha!
# Keep looking for serial
pos=U64(prev)
- if not pos: raise KeyError, serial
+ if not pos:
+ raise POSKeyError(serial)
continue
if vlen:
@@ -682,6 +686,80 @@
finally:
self._lock_release()
+ def restore(self, oid, serial, data, version, transaction):
+ # A lot like store() but without all the consistency checks. This
+ # should only be used when we /know/ the data is good, hence the
+ # method name. While the signature looks like store() there are some
+ # differences:
+ #
+ # - serial is the serial number of /this/ revision, not of the
+ # previous revision. It is used instead of self._serial, which is
+ # ignored.
+ #
+ # - Nothing is returned
+ #
+ # - data can be None, which indicates a George Bailey object (i.e. one
+ # who's creation has been transactionally undone).
+ if self._is_read_only:
+ raise POSException.ReadOnlyError()
+ if transaction is not self._transaction:
+ raise POSException.StorageTransactionError(self, transaction)
+
+ self._lock_acquire()
+ try:
+ # Position of the non-version data
+ pnv = None
+ # We need to get some information about previous revisions of the
+ # object. Specifically, we need the position of the non-version
+ # data if this update is in a version. We also need the position
+ # of the previous record in this version.
+ old = self._index_get(oid, 0)
+ if old:
+ self._file.seek(old)
+ # Read the previous revision record
+ h = self._file.read(42)
+ doid,oserial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+ if doid != oid:
+ raise CorruptedDataError, h
+ # Calculate the file position in the temporary file
+ here = self._pos + self._tfile.tell() + self._thl
+ # And update the temp file index
+ self._tindex[oid] = here
+ # Write the recovery data record
+ if data is None:
+ dlen = 0
+ else:
+ dlen = len(data)
+ self._tfile.write(pack('>8s8s8s8sH8s',
+ oid, serial, p64(old), p64(self._pos),
+ len(version), p64(dlen)))
+ # We need to write some version information if this revision is
+ # happening in a version.
+ if version:
+ # If there's a previous revision in this version, write the
+ # position, otherwise write the position of the previous
+ # non-version revision.
+ if pnv:
+ self._tfile.write(pnv)
+ else:
+ self._tfile.write(p64(old))
+ # Link to the last record for this version
+ pv = self._tvindex.get(version, 0)
+ if not pv:
+ self._vindex_get(version, 0)
+ self._tfile.write(p64(pv))
+ self._tvindex[version] = here
+ self._tfile.write(version)
+ # And finally, write the data
+ if data is None:
+ # Write a zero backpointer, which is indication used to
+ # represent an un-creation transaction.
+ self._tfile.write(z64)
+ else:
+ self._tfile.write(data)
+ finally:
+ self._lock_release()
+
def supportsUndo(self):
return 1
@@ -942,7 +1020,7 @@
def transactionalUndo(self, transaction_id, transaction):
"""Undo a transaction, given by transaction_id.
- Do so by writing new data that reverses tyhe action taken by
+ Do so by writing new data that reverses the action taken by
the transaction."""
# Usually, we can get by with just copying a data pointer, by
# writing a file position rather than a pickle. Sometimes, we
@@ -1944,7 +2022,8 @@
while 1:
old=U64(back)
- if not old: raise KeyError, oid
+ if not old:
+ raise POSKeyError(oid)
seek(old)
h=read(42)
doid,serial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h)
@@ -1961,7 +2040,8 @@
while 1:
old=U64(back)
- if not old: raise KeyError, oid
+ if not old:
+ raise POSKeyError(oid)
seek(old)
h=read(42)
doid,serial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h)
@@ -2014,6 +2094,8 @@
"""Iterate over the transactions in a FileStorage file.
"""
_ltid=z64
+
+ _file = None
def __init__(self, file, start=None, stop=None):
if isinstance(file, StringType):
@@ -2030,6 +2112,12 @@
self._skip_to_start(start)
self._stop = stop
+ def close(self):
+ file = self._file
+ if file is not None:
+ self._file = None
+ file.close()
+
def _skip_to_start(self, start):
# Scan through the transaction records doing almost no sanity
# checks.
@@ -2057,6 +2145,10 @@
self._file.name, pos, U64(rtl), U64(stl))
def next(self, index=0):
+ if self._file is None:
+ # A closed iterator. XXX: Is IOError the best we can do? For
+ # now, mimic a read on a closed file.
+ raise IOError, 'iterator is closed'
file=self._file
seek=file.seek
read=file.read
=== StandaloneZODB/ZODB/POSException.py 1.9 => 1.10 ===
"""
+class POSKeyError(KeyError, POSError):
+ """Key not found in database
+ """
+
+ def __str__(self):
+ return "%016x" % utils.U64(self.args[0])
+
class TransactionError(POSError):
"""An error occured due to normal transaction processing
"""
=== StandaloneZODB/ZODB/fsdump.py 1.1 => 1.2 ===
j = 0
for rec in trans:
- modname, classname = get_pickle_metadata(rec.data)
- dig = md5.new(rec.data).hexdigest()
- fullclass = "%s.%s" % (modname, classname)
+ if rec.data is None:
+ fullclass = "undo or abort of object creation"
+ else:
+ modname, classname = get_pickle_metadata(rec.data)
+ dig = md5.new(rec.data).hexdigest()
+ fullclass = "%s.%s" % (modname, classname)
# special case for testing purposes
if fullclass == "ZODB.tests.MinPO.MinPO":
obj = zodb_unpickle(rec.data)