[Zope-Checkins] CVS: StandaloneZODB/ZODB - BaseStorage.py:1.17 Connection.py:1.62 DemoStorage.py:1.9 FileStorage.py:1.77 MappingStorage.py:1.5 POSException.py:1.9

Jeremy Hylton jeremy@zope.com
Thu, 17 Jan 2002 12:34:33 -0500


Update of /cvs-repository/StandaloneZODB/ZODB
In directory cvs.zope.org:/tmp/cvs-serv20058

Modified Files:
	BaseStorage.py Connection.py DemoStorage.py FileStorage.py 
	MappingStorage.py POSException.py 
Log Message:
Merge Standby-branch to trunk (mostly).

The Standby-branch was branched from the StandaloneZODB-1_0-branch,
which includes the BTrees-fsIndex code.  I didn't include that change
in the merge, but everything else.  Terse summary follows:

BaseStorage.py:
    Add read-only storage feature.
    Add TransactionRecord and DataRecord marker classes for iteration.
    Reformat some lines.

FileStorage.py:
    Add read-only storage feature.
    Greg Ward's ConflictError patch
    Reformat some lines.
    Add lastTransaction(), lastSerialno().
    Add bounds support to iterator().
    Use TransactionRecord and DataRecord.

Connection.py:
DemoStorage.py:
MappingStorage.py:
    Greg Ward's ConflictError patch

POSException.py:
    Greg Ward's ConflictError patch
    Add ReadOnlyError.



=== StandaloneZODB/ZODB/BaseStorage.py 1.16 => 1.17 ===
     _serial=z64       # Transaction serial number
     _tstatus=' '      # Transaction status, used for copying data
+    _is_read_only = 0
 
     def __init__(self, name, base=None):
         
@@ -42,8 +43,10 @@
         t=time.time()
         t=self._ts=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
         self._serial=`t`
-        if base is None: self._oid='\0\0\0\0\0\0\0\0'
-        else:            self._oid=base._oid
+        if base is None:
+            self._oid='\0\0\0\0\0\0\0\0'
+        else:
+            self._oid=base._oid
 
     def abortVersion(self, src, transaction):
         if transaction is not self._transaction:
@@ -55,15 +58,24 @@
             raise POSException.StorageTransactionError(self, transaction)
         return []
 
-    def close(self): pass
+    def close(self):
+        pass
 
-    def getName(self): return self.__name__
-    def getSize(self): return len(self)*300 # WAG!
-    def history(self, oid, version, length=1): pass
+    def getName(self):
+        return self.__name__
+    
+    def getSize(self):
+        return len(self)*300 # WAG!
+    
+    def history(self, oid, version, length=1):
+        pass
                     
-    def modifiedInVersion(self, oid): return ''
+    def modifiedInVersion(self, oid):
+        return ''
 
     def new_oid(self, last=None):
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
         if last is None:
             self._lock_acquire()
             try:
@@ -79,10 +91,17 @@
             if d < 255: return last[:-1]+chr(d+1)+'\0'*(8-len(last))
             else:       return self.new_oid(last[:-1])
 
-    def registerDB(self, db, limit): pass # we don't care
+    def registerDB(self, db, limit):
+        pass # we don't care
 
-    def supportsUndo(self): return 0
-    def supportsVersions(self): return 0
+    def isReadOnly(self):
+        return self._is_read_only
+    
+    def supportsUndo(self):
+        return 0
+    
+    def supportsVersions(self):
+        return 0
         
     def tpc_abort(self, transaction):
         self._lock_acquire()
@@ -171,15 +190,22 @@
         pass
 
     def undo(self, transaction_id):
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
         raise POSException.UndoError, 'non-undoable transaction'
 
-    def undoLog(self, first, last, filter=None): return ()
+    def undoLog(self, first, last, filter=None):
+        return ()
 
-    def versionEmpty(self, version): return 1
+    def versionEmpty(self, version):
+        return 1
 
-    def versions(self, max=None): return ()
+    def versions(self, max=None):
+        return ()
 
-    def pack(self, t, referencesf): pass
+    def pack(self, t, referencesf):
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
 
     def getSerial(self, oid):
         self._lock_acquire()
@@ -232,3 +258,9 @@
                 
             self.tpc_vote(transaction)
             self.tpc_finish(transaction)
+
+class TransactionRecord:
+    """Abstract base class for iterator protocol"""
+
+class DataRecord:
+    """Abstract base class for iterator protocol"""


=== StandaloneZODB/ZODB/Connection.py 1.61 => 1.62 ===
 
 from cPickleCache import PickleCache
-from POSException import ConflictError
+from POSException import ConflictError, ReadConflictError
 from ExtensionClass import Base
 import ExportImport, TmpStore
 from zLOG import LOG, ERROR, BLATHER
@@ -248,7 +248,7 @@
                 or
                 invalid(None)
                 ):
-                raise ConflictError, `oid`
+                raise ConflictError(object=object)
             self._invalidating.append(oid)
 
         else:
@@ -315,7 +315,7 @@
                     or
                     invalid(None)
                     ):
-                    raise ConflictError, `oid`
+                    raise ConflictError(object=object)
                 self._invalidating.append(oid)
                 
             klass = object.__class__
@@ -459,7 +459,7 @@
             if invalid(oid) or invalid(None):
                 if not hasattr(object.__class__, '_p_independent'):
                     get_transaction().register(self)
-                    raise ConflictError(`oid`, `object.__class__`)
+                    raise ReadConflictError(object=object)
                 invalid=1
             else:
                 invalid=0
@@ -484,7 +484,7 @@
                     except KeyError: pass
                 else:
                     get_transaction().register(self)
-                    raise ConflictError(`oid`, `object.__class__`)
+                    raise ConflictError(object=object)
 
         except ConflictError:
             raise
@@ -544,7 +544,7 @@
 
     def tpc_begin(self, transaction, sub=None):
         if self._invalid(None): # Some nitwit invalidated everything!
-            raise ConflictError, "transaction already invalidated"
+            raise ConflictError("transaction already invalidated")
         self._invalidating=[]
         self._creating=[]
 


=== StandaloneZODB/ZODB/DemoStorage.py 1.8 => 1.9 ===
                     nv=old
 
-                if serial != oserial: raise POSException.ConflictError
+                if serial != oserial:
+                    raise POSException.ConflictError(serials=(oserial, serial))
                 
             serial=self._serial
             r=[oid, serial, old, version and (version, nv) or None, data]


=== StandaloneZODB/ZODB/FileStorage.py 1.76 => 1.77 ===
 __version__='$Revision$'[11:-2]
 
-import struct, time, os, bpthread, string, base64, sys
+import struct, time, os, string, base64, sys
 from struct import pack, unpack
-from cPickle import loads
 import POSException
 from POSException import UndoError
 from TimeStamp import TimeStamp
 from lock_file import lock_file
 from utils import t32, p64, U64, cp
-from zLOG import LOG, WARNING, ERROR, PANIC, register_subsystem
+from zLOG import LOG, BLATHER, WARNING, ERROR, PANIC, register_subsystem
 register_subsystem('ZODB FS')
 import BaseStorage
-from cPickle import Pickler, Unpickler
+from cPickle import Pickler, Unpickler, loads
 import ConflictResolution
 
 try: from posix import fsync
@@ -185,6 +184,7 @@
             create = 1
 
         if read_only:
+            self._is_read_only = 1
             if create:
                 raise ValueError, "can\'t create a read-only file"
         elif stop is not None:
@@ -242,6 +242,7 @@
                 self._file, file_name, index, vindex, tindex, stop,
                 read_only=read_only,
                 )
+        self._ltid = tid
 
         self._ts = tid = TimeStamp(tid)
         t = time.time()
@@ -262,7 +263,8 @@
         self._index_get=index.get
         self._vindex_get=vindex.get
 
-    def __len__(self): return len(self._index)
+    def __len__(self):
+        return len(self._index)
 
     def _newIndexes(self):
         # hook to use something other than builtin dict
@@ -389,13 +391,20 @@
 
     def close(self):
         self._file.close()
-        if hasattr(self,'_lock_file'):  self._lock_file.close()
-        if self._tfile:                 self._tfile.close()
-        try: self._save_index()
-        except: pass # We don't care if this fails.
+        if hasattr(self,'_lock_file'):
+            self._lock_file.close()
+        if self._tfile:
+            self._tfile.close()
+        try:
+            self._save_index()
+        except:
+            # XXX should log the error, though
+            pass # We don't care if this fails.
         
     def commitVersion(self, src, dest, transaction, abort=None):
         # We are going to commit by simply storing back pointers.
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
         if not (src and isinstance(src, StringType)
                 and isinstance(dest, StringType)):
             raise POSException.VersionCommitError('Invalid source version')
@@ -413,9 +422,8 @@
         
         self._lock_acquire()
         try:
-            file=self._file
-            read=file.read
-            seek=file.seek
+            read=self._file.read
+            seek=self._file.seek
             tfile=self._tfile
             write=tfile.write
             tindex=self._tindex
@@ -607,6 +615,8 @@
         finally: self._lock_release()
 
     def store(self, oid, serial, data, version, transaction):
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
         if transaction is not self._transaction:
             raise POSException.StorageTransactionError(self, transaction)
 
@@ -632,8 +642,8 @@
                 if serial != oserial:
                     data=self.tryToResolveConflict(oid, oserial, serial, data)
                     if not data:
-                        raise POSException.ConflictError, (
-                            serial, oserial)
+                        raise POSException.ConflictError(oid=oid,
+                                                serials=(oserial, serial))
             else:
                 oserial=serial
                     
@@ -672,13 +682,17 @@
         finally:
             self._lock_release()
 
-    def supportsUndo(self): return 1
-    def supportsVersions(self): return 1
+    def supportsUndo(self):
+        return 1
+    
+    def supportsVersions(self):
+        return 1
 
     def _clear_temp(self):
         self._tindex.clear()
         self._tvindex.clear()
-        self._tfile.seek(0)
+        if self._tfile is not None:
+            self._tfile.seek(0)
 
     def _begin(self, tid, u, d, e):
         self._thl=23+len(u)+len(d)+len(e)
@@ -702,9 +716,9 @@
 
             # We have to check lengths here because struct.pack
             # doesn't raise an exception on overflow!
-            if luser > 65535: raise FileStorageError, 'user name too long'
-            if ldesc > 65535: raise FileStorageError, 'description too long'
-            if lext  > 65535: raise FileStorageError, 'too much extension data'
+            if luser > 65535: raise FileStorageError('user name too long')
+            if ldesc > 65535: raise FileStorageError('description too long')
+            if lext > 65535: raise FileStorageError('too much extension data')
 
             tlen=self._thl
             pos=self._pos
@@ -718,7 +732,7 @@
                 # suspect.
                 write(pack(
                     ">8s" "8s" "c"  "H"        "H"        "H"
-                     ,tid, stl, 'c', luser,     ldesc,     lext,
+                     ,tid, stl,'c',  luser,     ldesc,     lext,
                     ))
                 if user: write(user)
                 if desc: write(desc)
@@ -754,6 +768,7 @@
 
             self._index.update(self._tindex)
             self._vindex.update(self._tvindex)
+        self._ltid = tid
 
     def _abort(self):
         if self._nextpos:
@@ -761,6 +776,8 @@
             self._nextpos=0
 
     def undo(self, transaction_id):
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
         self._lock_acquire()
         try:
             self._clear_index()
@@ -808,7 +825,8 @@
             return t.keys()            
         finally: self._lock_release()
 
-    def supportsTransactionalUndo(self): return 1
+    def supportsTransactionalUndo(self):
+        return 1
 
     def _undoDataInfo(self, oid, pos, tpos):
         """Return the serial, data pointer, data, and version for the oid
@@ -930,7 +948,9 @@
         # writing a file position rather than a pickle. Sometimes, we
         # may do conflict resolution, in which case we actually copy
         # new data that results from resolution.
-        
+
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
         if transaction is not self._transaction:
             raise POSException.StorageTransactionError(self, transaction)
         
@@ -1175,6 +1195,8 @@
         the associated data are copied, since the old records are not copied.
         """
 
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
         # Ugh, this seems long
         
         packing=1 # are we in the packing phase (or the copy phase)
@@ -1551,8 +1573,32 @@
             self._packt=z64
             _lock_release()
 
-    def iterator(self):
-        return FileIterator(self._file_name)
+    def iterator(self, start=None, stop=None):
+        return FileIterator(self._file_name, start, stop)
+
+    def lastTransaction(self):
+        """Return transaction id for last committed transaction"""
+        return self._ltid
+
+    def lastSerial(self, oid):
+        """Return last serialno committed for object oid.
+
+        If there is no serialno for this oid -- which can only occur
+        if it is a new object -- return None.
+        """
+        try:
+            pos = self._index[oid]
+        except KeyError:
+            return None
+        self._file.seek(pos)
+        # first 8 bytes are oid, second 8 bytes are serialno
+        h = self._file.read(16)
+        if len(h) < 16:
+            raise CorruptedDataError, h
+        if h[:8] != oid:
+            h = h + self._file.read(26) # get rest of header
+            raise CorruptedDataError, h
+        return h[8:]
 
 def shift_transactions_forward(index, vindex, tindex, file, pos, opos):
     """Copy transactions forward in the data file
@@ -1969,14 +2015,46 @@
     """
     _ltid=z64
     
-    def __init__(self, file):
+    def __init__(self, file, start=None, stop=None):
         if isinstance(file, StringType):
-            file=open(file, 'rb')
-        self._file=file
-        if file.read(4) != packed_version: raise FileStorageFormatError, name
+            file = open(file, 'rb')
+        self._file = file
+        if file.read(4) != packed_version:
+            raise FileStorageFormatError, name
         file.seek(0,2)
-        self._file_size=file.tell()
-        self._pos=4L
+        self._file_size = file.tell()
+        self._pos = 4L
+        assert start is None or isinstance(start, StringType)
+        assert stop is None or isinstance(stop, StringType)
+        if start:
+            self._skip_to_start(start)
+        self._stop = stop
+
+    def _skip_to_start(self, start):
+        # Scan through the transaction records doing almost no sanity
+        # checks. 
+        while 1:
+            self._file.seek(self._pos)
+            h = self._file.read(16)
+            if len(h) < 16:
+                return
+            tid, stl = unpack(">8s8s", h)
+            if tid >= start:
+                return
+            tl = U64(stl)
+            try:
+                self._pos += tl + 8
+            except OverflowError:
+                self._pos = long(self._pos) + tl + 8
+            if __debug__:
+                # Sanity check
+                self._file.seek(self._pos - 8, 0)
+                rtl = self._file.read(8)
+                if rtl != stl:
+                    pos = self._file.tell() - 8
+                    panic("%s has inconsistent transaction length at %s "
+                          "(%s != %s)",
+                          self._file.name, pos, U64(rtl), U64(stl))
 
     def next(self, index=0):
         file=self._file
@@ -1984,6 +2062,7 @@
         read=file.read
         pos=self._pos
 
+        LOG("ZODB FS", BLATHER, "next(%d)" % index)
         while 1:
             # Read the transaction record
             seek(pos)
@@ -1994,7 +2073,7 @@
             if el < 0: el=t32-el
 
             if tid <= self._ltid:
-                warn("%s time-stamp reduction at %s", name, pos)
+                warn("%s time-stamp reduction at %s", self._file.name, pos)
             self._ltid=tid
 
             tl=U64(stl)
@@ -2004,11 +2083,12 @@
                 # cleared.  They may also be corrupted,
                 # in which case, we don't want to totally lose the data.
                 warn("%s truncated, possibly due to damaged records at %s",
-                     name, pos)
+                     self._file.name, pos)
                 break
 
             if status not in ' up':
-                warn('%s has invalid status, %s, at %s', name, status, pos)
+                warn('%s has invalid status, %s, at %s', self._file.name,
+                     status, pos)
 
             if tl < (23+ul+dl+el):
                 # We're in trouble. Find out if this is bad data in
@@ -2022,16 +2102,24 @@
                 # reasonable:
                 if self._file_size - rtl < pos or rtl < 23:
                     nearPanic('%s has invalid transaction header at %s',
-                              name, pos)
+                              self._file.name, pos)
                     warn("It appears that there is invalid data at the end of "
                          "the file, possibly due to a system crash.  %s "
                          "truncated to recover from bad data at end."
-                         % name)
+                         % self._file.name)
                     break
                 else:
-                    warn('%s has invalid transaction header at %s', name, pos)
+                    warn('%s has invalid transaction header at %s',
+                         self._file.name, pos)
                     break
 
+            if self._stop is not None:
+                LOG("ZODB FS", BLATHER,
+                    ("tid %x > stop %x ? %d" %
+                     (U64(tid), U64(self._stop), tid > self._stop)))
+            if self._stop is not None and tid > self._stop:
+                raise IndexError, index
+
             tpos=pos
             tend=tpos+tl
 
@@ -2041,7 +2129,7 @@
                 h=read(8)
                 if h != stl:
                     panic('%s has inconsistent transaction length at %s',
-                          name, pos)
+                          self._file.name, pos)
                 pos=tend+8
                 continue
 
@@ -2067,7 +2155,7 @@
             h=read(8)
             if h != stl:
                 warn("%s redundant transaction length check failed at %s",
-                     name, pos)
+                     self._file.name, pos)
                 break
             self._pos=pos+8
 
@@ -2075,7 +2163,7 @@
 
         raise IndexError, index
     
-class RecordIterator(Iterator):
+class RecordIterator(Iterator, BaseStorage.TransactionRecord):
     """Iterate over the transactions in a FileStorage file.
     """
     def __init__(self, tid, status, user, desc, ext, pos, stuff):
@@ -2127,9 +2215,8 @@
             return r
         
         raise IndexError, index
-    
 
-class Record:
+class Record(BaseStorage.DataRecord):
     """An abstract database record
     """
     def __init__(self, *args):


=== StandaloneZODB/ZODB/MappingStorage.py 1.4 => 1.5 ===
                 old=self._index[oid]
                 oserial=old[:8]
-                if serial != oserial: raise POSException.ConflictError
+                if serial != oserial:
+                    raise POSException.ConflictError(serials=(oserial, serial))
                 
             serial=self._serial
             self._tindex.append((oid,serial+data))


=== StandaloneZODB/ZODB/POSException.py 1.8 => 1.9 ===
 # 
 ##############################################################################
-'''BoboPOS-defined exceptions
+"""BoboPOS-defined exceptions
 
-$Id$'''
-__version__='$Revision$'[11:-2]
+$Id$"""
+__version__ = '$Revision$'.split()[-2:][0]
 
 from string import join
-StringType=type('')
-DictType=type({})
+from types import StringType, DictType
+from ZODB import utils
 
 class POSError(Exception):
     """Persistent object system error
@@ -28,10 +28,94 @@
     """
 
 class ConflictError(TransactionError):
-    """Two transactions tried to modify the same object at once
+    """Two transactions tried to modify the same object at once.  This
+    transaction should be resubmitted.
 
-    This transaction should be resubmitted.
-    """
+    Instance attributes:
+      oid : string
+        the OID (8-byte packed string) of the object in conflict
+      class_name : string
+        the fully-qualified name of that object's class
+      message : string
+        a human-readable explanation of the error
+      serials : (string, string)
+        a pair of 8-byte packed strings; these are the serial numbers
+        (old and new) of the object in conflict.  (Serial numbers are
+        closely related [equal?] to transaction IDs; a ConflictError may
+        be triggered by a serial number mismatch.)
+
+    The caller should pass either object or oid as a keyword argument,
+    but not both of them.  If object is passed, it should be a
+    persistent object with an _p_oid attribute.
+    """
+
+    def __init__(self, message=None, object=None, oid=None, serials=None):
+        if message is None:
+            self.message = "database conflict error"
+        else:
+            self.message = message
+
+        if object is None:
+            self.oid = None
+            self.class_name = None
+        else:
+            self.oid = object._p_oid
+            klass = object.__class__
+            self.class_name = klass.__module__ + "." + klass.__name__
+
+        if oid is not None:
+            assert self.oid is None
+            self.oid = oid
+
+        self.serials = serials
+
+    def __str__(self):
+        extras = []
+        if self.oid:
+            extras.append("oid %016x" % utils.U64(self.oid))
+        if self.class_name:
+            extras.append("class %s" % self.class_name)
+        if self.serials:
+            extras.append("serial was %016x, now %016x" %
+                          tuple(map(utils.U64, self.serials)))
+        if extras:
+            return "%s (%s)" % (self.message, ", ".join(extras))
+        else:
+            return self.message
+
+    def get_oid(self):
+        return self.oid
+
+    def get_class_name(self):
+        return self.class_name
+
+    def get_old_serial(self):
+        return self.serials[0]
+
+    def get_new_serial(self):
+        return self.serials[1]
+
+    def get_serials(self):
+        return self.serials
+
+
+class ReadConflictError(ConflictError):
+    """A conflict detected at read time -- attempt to read an object
+    that has changed in another transaction (eg. another thread
+    or process).
+    """
+    def __init__(self, message=None, object=None, serials=None):
+        if message is None:
+            message = "database read conflict error"
+        ConflictError.__init__(self, message=message, object=object,
+                               serials=serials)
+
+class BTreesConflictError(ConflictError):
+    """A special subclass for BTrees conflict errors, which return
+    an undocumented four-tuple."""
+    def __init__(self, *btree_args):
+        ConflictError.__init__(self, message="BTrees conflict error")
+        self.btree = btree_args
 
 class VersionError(POSError):
     """An error in handling versions occurred
@@ -82,6 +166,10 @@
 
 class MountedStorageError(StorageError):
     """Unable to access mounted storage.
+    """
+
+class ReadOnlyError(StorageError):
+    """Unable to modify objects in a read-only storage.
     """
 
 class ExportError(POSError):