[Zope-Checkins] CVS: StandaloneZODB/ZODB - BaseStorage.py:1.16.8.2 DB.py:1.36.6.2 FileStorage.py:1.76.8.2 cPersistence.c:1.47.8.2 cPickleCache.c:1.37.8.2

Jeremy Hylton jeremy@zope.com
Tue, 18 Dec 2001 12:14:15 -0500


Update of /cvs-repository/StandaloneZODB/ZODB
In directory cvs.zope.org:/tmp/cvs-serv31892

Modified Files:
      Tag: Standby-branch
	BaseStorage.py DB.py FileStorage.py cPersistence.c 
	cPickleCache.c 
Log Message:
Merge in jeremy-Standby-branch


=== StandaloneZODB/ZODB/BaseStorage.py 1.16.8.1 => 1.16.8.2 ===
         t=self._ts=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
         self._serial=`t`
-        if base is None: self._oid='\0\0\0\0\0\0\0\0'
-        else:            self._oid=base._oid
+        if base is None:
+            self._oid='\0\0\0\0\0\0\0\0'
+        else:
+            self._oid=base._oid
 
     def abortVersion(self, src, transaction):
         if transaction is not self._transaction:
@@ -232,3 +234,9 @@
                 
             self.tpc_vote(transaction)
             self.tpc_finish(transaction)
+
+class TransactionRecord:
+    """Abstract base class for iterator protocol"""
+
+class DataRecord:
+    """Abstract base class for iterator protocol"""


=== StandaloneZODB/ZODB/DB.py 1.36.6.1 => 1.36.6.2 ===


=== StandaloneZODB/ZODB/FileStorage.py 1.76.8.1 => 1.76.8.2 ===
                 read_only=read_only,
                 )
+        self._ltid = tid
 
         self._ts = tid = TimeStamp(tid)
         t = time.time()
@@ -395,10 +396,15 @@
 
     def close(self):
         self._file.close()
-        if hasattr(self,'_lock_file'):  self._lock_file.close()
-        if self._tfile:                 self._tfile.close()
-        try: self._save_index()
-        except: pass # We don't care if this fails.
+        if hasattr(self,'_lock_file'):
+            self._lock_file.close()
+        if self._tfile:
+            self._tfile.close()
+        try:
+            self._save_index()
+        except:
+            # XXX should log the error, though
+            pass # We don't care if this fails.
         
     def commitVersion(self, src, dest, transaction, abort=None):
         # We are going to commit by simply storing back pointers.
@@ -419,9 +425,8 @@
         
         self._lock_acquire()
         try:
-            file=self._file
-            read=file.read
-            seek=file.seek
+            read=self._file.read
+            seek=self._file.seek
             tfile=self._tfile
             write=tfile.write
             tindex=self._tindex
@@ -708,9 +713,9 @@
 
             # We have to check lengths here because struct.pack
             # doesn't raise an exception on overflow!
-            if luser > 65535: raise FileStorageError, 'user name too long'
-            if ldesc > 65535: raise FileStorageError, 'description too long'
-            if lext  > 65535: raise FileStorageError, 'too much extension data'
+            if luser > 65535: raise FileStorageError('user name too long')
+            if ldesc > 65535: raise FileStorageError('description too long')
+            if lext > 65535: raise FileStorageError('too much extension data')
 
             tlen=self._thl
             pos=self._pos
@@ -724,7 +729,7 @@
                 # suspect.
                 write(pack(
                     ">8s" "8s" "c"  "H"        "H"        "H"
-                     ,tid, stl, 'c', luser,     ldesc,     lext,
+                     ,tid, stl,'c',  luser,     ldesc,     lext,
                     ))
                 if user: write(user)
                 if desc: write(desc)
@@ -760,6 +765,7 @@
 
             self._index.update(self._tindex)
             self._vindex.update(self._tvindex)
+        self._ltid = tid
 
     def _abort(self):
         if self._nextpos:
@@ -1557,8 +1563,32 @@
             self._packt=z64
             _lock_release()
 
-    def iterator(self):
-        return FileIterator(self._file_name)
+    def iterator(self, start=None, stop=None):
+        return FileIterator(self._file_name, start, stop)
+
+    def lastTransaction(self):
+        """Return transaction id for last committed transaction"""
+        return self._ltid
+
+    def lastSerial(self, oid):
+        """Return last serialno committed for object oid.
+
+        If there is no serialno for this oid -- which can only occur
+        if it is a new object -- return None.
+        """
+        try:
+            pos = self._index[oid]
+        except KeyError:
+            return None
+        self._file.seek(pos)
+        # first 8 bytes are oid, second 8 bytes are serialno
+        h = self._file.read(16)
+        if len(h) < 16:
+            raise CorruptedDataError, h
+        if h[:8] != oid:
+            h = h + self._file.read(26) # get rest of header
+            raise CorruptedDataError, h
+        return h[8:]
 
 def shift_transactions_forward(index, vindex, tindex, file, pos, opos):
     """Copy transactions forward in the data file
@@ -1975,14 +2005,41 @@
     """
     _ltid=z64
     
-    def __init__(self, file):
+    def __init__(self, file, start=None, stop=None):
         if isinstance(file, StringType):
-            file=open(file, 'rb')
-        self._file=file
-        if file.read(4) != packed_version: raise FileStorageFormatError, name
+            file = open(file, 'rb')
+        self._file = file
+        if file.read(4) != packed_version:
+            raise FileStorageFormatError, name
         file.seek(0,2)
-        self._file_size=file.tell()
-        self._pos=4L
+        self._file_size = file.tell()
+        self._pos = 4L
+        assert start is None or isinstance(start, StringType)
+        assert stop is None or isinstance(stop, StringType)
+        if start:
+            self._skip_to_start(start)
+        self._stop = stop
+
+    def _skip_to_start(self, start):
+        # Scan through the transaction records doing almost no sanity
+        # checks. 
+        self._file.seek(self._pos)
+        while 1:
+            self._pos = self._file.tell()
+            h = self._file.read(16)
+            if len(h) < 16:
+                return
+            tid, stl = unpack(">8s8s", h)
+            if tid >= start:
+                return
+            tl = U64(stl)
+            self._file.seek(tl - 8, 1)
+            rtl = self._file.read(8)
+            if rtl != stl:
+                pos = self._file.tell() - 8
+                panic("%s has inconsistent transaction length at %s "
+                      "(%s != %s)",
+                      self._file.name, pos, rtl, stl)
 
     def next(self, index=0):
         file=self._file
@@ -1990,6 +2047,7 @@
         read=file.read
         pos=self._pos
 
+        LOG("ZODB FS", -100, "next(%d)" % index)
         while 1:
             # Read the transaction record
             seek(pos)
@@ -2000,7 +2058,7 @@
             if el < 0: el=t32-el
 
             if tid <= self._ltid:
-                warn("%s time-stamp reduction at %s", name, pos)
+                warn("%s time-stamp reduction at %s", self._file.name, pos)
             self._ltid=tid
 
             tl=U64(stl)
@@ -2010,11 +2068,12 @@
                 # cleared.  They may also be corrupted,
                 # in which case, we don't want to totally lose the data.
                 warn("%s truncated, possibly due to damaged records at %s",
-                     name, pos)
+                     self._file.name, pos)
                 break
 
             if status not in ' up':
-                warn('%s has invalid status, %s, at %s', name, status, pos)
+                warn('%s has invalid status, %s, at %s', self._file.name,
+                     status, pos)
 
             if tl < (23+ul+dl+el):
                 # We're in trouble. Find out if this is bad data in
@@ -2028,16 +2087,24 @@
                 # reasonable:
                 if self._file_size - rtl < pos or rtl < 23:
                     nearPanic('%s has invalid transaction header at %s',
-                              name, pos)
+                              self._file.name, pos)
                     warn("It appears that there is invalid data at the end of "
                          "the file, possibly due to a system crash.  %s "
                          "truncated to recover from bad data at end."
-                         % name)
+                         % self._file.name)
                     break
                 else:
-                    warn('%s has invalid transaction header at %s', name, pos)
+                    warn('%s has invalid transaction header at %s',
+                         self._file.name, pos)
                     break
 
+            if self._stop is not None:
+                LOG("ZODB FS", -100,
+                    ("tid %s > stop %s ? %d" %
+                     (repr(tid), repr(self._stop), tid > self._stop)))
+            if self._stop is not None and tid > self._stop:
+                raise IndexError, index
+
             tpos=pos
             tend=tpos+tl
 
@@ -2047,7 +2114,7 @@
                 h=read(8)
                 if h != stl:
                     panic('%s has inconsistent transaction length at %s',
-                          name, pos)
+                          self._file.name, pos)
                 pos=tend+8
                 continue
 
@@ -2073,7 +2140,7 @@
             h=read(8)
             if h != stl:
                 warn("%s redundant transaction length check failed at %s",
-                     name, pos)
+                     self._file.name, pos)
                 break
             self._pos=pos+8
 
@@ -2081,7 +2148,7 @@
 
         raise IndexError, index
     
-class RecordIterator(Iterator):
+class RecordIterator(Iterator, BaseStorage.TransactionRecord):
     """Iterate over the transactions in a FileStorage file.
     """
     def __init__(self, tid, status, user, desc, ext, pos, stuff):
@@ -2133,9 +2200,8 @@
             return r
         
         raise IndexError, index
-    
 
-class Record:
+class Record(BaseStorage.DataRecord):
     """An abstract database record
     """
     def __init__(self, *args):


=== StandaloneZODB/ZODB/cPersistence.c 1.47.8.1 => 1.47.8.2 ===


=== StandaloneZODB/ZODB/cPickleCache.c 1.37.8.1 => 1.37.8.2 ===
 	      {
 		PyTuple_SET_ITEM(t, 0, v);
-		v = PyObject_CallObject(self->setklassstate, t);
+		v = PyObject_Call(self->setklassstate, t, NULL);
 		PyTuple_SET_ITEM(t, 0, NULL);
 		Py_DECREF(t);
 	      }