[Zope3-checkins] CVS: Zope3/src/zodb/storage - file.py:1.8.4.6

Jeremy Hylton jeremy@zope.com
Mon, 10 Mar 2003 18:33:44 -0500


Update of /cvs-repository/Zope3/src/zodb/storage
In directory cvs.zope.org:/tmp/cvs-serv32018/storage

Modified Files:
      Tag: opaque-pickles-branch
	file.py 
Log Message:
Many (mostly) small changes to fix refs handling.

Make sure the references are always read before the data.
Add use of _read_data_header() in a few more places.
_loadBack_impl() returns the refs, too, since any time you read data
    from a backpointer, you'd also need the refs from the backpointer.
    The only case that's come up so far is an iterator's .refs attr.
Fix off-by-8 bug in _txn_undo_write().  The end of the data records
    is 8 bytes before the end of the txn record.
Rename _transactionalUndoRecord() to _undo_record() and have it
    return refs along with data.   XXX The conflict resolution case
    is not right yet.
Fix DataHeader.recordlen() to account for the version length.


=== Zope3/src/zodb/storage/file.py 1.8.4.5 => 1.8.4.6 ===
--- Zope3/src/zodb/storage/file.py:1.8.4.5	Mon Mar 10 15:59:10 2003
+++ Zope3/src/zodb/storage/file.py	Mon Mar 10 18:33:43 2003
@@ -353,7 +353,6 @@
 
                 if h.version:
                     vindex[h.version] = pos
-                    dlen += 16 + len(h.version)
 
                 if pos + dlen > tend or h.tloc != tpos:
                     if recover:
@@ -458,27 +457,28 @@
                 # If backpointer is 0, object does not currently exist.
                 raise POSKeyError(oid)
             h = self._read_data_header(back)
-            self._file.read(h.nrefs * 8)
+            refs = self._file.read(h.nrefs * 8)
             if h.plen:
-                return self._file.read(h.plen), h.serial, back, h.tloc
+                return self._file.read(h.plen), refs, h.serial, back, h.tloc
             back = h.back
 
     def _loadBack(self, oid, back):
-        data, serial, old, tloc = self._loadBack_impl(oid, back)
+        data, refs, serial, old, tloc = self._loadBack_impl(oid, back)
         return data, serial
 
     def _loadBackPOS(self, oid, back):
         """Return position of data record for backpointer."""
-        data, serial, old, tloc = self._loadBack_impl(oid, back)
+        data, refs, serial, old, tloc = self._loadBack_impl(oid, back)
         return old
 
     def _loadBackTxn(self, oid, back):
         """Return data, serial, and txn id for backpointer."""
-        data, serial, old, tloc = self._loadBack_impl(oid, back)
+        data, refs, serial, old, tloc = self._loadBack_impl(oid, back)
         self._file.seek(tloc)
         h = self._file.read(TRANS_HDR_LEN)
         tid = h[:8]
-        return data, serial, tid
+        refs = splitrefs(refs)
+        return data, refs, serial, tid
 
     def getTxnFromData(self, oid, back):
         """Return transaction id for data at back."""
@@ -893,33 +893,29 @@
         h = self._file.read(TRANS_HDR_LEN)
         tid, tl, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
         self._file.read(ul + dl + el)
-        tend = tpos + tl + 8
+        tend = tpos + tl 
         pos = self._file.tell()
         while pos < tend:
-            h = self._file.read(DATA_HDR_LEN)
-            _oid, serial, prev, tpos, vl, nr, dl = struct.unpack(DATA_HDR, h)
-            reclen = DATA_HDR_LEN + vl + dl + (nr * 8)
-            if vl:
-                reclen += 16
-            if _oid == oid:
-                if vl:
-                    self._file.read(vl + 16)
+            h = self._read_data_header(pos)
+            if h.oid == oid:
+                if h.version:
+                    self._file.read(h.vlen + 16)
                 # Read past any references data
-                self._file.read(nr * 8)
+                self._file.read(h.nrefs * 8)
                 # Make sure this looks like the right data record
-                if dl == 0:
+                if h.plen == 0:
                     # This is also a backpointer.  Gotta trust it.
                     return pos
-                if dl != len(data):
+                if h.plen != len(data):
                     # The expected data doesn't match what's in the
                     # backpointer.  Something is wrong.
                     error("Mismatch between data and backpointer at %d", pos)
                     return 0
-                _data = self._file.read(dl)
+                _data = self._file.read(h.plen)
                 if data != _data:
                     return 0
                 return pos
-            pos += reclen
+            pos += h.recordlen()
             self._file.seek(pos)
         return 0
 
@@ -1181,11 +1177,11 @@
         assert oid == h[:8]
         return h[8:]
 
-    def _transactionalUndoRecord(self, oid, pos, serial, pre, version):
-        """Get the indo information for a data record
+    def _undo_record(self, h, pos):
+        """Get the undo information for a data record
 
-        Return a 5-tuple consisting of a pickle, data pointer,
-        version, packed non-version data pointer, and current
+        Return a 6-tuple consisting of a pickle, references, data
+        pointer, version, packed non-version data pointer, and current
         position.  If the pickle is true, then the data pointer must
         be 0, but the pickle can be empty *and* the pointer 0.
         """
@@ -1193,17 +1189,17 @@
         copy = 1 # Can we just copy a data pointer
 
         # First check if it is possible to undo this record.
-        tpos = self._tindex.get(oid, 0)
-        ipos = self._index.get(oid, 0)
+        tpos = self._tindex.get(h.oid, 0)
+        ipos = self._index.get(h.oid, 0)
         tipos = tpos or ipos
 
         if tipos != pos:
             # Eek, a later transaction modified the data, but,
             # maybe it is pointing at the same data we are.
             cserial, cdataptr, cdata, cver = self._undoDataInfo(
-                oid, ipos, tpos)
+                h.oid, ipos, tpos)
             # Versions of undone record and current record *must* match!
-            if cver != version:
+            if cver != h.version:
                 raise UndoError(oid, 'Current and undone versions differ')
 
             if cdataptr != pos:
@@ -1214,46 +1210,51 @@
                         cdataptr == tipos
                         or
                         # Backpointers are different
-                        self._loadBackPOS(oid, pos) !=
-                        self._loadBackPOS(oid, cdataptr)
+                        self._loadBackPOS(h.oid, pos) !=
+                        self._loadBackPOS(h.oid, cdataptr)
                         ):
-                        if pre and not tpos:
+                        if h.prev and not tpos:
                             copy = 0 # we'll try to do conflict resolution
                         else:
                             # We bail if:
                             # - We don't have a previous record, which should
                             #   be impossible.
-                            raise UndoError(oid, "No previous record")
+                            raise UndoError(h.oid, "No previous record")
                 except KeyError:
                     # LoadBack gave us a key error. Bail.
-                    raise UndoError(oid, "_loadBack() failed")
+                    raise UndoError(h.oid, "_loadBack() failed")
 
         # Return the data that should be written in the undo record.
-        if not pre:
+        if not h.prev:
             # There is no previous revision, because the object creation
             # is being undone.
-            return '', 0, '', '', ipos
+            return "", "", 0, "", "", ipos
 
-        version, snv = self._getVersion(oid, pre)
+        version, snv = self._getVersion(h.oid, h.prev)
         if copy:
             # we can just copy our previous-record pointer forward
-            return '', pre, version, snv, ipos
+            return "", "", h.prev, version, snv, ipos
 
         try:
             # returns data, serial tuple
-            bdata = self._loadBack(oid, pre)[0]
+            bdata = self._loadBack(h.oid, h.prev)[0]
         except KeyError:
             # couldn't find oid; what's the real explanation for this?
-            raise UndoError(oid, "_loadBack() failed")
+            raise UndoError(h.oid, "_loadBack() failed")
+
+
+        # XXX conflict resolution needs to give us new references, but
+        # that code isn't written yet
+        
         try:
-            data = self.resolveConflict(oid, cserial, serial, bdata, cdata)
+            data = self.resolveConflict(h.oid, cserial, h.serial, bdata, cdata)
         except interfaces.ConflictError:
             data = None
 
         if data is None:
-            raise UndoError(oid,
+            raise UndoError(h.oid,
                             "Some data were modified by a later transaction")
-        return data, 0, version, snv, ipos
+        return data, "", 0, h.version, snv, ipos
 
 
     # undoLog() returns a description dict that includes an id entry.
@@ -1354,32 +1355,23 @@
         failed = failures.has_key
         # Read the data records for this transaction
         while pos < tend:
-            self._file.seek(pos)
-            h = self._file.read(DATA_HDR_LEN)
-            oid, serial, prev, tloc, vlen, nrefs, plen = struct.unpack(
-                DATA_HDR, h)
-            if failed(oid):
-                del failures[oid] # second chance!
-            if vlen:
-                dlen = DATA_VERSION_HDR_LEN + vlen + (plen or 8) + (nrefs * 8)
-                self._file.seek(16, 1)
-                version = self._file.read(vlen)
-            else:
-                dlen = DATA_HDR_LEN + (plen or 8)
-                version = ''
-
-            refsdata = self._file.read(nrefs * 8)
+            h = self._read_data_header(pos)
+            if failed(h.oid):
+                del failures[h.oid] # second chance!
+            if h.version:
+                self._file.read(16 + h.vlen)
+            self._file.read(h.nrefs * 8)
 
             try:
-                p, prev, v, snv, ipos = self._transactionalUndoRecord(
-                    oid, pos, serial, prev, version)
+                p, refs, prev, v, snv, ipos = self._undo_record(h, pos)
             except UndoError, v:
                 # Don't fail right away. We may be redeemed later!
-                failures[oid] = v
+                failures[h.oid] = v
             else:
                 plen = len(p)
+                nrefs = len(refs) / 8
                 self._tfile.write(pack(DATA_HDR,
-                                       oid, self._serial, ipos,
+                                       h.oid, self._serial, ipos,
                                        otloc, len(v), nrefs, plen))
                 if v:
                     vprev = self._tvindex.get(v, 0) or self._vindex.get(v, 0)
@@ -1389,15 +1381,14 @@
                 else:
                     odlen = DATA_HDR_LEN + (plen or 8)
 
-                self._tfile.write(refsdata)
-
+                self._tfile.write(refs)
                 if p:
                     self._tfile.write(p)
                 else:
                     self._tfile.write(p64(prev))
-                tindex[oid] = here
+                tindex[h.oid] = here
                 here += odlen + nrefs * 8
-            pos += dlen
+            pos += h.recordlen()
             if pos > tend:
                 raise UndoError(None, "non-undoable transaction")
         if failures:
@@ -1405,7 +1396,6 @@
 
         return tindex
 
-
     def versionEmpty(self, version):
         if not version:
             # The interface is silent on this case. I think that this should
@@ -1702,7 +1692,6 @@
                 h = self._read_data_header(pos)
                 dlen = h.recordlen()
                 if h.version:
-                    dlen += 16 + h.vlen
                     if packing and pindex.get(oid, 0) != pos:
                         # This is not the most current record, or the
                         # oid is no longer referenced so skip it.
@@ -2066,8 +2055,6 @@
             # Read the data records for this transaction
             h = self._read_data_header(pos)
             dlen = h.recordlen()
-            if h.version:
-                dlen += 16 + h.vlen
             if pos + dlen > self._tend or h.tloc != self._tpos:
                 warn("%s data record exceeds transaction record at %s",
                      file.name, pos)
@@ -2076,10 +2063,9 @@
             pos += dlen
             prev_txn = None
 
-            refsdata = self._file.read(h.nrefs * 8)
-            refs = splitrefs(refsdata)
-
             if h.plen:
+                refsdata = self._file.read(h.nrefs * 8)
+                refs = splitrefs(refsdata)
                 data = self._file.read(h.plen)
             else:
                 if not h.back:
@@ -2089,8 +2075,9 @@
                     # transaction that created it.  Return None
                     # instead of a pickle to indicate this.
                     data = None
+                    refs = []
                 else:
-                    data, _s, tid = self._loadBackTxn(h.oid, h.back)
+                    data, refs, _s, tid = self._loadBackTxn(h.oid, h.back)
                     prev_txn = self.getTxnFromData(h.oid, h.back)
 
             yield Record(h.oid, h.serial, h.version, data, prev_txn, refs)
@@ -2180,6 +2167,7 @@
         self.serial = serial
         self.prev = prev
         self.tloc = tloc
+
         self.vlen = vlen
         self.nrefs = nrefs
         self.plen = plen
@@ -2194,7 +2182,10 @@
         self.version = buf[16:]
 
     def recordlen(self):
-        return DATA_HDR_LEN + (self.nrefs * 8) + (self.plen or 8)
+        rlen = DATA_HDR_LEN + (self.nrefs * 8) + (self.plen or 8)
+        if self.version:
+            rlen += 16 + self.vlen
+        return rlen
 
 
 def cleanup(filename):