[Zodb-checkins] CVS: ZODB4/src/zodb - conflict.py:1.9 connection.py:1.17 db.py:1.13 export.py:1.5 interfaces.py:1.15 lockfile.py:1.6 serialize.py:1.15 utils.py:1.4

Barry Warsaw barry@wooz.org
Thu, 13 Mar 2003 16:32:59 -0500


Update of /cvs-repository/ZODB4/src/zodb
In directory cvs.zope.org:/tmp/cvs-serv27419/src/zodb

Modified Files:
	conflict.py connection.py db.py export.py interfaces.py 
	lockfile.py serialize.py utils.py 
Log Message:
> I believe we're ready to merge back to the head.

merging the opaque-pickles-branch back into the head



=== ZODB4/src/zodb/conflict.py 1.8 => 1.9 ===
--- ZODB4/src/zodb/conflict.py:1.8	Fri Mar  7 18:09:51 2003
+++ ZODB4/src/zodb/conflict.py	Thu Mar 13 16:32:27 2003
@@ -183,4 +183,4 @@
 
         writer = ResolvedObjectWriter()
         obj = ResolvedObjectAdapter(get_self(resolve), resolved)
-        return writer.getState(obj)
+        return writer.getState(obj)[0]


=== ZODB4/src/zodb/connection.py 1.16 => 1.17 ===
--- ZODB4/src/zodb/connection.py:1.16	Thu Mar 13 13:48:55 2003
+++ ZODB4/src/zodb/connection.py	Thu Mar 13 16:32:27 2003
@@ -37,21 +37,20 @@
 $Id$
 """
 
-import cPickle
-from cStringIO import StringIO
 import logging
+import struct
 import tempfile
 import threading
-import time
 from types import StringType, ClassType, TupleType
 
 from zodb import interfaces
 from zodb.conflict import ResolvedSerial
 from zodb.export import ExportImport
 from zodb.interfaces import *
-from zodb.serialize import ConnectionObjectReader, ObjectWriter
+from zodb.serialize import ConnectionObjectReader, ObjectWriter, findrefs
 from zodb.storage.interfaces import IStorage
-from zodb.utils import p64, u64, Set, z64
+from zodb.storage.base import splitrefs
+from zodb.utils import p64, u64, Set
 
 from transaction import get_transaction
 from transaction.interfaces import IDataManager, IRollback, TransactionError
@@ -130,7 +129,7 @@
     # root(), sync(), get()
 
     def root(self):
-        return self.get(z64)
+        return self.get(ZERO)
 
     def sync(self):
         if self._txn:
@@ -155,7 +154,7 @@
         obj._p_serial = serial
 
         self._cache[oid] = obj
-        if oid == z64:
+        if oid == ZERO:
             # Keep a reference to the root so that the pickle cache
             # won't evict it.  XXX Not sure if this is necessary.  If
             # the cache is LRU, it should know best if the root is needed.
@@ -356,10 +355,9 @@
 
     def savepoint(self, txn):
         if self._tmp is None:
-            tmp = TmpStore(self._version)
+            tmp = TmpStore(self._db, self._storage, self._version)
             self._tmp = self._storage
             self._storage = tmp
-            tmp.registerDB(self._db)
         self._modified = Set()
         self._created = Set()
         self._storage.tpcBegin(txn)
@@ -385,7 +383,7 @@
                 del self._cache[oid]
 
     def _invalidate_modified(self):
-        self._db.invalidate(self._modified, self)
+        self._db.invalidate(self._modified, self, self._version)
 
     def _flush_invalidations(self):
         self._inv_lock.acquire()
@@ -472,8 +470,9 @@
                 self._inv_lock.release()
             self._modified.add(oid)
 
-        s = self._storage.store(oid, serial, writer.getState(pobject),
-                                self._version, transaction)
+        data, refs = writer.getState(pobject)
+        s = self._storage.store(oid, serial, data, refs, self._version,
+                                transaction)
         # Put the object in the cache before handling the
         # response, just in case the response contains the
         # serial number for a newly created object
@@ -495,9 +494,11 @@
         self._created |= tmp._created
 
         for oid in tmp._index:
-            data, serial = tmp.load(oid, tmp._bver)
-            s = self._storage.store(oid, serial, data, self._version, txn)
+            data, refs, serial = tmp.loadrefs(oid, tmp._bver)
+            s = self._storage.store(oid, serial, data, refs,
+                                    self._version, txn)
             self._handle_serial(s, oid, change=False)
+        tmp.close()
 
     def _abort_sub(self):
         # Abort work done in subtransactions.
@@ -509,6 +510,7 @@
 
         self._cache.invalidateMany(tmp._index)
         self._invalidate_created(tmp._created)
+        tmp.close()
 
 class Rollback:
     """Rollback changes associated with savepoint"""
@@ -533,15 +535,41 @@
         self._tmp_undo.rollback()
         self._conn._cache.invalidateMany(self._conn._modified)
 
+class UndoInfo:
+    """A helper class for rollback.
+
+    The class stores the state necessary for rolling back to a
+    particular time.
+    """
+
+    def __init__(self, store, pos, index):
+        self._store = store
+        self._pos = pos
+        self._index = index
+
+    def current(self, cur_store):
+        """Return true if the UndoInfo is for cur_store."""
+        return self._store is cur_store
+
+    def rollback(self):
+        self._store.rollback(self._pos, self._index)
+
+
 class TmpStore:
     """A storage to support savepoints."""
 
     _bver = ''
 
-    # XXX This storage doesn't implementation the full API yet
-    __implements__ = IStorage
+    # The header format is oid, serial, nrefs, len(data).  Following
+    # the header are the refs and the data, where the size of refs is
+    # nrefs * 8.
+
+    _fmt = ">8s8sQI"
+    _header_size = 28
 
-    def __init__(self, base_version):
+    def __init__(self, db, storage, base_version):
+        self._db = db
+        self._storage = storage
         self._transaction = None
         if base_version:
             self._bver = base_version
@@ -564,39 +592,36 @@
         pos = self._index.get(oid, None)
         if pos is None:
             return self._storage.load(oid, self._bver)
+        data, refs, serial = self.loadrefs(oid, version)
+        return data, serial
+
+    def loadrefs(self, oid, version):
+        # A version of load the returns data, refs, and serial.
+        pos = self._index.get(oid)
+        # We only call loadrefs() for objects in the TmpStore.
+        assert pos is not None
         self._file.seek(pos)
-        h = self._file.read(24)
-        if h[:8] != oid:
-            raise interfaces.StorageSystemError('Bad temporary storage')
-        size = u64(h[16:])
-        serial = h[8:16]
-        return self._file.read(size), serial
-
-    # XXX clarify difference between self._storage & self._db._storage
-
-    def modifiedInVersion(self, oid):
-        if self._index.has_key(oid):
-            return self._bver
-        return self._storage.modifiedInVersion(oid)
+        buf = self._file.read(self._header_size)
+        oid, serial, nrefs, size = struct.unpack(self._fmt, buf)
+        refs = self._file.read(nrefs * 8)
+        data = self._file.read(size)
+        return data, splitrefs(refs), serial
 
     def newObjectId(self):
         return self._storage.newObjectId()
 
-    def registerDB(self, db):
-        self._db = db
-        self._storage = db._storage
-
-    def store(self, oid, serial, data, version, transaction):
+    def store(self, oid, serial, data, refs, version, transaction):
         if transaction is not self._transaction:
             raise interfaces.StorageTransactionError(self, transaction)
         self._file.seek(self._pos)
-        l = len(data)
         if serial is None:
-            serial = z64
-        self._file.write(oid + serial + p64(l))
+            serial = ZERO
+        buf = struct.pack(self._fmt, oid, serial, len(refs), len(data))
+        self._file.write(buf)
+        self._file.write("".join(refs))
         self._file.write(data)
         self._tindex[oid] = self._pos
-        self._pos += l + 24
+        self._pos += len(refs) * 8 + len(data) + self._header_size
         return serial
 
     def tpcAbort(self, transaction):
@@ -642,22 +667,3 @@
         self._tpos = self._pos = pos
         self._index = index
         self._tindex.clear()
-
-class UndoInfo:
-    """A helper class for rollback.
-
-    The class stores the state necessary for rolling back to a
-    particular time.
-    """
-
-    def __init__(self, store, pos, index):
-        self._store = store
-        self._pos = pos
-        self._index = index
-
-    def current(self, cur_store):
-        """Return true if the UndoInfo is for cur_store."""
-        return self._store is cur_store
-
-    def rollback(self):
-        self._store.rollback(self._pos, self._index)


=== ZODB4/src/zodb/db.py 1.12 => 1.13 ===
--- ZODB4/src/zodb/db.py:1.12	Thu Mar 13 13:48:55 2003
+++ ZODB4/src/zodb/db.py	Thu Mar 13 16:32:27 2003
@@ -28,7 +28,8 @@
 from zodb.connection import Connection
 from zodb.serialize import getDBRoot
 from zodb.ztransaction import Transaction
-from zodb.utils import z64, Set
+from zodb.interfaces import ZERO
+from zodb.utils import Set
 
 from transaction import get_transaction
 from transaction.interfaces import IDataManager
@@ -80,12 +81,15 @@
         self._checkVersion()
         storage.registerDB(self)
         try:
-            storage.load(z64, "")
+            storage.load(ZERO, "")
         except KeyError:
             # Create the database's root in the storage if it doesn't exist
             t = Transaction(description="initial database creation")
             storage.tpcBegin(t)
-            storage.store(z64, None, getDBRoot(), '', t)
+            # Because this is the initial root object, we know it can't have
+            # any references, so include a longer comment then it would take
+            # to unpack getDBRoot()'s return value.
+            storage.store(ZERO, None, getDBRoot()[0], [], '', t)
             storage.tpcVote(t)
             storage.tpcFinish(t)
 


=== ZODB4/src/zodb/export.py 1.4 => 1.5 ===
--- ZODB4/src/zodb/export.py:1.4	Wed Feb  5 18:28:34 2003
+++ ZODB4/src/zodb/export.py	Thu Mar 13 16:32:27 2003
@@ -103,7 +103,7 @@
         """
         copier = ObjectCopier(self, self._storage, self._created)
 
-        while 1:
+        while True:
             h = file.read(16)
             if h == export_end_marker:
                 break
@@ -116,7 +116,7 @@
 
             # XXX I think it would be better if copier.copy()
             # returned an oid and a new pickle so that this logic
-            # wasn't smeared across to modules.
+            # wasn't smeared across two modules.
             oid = h[:8]
             new_ref = copier.oids.get(oid)
             if new_ref is None:
@@ -127,5 +127,6 @@
             else:
                 newObjectId = new_ref[0]
 
-            new = copier.copy(p)
-            self._storage.store(newObjectId, None, new, self._version, txn)
+            data, refs = copier.copy(p)
+            self._storage.store(newObjectId, None, data, refs,
+                                self._version, txn)


=== ZODB4/src/zodb/interfaces.py 1.14 => 1.15 ===
--- ZODB4/src/zodb/interfaces.py:1.14	Thu Mar 13 13:48:55 2003
+++ ZODB4/src/zodb/interfaces.py	Thu Mar 13 16:32:27 2003
@@ -35,25 +35,32 @@
 from transaction.interfaces \
      import TransactionError, RollbackError, ConflictError as _ConflictError
 
-__all__ = ['POSError',
-           'POSKeyError',
-           'ConflictError',
-           'ReadConflictError',
-           'DanglingReferenceError',
-           'InvalidObjectReference',
-           'VersionError',
-           'VersionCommitError',
-           'VersionLockError',
-           'UndoError',
-           'MultipleUndoErrors',
-           'ExportError',
-           'Unsupported',
+__all__ = [
+    # Constants
+    'ZERO', 'MAXTID',
+    # Exceptions
+    'POSError',
+    'POSKeyError',
+    'ConflictError',
+    'ReadConflictError',
+    'DanglingReferenceError',
+    'VersionError',
+    'VersionCommitError',
+    'VersionLockError',
+    'UndoError',
+    'MultipleUndoErrors',
+    'ExportError',
+    'Unsupported',
+    'InvalidObjectReference',
+    # Interfaces
+    'IAppConnection',
+    'IConnection',
+    'ITransaction',
+    'ITransactionAttrs',
+    ]
 
-           'IAppConnection',
-           'IConnection',
-           'ITransaction',
-           'ITransactionAttrs',
-           ]
+ZERO = '\0'*8
+MAXTID = '\377'*8
 
 def _fmt_oid(oid):
     return "%016x" % zodb.utils.u64(oid)


=== ZODB4/src/zodb/lockfile.py 1.5 => 1.6 ===
--- ZODB4/src/zodb/lockfile.py:1.5	Thu Mar 13 13:48:55 2003
+++ ZODB4/src/zodb/lockfile.py	Thu Mar 13 16:32:27 2003
@@ -45,7 +45,7 @@
         pass
 
 
-
+
 # This is a better interface to use than the lockfile.lock_file() interface.
 # Creating the instance acquires the lock.  The file remains open.  Calling
 # close both closes and unlocks the lock file.


=== ZODB4/src/zodb/serialize.py 1.14 => 1.15 ===
--- ZODB4/src/zodb/serialize.py:1.14	Mon Mar 10 18:45:57 2003
+++ ZODB4/src/zodb/serialize.py	Thu Mar 13 16:32:27 2003
@@ -65,10 +65,12 @@
 
 __metaclass__ = type
 
-from cStringIO import StringIO
-import cPickle
-from types import StringType, TupleType
 import logging
+import cPickle
+from cStringIO import StringIO
+
+from zodb.interfaces import ZERO
+
 
 def getClassMetadata(obj):
     if obj._p_state == 3:
@@ -81,7 +83,7 @@
 
 class RootJar:
     def newObjectId(self):
-        return "\0" * 8
+        return ZERO
 
 def getDBRoot():
     """Return a serialized database root object."""
@@ -123,7 +125,7 @@
         # if isinstance(oid, types.MemberDescriptor):
         # -- but I can't because the type doesn't have a canonical name.
         # Instead, we'll assert that an oid must always be a string
-        if not (oid is None or isinstance(oid, StringType)):
+        if not (oid is None or isinstance(oid, str)):
             # XXX log a warning
             return None
 
@@ -143,7 +145,9 @@
         return NewObjectIterator(self._stack)
 
     def getState(self, obj):
-        return self._dump(getClassMetadata(obj), obj.__getstate__())
+        data = self._dump(getClassMetadata(obj), obj.__getstate__())
+        refs = findrefs(data)
+        return data, refs
 
     def _dump(self, classmeta, state):
         # To reuse the existing cStringIO object, we must reset
@@ -236,7 +240,7 @@
 
     def _persistent_load(self, oid):
         # persistent_load function to pass to ObjectReader
-        if isinstance(oid, TupleType):
+        if isinstance(oid, tuple):
             # XXX We get here via new_persistent_id()
 
             # Quick instance reference.  We know all we need to know
@@ -273,7 +277,7 @@
         self._cache = oids
 
     def _persistent_load(self, oid):
-        if isinstance(oid, TupleType):
+        if isinstance(oid, tuple):
             oid, classmeta = oid
         else:
             classmeta = None
@@ -307,7 +311,8 @@
 
     def copy(self, pickle):
         classmeta, state = self._reader.readPickle(pickle)
-        return self._writer._dump(classmeta, state)
+        data = self._writer._dump(classmeta, state)
+        return data, findrefs(data)
 
 def findrefs(p):
     f = StringIO(p)
@@ -321,7 +326,7 @@
     # Iterator over L and convert persistent references to simple oids.
     oids = []
     for ref in L:
-        if isinstance(ref, TupleType):
+        if isinstance(ref, tuple):
             oids.append(ref[0])
         else:
             oids.append(ref)


=== ZODB4/src/zodb/utils.py 1.3 => 1.4 ===
--- ZODB4/src/zodb/utils.py:1.3	Wed Mar  5 17:14:30 2003
+++ ZODB4/src/zodb/utils.py	Thu Mar 13 16:32:27 2003
@@ -15,8 +15,6 @@
 import struct
 import time
 
-z64 = "\0" * 8
-
 def p64(v):
     """Pack an integer or long into a 8-byte string"""
     return struct.pack(">Q", v)