[Zodb-checkins] CVS: Zope/lib/python/ZODB - BaseStorage.py:1.16.16.2 Connection.py:1.61.16.2 DemoStorage.py:1.8.16.2 FileStorage.py:1.76.16.2 MappingStorage.py:1.4.16.2 POSException.py:1.8.16.2 cPickleCache.c:1.37.16.2

Fred L. Drake, Jr. fdrake@acm.org
Wed, 20 Feb 2002 10:05:38 -0500


Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv1061

Modified Files:
      Tag: Zope-2_5-branch
	BaseStorage.py Connection.py DemoStorage.py FileStorage.py 
	MappingStorage.py POSException.py cPickleCache.c 
Log Message:
Merge ZODB from trunk.

=== Zope/lib/python/ZODB/BaseStorage.py 1.16.16.1 => 1.16.16.2 ===
     _serial=z64       # Transaction serial number
     _tstatus=' '      # Transaction status, used for copying data
+    _is_read_only = 0
 
     def __init__(self, name, base=None):
         
@@ -43,8 +44,10 @@
         t=time.time()
         t=self._ts=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
         self._serial=`t`
-        if base is None: self._oid='\0\0\0\0\0\0\0\0'
-        else:            self._oid=base._oid
+        if base is None:
+            self._oid='\0\0\0\0\0\0\0\0'
+        else:
+            self._oid=base._oid
 
     def abortVersion(self, src, transaction):
         if transaction is not self._transaction:
@@ -56,15 +59,24 @@
             raise POSException.StorageTransactionError(self, transaction)
         return []
 
-    def close(self): pass
+    def close(self):
+        pass
 
-    def getName(self): return self.__name__
-    def getSize(self): return len(self)*300 # WAG!
-    def history(self, oid, version, length=1): pass
+    def getName(self):
+        return self.__name__
+    
+    def getSize(self):
+        return len(self)*300 # WAG!
+    
+    def history(self, oid, version, length=1):
+        pass
                     
-    def modifiedInVersion(self, oid): return ''
+    def modifiedInVersion(self, oid):
+        return ''
 
     def new_oid(self, last=None):
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
         if last is None:
             self._lock_acquire()
             try:
@@ -80,10 +92,17 @@
             if d < 255: return last[:-1]+chr(d+1)+'\0'*(8-len(last))
             else:       return self.new_oid(last[:-1])
 
-    def registerDB(self, db, limit): pass # we don't care
+    def registerDB(self, db, limit):
+        pass # we don't care
 
-    def supportsUndo(self): return 0
-    def supportsVersions(self): return 0
+    def isReadOnly(self):
+        return self._is_read_only
+    
+    def supportsUndo(self):
+        return 0
+    
+    def supportsVersions(self):
+        return 0
         
     def tpc_abort(self, transaction):
         self._lock_acquire()
@@ -172,15 +191,22 @@
         pass
 
     def undo(self, transaction_id):
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
         raise POSException.UndoError, 'non-undoable transaction'
 
-    def undoLog(self, first, last, filter=None): return ()
+    def undoLog(self, first, last, filter=None):
+        return ()
 
-    def versionEmpty(self, version): return 1
+    def versionEmpty(self, version):
+        return 1
 
-    def versions(self, max=None): return ()
+    def versions(self, max=None):
+        return ()
 
-    def pack(self, t, referencesf): pass
+    def pack(self, t, referencesf):
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
 
     def getSerial(self, oid):
         self._lock_acquire()
@@ -202,7 +228,24 @@
         """
         _ts=None
         ok=1
-        preindex={}; preget=preindex.get   # waaaa
+        preindex={};
+        preget=preindex.get   # waaaa
+        # restore() is a new storage API method which has an identical
+        # signature to store() except that it does not return anything.
+        # Semantically, restore() is also identical to store() except that it
+        # doesn't do the ConflictError or VersionLockError consistency
+        # checks.  The reason to use restore() over store() in this method is
+        # that store() cannot be used to copy transactions spanning a version
+        # commit or abort, or over transactional undos.
+        #
+        # We'll use restore() if it's available, otherwise we'll fall back to
+        # using store().  However, if we use store, then
+        # copyTransactionsFrom() may fail with VersionLockError or
+        # ConflictError.
+        if hasattr(self, 'restore'):
+            restoring = 1
+        else:
+            restoring = 0
         for transaction in other.iterator():
             
             tid=transaction.tid
@@ -227,9 +270,18 @@
             for r in transaction:
                 oid=r.oid
                 if verbose: print `oid`, r.version, len(r.data)
-                pre=preget(oid, None)
-                s=self.store(oid, pre, r.data, r.version, transaction)
-                preindex[oid]=s
+                if restoring:
+                    self.restore(oid, r.serial, r.data, r.version, transaction)
+                else:
+                    pre=preget(oid, None)
+                    s=self.store(oid, pre, r.data, r.version, transaction)
+                    preindex[oid]=s
                 
             self.tpc_vote(transaction)
             self.tpc_finish(transaction)
+
+class TransactionRecord:
+    """Abstract base class for iterator protocol"""
+
+class DataRecord:
+    """Abstract base class for iterator protocol"""


=== Zope/lib/python/ZODB/Connection.py 1.61.16.1 => 1.61.16.2 ===
 
 from cPickleCache import PickleCache
-from POSException import ConflictError
+from POSException import ConflictError, ReadConflictError
 from ExtensionClass import Base
 import ExportImport, TmpStore
 from zLOG import LOG, ERROR, BLATHER
@@ -249,7 +249,7 @@
                 or
                 invalid(None)
                 ):
-                raise ConflictError, `oid`
+                raise ConflictError(object=object)
             self._invalidating.append(oid)
 
         else:
@@ -316,7 +316,7 @@
                     or
                     invalid(None)
                     ):
-                    raise ConflictError, `oid`
+                    raise ConflictError(object=object)
                 self._invalidating.append(oid)
                 
             klass = object.__class__
@@ -460,7 +460,7 @@
             if invalid(oid) or invalid(None):
                 if not hasattr(object.__class__, '_p_independent'):
                     get_transaction().register(self)
-                    raise ConflictError(`oid`, `object.__class__`)
+                    raise ReadConflictError(object=object)
                 invalid=1
             else:
                 invalid=0
@@ -485,7 +485,7 @@
                     except KeyError: pass
                 else:
                     get_transaction().register(self)
-                    raise ConflictError(`oid`, `object.__class__`)
+                    raise ConflictError(object=object)
 
         except ConflictError:
             raise
@@ -545,7 +545,7 @@
 
     def tpc_begin(self, transaction, sub=None):
         if self._invalid(None): # Some nitwit invalidated everything!
-            raise ConflictError, "transaction already invalidated"
+            raise ConflictError("transaction already invalidated")
         self._invalidating=[]
         self._creating=[]
 


=== Zope/lib/python/ZODB/DemoStorage.py 1.8.16.1 => 1.8.16.2 ===
                     nv=old
 
-                if serial != oserial: raise POSException.ConflictError
+                if serial != oserial:
+                    raise POSException.ConflictError(serials=(oserial, serial))
                 
             serial=self._serial
             r=[oid, serial, old, version and (version, nv) or None, data]


=== Zope/lib/python/ZODB/FileStorage.py 1.76.16.1 => 1.76.16.2 === (1054/1154 lines abridged)
 __version__='$Revision$'[11:-2]
 
-import struct, time, os, bpthread, string, base64, sys
+import struct, time, os, string, base64, sys
 from struct import pack, unpack
-from cPickle import loads
 import POSException
-from POSException import UndoError
+from POSException import UndoError, POSKeyError
 from TimeStamp import TimeStamp
 from lock_file import lock_file
 from utils import t32, p64, U64, cp
-from zLOG import LOG, WARNING, ERROR, PANIC, register_subsystem
+from zLOG import LOG, BLATHER, WARNING, ERROR, PANIC, register_subsystem
 register_subsystem('ZODB FS')
 import BaseStorage
-from cPickle import Pickler, Unpickler
+from cPickle import Pickler, Unpickler, loads
 import ConflictResolution
 
 try:
@@ -143,6 +142,10 @@
 from types import StringType
 
 z64='\0'*8
+# constants to support various header sizes
+TRANS_HDR_LEN = 23
+DATA_HDR_LEN = 42
+DATA_VERSION_HDR_LEN = 58
 
 def warn(message, *data):
     LOG('ZODB FS',WARNING, "%s  warn: %s\n" % (packed_version, (message % data)))
@@ -192,6 +195,7 @@
             create = 1
 
         if read_only:
+            self._is_read_only = 1
             if create:
                 raise ValueError, "can\'t create a read-only file"
         elif stop is not None:
@@ -249,6 +253,7 @@
                 self._file, file_name, index, vindex, tindex, stop,
                 read_only=read_only,
                 )
+        self._ltid = tid
 
         self._ts = tid = TimeStamp(tid)
         t = time.time()
@@ -269,7 +274,8 @@
         self._index_get=index.get

[-=- -=- -=- 1054 lines omitted -=- -=- -=-]

             # Read the data records for this transaction
-
             seek(pos)
-            h=read(42)
+            h=read(DATA_HDR_LEN)
             oid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
             prev=U64(sprev)
             tloc=U64(stloc)
             plen=U64(splen)
 
-            dlen=42+(plen or 8)
+            dlen=DATA_HDR_LEN+(plen or 8)
 
             if vlen:
                 dlen=dlen+(16+vlen)
@@ -2124,19 +2312,28 @@
                 break
 
             self._pos=pos+dlen
-            if plen: p=read(plen)
+            if plen:
+                p = read(plen)
             else:
-                p=read(8)
-                p=_loadBack(file, oid, p)[0]
+                p = read(8)
+                if p == z64:
+                    # If the backpointer is 0 (encoded as z64), then
+                    # this transaction undoes the object creation.  It
+                    # either aborts the version that created the
+                    # object or undid the transaction that created it.
+                    # Return None instead of a pickle to indicate
+                    # this.
+                    p = None
+                else:
+                    p = _loadBack(file, oid, p)[0]
                 
-            r=Record(oid, serial, version, p)
+            r = Record(oid, serial, version, p)
             
             return r
         
         raise IndexError, index
-    
 
-class Record:
+class Record(BaseStorage.DataRecord):
     """An abstract database record
     """
     def __init__(self, *args):


=== Zope/lib/python/ZODB/MappingStorage.py 1.4.16.1 => 1.4.16.2 ===
                 old=self._index[oid]
                 oserial=old[:8]
-                if serial != oserial: raise POSException.ConflictError
+                if serial != oserial:
+                    raise POSException.ConflictError(serials=(oserial, serial))
                 
             serial=self._serial
             self._tindex.append((oid,serial+data))


=== Zope/lib/python/ZODB/POSException.py 1.8.16.1 => 1.8.16.2 ===
 # 
 ##############################################################################
-'''BoboPOS-defined exceptions
+"""BoboPOS-defined exceptions
 
-$Id$'''
-__version__='$Revision$'[11:-2]
+$Id$"""
+__version__='$Revision$'.split()[-2:][0]
 
 from string import join
-StringType=type('')
-DictType=type({})
+from types import StringType, DictType
+from ZODB import utils
 
 class POSError(Exception):
     """Persistent object system error
     """
 
+class POSKeyError(KeyError, POSError):
+    """Key not found in database
+    """
+
+    def __str__(self):
+        return "%016x" % utils.U64(self.args[0])
+
 class TransactionError(POSError):
     """An error occured due to normal transaction processing
     """
 
 class ConflictError(TransactionError):
-    """Two transactions tried to modify the same object at once
+    """Two transactions tried to modify the same object at once.  This
+    transaction should be resubmitted.
+  
+    Instance attributes:
+      oid : string
+        the OID (8-byte packed string) of the object in conflict
+      class_name : string
+        the fully-qualified name of that object's class
+      message : string
+        a human-readable explanation of the error
+      serials : (string, string)
+        a pair of 8-byte packed strings; these are the serial numbers
+        (old and new) of the object in conflict.  (Serial numbers are
+        closely related [equal?] to transaction IDs; a ConflictError may
+        be triggered by a serial number mismatch.)
+
+    The caller should pass either object or oid as a keyword argument,
+    but not both of them.  If object is passed, it should be a
+    persistent object with an _p_oid attribute.
+    """
+
+    def __init__(self, message=None, object=None, oid=None, serials=None):
+        if message is None:
+            self.message = "database conflict error"
+        else:
+            self.message = message
+
+        if object is None or not hasattr(object, '_p_oid'):
+            self.oid = None
+            self.class_name = None
+        else:
+            self.oid = object._p_oid
+            klass = object.__class__
+            self.class_name = klass.__module__ + "." + klass.__name__
+
+        if oid is not None:
+            assert self.oid is None
+            self.oid = oid
+
+        self.serials = serials
+
+    def __str__(self):
+        extras = []
+        if self.oid:
+            extras.append("oid %016x" % utils.U64(self.oid))
+        if self.class_name:
+            extras.append("class %s" % self.class_name)
+        if self.serials:
+            extras.append("serial was %016x, now %016x" %
+                          tuple(map(utils.U64, self.serials)))
+        if extras:
+            return "%s (%s)" % (self.message, ", ".join(extras))
+        else:
+            return self.message
+
+    def get_oid(self):
+        return self.oid
+
+    def get_class_name(self):
+        return self.class_name
+
+    def get_old_serial(self):
+        return self.serials[0]
+
+    def get_new_serial(self):
+        return self.serials[1]
+
+    def get_serials(self):
+        return self.serials
+
+class ReadConflictError(ConflictError):
+    """A conflict detected at read time -- attempt to read an object
+    that has changed in another transaction (eg. another thread
+    or process).
+    """
+    def __init__(self, message=None, object=None, serials=None):
+        if message is None:
+            message = "database read conflict error"
+        ConflictError.__init__(self, message=message, object=object,
+                               serials=serials)
+
+class BTreesConflictError(ConflictError):
+    """A special subclass for BTrees conflict errors, which return
+    an undocumented four-tuple."""
+    def __init__(self, *btree_args):
+        ConflictError.__init__(self, message="BTrees conflict error")
+        self.btree = btree_args
 
-    This transaction should be resubmitted.
-    """
 
 class VersionError(POSError):
     """An error in handling versions occurred
@@ -83,6 +174,10 @@
 
 class MountedStorageError(StorageError):
     """Unable to access mounted storage.
+    """
+
+class ReadOnlyError(StorageError):
+    """Unable to modify objects in a read-only storage.
     """
 
 class ExportError(POSError):


=== Zope/lib/python/ZODB/cPickleCache.c 1.37.16.1 => 1.37.16.2 === (851/951 lines abridged)
 "$Id$\n";
 
-#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
-#define UNLESS(E) if(!(E))
-#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
-#define OBJECT(O) ((PyObject*)O)
-
 /* Compute the current time in the units and range used for peristent
    objects. */
 #define PER_TIME() ((long)(time(NULL) / 3)) % 65536
@@ -34,23 +29,23 @@
 static PyObject *py_reload, *py__p_jar, *py__p_changed;
 
 typedef struct {
-  PyObject_HEAD
-  PyObject *data;
-  PyObject *jar;
-  PyObject *setklassstate;
-  int position;
-  int cache_size;
-  int cache_age;
-  /* Cache statistics */
-  int sum_deal;
-  int sum_deac;
-  double sum_age;
-  int n, na;
-  time_t last_check;		/* Time of last gc */
-  double mean_age;
-  double mean_deal;
-  double mean_deac;
-  double df, dfa;			/* Degees of freedom for above stats */
+    PyObject_HEAD
+    PyObject *data;
+    PyObject *jar;
+    PyObject *setklassstate;
+    int position;
+    int cache_size;
+    int cache_age;
+    /* Cache statistics */
+    int sum_deal;
+    int sum_deac;
+    double sum_age;
+    int n, na;
+    time_t last_check;		/* Time of last gc */
+    double mean_age;
+    double mean_deal;
+    double mean_deac;
+    double df, dfa;			/* Degees of freedom for above stats */
 } ccobject;
 

[-=- -=- -=- 851 lines omitted -=- -=- -=-]

 };
 
 static PyObject *
@@ -642,9 +644,9 @@
   int cache_size=100, cache_age=1000;
   PyObject *jar;
 
-  UNLESS(PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
+  if (!PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
       return NULL;
-  return (PyObject*)newccobject(jar, cache_size,cache_age);
+  return (PyObject *)newccobject(jar, cache_size, cache_age);
 }
 
 static struct PyMethodDef cCM_methods[] = {
@@ -655,27 +657,17 @@
 void
 initcPickleCache(void)
 {
-  PyObject *m, *d, *s;
-  char *rev="$Revision$";
+  PyObject *m;
 
-  Cctype.ob_type=&PyType_Type;
+  Cctype.ob_type = &PyType_Type;
 
-  UNLESS(ExtensionClassImported) return;
+  if (!ExtensionClassImported)
+      return;
 
   m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string,
 		     (PyObject*)NULL, PYTHON_API_VERSION);
 
-  d = PyModule_GetDict(m);
-
-  py_reload=PyString_FromString("reload");
-  py__p_jar=PyString_FromString("_p_jar");
-  py__p_changed=PyString_FromString("_p_changed");
-
-  s = PyString_FromStringAndSize(rev+11,strlen(rev+11)-2);
-  PyDict_SetItemString(d,"__version__", s);
-  Py_XDECREF(s);
-
-  /* Check for errors */
-  if (PyErr_Occurred())
-    Py_FatalError("can't initialize module cPickleCache");
+  py_reload = PyString_InternFromString("reload");
+  py__p_jar = PyString_InternFromString("_p_jar");
+  py__p_changed = PyString_InternFromString("_p_changed");
 }