[Zodb-checkins] CVS: ZODB3/ZODB - fspack.py:1.8.2.1 storage.xml:1.1.2.1 Connection.py:1.91.2.1 FileStorage.py:1.129.8.1 cPersistence.c:1.69.2.1 cPickleCache.c:1.82.2.1 component.xml:1.7.8.2 config.py:1.10.12.1 fsdump.py:1.6.30.1 fsrecover.py:1.6.8.1 StorageConfig.py:NONE StorageTypes.py:NONE

Jeremy Hylton jeremy at zope.com
Thu May 29 13:31:23 EDT 2003


Update of /cvs-repository/ZODB3/ZODB
In directory cvs.zope.org:/tmp/cvs-serv13391/ZODB

Modified Files:
      Tag: ZODB3-auth-branch
	Connection.py FileStorage.py cPersistence.c cPickleCache.c 
	component.xml config.py fsdump.py fsrecover.py 
Added Files:
      Tag: ZODB3-auth-branch
	fspack.py storage.xml 
Removed Files:
      Tag: ZODB3-auth-branch
	StorageConfig.py StorageTypes.py 
Log Message:
Merge trunk to auth branch.


=== Added File ZODB3/ZODB/fspack.py === (777/877 lines abridged)
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""FileStorage helper to perform pack.

A storage contains an ordered set of object revisions.  When a storage
is packed, object revisions that are not reachable as of the pack time
are deleted.  The notion of reachability is complicated by
backpointers -- object revisions that point to earlier revisions of
the same object.

An object revisions is reachable at a certain time if it is reachable
from the revision of the root at that time or if it is reachable from
a backpointer after that time.
"""

# This module contains code backported from ZODB4 from the
# zodb.storage.file package.  It's been edited heavily to work with
# ZODB3 code and storage layout.

import os
import struct
from types import StringType

from ZODB.referencesf import referencesf
from ZODB.utils import p64, u64, z64
from zLOG import LOG, BLATHER, WARNING, ERROR, PANIC

try:
    from ZODB.fsIndex import fsIndex
except ImportError:
    def fsIndex():
        return {}

class CorruptedError(Exception):
    pass

class CorruptedDataError(CorruptedError):

    def __init__(self, oid=None, buf=None, pos=None):

[-=- -=- -=- 777 lines omitted -=- -=- -=-]

                ipos = self.copyOne(ipos)
        except CorruptedDataError, err:
            # The last call to copyOne() will raise
            # CorruptedDataError, because it will attempt to read past
            # the end of the file.  Double-check that the exception
            # occurred for this reason.
            self._file.seek(0, 2)
            endpos = self._file.tell()
            if endpos != err.pos:
                raise

    def copyOne(self, ipos):
        # The call below will raise CorruptedDataError at EOF.
        th = self._read_txn_header(ipos)
        self._lock_counter += 1
        if self._lock_counter % 20 == 0:
            self._commit_lock_release()
        pos = self._tfile.tell()
        self._copier.setTxnPos(pos)
        self._tfile.write(th.asString())
        tend = ipos + th.tlen
        ipos += th.headerlen()

        while ipos < tend:
            h = self._read_data_header(ipos)
            ipos += h.recordlen()
            prev_txn = None
            if h.plen:
                data = self._file.read(h.plen)
            else:
                data = self.fetchBackpointer(h.oid, h.back)
                if h.back:
                    prev_txn = self.getTxnFromData(h.oid, h.back)

            self._copier.copy(h.oid, h.serial, data, h.version,
                              prev_txn, pos, self._tfile.tell())

        tlen = self._tfile.tell() - pos
        assert tlen == th.tlen
        self._tfile.write(p64(tlen))
        ipos += 8

        self.index.update(self.tindex)
        self.tindex.clear()
        self.vindex.update(self.tvindex)
        self.tvindex.clear()
        if self._lock_counter % 20 == 0:
            self._commit_lock_acquire()
        return ipos



=== Added File ZODB3/ZODB/storage.xml ===
<schema>
<import package="ZODB"/>
<section type="ZODB.storage" name="*" attribute="storage"/>
</schema>


=== ZODB3/ZODB/Connection.py 1.91 => 1.91.2.1 ===
--- ZODB3/ZODB/Connection.py:1.91	Wed Apr 23 16:36:02 2003
+++ ZODB3/ZODB/Connection.py	Thu May 29 12:30:51 2003
@@ -316,7 +316,8 @@
         
         oid = object._p_oid
         if self._conflicts.has_key(oid):
-            raise ReadConflictError(oid)
+            self.getTransaction().register(object)
+            raise ReadConflictError(object=object)
         
         invalid = self._invalid
         if oid is None or object._p_jar is not self:
@@ -590,7 +591,7 @@
                     # Defer _p_independent() call until state is loaded.
                     return 1
                 else:
-                    self.getTransaction().register(self)
+                    self.getTransaction().register(obj)
                     self._conflicts[obj._p_oid] = 1
                     raise ReadConflictError(object=obj)
             else:


=== ZODB3/ZODB/FileStorage.py 1.129 => 1.129.8.1 === (415/515 lines abridged)
--- ZODB3/ZODB/FileStorage.py:1.129	Mon Mar 17 13:58:30 2003
+++ ZODB3/ZODB/FileStorage.py	Thu May 29 12:30:51 2003
@@ -127,16 +127,15 @@
 from types import StringType, DictType
 from struct import pack, unpack
 
-try:
-    from posix import fsync
-except:
-    fsync = None
+# Not all platforms have fsync
+fsync = getattr(os, "fsync", None)
 
 from ZODB import BaseStorage, ConflictResolution, POSException
 from ZODB.POSException import UndoError, POSKeyError, MultipleUndoErrors
 from ZODB.TimeStamp import TimeStamp
 from ZODB.lock_file import LockFile
 from ZODB.utils import p64, u64, cp, z64
+from ZODB.fspack import FileStoragePacker
 
 try:
     from ZODB.fsIndex import fsIndex
@@ -596,6 +595,8 @@
             pos=_index[oid]
         except KeyError:
             raise POSKeyError(oid)
+        except TypeError:
+            raise TypeError, 'invalid oid %r' % (oid,)
         file.seek(pos)
         read=file.read
         h=read(DATA_HDR_LEN)
@@ -617,6 +618,8 @@
             pos = _index[oid]
         except KeyError:
             raise POSKeyError(oid)
+        except TypeError:
+            raise TypeError, 'invalid oid %r' % (oid,)
         file.seek(pos)
         read = file.read
         h = read(DATA_HDR_LEN)
@@ -656,6 +659,8 @@
                 pos = self._index[oid]
             except KeyError:
                 raise POSKeyError(oid)
+            except TypeError:
+                raise TypeError, 'invalid oid %r' % (oid,)
             while 1:
                 seek(pos)
                 h=read(DATA_HDR_LEN)
@@ -687,6 +692,8 @@

[-=- -=- -=- 415 lines omitted -=- -=- -=-]

-
-            # OK, we're beyond the point of no return
-            os.rename(name+'.pack', name)
-            self._file=open(name,'r+b')
-            self._initIndex(index, vindex, tindex, tvindex)
-            self._pos=opos
-            self._save_index()
+                self._file.close()
+                try:
+                    if os.path.exists(oldpath):
+                        os.remove(oldpath)
+                    os.rename(self._file_name, oldpath)
+                except Exception, msg:
+                    self._file = open(self._file_name, 'r+b')
+                    raise
 
+                # OK, we're beyond the point of no return
+                os.rename(self._file_name + '.pack', self._file_name)
+                self._file = open(self._file_name, 'r+b')
+                self._initIndex(p.index, p.vindex, p.tindex, p.tvindex)
+                self._pos = opos
+                self._save_index()
+            finally:
+                self._lock_release()
         finally:
-
-            if locked:
-                _commit_lock_release()
-                _lock_release()
-
-            _lock_acquire()
-            self._packt=z64
-            _lock_release()
+            if p.locked:
+                self._commit_lock_release()
+            self._lock_acquire()
+            self._packt = z64
+            self._lock_release()
 
     def iterator(self, start=None, stop=None):
         return FileIterator(self._file_name, start, stop)
@@ -1826,6 +1521,8 @@
             pos = self._index[oid]
         except KeyError:
             return None
+        except TypeError:
+            raise TypeError, 'invalid oid %r' % (oid,)
         self._file.seek(pos)
         # first 8 bytes are oid, second 8 bytes are serialno
         h = self._file.read(16)


=== ZODB3/ZODB/cPersistence.c 1.69 => 1.69.2.1 ===
--- ZODB3/ZODB/cPersistence.c:1.69	Wed Apr 23 16:05:51 2003
+++ ZODB3/ZODB/cPersistence.c	Thu May 29 12:30:51 2003
@@ -192,7 +192,13 @@
     self->ring.prev = NULL;
     self->ring.next = NULL;
     self->state = cPersistent_GHOST_STATE;
-    /* self->ring held a reference to the object. */
+
+    /* We remove the reference to the just ghosted object that the ring
+     * holds.  Note that the dictionary of oids->objects has an uncounted
+     * reference, so if the ring's reference was the only one, this frees
+     * the ghost object.  Note further that the object's dealloc knows to
+     * inform the dictionary that it is going away.
+     */
     Py_DECREF(self);
 }
 
@@ -665,8 +671,10 @@
 	      Py_DECREF(meth);
 	      return 0;
 	    }
-	  if (PyObject_IsTrue(v)) return changed(self);
-	  if (self->state >= 0) self->state=cPersistent_UPTODATE_STATE;
+	  if (PyObject_IsTrue(v)) 
+	      return changed(self);
+	  if (self->state >= 0) 
+	      self->state=cPersistent_UPTODATE_STATE;
 	  return 0;
 	}
     }


=== ZODB3/ZODB/cPickleCache.c 1.82 => 1.82.2.1 ===
--- ZODB3/ZODB/cPickleCache.c:1.82	Wed Apr 23 16:05:51 2003
+++ ZODB3/ZODB/cPickleCache.c	Thu May 29 12:30:51 2003
@@ -536,8 +536,6 @@
        call above creates artificial references to v.
     */
     _Py_RefTotal--;
-    /* XXX it may be a problem that v->ob_type is still NULL? 
-       I don't understand what this comment means.  --jeremy */
     assert(v->ob_type);
 #else
     Py_INCREF(v);
@@ -628,8 +626,6 @@
 static PyObject *
 cc_getattr(ccobject *self, char *name)
 {
-  PyObject *r;
-
   if(*name=='c')
     {
       if(strcmp(name,"cache_age")==0)
@@ -648,16 +644,10 @@
 	  return PyDict_Copy(self->data);
 	}
     }
-  if((*name=='h' && strcmp(name, "has_key")==0) ||
-     (*name=='i' && strcmp(name, "items")==0) ||
-     (*name=='k' && strcmp(name, "keys")==0)
-     )
-    return PyObject_GetAttrString(self->data, name);
-
-  if((r=Py_FindMethod(cc_methods, (PyObject *)self, name)))
-    return r;
-  PyErr_Clear();
-  return PyObject_GetAttrString(self->data, name);
+
+  if (strcmp(name, "items") == 0)
+      return PyObject_GetAttrString(self->data, name);
+  return Py_FindMethod(cc_methods, (PyObject *)self, name);
 }
 
 static int


=== ZODB3/ZODB/component.xml 1.7.8.1 => 1.7.8.2 ===
--- ZODB3/ZODB/component.xml:1.7.8.1	Wed May 28 14:37:30 2003
+++ ZODB3/ZODB/component.xml	Thu May 29 12:30:51 2003
@@ -66,15 +66,62 @@
                implements="ZODB.storage">
     <multikey name="server" datatype="socket-address" required="yes"/>
     <key name="storage" default="1"/>
+      <description>
+        The name of the storage that the client wants to use.  If the
+        ZEO server serves more than one storage, the client selects
+        the storage it wants to use by name.  The default name is '1',
+        which is also the default name for the ZEO server.
+      </description>
     <key name="cache-size" datatype="integer" default="20000000"/>
+      <description>
+        The maximum size of the client cache, in bytes.
+      </description>
     <key name="name" default=""/>
+      <description>
+        The storage name.  If unspecified, the address of the server
+        will be used as the name.
+      </description>
     <key name="client"/>
+      <description>
+        Enables persistent cache files.  The string passed here is
+        used to construct the cache filenames.  If it is not
+        specified, the client creates a temporary cache that will
+        only be used by the current object.
+      </description>
     <key name="var"/>
+      <description>
+        The directory where persistent cache files are stored.  By
+        default cache files, if they are persistent, are stored in 
+        the current directory.
+      </description>
     <key name="min-disconnect-poll" datatype="integer" default="5"/>
+      <description>
+        The minimum delay in seconds between attempts to connect to
+        the server, in seconds.  Defaults to 5 seconds.
+      </description>
     <key name="max-disconnect-poll" datatype="integer" default="300"/>
+      <description>
+        The maximum delay in seconds between attempts to connect to
+        the server, in seconds.  Defaults to 300 seconds.
+      </description>
     <key name="wait" datatype="boolean" default="on"/>
+      <description>
+        A boolean indicating whether the constructor should wait
+        for the client to connect to the server and verify the cache
+        before returning.  The default is true.
+      </description>
     <key name="read-only" datatype="boolean" default="off"/>
+      <description>
+        A flag indicating whether this should be a read-only storage,
+        defaulting to false (i.e. writing is allowed by default).
+      </description>
     <key name="read-only-fallback" datatype="boolean" default="off"/>
+      <description>
+        A flag indicating whether a read-only remote storage should be
+        acceptable as a fallback when no writable storages are
+        available.  Defaults to false.  At most one of read_only and
+        read_only_fallback should be true.
+      </description>
     <key name="realm" required="no">
       <description>
         The authentication realm of the server.  Some authentication
@@ -82,7 +129,6 @@
         that are accepted by this server.
       </description>
     </key>
-
   </sectiontype>
 
   <sectiontype name="demostorage" datatype=".DemoStorage"


=== ZODB3/ZODB/config.py 1.10 => 1.10.12.1 ===
--- ZODB3/ZODB/config.py:1.10	Tue Jan 28 18:15:56 2003
+++ ZODB3/ZODB/config.py	Thu May 29 12:30:51 2003
@@ -22,27 +22,50 @@
 
 import ZODB
 
-schema_path = os.path.join(ZODB.__path__[0], "config.xml")
-_schema = None
+db_schema_path = os.path.join(ZODB.__path__[0], "config.xml")
+_db_schema = None
 
-def getSchema():
-    global _schema
-    if _schema is None:
-        _schema = ZConfig.loadSchema(schema_path)
-    return _schema
+s_schema_path = os.path.join(ZODB.__path__[0], "storage.xml")
+_s_schema = None
+
+def getDbSchema():
+    global _db_schema
+    if _db_schema is None:
+        _db_schema = ZConfig.loadSchema(db_schema_path)
+    return _db_schema
+
+def getStorageSchema():
+    global _s_schema
+    if _s_schema is None:
+        _s_schema = ZConfig.loadSchema(s_schema_path)
+    return _s_schema
 
 def databaseFromString(s):
     return databaseFromFile(StringIO.StringIO(s))
 
 def databaseFromFile(f):
-    config, handle = ZConfig.loadConfigFile(getSchema(), f)
+    config, handle = ZConfig.loadConfigFile(getDbSchema(), f)
     return databaseFromConfig(config.database)
 
 def databaseFromURL(url):
-    config, handler = ZConfig.loadConfig(getSchema(), url)
+    config, handler = ZConfig.loadConfig(getDbSchema(), url)
     return databaseFromConfig(config.database)
 
 def databaseFromConfig(section):
+    return section.open()
+
+def storageFromString(s):
+    return storageFromFile(StringIO.StringIO(s))
+
+def storageFromFile(f):
+    config, handle = ZConfig.loadConfigFile(getStorageSchema(), f)
+    return storageFromConfig(config.storage)
+
+def storageFromURL(url):
+    config, handler = ZConfig.loadConfig(getStorageSchema(), url)
+    return storageFromConfig(config.storage)
+
+def storageFromConfig(section):
     return section.open()
 
 


=== ZODB3/ZODB/fsdump.py 1.6 => 1.6.30.1 ===
--- ZODB3/ZODB/fsdump.py:1.6	Tue Dec  3 13:46:36 2002
+++ ZODB3/ZODB/fsdump.py	Thu May 29 12:30:51 2003
@@ -126,7 +126,7 @@
             self.dump_data(pos)
         stlen2 = self.file.read(8)
         print >> self.dest, "redundant trec len: %d" % u64(stlen2)
-        return True
+        return 1
 
     def dump_data(self, tloc):
         pos = self.file.tell()


=== ZODB3/ZODB/fsrecover.py 1.6 => 1.6.8.1 ===
--- ZODB3/ZODB/fsrecover.py:1.6	Mon Mar 17 14:26:09 2003
+++ ZODB3/ZODB/fsrecover.py	Thu May 29 12:30:51 2003
@@ -11,8 +11,6 @@
 # FOR A PARTICULAR PURPOSE
 #
 ##############################################################################
-
-
 """Simple script for repairing damaged FileStorage files.
 
 Usage: %s [-f] input output
@@ -84,220 +82,272 @@
 
 import getopt, ZODB.FileStorage, struct, time
 from struct import unpack
-from ZODB.utils import t32, p64, U64
+from ZODB.utils import t32, p64, u64
 from ZODB.TimeStamp import TimeStamp
 from cPickle import loads
 from ZODB.FileStorage import RecordIterator
 
-class EOF(Exception): pass
 class ErrorFound(Exception): pass
 
 def error(mess, *args):
     raise ErrorFound(mess % args)
 
-def read_transaction_header(file, pos, file_size):
+def read_txn_header(f, pos, file_size, outp, ltid):
     # Read the transaction record
-    seek=file.seek
-    read=file.read
-
-    seek(pos)
-    h=read(23)
-    if len(h) < 23: raise EOF
+    f.seek(pos)
+    h = f.read(23)
+    if len(h) < 23:
+        raise EOFError
 
     tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
     if el < 0: el=t32-el
 
-    tl=U64(stl)
-
-    if status=='c': raise EOF
+    tl = u64(stl)
 
-    if pos+(tl+8) > file_size:
+    if pos + (tl + 8) > file_size:
         error("bad transaction length at %s", pos)
 
-    if status not in ' up':
-        error('invalid status, %s, at %s', status, pos)
+    if tl < (23 + ul + dl + el):
+        error("invalid transaction length, %s, at %s", tl, pos)
 
-    if tl < (23+ul+dl+el):
-        error('invalid transaction length, %s, at %s', tl, pos)
+    if ltid and tid < ltid:
+        error("time-stamp reducation %s < %s, at %s", u64(tid), u64(ltid), pos)
 
-    tpos=pos
-    tend=tpos+tl
+    if status == "c":
+        truncate(f, pos, file_size, output)
+        raise EOFError
 
-    if status=='u':
+    if status not in " up":
+        error("invalid status, %r, at %s", status, pos)
+
+    tpos = pos
+    tend = tpos + tl
+
+    if status == "u":
         # Undone transaction, skip it
-        seek(tend)
-        h=read(8)
-        if h != stl: error('inconsistent transaction length at %s', pos)
-        pos=tend+8
-        return pos, None
-
-    pos=tpos+(23+ul+dl+el)
-    user=read(ul)
-    description=read(dl)
+        f.seek(tend)
+        h = f.read(8)
+        if h != stl:
+            error("inconsistent transaction length at %s", pos)
+        pos = tend + 8
+        return pos, None, tid
+
+    pos = tpos+(23+ul+dl+el)
+    user = f.read(ul)
+    description = f.read(dl)
     if el:
-        try: e=loads(read(el))
+        try: e=loads(f.read(el))
         except: e={}
     else: e={}
 
     result = RecordIterator(tid, status, user, description, e, pos, tend,
-                            file, tpos)
-    pos=tend
+                            f, tpos)
+    pos = tend
 
     # Read the (intentionally redundant) transaction length
-    seek(pos)
-    h=read(8)
+    f.seek(pos)
+    h = f.read(8)
     if h != stl:
         error("redundant transaction length check failed at %s", pos)
-    pos=pos+8
+    pos += 8
 
-    return pos, result
+    return pos, result, tid
 
-def scan(file, pos, file_size):
-    seek=file.seek
-    read=file.read
+def truncate(f, pos, file_size, outp):
+    """Copy data from pos to end of f to a .trNNN file."""
+
+    i = 0
     while 1:
-        seek(pos)
-        data=read(8096)
-        if not data: return 0
+        trname = outp + ".tr%d" % i
+        if os.path.exists(trname):
+            i += 1
+    tr = open(trname, "wb")
+    copy(f, tr, file_size - pos)
+    f.seek(pos)
+    tr.close()
+
+def copy(src, dst, n):
+    while n:
+        buf = src.read(8096)
+        if not buf:
+            break
+        if len(buf) > n:
+            buf = buf[:n]
+        dst.write(buf)
+        n -= len(buf)
+
+def scan(f, pos):
+    """Return a potential transaction location following pos in f.
+
+    This routine scans forward from pos looking for the last data
+    record in a transaction.  A period '.' always occurs at the end of
+    a pickle, and an 8-byte transaction length follows the last
+    pickle.  If a period is followed by a plausible 8-byte transaction
+    length, assume that we have found the end of a transaction.
+
+    The caller should try to verify that the returned location is
+    actually a transaction header.
+    """
+    while 1:
+        f.seek(pos)
+        data = f.read(8096)
+        if not data:
+            return 0
 
-        s=0
+        s = 0
         while 1:
-            l=data.find('.', s)
+            l = data.find(".", s)
             if l < 0:
-                pos=pos+8096
+                pos += len(data)
                 break
-            if l > 8080:
-                pos = pos + l
+            # If we are less than 8 bytes from the end of the
+            # string, we need to read more data.
+            s = l + 1
+            if s > len(data) - 8:
+                pos += l
                 break
-            s=l+1
-            tl=U64(data[s:s+8])
+            tl = u64(data[s:s+8])
             if tl < pos:
                 return pos + s + 8
 
 def iprogress(i):
-    if i%2: print '.',
-    else: print (i/2)%10,
+    if i % 2:
+        print ".",
+    else:
+        print (i/2) % 10,
     sys.stdout.flush()
 
 def progress(p):
-    for i in range(p): iprogress(i)
-
-def recover(argv=sys.argv):
+    for i in range(p):
+        iprogress(i)
 
+def main():
     try:
-        opts, (inp, outp) = getopt.getopt(argv[1:], 'fv:pP:')
-        force = partial = verbose = 0
-        pack = None
-        for opt, v in opts:
-            if opt == '-v': verbose = int(v)
-            elif opt == '-p': partial=1
-            elif opt == '-f': force=1
-            elif opt == '-P': pack=time.time()-float(v)
-
-
-        force = filter(lambda opt: opt[0]=='-f', opts)
-        partial = filter(lambda opt: opt[0]=='-p', opts)
-        verbose = filter(lambda opt: opt[0]=='-v', opts)
-        verbose = verbose and int(verbose[0][1]) or 0
-        print 'Recovering', inp, 'into', outp
-    except:
+        opts, (inp, outp) = getopt.getopt(sys.argv[1:], "fv:pP:")
+    except getopt.error:
         die()
         print __doc__ % argv[0]
+        
+    force = partial = verbose = 0
+    pack = None
+    for opt, v in opts:
+        if opt == "-v":
+            verbose = int(v)
+        elif opt == "-p":
+            partial = 1
+        elif opt == "-f":
+            force = 1
+        elif opt == "-P":
+            pack = time.time() - float(v)
 
+    recover(inp, outp, verbose, partial, force, pack)
+
+def recover(inp, outp, verbose=0, partial=0, force=0, pack=0):
+    print "Recovering", inp, "into", outp
 
     if os.path.exists(outp) and not force:
         die("%s exists" % outp)
 
-    file=open(inp, "rb")
-    seek=file.seek
-    read=file.read
-    if read(4) != ZODB.FileStorage.packed_version:
+    f = open(inp, "rb")
+    if f.read(4) != ZODB.FileStorage.packed_version:
         die("input is not a file storage")
 
-    seek(0,2)
-    file_size=file.tell()
+    f.seek(0,2)
+    file_size = f.tell()
 
-    ofs=ZODB.FileStorage.FileStorage(outp, create=1)
-    _ts=None
-    ok=1
-    prog1=0
-    preindex={}; preget=preindex.get   # waaaa
-    undone=0
+    ofs = ZODB.FileStorage.FileStorage(outp, create=1)
+    _ts = None
+    ok = 1
+    prog1 = 0
+    undone = 0
 
-    pos=4
+    pos = 4
+    ltid = None
     while pos:
-
         try:
-            npos, transaction = read_transaction_header(file, pos, file_size)
-        except EOF:
+            npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
+        except EOFError:
             break
-        except:
-            print "\n%s: %s\n" % sys.exc_info()[:2]
-            if not verbose: progress(prog1)
-            pos = scan(file, pos, file_size)
+        except (KeyboardInterrupt, SystemExit):
+            raise
+        except Exception, err:
+            print "error reading txn header:", err
+            if not verbose:
+                progress(prog1)
+            pos = scan(f, pos)
+            if verbose > 1:
+                print "looking for valid txn header at", pos
             continue
+        ltid = tid
 
-        if transaction is None:
+        if txn is None:
             undone = undone + npos - pos
-            pos=npos
+            pos = npos
             continue
         else:
-            pos=npos
+            pos = npos
 
-        tid=transaction.tid
+        tid = txn.tid
 
         if _ts is None:
-            _ts=TimeStamp(tid)
+            _ts = TimeStamp(tid)
         else:
-            t=TimeStamp(tid)
+            t = TimeStamp(tid)
             if t <= _ts:
-                if ok: print ('Time stamps out of order %s, %s' % (_ts, t))
-                ok=0
-                _ts=t.laterThan(_ts)
-                tid=`_ts`
+                if ok:
+                    print ("Time stamps out of order %s, %s" % (_ts, t))
+                ok = 0
+                _ts = t.laterThan(_ts)
+                tid = `_ts`
             else:
                 _ts = t
                 if not ok:
-                    print ('Time stamps back in order %s' % (t))
-                    ok=1
-
-        if verbose:
-            print 'begin',
-            if verbose > 1: print
-            sys.stdout.flush()
+                    print ("Time stamps back in order %s" % (t))
+                    ok = 1
 
-        ofs.tpc_begin(transaction, tid, transaction.status)
+        ofs.tpc_begin(txn, tid, txn.status)
 
         if verbose:
-            print 'begin', pos, _ts,
-            if verbose > 1: print
+            print "begin", pos, _ts,
+            if verbose > 1:
+                print
             sys.stdout.flush()
 
-        nrec=0
+        nrec = 0
         try:
-            for r in transaction:
-                oid=r.oid
-                if verbose > 1: print U64(oid), r.version, len(r.data)
-                pre=preget(oid, None)
-                s=ofs.store(oid, pre, r.data, r.version, transaction)
-                preindex[oid]=s
-                nrec=nrec+1
-        except:
+            for r in txn:
+                if verbose > 1:
+                    if r.data is None:
+                        l = "bp"
+                    else:
+                        l = len(r.data)
+                        
+                    print "%7d %s %s" % (u64(r.oid), l, r.version)
+                s = ofs.restore(r.oid, r.serial, r.data, r.version,
+                                r.data_txn, txn)
+                nrec += 1
+        except (KeyboardInterrupt, SystemExit):
+            raise
+        except Exception, err:
             if partial and nrec:
-                ofs._status='p'
-                ofs.tpc_vote(transaction)
-                ofs.tpc_finish(transaction)
-                if verbose: print 'partial'
+                ofs._status = "p"
+                ofs.tpc_vote(txn)
+                ofs.tpc_finish(txn)
+                if verbose:
+                    print "partial"
             else:
-                ofs.tpc_abort(transaction)
-            print "\n%s: %s\n" % sys.exc_info()[:2]
-            if not verbose: progress(prog1)
-            pos = scan(file, pos, file_size)
+                ofs.tpc_abort(txn)
+            print "error copying transaction:", err
+            if not verbose:
+                progress(prog1)
+            pos = scan(f, pos)
+            if verbose > 1:
+                print "looking for valid txn header at", pos
         else:
-            ofs.tpc_vote(transaction)
-            ofs.tpc_finish(transaction)
+            ofs.tpc_vote(txn)
+            ofs.tpc_finish(txn)
             if verbose:
-                print 'finish'
+                print "finish"
                 sys.stdout.flush()
 
         if not verbose:
@@ -320,5 +370,6 @@
 
     ofs.close()
 
+if __name__ == "__main__":
+    main()
 
-if __name__=='__main__': recover()

=== Removed File ZODB3/ZODB/StorageConfig.py ===

=== Removed File ZODB3/ZODB/StorageTypes.py ===




More information about the Zodb-checkins mailing list