[Zodb-checkins] CVS: ZODB4/src/zodb/storage/file - recover.py:1.2 main.py:1.7 __init__.py:1.4

Jeremy Hylton jeremy at zope.com
Thu Jun 19 18:41:40 EDT 2003


Update of /cvs-repository/ZODB4/src/zodb/storage/file
In directory cvs.zope.org:/tmp/cvs-serv15960/src/zodb/storage/file

Modified Files:
	main.py __init__.py 
Added Files:
	recover.py 
Log Message:
Merge ZODB3-2-merge branch to the head.

This completes the porting of bug fixes and random improvements from
ZODB 3.2 to ZODB 4.


=== ZODB4/src/zodb/storage/file/recover.py 1.1 => 1.2 ===
--- /dev/null	Thu Jun 19 17:41:40 2003
+++ ZODB4/src/zodb/storage/file/recover.py	Thu Jun 19 17:41:09 2003
@@ -0,0 +1,370 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Simple script for repairing damaged FileStorage files.
+
+Usage: %s [-f] input output
+
+Recover data from a FileStorage data file, skipping over damaged
+data. Any damaged data will be lost. This could lead to useless output
+of critical data were lost.
+
+Options:
+
+    -f
+       Force output to output file even if it exists
+
+    -v level
+
+       Set the verbosity level:
+
+         0 -- Show progress indicator (default)
+
+         1 -- Show transaction times and sizes
+
+         2 -- Show transaction times and sizes, and
+              show object (record) ids, versions, and sizes.
+
+    -p
+
+       Copy partial transactions. If a data record in the middle of a
+       transaction is bad, the data up to the bad data are packed. The
+       output record is marked as packed. If this option is not used,
+       transaction with any bad data are skipped.
+
+    -P t
+
+       Pack data to t seconds in the past. Note that is the "-p"
+       option is used, then t should be 0.
+
+
+Important note: The ZODB package must be imporatble.  You may need
+                to adjust PYTHONPATH accordingly.
+
+"""
+
+# Algorithm:
+#
+#     position to start of input
+#     while 1:
+#         if end of file: break
+#          try: copy_transaction
+#          except:
+#                 scan for transaction
+#                 continue
+
+import sys, os
+import time
+import struct
+import getopt
+
+from struct import unpack
+from cPickle import loads
+
+if __name__ == '__main__' and len(sys.argv) < 3:
+    print __doc__ % sys.argv[0]
+
+def die(mess=''):
+    if not mess: mess="%s: %s" % sys.exc_info()[:2]
+    print mess+'\n'
+    sys.exit(1)
+
+from zodb.storage.file.main import FileStorage, RecordIterator
+from zodb.storage.file.format import FileStorageFormatter
+from zodb.timestamp import TimeStamp
+from zodb.utils import p64, u64
+
+magic = "FS" + FileStorageFormatter._format_version 
+
+class ErrorFound(Exception): pass
+
+def error(mess, *args):
+    raise ErrorFound(mess % args)
+
+def read_txn_header(f, pos, file_size, outp, ltid):
+    # Read the transaction record
+    f.seek(pos)
+    h = f.read(23)
+    if len(h) < 23:
+        raise EOFError
+
+    tid, tl, status, ul, dl, el = unpack(">QQcHHH",h)
+
+    if pos + (tl + 8) > file_size:
+        error("bad transaction length at %s", pos)
+
+    if tl < (23 + ul + dl + el):
+        error("invalid transaction length, %s, at %s", tl, pos)
+
+    if ltid and tid < ltid:
+        error("time-stamp reducation %s < %s, at %s", u64(tid), u64(ltid), pos)
+
+    if status == "c":
+        truncate(f, pos, file_size, output)
+        raise EOFError
+
+    if status not in " up":
+        error("invalid status, %r, at %s", status, pos)
+
+    tpos = pos
+    tend = tpos + tl
+
+    if status == "u":
+        # Undone transaction, skip it
+        f.seek(tend)
+        rtl = unpack(">Q", f.read(8))[0]
+        if rtl != stl:
+            error("inconsistent transaction length at %s", pos)
+        pos = tend + 8
+        return pos, None, tid
+
+    pos = tpos+(23+ul+dl+el)
+    user = f.read(ul)
+    description = f.read(dl)
+    if el:
+        try: e=loads(f.read(el))
+        except: e={}
+    else: e={}
+
+    result = RecordIterator(tid, status, user, description, e, pos, tend,
+                            f, tpos)
+    pos = tend
+
+    # Read the (intentionally redundant) transaction length
+    f.seek(pos)
+    h = f.read(8)
+    if h != stl:
+        error("redundant transaction length check failed at %s", pos)
+    pos += 8
+
+    return pos, result, tid
+
+def truncate(f, pos, file_size, outp):
+    """Copy data from pos to end of f to a .trNNN file."""
+
+    i = 0
+    while 1:
+        trname = outp + ".tr%d" % i
+        if os.path.exists(trname):
+            i += 1
+    tr = open(trname, "wb")
+    copy(f, tr, file_size - pos)
+    f.seek(pos)
+    tr.close()
+
+def copy(src, dst, n):
+    while n:
+        buf = src.read(8096)
+        if not buf:
+            break
+        if len(buf) > n:
+            buf = buf[:n]
+        dst.write(buf)
+        n -= len(buf)
+
+def scan(f, pos):
+    """Return a potential transaction location following pos in f.
+
+    This routine scans forward from pos looking for the last data
+    record in a transaction.  A period '.' always occurs at the end of
+    a pickle, and an 8-byte transaction length follows the last
+    pickle.  If a period is followed by a plausible 8-byte transaction
+    length, assume that we have found the end of a transaction.
+
+    The caller should try to verify that the returned location is
+    actually a transaction header.
+    """
+    while 1:
+        f.seek(pos)
+        data = f.read(8096)
+        if not data:
+            return 0
+
+        s = 0
+        while 1:
+            l = data.find(".", s)
+            if l < 0:
+                pos += len(data)
+                break
+            # If we are less than 8 bytes from the end of the
+            # string, we need to read more data.
+            s = l + 1
+            if s > len(data) - 8:
+                pos += l
+                break
+            tl = u64(data[s:s+8])
+            if tl < pos:
+                return pos + s + 8
+
+def iprogress(i):
+    if i % 2:
+        print ".",
+    else:
+        print (i/2) % 10,
+    sys.stdout.flush()
+
+def progress(p):
+    for i in range(p):
+        iprogress(i)
+
+def main():
+    try:
+        opts, (inp, outp) = getopt.getopt(sys.argv[1:], "fv:pP:")
+    except getopt.error:
+        die()
+        print __doc__ % argv[0]
+        
+    force = partial = verbose = 0
+    pack = None
+    for opt, v in opts:
+        if opt == "-v":
+            verbose = int(v)
+        elif opt == "-p":
+            partial = 1
+        elif opt == "-f":
+            force = 1
+        elif opt == "-P":
+            pack = time.time() - float(v)
+
+    recover(inp, outp, verbose, partial, force, pack)
+
+def recover(inp, outp, verbose=0, partial=0, force=0, pack=0):
+    print "Recovering", inp, "into", outp
+
+    if os.path.exists(outp) and not force:
+        die("%s exists" % outp)
+
+    f = open(inp, "rb")
+    if f.read(4) != magic:
+        die("input is not a file storage")
+
+    f.seek(0,2)
+    file_size = f.tell()
+
+    ofs = FileStorage(outp, create=1)
+    _ts = None
+    ok = 1
+    prog1 = 0
+    undone = 0
+
+    pos = 4L
+    ltid = None
+    while pos:
+        try:
+            npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
+        except EOFError:
+            break
+        except (KeyboardInterrupt, SystemExit):
+            raise
+        except Exception, err:
+            print "error reading txn header:", err
+            if not verbose:
+                progress(prog1)
+            pos = scan(f, pos)
+            if verbose > 1:
+                print "looking for valid txn header at", pos
+            continue
+        ltid = tid
+
+        if txn is None:
+            undone = undone + npos - pos
+            pos = npos
+            continue
+        else:
+            pos = npos
+
+        tid = txn.tid
+
+        if _ts is None:
+            _ts = TimeStamp(tid)
+        else:
+            t = TimeStamp(tid)
+            if t <= _ts:
+                if ok:
+                    print ("Time stamps out of order %s, %s" % (_ts, t))
+                ok = 0
+                _ts = t.laterThan(_ts)
+                tid = `_ts`
+            else:
+                _ts = t
+                if not ok:
+                    print ("Time stamps back in order %s" % (t))
+                    ok = 1
+
+        ofs.tpc_begin(txn, tid, txn.status)
+
+        if verbose:
+            print "begin", pos, _ts,
+            if verbose > 1:
+                print
+            sys.stdout.flush()
+
+        nrec = 0
+        try:
+            for r in txn:
+                if verbose > 1:
+                    if r.data is None:
+                        l = "bp"
+                    else:
+                        l = len(r.data)
+                        
+                    print "%7d %s %s" % (u64(r.oid), l, r.version)
+                s = ofs.restore(r.oid, r.serial, r.data, r.version,
+                                r.data_txn, txn)
+                nrec += 1
+        except (KeyboardInterrupt, SystemExit):
+            raise
+        except Exception, err:
+            if partial and nrec:
+                ofs._status = "p"
+                ofs.tpc_vote(txn)
+                ofs.tpc_finish(txn)
+                if verbose:
+                    print "partial"
+            else:
+                ofs.tpc_abort(txn)
+            print "error copying transaction:", err
+            if not verbose:
+                progress(prog1)
+            pos = scan(f, pos)
+            if verbose > 1:
+                print "looking for valid txn header at", pos
+        else:
+            ofs.tpc_vote(txn)
+            ofs.tpc_finish(txn)
+            if verbose:
+                print "finish"
+                sys.stdout.flush()
+
+        if not verbose:
+            prog = pos * 20l / file_size
+            while prog > prog1:
+                prog1 = prog1 + 1
+                iprogress(prog1)
+
+
+    bad = file_size - undone - ofs._pos
+
+    print "\n%s bytes removed during recovery" % bad
+    if undone:
+        print "%s bytes of undone transaction data were skipped" % undone
+
+    if pack is not None:
+        print "Packing ..."
+        ofs.pack(pack)
+
+    ofs.close()
+
+if __name__ == "__main__":
+    main()
+


=== ZODB4/src/zodb/storage/file/main.py 1.6 => 1.7 ===
--- ZODB4/src/zodb/storage/file/main.py:1.6	Fri Jun  6 11:24:21 2003
+++ ZODB4/src/zodb/storage/file/main.py	Thu Jun 19 17:41:09 2003
@@ -387,7 +387,7 @@
             self._file.read(h.nrefs * 8)
             if h.plen:
                 return self._file.read(h.plen)
-            return self._loadBack(oid, h.pnv)[0]
+            return self._loadBack(oid, h.back)[0]
         finally:
             self._lock_release()
 


=== ZODB4/src/zodb/storage/file/__init__.py 1.3 => 1.4 ===
--- ZODB4/src/zodb/storage/file/__init__.py:1.3	Fri Apr 25 15:28:46 2003
+++ ZODB4/src/zodb/storage/file/__init__.py	Thu Jun 19 17:41:09 2003
@@ -11,4 +11,4 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
-from zodb.storage.file.main import FileStorage
+from zodb.storage.file.main import FileStorage, FileIterator




More information about the Zodb-checkins mailing list