[Zope-CVS] CVS: Packages/PartitionedFileStorage - FileStorage_py_1_129.patch:1.1 PartitionedFile.py:1.1

Shane Hathaway shane@zope.com
Wed, 23 Apr 2003 14:25:41 -0400


Update of /cvs-repository/Packages/PartitionedFileStorage
In directory cvs.zope.org:/tmp/cvs-serv8282

Added Files:
	FileStorage_py_1_129.patch PartitionedFile.py 
Log Message:
Checked in latest version of PartitionedFileStorage.

=== Added File Packages/PartitionedFileStorage/FileStorage_py_1_129.patch ===
Index: FileStorage.py
===================================================================
RCS file: /cvs-repository/Zope/lib/python/ZODB/FileStorage.py,v
retrieving revision 1.129
diff -u -u -r1.129 FileStorage.py
--- FileStorage.py	17 Mar 2003 18:58:30 -0000	1.129
+++ FileStorage.py	23 Apr 2003 18:14:01 -0000
@@ -203,6 +203,43 @@
 
 packed_version='FS21'
 
+
+class FileOperations:
+    '''This class implements some basic operations on files.
+    PartitionedFileOperations provides an alternate implementation.
+    '''
+    def open(self, name, mode='r', bufsize=0):
+        global open
+        return open(name, mode, bufsize)
+
+    def exists(self, name):
+        return os.path.exists(name)
+
+    def rename(self, oldname, newname):
+        os.rename(oldname, newname)
+
+    def remove(self, name):
+        os.remove(name)
+
+    def fsync(self, file):
+        global fsync
+        if fsync is not None:
+            fsync(file.fileno())
+
+defaultFops = FileOperations()
+
+
+# Set the PARTITIONED_FILE_SIZE environment variable to use
+# PartitionedFile support.  Note: To convert a partitioned file back
+# to a normal file, just 'cat' (Unix) or 'copy' (DOS/Windows) the
+# partitions together in order.
+PARTITIONED_FILE_SIZE = long(os.environ.get('PARTITIONED_FILE_SIZE', 0))
+if PARTITIONED_FILE_SIZE > 0:
+    from PartitionedFile import PartitionedFileOperations
+    defaultFops = PartitionedFileOperations(partlen=PARTITIONED_FILE_SIZE)
+
+
+
 class FileStorage(BaseStorage.BaseStorage,
                   ConflictResolution.ConflictResolvingStorage):
     # default pack time is 0
@@ -211,7 +248,10 @@
     _records_before_save = 10000
 
     def __init__(self, file_name, create=0, read_only=0, stop=None,
-                 quota=None):
+                 quota=None, fops=None):
+        if fops is None:
+            fops = defaultFops
+        self.fops = fops
 
         if read_only:
             self._is_read_only = 1
@@ -243,7 +283,8 @@
         self._file = None
         if not create:
             try:
-                self._file = open(file_name, read_only and 'rb' or 'r+b')
+                self._file = self.fops.open(
+                    file_name, read_only and 'rb' or 'r+b')
             except IOError, exc:
                 if exc.errno == errno.EFBIG:
                     # The file is too big to open.  Fail visibly.
@@ -254,15 +295,15 @@
                 # If something else went wrong, it's hard to guess
                 # what the problem was.  If the file does not exist,
                 # create it.  Otherwise, fail.
-                if os.path.exists(file_name):
+                if self.fops.exists(file_name):
                     raise
                 else:
                     create = 1
 
         if self._file is None and create:
-            if os.path.exists(file_name):
-                os.remove(file_name)
-            self._file = open(file_name, 'w+b')
+            if self.fops.exists(file_name):
+                self.fops.remove(file_name)
+            self._file = self.fops.open(file_name, 'w+b')
             self._file.write(packed_version)
 
         r = self._restore_index()
@@ -1010,7 +1051,7 @@
             file.write(self._tstatus)
             file.flush()
 
-            if fsync is not None: fsync(file.fileno())
+            self.fops.fsync(file)
 
             self._pos = nextpos
             
@@ -1445,7 +1486,7 @@
         _commit_lock_release=self._commit_lock_release
         index, vindex, tindex, tvindex = self._newIndexes()
         name=self.__name__
-        file=open(name, 'rb')
+        file=self.fops.open(name, 'r+b')
         stop=`apply(TimeStamp, time.gmtime(t)[:5]+(t%60,))`
         if stop==z64: raise FileStorageError, 'Invalid pack time'
 
@@ -1502,7 +1543,7 @@
             # Step 2, copy data and compute new index based on new positions.
             index, vindex, tindex, tvindex = self._newIndexes()
 
-            ofile=open(name+'.pack', 'w+b')
+            ofile=self.fops.open(name+'.pack', 'w+b')
 
             # Index for non-version data.  This is a temporary structure
             # to reduce I/O during packing
@@ -1546,7 +1587,7 @@
                         # continuing
                         ofile.close()
                         file.close()
-                        os.remove(name+'.pack')
+                        self.fops.remove(name+'.pack')
                         return
 
                     packing=0
@@ -1784,17 +1825,17 @@
             file.close()
             self._file.close()
             try:
-                if os.path.exists(name+'.old'):
-                    os.remove(name+'.old')
-                os.rename(name, name+'.old')
+                if self.fops.exists(name+'.old'):
+                    self.fops.remove(name+'.old')
+                self.fops.rename(name, name+'.old')
             except:
                 # Waaa
-                self._file=open(name,'r+b')
+                self._file=self.fops.open(name, 'r+b')
                 raise
 
             # OK, we're beyond the point of no return
-            os.rename(name+'.pack', name)
-            self._file=open(name,'r+b')
+            self.fops.rename(name+'.pack', name)
+            self._file=self.fops.open(name, 'r+b')
             self._initIndex(index, vindex, tindex, tvindex)
             self._pos=opos
             self._save_index()
@@ -1975,8 +2016,10 @@
 
     return p, s
 
-def recover(file_name):
-    file=open(file_name, 'r+b')
+def recover(file_name, fops=None):
+    if fops is None:
+        fops = defaultFops
+    file=fops.open(file_name, 'r+b')
     index={}
     vindex={}
     tindex={}
@@ -2224,7 +2267,9 @@
     # seek to transaction header, where tid is first 8 bytes
     return file.read(8)
 
-def _truncate(file, name, pos):
+def _truncate(file, name, pos, fops=None):
+    if fops is None:
+        fops = defaultFops
     seek=file.seek
     seek(0,2)
     file_size=file.tell()
@@ -2232,11 +2277,11 @@
         i=0
         while 1:
             oname='%s.tr%s' % (name, i)
-            if os.path.exists(oname):
+            if fops.exists(oname):
                 i=i+1
             else:
                 warn("Writing truncated data from %s to %s", name, oname)
-                o=open(oname,'wb')
+                o = fops.open(oname, 'wb')
                 seek(pos)
                 cp(file, o, file_size-pos)
                 o.close()
@@ -2271,9 +2316,11 @@
     _ltid = z64
     _file = None
 
-    def __init__(self, file, start=None, stop=None):
+    def __init__(self, file, start=None, stop=None, fops=None):
+        if fops is None:
+            fops = defaultFops
         if isinstance(file, StringType):
-            file = open(file, 'rb')
+            file = fops.open(file, 'rb')
         self._file = file
         if file.read(4) != packed_version:
             raise FileStorageFormatError, file.name


=== Added File Packages/PartitionedFileStorage/PartitionedFile.py === (564/664 lines abridged)
##############################################################################
# 
# Zope Public License (ZPL) Version 1.0
# -------------------------------------
# 
# Copyright (c) Digital Creations.  All rights reserved.
# 
# This license has been certified as Open Source(tm).
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
# 
# 1. Redistributions in source code must retain the above copyright
#    notice, this list of conditions, and the following disclaimer.
# 
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions, and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
# 
# 3. Digital Creations requests that attribution be given to Zope
#    in any manner possible. Zope includes a "Powered by Zope"
#    button that is installed by default. While it is not a license
#    violation to remove this button, it is requested that the
#    attribution remain. A significant investment has been put
#    into Zope, and this effort will continue if the Zope community
#    continues to grow. This is one way to assure that growth.
# 
# 4. All advertising materials and documentation mentioning
#    features derived from or use of this software must display
#    the following acknowledgement:
# 
#      "This product includes software developed by Digital Creations
#      for use in the Z Object Publishing Environment
#      (http://www.zope.org/)."
# 
#    In the event that the product being advertised includes an
#    intact Zope distribution (with copyright and license included)
#    then this clause is waived.
# 
# 5. Names associated with Zope or Digital Creations must not be used to
#    endorse or promote products derived from this software without
#    prior written permission from Digital Creations.
# 
# 6. Modified redistributions of any form whatsoever must retain
#    the following acknowledgment:
# 
#      "This product includes software developed by Digital Creations
#      for use in the Z Object Publishing Environment

[-=- -=- -=- 564 lines omitted -=- -=- -=-]

            self.write(line)  # According to the spec, doesn't add a newline.

    def fsync(self):
        '''
        Requests an immediate flush of all data to physical media.
        '''
        if posixfsync is not None:
            state = self.__state
            for file in state.files:
                posixfsync(file.fileno())


if __name__ == '__main__':
    # Test the seek, write, and read functions.
    import whrandom
    generator = whrandom.whrandom()
    while 1:
        fops = PartitionedFileOperations(partlen=991)
        try: os.mkdir('test')
        except: pass

        # Notice how easy it is to open a PartitionedFile:
        file = fops.open('test/parts', 'wb')

        # Here's how you do the same thing without partitions:
        # file = open('test/parts', 'wb')

        # Randomly shove data in the file, storing the same
        # data in a string at the same time.  Then read
        # the file contents and see if it compares.
        # Do this repeatedly.
        MAXPOS = 12349
        compare = ''
        blank = '\0' * MAXPOS
        for n in range(500):
            s = ('%04d' % n) * 495
            location = int(generator.random() * MAXPOS)
            compare = (compare + blank)[:location] + s + \
                      compare[location + len(s):]
            file.seek(location)
            file.write(s)
        file.close()
        file = fops.open('test/parts', 'r')
        # file = open('test/parts', 'r')
        if file.read() == compare:
            print 'Passed.'
        else:
            print 'FAILED! ' * 10
        file.close()