[Zodb-checkins] SVN: ZODB/branches/zcZODB-3.8/src/Z Merged Dieter's branch that ads an option to drop a zeo cache rather

Jim Fulton jim at zope.com
Fri Sep 26 11:26:36 EDT 2008


Log message for revision 91524:
  Merged Dieter's branch that ads an option to drop a zeo cache rather
  than verifying it. (91047 from trunk)
  

Changed:
  U   ZODB/branches/zcZODB-3.8/src/ZEO/ClientStorage.py
  U   ZODB/branches/zcZODB-3.8/src/ZEO/component.xml
  U   ZODB/branches/zcZODB-3.8/src/ZEO/tests/testZEO.py
  U   ZODB/branches/zcZODB-3.8/src/ZODB/component.xml
  U   ZODB/branches/zcZODB-3.8/src/ZODB/config.py

-=-
Modified: ZODB/branches/zcZODB-3.8/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZEO/ClientStorage.py	2008-09-26 15:08:13 UTC (rev 91523)
+++ ZODB/branches/zcZODB-3.8/src/ZEO/ClientStorage.py	2008-09-26 15:26:35 UTC (rev 91524)
@@ -114,6 +114,7 @@
                  wait_for_server_on_startup=None, # deprecated alias for wait
                  wait=None, wait_timeout=None,
                  read_only=0, read_only_fallback=0,
+                 drop_cache_rather_verify=False,
                  username='', password='', realm=None,
                  blob_dir=None, shared_blob_dir=False):
         """ClientStorage constructor.
@@ -188,6 +189,9 @@
 
         realm -- not documented.
 
+        drop_cache_rather_verify -- a flag indicating that the cache
+            should be dropped rather than expensively verified.
+
         blob_dir -- directory path for blob data.  'blob data' is data that
             is retrieved via the loadBlob API.
 
@@ -210,6 +214,14 @@
         if debug:
             log2("ClientStorage(): debug argument is no longer used")
 
+        # Remember some parameters for "_setupCache"
+        self._var_ = var
+        self._storage_ = storage
+        self._client_ = client
+        self._cache_size_ = cache_size
+
+        self._drop_cache_rather_verify = drop_cache_rather_verify
+
         # wait defaults to True, but wait_for_server_on_startup overrides
         # if not None
         if wait_for_server_on_startup is not None:
@@ -331,13 +343,7 @@
         else:
             self.fshelper = None
 
-        # Decide whether to use non-temporary files
-        if client is not None:
-            dir = var or os.getcwd()
-            cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
-        else:
-            cache_path = None
-        self._cache = self.ClientCacheClass(cache_path, size=cache_size)
+        self._setupCache()
 
         self._rpc_mgr = self.ConnectionManagerClass(addr, self,
                                                     tmin=min_disconnect_poll,
@@ -352,6 +358,19 @@
             if not self._rpc_mgr.attempt_connect():
                 self._rpc_mgr.connect()
 
+    def _setupCache(self):
+        '''create and open the cache.'''
+        # Decide whether to use non-temporary files
+        storage = self._storage_
+        client = self._client_
+        cache_size = self._cache_size_
+        if client is not None:
+            dir = self._var_ or os.getcwd()
+            cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
+        else:
+            cache_path = None
+        self._cache = self.ClientCacheClass(cache_path, size=cache_size)
+
     def _wait(self, timeout=None):
         if timeout is not None:
             deadline = time.time() + timeout
@@ -1231,6 +1250,23 @@
                 self.finish_verification(pair)
                 return "quick verification"
 
+        # From this point on, we do not have complete information about
+        # the missed transactions.  The reason is that cache
+        # verification only checks objects in the client cache and
+        # there may be objects in the object caches that aren't in the
+        # client cach that would need verification too. We avoid that
+        # problem by just invalidating the objects in the object caches.
+        if self._db is not None:
+            self._db.invalidateCache()
+
+        if self._cache and self._drop_cache_rather_verify:
+            log2("dropping cache")
+            self._cache.close()
+            self._setupCache() # creates a new cache
+            self._server = server
+            self._ready.set()
+            return "cache dropped"
+
         log2("Verifying cache")
 
         # TODO:  should batch these operations for efficiency; would need

Modified: ZODB/branches/zcZODB-3.8/src/ZEO/component.xml
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZEO/component.xml	2008-09-26 15:08:13 UTC (rev 91523)
+++ ZODB/branches/zcZODB-3.8/src/ZEO/component.xml	2008-09-26 15:26:35 UTC (rev 91524)
@@ -102,6 +102,16 @@
       <metadefault>$INSTANCE/var/ZEO.pid (or $clienthome/ZEO.pid)</metadefault>
     </key>
 
+    <!-- DM 2006-06-12: added option -->
+    <key name="drop-cache-rather-verify" datatype="boolean"
+         required="no" default="false">
+       <description>
+         indicates that the cache should be dropped rather than
+	 verified when the verification optimization is not
+	 available (e.g. when the ZEO server restarted).
+       </description>
+    </key>
+
   </sectiontype>
 
 </component>

Modified: ZODB/branches/zcZODB-3.8/src/ZEO/tests/testZEO.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZEO/tests/testZEO.py	2008-09-26 15:08:13 UTC (rev 91523)
+++ ZODB/branches/zcZODB-3.8/src/ZEO/tests/testZEO.py	2008-09-26 15:26:35 UTC (rev 91524)
@@ -59,6 +59,8 @@
 class DummyDB:
     def invalidate(self, *args):
         pass
+    def invalidateCache(*unused):
+        pass
 
 class OneTimeTests(unittest.TestCase):
 
@@ -127,6 +129,58 @@
         finally:
             storage2.close()
 
+    def checkDropCacheRatherVerifyImplementation(self):
+        # As it is quite difficult to set things up such that the verification
+        # optimizations do not step in, we emulate both the cache
+        # as well as the server.
+        from ZODB.TimeStamp import TimeStamp
+        class CacheEmulator(object):
+            # the settings below would be inconsitent for a normal cache
+            # but they are sufficient for our test setup
+            def __len__(self): return 1 # claim not to be empty
+            def contents(self): return () # do not invalidate anything
+            def getLastTid(self): return
+            def close(self): pass
+        class ServerEmulator(object):
+            def verify(*unused): pass
+            def endZeoVerify(*unused): pass
+            def lastTransaction(*unused): pass
+        storage = self._storage
+        storage._cache = cache = CacheEmulator()
+        server = ServerEmulator()
+        # test the standard behaviour
+        self.assertEqual(storage.verify_cache(server), "full verification")
+        # test the "drop cache rather verify" behaviour
+        storage._drop_cache_rather_verify = True
+        self.assertEqual(storage.verify_cache(server), "cache dropped")
+        # verify that we got a new cache
+        self.assert_(cache != storage._cache)
+
+
+class ConfigurationTests(unittest.TestCase):
+    def checkDropCacheRatherVerifyConfiguration(self):
+        from ZODB.config import storageFromString
+        # the default is to do verification and not drop the cache
+        cs = storageFromString('''
+        <zeoclient>
+          server localhost:9090
+          wait false
+        </zeoclient>
+        ''')
+        self.assertEqual(cs._drop_cache_rather_verify, False)
+        cs.close()
+        # now for dropping
+        cs = storageFromString('''
+        <zeoclient>
+          server localhost:9090
+          wait false
+          drop-cache-rather-verify true
+        </zeoclient>
+        ''')
+        self.assertEqual(cs._drop_cache_rather_verify, True)
+        cs.close()
+
+
 class GenericTests(
     # Base class for all ZODB tests
     StorageTestBase.StorageTestBase,
@@ -952,7 +1006,9 @@
     """
 
 test_classes = [FileStorageTests, MappingStorageTests, DemoStorageTests,
-                BlobAdaptedFileStorageTests, BlobWritableCacheTests]
+                BlobAdaptedFileStorageTests, BlobWritableCacheTests,
+                ConfigurationTests,
+                ]
 
 def test_suite():
     suite = unittest.TestSuite()

Modified: ZODB/branches/zcZODB-3.8/src/ZODB/component.xml
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZODB/component.xml	2008-09-26 15:08:13 UTC (rev 91523)
+++ ZODB/branches/zcZODB-3.8/src/ZODB/component.xml	2008-09-26 15:26:35 UTC (rev 91524)
@@ -163,6 +163,13 @@
         that are accepted by this server.
       </description>
     </key>
+    <!-- DM 2008-05-15: added -->
+    <key name="drop-cache-rather-verify" datatype="boolean" default="off">
+      <description>
+        A flag indicating whether the client cache should be dropped
+        instead of an expensive verification.
+      </description>
+    </key>
   </sectiontype>
 
   <sectiontype name="demostorage" datatype=".DemoStorage"

Modified: ZODB/branches/zcZODB-3.8/src/ZODB/config.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZODB/config.py	2008-09-26 15:08:13 UTC (rev 91523)
+++ ZODB/branches/zcZODB-3.8/src/ZODB/config.py	2008-09-26 15:26:35 UTC (rev 91524)
@@ -164,6 +164,7 @@
             wait=self.config.wait,
             read_only=self.config.read_only,
             read_only_fallback=self.config.read_only_fallback,
+            drop_cache_rather_verify=self.config.drop_cache_rather_verify,
             username=self.config.username,
             password=self.config.password,
             realm=self.config.realm)



More information about the Zodb-checkins mailing list