[Zope-Checkins] CVS: Zope2 - testCatalog.py:1.1.2.19

Digicool CVS cvs@digicool.com
Wed, 7 Mar 2001 14:07:35 -0500


Update of /mnt/cvs-repository/Zope2/lib/python/Products/ZCatalog/tests
In directory yetix:/work/Zope2/CatalogUnit/lib/python/Products/ZCatalog/tests

Modified Files:
      Tag: Catalog-Unittest-Branch
	testCatalog.py 
Log Message:
Code cleanup



--- Updated File testCatalog.py in package Zope2 --
--- testCatalog.py	2001/03/07 12:46:32	1.1.2.18
+++ testCatalog.py	2001/03/07 14:07:51	1.1.2.19
@@ -7,6 +7,9 @@
     Andreas Jung, andreas@digicool.com
     
     $Log$
+    Revision 1.1.2.19  2001/03/07 14:07:51  andreas
+    Code cleanup
+
     Revision 1.1.2.18  2001/03/07 12:46:32  andreas
     added advanced tests
 
@@ -18,52 +21,6 @@
     - tests must now be started in the lib/python directory
     - older input sets are no longer valid (must be recreated)
 
-    Revision 1.1.2.15  2001/03/02 17:03:03  andreas
-    changed default settings
-
-    Revision 1.1.2.14  2001/03/02 15:16:47  andreas
-    version for release
-
-    Revision 1.1.2.13  2001/03/02 00:41:33  andreas
-    SHould now be a "final" version
-
-    Revision 1.1.2.12  2001/03/01 23:46:16  andreas
-    complete thread handling rewrite
-
-    Revision 1.1.2.11  2001/03/01 18:35:50  andreas
-    simple tests are now doing benchmarks
-
-    Revision 1.1.2.10  2001/02/28 20:23:23  andreas
-    minor changes
-
-    Revision 1.1.2.9  2001/02/28 18:39:20  andreas
-    misc changes
-
-    Revision 1.1.2.8  2001/02/28 16:51:32  andreas
-    added benchmarks
-
-    Revision 1.1.2.7  2001/02/28 16:02:15  andreas
-    fixed bug in generation of keywords index
-
-    Revision 1.1.2.6  2001/02/28 15:31:19  andreas
-    updated tests
-
-    Revision 1.1.2.5  2001/02/27 21:06:18  andreas
-    minor changes
-
-    Revision 1.1.2.4  2001/02/27 20:43:08  andreas
-    added -d option
-
-    Revision 1.1.2.3  2001/02/27 20:26:24  andreas
-    added prelimary stress test
-
-    Revision 1.1.2.2  2001/02/27 19:33:55  andreas
-    detabbed version
-
-    Revision 1.1.2.1  2001/02/27 19:27:58  andreas
-    first lame version
-
-
 """
 
 import os,sys
@@ -82,13 +39,8 @@
 import Zope
 import ZODB, ZODB.FileStorage
 from Products.ZCatalog import Catalog,Vocabulary
-from SearchIndex.UnIndex import UnIndex
-from SearchIndex.UnTextIndex import UnTextIndex
-from SearchIndex.UnKeywordIndex import UnKeywordIndex
-from SearchIndex.Lexicon import Lexicon, stop_word_dict
 import Persistence
 import ExtensionClass
-import DateTime
 import dispatcher,keywords
 from zLOG import LOG
 
@@ -102,8 +54,6 @@
 # maximum number of threads for stress testa
 numThreads = 4
 
-# dictionary with test words
-dictFile = "/usr/share/dict/words"
 
 # number of iterations for searches
 searchIterations = 1000
@@ -120,36 +70,7 @@
 # Don't change anything below
 #
 
-def myLOG(*args):
-    args = map(str,args)
-    LOG('catalog',0,'bench', string.join(args , ' '))
-    open('bench.log','a').write( string.join(args," ") + "\n")
-
-
-class Timer:
-
-    def __init__(self,name=''):
-        self.name = name
-        self.start()
 
-    def start(self):
-        self.ts = time.time()
-
-    def end(self):
-        self.te = time.time()
-        if thread.get_ident() == mainThreadID:
-            myLOG('bench THMain  ' , self.__repr__())
-        else:
-            myLOG('bench TH%-6s' % thread.get_ident(),self.__repr__())
-
-
-    def __repr__(self):
-        return "%-60s: %8.3f sec" % (self.name,self.te-self.ts)
-
-    def __str__(self):
-        return self.__repr__()        
-
-
 class testZODB:
     """ some wrapper stuff around ZODB """
 
@@ -301,9 +222,10 @@
 class testSearches(dispatcher.Dispatcher,TestCase):
     """ test searches """
 
-    def __init__(self,func):
+    def __init__(self,func,args=(),kw={}):
 
-        TestCase.__init__(self,func)
+
+        TestCase.__init__(self,func,args,kw)
         dispatcher.Dispatcher.__init__(self)
 
         self.init_phase = 0
@@ -337,7 +259,15 @@
         self.zodb = self.catalog = None		
 
     def log_zodb_size(self,s):
-        self.logn("Size of ZODB (data/work/Data.fs) %s test : %d" % (s,os.stat("data/work/Data.fs")[6]) )
+        self.logn("Size of ZODB (data/work/Data.fs) %s test : %s" % (s,self.size2size(os.stat("data/work/Data.fs")[6])) )
+
+
+    def size2size(self,n):
+        import math
+        if n <1024.0: return "%8.3lf Bytes" % n
+        if n <1024.0*1024.0: return "%8.3lf KB" % (1.0*n/1024.0)
+        if n <1024.0*1024.0*1024.0: return "%8.3lf MB" % (1.0*n/1024.0/1024.0)
+
         
 
     #############################################################        
@@ -345,15 +275,11 @@
     #############################################################        
 
 
-    def testFulltextIndex(self,*args):
+    def testFulltextIndex(self,args,kw):
         """ benchmark FulltextIndex """
-        self.dispatcher('funcFulltextIndex' , ('funcFulltextIndex',1 , () , {} ) )
+        self.dispatcher('funcFulltextIndex' , ('funcFulltextIndex', kw["numThreads"] , () , {} ) )
 
-    def testFulltextIndexTH(self,*args):
-        """ thread benchmark FulltextIndex """
-        self.dispatcher('funcFulltextIndex' , ('funcFulltextIndex',numThreads , () , {} ) )
 
-
     def funcFulltextIndex(self,*args):
         """ benchmark FulltextIndex """
 
@@ -371,13 +297,9 @@
     # Field index test
     #############################################################        
 
-    def testFieldIndex(self,*args):
+    def testFieldIndex(self,args,kw):
         """ benchmark field index"""
-        self.dispatcher('funcFieldIndex' , ('funcFieldIndex',1 , () , {} ) )
-
-    def testFieldIndexTH(self,*args):
-        """ thread benchmark field index"""
-        self.dispatcher('funcFieldIndex' , ('funcFieldIndex',numThreads , () , {} ) )
+        self.dispatcher('funcFieldIndex' , ('funcFieldIndex',kw["numThreads"] , () , {} ) )
 
 
     def funcFieldIndex(self,*args):
@@ -399,15 +321,11 @@
     # Keyword index test
     #############################################################        
 
-    def testKeywordIndex(self,*args):
+    def testKeywordIndex(self,args,kw):
         """ benchmark Keyword index"""
-        self.dispatcher('funcKeywordIndex' , ('funcKeywordIndex',1 , () , {} ) )
+        self.dispatcher('funcKeywordIndex' , ('funcKeywordIndex', kw["numThreads"] , () , {} ) )
 
-    def testKeywordIndexTH(self,*args):
-        """ thread benchmark Keyword index"""
-        self.dispatcher('funcKeywordIndex' , ('funcKeywordIndex',numThreads, () , {} ) )
 
-
     def funcKeywordIndex(self,*args):
         """ benchmark KeywordIndex """
 
@@ -425,13 +343,9 @@
     # Field range index test
     #############################################################        
 
-    def testFieldRangeIndex(self,*args):
+    def testFieldRangeIndex(self,args,kw):
         """ benchmark field range index"""
-        self.dispatcher('funcFieldRangeIndex' , ('funcFieldRangeIndex',1 , () , {} ) )
-
-    def testFieldRangeIndexTH(self,*args):
-        """ thread benchmark field range index"""
-        self.dispatcher('funcFieldRangeIndex' , ('funcFieldRangeIndex',numThreads, () , {} ) )
+        self.dispatcher('funcFieldRangeIndex' , ('funcFieldRangeIndex', kw["numThreads"] , () , {} ) )
 
 
     def funcFieldRangeIndex(self,*args):
@@ -465,13 +379,9 @@
     # Keyword + range index test
     #############################################################        
 
-    def testKeywordRangeIndex(self,*args):
+    def testKeywordRangeIndex(self,args,kw):
         """ benchmark Keyword range index"""
-        self.dispatcher('funcKeywordRangeIndex' , ('funcKeywordRangeIndex',1 , () , {} ) )
-
-    def testKeywordRangeIndexTH(self,*args):
-        """ thread benchmark keyword range index"""
-        self.dispatcher('funcKeywordRangeIndex' , ('funcKeywordRangeIndex',numThreads , () , {} ) )
+        self.dispatcher('funcKeywordRangeIndex' , ('funcKeywordRangeIndex', kw["numThreads"] , () , {} ) )
 
 
     def funcKeywordRangeIndex(self,*args):
@@ -501,7 +411,6 @@
         conflicts = 0
         cat,msg_ids ,keywords= self.get_catalog()
 
-        T = Timer('testUpdates of objects (100 iterations)')
 
         for i in range(updateIterations):
 
@@ -521,7 +430,6 @@
         except:
             conflicts = conflicts + 1
 
-        T.end()
 
         self.conflicts[thread.get_ident()] = conflicts
         self.threads[thread.get_ident()] = 1            
@@ -532,7 +440,7 @@
     # Test full reindexing
     #############################################################        
 
-    def testReindexing(self,*args):
+    def testReindexing(self,args,kw):
         """ test reindexing of existing data """
         self.dispatcher("testReindexing" , ("funcReindexing",1 , (mbox,1000) , {} ))
 
@@ -548,8 +456,6 @@
         mb = mailbox.UnixMailbox(open(mbox,"r"))
         i = 0
 
-        print numfiles
-
         msg = mb.next()
         while msg and i<numfiles:
 
@@ -564,8 +470,7 @@
 
             msg = mb.next()
             i = i+1
-#            if i%100==0: print i
-            print i
+            if i%100==0: print i
 
         env = self.th_teardown(env,conflicts=conflicts)
 
@@ -574,10 +479,9 @@
     # Test full reindexing
     #############################################################        
     
-    def testIncrementalIndexing(self,*args):
+    def testIncrementalIndexing(self,args,kw):
         """ testing incremental indexing """
-
-        self.dispatcher("testIncrementalIndexing" , ("funcReindexing",1, (mbox2,100) , {}))
+        self.dispatcher("testIncrementalIndexing" , ("funcReindexing",1, (mbox2,1000) , {}))
 
 
     def get_catalog(self):
@@ -645,16 +549,16 @@
     if '-b' in optsLst:
 
         basic_tests = [
-             testSearches("testFulltextIndex"),
-             testSearches("testFulltextIndexTH"),
-             testSearches("testFieldIndex"),
-             testSearches("testFieldIndexTH"),
-             testSearches("testFieldRangeIndex"),
-             testSearches("testFieldRangeIndexTH"),
-             testSearches("testKeywordIndex"),
-             testSearches("testKeywordIndexTH"),
-             testSearches("testKeywordRangeIndex"),
-             testSearches("testKeywordRangeIndexTH")
+             testSearches("testFulltextIndex",(),{"numThreads": 1}),
+             testSearches("testFulltextIndex",(),{"numThreads": 4}),
+             testSearches("testFieldIndex",(),{"numThreads": 1}),
+             testSearches("testFieldIndex",(),{"numThreads": 4}),
+             testSearches("testFieldRangeIndex",(),{"numThreads": 1}),
+             testSearches("testFieldRangeIndex",(),{"numThreads": 4}),
+             testSearches("testKeywordIndex",(),{"numThreads": 1}),
+             testSearches("testKeywordIndex",(),{"numThreads": 4}),
+             testSearches("testKeywordRangeIndex",(),{"numThreads": 1}),
+             testSearches("testKeywordRangeIndex",(),{"numThreads": 4})
         ]
 
         testsuite1 = TestSuite()
@@ -666,9 +570,9 @@
     if '-a' in optsLst:
 
         basic_tests = [
-            testSearches("testReindexing"),
+            testSearches("testReindexing",(),{"numThreads" : 1}),
+            testSearches("testIncrementalIndexing",(),{"numThreads" : 1})
 #            testSearches("testUpdates"),
-#           testSearches("testIncrementalIndexing")
         ]
             
         testsuite1 = TestSuite()