[Zope-Checkins] SVN: Zope/trunk/ merge ajung-zcatalog-progress branch 26525:26606

Andreas Jung andreas at andreas-jung.com
Sat Jul 17 00:16:35 EDT 2004


Log message for revision 26607:
  merge ajung-zcatalog-progress branch  26525:26606


Changed:
  U   Zope/trunk/doc/CHANGES.txt
  U   Zope/trunk/lib/python/Products/ZCatalog/Catalog.py
  U   Zope/trunk/lib/python/Products/ZCatalog/CatalogPathAwareness.py
  A   Zope/trunk/lib/python/Products/ZCatalog/ProgressHandler.py
  U   Zope/trunk/lib/python/Products/ZCatalog/README.txt
  U   Zope/trunk/lib/python/Products/ZCatalog/ZCatalog.py
  U   Zope/trunk/lib/python/Products/ZCatalog/ZCatalogIndexes.py
  U   Zope/trunk/lib/python/Products/ZCatalog/__init__.py
  U   Zope/trunk/lib/python/Products/ZCatalog/dtml/catalogAdvanced.dtml


-=-
Modified: Zope/trunk/doc/CHANGES.txt
===================================================================
--- Zope/trunk/doc/CHANGES.txt	2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/doc/CHANGES.txt	2004-07-17 04:16:35 UTC (rev 26607)
@@ -12,10 +12,6 @@
        changes in C, brining the Python and C implementations back in
        sync.  See lib/python/AccessControl/PermissionRole.py.
 
-     - Renable C Zope security policy by implementing recent Python
-       changes in C, brining the Python and C implementations back in
-       sync.  See lib/python/AccessControl/ZopeSecurityPolicy.py.
-
      - Add cyclic-garbage collection support to C extension classes,
        especially to acquisition wrappers.
 
@@ -26,6 +22,10 @@
 
     Features added
 
+     - ZCatalog: added a new configuration option in the "Advanced" tab
+       to provide optional logging of the progress of long running 
+       reindexing or recataloging operations.
+
      - made Zope.configure return the starter instance to enable other
        methods to be called, such as starter.setupConfiguredLoggers()
 

Modified: Zope/trunk/lib/python/Products/ZCatalog/Catalog.py
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/Catalog.py	2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/Catalog.py	2004-07-17 04:16:35 UTC (rev 26607)
@@ -11,27 +11,27 @@
 #
 ##############################################################################
 
-from Persistence import Persistent
+import types
+import logging
+from bisect import bisect
+from random import randint
+
 import Acquisition
 import ExtensionClass
-from MultiMapping import MultiMapping
-import Record
 from Missing import MV
-import logging
+from Persistence import Persistent
 
-from Lazy import LazyMap, LazyFilter, LazyCat, LazyValues
-from CatalogBrains import AbstractCatalogBrain, NoBrainer
+import BTrees.Length
 from BTrees.IIBTree import intersection, weightedIntersection, IISet
 from BTrees.OIBTree import OIBTree
 from BTrees.IOBTree import IOBTree
-import BTrees.Length
+from Lazy import LazyMap, LazyCat, LazyValues
+from CatalogBrains import AbstractCatalogBrain, NoBrainer
 
-import time, sys, types
-from bisect import bisect
-from random import randint
 
 LOG = logging.getLogger('Zope.ZCatalog')
 
+
 try:
     from DocumentTemplate.cDocumentTemplate import safe_callable
 except ImportError:
@@ -278,7 +278,7 @@
 
         indexes = self.indexes
 
-        if isinstance(index_type, types.StringType):
+        if isinstance(index_type, str):
             raise TypeError,"""Catalog addIndex now requires the index type to
             be resolved prior to adding; create the proper index in the caller."""
 
@@ -755,7 +755,7 @@
         reverse = 0
         if sort_index is not None:
             order = self._get_sort_attr("order", args)
-            if (isinstance(order, types.StringType) and
+            if (isinstance(order, str) and
                 order.lower() in ('reverse', 'descending')):
                 reverse = 1
         # Perform searches with indexes and sort_index

Modified: Zope/trunk/lib/python/Products/ZCatalog/CatalogPathAwareness.py
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/CatalogPathAwareness.py	2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/CatalogPathAwareness.py	2004-07-17 04:16:35 UTC (rev 26607)
@@ -13,7 +13,6 @@
 
 """ZCatalog Findable class"""
 
-import urllib
 from Globals import DTMLFile
 from Acquisition import aq_base
 

Copied: Zope/trunk/lib/python/Products/ZCatalog/ProgressHandler.py (from rev 26606, Zope/branches/ajung-zcatalog-progress/lib/python/Products/ZCatalog/ProgressHandler.py)


Property changes on: Zope/trunk/lib/python/Products/ZCatalog/ProgressHandler.py
___________________________________________________________________
Name: svn:keywords
   + svn:eol-style

Modified: Zope/trunk/lib/python/Products/ZCatalog/README.txt
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/README.txt	2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/README.txt	2004-07-17 04:16:35 UTC (rev 26607)
@@ -52,3 +52,11 @@
        encoding (set in site.py of the Python installation) if any
        to the keyword uses a non-ascii encoding (e.g. using accented
        characters). 
+
+  Notes for Zope 2.8:
+
+       reindexIndex() and refreshCatalog() accept a new optional parameter 
+       'pghandler' which must be a handler instance implementing the 
+       IProgressHandler interface (see ProgressHandler.py). This can be useful
+       to provide logging during long running operations.
+ 

Modified: Zope/trunk/lib/python/Products/ZCatalog/ZCatalog.py
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/ZCatalog.py	2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/ZCatalog.py	2004-07-17 04:16:35 UTC (rev 26607)
@@ -15,6 +15,8 @@
 $Id$
 """
 
+import urllib, time, sys, string,logging
+
 from Globals import DTMLFile, MessageDialog
 import Globals
 
@@ -35,9 +37,8 @@
 from Products.PluginIndexes.common.PluggableIndex \
      import PluggableIndexInterface
 from Products.PluginIndexes.TextIndex import Splitter
-import urllib, time, sys
-import string,logging
 from IZCatalog import IZCatalog
+from ProgressHandler import ZLogHandler
 
 LOG = logging.getLogger('Zope.ZCatalog')
 
@@ -127,6 +128,7 @@
           'manage_catalogClear', 'manage_addColumn', 'manage_delColumn',
           'manage_addIndex', 'manage_delIndex', 'manage_clearIndex',
           'manage_reindexIndex', 'manage_main', 'availableSplitters',
+          'manage_setProgress',
 
           # these two are deprecated:
           'manage_delColumns', 'manage_deleteIndex'
@@ -250,7 +252,9 @@
         elapse = time.time()
         c_elapse = time.clock()
 
-        self.refreshCatalog(clear=1)
+        pgthreshold = self._getProgressThreshold()
+        handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
+        self.refreshCatalog(clear=1, pghandler=handler)
 
         elapse = time.time() - elapse
         c_elapse = time.clock() - c_elapse
@@ -263,7 +267,7 @@
                          'Total CPU time: %s' % (`elapse`, `c_elapse`)))
 
 
-    def refreshCatalog(self, clear=0):
+    def refreshCatalog(self, clear=0, pghandler=None):
         """ re-index everything we can find """
 
         cat = self._catalog
@@ -272,26 +276,27 @@
             paths = tuple(paths)
             cat.clear()
 
-        LOG('ZCatalog', BLATHER, 'Starting recataloging of ZCatalog at %s' % 
-             self.absolute_url(1))
         num_objects = len(paths)
+        if pghandler:
+            pghandler.init('Refreshing catalog: %s' % self.absolute_url(1), num_objects)
+
         for i in xrange(num_objects):
+            if pghandler: pghandler.report(i)
+
             p = paths[i]
             obj = self.resolve_path(p)
             if not obj:
                 obj = self.resolve_url(p, self.REQUEST)
             if obj is not None:
                 try:
-                    LOG('ZCatalog', BLATHER, 'Recataloging object %s (%d/%d)' %
-                         (p, i, num_objects))
-                    self.catalog_object(obj, p)
+                    self.catalog_object(obj, p, pghandler=pghandler)
                 except ConflictError:
                     raise
                 except:
-                    LOG('ZCatalog', ERROR, 'Recataloging object at %s failed' % p,
-                              error=sys.exc_info())
+                    LOG.error('Recataloging object at %s failed' % p,
+                              exc_info=sys.exc_info())
 
-        LOG('ZCatalog', BLATHER, 'Recataloging of ZCatalog at %s terminated' % self.absolute_url(1))
+        if pghandler: pghandler.finish()
 
     def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None):
         """ clears the whole enchilada """
@@ -460,10 +465,21 @@
                 '/manage_catalogIndexes?manage_tabs_message=Index%20Cleared')
 
 
-    def reindexIndex(self, name, REQUEST):
+    def reindexIndex(self, name, REQUEST, pghandler=None):
         if isinstance(name, str):
             name = (name,)
-        for p in self._catalog.uids.keys():
+
+        paths = self._catalog.uids.keys()
+        num_paths = len(paths)   # inefficient        
+
+        i = 0
+        if pghandler:
+            pghandler.init('reindexing %s' % name, num_paths)
+
+        for p in paths:
+            i+=1
+            if pghandler: pghandler.report(i)
+
             obj = self.resolve_path(p)
             if not obj:
                 obj = self.resolve_url(p, REQUEST)
@@ -475,7 +491,7 @@
                 # index via the UI
                 try:
                     self.catalog_object(obj, p, idxs=name,
-                                        update_metadata=0)
+                                        update_metadata=0, pghandler=pghandler)
                 except TypeError:
                     # Fall back to Zope 2.6.2 interface. This is necessary for
                     # products like CMF 1.4.2 and earlier that subclass from
@@ -485,8 +501,11 @@
                     warn('catalog_object interface of %s not up to date'
                          % self.__class__.__name__,
                          DeprecationWarning)
-                    self.catalog_object(obj, p, idxs=name)
+                    self.catalog_object(obj, p, idxs=name, pghandler=pghandler)
 
+        if pghandler:
+            pghandler.finish()
+
     def manage_reindexIndex(self, ids=None, REQUEST=None, RESPONSE=None,
                             URL1=None):
         """Reindex indexe(s) from a ZCatalog"""
@@ -495,7 +514,9 @@
                 message='No items were specified!',
                 action = "./manage_catalogIndexes",)
 
-        self.reindexIndex(ids, REQUEST)
+        pgthreshold = self._getProgressThreshold()
+        handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
+        self.reindexIndex(ids, REQUEST, handler)
 
         if REQUEST and RESPONSE:
             RESPONSE.redirect(
@@ -509,7 +530,7 @@
         return Splitter.availableSplitters
 
 
-    def catalog_object(self, obj, uid=None, idxs=None, update_metadata=1):
+    def catalog_object(self, obj, uid=None, idxs=None, update_metadata=1, pghandler=None):
         """ wrapper around catalog """
 
         if uid is None:
@@ -551,6 +572,8 @@
                 get_transaction().commit(1)
                 self._p_jar.cacheGC()
                 self._v_total = 0
+                if pghandler:
+                    pghandler.info('commiting subtransaction')
 
     def uncatalog_object(self, uid):
         """Wrapper around catalog """
@@ -861,6 +884,21 @@
                   '%s unchanged.' % (len(fixed), len(removed), unchanged),
           action='./manage_main')
 
+    def manage_setProgress(self, pgthreshold=0, RESPONSE=None, URL1=None):
+        """Set parameter to perform logging of reindexing operations very 
+           'pgthreshold' objects
+        """
+
+        self.pgthreshold = pgthreshold
+        if RESPONSE:
+            RESPONSE.redirect(
+                URL1 + '/manage_main?manage_tabs_message=Catalog%20Changed')
+
+    def _getProgressThreshold(self):
+        if not hasattr(self, 'pgthreshold'):
+            self.pgthreshold = 0
+        return self.pgthreshold
+
     def manage_convertBTrees(self, threshold=200):
         """Convert the catalog's data structures to use BTrees package"""
         assert type(threshold) is type(0)

Modified: Zope/trunk/lib/python/Products/ZCatalog/ZCatalogIndexes.py
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/ZCatalogIndexes.py	2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/ZCatalogIndexes.py	2004-07-17 04:16:35 UTC (rev 26607)
@@ -14,21 +14,16 @@
 """$Id$
 """
 
-from  Globals import DTMLFile, InitializeClass
+
+from Acquisition import Implicit
+from Persistence import Persistent
+from Globals import DTMLFile, InitializeClass
 from AccessControl.SecurityInfo import ClassSecurityInfo
-from AccessControl.Permissions import delete_objects, manage_zcatalog_indexes
-import Globals
+from AccessControl.Permissions import manage_zcatalog_indexes
 from OFS.Folder import Folder
-from OFS.FindSupport import FindSupport
-from OFS.History import Historical
 from OFS.SimpleItem import SimpleItem
-from OFS.ObjectManager import ObjectManager, IFAwareObjectManager
+from OFS.ObjectManager import IFAwareObjectManager
 
-import os, sys, time 
-
-from Acquisition import Implicit
-from Persistence import Persistent
-
 from Products.PluginIndexes.common.PluggableIndex import PluggableIndexInterface
 
 _marker = []

Modified: Zope/trunk/lib/python/Products/ZCatalog/__init__.py
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/__init__.py	2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/__init__.py	2004-07-17 04:16:35 UTC (rev 26607)
@@ -13,7 +13,7 @@
 
 """ZCatalog product"""
 
-import ZCatalog, Catalog, CatalogAwareness, CatalogPathAwareness, ZClasses
+import ZCatalog, CatalogAwareness, CatalogPathAwareness
 from Products.PluginIndexes.TextIndex import Vocabulary
 from ZClasses import createZClassForBase
 

Modified: Zope/trunk/lib/python/Products/ZCatalog/dtml/catalogAdvanced.dtml
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/dtml/catalogAdvanced.dtml	2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/dtml/catalogAdvanced.dtml	2004-07-17 04:16:35 UTC (rev 26607)
@@ -37,6 +37,19 @@
   </td>
 </tr>
 <tr>
+  <td align="left" valign="top">
+  <p class="form-help">Log progress of reindexing every N objects to the Zope logger (set to 0 to disabled logging)
+  </p>
+  </td>
+  <td align="right" valign="top">
+<form action="&dtml-URL1;">
+<input type="text" name="pgthreshold:int" value="<dtml-var pgthreshold missing="0">">
+<input class="form-element" type="submit" 
+ name="manage_setProgress:method" value=" Change ">
+</form>
+  </td>
+</tr>
+<tr>
   <td>
 
   </td>



More information about the Zope-Checkins mailing list