[Zope-Checkins] CVS: Zope2 - ZCatalogIndexes.py:1.2 Catalog.py:1.71 Vocabulary.py:1.14 ZCatalog.py:1.90 __init__.py:1.16

andreas@serenade.digicool.com andreas@serenade.digicool.com
Wed, 30 May 2001 11:58:08 -0400


Update of /cvs-repository/Zope2/lib/python/Products/ZCatalog
In directory serenade:/tmp/cvs-serv21362/lib/python/Products/ZCatalog

Modified Files:
	Catalog.py Vocabulary.py ZCatalog.py __init__.py 
Added Files:
	ZCatalogIndexes.py 
Log Message:
merged ajung-dropin-registry branch


--- Updated File ZCatalogIndexes.py in package Zope2 --

--- Updated File Catalog.py in package Zope2 --
--- Catalog.py	2001/04/27 18:07:13	1.70
+++ Catalog.py	2001/05/30 15:57:37	1.71
@@ -86,8 +86,7 @@
 from Persistence import Persistent
 import Acquisition
 import ExtensionClass
-from SearchIndex import UnIndex, UnTextIndex, UnKeywordIndex
-from SearchIndex.Lexicon import Lexicon
+from Products.PluginIndexes.TextIndex.Lexicon import Lexicon
 from MultiMapping import MultiMapping
 from string import lower
 import Record
@@ -101,7 +100,7 @@
 from BTrees.OIBTree import OIBTree
 from BTrees.IOBTree import IOBTree
 import BTrees.Length
-from SearchIndex.randid import randid
+from Products.PluginIndexes.common.randid import randid
 
 import time
 
@@ -321,9 +320,14 @@
             self.data[key] = tuple(rec)
 
     def addIndex(self, name, index_type):
-        """Create a new index, of one of the following index_types
+        """Create a new index, given a name and a index_type.  
 
-        Types: 'FieldIndex', 'TextIndex', 'KeywordIndex'.
+        Old format: index_type was a string, 'FieldIndex' 'TextIndex' or
+        'KeywordIndex' is no longer valid; the actual index must be instantiated
+        and passed in to addIndex.
+
+        New format: index_type is the actual index object to be stored.
+
         """
 
         if self.indexes.has_key(name):
@@ -336,21 +340,12 @@
         # pluggable and managable
 
         indexes = self.indexes
-        if index_type == 'FieldIndex':
-            indexes[name] = UnIndex.UnIndex(name)
-        elif index_type == 'TextIndex':
-            lexicon=self.lexicon
-            if type(lexicon) is type(''):
-                lexicon=getattr(self, lexicon).getLexicon()
-                
-            indexes[name] = UnTextIndex.UnTextIndex(name, None, None, lexicon)
-        elif index_type == 'KeywordIndex':
-            indexes[name] = UnKeywordIndex.UnKeywordIndex(name)
-        else:
-            raise 'Unknown Index Type', (
-                "%s invalid - must be one of %s"
-                % (index_type, ['FieldIndex', 'TextIndex', 'KeywordIndex'])
-                )
+
+        if type(index_type) == type(''):
+            raise TypeError,"""Catalog addIndex now requires the index type to
+            be resolved prior to adding; create the proper index in the caller."""
+
+        indexes[name] = index_type;
 
         self.indexes = indexes
 
@@ -364,9 +359,15 @@
         del indexes[name]
         self.indexes = indexes
 
+
+    def reindexIndex(self,name):
+
+        for p in self.paths.items():
+            print p
+
     # the cataloging API
 
-    def catalogObject(self, object, uid, threshold=None):
+    def catalogObject(self, object, uid, threshold=None,idxs=[]):
         """ 
         Adds an object to the Catalog by iteratively applying it
         all indexes.
@@ -425,7 +426,13 @@
             self.paths[index] = uid
             
         total = 0
-        for x in self.indexes.values():
+
+        if idxs==[]: use_indexes = self.indexes.keys()
+        else:        use_indexes = idxs
+
+        for item in use_indexes:
+            x = self.indexes[item]
+            
             ## tricky!  indexes need to acquire now, and because they
             ## are in a standard dict __getattr__ isn't used, so
             ## acquisition doesn't kick in, we must explicitly wrap!
@@ -512,25 +519,36 @@
     
 ## Searching engine.  You don't really have to worry about what goes
 ## on below here...  Most of this stuff came from ZTables with tweaks.
+## But I worry about :-)
 
-    def _indexedSearch(self, args, sort_index, append, used):
+    def _indexedSearch(self, request , sort_index, append, used):
         """
         Iterate through the indexes, applying the query to each one.
         """
 
-        rs=None
-        data=self.data
+        rs   = None             # resultset
+        data = self.data
         
         if used is None: used={}
         for i in self.indexes.keys():
+
             index = self.indexes[i].__of__(self)
             if hasattr(index,'_apply_index'):
-                r=index._apply_index(args)
+
+                r = None
+
+                # Optimization: we check if there is some work for the index.
+                # 
+                if request.has_key(index.id) :
+                    if len(request[index.id])>0:
+                        r=index._apply_index(request)
+
                 if r is not None:
                     r, u = r
-                    for name in u:
-                        used[name]=1
+                    for name in u: used[name]=1
                     w, rs = weightedIntersection(rs, r)
+
+
                         
         #assert rs==None or hasattr(rs, 'values') or hasattr(rs, 'keys')
         if rs is None:
@@ -594,6 +612,7 @@
         return used
 
     def searchResults(self, REQUEST=None, used=None, **kw):
+        
         # Get search arguments:
         if REQUEST is None and not kw:
             try: REQUEST=self.REQUEST
@@ -634,6 +653,7 @@
         
         # Perform searches with indexes and sort_index
         r=[]
+        
         used=self._indexedSearch(kw, sort_index, r.append, used)
         if not r:
             return LazyCat(r)

--- Updated File Vocabulary.py in package Zope2 --
--- Vocabulary.py	2001/03/19 19:26:17	1.13
+++ Vocabulary.py	2001/05/30 15:57:37	1.14
@@ -1,203 +1 @@
-##############################################################################
-# 
-# Zope Public License (ZPL) Version 1.0
-# -------------------------------------
-# 
-# Copyright (c) Digital Creations.  All rights reserved.
-# 
-# This license has been certified as Open Source(tm).
-# 
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-# 
-# 1. Redistributions in source code must retain the above copyright
-#    notice, this list of conditions, and the following disclaimer.
-# 
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions, and the following disclaimer in
-#    the documentation and/or other materials provided with the
-#    distribution.
-# 
-# 3. Digital Creations requests that attribution be given to Zope
-#    in any manner possible. Zope includes a "Powered by Zope"
-#    button that is installed by default. While it is not a license
-#    violation to remove this button, it is requested that the
-#    attribution remain. A significant investment has been put
-#    into Zope, and this effort will continue if the Zope community
-#    continues to grow. This is one way to assure that growth.
-# 
-# 4. All advertising materials and documentation mentioning
-#    features derived from or use of this software must display
-#    the following acknowledgement:
-# 
-#      "This product includes software developed by Digital Creations
-#      for use in the Z Object Publishing Environment
-#      (http://www.zope.org/)."
-# 
-#    In the event that the product being advertised includes an
-#    intact Zope distribution (with copyright and license included)
-#    then this clause is waived.
-# 
-# 5. Names associated with Zope or Digital Creations must not be used to
-#    endorse or promote products derived from this software without
-#    prior written permission from Digital Creations.
-# 
-# 6. Modified redistributions of any form whatsoever must retain
-#    the following acknowledgment:
-# 
-#      "This product includes software developed by Digital Creations
-#      for use in the Z Object Publishing Environment
-#      (http://www.zope.org/)."
-# 
-#    Intact (re-)distributions of any official Zope release do not
-#    require an external acknowledgement.
-# 
-# 7. Modifications are encouraged but must be packaged separately as
-#    patches to official Zope releases.  Distributions that do not
-#    clearly separate the patches from the original work must be clearly
-#    labeled as unofficial distributions.  Modifications which do not
-#    carry the name Zope may be packaged in any form, as long as they
-#    conform to all of the clauses above.
-# 
-# 
-# Disclaimer
-# 
-#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
-#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
-#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
-#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-#   SUCH DAMAGE.
-# 
-# 
-# This software consists of contributions made by Digital Creations and
-# many individuals on behalf of Digital Creations.  Specific
-# attributions are listed in the accompanying credits file.
-# 
-##############################################################################
-"""ZCatalog product"""
-
-from Globals import DTMLFile, MessageDialog
-import Globals, AccessControl.Role
-from Acquisition import Implicit
-from Persistence import Persistent
-from OFS.SimpleItem import Item
-from SearchIndex import Lexicon, GlobbingLexicon
-
-from SearchIndex.Lexicon import stop_word_dict
-
-manage_addVocabularyForm=DTMLFile('dtml/addVocabulary',globals())
-
-def manage_addVocabulary(self, id, title, globbing=None, REQUEST=None):
-    """Add a Vocabulary object
-    """
-    id=str(id)
-    title=str(title)
-    if globbing: globbing=1
-    
-    c=Vocabulary(id, title, globbing)
-    self._setObject(id, c)
-    if REQUEST is not None:
-        return self.manage_main(self,REQUEST)
-
-
-class Vocabulary(Item, Persistent, Implicit,
-                 AccessControl.Role.RoleManager,
-                 ):
-    """
-    A Vocabulary is a user-managable realization of a Lexicon object.
-
-    """
-
-    meta_type = "Vocabulary"
-    _isAVocabulary = 1
-
-    
-    manage_options=(
-        (
-        {'label': 'Vocabulary', 'action': 'manage_main',
-         'help' : ('ZCatalog', 'Vocabulary_Vocabulary.stx')},
-        {'label': 'Query', 'action': 'manage_query',
-         'help': ('ZCatalog', 'Vocabulary_Query.stx')},
-        )
-        +Item.manage_options
-        +AccessControl.Role.RoleManager.manage_options
-        )
-
-    __ac_permissions__=(
-
-        ('Manage Vocabulary',
-         ['manage_main', 'manage_vocab', 'manage_query'], 
-         ['Manager']),
-
-        ('Query Vocabulary',
-         ['query',],
-         ['Anonymous', 'Manager']), 
-        )
-
-    
-
-    manage_main = DTMLFile('dtml/manage_vocab', globals())
-    manage_query = DTMLFile('dtml/vocab_query', globals())
-
-    def __init__(self, id, title='', globbing=None):
-        """ create the lexicon to manage... """
-        self.id = id
-        self.title = title
-        self.globbing = not not globbing
-
-        if globbing:
-            self.lexicon = GlobbingLexicon.GlobbingLexicon()
-        else:
-            self.lexicon = Lexicon.Lexicon(stop_word_dict)
-
-    def getLexicon(self):
-        return self.lexicon
-
-    def query(self, pattern):
-        """ """
-        result = []
-        for x in self.lexicon.get(pattern):
-            if self.globbing:
-                result.append(self.lexicon._inverseLex[x])
-            else:
-                result.append(pattern)
-        return result
-            
-
-    def manage_insert(self, word='', URL1=None, RESPONSE=None):
-        """ doc string """
-        self.insert(word)
-
-        if RESPONSE:
-            RESPONSE.redirect(URL1 + '/manage_main')
-
-    def manage_stop_syn(self, stop_syn, REQUEST=None):
-        pass
-
-
-
-    def insert(self, word=''):
-        self.lexicon.set(word)
-
-    def words(self):
-        return self.lexicon._lexicon.items()
-
-    
-
-
-
-
-
-
-
-
-
-
+from Products.PluginIndexes.TextIndex.Vocabulary import *

--- Updated File ZCatalog.py in package Zope2 --
--- ZCatalog.py	2001/05/21 19:40:55	1.89
+++ ZCatalog.py	2001/05/30 15:57:37	1.90
@@ -86,20 +86,24 @@
 
 from Globals import DTMLFile, MessageDialog
 import Globals
+
 from OFS.Folder import Folder
 from OFS.FindSupport import FindSupport
+from OFS.ObjectManager import ObjectManager
 from DateTime import DateTime
-import string, urlparse, urllib, os, sys, time
-import Products
 from Acquisition import Implicit
 from Persistence import Persistent
 from DocumentTemplate.DT_Util import InstanceDict, TemplateDict
 from DocumentTemplate.DT_Util import Eval
 from AccessControl.Permission import name_trans
 from Catalog import Catalog, CatalogError
-from Vocabulary import Vocabulary
 from AccessControl import getSecurityManager, full_read_guard
 from zLOG import LOG, ERROR
+from ZCatalogIndexes import ZCatalogIndexes
+from Products.PluginIndexes.common.PluggableIndex import PluggableIndexInterface
+from Products.PluginIndexes.TextIndex.Vocabulary import Vocabulary
+from Products.PluginIndexes.TextIndex import Splitter
+import string,  urllib, os, sys, time
 
 StringType=type('')
 
@@ -157,9 +161,8 @@
          'action': 'manage_propertiesForm',
          'help': ('OFSP','Properties.stx')},
         {'label': 'Indexes',            # TAB: Indexes
-         'action': 'manage_catalogIndexes', 
-         'target': 'manage_main',
-         'help':('ZCatalog','ZCatalog_Indexes.stx')},
+         'action': 'Indexes/manage_workspace',
+         'help': ('ZCatalog','ZCatalog_Indexes.stx')},
         {'label': 'Metadata',           # TAB: Metadata
          'action': 'manage_catalogSchema', 
          'target':'manage_main',
@@ -216,6 +219,8 @@
     manage_objectInformation = DTMLFile('dtml/catalogObjectInformation',
                                         globals())
 
+    Indexes = ZCatalogIndexes()
+
     threshold=10000
     _v_total=0
     _v_transaction = None
@@ -228,6 +233,9 @@
             self=self.__of__(container)
         self.id=id
         self.title=title
+
+        self.vocabulary = None
+        self.availableSplitters = Splitter.availableSplitters
         
         self.threshold = 10000
         self._v_total = 0
@@ -235,26 +243,30 @@
         if vocab_id is None:
             v = Vocabulary('Vocabulary', 'Vocabulary', globbing=1)
             self._setObject('Vocabulary', v)
+            self.vocabulary = v
             self.vocab_id = 'Vocabulary'
         else:
             self.vocab_id = vocab_id
 
+
         self._catalog = Catalog(vocabulary=self.vocab_id)
+
+        self.addColumn('id')
+        self.addIndex('id', 'FieldIndex')
 
-        self._catalog.addColumn('id')
-        self._catalog.addIndex('id', 'FieldIndex')
+        self.addColumn('title')
+        self.addIndex('title', 'TextIndex')
 
-        self._catalog.addColumn('title')
-        self._catalog.addIndex('title', 'TextIndex')
+        self.addColumn('meta_type')
+        self.addIndex('meta_type', 'FieldIndex')
 
-        self._catalog.addColumn('meta_type')
-        self._catalog.addIndex('meta_type', 'FieldIndex')
+        self.addColumn('bobobase_modification_time')
+        self.addIndex('bobobase_modification_time', 'FieldIndex')
 
-        self._catalog.addColumn('bobobase_modification_time')
-        self._catalog.addIndex('bobobase_modification_time', 'FieldIndex')
+        self.addColumn('summary')
+        self.addIndex('PrincipiaSearchSource', 'TextIndex')
 
-        self._catalog.addColumn('summary')
-        self._catalog.addIndex('PrincipiaSearchSource', 'TextIndex')
+        self.addIndex('path','PathIndex')
 
     def __len__(self): return len(self._catalog)
 
@@ -380,7 +392,7 @@
 
     def manage_addColumn(self, name, REQUEST=None, RESPONSE=None, URL1=None):
         """ add a column """
-        self._catalog.addColumn(name)
+        self.addColumn(name)
 
         if REQUEST and RESPONSE:
             RESPONSE.redirect(URL1 + '/manage_catalogSchema?manage_tabs_message=Column%20Added')
@@ -388,28 +400,73 @@
     def manage_delColumns(self, names, REQUEST=None, RESPONSE=None, URL1=None):
         """ del a column """
         for name in names:
-            self._catalog.delColumn(name)
+            self.delColumn(name)
 
         if REQUEST and RESPONSE:
             RESPONSE.redirect(URL1 + '/manage_catalogSchema?manage_tabs_message=Column%20Deleted')
 
     def manage_addIndex(self, name, type, REQUEST=None, RESPONSE=None, URL1=None):
         """ add an index """
-        self._catalog.addIndex(name, type)
-        
+        self.addIndex(name, type)
+
         if REQUEST and RESPONSE:
-            RESPONSE.redirect(URL1 + '/manage_catalogIndexes?manage_tabs_message=Index%20Added')
+            RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Index%20Added')
         
-    def manage_delIndexes(self, names, REQUEST=None, RESPONSE=None, URL1=None):
+
+    def manage_deleteIndex(self, ids=None, REQUEST=None, RESPONSE=None,
+        URL1=None):
         """ del an index """
-        for name in names:
-            self._catalog.delIndex(name)
+        if not ids:
+            return MessageDialog(title='No items specified',
+                message='No items were specified!',
+                action = "./manage_main",)
+
+        for name in ids:
+            self.delIndex(name)
         
         if REQUEST and RESPONSE:
-            RESPONSE.redirect(URL1 + '/manage_catalogIndexes?manage_tabs_message=Index%20Deleted')
+            RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Index%20Deleted')
+
+    def manage_clearIndex(self, ids=None, REQUEST=None, RESPONSE=None,
+        URL1=None):
+        """ del an index """
+        if not ids:
+            return MessageDialog(title='No items specified',
+                message='No items were specified!',
+                action = "./manage_main",)
+
+        for name in ids:
+            self.clearIndex(name)
+        
+        if REQUEST and RESPONSE:
+            RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Index%20Cleared')
+
+
+    def reindexIndex(self,name,REQUEST):
+        
+        paths = tuple(self._catalog.paths.values())
+
+        for p in paths:
+            obj = self.resolve_path(p)
+            if not obj:
+                obj = self.resolve_url(p, REQUEST)
+            if obj is not None:
+                self.catalog_object(obj, p, idxs=[name])             
+
+    def manage_reindexIndex(self, ids=None, REQUEST=None, RESPONSE=None, URL1=None):
+        """ Reindex indexes from a ZCatalog"""
+        if not ids:
+            return MessageDialog(title='No items specified',
+                message='No items were specified!',
+                action = "./manage_main",)
 
+        for id in ids:
+            self.reindexIndex(id, REQUEST)
 
-    def catalog_object(self, obj, uid=None):
+        if REQUEST and RESPONSE:
+            RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Reindexing%20Performed')
+
+    def catalog_object(self, obj, uid=None, idxs=[]):
         """ wrapper around catalog """
 
         if uid is None:
@@ -423,7 +480,7 @@
         elif type(uid) is not StringType:
             raise CatalogError('The object unique id must be a string.')
 
-        self._catalog.catalogObject(obj, uid, None)
+        self._catalog.catalogObject(obj, uid, None,idxs)
         # None passed in to catalogObject as third argument indicates
         # that we shouldn't try to commit subtransactions within any
         # indexing code.  We throw away the result of the call to
@@ -530,13 +587,14 @@
 
     meta_types=() # Sub-object types that are specific to this object
     
-    def all_meta_types(self):
-        pmt=()
-        if hasattr(self, '_product_meta_types'): pmt=self._product_meta_types
-        elif hasattr(self, 'aq_acquire'):
-            try: pmt=self.aq_acquire('_product_meta_types')
-            except AttributeError:  pass
-        return self.meta_types+Products.meta_types+pmt
+    # Dont need this anymore -- we inherit from object manager
+    #def all_meta_types(self):
+    #    pmt=()
+    #    if hasattr(self, '_product_meta_types'): pmt=self._product_meta_types
+    #    elif hasattr(self, 'aq_acquire'):
+    #        try: pmt=self.aq_acquire('_product_meta_types')
+    #        except AttributeError:  pass
+    #    return self.meta_types+Products.meta_types+pmt
 
     def valid_roles(self):
         "Return list of valid roles"
@@ -728,6 +786,54 @@
         tt=time.time()-tt
         ct=time.clock()-ct
         return 'Finished conversion in %s seconds (%s cpu)' % (tt, ct)
+
+    #
+    # Indexing methods 
+    #
+
+    def addIndex(self, name, type):
+
+        # Convert the type by finding an appropriate product which supports
+        # this interface by that name.  Bleah
+
+        products = ObjectManager.all_meta_types(self, interfaces=(
+            PluggableIndexInterface,))
+
+        p = None
+
+        for prod in products:
+            if prod['name'] == type: 
+                p = prod
+                break
+
+        if p is None:
+            raise ValueError, "Index of type %s not found" % type
+
+        base = p['instance']
+
+        if base is None:
+            raise ValueError, "Index type %s does not support addIndex" % type
+
+        index = base(name, self)
+
+        self._catalog.addIndex(name,index)
+
+
+    def delIndex(self, name ):
+
+        self._catalog.delIndex(name)
+
+    def clearIndex(self, name):
+
+        self._catalog.indexes[name].clear()
+
+
+    def addColumn(self, name, default_value=None):
+        return self._catalog.addColumn(name, default_value)
+
+    def delColumn(self, name):
+        return self._catalog.delColumn(name)
+
     
 Globals.default__class_init__(ZCatalog)
 
@@ -787,6 +893,3 @@
         if not (role in pr):
             return 0
     return 1
-
-
-

--- Updated File __init__.py in package Zope2 --
--- __init__.py	2001/01/31 22:54:11	1.15
+++ __init__.py	2001/05/30 15:57:37	1.16
@@ -85,7 +85,8 @@
 
 """ZCatalog product"""
 
-import ZCatalog, Catalog, CatalogAwareness, Vocabulary, ZClasses
+import ZCatalog, Catalog, CatalogAwareness,  ZClasses
+from Products.PluginIndexes.TextIndex import Vocabulary
 from ZClasses import createZClassForBase
 
 createZClassForBase( ZCatalog.ZCatalog , globals()