[Zope-Checkins] CVS: Zope/lib/python/Products/PluginIndexes/TextIndexNG - DumbQueryParser.py:1.1.2.1 TextIndexNG.py:1.2.2.49 interactiveDemo.py:1.1.2.6

Andreas Jung andreas@digicool.com
Sat, 2 Mar 2002 09:35:01 -0500


Update of /cvs-repository/Zope/lib/python/Products/PluginIndexes/TextIndexNG
In directory cvs.zope.org:/tmp/cvs-serv23886

Modified Files:
      Tag: ajung-textindexng-branch
	TextIndexNG.py interactiveDemo.py 
Added Files:
      Tag: ajung-textindexng-branch
	DumbQueryParser.py 
Log Message:
added a new dumb query parser to that will be used as a temp, solution
to continue development until the more sophisticated query parser is
finished.

- all unittests will use the "official" query parser
- interactiveDemo.py and the interactive test function in the ZMI
  will use the DumpQueryParser


=== Added File Zope/lib/python/Products/PluginIndexes/TextIndexNG/DumbQueryParser.py ===
#!/usr/bin/env python2.1

##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
# 
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
# 
##############################################################################

"""
DumbQueryParser class

A very dumb parser as temporarily solution for the non-yet-finished parser
"""

__version__ = '$Id: DumbQueryParser.py,v 1.1.2.1 2002/03/02 14:35:00 andreasjung Exp $'

from queryparser.Collector import OperatorDict

class DumbQueryParser:
    """Dumb query parser """ 

    def __call__(self, query, operator='and'):

        res = []

        words = query.split(' ')
        words = [ x.strip() for x in words]

        for word in words:
            if word.startswith('!'):
                res.append( 'PL("%s")' % word[1:])
            else:
                res.append( 'LL("%s")' % word)

        s = ','.join(res)
        s = "%s(%s)" % (OperatorDict[operator], s)

        return s 


def test():

    import os, sys, re,traceback, atexit

    histfile = os.path.expanduser('~/.pyhist')
    try:
        import readline
        readline.read_history_file(histfile)
        atexit.register(readline.write_history_file,histfile)
    except: pass


    print "entering interactive query mode:"
    while 1:

        s = raw_input('> ')
        print s  

        try:

            P = DumbQueryParser()
            res = P(s)        
            print 'res:',res

        except:
            traceback.print_exc()


if __name__ == '__main__':
    test()


=== Zope/lib/python/Products/PluginIndexes/TextIndexNG/TextIndexNG.py 1.2.2.48 => 1.2.2.49 ===
 
 from queryparser.queryparser import QueryParser
+from DumbQueryParser import DumbQueryParser
 
 import Stemmer, Similarity
 import Thesaurus, StopWords, Normalizer
@@ -114,11 +115,12 @@
     _all_options = ('useSplitter', 'splitterMaxLen', 'splitterIndexNumbers',
          'splitterSingleChars', 'splitterCasefolding', 'useStemmer', 'useOperator',
          'useGlobbing', 'lexicon', 'nearDistance', 'useSimilarity',
-         'stopWords', 'thesaurus', 'characterMapping', 'useConverters'
+         'stopWords', 'thesaurus', 'characterMapping', 'useConverters',
+        'useDumbParser'
         )
 
     query_options = ("query","operator")
-    _valid_default_operators    =  ('and','or')
+    _valid_default_operators    =  ('and','or','near','quote')
  
 
     def __init__(self 
@@ -187,6 +189,9 @@
         # use converters from the ConvertersRegistry
         self.useConverters = getattr(extra,'useConverters',0) 
 
+        # check Parser
+        self.useDumbParser = getattr(extra,'useDumbParser',0)
+
         if verbose: self.debugOn()
         else:       self.debugOff()
 
@@ -519,11 +524,7 @@
         record = parseIndexRequest(request,self.id,self.query_options)
         if record.keys==None: return None
 
-        # Changed for 2.4
-        # We use the default operator that can me managed via the ZMI
-
-        qop = record.get('operator', self.useOperator)
-        query_operator = operator_dict.get(qop)
+        query_operator = record.get('operator', self.useOperator)
 
         if query_operator is None:
             raise exceptions.RuntimeError, ("Invalid operator '%s' "
@@ -532,7 +533,7 @@
 
         q = record.keys[0]
 
-        res = self.query( q )
+        res = self.query(q, query_operator )
 
         # this should be fixed
         bucket = IIBucket()
@@ -543,8 +544,8 @@
         return res[0], (self.id, )
 
 
-    def query(self, q):
-        """ to be finished """
+    def query(self, q, query_operator='and'):
+        """ evaluate query  """
 
         LL  = self.LexiconLookup
         PL  = self.SimilarityLookup
@@ -553,9 +554,12 @@
         txN = self.txNear
         txQ = self.txQuote
 
-        debug('Query: ',q)
+        debug('Query: ',q, query_operator)
 
-        parsed_query = QueryParser()(q) 
+        if self.useDumbParser:
+            parsed_query = DumbQueryParser()(q, query_operator)
+        else:
+            parsed_query = QueryParser()(q) 
 
         debug('parsed query: ', parsed_query)
 
@@ -654,7 +658,6 @@
         debug("\treturn: ",r)
 
         return r
-        
 
 
     def txUnion(self, *sets):
@@ -947,6 +950,6 @@
 
 def manage_addTextIndexNG(self, id, extra, REQUEST=None, RESPONSE=None, URL3=None):
     """Add a new TextIndexNG """
-    
+    extra.useDumbParser = 1
     return self.manage_addIndex(id, 'TextIndexNG', extra, REQUEST, RESPONSE, URL3)
 


=== Zope/lib/python/Products/PluginIndexes/TextIndexNG/interactiveDemo.py 1.1.2.5 => 1.1.2.6 ===
             help='max. distance between words for near search',default=5)
 
+    parser.add_option('-x','--dumbparser', action='store',type='int',
+            dest='extra_useDumbParser',
+            help='enable dumb query parser',default=0)
+
     parser.add_option('-c','--casefolding', action='store_true',
             dest='extra_splitterCasefolding', help='enable casefolding',
             default=1)
@@ -154,7 +158,6 @@
         if k.startswith('extra_'):
             setattr(E,k[6:],getattr(options,k))
 
-    
     cat = index_directory(  options.directory, 
                             E, 
                             options.verbose,