[Zope-Checkins] CVS: Zope/lib/python/RestrictedPython/compiler - consts.py:1.2.16.1 future.py:1.2.16.1 misc.py:1.2.16.1 pyassem.py:1.2.16.3 pycodegen.py:1.4.2.1 symbols.py:1.3.8.2

Shane Hathaway shane@digicool.com
Fri, 21 Dec 2001 14:13:41 -0500


Update of /cvs-repository/Zope/lib/python/RestrictedPython/compiler
In directory cvs.zope.org:/tmp/cvs-serv5071/compiler

Modified Files:
      Tag: Zope-2_4-branch
	consts.py future.py misc.py pyassem.py pycodegen.py symbols.py 
Log Message:
Backported stacksize and other compiler bugfixes.


=== Zope/lib/python/RestrictedPython/compiler/consts.py 1.2 => 1.2.16.1 ===
-CO_VARARGS = 1
-CO_VARKEYWORDS = 2
-
 # operation flags
 OP_ASSIGN = 'OP_ASSIGN'
 OP_DELETE = 'OP_DELETE'
@@ -12,3 +8,9 @@
 SC_FREE = 3
 SC_CELL = 4
 SC_UNKNOWN = 5
+
+CO_OPTIMIZED = 0x0001
+CO_NEWLOCALS = 0x0002
+CO_VARARGS = 0x0004
+CO_VARKEYWORDS = 0x0008
+CO_NESTED = 0x0010


=== Zope/lib/python/RestrictedPython/compiler/future.py 1.2 => 1.2.16.1 ===
 if __name__ == "__main__":
     import sys
-    from compiler import parseFile, walk
+    from transformer import parseFile
 
     for file in sys.argv[1:]:
         print file


=== Zope/lib/python/RestrictedPython/compiler/misc.py 1.2 => 1.2.16.1 ===
     def top(self):
         return self.stack[-1]
+    def __getitem__(self, index): # needed by visitContinue()
+        return self.stack[index]
+
+MANGLE_LEN = 256 # magic constant from compile.c
+
+def mangle(name, klass):
+    if not name.startswith('__'):
+        return name
+    if len(name) + 2 >= MANGLE_LEN:
+        return name
+    if name.endswith('__'):
+        return name
+    try:
+        i = 0
+        while klass[i] == '_':
+            i = i + 1
+    except IndexError:
+        return name
+    klass = klass[i:]
+
+    tlen = len(klass) + len(name)
+    if tlen > MANGLE_LEN:
+        klass = klass[:MANGLE_LEN-tlen]
+
+    return "_%s%s" % (klass, name)
+


=== Zope/lib/python/RestrictedPython/compiler/pyassem.py 1.2.16.2 => 1.2.16.3 ===
 
 from __future__ import nested_scopes
+
 import dis
 import new
 import string
@@ -8,6 +9,7 @@
 import types
 
 import misc
+from consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS
 
 def xxx_sort(l):
     l = l[:]
@@ -54,7 +56,7 @@
         # these edges to get the blocks emitted in the right order,
         # however. :-(  If a client needs to remove these edges, call
         # pruneEdges().
-        
+
         self.current.addNext(block)
         self.startBlock(block)
 
@@ -109,13 +111,13 @@
 
         # XXX This is a total mess.  There must be a better way to get
         # the code blocks in the right order.
-        
+
         self.fixupOrderHonorNext(blocks, default_next)
         self.fixupOrderForward(blocks, default_next)
 
     def fixupOrderHonorNext(self, blocks, default_next):
         """Fix one problem with DFS.
-        
+
         The DFS uses child block, but doesn't know about the special
         "next" block.  As a result, the DFS can order blocks so that a
         block isn't next to the right block for implicit control
@@ -195,19 +197,18 @@
             chains.remove(c)
             chains.insert(goes_before, c)
 
-
         del blocks[:]
         for c in chains:
             for b in c:
                 blocks.append(b)
-            
+
     def getBlocks(self):
         return self.blocks.elements()
 
     def getRoot(self):
         """Return nodes appropriate for use with dominator"""
         return self.entry
-    
+
     def getContainedGraphs(self):
         l = []
         for b in self.getBlocks():
@@ -246,7 +247,7 @@
     def __str__(self):
         insts = map(str, self.insts)
         return "<block %s %d:\n%s>" % (self.label, self.bid,
-                                       string.join(insts, '\n')) 
+                                       string.join(insts, '\n'))
 
     def emit(self, inst):
         op = inst[0]
@@ -268,7 +269,7 @@
         assert len(self.next) == 1, map(str, self.next)
 
     _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS',
-                        'JUMP_ABSOLUTE', 'JUMP_FORWARD')
+                        'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP')
 
     def pruneNext(self):
         """Remove bogus edge for unconditional transfers
@@ -312,11 +313,6 @@
         return contained
 
 # flags for code objects
-CO_OPTIMIZED = 0x0001
-CO_NEWLOCALS = 0x0002
-CO_VARARGS = 0x0004
-CO_VARKEYWORDS = 0x0008
-CO_NESTED = 0x0010
 
 # the FlowGraph is transformed in place; it exists in one of these states
 RAW = "RAW"
@@ -327,15 +323,17 @@
 class PyFlowGraph(FlowGraph):
     super_init = FlowGraph.__init__
 
-    def __init__(self, name, filename, args=(), optimized=0):
+    def __init__(self, name, filename, args=(), optimized=0, klass=None):
         self.super_init()
         self.name = name
+        assert isinstance(filename, types.StringType)
         self.filename = filename
         self.docstring = None
         self.args = args # XXX
         self.argcount = getArgCount(args)
+        self.klass = klass
         if optimized:
-            self.flags = CO_OPTIMIZED | CO_NEWLOCALS 
+            self.flags = CO_OPTIMIZED | CO_NEWLOCALS
         else:
             self.flags = 0
         self.consts = []
@@ -364,6 +362,10 @@
         if flag == CO_VARARGS:
             self.argcount = self.argcount - 1
 
+    def checkFlag(self, flag):
+        if self.flags & flag:
+            return 1
+
     def setFreeVars(self, names):
         self.freevars = list(names)
 
@@ -462,7 +464,6 @@
                 insts[i] = opname, offset
             elif self.hasjabs.has_elt(opname):
                 insts[i] = opname, begin[inst[1]]
-        self.stacksize = findDepth(self.insts)
         self.stage = FLAT
 
     hasjrel = misc.Set()
@@ -480,8 +481,7 @@
         for i in range(len(self.insts)):
             t = self.insts[i]
             if len(t) == 2:
-                opname = t[0]
-                oparg = t[1]
+                opname, oparg = t
                 conv = self._converters.get(opname, None)
                 if conv:
                     self.insts[i] = opname, conv(self, oparg)
@@ -501,10 +501,16 @@
         self.closure = self.cellvars + self.freevars
 
     def _lookupName(self, name, list):
-        """Return index of name in list, appending if necessary"""
+        """Return index of name in list, appending if necessary
+
+        This routine uses a list instead of a dictionary, because a
+        dictionary can't store two different keys if the keys have the
+        same value but different types, e.g. 2 and 2L.  The compiler
+        must treat these two separately, so it does an explicit type
+        comparison before comparing the values.
+        """
         t = type(name)
         for i in range(len(list)):
-            # must do a comparison on type first to prevent UnicodeErrors 
             if t == type(list[i]) and list[i] == name:
                 return i
         end = len(list)
@@ -523,9 +529,15 @@
     _convert_STORE_FAST = _convert_LOAD_FAST
     _convert_DELETE_FAST = _convert_LOAD_FAST
 
+    def _convert_LOAD_NAME(self, arg):
+        if self.klass is None:
+            self._lookupName(arg, self.varnames)
+        return self._lookupName(arg, self.names)
+
     def _convert_NAME(self, arg):
+        if self.klass is None:
+            self._lookupName(arg, self.varnames)
         return self._lookupName(arg, self.names)
-    _convert_LOAD_NAME = _convert_NAME
     _convert_STORE_NAME = _convert_NAME
     _convert_DELETE_NAME = _convert_NAME
     _convert_IMPORT_NAME = _convert_NAME
@@ -557,7 +569,7 @@
     for name, obj in locals().items():
         if name[:9] == "_convert_":
             opname = name[9:]
-            _converters[opname] = obj            
+            _converters[opname] = obj
     del name, obj, opname
 
     def makeByteCode(self):
@@ -587,13 +599,14 @@
 
     def newCodeObject(self):
         assert self.stage == DONE
-        if self.flags == 0:
+        if (self.flags & CO_NEWLOCALS) == 0:
             nlocals = 0
         else:
             nlocals = len(self.varnames)
         argcount = self.argcount
         if self.flags & CO_VARKEYWORDS:
             argcount = argcount - 1
+
         return new.code(argcount, nlocals, self.stacksize, self.flags,
                         self.lnotab.getCode(), self.getConsts(),
                         tuple(self.names), tuple(self.varnames),
@@ -613,7 +626,7 @@
                 elt = elt.getCode()
             l.append(elt)
         return tuple(l)
-            
+
 def isJump(opname):
     if opname[:4] == 'JUMP':
         return 1
@@ -707,17 +720,19 @@
 
     def getTable(self):
         return string.join(map(chr, self.lnotab), '')
-    
+
 class StackDepthTracker:
     # XXX 1. need to keep track of stack depth on jumps
     # XXX 2. at least partly as a result, this code is broken
 
-    def findDepth(self, insts):
+    def findDepth(self, insts, debug=0):
         depth = 0
         maxDepth = 0
         for i in insts:
             opname = i[0]
-            delta = self.effect.get(opname)
+            if debug:
+                print i,
+            delta = self.effect.get(opname, None)
             if delta is not None:
                 depth = depth + delta
             else:
@@ -732,10 +747,10 @@
                     meth = getattr(self, opname, None)
                     if meth is not None:
                         depth = depth + meth(i[1])
-            if depth < 0:
-                depth = 0
             if depth > maxDepth:
                 maxDepth = depth
+            if debug:
+                print depth, maxDepth
         return maxDepth
 
     effect = {
@@ -756,9 +771,8 @@
         'DELETE_SUBSCR': -2,
         # PRINT_EXPR?
         'PRINT_ITEM': -1,
-        'LOAD_LOCALS': 1,
         'RETURN_VALUE': -1,
-        'EXEC_STMT': -2,
+        'EXEC_STMT': -3,
         'BUILD_CLASS': -2,
         'STORE_NAME': -1,
         'STORE_ATTR': -2,
@@ -774,23 +788,20 @@
         # close enough...
         'SETUP_EXCEPT': 3,
         'SETUP_FINALLY': 3,
-        'FOR_ITER': 1,
+        'FOR_LOOP': 1,
         }
     # use pattern match
     patterns = [
         ('BINARY_', -1),
         ('LOAD_', 1),
         ]
-    
-    # special cases:
-    # UNPACK_SEQUENCE, BUILD_TUPLE,
-    # BUILD_LIST, CALL_FUNCTION, MAKE_FUNCTION, BUILD_SLICE
+
     def UNPACK_SEQUENCE(self, count):
         return count-1
     def BUILD_TUPLE(self, count):
-        return 1-count
+        return -count+1
     def BUILD_LIST(self, count):
-        return 1-count
+        return -count+1
     def CALL_FUNCTION(self, argc):
         hi, lo = divmod(argc, 256)
         return -(lo + hi * 2)
@@ -802,6 +813,9 @@
         return self.CALL_FUNCTION(argc)-2
     def MAKE_FUNCTION(self, argc):
         return -argc
+    def MAKE_CLOSURE(self, argc):
+        # XXX need to account for free variables too!
+        return -argc
     def BUILD_SLICE(self, argc):
         if argc == 2:
             return -1
@@ -809,5 +823,5 @@
             return -2
     def DUP_TOPX(self, argc):
         return argc
-    
+
 findDepth = StackDepthTracker().findDepth


=== Zope/lib/python/RestrictedPython/compiler/pycodegen.py 1.4 => 1.4.2.1 === (426/526 lines abridged)
 import pyassem, misc, future, symbols
 from consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL
-from pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,\
-     CO_NESTED, TupleArg
+from consts import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, CO_NESTED
+from pyassem import TupleArg
 
 # Do we have Python 1.x or Python 2.x?
 try:
@@ -30,15 +30,24 @@
     (1,1) : "CALL_FUNCTION_VAR_KW",
 }
 
+LOOP = 1
+EXCEPT = 2
+TRY_FINALLY = 3
+END_FINALLY = 4
+
 def compile(filename, display=0):
     f = open(filename)
     buf = f.read()
     f.close()
     mod = Module(buf, filename)
-    mod.compile(display)
-    f = open(filename + "c", "wb")
-    mod.dump(f)
-    f.close()
+    try:
+        mod.compile(display)
+    except SyntaxError:
+        raise
+    else:
+        f = open(filename + "c", "wb")
+        mod.dump(f)
+        f.close()
 
 class Module:
     def __init__(self, source, filename):
@@ -117,6 +126,12 @@
     def visitAssName(self, node):
         self.names.add(node.name)
 
+def is_constant_false(node):
+    if isinstance(node, ast.Const):
+        if not node.value:
+            return 1
+    return 0
+
 class CodeGenerator:
     """Defines basic code generator for Python bytecode

[-=- -=- -=- 426 lines omitted -=- -=- -=-]

+    __super_init = AbstractClassCode.__init__
+
+    def __init__(self, klass, scopes, filename):
+        self.scopes = scopes
+        self.scope = scopes[klass]
+        self.__super_init(klass, scopes, filename)
+        self.graph.setFreeVars(self.scope.get_free_vars())
+        self.graph.setCellVars(self.scope.get_cell_vars())
+
 class NestedClassCodeGenerator(AbstractClassCode,
                                NestedScopeMixin,
                                NestedScopeCodeGenerator):
     super_init = NestedScopeCodeGenerator.__init__ # call be other init
     __super_init = AbstractClassCode.__init__
 
-    def __init__(self, klass, filename, scopes):
+    def __init__(self, klass, scopes, filename):
+        assert isinstance(filename, types.StringType)
         self.scopes = scopes
         self.scope = scopes[klass]
-        self.__super_init(klass, filename, scopes)
+        self.__super_init(klass, scopes, filename)
         self.graph.setFreeVars(self.scope.get_free_vars())
         self.graph.setCellVars(self.scope.get_cell_vars())
         self.graph.setFlag(CO_NESTED)
@@ -1212,7 +1287,7 @@
 def findOp(node):
     """Find the op (DELETE, LOAD, STORE) in an AssTuple tree"""
     v = OpFinder()
-    walk(node, v, 0)
+    walk(node, v, verbose=0)
     return v.op
 
 class OpFinder:
@@ -1224,6 +1299,7 @@
         elif self.op != node.flags:
             raise ValueError, "mixed ops in stmt"
     visitAssAttr = visitAssName
+    visitSubscript = visitAssName
 
 class Delegator:
     """Base class to support delegation for augmented assignment nodes
@@ -1253,6 +1329,7 @@
 
 class AugSubscript(Delegator):
     pass
+
 
 wrapper = {
     ast.Getattr: AugGetattr,


=== Zope/lib/python/RestrictedPython/compiler/symbols.py 1.3.8.1 => 1.3.8.2 ===
 import ast
 from consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL, SC_UNKNOWN
+from misc import mangle
 import types
 
 import sys
@@ -36,13 +37,7 @@
     def mangle(self, name):
         if self.klass is None:
             return name
-        if not name.startswith('__'):
-            return name
-        if len(name) + 2 >= MANGLE_LEN:
-            return name
-        if name.endswith('__'):
-            return name
-        return "_%s%s" % (self.klass, name)
+        return mangle(name, self.klass)
 
     def add_def(self, name):
         self.defs[self.mangle(name)] = 1
@@ -295,6 +290,27 @@
                 name = name[:i]
             scope.add_def(asname or name)
 
+    def visitGlobal(self, node, scope):
+        for name in node.names:
+            scope.add_global(name)
+
+    def visitAssign(self, node, scope):
+        """Propagate assignment flag down to child nodes.
+
+        The Assign node doesn't itself contains the variables being
+        assigned to.  Instead, the children in node.nodes are visited
+        with the assign flag set to true.  When the names occur in
+        those nodes, they are marked as defs.
+
+        Some names that occur in an assignment target are not bound by
+        the assignment, e.g. a name occurring inside a slice.  The
+        visitor handles these nodes specially; they do not propagate
+        the assign flag to their children.
+        """
+        for n in node.nodes:
+            self.visit(n, scope, 1)
+        self.visit(node.expr, scope)
+
     def visitAssName(self, node, scope, assign=1):
         scope.add_def(node.name)
 
@@ -306,6 +322,13 @@
         for n in node.subs:
             self.visit(n, scope, 0)
 
+    def visitSlice(self, node, scope, assign=0):
+        self.visit(node.expr, scope, 0)
+        if node.lower:
+            self.visit(node.lower, scope, 0)
+        if node.upper:
+            self.visit(node.upper, scope, 0)
+
     def visitAugAssign(self, node, scope):
         # If the LHS is a name, then this counts as assignment.
         # Otherwise, it's just use.
@@ -314,15 +337,6 @@
             self.visit(node.node, scope, 1) # XXX worry about this
         self.visit(node.expr, scope)
 
-    def visitAssign(self, node, scope):
-        for n in node.nodes:
-            self.visit(n, scope, 1)
-        self.visit(node.expr, scope)
-
-    def visitGlobal(self, node, scope):
-        for name in node.names:
-            scope.add_global(name)
-
     # prune if statements if tests are false
 
     _const_types = types.StringType, types.IntType, types.FloatType
@@ -348,7 +362,8 @@
 
 if __name__ == "__main__":
     import sys
-    from compiler import parseFile, walk
+    from transformer import parseFile
+    from visitor import walk
     import symtable
 
     def get_names(syms):