More simplification and deletion of unused code.

- Move code out of misc.py and into the file it's used
  - Rewrite Stack() and flatten() to be more concise
- pgen2/driver.py: remove unneeded string/file adapters.

There are now FIVE differences in regtest.sh.  They seem small though.

TODO: verify no diff in output disassembly.

opy/_regtest/benchmarks.txt

@@ -29,3 +29,15 @@ lisa:     3.46 lines / ms
Slower than OSH parser!


2018/03/18
----------

Did a bunch of major simplifications.  Hm seems to be no faster?  Slightly
slower.  Not sure if this is signifiant.

Lisa.

real    0m20.079s
user    0m18.280s
sys     0m1.633s

opy/compiler2/ast.py

@@ -4,17 +4,18 @@
"""
from .consts import CO_VARARGS, CO_VARKEYWORDS


# NOTE: Similar to pyassem.flatten().
def flatten(seq):
    l = []
    for elt in seq:
        t = type(elt)
        if t is tuple or t is list:
            for elt2 in flatten(elt):
                l.append(elt2)
        if isinstance(elt, (tuple, list)):
            l.extend(flatten(elt))
        else:
            l.append(elt)
    return l


def flatten_nodes(seq):
    return [n for n in flatten(seq) if isinstance(n, Node)]

opy/compiler2/misc.py

@@ -1,25 +1,4 @@

def flatten(tup):
    elts = []
    for elt in tup:
        if isinstance(elt, tuple):
            elts = elts + flatten(elt)
        else:
            elts.append(elt)
    return elts

class Stack(object):
    def __init__(self):
        self.stack = []
        self.pop = self.stack.pop
    def __len__(self):
        return len(self.stack)
    def push(self, elt):
        self.stack.append(elt)
    def top(self):
        return self.stack[-1]
    def __getitem__(self, index): # needed by visitContinue()
        return self.stack[index]
# mangle() is used by both symbols and pycodegen.

MANGLE_LEN = 256 # magic constant from compile.c

opy/compiler2/pyassem.py

@@ -5,9 +5,20 @@
import types
import sys

from . import misc
from .consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS


# NOTE: Similar to ast.flatten().
def flatten(tup):
    elts = []
    for elt in tup:
        if isinstance(elt, tuple):
            elts.extend(flatten(elt))
        else:
            elts.append(elt)
    return elts


class FlowGraph(object):
    def __init__(self):
        self.current = self.entry = Block()
@@ -581,7 +592,7 @@ def getArgCount(args):
    if args:
        for arg in args:
            if isinstance(arg, TupleArg):
                numNames = len(misc.flatten(arg.names))
                numNames = len(flatten(arg.names))
                argcount = argcount - numNames
    return argcount

opy/compiler2/pycodegen.py

@@ -128,6 +128,16 @@ def is_constant_false(node):
            return 1
    return 0


class Stack(list):

    def push(self, elt):
        self.append(elt)

    def top(self):
        return self[-1]


class CodeGenerator(object):
    """Defines basic code generator for Python bytecode

@@ -151,8 +161,8 @@ def __init__(self):
            self.initClass()
            self.__class__.__initialized = 1
        self.checkClass()
        self.locals = misc.Stack()
        self.setups = misc.Stack()
        self.locals = Stack()
        self.setups = Stack()
        self.last_lineno = None
        self._setupGraphDelegation()
        self._div_op = "BINARY_DIVIDE"

opy/compiler2/transformer.py

@@ -203,11 +203,7 @@ def Node(*args):
        #return apply(ast.Node, args)

class Transformer(object):
    """Utility object for transforming Python parse trees.

    Exposes the following methods:
        tree = transform(ast_tree)
    """
    """Transform a parse tree into an AST."""

    def __init__(self):
        self._dispatch = {}
@@ -225,19 +221,7 @@ def __init__(self):
                               }
        self.encoding = None

    def transform(self, tree):
        """Transform an AST into a modified parse tree."""
        # We only use Pgen2Transformer
        # if not (isinstance(tree, tuple) or isinstance(tree, list)):
        #    tree = parser.st2tuple(tree, line_info=1)
        return self.compile_node(tree)

    # --------------------------------------------------------------
    #
    # PRIVATE METHODS
    #

    def compile_node(self, node):
    def transform(self, node):
        ### emit a line-number node?
        n = node[0]

@@ -261,6 +245,11 @@ def compile_node(self, node):

        raise WalkerError('unexpected node type %r' % type_repr(n))

    # --------------------------------------------------------------
    #
    # PRIVATE METHODS
    #

    def single_input(self, node):
        ### do we want to do anything about being "interactive" ?

opy/opy_main.py

@@ -231,13 +231,12 @@ def OpyCommandMain(argv):
    py_path = argv[1]
    out_path = argv[2]

    # TODO: It shouldn't suck the whole file in!  It should read it line by line.
    with open(py_path) as f:
      tokens = tokenize.generate_tokens(f.readline)
      parse_tree = dr.parse_tokens(tokens, start_symbol=FILE_INPUT)
      tr = transformer.Transformer()
      co = pycodegen.compile(parse_tree, py_path, 'exec', transformer=tr)
    log("Code length: %d", len(co.co_code))
    log("Compiled to %d bytes of bytecode", len(co.co_code))

    # Write the .pyc file
    with open(out_path, 'wb') as out_f:

opy/pgen2/driver.py

@@ -80,25 +80,3 @@ def parse_tokens(self, tokens, start_symbol=None, debug=False):
            raise parse.ParseError("incomplete input",
                                   type, value, (prefix, start))
        return p.rootnode

    def parse_stream_raw(self, stream, debug=False):
        """Parse a stream and return the syntax tree."""
        tokens = tokenize.generate_tokens(stream.readline)
        return self.parse_tokens(tokens, debug)

    def parse_stream(self, stream, debug=False):
        """Parse a stream and return the syntax tree."""
        return self.parse_stream_raw(stream, debug)

    def parse_file(self, filename, encoding=None, debug=False):
        """Parse a file and return the syntax tree."""
        stream = codecs.open(filename, "r", encoding)
        try:
            return self.parse_stream(stream, debug)
        finally:
            stream.close()

    def parse_string(self, text, debug=False):
        """Parse a string and return the syntax tree."""
        tokens = tokenize.generate_tokens(io.StringIO(text).readline)
        return self.parse_tokens(tokens, debug)