Add opyc actions: lex, eval, and repl.

opyc now exposes the whole compiler pipeline.

Also, remove unused code in opy_main.py.

opy/compiler2/pycodegen.py

@@ -89,17 +89,17 @@ class Expression(AbstractCompileMode):

    mode = "eval"

    def compile(self):
        tree = self._get_tree()
    def compile(self, transformer=None):
        tree = self._get_tree(transformer=transformer)
        gen = ExpressionCodeGenerator(tree)
        self.code = gen.getCode()

class Interactive(AbstractCompileMode):

    mode = "single"

    def compile(self):
        tree = self._get_tree()
    def compile(self, transformer=None):
        tree = self._get_tree(transformer=transformer)
        gen = InteractiveCodeGenerator(tree)
        self.code = gen.getCode()

opy/compiler2/transformer.py

@@ -25,9 +25,6 @@
#   http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.

# NOTE: For the unused parser.suite() and parser.expr()
import parser

from .ast import *
from .consts import CO_VARARGS, CO_VARKEYWORDS
from .consts import OP_ASSIGN, OP_DELETE, OP_APPLY
@@ -261,11 +258,13 @@ def transform(self, tree):

    def parsesuite(self, text):
        """Return a modified parse tree for the given suite text."""
        return self.transform(parser.suite(text))
        raise AssertionError("Shouldn't use stdlib parser")
        #return self.transform(parser.suite(text))

    def parseexpr(self, text):
        """Return a modified parse tree for the given expression text."""
        return self.transform(parser.expr(text))
        raise AssertionError("Shouldn't use stdlib parser")
        #return self.transform(parser.expr(text))

    def parsefile(self, file):
        """Return a modified parse tree for the contents of the given file."""
@@ -1569,6 +1568,11 @@ def parsesuite(self, text):
    #self.printer.Print(tree)
    return self.transform(tree)

  def parseexpr(self, text):
    tree = self.py_parser.expr(text)
    #self.printer.Print(tree)
    return self.transform(tree)


def debug_tree(tree):
    l = []

opy/opy_main.py

@@ -19,6 +19,7 @@
#sys.path.append(os.path.join(this_dir))

from .pgen2 import driver, pgen, grammar
from .pgen2 import token
from .pgen2 import tokenize
from . import pytree

@@ -81,12 +82,19 @@ def __init__(self, driver, start_symbol):
    self.driver = driver
    self.start_symbol = start_symbol

  # TODO: Remove this redundancy
  def suite(self, text):
    f = cStringIO.StringIO(text)
    tokens = tokenize.generate_tokens(f.readline)
    tree = self.driver.parse_tokens(tokens, start_symbol=self.start_symbol)
    return tree

  def expr(self, text):
    f = cStringIO.StringIO(text)
    tokens = tokenize.generate_tokens(f.readline)
    tree = self.driver.parse_tokens(tokens, start_symbol=self.start_symbol)
    return tree


def CountTupleTree(tu):
  """Count the nodes in a tuple parse tree."""
@@ -136,6 +144,25 @@ def Options():
  return p


# Emulating parser.st structures from parsermodule.c.
# They have a totuple() method, which outputs tuples like this.
def py2st(gr, raw_node):
  typ, value, context, children = raw_node
  # See pytree.Leaf
  if context:
    _, (lineno, column) = context
  else:
    lineno = 0  # default in Leaf
    column = 0

  if children:
    return (typ,) + tuple(children)
  else:
    return (typ, value, lineno, column)

# TODO: more actions:
# - lex, parse, ast, cfg, compile/eval/repl

# Made by the Makefile.
PICKLE_REL_PATH = '_build/opy/py27.grammar.pickle'

@@ -150,7 +177,7 @@ def OpyCommandMain(argv):
  except IndexError:
    raise args.UsageError('opy: Missing required subcommand.')

  if action in ('parse', 'compile'):
  if action in ('parse', 'compile', 'eval', 'repl'):
    loader = util.GetResourceLoader()
    f = loader.open(PICKLE_REL_PATH)
    gr = grammar.Grammar()
@@ -175,30 +202,7 @@ def OpyCommandMain(argv):
    FILE_INPUT = None
    symbols = None

  #do_glue = False
  do_glue = True

  if do_glue:  # Make it a flag
    # Emulating parser.st structures from parsermodule.c.
    # They have a totuple() method, which outputs tuples like this.
    def py2st(gr, raw_node):
      type, value, context, children = raw_node
      # See pytree.Leaf
      if context:
        _, (lineno, column) = context
      else:
        lineno = 0  # default in Leaf
        column = 0

      if children:
        return (type,) + tuple(children)
      else:
        return (type, value, lineno, column)
    convert = py2st
  else:
    convert = pytree.convert

  dr = driver.Driver(gr, convert=convert)
  dr = driver.Driver(gr, convert=py2st)

  if action == 'pgen2':
    grammar_path = argv[1]
@@ -220,6 +224,13 @@ def py2st(gr, raw_node):
    printer = TupleTreePrinter(HostStdlibNames())
    printer.Print(tree)

  elif action == 'lex':
    py_path = argv[1]
    with open(py_path) as f:
      tokens = tokenize.generate_tokens(f.readline)
      for typ, val, start, end, unused_line in tokens:
        print('%10s %10s %-10s %r' % (start, end, token.tok_name[typ], val))

  elif action == 'parse':
    py_path = argv[1]
    with open(py_path) as f:
@@ -236,19 +247,15 @@ def py2st(gr, raw_node):
      tree.PrettyPrint(sys.stdout)
      log('\tChildren: %d' % len(tree.children), file=sys.stderr)

  elif action == 'compile':
    # 'opy compile' is pgen2 + compiler2

  elif action == 'compile':  # 'opyc compile' is pgen2 + compiler2
    py_path = argv[1]
    out_path = argv[2]

    if do_glue:
      py_parser = Pgen2PythonParser(dr, FILE_INPUT)
      printer = TupleTreePrinter(transformer._names)
      tr = transformer.Pgen2Transformer(py_parser, printer)
    else:
      tr = transformer.Transformer()
    py_parser = Pgen2PythonParser(dr, FILE_INPUT)
    printer = TupleTreePrinter(transformer._names)
    tr = transformer.Pgen2Transformer(py_parser, printer)

    # TODO: It shouldn't suck the whole file in!  It should read it line by line.
    with open(py_path) as f:
      contents = f.read()
    co = pycodegen.compile(contents, py_path, 'exec', transformer=tr)
@@ -260,6 +267,34 @@ def py2st(gr, raw_node):
      out_f.write(h)
      marshal.dump(co, out_f)

  elif action == 'eval':  # Like compile, but parses to a code object and prints it
    py_expr = argv[1]

    py_parser = Pgen2PythonParser(dr, gr.symbol2number['eval_input'])
    printer = TupleTreePrinter(transformer._names)
    tr = transformer.Pgen2Transformer(py_parser, printer)

    co = pycodegen.compile(py_expr, '<eval input>', 'eval', transformer=tr)

    v = dis_tool.Visitor()
    v.show_code(co)
    print()
    print('RESULT:')
    print(eval(co))

  elif action == 'repl':  # Like eval in a loop
    while True:
      py_expr = raw_input('opy> ')
      py_parser = Pgen2PythonParser(dr, gr.symbol2number['eval_input'])
      printer = TupleTreePrinter(transformer._names)
      tr = transformer.Pgen2Transformer(py_parser, printer)

      co = pycodegen.compile(py_expr, '<eval input>', 'eval', transformer=tr)

      v = dis_tool.Visitor()
      v.show_code(co)
      print(eval(co))

  elif action == 'dis':
    pyc_path = argv[1]
    try:
@@ -294,29 +329,6 @@ def py2st(gr, raw_node):
          h.update(b)
      print('%6d %s %s' % (os.path.getsize(path), h.hexdigest(), path))

  # NOTE: Unused
  elif action == 'old-compile':
    py_path = argv[1]
    out_path = argv[2]

    if do_glue:
      py_parser = Pgen2PythonParser(dr, FILE_INPUT)
      printer = TupleTreePrinter(transformer._names)
      tr = transformer.Pgen2Transformer(py_parser, printer)
    else:
      tr = transformer.Transformer()

    f = open(py_path)
    contents = f.read()
    co = pycodegen.compile(contents, py_path, 'exec', transformer=tr)
    log("Code length: %d", len(co.co_code))

    # Write the .pyc file
    with open(out_path, 'wb') as out_f:
      h = pycodegen.getPycHeader(py_path)
      out_f.write(h)
      marshal.dump(co, out_f)

  # TODO: Not used
  elif action == 'compile2':
    in_path = argv[1]

opy/pytree.py

@@ -365,24 +365,6 @@ def _prefix_setter(self, prefix):

    prefix = property(_prefix_getter, _prefix_setter)

def convert(gr, raw_node):
    """
    Convert raw node information to a Node or Leaf instance.

    This is passed to the parser driver which calls it whenever a reduction of a
    grammar rule produces a new complete node, so that the tree is build
    strictly bottom-up.
    """
    type, value, context, children = raw_node
    if children or type in gr.number2symbol:
        # If there's exactly one child, return that child instead of
        # creating a new node.
        if len(children) == 1:
            return children[0]
        return Node(type, children, context=context)
    else:
        return Leaf(type, value, context=context)


class BasePattern(object):