'opyc compile -emit-docstring=0' omits docstrings.

(This is like python -OO)

- Wrote an args.OilFlags() parser to go along with this
- Simplify invocation of skeleton.Compile()
- Add some tests to test/opy.sh
- opyc dis: Conslidate printing of bit flags on CodeObject.
- rewrote a lot of comments, and remove unused code

core/args_test.py

@@ -176,6 +176,72 @@ def testParseLikeEcho(self):
     self.assertEqual(None, arg.n)
     self.assertEqual(0, i)
 
+  def testOilFlags(self):
+    s = args.OilFlags()
+    s.Flag('-docstring', args.Bool, default=True)
+    s.Flag('-out-file', args.Str)
+    s.Flag('-retries', args.Int)
+
+    arg, i = s.Parse(['-docstring=0', 'x', 'y'])
+    self.assertEqual(False, arg.docstring)
+    self.assertEqual(None, arg.out_file)
+    self.assertEqual(1, i)
+
+    # This turns it on too
+    arg, i = s.Parse(['-docstring', '0', 'x', 'y'])
+    self.assertEqual(True, arg.docstring)
+    self.assertEqual(None, arg.out_file)
+    self.assertEqual(1, i)
+
+    arg, i = s.Parse(['-out-file', 'out', 'y'])
+    self.assertEqual(True, arg.docstring)
+    self.assertEqual('out', arg.out_file)
+    self.assertEqual(2, i)
+
+    arg, i = s.Parse(['-retries', '3'])
+    self.assertEqual(3, arg.retries)
+
+    arg, i = s.Parse(['-retries=3'])
+    self.assertEqual(3, arg.retries)
+
+    # Like GNU: anything that starts with -- is parsed like an option.
+    self.assertRaises(args.UsageError, s.Parse, ['---'])
+
+    self.assertRaises(args.UsageError, s.Parse, ['-oops'])
+
+    # Invalid boolean arg
+    self.assertRaises(args.UsageError, s.Parse, ['--docstring=YEAH'])
+
+    arg, i = s.Parse(['--'])
+    self.assertEqual(1, i)
+
+    arg, i = s.Parse(['-'])
+    self.assertEqual(0, i)
+
+    arg, i = s.Parse(['abc'])
+    self.assertEqual(0, i)
+
+  def testFlagRegex(self):
+    import libc
+    CASES = [
+        '-',
+        '--',
+        '--+',
+        '---',  # invalid flag but valid arg
+        '---invalid',  # invalid flag but valid arg?
+        '-port',
+        '--port',
+        '--port-num',
+        '--port-num=8000',
+        '--port-num=',  # empty value
+        '--port-num=x=y',  # only first = matters
+
+        # We should point out the bad +.  It should match but then become an
+        # error.  It shoudl NOT be an arg!
+        '--port-num+',  # invalid
+        ]
+    for case in CASES:
+      print('%s\t%s' % (case, libc.regex_match(args._FLAG_ERE, case)))
 
 
 if __name__ == '__main__':

opy/compiler2/dis_tool.py

@@ -48,28 +48,17 @@ def to_hexstr(bytes_value, level=0, wrap=False):
         return template % tuple(ord(char) for char in bytes_value)
 
 
-# TODO: Unify this with _const() in consts.py.
-def build_flags_def(consts, co_flags_def):
-  for name in dir(consts):
-    if name.startswith('CO_'):
-      co_flags_def[name] = getattr(consts, name)
+def ShowFlags(flags):
+  flag_names = []
+  for bit in sorted(consts.VALUE_TO_NAME):
+    if flags & bit:
+      flag_names.append(consts.VALUE_TO_NAME[bit])
 
-
-_CO_FLAGS_DEF = {}
-build_flags_def(consts, _CO_FLAGS_DEF)
-
-
-def show_flags(value):
-    names = []
-    for name, bit in _CO_FLAGS_DEF.items():
-      if value & bit:
-        names.append(name)
-
-    h = "0x%05x" % value
-    if names:
-      return '%s %s' % (h, ' '.join(names))
-    else:
-      return h
+  h = "0x%05x" % flags
+  if flag_names:
+    return '%s %s' % (h, ' '.join(flag_names))
+  else:
+    return h
 
 
 def unpack_pyc(f):
@@ -243,7 +232,7 @@ def show_code(self, code, level=0):
       if isinstance(value, str):
         value = repr(value)
       elif name == "co_flags":
-        value = show_flags(value)
+        value = ShowFlags(value)
       elif name == "co_lnotab":
         value = "0x(%s)" % to_hexstr(value)
       print("%s%s%s" % (indent, (name+":").ljust(NAME_OFFSET), value))

opy/compiler2/pyassem.py

@@ -4,6 +4,8 @@
 import itertools
 import types
 
+from core import util
+
 from .consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS
 from opy.lib import dis
 
@@ -679,7 +681,7 @@ def MaxStackDepth(block_depths, entry_block, exit_block):
     return g.Max(entry_block, 0)
 
 
-def MakeCodeObject(frame, graph):
+def MakeCodeObject(frame, graph, comp_opt):
     """Order blocks, encode instructions, and create types.CodeType().
 
     Called by Compile below, and also recursively by ArgEncoder.
@@ -704,7 +706,12 @@ def MakeCodeObject(frame, graph):
     # What variables should be available at runtime?
 
     cellvars = ReorderCellVars(frame)
-    consts = [frame.docstring]
+
+    # NOTE: Modules docstrings are assigned to __doc__ in pycodegen.py.visitModule.
+    consts = []
+    if comp_opt.emit_docstring:
+      consts.append(frame.docstring)
+
     names = []
     # The closure list is used to track the order of cell variables and
     # free variables in the resulting code object.  The offsets used by
@@ -713,7 +720,7 @@ def MakeCodeObject(frame, graph):
 
     # Convert arguments from symbolic to concrete form.
     enc = ArgEncoder(frame.klass, consts, names, frame.varnames,
-                             closure)
+                     closure)
     # Mutates not only insts, but also appends to consts, names, etc.
     enc.Run(insts)
 

opy/compiler2/pycodegen.py

@@ -234,7 +234,7 @@ def set_lineno(self, node, force=False):
     def visitModule(self, node):
         self.scope = self.ctx.scopes[node]
         self.emit('SET_LINENO', 0)
-        if node.doc:
+        if node.doc and self.ctx.comp_opt.emit_docstring:
             self.emit('LOAD_CONST', node.doc)
             self.storeName('__doc__')
 
@@ -592,7 +592,7 @@ def _makeClosure(self, gen, args):
         frees = gen.scope.get_free_vars()
 
         # Recursive call!
-        co = pyassem.MakeCodeObject(gen.frame, gen.graph)
+        co = pyassem.MakeCodeObject(gen.frame, gen.graph, self.ctx.comp_opt)
 
         if frees:
             for name in frees:

opy/opy_main.py

@@ -4,7 +4,6 @@
 """
 from __future__ import print_function
 
-import cStringIO
 import hashlib
 import optparse
 import os
@@ -183,8 +182,8 @@ def WriteDisTables(pyc_path, co, out):
   # CO_VARARGS, CO_VAR_KEYWORDS, CO_GENERATOR, CO_NEWLOCALS (we only support
   # this?)  FUTURE_DIVISION, FUTURE_ABSOLUTE_IMPORT, etc.
   for flag in sorted(consts.VALUE_TO_NAME):
-    flag_name = consts.VALUE_TO_NAME[flag]
     if co.co_flags & flag:
+      flag_name = consts.VALUE_TO_NAME[flag]
       out.WriteFlagRow(pyc_path, co.co_name, flag_name)
 
   # Write a row for every constant
@@ -232,6 +231,9 @@ def OpyCommandMain(argv):
   except IndexError:
     raise args.UsageError('opy: Missing required subcommand.')
 
+  argv = argv[1:]  # TODO: Should I do input.ReadRequiredArg()?
+                   # That will shift the input.
+
   if action in (
       'parse', 'compile', 'dis', 'cfg', 'compile-ovm', 'eval', 'repl', 'run',
       'run-ovm'):
@@ -250,21 +252,23 @@ def OpyCommandMain(argv):
     symbols = Symbols(gr)
     pytree.Init(symbols)  # for type_repr() pretty printing
     transformer.Init(symbols)  # for _names and other dicts
-    tr = transformer.Transformer()
+
+    compiler = skeleton.Compiler(gr)
   else:
     # e.g. pgen2 doesn't use any of these.  Maybe we should make a different
     # tool.
-    gr = None
-    symbols = None
-    tr = None
+    compiler = None
+
+  compile_spec = args.OilFlags()
+  compile_spec.Flag('-emit-docstring', args.Bool, default=True)
 
   #
   # Actions
   #
 
   if action == 'pgen2':
-    grammar_path = argv[1]
-    pickle_path = argv[2]
+    grammar_path = argv[0]
+    pickle_path = argv[1]
     WriteGrammar(grammar_path, pickle_path)
 
   elif action == 'stdlib-parse':
@@ -283,14 +287,14 @@ def OpyCommandMain(argv):
     log('COUNT %d', n)
 
   elif action == 'lex':
-    py_path = argv[1]
+    py_path = argv[0]
     with open(py_path) as f:
       tokens = tokenize.generate_tokens(f.readline)
       for typ, val, start, end, unused_line in tokens:
         print('%10s %10s %-10s %r' % (start, end, token.tok_name[typ], val))
 
   elif action == 'parse':
-    py_path = argv[1]
+    py_path = argv[0]
     with open(py_path) as f:
       tokens = tokenize.generate_tokens(f.readline)
       p = parse.Parser(gr, convert=skeleton.py2st)
@@ -306,20 +310,25 @@ def OpyCommandMain(argv):
       tree.PrettyPrint(sys.stdout)
       log('\tChildren: %d' % len(tree.children), file=sys.stderr)
 
-  elif action == 'cfg':  # output fg
-    py_path = argv[1]
+  elif action == 'cfg':  # output Control Flow Graph
+    opt, i = compile_spec.Parse(argv)
+    py_path = argv[i]
     with open(py_path) as f:
-      graph = skeleton.Compile(f, py_path, gr, 'file_input', 'exec',
-                               return_cfg=True)
+      graph = compiler.Compile(f, opt, 'exec', return_cfg=True)
 
     print(graph)
 
   elif action == 'compile':  # 'opyc compile' is pgen2 + compiler2
-    py_path = argv[1]
-    out_path = argv[2]
+    # spec.Arg('action', ['foo', 'bar'])
+    # But that leads to some duplication.
+
+    opt, i = compile_spec.Parse(argv)
+
+    py_path = argv[i]
+    out_path = argv[i+1]
 
     with open(py_path) as f:
-      co = skeleton.Compile(f, py_path, gr, 'file_input', 'exec')
+      co = compiler.Compile(f, opt, 'exec')
 
     log("Compiled to %d bytes of bytecode", len(co.co_code))
 
@@ -330,12 +339,13 @@ def OpyCommandMain(argv):
       marshal.dump(co, out_f)
 
   elif action == 'compile-ovm':
-    py_path = argv[1]
-    out_path = argv[2]
+    opt, i = compile_spec.Parse(argv)
+    py_path = argv[i]
+    out_path = argv[i+1]
 
     # Compile to OVM bytecode
     with open(py_path) as f:
-      co = skeleton.Compile(f, py_path, gr, 'file_input', 'ovm')
+      co = compiler.Compile(f, opt, 'ovm')
 
     log("Compiled to %d bytes of bytecode", len(co.co_code))
     # Write the .pyc file
@@ -349,9 +359,10 @@ def OpyCommandMain(argv):
     log('Wrote only the bytecode to %r', out_path)
 
   elif action == 'eval':  # Like compile, but parses to a code object and prints it
-    py_expr = argv[1]
-    f = cStringIO.StringIO(py_expr)
-    co = skeleton.Compile(f, '<eval input>', gr, 'eval_input', 'eval')
+    opt, i = compile_spec.Parse(argv)
+    py_expr = argv[i]
+    f = skeleton.StringInput(py_expr, '<eval input>')
+    co = compiler.Compile(f, opt, 'eval')
 
     v = dis_tool.Visitor()
     v.show_code(co)
@@ -362,10 +373,10 @@ def OpyCommandMain(argv):
   elif action == 'repl':  # Like eval in a loop
     while True:
       py_expr = raw_input('opy> ')
-      f = cStringIO.StringIO(py_expr)
+      f = skeleton.StringInput(py_expr, '<REPL input>')
 
       # TODO: change this to 'single input'?  Why doesn't this work?
-      co = skeleton.Compile(f, '<REPL input>', gr, 'eval_input', 'eval')
+      co = skeleton.Compile(f, gr, 'eval')
 
       v = dis_tool.Visitor()
       v.show_code(co)
@@ -390,7 +401,7 @@ def OpyCommandMain(argv):
 
     if path.endswith('.py'):
       with open(path) as f:
-        co = skeleton.Compile(f, path, gr, 'file_input', 'exec')
+        co = skeleton.Compile(f, gr, 'exec')
 
       log("Compiled to %d bytes of bytecode", len(co.co_code))
       v.show_code(co)
@@ -417,20 +428,21 @@ def OpyCommandMain(argv):
           h.update(b)
       print('%6d %s %s' % (os.path.getsize(path), h.hexdigest(), path))
 
-  elif action == 'run':
+  elif action == 'run':  # Compile and run, without writing pyc file
     # TODO: Add an option like -v in __main__
 
     #level = logging.DEBUG if args.verbose else logging.WARNING
     #logging.basicConfig(level=level)
     #logging.basicConfig(level=logging.DEBUG)
 
-    # Compile and run, without writing pyc file
-    py_path = argv[1]
-    opy_argv = argv[1:]
+    opt, i = compile_spec.Parse(argv)
+
+    py_path = argv[i]
+    opy_argv = argv[i:]
 
     if py_path.endswith('.py'):
       with open(py_path) as f:
-        co = skeleton.Compile(f, py_path, gr, 'file_input', 'exec')
+        co = compiler.Compile(f, opt, 'exec')
       num_ticks = execfile.run_code_object(co, opy_argv)
 
     elif py_path.endswith('.pyc') or py_path.endswith('.opyc'):
@@ -442,16 +454,16 @@ def OpyCommandMain(argv):
     else:
       raise args.UsageError('Invalid path %r' % py_path)
 
-  elif action == 'run-ovm':
-    # Compile and run, without writing pyc file
+  elif action == 'run-ovm':  # Compile and run, without writing pyc file
+    opt, i = compile_spec.Parse(argv)
     py_path = argv[1]
     opy_argv = argv[1:]
 
     if py_path.endswith('.py'):
       #mode = 'exec'
       mode = 'ovm'  # OVM bytecode is different!
       with open(py_path) as f:
-        co = skeleton.Compile(f, py_path, gr, 'file_input', mode)
+        co = compiler.Compile(f, opt, mode)
       log('Compiled to %d bytes of OVM code', len(co.co_code))
       num_ticks = ovm.run_code_object(co, opy_argv)