Add the option to put xtrace output on --debug-file.

Also:

- Cleanup of completion initialization
- Cleanup of debug logging for completion
- Fix lint errors

bin/oil.py

@@ -167,7 +167,7 @@ def OshMain(argv0, argv, login_shell):
   # Controlled by env variable, flag, or hook?
   dumper = dev.CrashDumper(os.getenv('OSH_CRASH_DUMP_DIR', ''))
   ex = cmd_exec.Executor(mem, fd_state, funcs, comp_lookup, exec_opts,
-                         parse_ctx, dumper)
+                         parse_ctx, dumper, debug_f)
 
   # NOTE: The rc file can contain both commands and functions... ideally we
   # would only want to save nodes/lines for the functions.
@@ -222,11 +222,27 @@ def OshMain(argv0, argv, login_shell):
     # NOTE: We're using a different evaluator here.  The completion system can
     # also run functions... it gets the Executor through Executor._Complete.
     if HAVE_READLINE:
-      progress_f = ui.StatusLine()
-      splitter = legacy.SplitContext(mem)
+      splitter = legacy.SplitContext(mem)  # TODO: share with executor.
       ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter)
-      completion.Init(readline, pool, ex, comp_lookup, progress_f, debug_f,
-                      ev, parse_ctx)
+      progress_f = ui.StatusLine()
+      var_action = completion.VariablesActionInternal(ex.mem)
+      root_comp = completion.RootCompleter(ev, comp_lookup, var_action,
+                                           parse_ctx, progress_f, debug_f)
+      completion.Init(readline, root_comp, debug_f)
+
+      from core import comp_builtins
+      # register builtins and words
+      comp_builtins.Complete(['-E', '-A', 'command'], ex, comp_lookup)
+      # register path completion
+      comp_builtins.Complete(['-D', '-A', 'file'], ex, comp_lookup)
+
+      if 1:
+        # Something for fun, to show off.  Also: test that you don't repeatedly hit
+        # the file system / network / coprocess.
+        A1 = completion.WordsAction(['foo.py', 'foo', 'bar.py'])
+        A2 = completion.WordsAction(['m%d' % i for i in range(5)], delay=0.1)
+        C1 = completion.ChainedCompleter([A1, A2])
+        comp_lookup.RegisterName('slowc', C1)
 
     return main_loop.Interactive(opts, ex, c_parser, arena)
 

core/cmd_exec.py

@@ -107,19 +107,18 @@ class Executor(object):
   CompoundWord/WordPart.
   """
   def __init__(self, mem, fd_state, funcs, comp_lookup, exec_opts, parse_ctx,
-               dumper):
+               dumper, debug_f):
     """
     Args:
       mem: Mem instance for storing variables
       fd_state: FdState() for managing descriptors
-      funcs: registry of functions (these names are completed)
-      comp_lookup: completion pattern/action
+      funcs: dict of functions
+      comp_lookup: registry of completion hooks
       exec_opts: ExecOpts
       parse_ctx: for instantiating parsers
     """
     self.mem = mem
     self.fd_state = fd_state
-    # function space is different than var space.  Not hierarchical.
     self.funcs = funcs
     # Completion hooks, set by 'complete' builtin.
     self.comp_lookup = comp_lookup
@@ -131,8 +130,8 @@ def __init__(self, mem, fd_state, funcs, comp_lookup, exec_opts, parse_ctx,
     self.dumper = dumper
 
     self.splitter = legacy.SplitContext(self.mem)
-    self.word_ev = word_eval.NormalWordEvaluator(
-        mem, exec_opts, self.splitter, self)
+    self.word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, self.splitter,
+                                                 self)
     self.arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, self.word_ev)
     self.bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, self.word_ev)
 
@@ -150,7 +149,11 @@ def __init__(self, mem, fd_state, funcs, comp_lookup, exec_opts, parse_ctx,
 
     self.loop_level = 0  # for detecting bad top-level break/continue
 
-    self.tracer = Tracer(parse_ctx, exec_opts, mem, self.word_ev)
+    if 1:
+      trace_f = debug_f
+    else:
+      trace_f = util.DebugFile(sys.stderr)
+    self.tracer = Tracer(parse_ctx, exec_opts, mem, self.word_ev, trace_f)
     self.check_command_sub_status = False  # a hack
 
   def _EvalHelper(self, c_parser, source_name):
@@ -1398,9 +1401,10 @@ class Tracer(object):
     - set -x doesn't print line numbers!  OH but you can do that with
       PS4=$LINENO
   """
-  def __init__(self, parse_ctx, exec_opts, mem, word_ev):
+  def __init__(self, parse_ctx, exec_opts, mem, word_ev, f):
     """
     Args:
+      parse_ctx: For parsing PS4.
       exec_opts: For xtrace setting
       mem: for retrieving PS4
       word_ev: for evaluating PS4
@@ -1409,6 +1413,7 @@ def __init__(self, parse_ctx, exec_opts, mem, word_ev):
     self.exec_opts = exec_opts
     self.mem = mem
     self.word_ev = word_ev
+    self.f = f  # can be the --debug-file as well
 
     self.arena = alloc.SideArena('<$PS4>')
     self.parse_cache = {}  # PS4 value -> CompoundWord.  PS4 is scoped.
@@ -1463,7 +1468,7 @@ def OnSimpleCommand(self, argv):
 
     first_char, prefix = self._EvalPS4()
     cmd = ' '.join(pretty.Str(a) for a in argv)
-    print('%s%s%s' % (first_char, prefix, cmd), file=sys.stderr)
+    self.f.log('%s%s%s', first_char, prefix, cmd)
 
   def OnAssignment(self, lval, val, flags, lookup_mode):
     # NOTE: I think tracing should be on by default?  For post-mortem viewing.
@@ -1472,7 +1477,7 @@ def OnAssignment(self, lval, val, flags, lookup_mode):
 
     # Now we have to get the prefix
     first_char, prefix = self._EvalPS4()
-    print('%s%s%s = %s' % (first_char, prefix, lval, val), file=sys.stderr)
+    self.f.log('%s%s%s = %s', first_char, prefix, lval, val)
 
   def Event(self):
     """

core/cmd_exec_test.py

@@ -10,6 +10,7 @@
 """
 
 import unittest
+import sys
 
 from core import cmd_exec  # module under test
 from core import dev
@@ -18,6 +19,7 @@
 from core import process
 from core import state
 from core import test_lib
+from core import util
 
 from osh.meta import ast, Id
 from osh import parse_lib
@@ -43,8 +45,9 @@ def InitExecutor(arena=None):
   exec_opts = state.ExecOpts(mem, None)
   parse_ctx = parse_lib.ParseContext(arena, {})
   dumper = dev.CrashDumper('')
+  debug_f = util.DebugFile(sys.stderr)
   return cmd_exec.Executor(mem, fd_state, funcs, comp_funcs, exec_opts,
-                           parse_ctx, dumper)
+                           parse_ctx, dumper, debug_f)
 
 
 def InitEvaluator():

core/comp_builtins.py

@@ -3,8 +3,6 @@
 comp_builtins.py - Completion builtins
 """
 
-import os
-
 from core import args
 from core import builtin
 from core import completion
@@ -206,7 +204,7 @@ def Complete(argv, ex, comp_lookup):
 
   patterns = []
   for pat in patterns:
-    comp_lookup.RegisterGlob(pat, action)
+    comp_lookup.RegisterGlob(pat, chain)
 
   return 0
 

core/completion.py

@@ -334,6 +334,8 @@ def Matches(self, words, index, to_complete):
         self.cache[key] = listing
       names.extend(listing)
 
+    # TODO: Shouldn't do the prefix / space thing ourselves.  readline does
+    # that at the END of the line.
     for word in listing:
       if word.startswith(to_complete):
         yield word + ' '
@@ -512,14 +514,15 @@ def _GetCompletionType(w_parser, c_parser, ev, debug_f):
     pass
 
   # TODO: Need to show buf... Need a multiline display for debugging?
-  debug_f.log('prev_token %s  cur_token %s  cur_word %s',
-      prev_token, cur_token, cur_word)
-  debug_f.log('comp_state %s  error %s', comp_state, c_parser.Error())
-  # This one can be multiple lines
-  debug_f.log('node: %s %s', repr(node) if node else '<Parse Error>',
-                node.tag if node else '')
-  # This one can be multiple lines
-  debug_f.log('com_node: %s', repr(com_node) if com_node else '<None>')
+  if 0:
+    debug_f.log('prev_token %s  cur_token %s  cur_word %s',
+        prev_token, cur_token, cur_word)
+    debug_f.log('comp_state %s  error %s', comp_state, c_parser.Error())
+    # This one can be multiple lines
+    debug_f.log('node: %s %s', repr(node) if node else '<Parse Error>',
+                  node.tag if node else '')
+    # This one can be multiple lines
+    debug_f.log('com_node: %s', repr(com_node) if com_node else '<None>')
 
   # IMPORTANT: if the last token is Id.Ignored_Space, then we want to add a
   # dummy word!  empty word
@@ -591,9 +594,8 @@ class RootCompleter(object):
   """
   Provide completion of a buffer according to the configured rules.
   """
-  def __init__(self, pool, ev, comp_lookup, var_comp, parse_ctx, progress_f,
+  def __init__(self, ev, comp_lookup, var_comp, parse_ctx, progress_f,
                debug_f):
-    self.pool = pool
     self.ev = ev
     self.comp_lookup = comp_lookup
     # This can happen in any position, with any command
@@ -639,6 +641,8 @@ def Matches(self, buf):
     self.progress_f.Write('Completing %r ... (Ctrl-C to cancel)', buf)
     start_time = time.time()
 
+    self.debug_f.log('Using %s', chain)
+
     index = len(comp_words) - 1  # COMP_CWORD -1 when it's empty
     i = 0
     for m in chain.Matches(comp_words, index, to_complete):
@@ -664,11 +668,10 @@ def Matches(self, buf):
 
 
 class ReadlineCompleter(object):
-  def __init__(self, readline_mod, root_comp, debug_f, debug=False):
+  def __init__(self, readline_mod, root_comp, debug_f):
     self.readline_mod = readline_mod
     self.root_comp = root_comp
     self.debug_f = debug_f
-    self.debug = debug
 
     self.comp_iter = None  # current completion being processed
 
@@ -687,10 +690,9 @@ def _GetNextCompletion(self, state):
       # The current position of the cursor.  The thing being completed.
       end = self.readline_mod.get_endidx()
 
-      if self.debug:
-        self.debug_f.log(
-            'line: %r / begin - end: %d - %d, part: %r', buf, begin, end,
-            buf[begin:end])
+      self.debug_f.log(
+          'line: %r / begin - end: %d - %d, part: %r', buf, begin, end,
+          buf[begin:end])
 
       self.comp_iter = self.root_comp.Matches(buf)
 
@@ -751,28 +753,7 @@ def InitReadline(readline_mod, complete_cb):
   readline_mod.set_completer_delims(' ')
 
 
-def Init(readline_mod, pool, ex, comp_lookup, progress_f, debug_f, ev,
-         parse_ctx):
-
-  from core import comp_builtins
-
-  # TODO: Need a space
-  # register builtins and words
-  comp_builtins.Complete(['-E', '-A', 'command'], ex, comp_lookup)
-  # register path completion
-  comp_builtins.Complete(['-D', '-A', 'file'], ex, comp_lookup)
-
-  # Something for fun, to show off.  Also: test that you don't repeatedly hit
-  # the file system / network / coprocess.
-  A1 = WordsAction(['foo.py', 'foo', 'bar.py'])
-  A2 = WordsAction(['m%d' % i for i in range(5)], delay=0.1)
-  C1 = ChainedCompleter([A1, A2])
-  comp_lookup.RegisterName('slowc', C1)
-
-  var_comp = VariablesActionInternal(ex.mem)
-  root_comp = RootCompleter(pool, ev, comp_lookup, var_comp, parse_ctx,
-                            progress_f, debug_f)
-
+def Init(readline_mod, root_comp, debug_f):
   complete_cb = ReadlineCompleter(readline_mod, root_comp, debug_f)
   InitReadline(readline_mod, complete_cb)
 

core/completion_test.py

@@ -162,7 +162,7 @@ def testRootCompleter(self):
     arena = pool.NewArena()
     parse_ctx = parse_lib.ParseContext(arena, {})
     var_comp = V1
-    r = completion.RootCompleter(pool, ev, comp_lookup, var_comp, parse_ctx,
+    r = completion.RootCompleter(ev, comp_lookup, var_comp, parse_ctx,
                                  progress_f, debug_f)
 
     m = list(r.Matches('grep f'))

core/ui.py

@@ -16,7 +16,6 @@
 from asdl import encode
 from asdl import format as fmt
 from core import dev
-from core import word
 from osh import ast_lib
 from osh.meta import ast
 

core/util.py

@@ -256,9 +256,6 @@ def __init__(self, f):
   def log(self, msg, *args):
     if args:
       msg = msg % args
-    # TODO: Don't get pid every time.  Should there be one of these per
-    # process?
-    self.f.write('%d ' % os.getpid())
     self.f.write(msg)
     self.f.write('\n')
     self.f.flush()  # need to see it interacitvely