[translation] Instantiating objects in main() in bin/osh_eval.

- Create a vm::_Executor interface.  The shell executor won't be linked
  in to osh_eval at first.
  - Hacks in mycpp to support this.
- util.DebugFile() now type checks and translates.  Needed a flush()
  method on class Writer.

bin/oil.py

@@ -19,7 +19,10 @@
 import posix_ as posix
 import sys
 import time  # for perf measurement
-from typing import List, NoReturn
+from typing import List, Dict, NoReturn, TYPE_CHECKING
+
+if TYPE_CHECKING:
+  from _devbuild.gen.syntax_asdl import command__ShFunction
 
 _trace_path = posix.environ.get('_PY_TRACE')
 if _trace_path:

bin/osh_eval.py

@@ -21,6 +21,7 @@
 from core import util
 from core import state
 from core import ui
+from core.vm import _Executor  # reordered by mycpp
 from frontend import parse_lib
 from frontend import reader
 from mycpp import mylib
@@ -39,9 +40,9 @@
 if TYPE_CHECKING:
   from _devbuild.gen.syntax_asdl import command__ShFunction
   from _devbuild.gen.runtime_asdl import cmd_value__Argv
+  from osh.builtin_misc import _Builtin
   from osh.cmd_parse import CommandParser
   from pgen2.grammar import Grammar
-  from osh.builtin_misc import _Builtin
 
 
 if mylib.PYTHON:
@@ -249,15 +250,15 @@ def main(argv):
   cmd_ev.arith_ev = arith_ev
   cmd_ev.word_ev = word_ev
   cmd_ev.tracer = tracer
-  #cmd_ev.shell_ex = ex
+  cmd_ev.shell_ex = ex
 
   if 0:
     cmd_ev.ExecuteAndCatch(node)
 
   return 0
 
 
-class NullExecutor(object):
+class NullExecutor(_Executor):
 
   def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
     # type: (cmd_value__Argv, bool, bool) -> int

core/executor.py

@@ -17,6 +17,7 @@
 from core import error
 from core import process
 from core.util import log, e_die
+from core.vm import _Executor
 from frontend import args
 from frontend import consts
 from oil_lang import objects
@@ -39,7 +40,7 @@
   from osh import cmd_eval
 
 
-class ShellExecutor(object):
+class ShellExecutor(_Executor):
   """
   This CommandEvaluator is combined with the OSH language evaluators in osh/ to create
   a shell interpreter.
@@ -459,7 +460,6 @@ def RunProcessSub(self, node, op_id):
     else:
       raise AssertionError()
 
-
   def Time(self):
     # type: () -> None
     pass

core/util.py

@@ -12,7 +12,10 @@
 
 import sys
 from core import error
-from typing import IO, NoReturn, Any
+from typing import NoReturn, Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+  from mycpp import mylib
 
 
 class UserExit(Exception):
@@ -100,7 +103,7 @@ def ShowFdState():
 
 class DebugFile(object):
   def __init__(self, f):
-    # type: (IO[str]) -> None
+    # type: (mylib.Writer) -> None
     self.f = f
 
   def log(self, msg, *args):

core/vm.py

@@ -4,46 +4,102 @@
 """
 from __future__ import print_function
 
-from typing import TYPE_CHECKING
+from mycpp import mylib
+
+from typing import List, TYPE_CHECKING
 if TYPE_CHECKING:
+  from _devbuild.gen.id_kind_asdl import Id_t
+  from _devbuild.gen.runtime_asdl import cmd_value__Argv, redirect
+  from _devbuild.gen.syntax_asdl import (
+      command_t, command__Pipeline, command__Subshell
+  )
   from osh.sh_expr_eval import ArithEvaluator
   from osh.sh_expr_eval import BoolEvaluator
   from oil_lang.expr_eval import OilEvaluator
   from osh.word_eval import NormalWordEvaluator
   from osh.cmd_eval import CommandEvaluator
   from osh import prompt
   from core import dev
-  from core import executor
 
 
-def InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex, prompt_ev, tracer):
-  # type: (ArithEvaluator, BoolEvaluator, OilEvaluator, NormalWordEvaluator, CommandEvaluator, executor.ShellExecutor, prompt.Evaluator, dev.Tracer) -> None
-  arith_ev.word_ev = word_ev
-  bool_ev.word_ev = word_ev
+if mylib.PYTHON:
+  def InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex, prompt_ev, tracer):
+    # type: (ArithEvaluator, BoolEvaluator, OilEvaluator, NormalWordEvaluator, CommandEvaluator, _Executor, prompt.Evaluator, dev.Tracer) -> None
+    arith_ev.word_ev = word_ev
+    bool_ev.word_ev = word_ev
+
+    expr_ev.shell_ex = shell_ex
+    expr_ev.word_ev = word_ev
+
+    word_ev.arith_ev = arith_ev
+    word_ev.expr_ev = expr_ev
+    word_ev.prompt_ev = prompt_ev
+    word_ev.shell_ex = shell_ex
+
+    cmd_ev.shell_ex = shell_ex
+    cmd_ev.arith_ev = arith_ev
+    cmd_ev.bool_ev = bool_ev
+    cmd_ev.expr_ev = expr_ev
+    cmd_ev.word_ev = word_ev
+    cmd_ev.tracer = tracer
+
+    shell_ex.cmd_ev = cmd_ev
+
+    prompt_ev.word_ev = word_ev
+
+    arith_ev.CheckCircularDeps()
+    bool_ev.CheckCircularDeps()
+    expr_ev.CheckCircularDeps()
+    word_ev.CheckCircularDeps()
+    cmd_ev.CheckCircularDeps()
+    shell_ex.CheckCircularDeps()
+    prompt_ev.CheckCircularDeps()
+
+
+class _Executor(object):
+
+  def __init__(self):
+    # type: () -> None
+    self.cmd_ev = None  # type: CommandEvaluator
+
+  def CheckCircularDeps(self):
+    # type: () -> None
+    pass
+
+  def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
+    # type: (cmd_value__Argv, bool, bool) -> int
+    pass
+
+  def RunBackgroundJob(self, node):
+    # type: (command_t) -> int
+    pass
+
+  def RunPipeline(self, node):
+    # type: (command__Pipeline) -> int
+    pass
+
+  def RunSubshell(self, node):
+    # type: (command__Subshell) -> int
+    pass
+
+  def RunCommandSub(self, node):
+    # type: (command_t) -> str
+    pass
 
-  expr_ev.shell_ex = shell_ex
-  expr_ev.word_ev = word_ev
+  def RunProcessSub(self, node, op_id):
+    # type: (command_t, Id_t) -> str
+    pass
 
-  word_ev.arith_ev = arith_ev
-  word_ev.expr_ev = expr_ev
-  word_ev.prompt_ev = prompt_ev
-  word_ev.shell_ex = shell_ex
+  def Time(self):
+    # type: () -> None
+    pass
 
-  cmd_ev.shell_ex = shell_ex
-  cmd_ev.arith_ev = arith_ev
-  cmd_ev.bool_ev = bool_ev
-  cmd_ev.expr_ev = expr_ev
-  cmd_ev.word_ev = word_ev
-  cmd_ev.tracer = tracer
+  def PushRedirects(self, redirects):
+    # type: (List[redirect]) -> bool
+    pass
 
-  shell_ex.cmd_ev = cmd_ev
+  def PopRedirects(self):
+    # type: () -> None
+    pass
 
-  prompt_ev.word_ev = word_ev
 
-  arith_ev.CheckCircularDeps()
-  bool_ev.CheckCircularDeps()
-  expr_ev.CheckCircularDeps()
-  word_ev.CheckCircularDeps()
-  cmd_ev.CheckCircularDeps()
-  shell_ex.CheckCircularDeps()
-  prompt_ev.CheckCircularDeps()

mycpp/cppgen_pass.py

@@ -1670,7 +1670,9 @@ def visit_func_def(self, o: 'mypy.nodes.FuncDef') -> T:
             class_name == 'SplitContext' and
               func_name in ('SplitForWordEval', '_GetSplitter') or
             class_name is None and 
-              func_name in ('maybe_encode', 'maybe_shell_encode')
+              func_name in ('maybe_encode', 'maybe_shell_encode') or
+            # virtual function
+            func_name == 'RunSimpleCommand'
           ):
 
           default_val = o.arguments[-1].initializer
@@ -1784,6 +1786,9 @@ def visit_class_def(self, o: 'mypy.nodes.ClassDef') -> T:
             # TODO: inherit from std::exception?
             if b.name != 'object' and b.name != 'Exception':
               base_class_name = b.name
+          elif isinstance(b, MemberExpr):
+            # TODO: handle vm::_Executor here?
+            pass
 
         # Forward declare types because they may be used in prototypes
         if self.forward_decl:
@@ -1810,7 +1815,9 @@ def visit_class_def(self, o: 'mypy.nodes.ClassDef') -> T:
 
           # e.g. class TextOutput : public ColorOutput
           if base_class_name:
-            self.decl_write(' : public %s', base_class_name)
+            # Hack because usually we don't inherit across modules.  It's a bit complicated.
+            b = 'vm::_Executor' if base_class_name == '_Executor' else base_class_name
+            self.decl_write(' : public %s', b)
 
           self.decl_write(' {\n')
           self.decl_write_ind(' public:\n')

mycpp/mycpp_main.py

@@ -66,7 +66,7 @@ def ModulesToCompile(result, mod_names):
 
   # Somehow the MyPy builder reorders the modules.
   for name, module in result.files.items():
-    if name == 'asdl.runtime':
+    if name in ('asdl.runtime', 'core.vm'):
       yield name, module
 
   for name, module in result.files.items():
@@ -76,7 +76,7 @@ def ModulesToCompile(result, mod_names):
       continue
 
     # Don't do it a second time!
-    if name == 'asdl.runtime':
+    if name in ('asdl.runtime', 'core.vm'):
       continue
 
     yield name, module

mycpp/mylib.cc

@@ -298,6 +298,10 @@ void CFileWriter::write(Str* s) {
   fwrite(s->data_, s->len_, 1, f_);
 }
 
+void CFileWriter::flush() {
+  ::fflush(f_);
+}
+
 bool CFileWriter::isatty() {
   return ::isatty(fileno(f_));
 }

mycpp/mylib.h

@@ -827,15 +827,18 @@ inline LineReader* open(Str* path) {
 class Writer {
  public:
   virtual void write(Str* s) = 0;
+  virtual void flush() = 0;
   virtual bool isatty() = 0;
 };
 
 class BufWriter : public Writer {
  public:
   BufWriter() : data_(nullptr), len_(0) {
   }
-  virtual void write(Str* s);
-  virtual bool isatty() {
+  virtual void write(Str* s) override;
+  virtual void flush() override {
+  }
+  virtual bool isatty() override {
     return false;
   }
   // For cStringIO API
@@ -884,8 +887,9 @@ class CFileWriter : public Writer {
  public:
   explicit CFileWriter(FILE* f) : f_(f) {
   }
-  virtual bool isatty();
-  virtual void write(Str* s);
+  virtual void write(Str* s) override;
+  virtual void flush() override;
+  virtual bool isatty() override;
 
  private:
   FILE* f_;

mycpp/mylib.pyi

@@ -20,6 +20,7 @@ def Stdin() -> LineReader: ...
 
 class Writer:
   def write(self, s: str) -> None: ...
+  def flush(self) -> None: ...
   def isatty(self) -> bool: ...
 
 class BufWriter(Writer):

oil_lang/expr_eval.py

@@ -30,7 +30,7 @@
       lvalue_t, lvalue__Named, lvalue__ObjIndex, lvalue__ObjAttr,
   )
   from _devbuild.gen.syntax_asdl import arg_list
-  from core.executor import ShellExecutor
+  from core.vm import _Executor
   from core.ui import ErrorFormatter
   from core.state import Mem
   from osh.word_eval import StringWordEvaluator
@@ -53,7 +53,7 @@ def __init__(self,
                errfmt,  # type: ErrorFormatter
                ):
     # type: (...) -> None
-    self.shell_ex = None  # type: ShellExecutor
+    self.shell_ex = None  # type: _Executor
     self.word_ev = None  # type: StringWordEvaluator
 
     self.mem = mem

osh/cmd_eval.py

@@ -97,8 +97,8 @@
   )
   from core.alloc import Arena
   from core import dev
-  from core.executor import ShellExecutor
   from core import optview
+  from core.vm import _Executor
   from oil_lang import expr_eval
   from osh import word_eval
   from osh import builtin_process
@@ -215,7 +215,7 @@ def __init__(self,
                 TODO: This should only be for assignment builtins?
       cmd_deps: A bundle of stateless code
     """
-    self.shell_ex = None  # type: ShellExecutor
+    self.shell_ex = None  # type: _Executor
     self.arith_ev = None  # type: sh_expr_eval.ArithEvaluator
     self.bool_ev = None  # type: sh_expr_eval.BoolEvaluator
     self.expr_ev = None  # type: expr_eval.OilEvaluator

osh/word_eval.py

@@ -48,9 +48,9 @@
   from _devbuild.gen.id_kind_asdl import Id_t
   from _devbuild.gen.syntax_asdl import command_t, speck, word_part_t
   from _devbuild.gen.option_asdl import builtin_t
-  from core import executor
   from core import optview
   from core.ui import ErrorFormatter
+  from core.vm import _Executor
   from osh.split import SplitContext
   from core.state import Mem
   from osh import prompt
@@ -1890,7 +1890,7 @@ class NormalWordEvaluator(AbstractWordEvaluator):
   def __init__(self, mem, exec_opts, splitter, errfmt):
     # type: (Mem, optview.Exec, SplitContext, ErrorFormatter) -> None
     AbstractWordEvaluator.__init__(self, mem, exec_opts, splitter, errfmt)
-    self.shell_ex = None  # type: executor.ShellExecutor
+    self.shell_ex = None  # type: _Executor
 
   def CheckCircularDeps(self):
     # type: () -> None

types/osh-eval-manifest.txt

@@ -13,6 +13,7 @@
 ./core/state.py
 ./core/ui.py
 ./core/util.py
+./core/vm.py
 ./_devbuild/gen/grammar_nt.py
 ./_devbuild/gen/hnode_asdl.py
 ./_devbuild/gen/id_kind_asdl.py