Change the LST representation of here docs and redirects.

This will make osh2oil translation of here docs easier.

- add the 'word here_begin' field to HereDoc
- Use 'token op' instead of 'id op_id' so we have a more natural way of
  getting location information.

Also:

- Add a stub for strict-var-eval.  This determines if names are known
  statically.
- Add test for conversion to setargv.
- Add test for conversion to "sh-builtin export"

core/cmd_exec.py

@@ -446,9 +446,9 @@ def _EvalLhs(self, node, spid):
    raise AssertionError(node.tag)

  def _EvalRedirect(self, n):
    fd = REDIR_DEFAULT_FD[n.op_id] if n.fd == const.NO_INTEGER else n.fd
    fd = REDIR_DEFAULT_FD[n.op.id] if n.fd == const.NO_INTEGER else n.fd
    if n.tag == redir_e.Redir:
      redir_type = REDIR_ARG_TYPES[n.op_id]  # could be static in the LST?
      redir_type = REDIR_ARG_TYPES[n.op.id]  # could be static in the LST?

      if redir_type == redir_arg_type_e.Path:
        # NOTE: no globbing.  You can write to a file called '*.py'.
@@ -462,7 +462,7 @@ def _EvalRedirect(self, n):
          util.error("Redirect filename can't be empty")
          return None

        return runtime.PathRedirect(n.op_id, fd, filename)
        return runtime.PathRedirect(n.op.id, fd, filename)

      elif redir_type == redir_arg_type_e.Desc:  # e.g. 1>&2
        val = self.word_ev.EvalWordToString(n.arg_word)
@@ -480,7 +480,7 @@ def _EvalRedirect(self, n):
              "Redirect descriptor should look like an integer, got %s", val)
          return None

        return runtime.DescRedirect(n.op_id, fd, target_fd)
        return runtime.DescRedirect(n.op.id, fd, target_fd)

      elif redir_type == redir_arg_type_e.Here:  # here word
        # TODO: decay should be controlled by an option

core/state.py

@@ -88,6 +88,7 @@ def Disable(self):
    (None, 'strict-array'),
    (None, 'strict-arith'),
    (None, 'strict-word-eval'),
    (None, 'strict-var-eval'),

    (None, 'vi'),
    (None, 'emacs'),
@@ -159,8 +160,11 @@ def __init__(self, mem):
    #

    self.strict_arith = False  # e.g. $(( x )) where x doesn't look like integer
    #self.strict_word = False  # word splitting, etc.
    self.strict_word_eval = False
    # Whether we statically know variables, e.g. $PYTHONPATH vs.
    # $ENV['PYTHONPATH'], and behavior of 'or' and 'if' expressions.
    # This is off by default because we want the interactive shell to match.
    self.strict_var_eval = False
    self.strict_scope = False  # disable dynamic scope
    # TODO: strict_bool.  Some of this is covered by arithmetic, e.g. -eq.

osh/cmd_parse.py

@@ -96,10 +96,10 @@ def _MaybeReadHereDocs(self):
          # reporting other errors.
          # Attribute it to the << in <<EOF for now.
          p_die("Couldn't find terminator for here doc that starts here",
                span_id=h.spids[0])
                token=h.op)

        # NOTE: Could do this runtime to preserve LST.
        if h.op_id == Id.Redir_DLessDash:
        if h.op.id == Id.Redir_DLessDash:
          line = line.lstrip('\t')
        if line.rstrip() == h.here_end:
          break
@@ -211,7 +211,7 @@ def ParseRedirect(self):
    if not self._Peek(): return None
    assert self.c_kind == Kind.Redir, self.cur_word

    left_spid = self.cur_word.token.span_id
    op = self.cur_word.token

    # For now only supporting single digit descriptor
    first_char = self.cur_word.token.val[0]
@@ -220,33 +220,32 @@ def ParseRedirect(self):
    else:
      fd = const.NO_INTEGER

    if self.c_id in (Id.Redir_DLess, Id.Redir_DLessDash):  # here doc
    if op.id in (Id.Redir_DLess, Id.Redir_DLessDash):  # here doc
      node = ast.HereDoc()
      node.op_id = self.c_id
      node.op = op
      node.body = None  # not read yet
      node.fd = fd
      node.was_filled = False
      node.spids.append(left_spid)
      self._Next()

      if not self._Peek(): return None
      node.here_begin = self.cur_word
      # "If any character in word is quoted, the delimiter shall be formed by
      # performing quote removal on word, and the here-document lines shall not
      # be expanded. Otherwise, the delimiter shall be the word itself."
      # NOTE: \EOF counts, or even E\OF
      ok, node.here_end, quoted = word.StaticEval(self.cur_word)
      ok, node.here_end, quoted = word.StaticEval(node.here_begin)
      if not ok:
        p_die('Invalid here doc delimiter', word=self.cur_word)
        p_die('Invalid here doc delimiter', word=node.here_begin)
      node.do_expansion = not quoted
      self._Next()

      self.pending_here_docs.append(node)  # will be filled on next newline.

    else:
      node = ast.Redir()
      node.op_id = self.c_id
      node.op = op
      node.fd = fd
      node.spids.append(left_spid)
      self._Next()

      if not self._Peek(): return None

osh/osh.asdl

@@ -140,9 +140,11 @@ module osh
  -- TODO : id -> token for translation?

  redir = 
    Redir(id op_id, int fd, word arg_word)
  | HereDoc(id op_id, int fd, word? body, int do_expansion,
            string here_end, bool was_filled)
    Redir(token op, int fd, word arg_word)
  | HereDoc(token op, int fd,
            word here_begin,  -- For translation
            string here_end, bool do_expansion, -- Derived from here_begin
            word? body, bool was_filled)

  assign_op = Equal | PlusEqual
  assign_pair = (lhs_expr lhs, assign_op op, word? rhs)

spec/here-doc.test.sh

@@ -39,10 +39,13 @@ EOF3
## END

#### Here doc with bad var delimiter
# Most shells accept this, but OSH is stricter.
cat <<${a}
here
${a}
## stdout: here
## OK osh stdout-json: ""
## OK osh status: 2

#### Here doc with bad comsub delimiter
# bash is OK with this; dash isn't.  Should be a parse error.

test/osh2oil.sh

@@ -232,6 +232,14 @@ OIL


export-readonly() {
  # Dynamic export falls back on sh-builtin

  osh0-oil3 << 'OSH' 3<< 'OIL'
export "$@"
OSH
sh-builtin export @Argv
OIL

  # Separate definition and attribute?
  osh0-oil3 << 'OSH' 3<< 'OIL'
export FOO
@@ -545,7 +553,7 @@ OIL
  osh0-oil3 << 'OSH' 3<< 'OIL'
set a b c
OSH
set -- a b c
setargv a b c
OIL
}

test/spec.sh

@@ -417,7 +417,7 @@ here-doc() {
  # - On Debian, the whole process hangs.
  # Is this due to Python 3.2 vs 3.4?  Either way osh doesn't implement the
  # functionality, so it's probably best to just implement it.
  sh-spec spec/here-doc.test.sh --osh-failures-allowed 1 --range 0-30 \
  sh-spec spec/here-doc.test.sh --range 0-30 \
    ${REF_SHELLS[@]} $OSH "$@"
}

tools/osh2oil.py

@@ -231,7 +231,9 @@ def End(self):

  def DoRedirect(self, node, local_symbols):
    #print(node, file=sys.stderr)
    self.cursor.PrintUntil(node.spids[0])
    op_spid = node.op.span_id
    op_id = node.op.id
    self.cursor.PrintUntil(op_spid)

    # TODO:
    # - Do < and <& the same way.
@@ -240,10 +242,10 @@ def DoRedirect(self, node, local_symbols):

    if node.tag == redir_e.Redir:
      if node.fd == const.NO_INTEGER:
        if node.op_id == Id.Redir_Great:
        if op_id == Id.Redir_Great:
          self.f.write('>')  # Allow us to replace the operator
          self.cursor.SkipUntil(node.spids[0] + 1)
        elif node.op_id == Id.Redir_GreatAnd:
          self.cursor.SkipUntil(op_spid + 1)
        elif op_id == Id.Redir_GreatAnd:
          self.f.write('> !')  # Replace >& 2 with > !2
          spid = word.LeftMostSpanForWord(node.arg_word)
          self.cursor.SkipUntil(spid)
@@ -253,10 +255,10 @@ def DoRedirect(self, node, local_symbols):
        # NOTE: Spacing like !2>err.txt vs !2 > err.txt can be done in the
        # formatter.
        self.f.write('!%d ' % node.fd)
        if node.op_id == Id.Redir_Great:
        if op_id == Id.Redir_Great:
          self.f.write('>')
          self.cursor.SkipUntil(node.spids[0] + 1)
        elif node.op_id == Id.Redir_GreatAnd:
          self.cursor.SkipUntil(op_spid + 1)
        elif op_id == Id.Redir_GreatAnd:
          self.f.write('> !')  # Replace 1>& 2 with !1 > !2
          spid = word.LeftMostSpanForWord(node.arg_word)
          self.cursor.SkipUntil(spid)