Fix translation of here docs with <<-

The HereDocLineReader has to return the start_offset separately.  And
then the LineLexer can be started at the offset, skipping leading tabs.

Added 'here-doc' to the list of passing osh2oil tests!

core/lexer.py

@@ -36,13 +36,13 @@ def __init__(self, match_func, line, arena):
    self.arena_skip = False  # For MaybeUnreadOne
    self.last_span_id = const.NO_INTEGER  # For MaybeUnreadOne

    self.Reset(line, -1)  # Invalid line_id to start
    self.Reset(line, -1, 0)  # Invalid line_id to start

  def Reset(self, line, line_id):
  def Reset(self, line, line_id, line_pos):
    #assert line, repr(line)  # can't be empty or None
    self.line = line
    self.line_pos = 0
    self.line_id = line_id
    self.line_pos = line_pos

  def MaybeUnreadOne(self):
    """Return True if we can unread one character, or False otherwise.
@@ -188,14 +188,14 @@ def PushHint(self, old_id, new_id):
  def _Read(self, lex_mode):
    t = self.line_lexer.Read(lex_mode)
    if t.id == Id.Eol_Tok:  # hit \0
      line_id, line = self.line_reader.GetLine()
      line_id, line, line_pos = self.line_reader.GetLine()

      if line is None:  # no more lines
        span_id = self.line_lexer.GetSpanIdForEof()
        t = ast.token(Id.Eof_Real, '', span_id)
        return t

      self.line_lexer.Reset(line, line_id)
      self.line_lexer.Reset(line, line_id, line_pos)
      t = self.line_lexer.Read(lex_mode)

    # e.g. translate ) or ` into EOF

core/reader.py

@@ -23,14 +23,14 @@ def __init__(self, arena):
  def GetLine(self):
    line = self._GetLine()
    if line is None:
      return -1, None
      return -1, None, 0

    if self.arena:
      line_id = self.arena.AddLine(line, self.line_num)
    else:
      line_id = -1
    self.line_num += 1
    return line_id, line
    return line_id, line, 0

  def Reset(self):
    # Should never be called?
@@ -109,8 +109,12 @@ def __init__(self, lines, arena):

  def GetLine(self):
    if self.pos == self.num_lines:
      return -1, None
      return -1, None, 0
    line_id, line, start_offset = self.lines[self.pos]

    self.pos += 1
    return line_id, line[start_offset:]

    # NOTE: we return a partial line, but we also want the lexer to create
    # tokens with the correct line_spans.  So we have to tell it 'start_offset'
    # as well.
    return line_id, line, start_offset

osh/cmd_parse.py

@@ -105,7 +105,8 @@ def _MaybeReadHereDocs(self):
      # echo 3) 4
      # EOF
      while True:
        line_id, line = self.line_reader.GetLine()
        line_id, line, unused_offset = self.line_reader.GetLine()
        assert unused_offset == 0

        if not line:  # EOF
          # An unterminated here doc is just a warning in bash.  We make it
@@ -135,8 +136,6 @@ def _MaybeReadHereDocs(self):

        here_lines.append((line_id, line, start_offset))

      line_reader = reader.HereDocLineReader(here_lines, self.arena)

      parts = []
      if delim_quoted:  # << 'EOF'
        # Create a line_span and a token for each line.
@@ -151,6 +150,8 @@ def _MaybeReadHereDocs(self):

      else:
        from osh import parse_lib  # Avoid circular import

        line_reader = reader.HereDocLineReader(here_lines, self.arena)
        w_parser = parse_lib.MakeWordParserForHereDoc(line_reader, self.arena)

        # NOTE: There can be different kinds of parse errors in here docs.

test/arena.sh

@@ -27,7 +27,10 @@ here-doc() {
  _compare test/arena/here-dq.sh
  _compare test/arena/here-sq.sh
  _compare test/arena/here-multiple.sh
  _compare test/arena/here-dq-indented.sh

  # This is a known exception to the arena invariant.  The leading tabs aren't
  # preserved, because we don't need them for osh2oil translation.
  #_compare test/arena/here-dq-indented.sh
}

tilde() {

test/arena/here-dq-indented.sh

@@ -1,7 +1,6 @@
#!/bin/bash

	cat <<-ONE
	cat <<-EOF
	indented
	body
	ONE
OSH
	EOF

test/osh2oil.sh

@@ -1142,6 +1142,7 @@ readonly -a PASSING=(
  more-env
  line-breaks
  redirect
  here-doc
  pipeline
  and-or