Fix arena invariant with here-sq.test.sh.

We add a line_span for every line of the body in the 'EOF'
(single-quoted) case.

Also, clarify a here doc spec test: the closing delimiter can have
leading tabs when <<- is used.

osh/cmd_parse.py

@@ -81,6 +81,8 @@ def GetCompletionState(self):
    return self.completion_stack

  def _MaybeReadHereDocs(self):
    """Fill the 'body' attribute of pending here doc nodes."""

    for h in self.pending_here_docs:
      here_end_line = None
      here_end_line_id = -1
@@ -95,10 +97,7 @@ def _MaybeReadHereDocs(self):
        p_die('Invalid here doc delimiter', word=h.here_begin)
      do_expansion = not quoted

      #log('HERE %r' % h.here_end)
      while True:
        # If op is <<-, strip off all leading tabs (NOT spaces).
        # (in C++, just bump the start?)
        line_id, line = self.line_reader.GetLine()

        if not line:  # EOF
@@ -109,9 +108,9 @@ def _MaybeReadHereDocs(self):
          p_die("Couldn't find terminator for here doc that starts here",
                token=h.op)

        # NOTE: Could do this runtime to preserve LST.
        # If op is <<-, strip off ALL leading tabs -- not spaces, and not just
        # the first tab.
        if h.op.id == Id.Redir_DLessDash:
          # NOTE: Stripping multiple leading tabs is correct!
          line = line.lstrip('\t')
        if line.rstrip() == delimiter:
          here_end_line = line
@@ -143,12 +142,15 @@ def _MaybeReadHereDocs(self):
        assert w is not None
        h.body = w
      else:
        # Each line is a single span.  TODO: Add span_id to token.
        tokens = [
            ast.token(Id.Lit_Chars, line, const.NO_INTEGER)
            for _, line in lines]
        parts = [ast.LiteralPart(t) for t in tokens]
        h.body = ast.CompoundWord(parts)
        # Create a line_span and a token for each line.
        tokens = []
        for line_id, line in lines:
          line_span = ast.line_span(line_id, 0, len(line))
          span_id = self.arena.AddLineSpan(line_span)
          t = ast.token(Id.Lit_Chars, line, span_id)
          tokens.append(t)
        # LiteralPart for each line.
        h.body = ast.CompoundWord([ast.LiteralPart(t) for t in tokens])

      # Create a span with the end terminator.  Maintains the invariant that
      # the spans "add up".

spec/here-doc.test.sh

@@ -297,7 +297,7 @@ cat <<-EOF
	2
		3  # 2 tabs are both stripped
  4  # spaces are preserved
EOF
	EOF
## STDOUT:
1
2