Reproduced 2 wild test failures with smaller tests.

- Notes about the span representation.
- Update some comments.
- 'oshc spans' prints to stdout, no stderr.

core/ui.py

@@ -102,6 +102,8 @@ def PrintFilenameAndLine(span_id, arena, f=sys.stderr):
  col = line_span.col
  length = line_span.length

  # TODO: If the line is blank, it would be nice to print the last non-blank
  # line too?
  print('Line %d of %r' % (line_num+1, path), file=f)
  print('  ' + line.rstrip(), file=f)
  f.write('  ')
@@ -126,6 +128,14 @@ def PrettyPrintError(parse_error, arena, f=sys.stderr):
  else:
    span_id = const.NO_INTEGER

  # TODO: Should there be a special span_id of 0 for EOF?  const.NO_INTEGER
  # means there is no location info, but 0 could mean that the location is EOF.
  # So then you query the arena for the last line in that case?
  # Eof_Real is the ONLY token with 0 span, because it's invisible!
  # Well Eol_Tok is a sentinel with a span_id of const.NO_INTEGER.  I think
  # that is OK.
  # Problem: the column for Eof could be useful.

  if span_id == const.NO_INTEGER:  # Any clause above might return this.
    # This is usually a bug.
    print('*** Error has no source location info ***', file=f)

osh/cmd_parse.py

@@ -492,7 +492,6 @@ def _ExpandAliases(self, words, cur_aliases):
    Returns:
      A command node if any aliases were expanded, but None otherwise.
    """
    #log('_ExpandAliases')
    # The last char that we might parse.
    right_spid = word.RightMostSpanForWord(words[-1])
    first_word_str = None  # for error message
@@ -1637,7 +1636,6 @@ def ParseLogicalLine(self):
    self._Peek()

    if self.c_id == Id.Eof_Real:
      # TODO: Assert that there are no pending here docs
      return None

    node = self._ParseCommandLine()

osh/word_parse.py

@@ -658,7 +658,8 @@ def _ReadCommandSubPart(self, token_type):
    else:
      raise AssertionError(self.token_type)

    c_parser = self.parse_ctx.MakeParserForCommandSub(self.line_reader, self.lexer)
    c_parser = self.parse_ctx.MakeParserForCommandSub(self.line_reader,
                                                      self.lexer)

    # NOTE: This doesn't use something like main_loop because we don't want to
    # interleave parsing and execution!  Unlike 'source' and 'eval'.

spec/parse-errors.test.sh

@@ -109,6 +109,11 @@ $(x
## status: 2
## OK mksh status: 1

#### incomplete backticks
`x
## status: 2
## OK mksh status: 1

#### misplaced ;;
echo 1 ;; echo 2
## stdout-json: ""

test/osh2oil.sh

@@ -346,6 +346,15 @@ OSH
indented
body
'''
OIL

  # Bug fix: the combination of an empty here doc and a redirect afterward.
  osh0-oil3 << 'OSH' 3<< 'OIL'
  cat <<EOF >expect
EOF
OSH
  cat << """ >expect
"""
OIL
}

test/spec.sh

@@ -388,7 +388,7 @@ explore-parsing() {
}

parse-errors() {
  sh-spec spec/parse-errors.test.sh --osh-failures-allowed 1 \
  sh-spec spec/parse-errors.test.sh --osh-failures-allowed 2 \
    ${REF_SHELLS[@]} $OSH_LIST "$@"
}

tools/osh2oil.py

@@ -45,6 +45,7 @@ def PrintUntil(self, until_span_id):
      span = self.arena.GetLineSpan(span_id)
      #log('SPAN %s', span)

      assert span.line_id != -1, 'Invalid span %d: %s' % (span_id, span)
      line = self.arena.GetLine(span.line_id)
      piece = line[span.col : span.col + span.length]
      self.f.write(piece)
@@ -78,7 +79,7 @@ def PrintSpans(arena):
  for i, span in enumerate(arena.spans):
    line = arena.GetLine(span.line_id)
    piece = line[span.col : span.col + span.length]
    print('%5d %r' % (i, piece), file=sys.stderr)
    print('%5d %r' % (i, piece))
  print('(%d spans)' % len(arena.spans), file=sys.stderr)