Reproduced 2 wild test failures with smaller tests.

- Notes about the span representation.
- Update some comments.
- 'oshc spans' prints to stdout, no stderr.

core/ui.py

@@ -102,6 +102,8 @@ def PrintFilenameAndLine(span_id, arena, f=sys.stderr):
   col = line_span.col
   length = line_span.length
 
+  # TODO: If the line is blank, it would be nice to print the last non-blank
+  # line too?
   print('Line %d of %r' % (line_num+1, path), file=f)
   print('  ' + line.rstrip(), file=f)
   f.write('  ')
@@ -126,6 +128,14 @@ def PrettyPrintError(parse_error, arena, f=sys.stderr):
   else:
     span_id = const.NO_INTEGER
 
+  # TODO: Should there be a special span_id of 0 for EOF?  const.NO_INTEGER
+  # means there is no location info, but 0 could mean that the location is EOF.
+  # So then you query the arena for the last line in that case?
+  # Eof_Real is the ONLY token with 0 span, because it's invisible!
+  # Well Eol_Tok is a sentinel with a span_id of const.NO_INTEGER.  I think
+  # that is OK.
+  # Problem: the column for Eof could be useful.
+
   if span_id == const.NO_INTEGER:  # Any clause above might return this.
     # This is usually a bug.
     print('*** Error has no source location info ***', file=f)

osh/cmd_parse.py

@@ -492,7 +492,6 @@ def _ExpandAliases(self, words, cur_aliases):
     Returns:
       A command node if any aliases were expanded, but None otherwise.
     """
-    #log('_ExpandAliases')
     # The last char that we might parse.
     right_spid = word.RightMostSpanForWord(words[-1])
     first_word_str = None  # for error message
@@ -1637,7 +1636,6 @@ def ParseLogicalLine(self):
     self._Peek()
 
     if self.c_id == Id.Eof_Real:
-      # TODO: Assert that there are no pending here docs
       return None
 
     node = self._ParseCommandLine()

osh/word_parse.py

@@ -658,7 +658,8 @@ def _ReadCommandSubPart(self, token_type):
     else:
       raise AssertionError(self.token_type)
 
-    c_parser = self.parse_ctx.MakeParserForCommandSub(self.line_reader, self.lexer)
+    c_parser = self.parse_ctx.MakeParserForCommandSub(self.line_reader,
+                                                      self.lexer)
 
     # NOTE: This doesn't use something like main_loop because we don't want to
     # interleave parsing and execution!  Unlike 'source' and 'eval'.

spec/parse-errors.test.sh

@@ -109,6 +109,11 @@ $(x
 ## status: 2
 ## OK mksh status: 1
 
+#### incomplete backticks
+`x
+## status: 2
+## OK mksh status: 1
+
 #### misplaced ;;
 echo 1 ;; echo 2
 ## stdout-json: ""

test/osh2oil.sh

@@ -346,6 +346,15 @@ OSH
 indented
 body
 '''
+OIL
+
+  # Bug fix: the combination of an empty here doc and a redirect afterward.
+  osh0-oil3 << 'OSH' 3<< 'OIL'
+  cat <<EOF >expect
+EOF
+OSH
+  cat << """ >expect
+"""
 OIL
 }
 

test/spec.sh

@@ -388,7 +388,7 @@ explore-parsing() {
 }
 
 parse-errors() {
-  sh-spec spec/parse-errors.test.sh --osh-failures-allowed 1 \
+  sh-spec spec/parse-errors.test.sh --osh-failures-allowed 2 \
     ${REF_SHELLS[@]} $OSH_LIST "$@"
 }
 

tools/osh2oil.py

@@ -45,6 +45,7 @@ def PrintUntil(self, until_span_id):
       span = self.arena.GetLineSpan(span_id)
       #log('SPAN %s', span)
 
+      assert span.line_id != -1, 'Invalid span %d: %s' % (span_id, span)
       line = self.arena.GetLine(span.line_id)
       piece = line[span.col : span.col + span.length]
       self.f.write(piece)
@@ -78,7 +79,7 @@ def PrintSpans(arena):
   for i, span in enumerate(arena.spans):
     line = arena.GetLine(span.line_id)
     piece = line[span.col : span.col + span.length]
-    print('%5d %r' % (i, piece), file=sys.stderr)
+    print('%5d %r' % (i, piece))
   print('(%d spans)' % len(arena.spans), file=sys.stderr)