[Bug #20579] ripper: Dispatch spaces at END-OF-INPUT without newline

Author: nobu

parse.y

@@ -10684,6 +10684,9 @@ parser_yylex(struct parser_params *p)
                 p->lex.nextline = p->lex.lastline;
                 set_lastline(p, prevline);
               case -1:		/* EOF no decrement*/
+                if (c == -1 && space_seen) {
+                    dispatch_scan_event(p, tSP);
+                }
                 lex_goto_eol(p);
                 if (c != -1) {
                     token_flush(p);

test/ripper/test_lexer.rb

@@ -499,6 +499,16 @@ def test_invalid_escape_string
     assert_lexer(expected, code)
   end
 
+  def test_spaces_at_eof
+    code = "1\n\t \t"
+    expected = [
+      [[1, 0], :on_int, "1", state(:EXPR_END)],
+      [[1, 1], :on_nl, "\n", state(:EXPR_BEG)],
+      [[2, 0], :on_sp, "\t \t", state(:EXPR_END)],
+    ]
+    assert_lexer(expected, code)
+  end
+
   def assert_lexer(expected, code)
     assert_equal(code, Ripper.tokenize(code).join(""))
     assert_equal(expected, result = Ripper.lex(code),