Skip to content

Commit

Permalink
Merge pull request #303 from yui-knk/fix_location_of_user_code
Browse files Browse the repository at this point in the history 
Fix location of user code
  • Loading branch information
@yui-knk
yui-knk committed Dec 23, 2023
2 parents 34b3640 + 1cd69eb commit 19a76c7
Show file tree
Hide file tree
Showing 5 changed files with 57 additions and 25 deletions.
14 changes: 10 additions & 4 deletions lib/lrama/lexer.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def column
def location
Location.new(
first_line: @head_line, first_column: @head_column,
last_line: @line, last_column: column
last_line: line, last_column: column
)
end

Expand All @@ -79,8 +79,7 @@ def lex_token
end
end

@head_line = line
@head_column = column
reset_first_position

case
when @scanner.eos?
Expand Down Expand Up @@ -118,6 +117,8 @@ def lex_token
def lex_c_code
nested = 0
code = ''
reset_first_position

while !@scanner.eos? do
case
when @scanner.scan(/{/)
Expand Down Expand Up @@ -167,9 +168,14 @@ def lex_comment
end
end

def reset_first_position
@head_line = line
@head_column = column
end

def newline
@line += 1
@head = @scanner.pos + 1
@head = @scanner.pos
end
end
end
2 changes: 1 addition & 1 deletion lib/lrama/parser.rb
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion parser.y
View file
Original file line number Diff line number Diff line change
Expand Up @@ -585,7 +585,7 @@ def raise_parse_error(error_message, line, first_column, last_column)
end
def blanks(text, first_column)
text[0..first_column].gsub(/[^\t]/, ' ')
text[0...first_column].gsub(/[^\t]/, ' ')
end
def carrets(text, first_column, last_column)
Expand Down
52 changes: 39 additions & 13 deletions spec/lrama/lexer_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
expect(lexer.next_token).to eq(['%{', '%{'])

lexer.status = :c_declaration; lexer.end_symbol = '%}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n// Prologue\n")])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n// Prologue\n")])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 7, first_column: 2, last_line: 9, last_column: 0)
lexer.status = :initial

expect(lexer.next_token).to eq(['%}', '%}'])
Expand All @@ -27,7 +29,9 @@
expect(lexer.next_token).to eq(['{', '{'])

lexer.status = :c_declaration; lexer.end_symbol = '}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n print_int();\n")])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n print_int();\n")])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 15, first_column: 10, last_line: 17, last_column: 0)
lexer.status = :initial

expect(lexer.next_token).to eq(['}', '}'])
Expand All @@ -36,7 +40,9 @@
expect(lexer.next_token).to eq(['{', '{'])

lexer.status = :c_declaration; lexer.end_symbol = '}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n print_token();\n")])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n print_token();\n")])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 18, first_column: 10, last_line: 20, last_column: 0)
lexer.status = :initial

expect(lexer.next_token).to eq(['}', '}'])
Expand All @@ -46,23 +52,29 @@
expect(lexer.next_token).to eq(['{', '{'])

lexer.status = :c_declaration; lexer.end_symbol = '}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: 'struct lex_params *p')])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: 'struct lex_params *p')])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 22, first_column: 12, last_line: 22, last_column: 32)
lexer.status = :initial

expect(lexer.next_token).to eq(['}', '}'])
expect(lexer.next_token).to eq(['%parse-param', '%parse-param'])
expect(lexer.next_token).to eq(['{', '{'])

lexer.status = :c_declaration; lexer.end_symbol = '}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: 'struct parse_params *p')])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: 'struct parse_params *p')])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 23, first_column: 14, last_line: 23, last_column: 36)
lexer.status = :initial

expect(lexer.next_token).to eq(['}', '}'])
expect(lexer.next_token).to eq(['%initial-action', '%initial-action'])
expect(lexer.next_token).to eq(['{', '{'])

lexer.status = :c_declaration; lexer.end_symbol = '}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n initial_action_func(@$);\n")])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n initial_action_func(@$);\n")])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 26, first_column: 1, last_line: 28, last_column: 0)
lexer.status = :initial

expect(lexer.next_token).to eq(['}', '}'])
Expand All @@ -71,7 +83,9 @@
expect(lexer.next_token).to eq(['{', '{'])

lexer.status = :c_declaration; lexer.end_symbol = '}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n int i;\n long l;\n char *str;\n")])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n int i;\n long l;\n char *str;\n")])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 30, first_column: 8, last_line: 34, last_column: 0)
lexer.status = :initial

expect(lexer.next_token).to eq(['}', '}'])
Expand Down Expand Up @@ -147,7 +161,9 @@
expect(lexer.next_token).to eq(['{', '{'])

lexer.status = :c_declaration; lexer.end_symbol = '}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: " code 1 ")])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: " code 1 ")])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 63, first_column: 11, last_line: 63, last_column: 19)
lexer.status = :initial

expect(lexer.next_token).to eq(['}', '}'])
Expand All @@ -156,7 +172,9 @@
expect(lexer.next_token).to eq(['{', '{'])

lexer.status = :c_declaration; lexer.end_symbol = '}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: " code 2 ")])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: " code 2 ")])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 64, first_column: 23, last_line: 64, last_column: 31)
lexer.status = :initial

expect(lexer.next_token).to eq(['}', '}'])
Expand All @@ -166,7 +184,9 @@
expect(lexer.next_token).to eq(['{', '{'])

lexer.status = :c_declaration; lexer.end_symbol = '}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: " code 3 ")])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: " code 3 ")])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 64, first_column: 58, last_line: 64, last_column: 66)
lexer.status = :initial

expect(lexer.next_token).to eq(['}', '}'])
Expand All @@ -177,7 +197,9 @@
expect(lexer.next_token).to eq(['{', '{'])

lexer.status = :c_declaration; lexer.end_symbol = '}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: " code 4 ")])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: " code 4 ")])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 65, first_column: 23, last_line: 65, last_column: 31)
lexer.status = :initial

expect(lexer.next_token).to eq(['}', '}'])
Expand All @@ -187,7 +209,9 @@
expect(lexer.next_token).to eq(['{', '{'])

lexer.status = :c_declaration; lexer.end_symbol = '}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: " code 5 ")])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: " code 5 ")])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 65, first_column: 58, last_line: 65, last_column: 66)
lexer.status = :initial

expect(lexer.next_token).to eq(['}', '}'])
Expand Down Expand Up @@ -232,7 +256,9 @@
expect(lexer.next_token).to eq(['%{', '%{'])

lexer.status = :c_declaration; lexer.end_symbol = '%}'
expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n// Prologue\n")])
token = lexer.next_token
expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: "\n// Prologue\n")])
expect(token[1].location).to eq Lrama::Lexer::Location.new(first_line: 7, first_column: 2, last_line: 9, last_column: 0)
lexer.status = :initial

expect(lexer.next_token).to eq(['%}', '%}'])
Expand Down
12 changes: 6 additions & 6 deletions spec/lrama/parser_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1795,7 +1795,7 @@ class : keyword_class tSTRING %prec tPLUS keyword_end { code 1 }
parser = Lrama::Parser.new(y, "parse.y")

expect { parser.parse }.to raise_error(ParseError, <<~ERROR)
parse.y:31:41: ident after %prec
parse.y:31:42: ident after %prec
class : keyword_class tSTRING %prec tPLUS keyword_end { code 1 }
^^^^^^^^^^^
ERROR
Expand All @@ -1816,7 +1816,7 @@ class : keyword_class { code 2 } tSTRING %prec "=" '!' keyword_end { code 3 }
parser = Lrama::Parser.new(y, "parse.y")

expect { parser.parse }.to raise_error(ParseError, <<~ERROR)
parse.y:31:50: char after %prec
parse.y:31:51: char after %prec
class : keyword_class { code 2 } tSTRING %prec "=" '!' keyword_end { code 3 }
^^^
ERROR
Expand All @@ -1837,7 +1837,7 @@ class : keyword_class { code 4 } tSTRING '?' keyword_end %prec tEQ { code 5 } {
parser = Lrama::Parser.new(y, "parse.y")

expect { parser.parse }.to raise_error(ParseError, <<~ERROR)
parse.y:31:77: multiple User_code after %prec
parse.y:31:78: multiple User_code after %prec
class : keyword_class { code 4 } tSTRING '?' keyword_end %prec tEQ { code 5 } { code 6 }
^
ERROR
Expand Down Expand Up @@ -2385,7 +2385,7 @@ class : keyword_class tSTRING keyword_end { code 1 }
;
INPUT
expect { Lrama::Parser.new(y, "error_messages/parse.y").parse }.to raise_error(ParseError, <<~ERROR)
error_messages/parse.y:5:7: parse error on value 'invalid' (IDENTIFIER)
error_messages/parse.y:5:8: parse error on value 'invalid' (IDENTIFIER)
%expect invalid
^^^^^^^
ERROR
Expand All @@ -2407,7 +2407,7 @@ class : keyword_class tSTRING keyword_end { code 1 }
;
INPUT
expect { Lrama::Parser.new(y, "error_messages/parse.y").parse }.to raise_error(ParseError, <<~ERROR)
error_messages/parse.y:5:9: parse error on value 10 (INTEGER)
error_messages/parse.y:5:10: parse error on value 10 (INTEGER)
%expect 0 10
^^
ERROR
Expand All @@ -2429,7 +2429,7 @@ class : keyword_class tSTRING keyword_end { code 1 }
;
INPUT
expect { Lrama::Parser.new(y, "error_messages/parse.y").parse }.to raise_error(ParseError, <<~ERROR)
error_messages/parse.y:5:8: parse error on value 'invalid' (IDENTIFIER)
error_messages/parse.y:5:9: parse error on value 'invalid' (IDENTIFIER)
%expect\t\tinvalid
\t\t^^^^^^^
ERROR
Expand Down

0 comments on commit 19a76c7

Please sign in to comment.