Skip to content

Commit

Permalink
d: change the return value of yylex from TokenKind to YYParser.Symbol
Browse files Browse the repository at this point in the history 
The complete symbol approach was deemed to be the right approach for Dlang.
Now, the user can return from yylex() an instance of YYParser.Symbol structure,
which binds together the TokenKind, the semantic value and the location. Before,
the last two were reported separately to the parser.
Only the user API is changed, Bisons's internal structure is kept the same.

* data/skeletons/d.m4 (struct YYParser.Symbol): New.
* data/skeletons/lalr1.d: Change the return value.
* doc/bison.texi: Document it.
* examples/d/calc/calc.y, examples/d/simple/calc.y: Demonstrate it.
* tests/calc.at, tests/scanner.at: Test it.
  • Loading branch information
@adelavais @akimd
adelavais authored and akimd committed Nov 20, 2020
1 parent 2ca158c commit 10305f3
Show file tree
Hide file tree
Showing 8 changed files with 95 additions and 62 deletions.
35 changes: 35 additions & 0 deletions data/skeletons/d.m4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -446,3 +446,38 @@ m4_define([b4_var_decls],
], [$@])])
m4_define([b4_var_decl],
[ protected $1;])


# b4_symbol_type_define
# ---------------------
# Define symbol_type, the external type for symbols used for symbol
# constructors.
m4_define([b4_symbol_type_define],
[[
/**
* A complete symbol
*/
struct Symbol
{
private SymbolKind kind;
private ]b4_yystype[ value;]b4_locations_if([[
private YYLocation location_;]])[
this(TokenKind token]b4_locations_if([[, YYLocation loc]])[)
{
kind = yytranslate_(token);]b4_locations_if([
location_ = loc;])[
}
static foreach (member; __traits(allMembers, YYSemanticType))
{
this(TokenKind token, typeof(mixin("YYSemanticType." ~ member)) val]b4_locations_if([[, YYLocation loc]])[)
{
kind = yytranslate_(token);
mixin("value." ~ member ~ " = val;");]b4_locations_if([
location_ = loc;])[
}
}
SymbolKind token() { return kind; }
]b4_yystype[ semanticValue() { return value; }]b4_locations_if([[
YYLocation location() { return location_; }]])[
}
]])
19 changes: 9 additions & 10 deletions data/skeletons/lalr1.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public interface Lexer
* to the next token and prepares to return the semantic value
* ]b4_locations_if([and beginning/ending positions ])[of the token.
* @@return the token identifier corresponding to the next token. */
TokenKind yylex ();
]b4_parser_class[.Symbol yylex ();
/**
* Entry point for error reporting. Emits an error
Expand Down Expand Up @@ -290,7 +290,7 @@ b4_user_union_members
yyDebugStream.writeln (s);
}
]])[
private final TokenKind yylex () {
private final ]b4_parser_class[.Symbol yylex () {
return yylexer.yylex ();
}
Expand Down Expand Up @@ -411,7 +411,9 @@ b4_locations_if([, ref ]b4_location_type[ yylocationp])[)
yycdebugln (message);
}
}
]])[
]])
b4_symbol_type_define
[
/**
* Parse input from the scanner that was specified at object construction
* time. Return whether the end of the input was reached successfully.
Expand Down Expand Up @@ -493,13 +495,10 @@ m4_popdef([b4_at_dollar])])dnl
if (yychar == TokenKind.]b4_symbol(empty, id)[)
{]b4_parse_trace_if([[
yycdebugln ("Reading a token");]])[
yychar = yylex ();]b4_locations_if([[
static if (yy_location_is_class) {
yylloc = new ]b4_location_type[(yylexer.startPos, yylexer.endPos);
} else {
yylloc = ]b4_location_type[(yylexer.startPos, yylexer.endPos);
}]])
yylval = yylexer.semanticVal;[
Symbol yysymbol = yylex();
yychar = yysymbol.token();
yylval = yysymbol.semanticValue();]b4_locations_if([[
yylloc = yysymbol.location();]])[
}
/* Convert token to internal form. */
Expand Down
6 changes: 3 additions & 3 deletions doc/bison.texi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14011,9 +14011,9 @@ This method is defined by the user to emit an error message. The first
parameter is omitted if location tracking is not active.
@end deftypemethod

@deftypemethod {Lexer} {TokenKind} yylex()
Return the next token. Its type is the return value, its semantic value and
location are saved and returned by the their methods in the interface.
@deftypemethod {Lexer} {YYParser.Symbol} yylex()
Return the next token. The return value is of type YYParser.Symbol, which
binds together the TokenKind, the semantic value and the location.
@end deftypemethod

@deftypemethod {Lexer} {YYPosition} getStartPos()
Expand Down
20 changes: 10 additions & 10 deletions examples/d/calc/calc.y
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
return semanticVal_;
}

TokenKind yylex()
Calc.Symbol yylex()
{
import std.uni : isWhite, isNumber;

Expand All @@ -127,7 +127,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
}

if (input.empty)
return TokenKind.YYEOF;
return Calc.Symbol(TokenKind.YYEOF, new YYLocation(startPos, endPos));

// Numbers.
if (input.front.isNumber)
Expand All @@ -143,7 +143,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
}
start = end;
end.column += lenChars;
return TokenKind.NUM;
return Calc.Symbol(TokenKind.NUM, semanticVal_.ival, new YYLocation(startPos, endPos));
}

// Individual characters
Expand All @@ -153,17 +153,17 @@ if (isInputRange!R && is(ElementType!R : dchar))
end.column++;
switch (ch)
{
case '+': return TokenKind.PLUS;
case '-': return TokenKind.MINUS;
case '*': return TokenKind.STAR;
case '/': return TokenKind.SLASH;
case '(': return TokenKind.LPAR;
case ')': return TokenKind.RPAR;
case '+': return Calc.Symbol(TokenKind.PLUS, new YYLocation(startPos, endPos));
case '-': return Calc.Symbol(TokenKind.MINUS, new YYLocation(startPos, endPos));
case '*': return Calc.Symbol(TokenKind.STAR, new YYLocation(startPos, endPos));
case '/': return Calc.Symbol(TokenKind.SLASH, new YYLocation(startPos, endPos));
case '(': return Calc.Symbol(TokenKind.LPAR, new YYLocation(startPos, endPos));
case ')': return Calc.Symbol(TokenKind.RPAR, new YYLocation(startPos, endPos));
case '\n':
{
end.line++;
end.column = 1;
return TokenKind.EOL;
return Calc.Symbol(TokenKind.EOL, new YYLocation(startPos, endPos));
}
default: assert(0);
}
Expand Down
21 changes: 10 additions & 11 deletions examples/d/simple/calc.y
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
return semanticVal_;
}

TokenKind yylex()
Calc.Symbol yylex()
{
import std.uni : isWhite, isNumber;

Expand All @@ -118,28 +118,27 @@ if (isInputRange!R && is(ElementType!R : dchar))
input.popFront;

if (input.empty)
return TokenKind.YYEOF;
return Calc.Symbol(TokenKind.YYEOF);

// Numbers.
if (input.front.isNumber)
{
import std.conv : parse;
semanticVal_.ival = input.parse!int;
return TokenKind.NUM;
return Calc.Symbol(TokenKind.NUM, input.parse!int);
}

// Individual characters
auto ch = input.front;
input.popFront;
switch (ch)
{
case '+': return TokenKind.PLUS;
case '-': return TokenKind.MINUS;
case '*': return TokenKind.STAR;
case '/': return TokenKind.SLASH;
case '(': return TokenKind.LPAR;
case ')': return TokenKind.RPAR;
case '\n': return TokenKind.EOL;
case '+': return Calc.Symbol(TokenKind.PLUS);
case '-': return Calc.Symbol(TokenKind.MINUS);
case '*': return Calc.Symbol(TokenKind.STAR);
case '/': return Calc.Symbol(TokenKind.SLASH);
case '(': return Calc.Symbol(TokenKind.LPAR);
case ')': return Calc.Symbol(TokenKind.RPAR);
case '\n': return Calc.Symbol(TokenKind.EOL);
default: assert(0);
}
}
Expand Down
30 changes: 15 additions & 15 deletions tests/calc.at
View file
Original file line number Diff line number Diff line change
Expand Up @@ -590,7 +590,7 @@ class CalcLexer(R) : Lexer
return res;
}

TokenKind yylex ()
YYParser.Symbol yylex ()
{]AT_LOCATION_IF([[
location.begin = location.end;]])[

Expand All @@ -606,13 +606,13 @@ class CalcLexer(R) : Lexer

// EOF.
if (input.empty)
return TokenKind.]AT_TOKEN_PREFIX[EOF;
return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EOF]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);

// Numbers.
if (input.front.isNumber)
{
semanticVal_.ival = parseInt;
return TokenKind.]AT_TOKEN_PREFIX[NUM;
return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[NUM, semanticVal_.ival]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
}

// Individual characters
Expand All @@ -630,22 +630,22 @@ class CalcLexer(R) : Lexer
if (c == '#')
{
stderr.writeln (]AT_LOCATION_IF([location, ": ", ])["syntax error: invalid character: '#'");
return TokenKind.]AT_TOKEN_PREFIX[YYerror;
return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[YYerror]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
}

switch (c)
{
case '+': return TokenKind.]AT_TOKEN_PREFIX[PLUS;
case '-': return TokenKind.]AT_TOKEN_PREFIX[MINUS;
case '*': return TokenKind.]AT_TOKEN_PREFIX[STAR;
case '/': return TokenKind.]AT_TOKEN_PREFIX[SLASH;
case '(': return TokenKind.]AT_TOKEN_PREFIX[LPAR;
case ')': return TokenKind.]AT_TOKEN_PREFIX[RPAR;
case '\n': return TokenKind.]AT_TOKEN_PREFIX[EOL;
case '=': return TokenKind.]AT_TOKEN_PREFIX[EQUAL;
case '^': return TokenKind.]AT_TOKEN_PREFIX[POW;
case '!': return TokenKind.]AT_TOKEN_PREFIX[NOT;
default: return TokenKind.]AT_TOKEN_PREFIX[YYUNDEF;
case '+': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[PLUS]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '-': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[MINUS]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '*': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[STAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '/': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[SLASH]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '(': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[LPAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case ')': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[RPAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '\n': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EOL]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '=': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EQUAL]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '^': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[POW]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '!': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[NOT]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
default: return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[YYUNDEF]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions tests/d.at
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class CalcLexer(R) : Lexer
YYSemanticType semanticVal_;
YYSemanticType semanticVal() @property { return semanticVal_; }

TokenKind yylex()
YYParser.Symbol yylex()
{
$2
}
Expand Down Expand Up @@ -143,16 +143,16 @@ AT_KEYWORDS([d])

AT_CHECK_D_MINIMAL_W_LEXER([
%define api.token.raw true
%union { int ival; }], [return TokenKind.END;])
%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 3,]])

AT_CHECK_D_MINIMAL_W_LEXER([
%define api.token.raw false
%union { int ival; }], [return TokenKind.END;])
%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 258,]])

AT_CHECK_D_MINIMAL_W_LEXER([
%union { int ival; }], [return TokenKind.END;])
%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 3,]])

AT_CLEANUP
18 changes: 9 additions & 9 deletions tests/scanner.at
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,12 +121,12 @@ class YYLexer(R) : Lexer
return semanticVal_;
}

TokenKind yylex ()
YYParser.Symbol yylex ()
{
import std.uni : isNumber;
// Handle EOF.
if (input.empty)
return TokenKind.END;
return YYParser.Symbol(TokenKind.END);

auto c = input.front;
input.popFront;
Expand All @@ -136,13 +136,13 @@ class YYLexer(R) : Lexer
{
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
semanticVal_.val = c - '0';
return TokenKind.NUM;
case '+': return TokenKind.PLUS;
case '-': return TokenKind.MINUS;
case '*': return TokenKind.STAR;
case '/': return TokenKind.SLASH;
case '(': return TokenKind.LPAR;
case ')': return TokenKind.RPAR;
return YYParser.Symbol(TokenKind.NUM, semanticVal_.val);
case '+': return YYParser.Symbol(TokenKind.PLUS);
case '-': return YYParser.Symbol(TokenKind.MINUS);
case '*': return YYParser.Symbol(TokenKind.STAR);
case '/': return YYParser.Symbol(TokenKind.SLASH);
case '(': return YYParser.Symbol(TokenKind.LPAR);
case ')': return YYParser.Symbol(TokenKind.RPAR);
default: assert(0);
}
}
Expand Down

0 comments on commit 10305f3

Please sign in to comment.