Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

don't allow a token starting with an asterisk directly following .* #6858

Merged
merged 6 commits into from
Oct 30, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/docgen.zig
Original file line number Diff line number Diff line change
Expand Up @@ -972,7 +972,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: anytype, source_token:
.Tilde,
=> try writeEscaped(out, src[token.loc.start..token.loc.end]),

.Invalid, .Invalid_ampersands => return parseError(
.Invalid, .Invalid_ampersands, .Invalid_periodasterisks => return parseError(
docgen_tokenizer,
source_token,
"syntax error",
Expand Down
4 changes: 4 additions & 0 deletions lib/std/zig/ast.zig
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ pub const Error = union(enum) {
ExpectedBlockOrField: ExpectedBlockOrField,
DeclBetweenFields: DeclBetweenFields,
InvalidAnd: InvalidAnd,
AsteriskAfterPointerDereference: AsteriskAfterPointerDereference,

pub fn render(self: *const Error, tokens: []const Token.Id, stream: anytype) !void {
switch (self.*) {
Expand Down Expand Up @@ -222,6 +223,7 @@ pub const Error = union(enum) {
.ExpectedBlockOrField => |*x| return x.render(tokens, stream),
.DeclBetweenFields => |*x| return x.render(tokens, stream),
.InvalidAnd => |*x| return x.render(tokens, stream),
.AsteriskAfterPointerDereference => |*x| return x.render(tokens, stream),
}
}

Expand Down Expand Up @@ -275,6 +277,7 @@ pub const Error = union(enum) {
.ExpectedBlockOrField => |x| return x.token,
.DeclBetweenFields => |x| return x.token,
.InvalidAnd => |x| return x.token,
.AsteriskAfterPointerDereference => |x| return x.token,
}
}

Expand Down Expand Up @@ -323,6 +326,7 @@ pub const Error = union(enum) {
pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");
pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields");
pub const InvalidAnd = SimpleError("`&&` is invalid. Note that `and` is boolean AND.");
pub const AsteriskAfterPointerDereference = SimpleError("`.*` can't be followed by `*`. Are you missing a space?");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are 2 spaces before Are you missing a space?.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's the user's missing space, silly!


pub const ExpectedCall = struct {
node: *Node,
Expand Down
13 changes: 13 additions & 0 deletions lib/std/zig/parse.zig
Original file line number Diff line number Diff line change
Expand Up @@ -2701,6 +2701,19 @@ const Parser = struct {
return &node.base;
}

if (p.eatToken(.Invalid_periodasterisks)) |period_asterisk| {
try p.errors.append(p.gpa, .{
.AsteriskAfterPointerDereference = .{ .token = period_asterisk },
});
const node = try p.arena.allocator.create(Node.SimpleSuffixOp);
node.* = .{
.base = .{ .tag = .Deref },
.lhs = lhs,
.rtoken = period_asterisk,
};
return &node.base;
}

if (p.eatToken(.Period)) |period| {
if (try p.parseIdentifier()) |identifier| {
const node = try p.arena.allocator.create(Node.SimpleInfixOp);
Expand Down
18 changes: 18 additions & 0 deletions lib/std/zig/parser_test.zig
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,24 @@ test "recovery: invalid global error set access" {
});
}

test "recovery: invalid asterisk after pointer dereference" {
try testError(
\\test "" {
\\ var sequence = "repeat".*** 10;
\\}
, &[_]Error{
.AsteriskAfterPointerDereference,
});
try testError(
\\test "" {
\\ var sequence = "repeat".** 10&&a;
\\}
, &[_]Error{
.AsteriskAfterPointerDereference,
.InvalidAnd,
});
}

test "recovery: missing semicolon after if, for, while stmt" {
try testError(
\\test "" {
Expand Down
46 changes: 43 additions & 3 deletions lib/std/zig/tokenizer.zig
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ pub const Token = struct {
pub const Id = enum {
Invalid,
Invalid_ampersands,
Invalid_periodasterisks,
Identifier,
StringLiteral,
MultilineStringLiteralLine,
Expand Down Expand Up @@ -201,6 +202,7 @@ pub const Token = struct {
return switch (id) {
.Invalid => "Invalid",
.Invalid_ampersands => "&&",
.Invalid_periodasterisks => ".**",
.Identifier => "Identifier",
.StringLiteral => "StringLiteral",
.MultilineStringLiteralLine => "MultilineStringLiteralLine",
Expand Down Expand Up @@ -403,6 +405,7 @@ pub const Tokenizer = struct {
angle_bracket_angle_bracket_right,
period,
period_2,
period_asterisk,
saw_at_sign,
};

Expand Down Expand Up @@ -979,9 +982,7 @@ pub const Tokenizer = struct {
state = .period_2;
},
'*' => {
result.id = .PeriodAsterisk;
self.index += 1;
break;
state = .period_asterisk;
},
else => {
result.id = .Period;
Expand All @@ -1001,6 +1002,17 @@ pub const Tokenizer = struct {
},
},

.period_asterisk => switch (c) {
'*' => {
result.id = .Invalid_periodasterisks;
break;
},
else => {
result.id = .PeriodAsterisk;
break;
},
},

.slash => switch (c) {
'/' => {
state = .line_comment_start;
Expand Down Expand Up @@ -1376,6 +1388,9 @@ pub const Tokenizer = struct {
.period_2 => {
result.id = .Ellipsis2;
},
.period_asterisk => {
result.id = .PeriodAsterisk;
},
.pipe => {
result.id = .Pipe;
},
Expand Down Expand Up @@ -1762,6 +1777,31 @@ test "correctly parse pointer assignment" {
});
}

test "correctly parse pointer dereference followed by asterisk" {
testTokenize("\"b\".* ** 10", &[_]Token.Id{
.StringLiteral,
.PeriodAsterisk,
.AsteriskAsterisk,
.IntegerLiteral,
});

testTokenize("(\"b\".*)** 10", &[_]Token.Id{
.LParen,
.StringLiteral,
.PeriodAsterisk,
.RParen,
.AsteriskAsterisk,
.IntegerLiteral,
});

testTokenize("\"b\".*** 10", &[_]Token.Id{
.StringLiteral,
.Invalid_periodasterisks,
.AsteriskAsterisk,
.IntegerLiteral,
});
}

test "tokenizer - range literals" {
testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral });
Expand Down
17 changes: 15 additions & 2 deletions src/stage1/tokenizer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@ enum TokenizeState {
TokenizeStateSawGreaterThanGreaterThan,
TokenizeStateSawDot,
TokenizeStateSawDotDot,
TokenizeStateSawDotStar,
TokenizeStateSawAtSign,
TokenizeStateCharCode,
TokenizeStateError,
Expand Down Expand Up @@ -566,9 +567,8 @@ void tokenize(Buf *buf, Tokenization *out) {
set_token_id(&t, t.cur_tok, TokenIdEllipsis2);
break;
case '*':
t.state = TokenizeStateStart;
t.state = TokenizeStateSawDotStar;
set_token_id(&t, t.cur_tok, TokenIdDotStar);
end_token(&t);
break;
default:
t.pos -= 1;
Expand All @@ -591,6 +591,18 @@ void tokenize(Buf *buf, Tokenization *out) {
continue;
}
break;
case TokenizeStateSawDotStar:
switch (c) {
case '*':
tokenize_error(&t, "`.*` can't be followed by `*`. Are you missing a space?");
break;
default:
t.pos -= 1;
end_token(&t);
t.state = TokenizeStateStart;
continue;
}
break;
case TokenizeStateSawGreaterThan:
switch (c) {
case '=':
Expand Down Expand Up @@ -1481,6 +1493,7 @@ void tokenize(Buf *buf, Tokenization *out) {
case TokenizeStateSawGreaterThan:
case TokenizeStateSawGreaterThanGreaterThan:
case TokenizeStateSawDot:
case TokenizeStateSawDotStar:
case TokenizeStateSawAtSign:
case TokenizeStateSawStarPercent:
case TokenizeStateSawPlusPercent:
Expand Down
8 changes: 8 additions & 0 deletions test/compile_errors.zig
Original file line number Diff line number Diff line change
Expand Up @@ -8195,4 +8195,12 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
, &[_][]const u8{
"tmp.zig:4:9: error: expected type '*c_void', found '?*c_void'",
});

cases.add("Issue #6823: don't allow .* to be followed by **",
\\fn foo() void {
\\ var sequence = "repeat".*** 10;
\\}
, &[_][]const u8{
"tmp.zig:2:30: error: `.*` can't be followed by `*`. Are you missing a space?",
});
}