From d4ff27180b31d7a4a5c284453b37d5301aece06d Mon Sep 17 00:00:00 2001 From: Benjamin Feng Date: Wed, 12 Jun 2019 14:00:25 -0500 Subject: [PATCH 1/5] Tokenize '&&' as AmpersandAmpersand --- std/zig/tokenizer.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 7bd5c537d3c0..df39c04f91b0 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -125,6 +125,7 @@ pub const Token = struct { SlashEqual, Comma, Ampersand, + AmpersandAmpersand, AmpersandEqual, QuestionMark, AngleBracketLeft, @@ -484,6 +485,10 @@ pub const Tokenizer = struct { }, State.Ampersand => switch (c) { + '&' => { + result.id = Token.Id.AmpersandAmpersand; + break; + }, '=' => { result.id = Token.Id.AmpersandEqual; self.index += 1; From 4708fb23c0d96de171d173e252689d26cd01b102 Mon Sep 17 00:00:00 2001 From: Benjamin Feng Date: Thu, 20 Jun 2019 22:06:20 -0500 Subject: [PATCH 2/5] Generate parse error from && --- std/zig/ast.zig | 4 ++++ std/zig/parse.zig | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 79d1ae8dada1..72a5234f6159 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -113,6 +113,7 @@ pub const Tree = struct { pub const Error = union(enum) { InvalidToken: InvalidToken, + InvalidAmpersandAmpersand: InvalidAmpersandAmpersand, ExpectedContainerMembers: ExpectedContainerMembers, ExpectedStringLiteral: ExpectedStringLiteral, ExpectedIntegerLiteral: ExpectedIntegerLiteral, @@ -161,6 +162,7 @@ pub const Error = union(enum) { switch (self.*) { // TODO https://github.com/ziglang/zig/issues/683 @TagType(Error).InvalidToken => |*x| return x.render(tokens, stream), + @TagType(Error).InvalidAmpersandAmpersand => |*x| return x.render(tokens, stream), @TagType(Error).ExpectedContainerMembers => |*x| return x.render(tokens, stream), @TagType(Error).ExpectedStringLiteral => |*x| return x.render(tokens, stream), @TagType(Error).ExpectedIntegerLiteral => |*x| return x.render(tokens, stream), @@ -211,6 +213,7 @@ pub const Error = union(enum) { switch (self.*) { // TODO https://github.com/ziglang/zig/issues/683 @TagType(Error).InvalidToken => |x| return x.token, + @TagType(Error).InvalidAmpersandAmpersand => |x| return x.token, @TagType(Error).ExpectedContainerMembers => |x| return x.token, @TagType(Error).ExpectedStringLiteral => |x| return x.token, @TagType(Error).ExpectedIntegerLiteral => |x| return x.token, @@ -291,6 +294,7 @@ pub const Error = union(enum) { pub const ExpectedDerefOrUnwrap = SingleTokenError("Expected pointer dereference or optional unwrap, found {}"); pub const ExpectedSuffixOp = SingleTokenError("Expected pointer dereference, optional unwrap, or field access, found {}"); + pub const InvalidAmpersandAmpersand = SimpleError("Invalid token '&&', 'and' performs boolean AND"); pub const ExpectedParamType = SimpleError("Expected parameter type"); pub const ExpectedPubItem = SimpleError("Pub must be followed by fn decl, var decl, or container member"); pub const UnattachedDocComment = SimpleError("Unattached documentation comment"); diff --git a/std/zig/parse.zig b/std/zig/parse.zig index 4f3fb76b5436..0daa9f35a8af 100644 --- a/std/zig/parse.zig +++ b/std/zig/parse.zig @@ -774,7 +774,7 @@ fn parseBoolAndExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node arena, it, tree, - SimpleBinOpParseFn(.Keyword_and, Node.InfixOp.Op.BoolAnd), + parseAmpersandAmpersandOp, parseCompareExpr, .Infinitely, ); @@ -2687,6 +2687,18 @@ fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) No // Helper parsers not included in the grammar +fn parseAmpersandAmpersandOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { + const op_parse_and = SimpleBinOpParseFn(.Keyword_and, Node.InfixOp.Op.BoolAnd); + const op_token = eatToken(it, .AmpersandAmpersand); + if (op_token != null) { + try tree.errors.push(AstError{ + .InvalidAmpersandAmpersand = AstError.InvalidAmpersandAmpersand{ .token = it.index }, + }); + return error.ParseError; + } + return op_parse_and(arena, it, tree); +} + fn parseBuiltinCall(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { const token = eatToken(it, .Builtin) orelse return null; const params = (try parseFnCallArguments(arena, it, tree)) orelse { From 484f8a4cc217fc08d09160ea02d8de6e122b07e9 Mon Sep 17 00:00:00 2001 From: Benjamin Feng Date: Thu, 20 Jun 2019 23:11:41 -0500 Subject: [PATCH 3/5] Switch old switch / @TagType() to use inferred enums --- std/zig/ast.zig | 198 ++++++++++++++++++++++++------------------------ 1 file changed, 98 insertions(+), 100 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 72a5234f6159..f648ced94386 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -160,103 +160,101 @@ pub const Error = union(enum) { pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void { switch (self.*) { - // TODO https://github.com/ziglang/zig/issues/683 - @TagType(Error).InvalidToken => |*x| return x.render(tokens, stream), - @TagType(Error).InvalidAmpersandAmpersand => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedContainerMembers => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedStringLiteral => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedIntegerLiteral => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedPubItem => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedIdentifier => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedStatement => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedVarDeclOrFn => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedVarDecl => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedReturnType => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedAggregateKw => |*x| return x.render(tokens, stream), - @TagType(Error).UnattachedDocComment => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedEqOrSemi => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedSemiOrLBrace => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedSemiOrElse => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedLabelOrLBrace => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedLBrace => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedColonOrRParen => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedLabelable => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedInlinable => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedAsmOutputReturnOrType => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedCall => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedCallOrFnProto => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedSliceOrRBracket => |*x| return x.render(tokens, stream), - @TagType(Error).ExtraAlignQualifier => |*x| return x.render(tokens, stream), - @TagType(Error).ExtraConstQualifier => |*x| return x.render(tokens, stream), - @TagType(Error).ExtraVolatileQualifier => |*x| return x.render(tokens, stream), - @TagType(Error).ExtraAllowZeroQualifier => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedTypeExpr => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedPrimaryTypeExpr => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedParamType => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedExpr => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedPrimaryExpr => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedToken => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedCommaOrEnd => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedParamList => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedPayload => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedBlockOrAssignment => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedBlockOrExpression => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedExprOrAssignment => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedPrefixExpr => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedLoopExpr => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedDerefOrUnwrap => |*x| return x.render(tokens, stream), - @TagType(Error).ExpectedSuffixOp => |*x| return x.render(tokens, stream), + .InvalidToken => |*x| return x.render(tokens, stream), + .InvalidAmpersandAmpersand => |*x| return x.render(tokens, stream), + .ExpectedContainerMembers => |*x| return x.render(tokens, stream), + .ExpectedStringLiteral => |*x| return x.render(tokens, stream), + .ExpectedIntegerLiteral => |*x| return x.render(tokens, stream), + .ExpectedPubItem => |*x| return x.render(tokens, stream), + .ExpectedIdentifier => |*x| return x.render(tokens, stream), + .ExpectedStatement => |*x| return x.render(tokens, stream), + .ExpectedVarDeclOrFn => |*x| return x.render(tokens, stream), + .ExpectedVarDecl => |*x| return x.render(tokens, stream), + .ExpectedReturnType => |*x| return x.render(tokens, stream), + .ExpectedAggregateKw => |*x| return x.render(tokens, stream), + .UnattachedDocComment => |*x| return x.render(tokens, stream), + .ExpectedEqOrSemi => |*x| return x.render(tokens, stream), + .ExpectedSemiOrLBrace => |*x| return x.render(tokens, stream), + .ExpectedSemiOrElse => |*x| return x.render(tokens, stream), + .ExpectedLabelOrLBrace => |*x| return x.render(tokens, stream), + .ExpectedLBrace => |*x| return x.render(tokens, stream), + .ExpectedColonOrRParen => |*x| return x.render(tokens, stream), + .ExpectedLabelable => |*x| return x.render(tokens, stream), + .ExpectedInlinable => |*x| return x.render(tokens, stream), + .ExpectedAsmOutputReturnOrType => |*x| return x.render(tokens, stream), + .ExpectedCall => |*x| return x.render(tokens, stream), + .ExpectedCallOrFnProto => |*x| return x.render(tokens, stream), + .ExpectedSliceOrRBracket => |*x| return x.render(tokens, stream), + .ExtraAlignQualifier => |*x| return x.render(tokens, stream), + .ExtraConstQualifier => |*x| return x.render(tokens, stream), + .ExtraVolatileQualifier => |*x| return x.render(tokens, stream), + .ExtraAllowZeroQualifier => |*x| return x.render(tokens, stream), + .ExpectedTypeExpr => |*x| return x.render(tokens, stream), + .ExpectedPrimaryTypeExpr => |*x| return x.render(tokens, stream), + .ExpectedParamType => |*x| return x.render(tokens, stream), + .ExpectedExpr => |*x| return x.render(tokens, stream), + .ExpectedPrimaryExpr => |*x| return x.render(tokens, stream), + .ExpectedToken => |*x| return x.render(tokens, stream), + .ExpectedCommaOrEnd => |*x| return x.render(tokens, stream), + .ExpectedParamList => |*x| return x.render(tokens, stream), + .ExpectedPayload => |*x| return x.render(tokens, stream), + .ExpectedBlockOrAssignment => |*x| return x.render(tokens, stream), + .ExpectedBlockOrExpression => |*x| return x.render(tokens, stream), + .ExpectedExprOrAssignment => |*x| return x.render(tokens, stream), + .ExpectedPrefixExpr => |*x| return x.render(tokens, stream), + .ExpectedLoopExpr => |*x| return x.render(tokens, stream), + .ExpectedDerefOrUnwrap => |*x| return x.render(tokens, stream), + .ExpectedSuffixOp => |*x| return x.render(tokens, stream), } } pub fn loc(self: *const Error) TokenIndex { switch (self.*) { - // TODO https://github.com/ziglang/zig/issues/683 - @TagType(Error).InvalidToken => |x| return x.token, - @TagType(Error).InvalidAmpersandAmpersand => |x| return x.token, - @TagType(Error).ExpectedContainerMembers => |x| return x.token, - @TagType(Error).ExpectedStringLiteral => |x| return x.token, - @TagType(Error).ExpectedIntegerLiteral => |x| return x.token, - @TagType(Error).ExpectedPubItem => |x| return x.token, - @TagType(Error).ExpectedIdentifier => |x| return x.token, - @TagType(Error).ExpectedStatement => |x| return x.token, - @TagType(Error).ExpectedVarDeclOrFn => |x| return x.token, - @TagType(Error).ExpectedVarDecl => |x| return x.token, - @TagType(Error).ExpectedReturnType => |x| return x.token, - @TagType(Error).ExpectedAggregateKw => |x| return x.token, - @TagType(Error).UnattachedDocComment => |x| return x.token, - @TagType(Error).ExpectedEqOrSemi => |x| return x.token, - @TagType(Error).ExpectedSemiOrLBrace => |x| return x.token, - @TagType(Error).ExpectedSemiOrElse => |x| return x.token, - @TagType(Error).ExpectedLabelOrLBrace => |x| return x.token, - @TagType(Error).ExpectedLBrace => |x| return x.token, - @TagType(Error).ExpectedColonOrRParen => |x| return x.token, - @TagType(Error).ExpectedLabelable => |x| return x.token, - @TagType(Error).ExpectedInlinable => |x| return x.token, - @TagType(Error).ExpectedAsmOutputReturnOrType => |x| return x.token, - @TagType(Error).ExpectedCall => |x| return x.node.firstToken(), - @TagType(Error).ExpectedCallOrFnProto => |x| return x.node.firstToken(), - @TagType(Error).ExpectedSliceOrRBracket => |x| return x.token, - @TagType(Error).ExtraAlignQualifier => |x| return x.token, - @TagType(Error).ExtraConstQualifier => |x| return x.token, - @TagType(Error).ExtraVolatileQualifier => |x| return x.token, - @TagType(Error).ExtraAllowZeroQualifier => |x| return x.token, - @TagType(Error).ExpectedTypeExpr => |x| return x.token, - @TagType(Error).ExpectedPrimaryTypeExpr => |x| return x.token, - @TagType(Error).ExpectedParamType => |x| return x.token, - @TagType(Error).ExpectedExpr => |x| return x.token, - @TagType(Error).ExpectedPrimaryExpr => |x| return x.token, - @TagType(Error).ExpectedToken => |x| return x.token, - @TagType(Error).ExpectedCommaOrEnd => |x| return x.token, - @TagType(Error).ExpectedParamList => |x| return x.token, - @TagType(Error).ExpectedPayload => |x| return x.token, - @TagType(Error).ExpectedBlockOrAssignment => |x| return x.token, - @TagType(Error).ExpectedBlockOrExpression => |x| return x.token, - @TagType(Error).ExpectedExprOrAssignment => |x| return x.token, - @TagType(Error).ExpectedPrefixExpr => |x| return x.token, - @TagType(Error).ExpectedLoopExpr => |x| return x.token, - @TagType(Error).ExpectedDerefOrUnwrap => |x| return x.token, - @TagType(Error).ExpectedSuffixOp => |x| return x.token, + .InvalidToken => |x| return x.token, + .InvalidAmpersandAmpersand => |x| return x.token, + .ExpectedContainerMembers => |x| return x.token, + .ExpectedStringLiteral => |x| return x.token, + .ExpectedIntegerLiteral => |x| return x.token, + .ExpectedPubItem => |x| return x.token, + .ExpectedIdentifier => |x| return x.token, + .ExpectedStatement => |x| return x.token, + .ExpectedVarDeclOrFn => |x| return x.token, + .ExpectedVarDecl => |x| return x.token, + .ExpectedReturnType => |x| return x.token, + .ExpectedAggregateKw => |x| return x.token, + .UnattachedDocComment => |x| return x.token, + .ExpectedEqOrSemi => |x| return x.token, + .ExpectedSemiOrLBrace => |x| return x.token, + .ExpectedSemiOrElse => |x| return x.token, + .ExpectedLabelOrLBrace => |x| return x.token, + .ExpectedLBrace => |x| return x.token, + .ExpectedColonOrRParen => |x| return x.token, + .ExpectedLabelable => |x| return x.token, + .ExpectedInlinable => |x| return x.token, + .ExpectedAsmOutputReturnOrType => |x| return x.token, + .ExpectedCall => |x| return x.node.firstToken(), + .ExpectedCallOrFnProto => |x| return x.node.firstToken(), + .ExpectedSliceOrRBracket => |x| return x.token, + .ExtraAlignQualifier => |x| return x.token, + .ExtraConstQualifier => |x| return x.token, + .ExtraVolatileQualifier => |x| return x.token, + .ExtraAllowZeroQualifier => |x| return x.token, + .ExpectedTypeExpr => |x| return x.token, + .ExpectedPrimaryTypeExpr => |x| return x.token, + .ExpectedParamType => |x| return x.token, + .ExpectedExpr => |x| return x.token, + .ExpectedPrimaryExpr => |x| return x.token, + .ExpectedToken => |x| return x.token, + .ExpectedCommaOrEnd => |x| return x.token, + .ExpectedParamList => |x| return x.token, + .ExpectedPayload => |x| return x.token, + .ExpectedBlockOrAssignment => |x| return x.token, + .ExpectedBlockOrExpression => |x| return x.token, + .ExpectedExprOrAssignment => |x| return x.token, + .ExpectedPrefixExpr => |x| return x.token, + .ExpectedLoopExpr => |x| return x.token, + .ExpectedDerefOrUnwrap => |x| return x.token, + .ExpectedSuffixOp => |x| return x.token, } } @@ -1712,15 +1710,15 @@ pub const Node = struct { i -= 1; switch (self.op) { - @TagType(Op).Call => |*call_info| { + .Call => |*call_info| { if (i < call_info.params.len) return call_info.params.at(i).*; i -= call_info.params.len; }, - Op.ArrayAccess => |index_expr| { + .ArrayAccess => |index_expr| { if (i < 1) return index_expr; i -= 1; }, - @TagType(Op).Slice => |range| { + .Slice => |range| { if (i < 1) return range.start; i -= 1; @@ -1729,16 +1727,16 @@ pub const Node = struct { i -= 1; } }, - Op.ArrayInitializer => |*exprs| { + .ArrayInitializer => |*exprs| { if (i < exprs.len) return exprs.at(i).*; i -= exprs.len; }, - Op.StructInitializer => |*fields| { + .StructInitializer => |*fields| { if (i < fields.len) return fields.at(i).*; i -= fields.len; }, - Op.UnwrapOptional, - Op.Deref, + .UnwrapOptional, + .Deref, => {}, } @@ -1747,7 +1745,7 @@ pub const Node = struct { pub fn firstToken(self: *const SuffixOp) TokenIndex { switch (self.op) { - @TagType(Op).Call => |*call_info| if (call_info.async_attr) |async_attr| return async_attr.firstToken(), + .Call => |*call_info| if (call_info.async_attr) |async_attr| return async_attr.firstToken(), else => {}, } return self.lhs.firstToken(); From 111d3792e0f2d92f7b9f73fb8265ee17fc229eb4 Mon Sep 17 00:00:00 2001 From: Benjamin Feng Date: Mon, 24 Jun 2019 23:31:59 -0500 Subject: [PATCH 4/5] Docgen invalid for && --- doc/docgen.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index bff110449c9e..831241de0f37 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -929,7 +929,9 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok std.zig.Token.Id.BracketStarCBracket, => try writeEscaped(out, src[token.start..token.end]), - std.zig.Token.Id.Invalid => return parseError( + std.zig.Token.Id.AmpersandAmpersand, + std.zig.Token.Id.Invalid, + => return parseError( docgen_tokenizer, source_token, "syntax error", From 8ea8cff4912bf66b8e95b7518877c7f55c7a9356 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 19 Jul 2019 17:54:06 -0400 Subject: [PATCH 5/5] slightly simpler implementation --- doc/docgen.zig | 238 +++++++++++++++++++++--------------------- std/zig/ast.zig | 19 ++-- std/zig/parse.zig | 14 +-- std/zig/tokenizer.zig | 5 +- 4 files changed, 135 insertions(+), 141 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 831241de0f37..3d3dcba76d5c 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -742,101 +742,101 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok const token = tokenizer.next(); try writeEscaped(out, src[index..token.start]); switch (token.id) { - std.zig.Token.Id.Eof => break, - - std.zig.Token.Id.Keyword_align, - std.zig.Token.Id.Keyword_and, - std.zig.Token.Id.Keyword_asm, - std.zig.Token.Id.Keyword_async, - std.zig.Token.Id.Keyword_await, - std.zig.Token.Id.Keyword_break, - std.zig.Token.Id.Keyword_cancel, - std.zig.Token.Id.Keyword_catch, - std.zig.Token.Id.Keyword_comptime, - std.zig.Token.Id.Keyword_const, - std.zig.Token.Id.Keyword_continue, - std.zig.Token.Id.Keyword_defer, - std.zig.Token.Id.Keyword_else, - std.zig.Token.Id.Keyword_enum, - std.zig.Token.Id.Keyword_errdefer, - std.zig.Token.Id.Keyword_error, - std.zig.Token.Id.Keyword_export, - std.zig.Token.Id.Keyword_extern, - std.zig.Token.Id.Keyword_for, - std.zig.Token.Id.Keyword_if, - std.zig.Token.Id.Keyword_inline, - std.zig.Token.Id.Keyword_nakedcc, - std.zig.Token.Id.Keyword_noalias, - std.zig.Token.Id.Keyword_or, - std.zig.Token.Id.Keyword_orelse, - std.zig.Token.Id.Keyword_packed, - std.zig.Token.Id.Keyword_promise, - std.zig.Token.Id.Keyword_pub, - std.zig.Token.Id.Keyword_resume, - std.zig.Token.Id.Keyword_return, - std.zig.Token.Id.Keyword_linksection, - std.zig.Token.Id.Keyword_stdcallcc, - std.zig.Token.Id.Keyword_struct, - std.zig.Token.Id.Keyword_suspend, - std.zig.Token.Id.Keyword_switch, - std.zig.Token.Id.Keyword_test, - std.zig.Token.Id.Keyword_threadlocal, - std.zig.Token.Id.Keyword_try, - std.zig.Token.Id.Keyword_union, - std.zig.Token.Id.Keyword_unreachable, - std.zig.Token.Id.Keyword_usingnamespace, - std.zig.Token.Id.Keyword_var, - std.zig.Token.Id.Keyword_volatile, - std.zig.Token.Id.Keyword_allowzero, - std.zig.Token.Id.Keyword_while, + .Eof => break, + + .Keyword_align, + .Keyword_and, + .Keyword_asm, + .Keyword_async, + .Keyword_await, + .Keyword_break, + .Keyword_cancel, + .Keyword_catch, + .Keyword_comptime, + .Keyword_const, + .Keyword_continue, + .Keyword_defer, + .Keyword_else, + .Keyword_enum, + .Keyword_errdefer, + .Keyword_error, + .Keyword_export, + .Keyword_extern, + .Keyword_for, + .Keyword_if, + .Keyword_inline, + .Keyword_nakedcc, + .Keyword_noalias, + .Keyword_or, + .Keyword_orelse, + .Keyword_packed, + .Keyword_promise, + .Keyword_pub, + .Keyword_resume, + .Keyword_return, + .Keyword_linksection, + .Keyword_stdcallcc, + .Keyword_struct, + .Keyword_suspend, + .Keyword_switch, + .Keyword_test, + .Keyword_threadlocal, + .Keyword_try, + .Keyword_union, + .Keyword_unreachable, + .Keyword_usingnamespace, + .Keyword_var, + .Keyword_volatile, + .Keyword_allowzero, + .Keyword_while, => { try out.write(""); try writeEscaped(out, src[token.start..token.end]); try out.write(""); }, - std.zig.Token.Id.Keyword_fn => { + .Keyword_fn => { try out.write(""); try writeEscaped(out, src[token.start..token.end]); try out.write(""); next_tok_is_fn = true; }, - std.zig.Token.Id.Keyword_undefined, - std.zig.Token.Id.Keyword_null, - std.zig.Token.Id.Keyword_true, - std.zig.Token.Id.Keyword_false, + .Keyword_undefined, + .Keyword_null, + .Keyword_true, + .Keyword_false, => { try out.write(""); try writeEscaped(out, src[token.start..token.end]); try out.write(""); }, - std.zig.Token.Id.StringLiteral, - std.zig.Token.Id.MultilineStringLiteralLine, - std.zig.Token.Id.CharLiteral, + .StringLiteral, + .MultilineStringLiteralLine, + .CharLiteral, => { try out.write(""); try writeEscaped(out, src[token.start..token.end]); try out.write(""); }, - std.zig.Token.Id.Builtin => { + .Builtin => { try out.write(""); try writeEscaped(out, src[token.start..token.end]); try out.write(""); }, - std.zig.Token.Id.LineComment, - std.zig.Token.Id.DocComment, - std.zig.Token.Id.ShebangLine, + .LineComment, + .DocComment, + .ShebangLine, => { try out.write(""); try writeEscaped(out, src[token.start..token.end]); try out.write(""); }, - std.zig.Token.Id.Identifier => { + .Identifier => { if (prev_tok_was_fn) { try out.write(""); try writeEscaped(out, src[token.start..token.end]); @@ -864,74 +864,72 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok } }, - std.zig.Token.Id.IntegerLiteral, - std.zig.Token.Id.FloatLiteral, + .IntegerLiteral, + .FloatLiteral, => { try out.write(""); try writeEscaped(out, src[token.start..token.end]); try out.write(""); }, - std.zig.Token.Id.Bang, - std.zig.Token.Id.Pipe, - std.zig.Token.Id.PipePipe, - std.zig.Token.Id.PipeEqual, - std.zig.Token.Id.Equal, - std.zig.Token.Id.EqualEqual, - std.zig.Token.Id.EqualAngleBracketRight, - std.zig.Token.Id.BangEqual, - std.zig.Token.Id.LParen, - std.zig.Token.Id.RParen, - std.zig.Token.Id.Semicolon, - std.zig.Token.Id.Percent, - std.zig.Token.Id.PercentEqual, - std.zig.Token.Id.LBrace, - std.zig.Token.Id.RBrace, - std.zig.Token.Id.LBracket, - std.zig.Token.Id.RBracket, - std.zig.Token.Id.Period, - std.zig.Token.Id.Ellipsis2, - std.zig.Token.Id.Ellipsis3, - std.zig.Token.Id.Caret, - std.zig.Token.Id.CaretEqual, - std.zig.Token.Id.Plus, - std.zig.Token.Id.PlusPlus, - std.zig.Token.Id.PlusEqual, - std.zig.Token.Id.PlusPercent, - std.zig.Token.Id.PlusPercentEqual, - std.zig.Token.Id.Minus, - std.zig.Token.Id.MinusEqual, - std.zig.Token.Id.MinusPercent, - std.zig.Token.Id.MinusPercentEqual, - std.zig.Token.Id.Asterisk, - std.zig.Token.Id.AsteriskEqual, - std.zig.Token.Id.AsteriskAsterisk, - std.zig.Token.Id.AsteriskPercent, - std.zig.Token.Id.AsteriskPercentEqual, - std.zig.Token.Id.Arrow, - std.zig.Token.Id.Colon, - std.zig.Token.Id.Slash, - std.zig.Token.Id.SlashEqual, - std.zig.Token.Id.Comma, - std.zig.Token.Id.Ampersand, - std.zig.Token.Id.AmpersandEqual, - std.zig.Token.Id.QuestionMark, - std.zig.Token.Id.AngleBracketLeft, - std.zig.Token.Id.AngleBracketLeftEqual, - std.zig.Token.Id.AngleBracketAngleBracketLeft, - std.zig.Token.Id.AngleBracketAngleBracketLeftEqual, - std.zig.Token.Id.AngleBracketRight, - std.zig.Token.Id.AngleBracketRightEqual, - std.zig.Token.Id.AngleBracketAngleBracketRight, - std.zig.Token.Id.AngleBracketAngleBracketRightEqual, - std.zig.Token.Id.Tilde, - std.zig.Token.Id.BracketStarBracket, - std.zig.Token.Id.BracketStarCBracket, + .Bang, + .Pipe, + .PipePipe, + .PipeEqual, + .Equal, + .EqualEqual, + .EqualAngleBracketRight, + .BangEqual, + .LParen, + .RParen, + .Semicolon, + .Percent, + .PercentEqual, + .LBrace, + .RBrace, + .LBracket, + .RBracket, + .Period, + .Ellipsis2, + .Ellipsis3, + .Caret, + .CaretEqual, + .Plus, + .PlusPlus, + .PlusEqual, + .PlusPercent, + .PlusPercentEqual, + .Minus, + .MinusEqual, + .MinusPercent, + .MinusPercentEqual, + .Asterisk, + .AsteriskEqual, + .AsteriskAsterisk, + .AsteriskPercent, + .AsteriskPercentEqual, + .Arrow, + .Colon, + .Slash, + .SlashEqual, + .Comma, + .Ampersand, + .AmpersandEqual, + .QuestionMark, + .AngleBracketLeft, + .AngleBracketLeftEqual, + .AngleBracketAngleBracketLeft, + .AngleBracketAngleBracketLeftEqual, + .AngleBracketRight, + .AngleBracketRightEqual, + .AngleBracketAngleBracketRight, + .AngleBracketAngleBracketRightEqual, + .Tilde, + .BracketStarBracket, + .BracketStarCBracket, => try writeEscaped(out, src[token.start..token.end]), - std.zig.Token.Id.AmpersandAmpersand, - std.zig.Token.Id.Invalid, - => return parseError( + .Invalid, .Invalid_ampersands => return parseError( docgen_tokenizer, source_token, "syntax error", diff --git a/std/zig/ast.zig b/std/zig/ast.zig index f648ced94386..38bd94339f5b 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -113,7 +113,6 @@ pub const Tree = struct { pub const Error = union(enum) { InvalidToken: InvalidToken, - InvalidAmpersandAmpersand: InvalidAmpersandAmpersand, ExpectedContainerMembers: ExpectedContainerMembers, ExpectedStringLiteral: ExpectedStringLiteral, ExpectedIntegerLiteral: ExpectedIntegerLiteral, @@ -161,7 +160,6 @@ pub const Error = union(enum) { pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void { switch (self.*) { .InvalidToken => |*x| return x.render(tokens, stream), - .InvalidAmpersandAmpersand => |*x| return x.render(tokens, stream), .ExpectedContainerMembers => |*x| return x.render(tokens, stream), .ExpectedStringLiteral => |*x| return x.render(tokens, stream), .ExpectedIntegerLiteral => |*x| return x.render(tokens, stream), @@ -211,7 +209,6 @@ pub const Error = union(enum) { pub fn loc(self: *const Error) TokenIndex { switch (self.*) { .InvalidToken => |x| return x.token, - .InvalidAmpersandAmpersand => |x| return x.token, .ExpectedContainerMembers => |x| return x.token, .ExpectedStringLiteral => |x| return x.token, .ExpectedIntegerLiteral => |x| return x.token, @@ -292,7 +289,6 @@ pub const Error = union(enum) { pub const ExpectedDerefOrUnwrap = SingleTokenError("Expected pointer dereference or optional unwrap, found {}"); pub const ExpectedSuffixOp = SingleTokenError("Expected pointer dereference, optional unwrap, or field access, found {}"); - pub const InvalidAmpersandAmpersand = SimpleError("Invalid token '&&', 'and' performs boolean AND"); pub const ExpectedParamType = SimpleError("Expected parameter type"); pub const ExpectedPubItem = SimpleError("Pub must be followed by fn decl, var decl, or container member"); pub const UnattachedDocComment = SimpleError("Unattached documentation comment"); @@ -322,8 +318,19 @@ pub const Error = union(enum) { expected_id: Token.Id, pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void { - const token_name = @tagName(tokens.at(self.token).id); - return stream.print("expected {}, found {}", @tagName(self.expected_id), token_name); + const found_token = tokens.at(self.token); + switch (found_token.id) { + .Invalid_ampersands => { + return stream.print("`&&` is invalid. Note that `and` is boolean AND."); + }, + .Invalid => { + return stream.print("expected {}, found invalid bytes", @tagName(self.expected_id)); + }, + else => { + const token_name = @tagName(found_token.id); + return stream.print("expected {}, found {}", @tagName(self.expected_id), token_name); + }, + } } }; diff --git a/std/zig/parse.zig b/std/zig/parse.zig index 0daa9f35a8af..59acf9989084 100644 --- a/std/zig/parse.zig +++ b/std/zig/parse.zig @@ -774,7 +774,7 @@ fn parseBoolAndExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node arena, it, tree, - parseAmpersandAmpersandOp, + SimpleBinOpParseFn(.Keyword_and, .BoolAnd), parseCompareExpr, .Infinitely, ); @@ -2687,18 +2687,6 @@ fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) No // Helper parsers not included in the grammar -fn parseAmpersandAmpersandOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const op_parse_and = SimpleBinOpParseFn(.Keyword_and, Node.InfixOp.Op.BoolAnd); - const op_token = eatToken(it, .AmpersandAmpersand); - if (op_token != null) { - try tree.errors.push(AstError{ - .InvalidAmpersandAmpersand = AstError.InvalidAmpersandAmpersand{ .token = it.index }, - }); - return error.ParseError; - } - return op_parse_and(arena, it, tree); -} - fn parseBuiltinCall(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { const token = eatToken(it, .Builtin) orelse return null; const params = (try parseFnCallArguments(arena, it, tree)) orelse { diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index df39c04f91b0..4539e1e5b271 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -77,6 +77,7 @@ pub const Token = struct { pub const Id = enum { Invalid, + Invalid_ampersands, Identifier, StringLiteral, MultilineStringLiteralLine, @@ -125,7 +126,6 @@ pub const Token = struct { SlashEqual, Comma, Ampersand, - AmpersandAmpersand, AmpersandEqual, QuestionMark, AngleBracketLeft, @@ -486,7 +486,8 @@ pub const Tokenizer = struct { State.Ampersand => switch (c) { '&' => { - result.id = Token.Id.AmpersandAmpersand; + result.id = Token.Id.Invalid_ampersands; + self.index += 1; break; }, '=' => {