support missing last semicolon

goccy · Sep 12, 2013 · 610779b · 610779b
1 parent 307b879
commit 610779b
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 29 deletions.
diff --git a/include/parser.hpp b/include/parser.hpp
@@ -262,7 +262,9 @@ class Parser {
 	bool isSingleTermOperator(ParseContext *pctx, Token *tk);
 	bool isIrregularFunction(ParseContext *pctx, Token *tk);
 	bool isMissingSemicolon(Enum::Token::Type::Type prev_type, Enum::Token::Type::Type type, Tokens *tokens);
+	bool isMissingSemicolon(Tokens *tokens);
 	bool canGrouping(Token *tk, Token *next_tk);
+	Token *replaceToStmt(Tokens *tokens, Token *cur_tk, size_t offset);
 	void parseStmt(ParseContext *pctx, Node *stmt);
 	void parseExpr(ParseContext *pctx, Node *expr);
 	void parseToken(ParseContext *pctx, Token *tk);

diff --git a/src/compiler/parser/Compiler_parser.cpp b/src/compiler/parser/Compiler_parser.cpp
@@ -250,6 +250,41 @@ bool Parser::isMissingSemicolon(TokenType::Type prev_type, TokenType::Type type,
 	return false;
 }
 
+bool Parser::isMissingSemicolon(Tokens *tokens)
+{
+	using namespace TokenType;
+	Token *tk = tokens->back();
+	if (tk->stype != SyntaxType::Stmt) {
+		size_t size = tokens->size();
+		for (size_t i = 0; i < size; i++) {
+			if (tokens->at(i)->stype == SyntaxType::Stmt) {
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+Token *Parser::replaceToStmt(Tokens *cur_tokens, Token *cur_tk, size_t offset)
+{
+	Tokens *stmt = new Tokens();
+	for (size_t i = 0; i < offset - 1; i++) {
+		Token *tk = cur_tokens->back();
+		i += (tk->total_token_num > 0) ? tk->total_token_num - 1 : 0;
+		stmt->insert(stmt->begin(), tk);
+		cur_tokens->pop_back();
+	}
+	Token *semicolon = new Token(";", cur_tk->finfo);
+	semicolon->info.type = TokenType::SemiColon;
+	semicolon->info.name = "SemiColon";
+	semicolon->info.kind = TokenKind::StmtEnd;
+	stmt->push_back(semicolon);
+	Token *stmt_ = new Token(stmt);
+	stmt_->stype = SyntaxType::Stmt;
+	cur_tokens->push_back(stmt_);
+	return stmt_;
+}
+
 Token *Parser::parseSyntax(Token *start_token, Tokens *tokens)
 {
 	using namespace TokenType;
@@ -308,23 +343,7 @@ Token *Parser::parseSyntax(Token *start_token, Tokens *tokens)
 		}
 		case RightBrace: case RightBracket: case RightParenthesis: {
 			if (isMissingSemicolon(prev_type, t->info.type, new_tokens)) {
-				size_t k = pos - intermediate_pos;
-				Tokens *stmt = new Tokens();
-				for (size_t j = 0; j < k - 1; j++) {
-					Token *tk = new_tokens->back();
-					j += (tk->total_token_num > 0) ? tk->total_token_num - 1 : 0;
-					stmt->insert(stmt->begin(), tk);
-					new_tokens->pop_back();
-				}
-				Token *semicolon = new Token(";", t->finfo);
-				semicolon->info.type = SemiColon;
-				semicolon->info.name = "SemiColon";
-				semicolon->info.kind = TokenKind::StmtEnd;
-				stmt->push_back(semicolon);
-				Token *stmt_ = new Token(stmt);
-				stmt_->stype = SyntaxType::Stmt;
-				new_tokens->push_back(stmt_);
-				prev_syntax = stmt_;
+				prev_syntax = replaceToStmt(new_tokens, t, pos - intermediate_pos);
 			}
 			new_tokens->push_back(t);
 			return new Token(new_tokens);
@@ -333,19 +352,8 @@ Token *Parser::parseSyntax(Token *start_token, Tokens *tokens)
 		case SemiColon: {
 			size_t k = pos - intermediate_pos;
 			if (start_pos == intermediate_pos) k++;
-			Tokens *stmt = new Tokens();
-			for (size_t j = 0; j < k - 1; j++) {
-				Token *tk = new_tokens->back();
-				j += (tk->total_token_num > 0) ? tk->total_token_num - 1 : 0;
-				stmt->insert(stmt->begin(), tk);
-				new_tokens->pop_back();
-			}
-			stmt->push_back(t);
-			Token *stmt_ = new Token(stmt);
-			stmt_->stype = SyntaxType::Stmt;
-			new_tokens->push_back(stmt_);
+			prev_syntax = replaceToStmt(new_tokens, t, k);
 			intermediate_pos = pos;
-			prev_syntax = stmt_;
 			break;
 		}
 		default:
@@ -358,6 +366,9 @@ Token *Parser::parseSyntax(Token *start_token, Tokens *tokens)
 		if (pos == end_pos) Parser_exception("nothing end flagment", t->finfo.start_line_num);
 		pos++;
 	}
+	if (isMissingSemicolon(new_tokens)) {
+		replaceToStmt(new_tokens, new_tokens->back(), pos - intermediate_pos);
+	}
 	return new Token(new_tokens);
 }
 

diff --git a/t/module.t b/t/module.t
@@ -30,6 +30,21 @@ subtest 'module version argument' => sub {
     is($ast->data, 'ModuleName');
     is($ast->args->data, '54');
 
+    $tokens = Compiler::Lexer->new('-')->tokenize('use Foo; use Bar');
+    $ast = Compiler::Parser->new->parse($tokens);
+    Compiler::Parser::AST::Renderer->new->render($ast);
+    is(ref $ast, 'Compiler::Parser::Node::Module');
+    is(ref $ast->next, 'Compiler::Parser::Node::Module');
+
+    $tokens = Compiler::Lexer->new('-')->tokenize('use Foo; my $x = sub { }');
+    $ast = Compiler::Parser->new->parse($tokens);
+    Compiler::Parser::AST::Renderer->new->render($ast);
+    is(ref $ast, 'Compiler::Parser::Node::Module');
+    is(ref $ast->next, 'Compiler::Parser::Node::Branch');
+    is(ref $ast->next->left, 'Compiler::Parser::Node::Leaf');
+    is(ref $ast->next->right, 'Compiler::Parser::Node::Function');
+    is(ref $ast->next->right->body, 'Compiler::Parser::Node::HashRef');
+
 };
 
 done_testing;