From 5b1962cef056a19af6363f8ccf9cd05baef87d28 Mon Sep 17 00:00:00 2001 From: Saif Eddin Gmati <29315886+azjezz@users.noreply.github.com> Date: Thu, 1 Dec 2022 01:33:38 +0100 Subject: [PATCH 01/13] chore: refactor project structure - added `prelude` - moved `ast` to `parser` - moved `Lexer` from `crate::lexer::lexer::Lexer` to `crate::lexer::Lexer` - renamed `LexerError` to `SyntaxError` - moved `SyntaxError` to `token::error` - implemented `Display` for `SyntaxError` - add `tokens.txt` to integration tests - removed all unit tests for `Lexer` ( everything is tested in integration tests ) - improved syntax error assertations --- .github/workflows/tests.yml | 2 +- bin/snapshot.rs | 18 +- build.rs | 60 +- src/lexer/error.rs | 58 + src/lexer/lexer.rs | 1776 ------------------------------ src/lexer/macros.rs | 15 + src/lexer/mod.rs | 1170 +++++++++++++++++++- src/lexer/token.rs | 2 +- src/lib.rs | 12 +- src/main.rs | 42 +- src/{ => parser}/ast.rs | 15 +- src/parser/block.rs | 11 +- src/parser/classish.rs | 29 +- src/parser/classish_statement.rs | 34 +- src/parser/comments.rs | 12 +- src/parser/error.rs | 12 +- src/parser/flags.rs | 16 +- src/parser/functions.rs | 24 +- src/parser/ident.rs | 24 +- src/parser/mod.rs | 23 +- src/parser/params.rs | 24 +- src/parser/precedence.rs | 2 +- src/parser/punc.rs | 23 +- src/parser/vars.rs | 11 +- src/prelude.rs | 8 + tests/0001/tokens.txt | 452 ++++++++ tests/0002/tokens.txt | 126 +++ tests/0003/tokens.txt | 157 +++ tests/0004/tokens.txt | 166 +++ tests/0005/tokens.txt | 143 +++ tests/0006/tokens.txt | 103 ++ tests/0007/tokens.txt | 215 ++++ tests/0008/tokens.txt | 395 +++++++ tests/0009/tokens.txt | 167 +++ tests/0010/tokens.txt | 1083 ++++++++++++++++++ tests/0011/tokens.txt | 1152 +++++++++++++++++++ tests/0012/tokens.txt | 1300 ++++++++++++++++++++++ tests/0013/tokens.txt | 420 +++++++ tests/0014/tokens.txt | 434 ++++++++ tests/0015/tokens.txt | 230 ++++ tests/0016/tokens.txt | 78 ++ tests/0017/tokens.txt | 220 ++++ tests/0018/tokens.txt | 235 ++++ tests/0019/tokens.txt | 1601 +++++++++++++++++++++++++++ tests/0020/tokens.txt | 811 ++++++++++++++ tests/0021/tokens.txt | 552 ++++++++++ tests/0022/tokens.txt | 265 +++++ tests/0023/tokens.txt | 505 +++++++++ tests/0024/tokens.txt | 414 +++++++ tests/0025/tokens.txt | 25 + tests/0026/tokens.txt | 34 + tests/0027/tokens.txt | 25 + tests/0028/tokens.txt | 34 + tests/0029/tokens.txt | 91 ++ tests/0030/tokens.txt | 193 ++++ tests/0031/tokens.txt | 159 +++ tests/0032/tokens.txt | 186 ++++ tests/0033/tokens.txt | 241 ++++ tests/0034/tokens.txt | 59 + tests/0035/tokens.txt | 55 + tests/0036/tokens.txt | 64 ++ tests/0037/tokens.txt | 80 ++ tests/0038/tokens.txt | 264 +++++ tests/0039/tokens.txt | 64 ++ tests/0040/tokens.txt | 122 ++ tests/0041/tokens.txt | 189 ++++ tests/0042/tokens.txt | 34 + tests/0043/tokens.txt | 41 + tests/0044/tokens.txt | 90 ++ tests/0045/tokens.txt | 108 ++ tests/0046/tokens.txt | 57 + tests/0047/tokens.txt | 73 ++ tests/0048/tokens.txt | 73 ++ tests/0049/tokens.txt | 121 ++ tests/0050/tokens.txt | 71 ++ tests/0051/tokens.txt | 80 ++ tests/0052/tokens.txt | 103 ++ tests/0053/tokens.txt | 80 ++ tests/0054/tokens.txt | 89 ++ tests/0055/tokens.txt | 105 ++ tests/0056/tokens.txt | 89 ++ tests/0057/tokens.txt | 105 ++ tests/0058/tokens.txt | 71 ++ tests/0059/tokens.txt | 71 ++ tests/0060/tokens.txt | 46 + tests/0061/tokens.txt | 85 ++ tests/0062/tokens.txt | 62 ++ tests/0063/tokens.txt | 78 ++ tests/0064/tokens.txt | 97 ++ tests/0065/tokens.txt | 25 + tests/0066/tokens.txt | 41 + tests/0067/tokens.txt | 18 + tests/0068/tokens.txt | 73 ++ tests/0069/tokens.txt | 69 ++ tests/0070/tokens.txt | 90 ++ tests/0071/tokens.txt | 27 + tests/0072/tokens.txt | 48 + tests/0073/tokens.txt | 43 + tests/0074/tokens.txt | 57 + tests/0075/tokens.txt | 66 ++ tests/0076/tokens.txt | 73 ++ tests/0077/tokens.txt | 50 + tests/0078/tokens.txt | 82 ++ tests/0079/tokens.txt | 34 + tests/0080/tokens.txt | 50 + tests/0081/tokens.txt | 64 ++ tests/0082/tokens.txt | 96 ++ tests/0083/tokens.txt | 94 ++ tests/0084/tokens.txt | 80 ++ tests/0085/tokens.txt | 48 + tests/0086/tokens.txt | 71 ++ tests/0087/tokens.txt | 119 ++ tests/0088/tokens.txt | 55 + tests/0089/tokens.txt | 64 ++ tests/0090/tokens.txt | 64 ++ tests/0091/tokens.txt | 69 ++ tests/0092/tokens.txt | 69 ++ tests/0093/tokens.txt | 48 + tests/0094/tokens.txt | 62 ++ tests/0095/tokens.txt | 34 + tests/0096/tokens.txt | 50 + tests/0097/tokens.txt | 64 ++ tests/0098/tokens.txt | 50 + tests/0099/tokens.txt | 64 ++ tests/0100/tokens.txt | 71 ++ tests/0101/tokens.txt | 53 + tests/0102/tokens.txt | 99 ++ tests/0103/tokens.txt | 34 + tests/0105/tokens.txt | 25 + tests/0106/tokens.txt | 34 + tests/0107/tokens.txt | 50 + tests/0108/tokens.txt | 41 + tests/0109/tokens.txt | 239 ++++ tests/0110/tokens.txt | 232 ++++ tests/0111/tokens.txt | 129 +++ tests/0112/tokens.txt | 129 +++ tests/0113/tokens.txt | 124 +++ tests/0114/tokens.txt | 117 ++ tests/0115/tokens.txt | 131 +++ tests/0116/tokens.txt | 124 +++ tests/0117/tokens.txt | 135 +++ tests/0118/tokens.txt | 87 ++ tests/0119/tokens.txt | 135 +++ tests/0120/tokens.txt | 85 ++ tests/0121/tokens.txt | 87 ++ tests/0122/tokens.txt | 87 ++ tests/0123/tokens.txt | 87 ++ tests/0124/tokens.txt | 99 ++ tests/0125/tokens.txt | 99 ++ tests/0126/tokens.txt | 94 ++ tests/0127/tokens.txt | 147 +++ tests/0128/tokens.txt | 138 +++ tests/0129/tokens.txt | 161 +++ tests/0130/tokens.txt | 99 ++ tests/0131/tokens.txt | 62 ++ tests/0133/code.php | 5 + tests/0133/lexer-error.txt | 1 + tests/0134/code.php | 4 + tests/0134/lexer-error.txt | 1 + tests/0135/code.php | 4 + tests/0135/lexer-error.txt | 1 + tests/0136/code.php | 4 + tests/0136/lexer-error.txt | 1 + tests/0137/code.php | 4 + tests/0137/lexer-error.txt | 1 + tests/0138/code.php | 3 + tests/0138/lexer-error.txt | 1 + tests/0139/code.php | 3 + tests/0139/lexer-error.txt | 1 + tests/0140/code.php | 3 + tests/0140/lexer-error.txt | 1 + tests/third_party_tests.rs | 6 +- 172 files changed, 21798 insertions(+), 1952 deletions(-) create mode 100644 src/lexer/error.rs delete mode 100644 src/lexer/lexer.rs create mode 100644 src/lexer/macros.rs rename src/{ => parser}/ast.rs (98%) create mode 100644 src/prelude.rs create mode 100644 tests/0001/tokens.txt create mode 100644 tests/0002/tokens.txt create mode 100644 tests/0003/tokens.txt create mode 100644 tests/0004/tokens.txt create mode 100644 tests/0005/tokens.txt create mode 100644 tests/0006/tokens.txt create mode 100644 tests/0007/tokens.txt create mode 100644 tests/0008/tokens.txt create mode 100644 tests/0009/tokens.txt create mode 100644 tests/0010/tokens.txt create mode 100644 tests/0011/tokens.txt create mode 100644 tests/0012/tokens.txt create mode 100644 tests/0013/tokens.txt create mode 100644 tests/0014/tokens.txt create mode 100644 tests/0015/tokens.txt create mode 100644 tests/0016/tokens.txt create mode 100644 tests/0017/tokens.txt create mode 100644 tests/0018/tokens.txt create mode 100644 tests/0019/tokens.txt create mode 100644 tests/0020/tokens.txt create mode 100644 tests/0021/tokens.txt create mode 100644 tests/0022/tokens.txt create mode 100644 tests/0023/tokens.txt create mode 100644 tests/0024/tokens.txt create mode 100644 tests/0025/tokens.txt create mode 100644 tests/0026/tokens.txt create mode 100644 tests/0027/tokens.txt create mode 100644 tests/0028/tokens.txt create mode 100644 tests/0029/tokens.txt create mode 100644 tests/0030/tokens.txt create mode 100644 tests/0031/tokens.txt create mode 100644 tests/0032/tokens.txt create mode 100644 tests/0033/tokens.txt create mode 100644 tests/0034/tokens.txt create mode 100644 tests/0035/tokens.txt create mode 100644 tests/0036/tokens.txt create mode 100644 tests/0037/tokens.txt create mode 100644 tests/0038/tokens.txt create mode 100644 tests/0039/tokens.txt create mode 100644 tests/0040/tokens.txt create mode 100644 tests/0041/tokens.txt create mode 100644 tests/0042/tokens.txt create mode 100644 tests/0043/tokens.txt create mode 100644 tests/0044/tokens.txt create mode 100644 tests/0045/tokens.txt create mode 100644 tests/0046/tokens.txt create mode 100644 tests/0047/tokens.txt create mode 100644 tests/0048/tokens.txt create mode 100644 tests/0049/tokens.txt create mode 100644 tests/0050/tokens.txt create mode 100644 tests/0051/tokens.txt create mode 100644 tests/0052/tokens.txt create mode 100644 tests/0053/tokens.txt create mode 100644 tests/0054/tokens.txt create mode 100644 tests/0055/tokens.txt create mode 100644 tests/0056/tokens.txt create mode 100644 tests/0057/tokens.txt create mode 100644 tests/0058/tokens.txt create mode 100644 tests/0059/tokens.txt create mode 100644 tests/0060/tokens.txt create mode 100644 tests/0061/tokens.txt create mode 100644 tests/0062/tokens.txt create mode 100644 tests/0063/tokens.txt create mode 100644 tests/0064/tokens.txt create mode 100644 tests/0065/tokens.txt create mode 100644 tests/0066/tokens.txt create mode 100644 tests/0067/tokens.txt create mode 100644 tests/0068/tokens.txt create mode 100644 tests/0069/tokens.txt create mode 100644 tests/0070/tokens.txt create mode 100644 tests/0071/tokens.txt create mode 100644 tests/0072/tokens.txt create mode 100644 tests/0073/tokens.txt create mode 100644 tests/0074/tokens.txt create mode 100644 tests/0075/tokens.txt create mode 100644 tests/0076/tokens.txt create mode 100644 tests/0077/tokens.txt create mode 100644 tests/0078/tokens.txt create mode 100644 tests/0079/tokens.txt create mode 100644 tests/0080/tokens.txt create mode 100644 tests/0081/tokens.txt create mode 100644 tests/0082/tokens.txt create mode 100644 tests/0083/tokens.txt create mode 100644 tests/0084/tokens.txt create mode 100644 tests/0085/tokens.txt create mode 100644 tests/0086/tokens.txt create mode 100644 tests/0087/tokens.txt create mode 100644 tests/0088/tokens.txt create mode 100644 tests/0089/tokens.txt create mode 100644 tests/0090/tokens.txt create mode 100644 tests/0091/tokens.txt create mode 100644 tests/0092/tokens.txt create mode 100644 tests/0093/tokens.txt create mode 100644 tests/0094/tokens.txt create mode 100644 tests/0095/tokens.txt create mode 100644 tests/0096/tokens.txt create mode 100644 tests/0097/tokens.txt create mode 100644 tests/0098/tokens.txt create mode 100644 tests/0099/tokens.txt create mode 100644 tests/0100/tokens.txt create mode 100644 tests/0101/tokens.txt create mode 100644 tests/0102/tokens.txt create mode 100644 tests/0103/tokens.txt create mode 100644 tests/0105/tokens.txt create mode 100644 tests/0106/tokens.txt create mode 100644 tests/0107/tokens.txt create mode 100644 tests/0108/tokens.txt create mode 100644 tests/0109/tokens.txt create mode 100644 tests/0110/tokens.txt create mode 100644 tests/0111/tokens.txt create mode 100644 tests/0112/tokens.txt create mode 100644 tests/0113/tokens.txt create mode 100644 tests/0114/tokens.txt create mode 100644 tests/0115/tokens.txt create mode 100644 tests/0116/tokens.txt create mode 100644 tests/0117/tokens.txt create mode 100644 tests/0118/tokens.txt create mode 100644 tests/0119/tokens.txt create mode 100644 tests/0120/tokens.txt create mode 100644 tests/0121/tokens.txt create mode 100644 tests/0122/tokens.txt create mode 100644 tests/0123/tokens.txt create mode 100644 tests/0124/tokens.txt create mode 100644 tests/0125/tokens.txt create mode 100644 tests/0126/tokens.txt create mode 100644 tests/0127/tokens.txt create mode 100644 tests/0128/tokens.txt create mode 100644 tests/0129/tokens.txt create mode 100644 tests/0130/tokens.txt create mode 100644 tests/0131/tokens.txt create mode 100644 tests/0133/code.php create mode 100644 tests/0133/lexer-error.txt create mode 100644 tests/0134/code.php create mode 100644 tests/0134/lexer-error.txt create mode 100644 tests/0135/code.php create mode 100644 tests/0135/lexer-error.txt create mode 100644 tests/0136/code.php create mode 100644 tests/0136/lexer-error.txt create mode 100644 tests/0137/code.php create mode 100644 tests/0137/lexer-error.txt create mode 100644 tests/0138/code.php create mode 100644 tests/0138/lexer-error.txt create mode 100644 tests/0139/code.php create mode 100644 tests/0139/lexer-error.txt create mode 100644 tests/0140/code.php create mode 100644 tests/0140/lexer-error.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 543371a4..2b5a364d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,6 +37,6 @@ jobs: run: | cargo fmt --all -- --check cargo clippy - + - name: test run: ./meta/test --all -- --skip third_party diff --git a/bin/snapshot.rs b/bin/snapshot.rs index 4f1efae1..378af610 100644 --- a/bin/snapshot.rs +++ b/bin/snapshot.rs @@ -1,4 +1,4 @@ -use php_parser_rs::{Lexer, Parser}; +use php_parser_rs::prelude::{Lexer, Parser}; use std::env; use std::fs::read_dir; use std::path::PathBuf; @@ -21,6 +21,7 @@ fn main() { for entry in entries { let code_filename = entry.join("code.php"); let ast_filename = entry.join("ast.txt"); + let tokens_filename = entry.join("tokens.txt"); let lexer_error_filename = entry.join("lexer-error.txt"); let parser_error_filename = entry.join("parser-error.txt"); @@ -32,6 +33,10 @@ fn main() { std::fs::remove_file(&ast_filename).unwrap(); } + if tokens_filename.exists() { + std::fs::remove_file(&tokens_filename).unwrap(); + } + if lexer_error_filename.exists() { std::fs::remove_file(&lexer_error_filename).unwrap(); } @@ -41,11 +46,17 @@ fn main() { } let code = std::fs::read_to_string(&code_filename).unwrap(); - let mut lexer = Lexer::new(None); + let mut lexer = Lexer::new(); let tokens = lexer.tokenize(code.as_bytes()); match tokens { Ok(tokens) => { + std::fs::write(tokens_filename, format!("{:#?}\n", tokens)).unwrap(); + println!( + "✅ generated `tokens.txt` for `{}`", + entry.to_string_lossy() + ); + let mut parser = Parser::new(None); let ast = parser.parse(tokens); match ast { @@ -67,7 +78,8 @@ fn main() { } } Err(error) => { - std::fs::write(lexer_error_filename, format!("{:?}\n", error)).unwrap(); + std::fs::write(lexer_error_filename, format!("{:?} -> {}\n", error, error)) + .unwrap(); println!( "✅ generated `lexer-error.txt` for `{}`", entry.to_string_lossy() diff --git a/build.rs b/build.rs index 7d63c68d..a4d3e13e 100644 --- a/build.rs +++ b/build.rs @@ -3,14 +3,19 @@ use std::fs::read_dir; use std::path::PathBuf; fn main() { - println!("cargo:rerun-if-changed=tests"); + let manifest = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tests = manifest.join("tests"); + let snapshot = manifest.join("bin").join("snapshot.rs"); + + println!("cargo:rerun-if-changed={}", tests.to_string_lossy()); + println!("cargo:rerun-if-changed={}", snapshot.to_string_lossy()); + println!("cargo:rerun-if-env-changed=BUILD_INTEGRATION_TESTS"); if env::var("BUILD_INTEGRATION_TESTS").unwrap_or_else(|_| "0".to_string()) == "0" { return; } - let manifest = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - let mut entries = read_dir(manifest.join("tests")) + let mut entries = read_dir(tests) .unwrap() .flatten() .map(|entry| entry.path()) @@ -26,6 +31,7 @@ fn main() { for entry in entries { let code_filename = entry.join("code.php"); let ast_filename = entry.join("ast.txt"); + let tokens_filename = entry.join("tokens.txt"); let lexer_error_filename = entry.join("lexer-error.txt"); let parser_error_filename = entry.join("parser-error.txt"); @@ -45,7 +51,12 @@ fn main() { entry.to_string_lossy() ); - content.push_str(&build_success_test(entry, code_filename, ast_filename)) + content.push_str(&build_success_test( + entry, + code_filename, + ast_filename, + tokens_filename, + )) } else if lexer_error_filename.exists() { assert!( !parser_error_filename.exists(), @@ -69,6 +80,7 @@ fn main() { entry, code_filename, parser_error_filename, + tokens_filename, )) } } @@ -77,21 +89,32 @@ fn main() { std::fs::write(dest, content).expect("failed to write to file"); } -fn build_success_test(entry: PathBuf, code_filename: PathBuf, ast_filename: PathBuf) -> String { +fn build_success_test( + entry: PathBuf, + code_filename: PathBuf, + ast_filename: PathBuf, + tokens_filename: PathBuf, +) -> String { format!( r#"#[test] fn test_success_{}() {{ - use php_parser_rs::{{Lexer, Parser}}; + use php_parser_rs::prelude::Parser; + use php_parser_rs::prelude::Lexer; use pretty_assertions::assert_str_eq; let code_filename = "{}"; let ast_filename = "{}"; + let tokens_filename = "{}"; let code = std::fs::read_to_string(&code_filename).unwrap(); let expected_ast = std::fs::read_to_string(&ast_filename).unwrap(); + let expected_tokens = std::fs::read_to_string(&tokens_filename).unwrap(); - let mut lexer = Lexer::new(None); + let mut lexer = Lexer::new(); let tokens = lexer.tokenize(code.as_bytes()).unwrap(); + + assert_str_eq!(expected_tokens.trim(), format!("{{:#?}}", tokens)); + let mut parser = Parser::new(None); let ast = parser.parse(tokens).unwrap(); @@ -101,7 +124,8 @@ fn test_success_{}() {{ "#, entry.file_name().unwrap().to_string_lossy(), code_filename.to_string_lossy(), - ast_filename.to_string_lossy() + ast_filename.to_string_lossy(), + tokens_filename.to_string_lossy(), ) } @@ -113,7 +137,7 @@ fn build_lexer_error_test( format!( r#"#[test] fn test_lexer_error_{}() {{ - use php_parser_rs::Lexer; + use php_parser_rs::prelude::Lexer; use pretty_assertions::assert_str_eq; let code_filename = "{}"; @@ -122,10 +146,13 @@ fn test_lexer_error_{}() {{ let code = std::fs::read_to_string(&code_filename).unwrap(); let expected_error = std::fs::read_to_string(&lexer_error_filename).unwrap(); - let mut lexer = Lexer::new(None); + let mut lexer = Lexer::new(); let error = lexer.tokenize(code.as_bytes()).err().unwrap(); - assert_str_eq!(expected_error.trim(), format!("{{:?}}", error)); + assert_str_eq!( + expected_error.trim(), + format!("{{:?}} -> {{}}", error, error.to_string()) + ); }} "#, @@ -139,22 +166,28 @@ fn build_parser_error_test( entry: PathBuf, code_filename: PathBuf, parser_error_filename: PathBuf, + tokens_filename: PathBuf, ) -> String { format!( r#"#[test] fn test_paser_error_{}() {{ - use php_parser_rs::{{Lexer, Parser}}; + use php_parser_rs::prelude::Parser; + use php_parser_rs::prelude::Lexer; use pretty_assertions::assert_str_eq; let code_filename = "{}"; + let tokens_filename = "{}"; let parser_error_filename = "{}"; let code = std::fs::read_to_string(&code_filename).unwrap(); + let expected_tokens = std::fs::read_to_string(&tokens_filename).unwrap(); let expected_error = std::fs::read_to_string(&parser_error_filename).unwrap(); - let mut lexer = Lexer::new(None); + let mut lexer = Lexer::new(); let tokens = lexer.tokenize(code.as_bytes()).unwrap(); + assert_str_eq!(expected_tokens.trim(), format!("{{:#?}}", tokens)); + let mut parser = Parser::new(None); let error = parser.parse(tokens).err().unwrap(); @@ -167,6 +200,7 @@ fn test_paser_error_{}() {{ "#, entry.file_name().unwrap().to_string_lossy(), code_filename.to_string_lossy(), + tokens_filename.to_string_lossy(), parser_error_filename.to_string_lossy() ) } diff --git a/src/lexer/error.rs b/src/lexer/error.rs new file mode 100644 index 00000000..5576a58b --- /dev/null +++ b/src/lexer/error.rs @@ -0,0 +1,58 @@ +use std::fmt::Display; + +use crate::lexer::token::Span; + +pub type LexResult = Result; + +#[derive(Debug, Eq, PartialEq)] +pub enum SyntaxError { + UnexpectedEndOfFile(Span), + UnexpectedError(Span), + UnexpectedCharacter(u8, Span), + InvalidHaltCompiler(Span), + InvalidOctalEscape(Span), + InvalidOctalLiteral(Span), + InvalidUnicodeEscape(Span), +} + +impl Display for SyntaxError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnexpectedEndOfFile(span) => write!( + f, + "Syntax Error: unexpected end of file on line {} column {}", + span.0, span.1 + ), + Self::UnexpectedError(span) => write!( + f, + "Syntax Error: unexpected error on line {} column {}", + span.0, span.1 + ), + Self::UnexpectedCharacter(char, span) => write!( + f, + "Syntax Error: unexpected character `{:?}` on line {} column {}", + *char as char, span.0, span.1 + ), + Self::InvalidHaltCompiler(span) => write!( + f, + "Syntax Error: invalid halt compiler on line {} column {}", + span.0, span.1 + ), + Self::InvalidOctalEscape(span) => write!( + f, + "Syntax Error: invalid octal escape on line {} column {}", + span.0, span.1 + ), + Self::InvalidOctalLiteral(span) => write!( + f, + "Syntax Error: invalid octal literal on line {} column {}", + span.0, span.1 + ), + Self::InvalidUnicodeEscape(span) => write!( + f, + "Syntax Error: invalid unicode escape on line {} column {}", + span.0, span.1 + ), + } + } +} diff --git a/src/lexer/lexer.rs b/src/lexer/lexer.rs deleted file mode 100644 index 131547cd..00000000 --- a/src/lexer/lexer.rs +++ /dev/null @@ -1,1776 +0,0 @@ -use std::num::IntErrorKind; - -use crate::{ByteString, OpenTagKind, Token, TokenKind}; - -#[derive(Debug, PartialEq, Eq)] -pub enum LexerState { - Initial, - Scripting, - Halted, - DoubleQuote, - LookingForVarname, - LookingForProperty, - VarOffset, -} - -#[allow(dead_code)] -#[derive(Default)] -pub struct LexerConfig { - short_tags: bool, -} - -#[allow(dead_code)] -pub struct Lexer { - config: LexerConfig, - state_stack: Vec, - chars: Vec, - cursor: usize, - current: Option, - col: usize, - line: usize, -} - -// Reusable pattern for the first byte of an identifier. -macro_rules! ident_start { - () => { - b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\x80'..=b'\xff' - }; -} - -// Reusable pattern for identifier after the first byte. -macro_rules! ident { - () => { - b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\x80'..=b'\xff' - }; -} - -impl Lexer { - pub fn new(config: Option) -> Self { - Self { - config: config.unwrap_or_default(), - state_stack: vec![LexerState::Initial], - chars: Vec::new(), - cursor: 0, - current: None, - line: 1, - col: 1, - } - } - - pub fn tokenize>( - &mut self, - input: &B, - ) -> Result, LexerError> { - let mut tokens = Vec::new(); - self.chars = input.as_ref().to_vec(); - - self.current = self.chars.first().copied(); - - while self.current.is_some() { - match self.state_stack.last().unwrap() { - // The "Initial" state is used to parse inline HTML. It is essentially a catch-all - // state that will build up a single token buffer until it encounters an open tag - // of some description. - LexerState::Initial => { - tokens.append(&mut self.initial()?); - } - // The scripting state is entered when an open tag is encountered in the source code. - // This tells the lexer to start analysing characters at PHP tokens instead of inline HTML. - LexerState::Scripting => { - self.skip_whitespace(); - - // If we have consumed whitespace and then reached the end of the file, we should break. - if self.current.is_none() { - break; - } - - tokens.push(self.scripting()?); - } - // The "Halted" state is entered when the `__halt_compiler` token is encountered. - // In this state, all the text that follows is no longer parsed as PHP as is collected - // into a single "InlineHtml" token (kind of cheating, oh well). - LexerState::Halted => { - tokens.push(Token { - kind: TokenKind::InlineHtml(self.chars[self.cursor..].into()), - span: (self.line, self.col), - }); - break; - } - // The double quote state is entered when inside a double-quoted string that - // contains variables. - LexerState::DoubleQuote => tokens.extend(self.double_quote()?), - // LookingForProperty is entered inside double quotes, - // backticks, or a heredoc, expecting a variable name. - // If one isn't found, it switches to scripting. - LexerState::LookingForVarname => { - if let Some(token) = self.looking_for_varname() { - tokens.push(token); - } - } - // LookingForProperty is entered inside double quotes, - // backticks, or a heredoc, expecting an arrow followed by a - // property name. - LexerState::LookingForProperty => { - tokens.push(self.looking_for_property()?); - } - LexerState::VarOffset => { - if self.current.is_none() { - break; - } - - tokens.push(self.var_offset()?); - } - } - } - - Ok(tokens) - } - - fn skip_whitespace(&mut self) { - while let Some(b' ' | b'\n' | b'\r' | b'\t') = self.current { - self.next(); - } - } - - fn initial(&mut self) -> Result, LexerError> { - let inline_span = (self.line, self.col); - let mut buffer = Vec::new(); - while let Some(char) = self.current { - if self.try_read(b" Result { - let span = (self.line, self.col); - let kind = match self.peek_buf() { - [b'@', ..] => { - self.next(); - - TokenKind::At - } - [b'!', b'=', b'=', ..] => { - self.skip(3); - TokenKind::BangDoubleEquals - } - [b'!', b'=', ..] => { - self.skip(2); - TokenKind::BangEquals - } - [b'!', ..] => { - self.next(); - TokenKind::Bang - } - [b'&', b'&', ..] => { - self.skip(2); - TokenKind::BooleanAnd - } - [b'&', b'=', ..] => { - self.skip(2); - TokenKind::AmpersandEquals - } - [b'&', ..] => { - self.next(); - TokenKind::Ampersand - } - [b'?', b'>', ..] => { - // This is a close tag, we can enter "Initial" mode again. - self.skip(2); - - self.enter_state(LexerState::Initial); - - TokenKind::CloseTag - } - [b'?', b'?', b'=', ..] => { - self.skip(3); - TokenKind::CoalesceEqual - } - [b'?', b'?', ..] => { - self.skip(2); - TokenKind::Coalesce - } - [b'?', b':', ..] => { - self.skip(2); - TokenKind::QuestionColon - } - [b'?', b'-', b'>', ..] => { - self.skip(3); - TokenKind::NullsafeArrow - } - [b'?', ..] => { - self.next(); - TokenKind::Question - } - [b'=', b'>', ..] => { - self.skip(2); - TokenKind::DoubleArrow - } - [b'=', b'=', b'=', ..] => { - self.skip(3); - TokenKind::TripleEquals - } - [b'=', b'=', ..] => { - self.skip(2); - TokenKind::DoubleEquals - } - [b'=', ..] => { - self.next(); - TokenKind::Equals - } - // Single quoted string. - [b'\'', ..] => { - self.next(); - self.tokenize_single_quote_string()? - } - [b'b' | b'B', b'\'', ..] => { - self.skip(2); - self.tokenize_single_quote_string()? - } - [b'"', ..] => { - self.next(); - self.tokenize_double_quote_string()? - } - [b'b' | b'B', b'"', ..] => { - self.skip(2); - self.tokenize_double_quote_string()? - } - [b'$', ident_start!(), ..] => { - self.next(); - self.tokenize_variable() - } - [b'$', ..] => { - self.next(); - TokenKind::Dollar - } - [b'.', b'=', ..] => { - self.skip(2); - TokenKind::DotEquals - } - [b'.', b'0'..=b'9', ..] => self.tokenize_number()?, - [b'.', b'.', b'.', ..] => { - self.skip(3); - TokenKind::Ellipsis - } - [b'.', ..] => { - self.next(); - TokenKind::Dot - } - &[b'0'..=b'9', ..] => self.tokenize_number()?, - &[b'\\', ident_start!(), ..] => { - self.next(); - - match self.scripting()? { - Token { - kind: - TokenKind::Identifier(ByteString(mut i)) - | TokenKind::QualifiedIdentifier(ByteString(mut i)), - .. - } => { - i.insert(0, b'\\'); - TokenKind::FullyQualifiedIdentifier(i.into()) - } - s => unreachable!("{:?}", s), - } - } - [b'\\', ..] => { - self.next(); - TokenKind::NamespaceSeparator - } - &[b @ ident_start!(), ..] => { - self.next(); - let mut qualified = false; - let mut last_was_slash = false; - - let mut buffer = vec![b]; - while let Some(next) = self.current { - if next.is_ascii_alphanumeric() || next == b'_' { - buffer.push(next); - self.next(); - last_was_slash = false; - continue; - } - - if next == b'\\' && !last_was_slash { - qualified = true; - last_was_slash = true; - buffer.push(next); - self.next(); - continue; - } - - break; - } - - if qualified { - TokenKind::QualifiedIdentifier(buffer.into()) - } else { - let kind = identifier_to_keyword(&buffer) - .unwrap_or_else(|| TokenKind::Identifier(buffer.into())); - - if kind == TokenKind::HaltCompiler { - match self.peek_buf() { - [b'(', b')', b';', ..] => { - self.skip(3); - self.enter_state(LexerState::Halted); - } - _ => return Err(LexerError::InvalidHaltCompiler), - } - } - - kind - } - } - [b'/', b'*', ..] => { - self.next(); - let mut buffer = vec![b'/']; - - while self.current.is_some() { - match self.peek_buf() { - [b'*', b'/', ..] => { - self.skip(2); - buffer.extend_from_slice(b"*/"); - break; - } - &[t, ..] => { - self.next(); - buffer.push(t); - } - [] => unreachable!(), - } - } - self.next(); - - if buffer.starts_with(b"/**") { - TokenKind::DocComment(buffer.into()) - } else { - TokenKind::Comment(buffer.into()) - } - } - [b'#', b'[', ..] => { - self.skip(2); - TokenKind::Attribute - } - &[ch @ b'/', b'/', ..] | &[ch @ b'#', ..] => { - let mut buffer = if ch == b'/' { - self.skip(2); - b"//".to_vec() - } else { - self.next(); - b"#".to_vec() - }; - - while let Some(c) = self.current { - if c == b'\n' { - break; - } - - buffer.push(c); - self.next(); - } - - self.next(); - - TokenKind::Comment(buffer.into()) - } - [b'/', b'=', ..] => { - self.skip(2); - TokenKind::SlashEquals - } - [b'/', ..] => { - self.next(); - TokenKind::Slash - } - [b'*', b'*', b'=', ..] => { - self.skip(3); - TokenKind::PowEquals - } - [b'*', b'*', ..] => { - self.skip(2); - TokenKind::Pow - } - [b'*', b'=', ..] => { - self.skip(2); - TokenKind::AsteriskEqual - } - [b'*', ..] => { - self.next(); - TokenKind::Asterisk - } - [b'|', b'|', ..] => { - self.skip(2); - TokenKind::Pipe - } - [b'|', b'=', ..] => { - self.skip(2); - TokenKind::PipeEquals - } - [b'|', ..] => { - self.next(); - TokenKind::Pipe - } - [b'^', b'=', ..] => { - self.skip(2); - TokenKind::CaretEquals - } - [b'^', ..] => { - self.next(); - TokenKind::Caret - } - [b'{', ..] => { - self.next(); - self.push_state(LexerState::Scripting); - TokenKind::LeftBrace - } - [b'}', ..] => { - self.next(); - self.pop_state(); - TokenKind::RightBrace - } - [b'(', ..] => { - self.next(); - - if self.try_read(b"int)") { - self.skip(4); - TokenKind::IntCast - } else if self.try_read(b"integer)") { - self.skip(8); - TokenKind::IntegerCast - } else if self.try_read(b"bool)") { - self.skip(5); - TokenKind::BoolCast - } else if self.try_read(b"boolean)") { - self.skip(8); - TokenKind::BooleanCast - } else if self.try_read(b"float)") { - self.skip(6); - TokenKind::FloatCast - } else if self.try_read(b"double)") { - self.skip(7); - TokenKind::DoubleCast - } else if self.try_read(b"real)") { - self.skip(5); - TokenKind::RealCast - } else if self.try_read(b"string)") { - self.skip(7); - TokenKind::StringCast - } else if self.try_read(b"binary)") { - self.skip(7); - TokenKind::BinaryCast - } else if self.try_read(b"array)") { - self.skip(6); - TokenKind::ArrayCast - } else if self.try_read(b"object)") { - self.skip(7); - TokenKind::ObjectCast - } else if self.try_read(b"unset)") { - self.skip(6); - TokenKind::UnsetCast - } else { - TokenKind::LeftParen - } - } - [b')', ..] => { - self.next(); - TokenKind::RightParen - } - [b';', ..] => { - self.next(); - TokenKind::SemiColon - } - [b'+', b'+', ..] => { - self.skip(2); - TokenKind::Increment - } - [b'+', b'=', ..] => { - self.skip(2); - TokenKind::PlusEquals - } - [b'+', ..] => { - self.next(); - TokenKind::Plus - } - [b'%', b'=', ..] => { - self.skip(2); - TokenKind::PercentEquals - } - [b'%', ..] => { - self.next(); - TokenKind::Percent - } - [b'-', b'-', ..] => { - self.skip(2); - TokenKind::Decrement - } - [b'-', b'>', ..] => { - self.skip(2); - TokenKind::Arrow - } - [b'-', b'=', ..] => { - self.skip(2); - TokenKind::MinusEquals - } - [b'-', ..] => { - self.next(); - TokenKind::Minus - } - [b'<', b'<', b'<', ..] => { - // TODO: Handle both heredocs and nowdocs. - self.skip(3); - - todo!("heredocs & nowdocs"); - } - [b'<', b'<', b'=', ..] => { - self.skip(3); - - TokenKind::LeftShiftEquals - } - [b'<', b'<', ..] => { - self.skip(2); - TokenKind::LeftShift - } - [b'<', b'=', b'>', ..] => { - self.skip(3); - TokenKind::Spaceship - } - [b'<', b'=', ..] => { - self.skip(2); - TokenKind::LessThanEquals - } - [b'<', b'>', ..] => { - self.skip(2); - TokenKind::AngledLeftRight - } - [b'<', ..] => { - self.next(); - TokenKind::LessThan - } - [b'>', b'>', b'=', ..] => { - self.skip(3); - TokenKind::RightShiftEquals - } - [b'>', b'>', ..] => { - self.skip(2); - TokenKind::RightShift - } - [b'>', b'=', ..] => { - self.skip(2); - TokenKind::GreaterThanEquals - } - [b'>', ..] => { - self.next(); - TokenKind::GreaterThan - } - [b',', ..] => { - self.next(); - TokenKind::Comma - } - [b'[', ..] => { - self.next(); - TokenKind::LeftBracket - } - [b']', ..] => { - self.next(); - TokenKind::RightBracket - } - [b':', b':', ..] => { - self.skip(2); - TokenKind::DoubleColon - } - [b':', ..] => { - self.next(); - TokenKind::Colon - } - &[b'~', ..] => { - self.next(); - TokenKind::BitwiseNot - } - &[b, ..] => unimplemented!( - " char: {}, line: {}, col: {}", - b as char, - self.line, - self.col - ), - // We should never reach this point since we have the empty checks surrounding - // the call to this function, but it's better to be safe than sorry. - [] => return Err(LexerError::UnexpectedEndOfFile), - }; - - Ok(Token { kind, span }) - } - - fn double_quote(&mut self) -> Result, LexerError> { - let span = (self.line, self.col); - let mut buffer = Vec::new(); - let kind = loop { - match self.peek_buf() { - [b'$', b'{', ..] => { - self.skip(2); - self.push_state(LexerState::LookingForVarname); - break TokenKind::DollarLeftBrace; - } - [b'{', b'$', ..] => { - // Intentionally only consume the left brace. - self.next(); - self.push_state(LexerState::Scripting); - break TokenKind::LeftBrace; - } - [b'"', ..] => { - self.next(); - self.enter_state(LexerState::Scripting); - break TokenKind::DoubleQuote; - } - [b'$', ident_start!(), ..] => { - self.next(); - let ident = self.consume_identifier(); - - match self.peek_buf() { - [b'[', ..] => self.push_state(LexerState::VarOffset), - [b'-', b'>', ident_start!(), ..] - | [b'?', b'-', b'>', ident_start!(), ..] => { - self.push_state(LexerState::LookingForProperty) - } - _ => {} - } - - break TokenKind::Variable(ident.into()); - } - &[b, ..] => { - self.next(); - buffer.push(b); - } - [] => return Err(LexerError::UnexpectedEndOfFile), - } - }; - - let mut tokens = Vec::new(); - if !buffer.is_empty() { - tokens.push(Token { - kind: TokenKind::StringPart(buffer.into()), - span, - }) - } - - tokens.push(Token { kind, span }); - Ok(tokens) - } - - fn looking_for_varname(&mut self) -> Option { - if let Some(ident) = self.peek_identifier() { - if let Some(b'[' | b'}') = self.peek_byte(ident.len()) { - let ident = ident.to_vec(); - let span = (self.line, self.col); - self.skip(ident.len()); - self.enter_state(LexerState::Scripting); - return Some(Token { - kind: TokenKind::Identifier(ident.into()), - span, - }); - } - } - - self.enter_state(LexerState::Scripting); - None - } - - fn looking_for_property(&mut self) -> Result { - let span = (self.line, self.col); - let kind = match self.peek_buf() { - [b'-', b'>', ..] => { - self.skip(2); - TokenKind::Arrow - } - [b'?', b'-', b'>', ..] => { - self.skip(3); - TokenKind::NullsafeArrow - } - &[ident_start!(), ..] => { - let buffer = self.consume_identifier(); - self.pop_state(); - TokenKind::Identifier(buffer.into()) - } - // Should be impossible as we already looked ahead this far inside double_quote. - _ => unreachable!(), - }; - Ok(Token { kind, span }) - } - - fn var_offset(&mut self) -> Result { - let span = (self.line, self.col); - let kind = match self.peek_buf() { - [b'$', ident_start!(), ..] => { - self.next(); - self.tokenize_variable() - } - &[b'0'..=b'9', ..] => { - // TODO: all integer literals are allowed, but only decimal integers with no underscores - // are actually treated as numbers. Others are treated as strings. - // Float literals are not allowed, but that could be handled in the parser. - self.tokenize_number()? - } - [b'[', ..] => { - self.next(); - TokenKind::LeftBracket - } - [b'-', ..] => { - self.next(); - TokenKind::Minus - } - [b']', ..] => { - self.next(); - self.pop_state(); - TokenKind::RightBracket - } - &[ident_start!(), ..] => { - let label = self.consume_identifier(); - TokenKind::Identifier(label.into()) - } - &[b, ..] => unimplemented!( - " char: {}, line: {}, col: {}", - b as char, - self.line, - self.col - ), - [] => return Err(LexerError::UnexpectedEndOfFile), - }; - Ok(Token { kind, span }) - } - - fn tokenize_single_quote_string(&mut self) -> Result { - let mut buffer = Vec::new(); - - loop { - match self.peek_buf() { - [b'\'', ..] => { - self.next(); - break; - } - &[b'\\', b @ b'\'' | b @ b'\\', ..] => { - self.skip(2); - buffer.push(b); - } - &[b, ..] => { - self.next(); - buffer.push(b); - } - [] => return Err(LexerError::UnexpectedEndOfFile), - } - } - - Ok(TokenKind::LiteralString(buffer.into())) - } - - fn tokenize_double_quote_string(&mut self) -> Result { - let mut buffer = Vec::new(); - - let constant = loop { - match self.peek_buf() { - [b'"', ..] => { - self.next(); - break true; - } - &[b'\\', b @ (b'"' | b'\\' | b'$'), ..] => { - self.skip(2); - buffer.push(b); - } - &[b'\\', b'n', ..] => { - self.skip(2); - buffer.push(b'\n'); - } - &[b'\\', b'r', ..] => { - self.skip(2); - buffer.push(b'\r'); - } - &[b'\\', b't', ..] => { - self.skip(2); - buffer.push(b'\t'); - } - &[b'\\', b'v', ..] => { - self.skip(2); - buffer.push(b'\x0b'); - } - &[b'\\', b'e', ..] => { - self.skip(2); - buffer.push(b'\x1b'); - } - &[b'\\', b'f', ..] => { - self.skip(2); - buffer.push(b'\x0c'); - } - &[b'\\', b'x', b @ (b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'), ..] => { - self.skip(3); - - let mut hex = String::from(b as char); - if let Some(b @ (b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F')) = self.current { - self.next(); - hex.push(b as char); - } - - let b = u8::from_str_radix(&hex, 16).unwrap(); - buffer.push(b); - } - &[b'\\', b'u', b'{', ..] => { - self.skip(3); - - let mut code_point = String::new(); - while let Some(b @ (b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F')) = self.current { - self.next(); - code_point.push(b as char); - } - - if code_point.is_empty() || self.current != Some(b'}') { - return Err(LexerError::InvalidUnicodeEscape); - } - self.next(); - - let c = if let Ok(c) = u32::from_str_radix(&code_point, 16) { - c - } else { - return Err(LexerError::InvalidUnicodeEscape); - }; - - if let Some(c) = char::from_u32(c) { - let mut tmp = [0; 4]; - let bytes = c.encode_utf8(&mut tmp); - buffer.extend(bytes.as_bytes()); - } else { - return Err(LexerError::InvalidUnicodeEscape); - } - } - &[b'\\', b @ b'0'..=b'7', ..] => { - self.skip(2); - - let mut octal = String::from(b as char); - if let Some(b @ b'0'..=b'7') = self.current { - self.next(); - octal.push(b as char); - } - if let Some(b @ b'0'..=b'7') = self.current { - self.next(); - octal.push(b as char); - } - - if let Ok(b) = u8::from_str_radix(&octal, 8) { - buffer.push(b); - } else { - return Err(LexerError::InvalidOctalEscape); - } - } - [b'$', ident_start!(), ..] | [b'{', b'$', ..] | [b'$', b'{', ..] => { - break false; - } - &[b, ..] => { - self.next(); - buffer.push(b); - } - [] => return Err(LexerError::UnexpectedEndOfFile), - } - }; - - Ok(if constant { - TokenKind::LiteralString(buffer.into()) - } else { - self.enter_state(LexerState::DoubleQuote); - TokenKind::StringPart(buffer.into()) - }) - } - - fn peek_identifier(&self) -> Option<&[u8]> { - let mut cursor = self.cursor; - if let Some(ident_start!()) = self.chars.get(cursor) { - cursor += 1; - while let Some(ident!()) = self.chars.get(cursor) { - cursor += 1; - } - Some(&self.chars[self.cursor..cursor]) - } else { - None - } - } - - fn consume_identifier(&mut self) -> Vec { - let ident = self.peek_identifier().unwrap().to_vec(); - self.skip(ident.len()); - - ident - } - - fn tokenize_variable(&mut self) -> TokenKind { - TokenKind::Variable(self.consume_identifier().into()) - } - - fn tokenize_number(&mut self) -> Result { - let mut buffer = String::new(); - - let (base, kind) = match self.peek_buf() { - [b'0', b'B' | b'b', ..] => { - self.skip(2); - (2, NumberKind::Int) - } - [b'0', b'O' | b'o', ..] => { - self.skip(2); - (8, NumberKind::Int) - } - [b'0', b'X' | b'x', ..] => { - self.skip(2); - (16, NumberKind::Int) - } - [b'0', ..] => (10, NumberKind::OctalOrFloat), - [b'.', ..] => (10, NumberKind::Float), - _ => (10, NumberKind::IntOrFloat), - }; - - if kind != NumberKind::Float { - self.read_digits(&mut buffer, base); - if kind == NumberKind::Int { - return parse_int(&buffer, base as u32); - } - } - - // Remaining cases: decimal integer, legacy octal integer, or float. - let is_float = matches!( - self.peek_buf(), - [b'.', ..] - | [b'e' | b'E', b'-' | b'+', b'0'..=b'9', ..] - | [b'e' | b'E', b'0'..=b'9', ..] - ); - if !is_float { - let base = if kind == NumberKind::OctalOrFloat { - 8 - } else { - 10 - }; - return parse_int(&buffer, base as u32); - } - - if self.current == Some(b'.') { - buffer.push('.'); - self.next(); - self.read_digits(&mut buffer, 10); - } - - if let Some(b'e' | b'E') = self.current { - buffer.push('e'); - self.next(); - if let Some(b @ (b'-' | b'+')) = self.current { - buffer.push(b as char); - self.next(); - } - self.read_digits(&mut buffer, 10); - } - - Ok(TokenKind::LiteralFloat(buffer.parse().unwrap())) - } - - fn read_digits(&mut self, buffer: &mut String, base: usize) { - if base == 16 { - self.read_digits_fn(buffer, u8::is_ascii_hexdigit); - } else { - let max = b'0' + base as u8; - self.read_digits_fn(buffer, |b| (b'0'..max).contains(b)); - }; - } - - fn read_digits_fn bool>(&mut self, buffer: &mut String, is_digit: F) { - if let Some(b) = self.current { - if is_digit(&b) { - self.next(); - buffer.push(b as char); - } else { - return; - } - } - loop { - match *self.peek_buf() { - [b, ..] if is_digit(&b) => { - self.next(); - buffer.push(b as char); - } - [b'_', b, ..] if is_digit(&b) => { - self.next(); - self.next(); - buffer.push(b as char); - } - _ => { - break; - } - } - } - } - - fn enter_state(&mut self, state: LexerState) { - *self.state_stack.last_mut().unwrap() = state; - } - - fn push_state(&mut self, state: LexerState) { - self.state_stack.push(state); - } - - fn pop_state(&mut self) { - self.state_stack.pop(); - } - - fn peek_buf(&self) -> &[u8] { - &self.chars[self.cursor..] - } - - fn peek_byte(&self, delta: usize) -> Option { - self.chars.get(self.cursor + delta).copied() - } - - fn try_read(&self, search: &'static [u8]) -> bool { - self.peek_buf().starts_with(search) - } - - fn skip(&mut self, count: usize) { - for _ in 0..count { - self.next(); - } - } - - fn next(&mut self) { - match self.current { - Some(b'\n') => { - self.line += 1; - self.col = 1; - } - Some(_) => self.col += 1, - _ => {} - } - self.cursor += 1; - self.current = self.chars.get(self.cursor).copied(); - } -} - -// Parses an integer literal in the given base and converts errors to LexerError. -// It returns a float token instead on overflow. -fn parse_int(buffer: &str, base: u32) -> Result { - match i64::from_str_radix(buffer, base) { - Ok(i) => Ok(TokenKind::LiteralInteger(i)), - Err(err) if err.kind() == &IntErrorKind::InvalidDigit => { - // The InvalidDigit error is only possible for legacy octal literals. - Err(LexerError::InvalidOctalLiteral) - } - Err(err) if err.kind() == &IntErrorKind::PosOverflow => { - // Parse as i128 so we can handle other bases. - // This means there's an upper limit on how large the literal can be. - let i = i128::from_str_radix(buffer, base).unwrap(); - Ok(TokenKind::LiteralFloat(i as f64)) - } - _ => Err(LexerError::UnexpectedError), - } -} - -fn identifier_to_keyword(ident: &[u8]) -> Option { - Some(match ident { - b"enddeclare" => TokenKind::EndDeclare, - b"endswitch" => TokenKind::EndSwitch, - b"endfor" => TokenKind::EndFor, - b"endwhile" => TokenKind::EndWhile, - b"endforeach" => TokenKind::EndForeach, - b"endif" => TokenKind::EndIf, - b"from" => TokenKind::From, - b"and" => TokenKind::LogicalAnd, - b"or" => TokenKind::LogicalOr, - b"xor" => TokenKind::LogicalXor, - b"print" => TokenKind::Print, - b"__halt_compiler" | b"__HALT_COMPILER" => TokenKind::HaltCompiler, - b"readonly" => TokenKind::Readonly, - b"global" => TokenKind::Global, - b"match" => TokenKind::Match, - b"abstract" => TokenKind::Abstract, - b"array" => TokenKind::Array, - b"as" => TokenKind::As, - b"break" => TokenKind::Break, - b"case" => TokenKind::Case, - b"catch" => TokenKind::Catch, - b"class" => TokenKind::Class, - b"clone" => TokenKind::Clone, - b"continue" => TokenKind::Continue, - b"const" => TokenKind::Const, - b"declare" => TokenKind::Declare, - b"default" => TokenKind::Default, - b"do" => TokenKind::Do, - b"echo" => TokenKind::Echo, - b"else" => TokenKind::Else, - b"elseif" => TokenKind::ElseIf, - b"enum" => TokenKind::Enum, - b"extends" => TokenKind::Extends, - b"false" | b"FALSE" => TokenKind::False, - b"final" => TokenKind::Final, - b"finally" => TokenKind::Finally, - b"fn" => TokenKind::Fn, - b"for" => TokenKind::For, - b"foreach" => TokenKind::Foreach, - b"function" => TokenKind::Function, - b"goto" => TokenKind::Goto, - b"if" => TokenKind::If, - b"include" => TokenKind::Include, - b"include_once" => TokenKind::IncludeOnce, - b"implements" => TokenKind::Implements, - b"interface" => TokenKind::Interface, - b"instanceof" => TokenKind::Instanceof, - b"namespace" => TokenKind::Namespace, - b"new" => TokenKind::New, - b"null" | b"NULL" => TokenKind::Null, - b"private" => TokenKind::Private, - b"protected" => TokenKind::Protected, - b"public" => TokenKind::Public, - b"require" => TokenKind::Require, - b"require_once" => TokenKind::RequireOnce, - b"return" => TokenKind::Return, - b"static" => TokenKind::Static, - b"switch" => TokenKind::Switch, - b"throw" => TokenKind::Throw, - b"trait" => TokenKind::Trait, - b"true" | b"TRUE" => TokenKind::True, - b"try" => TokenKind::Try, - b"use" => TokenKind::Use, - b"var" => TokenKind::Var, - b"yield" => TokenKind::Yield, - b"__DIR__" => TokenKind::DirConstant, - b"while" => TokenKind::While, - b"insteadof" => TokenKind::Insteadof, - _ => return None, - }) -} - -#[derive(Debug, Eq, PartialEq)] -enum NumberKind { - Int, - Float, - IntOrFloat, - OctalOrFloat, -} - -#[derive(Debug, Eq, PartialEq)] -pub enum LexerError { - UnexpectedEndOfFile, - UnexpectedError, - UnexpectedCharacter(u8), - InvalidHaltCompiler, - InvalidOctalEscape, - InvalidOctalLiteral, - InvalidUnicodeEscape, -} - -#[cfg(test)] -mod tests { - use super::Lexer; - use crate::{ByteString, LexerError, OpenTagKind, Token, TokenKind}; - - macro_rules! open { - () => { - TokenKind::OpenTag(OpenTagKind::Full) - }; - ($kind:expr) => { - TokenKind::OpenTag($kind) - }; - } - macro_rules! int { - ($i:expr) => { - TokenKind::LiteralInteger($i) - }; - } - - fn var>(v: B) -> TokenKind { - TokenKind::Variable(v.into()) - } - - #[test] - fn basic_tokens() { - assert_tokens("", &[open!(), TokenKind::CloseTag]); - } - - #[test] - fn close_tag_followed_by_content() { - assert_tokens( - " ", - &[ - open!(), - TokenKind::CloseTag, - TokenKind::InlineHtml(" ".into()), - ], - ); - } - - #[test] - fn inline_html() { - assert_tokens( - "Hello, world!\nb}" "#, - &[ - open!(), - TokenKind::StringPart("".into()), - TokenKind::LeftBrace, - TokenKind::Variable("a".into()), - TokenKind::Arrow, - TokenKind::Identifier("b".into()), - TokenKind::RightBrace, - TokenKind::DoubleQuote, - ], - ); - assert_tokens( - r#"b" "#, - &[ - open!(), - TokenKind::StringPart("".into()), - TokenKind::Variable("a".into()), - TokenKind::Arrow, - TokenKind::Identifier("b".into()), - TokenKind::DoubleQuote, - ], - ); - assert_tokens( - r#"" "#, - &[ - open!(), - TokenKind::StringPart("".into()), - TokenKind::Variable("a".into()), - TokenKind::StringPart("->".into()), - TokenKind::DoubleQuote, - ], - ); - assert_tokens( - r#"b" "#, - &[ - open!(), - TokenKind::StringPart("".into()), - TokenKind::Variable("a".into()), - TokenKind::NullsafeArrow, - TokenKind::Identifier("b".into()), - TokenKind::DoubleQuote, - ], - ); - assert_tokens( - r#"" "#, - &[ - open!(), - TokenKind::StringPart("".into()), - TokenKind::Variable("a".into()), - TokenKind::StringPart("?->".into()), - TokenKind::DoubleQuote, - ], - ); - assert_tokens( - r#" $", - &[open!(), TokenKind::Arrow, TokenKind::Dollar], - ); - } - - #[test] - fn math() { - assert_tokens( - ">(source: &B, expected: LexerError) { - let mut lexer = Lexer::new(None); - assert_eq!(lexer.tokenize(source), Err(expected)); - } - - fn assert_tokens>(source: &B, expected: &[TokenKind]) { - let mut kinds = vec![]; - - for token in get_tokens(source) { - kinds.push(token.kind); - } - - assert_eq!(kinds, expected); - } - - fn get_spans(source: &str) -> Vec<(usize, usize)> { - let tokens = get_tokens(source); - let mut spans = vec![]; - - for token in tokens { - spans.push(token.span); - } - - spans - } - - fn get_tokens>(source: &B) -> Vec { - let mut lexer = Lexer::new(None); - lexer.tokenize(source).unwrap() - } -} diff --git a/src/lexer/macros.rs b/src/lexer/macros.rs new file mode 100644 index 00000000..d8d2eb15 --- /dev/null +++ b/src/lexer/macros.rs @@ -0,0 +1,15 @@ +// Reusable pattern for the first byte of an identifier. +#[macro_export] +macro_rules! ident_start { + () => { + b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\x80'..=b'\xff' + }; +} + +// Reusable pattern for identifier after the first byte. +#[macro_export] +macro_rules! ident { + () => { + b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\x80'..=b'\xff' + }; +} diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index cea155ad..46dc993b 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -1,9 +1,1165 @@ -#![allow(clippy::module_inception)] +pub mod byte_string; +pub mod error; +mod macros; +pub mod token; -mod byte_string; -mod lexer; -mod token; +use std::num::IntErrorKind; -pub use byte_string::ByteString; -pub use lexer::{Lexer, LexerError}; -pub use token::{OpenTagKind, Span, Token, TokenKind}; +use crate::lexer::byte_string::ByteString; +use crate::lexer::error::SyntaxError; +use crate::lexer::token::OpenTagKind; +use crate::lexer::token::Span; +use crate::lexer::token::Token; +use crate::lexer::token::TokenKind; + +use crate::ident; +use crate::ident_start; + +#[derive(Debug, PartialEq, Eq)] +pub enum LexerState { + Initial, + Scripting, + Halted, + DoubleQuote, + LookingForVarname, + LookingForProperty, + VarOffset, +} + +pub struct Lexer { + state_stack: Vec, + chars: Vec, + cursor: usize, + current: Option, + span: Span, +} + +impl Default for Lexer { + fn default() -> Self { + Self::new() + } +} + +impl Lexer { + pub fn new() -> Self { + Self { + state_stack: vec![LexerState::Initial], + chars: Vec::new(), + cursor: 0, + current: None, + span: (1, 1), + } + } + + pub fn tokenize>( + &mut self, + input: &B, + ) -> Result, SyntaxError> { + let mut tokens = Vec::new(); + self.chars = input.as_ref().to_vec(); + + self.current = self.chars.first().copied(); + + while self.current.is_some() { + match self.state_stack.last().unwrap() { + // The "Initial" state is used to parse inline HTML. It is essentially a catch-all + // state that will build up a single token buffer until it encounters an open tag + // of some description. + LexerState::Initial => { + tokens.append(&mut self.initial()?); + } + // The scripting state is entered when an open tag is encountered in the source code. + // This tells the lexer to start analysing characters at PHP tokens instead of inline HTML. + LexerState::Scripting => { + self.skip_whitespace(); + + // If we have consumed whitespace and then reached the end of the file, we should break. + if self.current.is_none() { + break; + } + + tokens.push(self.scripting()?); + } + // The "Halted" state is entered when the `__halt_compiler` token is encountered. + // In this state, all the text that follows is no longer parsed as PHP as is collected + // into a single "InlineHtml" token (kind of cheating, oh well). + LexerState::Halted => { + tokens.push(Token { + kind: TokenKind::InlineHtml(self.chars[self.cursor..].into()), + span: self.span, + }); + break; + } + // The double quote state is entered when inside a double-quoted string that + // contains variables. + LexerState::DoubleQuote => tokens.extend(self.double_quote()?), + // LookingForProperty is entered inside double quotes, + // backticks, or a heredoc, expecting a variable name. + // If one isn't found, it switches to scripting. + LexerState::LookingForVarname => { + if let Some(token) = self.looking_for_varname() { + tokens.push(token); + } + } + // LookingForProperty is entered inside double quotes, + // backticks, or a heredoc, expecting an arrow followed by a + // property name. + LexerState::LookingForProperty => { + tokens.push(self.looking_for_property()?); + } + LexerState::VarOffset => { + if self.current.is_none() { + break; + } + + tokens.push(self.var_offset()?); + } + } + } + + Ok(tokens) + } + + fn skip_whitespace(&mut self) { + while let Some(b' ' | b'\n' | b'\r' | b'\t') = self.current { + self.next(); + } + } + + fn initial(&mut self) -> Result, SyntaxError> { + let inline_span = self.span; + let mut buffer = Vec::new(); + while let Some(char) = self.current { + if self.try_read(b" Result { + let span = self.span; + let kind = match self.peek_buf() { + [b'@', ..] => { + self.next(); + + TokenKind::At + } + [b'!', b'=', b'=', ..] => { + self.skip(3); + TokenKind::BangDoubleEquals + } + [b'!', b'=', ..] => { + self.skip(2); + TokenKind::BangEquals + } + [b'!', ..] => { + self.next(); + TokenKind::Bang + } + [b'&', b'&', ..] => { + self.skip(2); + TokenKind::BooleanAnd + } + [b'&', b'=', ..] => { + self.skip(2); + TokenKind::AmpersandEquals + } + [b'&', ..] => { + self.next(); + TokenKind::Ampersand + } + [b'?', b'>', ..] => { + // This is a close tag, we can enter "Initial" mode again. + self.skip(2); + + self.enter_state(LexerState::Initial); + + TokenKind::CloseTag + } + [b'?', b'?', b'=', ..] => { + self.skip(3); + TokenKind::CoalesceEqual + } + [b'?', b'?', ..] => { + self.skip(2); + TokenKind::Coalesce + } + [b'?', b':', ..] => { + self.skip(2); + TokenKind::QuestionColon + } + [b'?', b'-', b'>', ..] => { + self.skip(3); + TokenKind::NullsafeArrow + } + [b'?', ..] => { + self.next(); + TokenKind::Question + } + [b'=', b'>', ..] => { + self.skip(2); + TokenKind::DoubleArrow + } + [b'=', b'=', b'=', ..] => { + self.skip(3); + TokenKind::TripleEquals + } + [b'=', b'=', ..] => { + self.skip(2); + TokenKind::DoubleEquals + } + [b'=', ..] => { + self.next(); + TokenKind::Equals + } + // Single quoted string. + [b'\'', ..] => { + self.next(); + self.tokenize_single_quote_string()? + } + [b'b' | b'B', b'\'', ..] => { + self.skip(2); + self.tokenize_single_quote_string()? + } + [b'"', ..] => { + self.next(); + self.tokenize_double_quote_string()? + } + [b'b' | b'B', b'"', ..] => { + self.skip(2); + self.tokenize_double_quote_string()? + } + [b'$', ident_start!(), ..] => { + self.next(); + self.tokenize_variable() + } + [b'$', ..] => { + self.next(); + TokenKind::Dollar + } + [b'.', b'=', ..] => { + self.skip(2); + TokenKind::DotEquals + } + [b'.', b'0'..=b'9', ..] => self.tokenize_number()?, + [b'.', b'.', b'.', ..] => { + self.skip(3); + TokenKind::Ellipsis + } + [b'.', ..] => { + self.next(); + TokenKind::Dot + } + &[b'0'..=b'9', ..] => self.tokenize_number()?, + &[b'\\', ident_start!(), ..] => { + self.next(); + + match self.scripting()? { + Token { + kind: + TokenKind::Identifier(ByteString(mut i)) + | TokenKind::QualifiedIdentifier(ByteString(mut i)), + .. + } => { + i.insert(0, b'\\'); + TokenKind::FullyQualifiedIdentifier(i.into()) + } + s => unreachable!("{:?}", s), + } + } + [b'\\', ..] => { + self.next(); + TokenKind::NamespaceSeparator + } + &[b @ ident_start!(), ..] => { + self.next(); + let mut qualified = false; + let mut last_was_slash = false; + + let mut buffer = vec![b]; + while let Some(next) = self.current { + if next.is_ascii_alphanumeric() || next == b'_' { + buffer.push(next); + self.next(); + last_was_slash = false; + continue; + } + + if next == b'\\' && !last_was_slash { + qualified = true; + last_was_slash = true; + buffer.push(next); + self.next(); + continue; + } + + break; + } + + if qualified { + TokenKind::QualifiedIdentifier(buffer.into()) + } else { + let kind = identifier_to_keyword(&buffer) + .unwrap_or_else(|| TokenKind::Identifier(buffer.into())); + + if kind == TokenKind::HaltCompiler { + match self.peek_buf() { + [b'(', b')', b';', ..] => { + self.skip(3); + self.enter_state(LexerState::Halted); + } + _ => return Err(SyntaxError::InvalidHaltCompiler(self.span)), + } + } + + kind + } + } + [b'/', b'*', ..] => { + self.next(); + let mut buffer = vec![b'/']; + + while self.current.is_some() { + match self.peek_buf() { + [b'*', b'/', ..] => { + self.skip(2); + buffer.extend_from_slice(b"*/"); + break; + } + &[t, ..] => { + self.next(); + buffer.push(t); + } + [] => unreachable!(), + } + } + self.next(); + + if buffer.starts_with(b"/**") { + TokenKind::DocComment(buffer.into()) + } else { + TokenKind::Comment(buffer.into()) + } + } + [b'#', b'[', ..] => { + self.skip(2); + TokenKind::Attribute + } + &[ch @ b'/', b'/', ..] | &[ch @ b'#', ..] => { + let mut buffer = if ch == b'/' { + self.skip(2); + b"//".to_vec() + } else { + self.next(); + b"#".to_vec() + }; + + while let Some(c) = self.current { + if c == b'\n' { + break; + } + + buffer.push(c); + self.next(); + } + + self.next(); + + TokenKind::Comment(buffer.into()) + } + [b'/', b'=', ..] => { + self.skip(2); + TokenKind::SlashEquals + } + [b'/', ..] => { + self.next(); + TokenKind::Slash + } + [b'*', b'*', b'=', ..] => { + self.skip(3); + TokenKind::PowEquals + } + [b'*', b'*', ..] => { + self.skip(2); + TokenKind::Pow + } + [b'*', b'=', ..] => { + self.skip(2); + TokenKind::AsteriskEqual + } + [b'*', ..] => { + self.next(); + TokenKind::Asterisk + } + [b'|', b'|', ..] => { + self.skip(2); + TokenKind::Pipe + } + [b'|', b'=', ..] => { + self.skip(2); + TokenKind::PipeEquals + } + [b'|', ..] => { + self.next(); + TokenKind::Pipe + } + [b'^', b'=', ..] => { + self.skip(2); + TokenKind::CaretEquals + } + [b'^', ..] => { + self.next(); + TokenKind::Caret + } + [b'{', ..] => { + self.next(); + self.push_state(LexerState::Scripting); + TokenKind::LeftBrace + } + [b'}', ..] => { + self.next(); + self.pop_state(); + TokenKind::RightBrace + } + [b'(', ..] => { + self.next(); + + if self.try_read(b"int)") { + self.skip(4); + TokenKind::IntCast + } else if self.try_read(b"integer)") { + self.skip(8); + TokenKind::IntegerCast + } else if self.try_read(b"bool)") { + self.skip(5); + TokenKind::BoolCast + } else if self.try_read(b"boolean)") { + self.skip(8); + TokenKind::BooleanCast + } else if self.try_read(b"float)") { + self.skip(6); + TokenKind::FloatCast + } else if self.try_read(b"double)") { + self.skip(7); + TokenKind::DoubleCast + } else if self.try_read(b"real)") { + self.skip(5); + TokenKind::RealCast + } else if self.try_read(b"string)") { + self.skip(7); + TokenKind::StringCast + } else if self.try_read(b"binary)") { + self.skip(7); + TokenKind::BinaryCast + } else if self.try_read(b"array)") { + self.skip(6); + TokenKind::ArrayCast + } else if self.try_read(b"object)") { + self.skip(7); + TokenKind::ObjectCast + } else if self.try_read(b"unset)") { + self.skip(6); + TokenKind::UnsetCast + } else { + TokenKind::LeftParen + } + } + [b')', ..] => { + self.next(); + TokenKind::RightParen + } + [b';', ..] => { + self.next(); + TokenKind::SemiColon + } + [b'+', b'+', ..] => { + self.skip(2); + TokenKind::Increment + } + [b'+', b'=', ..] => { + self.skip(2); + TokenKind::PlusEquals + } + [b'+', ..] => { + self.next(); + TokenKind::Plus + } + [b'%', b'=', ..] => { + self.skip(2); + TokenKind::PercentEquals + } + [b'%', ..] => { + self.next(); + TokenKind::Percent + } + [b'-', b'-', ..] => { + self.skip(2); + TokenKind::Decrement + } + [b'-', b'>', ..] => { + self.skip(2); + TokenKind::Arrow + } + [b'-', b'=', ..] => { + self.skip(2); + TokenKind::MinusEquals + } + [b'-', ..] => { + self.next(); + TokenKind::Minus + } + [b'<', b'<', b'<', ..] => { + // TODO: Handle both heredocs and nowdocs. + self.skip(3); + + todo!("heredocs & nowdocs"); + } + [b'<', b'<', b'=', ..] => { + self.skip(3); + + TokenKind::LeftShiftEquals + } + [b'<', b'<', ..] => { + self.skip(2); + TokenKind::LeftShift + } + [b'<', b'=', b'>', ..] => { + self.skip(3); + TokenKind::Spaceship + } + [b'<', b'=', ..] => { + self.skip(2); + TokenKind::LessThanEquals + } + [b'<', b'>', ..] => { + self.skip(2); + TokenKind::AngledLeftRight + } + [b'<', ..] => { + self.next(); + TokenKind::LessThan + } + [b'>', b'>', b'=', ..] => { + self.skip(3); + TokenKind::RightShiftEquals + } + [b'>', b'>', ..] => { + self.skip(2); + TokenKind::RightShift + } + [b'>', b'=', ..] => { + self.skip(2); + TokenKind::GreaterThanEquals + } + [b'>', ..] => { + self.next(); + TokenKind::GreaterThan + } + [b',', ..] => { + self.next(); + TokenKind::Comma + } + [b'[', ..] => { + self.next(); + TokenKind::LeftBracket + } + [b']', ..] => { + self.next(); + TokenKind::RightBracket + } + [b':', b':', ..] => { + self.skip(2); + TokenKind::DoubleColon + } + [b':', ..] => { + self.next(); + TokenKind::Colon + } + &[b'~', ..] => { + self.next(); + TokenKind::BitwiseNot + } + &[b, ..] => unimplemented!( + " char: {}, line: {}, col: {}", + b as char, + self.span.0, + self.span.1 + ), + // We should never reach this point since we have the empty checks surrounding + // the call to this function, but it's better to be safe than sorry. + [] => return Err(SyntaxError::UnexpectedEndOfFile(self.span)), + }; + + Ok(Token { kind, span }) + } + + fn double_quote(&mut self) -> Result, SyntaxError> { + let span = self.span; + let mut buffer = Vec::new(); + let kind = loop { + match self.peek_buf() { + [b'$', b'{', ..] => { + self.skip(2); + self.push_state(LexerState::LookingForVarname); + break TokenKind::DollarLeftBrace; + } + [b'{', b'$', ..] => { + // Intentionally only consume the left brace. + self.next(); + self.push_state(LexerState::Scripting); + break TokenKind::LeftBrace; + } + [b'"', ..] => { + self.next(); + self.enter_state(LexerState::Scripting); + break TokenKind::DoubleQuote; + } + [b'$', ident_start!(), ..] => { + self.next(); + let ident = self.consume_identifier(); + + match self.peek_buf() { + [b'[', ..] => self.push_state(LexerState::VarOffset), + [b'-', b'>', ident_start!(), ..] + | [b'?', b'-', b'>', ident_start!(), ..] => { + self.push_state(LexerState::LookingForProperty) + } + _ => {} + } + + break TokenKind::Variable(ident.into()); + } + &[b, ..] => { + self.next(); + buffer.push(b); + } + [] => return Err(SyntaxError::UnexpectedEndOfFile(self.span)), + } + }; + + let mut tokens = Vec::new(); + if !buffer.is_empty() { + tokens.push(Token { + kind: TokenKind::StringPart(buffer.into()), + span, + }) + } + + tokens.push(Token { kind, span }); + Ok(tokens) + } + + fn looking_for_varname(&mut self) -> Option { + if let Some(ident) = self.peek_identifier() { + if let Some(b'[' | b'}') = self.peek_byte(ident.len()) { + let ident = ident.to_vec(); + let span = self.span; + self.skip(ident.len()); + self.enter_state(LexerState::Scripting); + return Some(Token { + kind: TokenKind::Identifier(ident.into()), + span, + }); + } + } + + self.enter_state(LexerState::Scripting); + None + } + + fn looking_for_property(&mut self) -> Result { + let span = self.span; + let kind = match self.peek_buf() { + [b'-', b'>', ..] => { + self.skip(2); + TokenKind::Arrow + } + [b'?', b'-', b'>', ..] => { + self.skip(3); + TokenKind::NullsafeArrow + } + &[ident_start!(), ..] => { + let buffer = self.consume_identifier(); + self.pop_state(); + TokenKind::Identifier(buffer.into()) + } + // Should be impossible as we already looked ahead this far inside double_quote. + _ => unreachable!(), + }; + Ok(Token { kind, span }) + } + + fn var_offset(&mut self) -> Result { + let span = self.span; + let kind = match self.peek_buf() { + [b'$', ident_start!(), ..] => { + self.next(); + self.tokenize_variable() + } + &[b'0'..=b'9', ..] => { + // TODO: all integer literals are allowed, but only decimal integers with no underscores + // are actually treated as numbers. Others are treated as strings. + // Float literals are not allowed, but that could be handled in the parser. + self.tokenize_number()? + } + [b'[', ..] => { + self.next(); + TokenKind::LeftBracket + } + [b'-', ..] => { + self.next(); + TokenKind::Minus + } + [b']', ..] => { + self.next(); + self.pop_state(); + TokenKind::RightBracket + } + &[ident_start!(), ..] => { + let label = self.consume_identifier(); + TokenKind::Identifier(label.into()) + } + &[b, ..] => unimplemented!( + " char: {}, line: {}, col: {}", + b as char, + self.span.0, + self.span.1 + ), + [] => return Err(SyntaxError::UnexpectedEndOfFile(self.span)), + }; + Ok(Token { kind, span }) + } + + fn tokenize_single_quote_string(&mut self) -> Result { + let mut buffer = Vec::new(); + + loop { + match self.peek_buf() { + [b'\'', ..] => { + self.next(); + break; + } + &[b'\\', b @ b'\'' | b @ b'\\', ..] => { + self.skip(2); + buffer.push(b); + } + &[b, ..] => { + self.next(); + buffer.push(b); + } + [] => return Err(SyntaxError::UnexpectedEndOfFile(self.span)), + } + } + + Ok(TokenKind::LiteralString(buffer.into())) + } + + fn tokenize_double_quote_string(&mut self) -> Result { + let mut buffer = Vec::new(); + + let constant = loop { + match self.peek_buf() { + [b'"', ..] => { + self.next(); + break true; + } + &[b'\\', b @ (b'"' | b'\\' | b'$'), ..] => { + self.skip(2); + buffer.push(b); + } + &[b'\\', b'n', ..] => { + self.skip(2); + buffer.push(b'\n'); + } + &[b'\\', b'r', ..] => { + self.skip(2); + buffer.push(b'\r'); + } + &[b'\\', b't', ..] => { + self.skip(2); + buffer.push(b'\t'); + } + &[b'\\', b'v', ..] => { + self.skip(2); + buffer.push(b'\x0b'); + } + &[b'\\', b'e', ..] => { + self.skip(2); + buffer.push(b'\x1b'); + } + &[b'\\', b'f', ..] => { + self.skip(2); + buffer.push(b'\x0c'); + } + &[b'\\', b'x', b @ (b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'), ..] => { + self.skip(3); + + let mut hex = String::from(b as char); + if let Some(b @ (b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F')) = self.current { + self.next(); + hex.push(b as char); + } + + let b = u8::from_str_radix(&hex, 16).unwrap(); + buffer.push(b); + } + &[b'\\', b'u', b'{', ..] => { + self.skip(3); + + let mut code_point = String::new(); + while let Some(b @ (b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F')) = self.current { + self.next(); + code_point.push(b as char); + } + + if code_point.is_empty() || self.current != Some(b'}') { + return Err(SyntaxError::InvalidUnicodeEscape(self.span)); + } + self.next(); + + let c = if let Ok(c) = u32::from_str_radix(&code_point, 16) { + c + } else { + return Err(SyntaxError::InvalidUnicodeEscape(self.span)); + }; + + if let Some(c) = char::from_u32(c) { + let mut tmp = [0; 4]; + let bytes = c.encode_utf8(&mut tmp); + buffer.extend(bytes.as_bytes()); + } else { + return Err(SyntaxError::InvalidUnicodeEscape(self.span)); + } + } + &[b'\\', b @ b'0'..=b'7', ..] => { + self.skip(2); + + let mut octal = String::from(b as char); + if let Some(b @ b'0'..=b'7') = self.current { + self.next(); + octal.push(b as char); + } + if let Some(b @ b'0'..=b'7') = self.current { + self.next(); + octal.push(b as char); + } + + if let Ok(b) = u8::from_str_radix(&octal, 8) { + buffer.push(b); + } else { + return Err(SyntaxError::InvalidOctalEscape(self.span)); + } + } + [b'$', ident_start!(), ..] | [b'{', b'$', ..] | [b'$', b'{', ..] => { + break false; + } + &[b, ..] => { + self.next(); + buffer.push(b); + } + [] => return Err(SyntaxError::UnexpectedEndOfFile(self.span)), + } + }; + + Ok(if constant { + TokenKind::LiteralString(buffer.into()) + } else { + self.enter_state(LexerState::DoubleQuote); + TokenKind::StringPart(buffer.into()) + }) + } + + fn peek_identifier(&self) -> Option<&[u8]> { + let mut cursor = self.cursor; + if let Some(ident_start!()) = self.chars.get(cursor) { + cursor += 1; + while let Some(ident!()) = self.chars.get(cursor) { + cursor += 1; + } + Some(&self.chars[self.cursor..cursor]) + } else { + None + } + } + + fn consume_identifier(&mut self) -> Vec { + let ident = self.peek_identifier().unwrap().to_vec(); + self.skip(ident.len()); + + ident + } + + fn tokenize_variable(&mut self) -> TokenKind { + TokenKind::Variable(self.consume_identifier().into()) + } + + fn tokenize_number(&mut self) -> Result { + let mut buffer = String::new(); + + let (base, kind) = match self.peek_buf() { + [b'0', b'B' | b'b', ..] => { + self.skip(2); + (2, NumberKind::Int) + } + [b'0', b'O' | b'o', ..] => { + self.skip(2); + (8, NumberKind::Int) + } + [b'0', b'X' | b'x', ..] => { + self.skip(2); + (16, NumberKind::Int) + } + [b'0', ..] => (10, NumberKind::OctalOrFloat), + [b'.', ..] => (10, NumberKind::Float), + _ => (10, NumberKind::IntOrFloat), + }; + + if kind != NumberKind::Float { + self.read_digits(&mut buffer, base); + if kind == NumberKind::Int { + return parse_int(&buffer, base as u32, self.span); + } + } + + // Remaining cases: decimal integer, legacy octal integer, or float. + let is_float = matches!( + self.peek_buf(), + [b'.', ..] + | [b'e' | b'E', b'-' | b'+', b'0'..=b'9', ..] + | [b'e' | b'E', b'0'..=b'9', ..] + ); + if !is_float { + let base = if kind == NumberKind::OctalOrFloat { + 8 + } else { + 10 + }; + return parse_int(&buffer, base as u32, self.span); + } + + if self.current == Some(b'.') { + buffer.push('.'); + self.next(); + self.read_digits(&mut buffer, 10); + } + + if let Some(b'e' | b'E') = self.current { + buffer.push('e'); + self.next(); + if let Some(b @ (b'-' | b'+')) = self.current { + buffer.push(b as char); + self.next(); + } + self.read_digits(&mut buffer, 10); + } + + Ok(TokenKind::LiteralFloat(buffer.parse().unwrap())) + } + + fn read_digits(&mut self, buffer: &mut String, base: usize) { + if base == 16 { + self.read_digits_fn(buffer, u8::is_ascii_hexdigit); + } else { + let max = b'0' + base as u8; + self.read_digits_fn(buffer, |b| (b'0'..max).contains(b)); + }; + } + + fn read_digits_fn bool>(&mut self, buffer: &mut String, is_digit: F) { + if let Some(b) = self.current { + if is_digit(&b) { + self.next(); + buffer.push(b as char); + } else { + return; + } + } + loop { + match *self.peek_buf() { + [b, ..] if is_digit(&b) => { + self.next(); + buffer.push(b as char); + } + [b'_', b, ..] if is_digit(&b) => { + self.next(); + self.next(); + buffer.push(b as char); + } + _ => { + break; + } + } + } + } + + fn enter_state(&mut self, state: LexerState) { + *self.state_stack.last_mut().unwrap() = state; + } + + fn push_state(&mut self, state: LexerState) { + self.state_stack.push(state); + } + + fn pop_state(&mut self) { + self.state_stack.pop(); + } + + fn peek_buf(&self) -> &[u8] { + &self.chars[self.cursor..] + } + + fn peek_byte(&self, delta: usize) -> Option { + self.chars.get(self.cursor + delta).copied() + } + + fn try_read(&self, search: &'static [u8]) -> bool { + self.peek_buf().starts_with(search) + } + + fn skip(&mut self, count: usize) { + for _ in 0..count { + self.next(); + } + } + + fn next(&mut self) { + match self.current { + Some(b'\n') => { + self.span.0 += 1; + self.span.1 = 1; + } + Some(_) => self.span.1 += 1, + _ => {} + } + self.cursor += 1; + self.current = self.chars.get(self.cursor).copied(); + } +} + +// Parses an integer literal in the given base and converts errors to SyntaxError. +// It returns a float token instead on overflow. +fn parse_int(buffer: &str, base: u32, span: Span) -> Result { + match i64::from_str_radix(buffer, base) { + Ok(i) => Ok(TokenKind::LiteralInteger(i)), + Err(err) if err.kind() == &IntErrorKind::InvalidDigit => { + // The InvalidDigit error is only possible for legacy octal literals. + Err(SyntaxError::InvalidOctalLiteral(span)) + } + Err(err) if err.kind() == &IntErrorKind::PosOverflow => { + // Parse as i128 so we can handle other bases. + // This means there's an upper limit on how large the literal can be. + let i = i128::from_str_radix(buffer, base).unwrap(); + Ok(TokenKind::LiteralFloat(i as f64)) + } + _ => Err(SyntaxError::UnexpectedError(span)), + } +} + +fn identifier_to_keyword(ident: &[u8]) -> Option { + Some(match ident { + b"enddeclare" => TokenKind::EndDeclare, + b"endswitch" => TokenKind::EndSwitch, + b"endfor" => TokenKind::EndFor, + b"endwhile" => TokenKind::EndWhile, + b"endforeach" => TokenKind::EndForeach, + b"endif" => TokenKind::EndIf, + b"from" => TokenKind::From, + b"and" => TokenKind::LogicalAnd, + b"or" => TokenKind::LogicalOr, + b"xor" => TokenKind::LogicalXor, + b"print" => TokenKind::Print, + b"__halt_compiler" | b"__HALT_COMPILER" => TokenKind::HaltCompiler, + b"readonly" => TokenKind::Readonly, + b"global" => TokenKind::Global, + b"match" => TokenKind::Match, + b"abstract" => TokenKind::Abstract, + b"array" => TokenKind::Array, + b"as" => TokenKind::As, + b"break" => TokenKind::Break, + b"case" => TokenKind::Case, + b"catch" => TokenKind::Catch, + b"class" => TokenKind::Class, + b"clone" => TokenKind::Clone, + b"continue" => TokenKind::Continue, + b"const" => TokenKind::Const, + b"declare" => TokenKind::Declare, + b"default" => TokenKind::Default, + b"do" => TokenKind::Do, + b"echo" => TokenKind::Echo, + b"else" => TokenKind::Else, + b"elseif" => TokenKind::ElseIf, + b"enum" => TokenKind::Enum, + b"extends" => TokenKind::Extends, + b"false" | b"FALSE" => TokenKind::False, + b"final" => TokenKind::Final, + b"finally" => TokenKind::Finally, + b"fn" => TokenKind::Fn, + b"for" => TokenKind::For, + b"foreach" => TokenKind::Foreach, + b"function" => TokenKind::Function, + b"goto" => TokenKind::Goto, + b"if" => TokenKind::If, + b"include" => TokenKind::Include, + b"include_once" => TokenKind::IncludeOnce, + b"implements" => TokenKind::Implements, + b"interface" => TokenKind::Interface, + b"instanceof" => TokenKind::Instanceof, + b"namespace" => TokenKind::Namespace, + b"new" => TokenKind::New, + b"null" | b"NULL" => TokenKind::Null, + b"private" => TokenKind::Private, + b"protected" => TokenKind::Protected, + b"public" => TokenKind::Public, + b"require" => TokenKind::Require, + b"require_once" => TokenKind::RequireOnce, + b"return" => TokenKind::Return, + b"static" => TokenKind::Static, + b"switch" => TokenKind::Switch, + b"throw" => TokenKind::Throw, + b"trait" => TokenKind::Trait, + b"true" | b"TRUE" => TokenKind::True, + b"try" => TokenKind::Try, + b"use" => TokenKind::Use, + b"var" => TokenKind::Var, + b"yield" => TokenKind::Yield, + b"__DIR__" => TokenKind::DirConstant, + b"while" => TokenKind::While, + b"insteadof" => TokenKind::Insteadof, + _ => return None, + }) +} + +#[derive(Debug, Eq, PartialEq)] +enum NumberKind { + Int, + Float, + IntOrFloat, + OctalOrFloat, +} diff --git a/src/lexer/token.rs b/src/lexer/token.rs index 14295aa2..fe55a8c1 100644 --- a/src/lexer/token.rs +++ b/src/lexer/token.rs @@ -1,6 +1,6 @@ use std::fmt::Display; -use crate::ByteString; +use crate::lexer::byte_string::ByteString; pub type Span = (usize, usize); diff --git a/src/lib.rs b/src/lib.rs index 1709348e..8f934357 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,3 @@ -mod ast; -mod lexer; -mod parser; - -pub use ast::*; -pub use lexer::*; -pub use parser::error::ParseError; -pub use parser::error::ParseResult; -pub use parser::Parser; +pub mod lexer; +pub mod parser; +pub mod prelude; diff --git a/src/main.rs b/src/main.rs index a001e08c..676b7437 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,16 +1,44 @@ -use php_parser_rs::{Lexer, Parser}; +use php_parser_rs::prelude::Lexer; +use php_parser_rs::prelude::Parser; fn main() { - let file = std::env::args().nth(1).unwrap(); - let contents = std::fs::read_to_string(&file).unwrap(); + let file = match std::env::args().nth(1) { + Some(file) => file, + None => { + println!("Usage: php-parser [file]"); - println!("> Parsing {}", file); + ::std::process::exit(0); + } + }; - let mut lexer = Lexer::new(None); - let tokens = lexer.tokenize(contents.as_bytes()).unwrap(); + let contents = match std::fs::read_to_string(&file) { + Ok(contents) => contents, + Err(error) => { + println!("Failed to read file: {}", error); + + ::std::process::exit(1); + } + }; + + let mut lexer = Lexer::new(); + let tokens = match lexer.tokenize(contents.as_bytes()) { + Ok(tokens) => tokens, + Err(error) => { + println!("{}", error); + + ::std::process::exit(1); + } + }; let mut parser = Parser::new(None); - let ast = parser.parse(tokens).unwrap(); + let ast = match parser.parse(tokens) { + Ok(ast) => ast, + Err(error) => { + println!("{}", error); + + ::std::process::exit(1); + } + }; dbg!(ast); } diff --git a/src/ast.rs b/src/parser/ast.rs similarity index 98% rename from src/ast.rs rename to src/parser/ast.rs index 3a982fa2..f61ad1c5 100644 --- a/src/ast.rs +++ b/src/parser/ast.rs @@ -1,6 +1,7 @@ use std::fmt::Display; -use crate::{ByteString, TokenKind}; +use crate::lexer::byte_string::ByteString; +use crate::lexer::token::TokenKind; pub type Block = Vec; pub type Program = Block; @@ -103,18 +104,6 @@ impl From<&ByteString> for Identifier { } } -impl From<&[u8]> for Identifier { - fn from(name: &[u8]) -> Self { - Self::from(ByteString::from(name)) - } -} - -impl From<&str> for Identifier { - fn from(name: &str) -> Self { - Self::from(ByteString::from(name)) - } -} - pub type ParamList = Vec; #[derive(Debug, PartialEq, Clone)] diff --git a/src/parser/block.rs b/src/parser/block.rs index ca9654ce..d1d4a3db 100644 --- a/src/parser/block.rs +++ b/src/parser/block.rs @@ -1,11 +1,10 @@ -use crate::TokenKind; - -use crate::Block; - -use super::{ParseResult, Parser}; +use crate::lexer::token::TokenKind; +use crate::parser::ast::Block; +use crate::parser::error::ParseResult; +use crate::parser::Parser; impl Parser { - pub(crate) fn block(&mut self, until: &TokenKind) -> ParseResult { + pub(in crate::parser) fn block(&mut self, until: &TokenKind) -> ParseResult { self.skip_comments(); let mut block = Block::new(); diff --git a/src/parser/classish.rs b/src/parser/classish.rs index 7c5ae80b..a7c774e8 100644 --- a/src/parser/classish.rs +++ b/src/parser/classish.rs @@ -1,17 +1,18 @@ -use super::ParseResult; +use crate::lexer::token::TokenKind; +use crate::parser::ast::BackedEnumType; +use crate::parser::ast::Block; +use crate::parser::ast::ClassFlag; +use crate::parser::ast::Expression; +use crate::parser::ast::Identifier; +use crate::parser::ast::Statement; +use crate::parser::error::ParseResult; +use crate::parser::Parser; + use crate::expect_token; use crate::expected_token_err; -use crate::BackedEnumType; -use crate::Block; -use crate::ClassFlag; -use crate::Expression; -use crate::Identifier; -use crate::Parser; -use crate::Statement; -use crate::TokenKind; impl Parser { - pub(crate) fn class_definition(&mut self) -> ParseResult { + pub(in crate::parser) fn class_definition(&mut self) -> ParseResult { let flags: Vec = self.class_flags()?.iter().map(|f| f.into()).collect(); expect_token!([TokenKind::Class], self, ["`class`"]); @@ -58,7 +59,7 @@ impl Parser { }) } - pub(crate) fn interface_definition(&mut self) -> ParseResult { + pub(in crate::parser) fn interface_definition(&mut self) -> ParseResult { expect_token!([TokenKind::Interface], self, ["`interface`"]); let name = self.ident()?; @@ -95,7 +96,7 @@ impl Parser { }) } - pub(crate) fn trait_definition(&mut self) -> ParseResult { + pub(in crate::parser) fn trait_definition(&mut self) -> ParseResult { expect_token!([TokenKind::Trait], self, ["`trait`"]); let name = self.ident()?; @@ -121,7 +122,7 @@ impl Parser { }) } - pub(crate) fn anonymous_class_definition(&mut self) -> ParseResult { + pub(in crate::parser) fn anonymous_class_definition(&mut self) -> ParseResult { self.next(); expect_token!([TokenKind::Class], self, ["`class`"]); @@ -173,7 +174,7 @@ impl Parser { }) } - pub(crate) fn enum_definition(&mut self) -> ParseResult { + pub(in crate::parser) fn enum_definition(&mut self) -> ParseResult { self.next(); let name = self.ident()?; diff --git a/src/parser/classish_statement.rs b/src/parser/classish_statement.rs index b2234b01..96d086e7 100644 --- a/src/parser/classish_statement.rs +++ b/src/parser/classish_statement.rs @@ -1,15 +1,16 @@ -use super::ParseResult; +use crate::lexer::token::TokenKind; +use crate::parser::ast::ClassFlag; +use crate::parser::ast::Identifier; +use crate::parser::ast::MethodFlag; +use crate::parser::ast::Statement; +use crate::parser::ast::TraitAdaptation; +use crate::parser::error::ParseError; +use crate::parser::error::ParseResult; +use crate::parser::precedence::Precedence; +use crate::parser::Parser; + use crate::expect_token; use crate::expected_token_err; -use crate::parser::precedence::Precedence; -use crate::ClassFlag; -use crate::Identifier; -use crate::MethodFlag; -use crate::ParseError; -use crate::Parser; -use crate::Statement; -use crate::TokenKind; -use crate::TraitAdaptation; #[derive(Debug)] pub enum ClassishDefinitionType { @@ -21,11 +22,14 @@ pub enum ClassishDefinitionType { } impl Parser { - pub(crate) fn class_statement(&mut self, flags: Vec) -> ParseResult { + pub(in crate::parser) fn class_statement( + &mut self, + flags: Vec, + ) -> ParseResult { self.complete_class_statement(ClassishDefinitionType::Class(flags)) } - pub(crate) fn interface_statement(&mut self) -> ParseResult { + pub(in crate::parser) fn interface_statement(&mut self) -> ParseResult { if self.current.kind == TokenKind::Const { return self.parse_classish_const(vec![]); } @@ -46,15 +50,15 @@ impl Parser { } } - pub(crate) fn trait_statement(&mut self) -> ParseResult { + pub(in crate::parser) fn trait_statement(&mut self) -> ParseResult { self.complete_class_statement(ClassishDefinitionType::Trait) } - pub(crate) fn anonymous_class_statement(&mut self) -> ParseResult { + pub(in crate::parser) fn anonymous_class_statement(&mut self) -> ParseResult { self.complete_class_statement(ClassishDefinitionType::AnonymousClass) } - pub(crate) fn enum_statement(&mut self, backed: bool) -> ParseResult { + pub(in crate::parser) fn enum_statement(&mut self, backed: bool) -> ParseResult { if self.current.kind == TokenKind::Case { self.next(); diff --git a/src/parser/comments.rs b/src/parser/comments.rs index 174550b5..29cdc313 100644 --- a/src/parser/comments.rs +++ b/src/parser/comments.rs @@ -1,9 +1,9 @@ -use crate::{Token, TokenKind}; - -use crate::Parser; +use crate::lexer::token::Token; +use crate::lexer::token::TokenKind; +use crate::parser::Parser; impl Parser { - pub(crate) fn skip_comments(&mut self) { + pub(in crate::parser) fn skip_comments(&mut self) { while matches!( self.current.kind, TokenKind::Comment(_) | TokenKind::DocComment(_) @@ -12,7 +12,7 @@ impl Parser { } } - pub(crate) fn gather_comments(&mut self) { + pub(in crate::parser) fn gather_comments(&mut self) { while matches!( self.current.kind, TokenKind::Comment(_) | TokenKind::DocComment(_) @@ -22,7 +22,7 @@ impl Parser { } } - pub(crate) fn clear_comments(&mut self) -> Vec { + pub(in crate::parser) fn clear_comments(&mut self) -> Vec { let c = self.comments.clone(); self.comments = vec![]; c diff --git a/src/parser/error.rs b/src/parser/error.rs index abfb2fdd..8dd99607 100644 --- a/src/parser/error.rs +++ b/src/parser/error.rs @@ -1,11 +1,11 @@ use std::fmt::Display; -use crate::Span; -use crate::Type; +use crate::lexer::token::Span; +use crate::parser::ast::Type; pub type ParseResult = Result; -#[derive(Debug)] +#[derive(Debug, Eq, PartialEq)] pub enum ParseError { ExpectedToken(Vec, Option, Span), MultipleModifiers(String, Span), @@ -13,10 +13,7 @@ pub enum ParseError { UnexpectedToken(String, Span), UnexpectedEndOfFile, StandaloneTypeUsedInCombination(Type, Span), - InvalidClassStatement(String, Span), - ConstantInTrait(Span), TryWithoutCatchOrFinally(Span), - InvalidCatchArgumentType(Span), VariadicPromotedProperty(Span), PromotedPropertyOutsideConstructor(Span), PromotedPropertyOnAbstractConstructor(Span), @@ -49,15 +46,12 @@ impl Display for ParseError { Self::MultipleModifiers(modifier, span) => write!(f, "Parse Error: Multiple {} modifiers are not allowed on line {} column {}", modifier, span.0, span.1), Self::MultipleAccessModifiers( span) => write!(f, "Parse Error: Multiple access type modifiers are not allowed on line {} column {}", span.0, span.1), Self::UnexpectedToken(message, span) => write!(f, "Parse error: unexpected token {} on line {} column {}", message, span.0, span.1), - Self::InvalidClassStatement(message, span) => write!(f, "Parse error: {} on line {} column {}", message, span.0, span.1), Self::UnexpectedEndOfFile => write!(f, "Parse error: unexpected end of file."), Self::FinalModifierOnAbstractClassMember(span) => write!(f, "Parse error: Cannot use the final modifier on an abstract class member on line {} column {}", span.0, span.1), Self::StaticModifierOnConstant(span) => write!(f, "Parse error: Cannot use 'static' as constant modifier on line {} column {}", span.0, span.1), Self::ReadonlyModifierOnConstant(span) => write!(f, "Parse error: Cannot use 'readonly' as constant modifier on line {} column {}", span.0, span.1), Self::FinalModifierOnPrivateConstant(span) => write!(f, "Parse error: Private constant cannot be final as it is not visible to other classes on line {} column {}", span.0, span.1), - Self::ConstantInTrait(span) => write!(f, "Parse error: traits cannot contain constants on line {} column {}", span.0, span.1), Self::TryWithoutCatchOrFinally(span) => write!(f, "Parse error: cannot use try without catch or finally on line {} column {}", span.0, span.1), - Self::InvalidCatchArgumentType(span) => write!(f, "Parse error: catch types must either describe a single type or union of types on line {} column {}", span.0, span.1), Self::StandaloneTypeUsedInCombination(r#type, span) => write!(f, "Parse error: {} can only be used as a standalone type on line {} column {}", r#type, span.0, span.1), Self::VariadicPromotedProperty(span) => write!(f, "Parse error: Cannot declare variadic promoted property on line {} column {}", span.0, span.1), Self::PromotedPropertyOutsideConstructor(span) => write!(f, "Parse error: Cannot declare promoted property outside a constructor on line {} column {}", span.0, span.1), diff --git a/src/parser/flags.rs b/src/parser/flags.rs index 643f86c6..278b8094 100644 --- a/src/parser/flags.rs +++ b/src/parser/flags.rs @@ -1,7 +1,7 @@ -use super::ParseResult; -use crate::ParseError; -use crate::Parser; -use crate::TokenKind; +use crate::lexer::token::TokenKind; +use crate::parser::error::ParseError; +use crate::parser::error::ParseResult; +use crate::parser::Parser; enum FlagTarget { Class, @@ -11,14 +11,14 @@ enum FlagTarget { } impl Parser { - pub(crate) fn class_flags(&mut self) -> ParseResult> { + pub(in crate::parser) fn class_flags(&mut self) -> ParseResult> { self.collect( vec![TokenKind::Final, TokenKind::Abstract, TokenKind::Readonly], FlagTarget::Class, ) } - pub(crate) fn class_members_flags(&mut self) -> ParseResult> { + pub(in crate::parser) fn class_members_flags(&mut self) -> ParseResult> { self.collect( vec![ TokenKind::Final, @@ -33,7 +33,7 @@ impl Parser { ) } - pub(crate) fn enum_members_flags(&mut self) -> ParseResult> { + pub(in crate::parser) fn enum_members_flags(&mut self) -> ParseResult> { self.collect( vec![ TokenKind::Final, @@ -46,7 +46,7 @@ impl Parser { ) } - pub(crate) fn promoted_property_flags(&mut self) -> ParseResult> { + pub(in crate::parser) fn promoted_property_flags(&mut self) -> ParseResult> { self.collect( vec![ TokenKind::Private, diff --git a/src/parser/functions.rs b/src/parser/functions.rs index 57f88f86..eac96e29 100644 --- a/src/parser/functions.rs +++ b/src/parser/functions.rs @@ -1,16 +1,16 @@ -use super::classish_statement::ClassishDefinitionType; -use super::params::ParamPosition; -use super::ParseResult; -use crate::ByteString; -use crate::ClassFlag; -use crate::MethodFlag; -use crate::ParseError; -use crate::Parser; -use crate::Statement; -use crate::TokenKind; +use crate::lexer::byte_string::ByteString; +use crate::lexer::token::TokenKind; +use crate::parser::ast::ClassFlag; +use crate::parser::ast::MethodFlag; +use crate::parser::ast::Statement; +use crate::parser::classish_statement::ClassishDefinitionType; +use crate::parser::error::ParseError; +use crate::parser::error::ParseResult; +use crate::parser::params::ParamPosition; +use crate::parser::Parser; impl Parser { - pub(crate) fn function(&mut self) -> ParseResult { + pub(in crate::parser) fn function(&mut self) -> ParseResult { self.next(); let by_ref = if self.current.kind == TokenKind::Ampersand { @@ -51,7 +51,7 @@ impl Parser { }) } - pub(crate) fn method( + pub(in crate::parser) fn method( &mut self, class_type: ClassishDefinitionType, flags: Vec, diff --git a/src/parser/ident.rs b/src/parser/ident.rs index 7b4f854c..21aa8f50 100644 --- a/src/parser/ident.rs +++ b/src/parser/ident.rs @@ -1,18 +1,20 @@ -use super::ParseResult; +use crate::lexer::byte_string::ByteString; +use crate::lexer::token::TokenKind; +use crate::parser::error::ParseResult; +use crate::parser::Parser; + use crate::expect_token; -use crate::Parser; -use crate::{ByteString, TokenKind}; impl Parser { /// Expect an unqualified identifier such as Foo or Bar. - pub(crate) fn ident(&mut self) -> ParseResult { + pub(in crate::parser) fn ident(&mut self) -> ParseResult { Ok(expect_token!([ TokenKind::Identifier(identifier) => identifier, ], self, "an identifier")) } /// Expect an unqualified or qualified identifier such as Foo, Bar or Foo\Bar. - pub(crate) fn name(&mut self) -> ParseResult { + pub(in crate::parser) fn name(&mut self) -> ParseResult { Ok(expect_token!([ TokenKind::Identifier(identifier) => identifier, TokenKind::QualifiedIdentifier(qualified) => qualified, @@ -20,7 +22,7 @@ impl Parser { } /// Expect an unqualified, qualified or fully qualified identifier such as Foo, Foo\Bar or \Foo\Bar. - pub(crate) fn full_name(&mut self) -> ParseResult { + pub(in crate::parser) fn full_name(&mut self) -> ParseResult { Ok(expect_token!([ TokenKind::Identifier(identifier) => identifier, TokenKind::QualifiedIdentifier(qualified) => qualified, @@ -28,13 +30,13 @@ impl Parser { ], self, "an identifier")) } - pub(crate) fn var(&mut self) -> ParseResult { + pub(in crate::parser) fn var(&mut self) -> ParseResult { Ok(expect_token!([ TokenKind::Variable(v) => v, ], self, "a variable")) } - pub(crate) fn full_name_maybe_type_keyword(&mut self) -> ParseResult { + pub(in crate::parser) fn full_name_maybe_type_keyword(&mut self) -> ParseResult { match self.current.kind { TokenKind::Array | TokenKind::Callable => { let r = Ok(self.current.kind.to_string().into()); @@ -45,7 +47,7 @@ impl Parser { } } - pub(crate) fn type_with_static(&mut self) -> ParseResult { + pub(in crate::parser) fn type_with_static(&mut self) -> ParseResult { Ok(match self.current.kind { TokenKind::Static | TokenKind::Null | TokenKind::True | TokenKind::False => { let str = self.current.kind.to_string(); @@ -56,7 +58,7 @@ impl Parser { }) } - pub(crate) fn ident_maybe_reserved(&mut self) -> ParseResult { + pub(in crate::parser) fn ident_maybe_reserved(&mut self) -> ParseResult { match self.current.kind { _ if is_reserved_ident(&self.current.kind) => { let string = self.current.kind.to_string().into(); @@ -68,7 +70,7 @@ impl Parser { } } -pub(crate) fn is_reserved_ident(kind: &TokenKind) -> bool { +pub fn is_reserved_ident(kind: &TokenKind) -> bool { matches!( kind, TokenKind::Static diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 253f30d3..54b4d660 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -3,22 +3,21 @@ use std::vec::IntoIter; use crate::expect_literal; use crate::expect_token; use crate::expected_token_err; -use crate::lexer::{Token, TokenKind}; +use crate::lexer::byte_string::ByteString; +use crate::lexer::token::Token; +use crate::lexer::token::TokenKind; +use crate::parser::ast::{ + ArrayItem, Block, Case, Catch, ClosureUse, Constant, DeclareItem, ElseIf, Expression, + IncludeKind, MagicConst, MatchArm, Program, Statement, StaticVar, StringPart, + TryBlockCaughtType, Type, Use, UseKind, +}; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; +use crate::parser::ident::is_reserved_ident; +use crate::parser::params::ParamPosition; use crate::parser::precedence::{Associativity, Precedence}; -use crate::{ - ast::{ - ArrayItem, ClosureUse, Constant, DeclareItem, ElseIf, IncludeKind, MagicConst, StaticVar, - StringPart, Use, UseKind, - }, - Block, Case, Catch, Expression, MatchArm, Program, Statement, Type, -}; -use crate::{ByteString, TryBlockCaughtType}; - -use self::ident::is_reserved_ident; -use self::params::ParamPosition; +pub mod ast; pub mod error; mod block; diff --git a/src/parser/params.rs b/src/parser/params.rs index 549a8bbf..1d1a7dba 100644 --- a/src/parser/params.rs +++ b/src/parser/params.rs @@ -1,12 +1,15 @@ -use crate::expect_token; +use crate::lexer::token::TokenKind; +use crate::parser::ast::Arg; +use crate::parser::ast::Expression; +use crate::parser::ast::Param; +use crate::parser::ast::ParamList; +use crate::parser::ast::PropertyFlag; use crate::parser::error::ParseError; -use crate::TokenKind; -use crate::{ - ast::{Arg, ParamList, PropertyFlag}, - Expression, Param, -}; +use crate::parser::error::ParseResult; +use crate::parser::precedence::Precedence; +use crate::parser::Parser; -use super::{precedence::Precedence, ParseResult, Parser}; +use crate::expect_token; #[derive(Debug)] pub enum ParamPosition { @@ -16,7 +19,10 @@ pub enum ParamPosition { } impl Parser { - pub(crate) fn param_list(&mut self, position: ParamPosition) -> Result { + pub(in crate::parser) fn param_list( + &mut self, + position: ParamPosition, + ) -> Result { let mut params = ParamList::new(); while !self.is_eof() && self.current.kind != TokenKind::RightParen { @@ -105,7 +111,7 @@ impl Parser { Ok(params) } - pub(crate) fn args_list(&mut self) -> ParseResult> { + pub(in crate::parser) fn args_list(&mut self) -> ParseResult> { let mut args = Vec::new(); while !self.is_eof() && self.current.kind != TokenKind::RightParen { diff --git a/src/parser/precedence.rs b/src/parser/precedence.rs index fa5c10e9..16cf1adf 100644 --- a/src/parser/precedence.rs +++ b/src/parser/precedence.rs @@ -1,4 +1,4 @@ -use crate::TokenKind; +use crate::lexer::token::TokenKind; #[allow(dead_code)] #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] diff --git a/src/parser/punc.rs b/src/parser/punc.rs index a8d6bb78..1eb5b9dc 100644 --- a/src/parser/punc.rs +++ b/src/parser/punc.rs @@ -1,42 +1,41 @@ -use crate::TokenKind; +use crate::lexer::token::TokenKind; +use crate::parser::error::ParseResult; +use crate::parser::Parser; use crate::expect_token; -use crate::Parser; - -use super::ParseResult; impl Parser { - pub(crate) fn semi(&mut self) -> ParseResult<()> { + pub(in crate::parser) fn semi(&mut self) -> ParseResult<()> { expect_token!([TokenKind::SemiColon], self, "`;`"); Ok(()) } - pub(crate) fn lbrace(&mut self) -> ParseResult<()> { + pub(in crate::parser) fn lbrace(&mut self) -> ParseResult<()> { expect_token!([TokenKind::LeftBrace], self, "`{`"); Ok(()) } - pub(crate) fn rbrace(&mut self) -> ParseResult<()> { + pub(in crate::parser) fn rbrace(&mut self) -> ParseResult<()> { expect_token!([TokenKind::RightBrace], self, "`}`"); Ok(()) } - pub(crate) fn lparen(&mut self) -> ParseResult<()> { + pub(in crate::parser) fn lparen(&mut self) -> ParseResult<()> { expect_token!([TokenKind::LeftParen], self, "`(`"); Ok(()) } - pub(crate) fn rparen(&mut self) -> ParseResult<()> { + pub(in crate::parser) fn rparen(&mut self) -> ParseResult<()> { expect_token!([TokenKind::RightParen], self, "`)`"); Ok(()) } - pub(crate) fn rbracket(&mut self) -> ParseResult<()> { + pub(in crate::parser) fn rbracket(&mut self) -> ParseResult<()> { expect_token!([TokenKind::RightBracket], self, "`]`"); Ok(()) } - pub(crate) fn optional_comma(&mut self) -> ParseResult<()> { + pub(in crate::parser) fn optional_comma(&mut self) -> ParseResult<()> { if self.current.kind == TokenKind::Comma { expect_token!([TokenKind::Comma], self, "`,`"); } @@ -44,7 +43,7 @@ impl Parser { Ok(()) } - pub(crate) fn colon(&mut self) -> ParseResult<()> { + pub(in crate::parser) fn colon(&mut self) -> ParseResult<()> { expect_token!([TokenKind::Colon], self, "`:`"); Ok(()) diff --git a/src/parser/vars.rs b/src/parser/vars.rs index 021859a5..29da2f78 100644 --- a/src/parser/vars.rs +++ b/src/parser/vars.rs @@ -1,9 +1,12 @@ -use super::{ParseError, ParseResult, Precedence}; -use crate::TokenKind; -use crate::{Expression, Parser}; +use crate::lexer::token::TokenKind; +use crate::parser::ast::Expression; +use crate::parser::error::ParseError; +use crate::parser::error::ParseResult; +use crate::parser::precedence::Precedence; +use crate::parser::Parser; impl Parser { - pub(crate) fn dynamic_variable(&mut self) -> ParseResult { + pub(in crate::parser) fn dynamic_variable(&mut self) -> ParseResult { self.next(); Ok(match &self.current.kind { diff --git a/src/prelude.rs b/src/prelude.rs new file mode 100644 index 00000000..8298ab6a --- /dev/null +++ b/src/prelude.rs @@ -0,0 +1,8 @@ +pub use crate::lexer::byte_string::*; +pub use crate::lexer::error::*; +pub use crate::lexer::token::*; +pub use crate::lexer::*; + +pub use crate::parser::ast::*; +pub use crate::parser::error::*; +pub use crate::parser::*; diff --git a/tests/0001/tokens.txt b/tests/0001/tokens.txt new file mode 100644 index 00000000..148e2e88 --- /dev/null +++ b/tests/0001/tokens.txt @@ -0,0 +1,452 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 3, + 14, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 21, + ), + }, + Token { + kind: Equals, + span: ( + 3, + 24, + ), + }, + Token { + kind: LiteralString( + "", + ), + span: ( + 3, + 26, + ), + }, + Token { + kind: Comma, + span: ( + 3, + 28, + ), + }, + Token { + kind: Array, + span: ( + 3, + 30, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 3, + 36, + ), + }, + Token { + kind: Equals, + span: ( + 3, + 39, + ), + }, + Token { + kind: LeftBracket, + span: ( + 3, + 41, + ), + }, + Token { + kind: RightBracket, + span: ( + 3, + 42, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 43, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 44, + ), + }, + Token { + kind: Identifier( + "never", + ), + span: ( + 3, + 46, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 52, + ), + }, + Token { + kind: Identifier( + "exit", + ), + span: ( + 4, + 5, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 9, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 4, + 10, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 12, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, + Token { + kind: Function, + span: ( + 7, + 1, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 7, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 13, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 7, + 14, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 7, + 18, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 20, + ), + }, + Token { + kind: Identifier( + "float", + ), + span: ( + 7, + 22, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 7, + 28, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 30, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 7, + 32, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 7, + 39, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 41, + ), + }, + Token { + kind: True, + span: ( + 7, + 43, + ), + }, + Token { + kind: Variable( + "d", + ), + span: ( + 7, + 48, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 50, + ), + }, + Token { + kind: False, + span: ( + 7, + 52, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 7, + 58, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 60, + ), + }, + Token { + kind: Null, + span: ( + 7, + 62, + ), + }, + Token { + kind: Variable( + "f", + ), + span: ( + 7, + 67, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 69, + ), + }, + Token { + kind: Colon, + span: ( + 7, + 70, + ), + }, + Token { + kind: Null, + span: ( + 7, + 72, + ), + }, + Token { + kind: Pipe, + span: ( + 7, + 76, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 7, + 77, + ), + }, + Token { + kind: Pipe, + span: ( + 7, + 83, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 7, + 84, + ), + }, + Token { + kind: Pipe, + span: ( + 7, + 87, + ), + }, + Token { + kind: Identifier( + "float", + ), + span: ( + 7, + 88, + ), + }, + Token { + kind: LeftBrace, + span: ( + 7, + 94, + ), + }, + Token { + kind: Return, + span: ( + 8, + 5, + ), + }, + Token { + kind: Null, + span: ( + 8, + 12, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, +] diff --git a/tests/0002/tokens.txt b/tests/0002/tokens.txt new file mode 100644 index 00000000..1386d00c --- /dev/null +++ b/tests/0002/tokens.txt @@ -0,0 +1,126 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 14, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 15, + ), + }, + Token { + kind: Identifier( + "never", + ), + span: ( + 3, + 17, + ), + }, + Token { + kind: Pipe, + span: ( + 3, + 22, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 3, + 23, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 30, + ), + }, + Token { + kind: Identifier( + "exit", + ), + span: ( + 4, + 5, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 9, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 4, + 10, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 12, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0003/tokens.txt b/tests/0003/tokens.txt new file mode 100644 index 00000000..623865d5 --- /dev/null +++ b/tests/0003/tokens.txt @@ -0,0 +1,157 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 14, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 15, + ), + }, + Token { + kind: Identifier( + "never", + ), + span: ( + 3, + 17, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 23, + ), + }, + Token { + kind: Try, + span: ( + 4, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 9, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 12, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 13, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 14, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: Catch, + span: ( + 6, + 7, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 14, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, +] diff --git a/tests/0004/tokens.txt b/tests/0004/tokens.txt new file mode 100644 index 00000000..544f8ad6 --- /dev/null +++ b/tests/0004/tokens.txt @@ -0,0 +1,166 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 14, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 15, + ), + }, + Token { + kind: Identifier( + "never", + ), + span: ( + 3, + 17, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 23, + ), + }, + Token { + kind: Try, + span: ( + 4, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 9, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 12, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 13, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 14, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: Catch, + span: ( + 6, + 7, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 13, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 6, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 16, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 18, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, +] diff --git a/tests/0005/tokens.txt b/tests/0005/tokens.txt new file mode 100644 index 00000000..f9820af6 --- /dev/null +++ b/tests/0005/tokens.txt @@ -0,0 +1,143 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 14, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 15, + ), + }, + Token { + kind: Identifier( + "never", + ), + span: ( + 3, + 17, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 23, + ), + }, + Token { + kind: Try, + span: ( + 4, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 9, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 12, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 13, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 14, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: Catch, + span: ( + 6, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 13, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, +] diff --git a/tests/0006/tokens.txt b/tests/0006/tokens.txt new file mode 100644 index 00000000..caa19f0e --- /dev/null +++ b/tests/0006/tokens.txt @@ -0,0 +1,103 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Include, + span: ( + 3, + 1, + ), + }, + Token { + kind: LiteralString( + "foo.php", + ), + span: ( + 3, + 9, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 18, + ), + }, + Token { + kind: IncludeOnce, + span: ( + 5, + 1, + ), + }, + Token { + kind: LiteralString( + "bar.php", + ), + span: ( + 5, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 23, + ), + }, + Token { + kind: Require, + span: ( + 7, + 1, + ), + }, + Token { + kind: LiteralString( + "baz.php", + ), + span: ( + 7, + 9, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 18, + ), + }, + Token { + kind: RequireOnce, + span: ( + 9, + 1, + ), + }, + Token { + kind: LiteralString( + "qux.php", + ), + span: ( + 9, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 23, + ), + }, +] diff --git a/tests/0007/tokens.txt b/tests/0007/tokens.txt new file mode 100644 index 00000000..cfd0d2b3 --- /dev/null +++ b/tests/0007/tokens.txt @@ -0,0 +1,215 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 3, + 6, + ), + }, + Token { + kind: Identifier( + "give_me_foo", + ), + span: ( + 3, + 8, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 21, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 5, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 5, + 4, + ), + }, + Token { + kind: LeftBracket, + span: ( + 5, + 6, + ), + }, + Token { + kind: LiteralString( + "single", + ), + span: ( + 6, + 5, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 6, + 14, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 6, + 17, + ), + }, + Token { + kind: Instanceof, + span: ( + 6, + 22, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 6, + 33, + ), + }, + Token { + kind: Comma, + span: ( + 6, + 36, + ), + }, + Token { + kind: LiteralString( + "multiple", + ), + span: ( + 7, + 5, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 7, + 16, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 7, + 19, + ), + }, + Token { + kind: Instanceof, + span: ( + 7, + 24, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 7, + 35, + ), + }, + Token { + kind: BooleanAnd, + span: ( + 7, + 39, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 7, + 42, + ), + }, + Token { + kind: Instanceof, + span: ( + 7, + 47, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 7, + 58, + ), + }, + Token { + kind: RightBracket, + span: ( + 8, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 2, + ), + }, +] diff --git a/tests/0008/tokens.txt b/tests/0008/tokens.txt new file mode 100644 index 00000000..4402f38a --- /dev/null +++ b/tests/0008/tokens.txt @@ -0,0 +1,395 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 3, + 4, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 3, + 6, + ), + }, + Token { + kind: Pow, + span: ( + 3, + 8, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 12, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 5, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 5, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 5, + 6, + ), + }, + Token { + kind: Question, + span: ( + 5, + 8, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 5, + 10, + ), + }, + Token { + kind: Colon, + span: ( + 5, + 12, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 5, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 15, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 7, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 7, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 7, + 6, + ), + }, + Token { + kind: Question, + span: ( + 7, + 8, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 7, + 10, + ), + }, + Token { + kind: Question, + span: ( + 7, + 12, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 7, + 14, + ), + }, + Token { + kind: Colon, + span: ( + 7, + 16, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 7, + 18, + ), + }, + Token { + kind: Colon, + span: ( + 7, + 20, + ), + }, + Token { + kind: LiteralInteger( + 5, + ), + span: ( + 7, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 23, + ), + }, + Token { + kind: Variable( + "d", + ), + span: ( + 9, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 9, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 9, + 6, + ), + }, + Token { + kind: QuestionColon, + span: ( + 9, + 8, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 9, + 11, + ), + }, + Token { + kind: QuestionColon, + span: ( + 9, + 13, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 9, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 17, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 11, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 11, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 11, + 6, + ), + }, + Token { + kind: Coalesce, + span: ( + 11, + 8, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 11, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 12, + ), + }, + Token { + kind: Variable( + "f", + ), + span: ( + 13, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 13, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 13, + 6, + ), + }, + Token { + kind: Coalesce, + span: ( + 13, + 8, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 13, + 11, + ), + }, + Token { + kind: Coalesce, + span: ( + 13, + 13, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 13, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 13, + 17, + ), + }, +] diff --git a/tests/0009/tokens.txt b/tests/0009/tokens.txt new file mode 100644 index 00000000..24810b5d --- /dev/null +++ b/tests/0009/tokens.txt @@ -0,0 +1,167 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftBracket, + span: ( + 3, + 5, + ), + }, + Token { + kind: LiteralString( + "bar", + ), + span: ( + 3, + 6, + ), + }, + Token { + kind: RightBracket, + span: ( + 3, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 12, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 5, + 1, + ), + }, + Token { + kind: LeftBracket, + span: ( + 5, + 5, + ), + }, + Token { + kind: LiteralString( + "bar", + ), + span: ( + 5, + 6, + ), + }, + Token { + kind: RightBracket, + span: ( + 5, + 11, + ), + }, + Token { + kind: LeftBracket, + span: ( + 5, + 12, + ), + }, + Token { + kind: LiteralString( + "baz", + ), + span: ( + 5, + 13, + ), + }, + Token { + kind: RightBracket, + span: ( + 5, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 19, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 7, + 1, + ), + }, + Token { + kind: LeftBracket, + span: ( + 7, + 5, + ), + }, + Token { + kind: LiteralString( + "bar", + ), + span: ( + 7, + 6, + ), + }, + Token { + kind: RightBracket, + span: ( + 7, + 11, + ), + }, + Token { + kind: Equals, + span: ( + 7, + 13, + ), + }, + Token { + kind: LiteralString( + "baz", + ), + span: ( + 7, + 15, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 20, + ), + }, +] diff --git a/tests/0010/tokens.txt b/tests/0010/tokens.txt new file mode 100644 index 00000000..49662042 --- /dev/null +++ b/tests/0010/tokens.txt @@ -0,0 +1,1083 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 3, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 3, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 3, + 13, + ), + }, + Token { + kind: DoubleEquals, + span: ( + 3, + 15, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 3, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 19, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 20, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 4, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 4, + 13, + ), + }, + Token { + kind: TripleEquals, + span: ( + 4, + 15, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 4, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 21, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 5, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 5, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 5, + 13, + ), + }, + Token { + kind: BangEquals, + span: ( + 5, + 15, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 5, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 19, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 20, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 6, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 6, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 6, + 13, + ), + }, + Token { + kind: BangDoubleEquals, + span: ( + 6, + 15, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 6, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 21, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 7, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 7, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 7, + 13, + ), + }, + Token { + kind: Plus, + span: ( + 7, + 15, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 7, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 19, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 8, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 8, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 8, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 8, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 8, + 13, + ), + }, + Token { + kind: Minus, + span: ( + 8, + 15, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 8, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 8, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 19, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 9, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 9, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 9, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 9, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 9, + 13, + ), + }, + Token { + kind: Slash, + span: ( + 9, + 15, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 9, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 9, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 19, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 10, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 10, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 10, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 10, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 10, + 13, + ), + }, + Token { + kind: Caret, + span: ( + 10, + 15, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 10, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 10, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 10, + 19, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 11, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 11, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 11, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 11, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 11, + 13, + ), + }, + Token { + kind: Asterisk, + span: ( + 11, + 15, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 11, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 11, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 19, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 12, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 12, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 12, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 12, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 12, + 13, + ), + }, + Token { + kind: RightShift, + span: ( + 12, + 15, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 12, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 12, + 19, + ), + }, + Token { + kind: SemiColon, + span: ( + 12, + 20, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 13, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 13, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 13, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 13, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 13, + 13, + ), + }, + Token { + kind: LeftShift, + span: ( + 13, + 15, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 13, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 13, + 19, + ), + }, + Token { + kind: SemiColon, + span: ( + 13, + 20, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 14, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 14, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 14, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 14, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 14, + 13, + ), + }, + Token { + kind: Pipe, + span: ( + 14, + 15, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 14, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 14, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 14, + 19, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 15, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 15, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 15, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 15, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 15, + 13, + ), + }, + Token { + kind: Ampersand, + span: ( + 15, + 15, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 15, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 15, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 15, + 19, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 16, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 16, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 16, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 16, + 11, + ), + }, + Token { + kind: BitwiseNot, + span: ( + 16, + 13, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 16, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 16, + 15, + ), + }, + Token { + kind: SemiColon, + span: ( + 16, + 16, + ), + }, + Token { + kind: Echo, + span: ( + 18, + 1, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 18, + 6, + ), + }, + Token { + kind: Plus, + span: ( + 18, + 8, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 18, + 10, + ), + }, + Token { + kind: Asterisk, + span: ( + 18, + 12, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 18, + 14, + ), + }, + Token { + kind: Slash, + span: ( + 18, + 16, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 18, + 18, + ), + }, + Token { + kind: Minus, + span: ( + 18, + 20, + ), + }, + Token { + kind: LiteralInteger( + 5, + ), + span: ( + 18, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 18, + 23, + ), + }, +] diff --git a/tests/0011/tokens.txt b/tests/0011/tokens.txt new file mode 100644 index 00000000..a93dbf48 --- /dev/null +++ b/tests/0011/tokens.txt @@ -0,0 +1,1152 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 3, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 3, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 13, + ), + }, + Token { + kind: DoubleEquals, + span: ( + 3, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 3, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 21, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 22, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 4, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 13, + ), + }, + Token { + kind: TripleEquals, + span: ( + 4, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 4, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 23, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 5, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 5, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 5, + 13, + ), + }, + Token { + kind: BangEquals, + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 5, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 21, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 22, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 6, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 6, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 6, + 13, + ), + }, + Token { + kind: BangDoubleEquals, + span: ( + 6, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 6, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 23, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 7, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 7, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 7, + 13, + ), + }, + Token { + kind: Plus, + span: ( + 7, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 7, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 21, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 8, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 8, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 8, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 8, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 8, + 13, + ), + }, + Token { + kind: Minus, + span: ( + 8, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 8, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 8, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 21, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 9, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 9, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 9, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 9, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 9, + 13, + ), + }, + Token { + kind: Slash, + span: ( + 9, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 9, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 9, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 21, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 10, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 10, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 10, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 10, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 10, + 13, + ), + }, + Token { + kind: Caret, + span: ( + 10, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 10, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 10, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 10, + 21, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 11, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 11, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 11, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 11, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 11, + 13, + ), + }, + Token { + kind: Asterisk, + span: ( + 11, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 11, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 11, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 21, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 12, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 12, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 12, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 12, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 12, + 13, + ), + }, + Token { + kind: RightShift, + span: ( + 12, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 12, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 12, + 21, + ), + }, + Token { + kind: SemiColon, + span: ( + 12, + 22, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 13, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 13, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 13, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 13, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 13, + 13, + ), + }, + Token { + kind: LeftShift, + span: ( + 13, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 13, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 13, + 21, + ), + }, + Token { + kind: SemiColon, + span: ( + 13, + 22, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 14, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 14, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 14, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 14, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 14, + 13, + ), + }, + Token { + kind: Pipe, + span: ( + 14, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 14, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 14, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 14, + 21, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 15, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 15, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 15, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 15, + 11, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 15, + 13, + ), + }, + Token { + kind: Ampersand, + span: ( + 15, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 15, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 15, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 15, + 21, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 16, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 16, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 16, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 16, + 11, + ), + }, + Token { + kind: BitwiseNot, + span: ( + 16, + 13, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 16, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 16, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 16, + 17, + ), + }, + Token { + kind: Echo, + span: ( + 18, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 18, + 6, + ), + }, + Token { + kind: Plus, + span: ( + 18, + 9, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 18, + 11, + ), + }, + Token { + kind: Asterisk, + span: ( + 18, + 14, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 18, + 16, + ), + }, + Token { + kind: Slash, + span: ( + 18, + 19, + ), + }, + Token { + kind: Variable( + "d", + ), + span: ( + 18, + 21, + ), + }, + Token { + kind: Minus, + span: ( + 18, + 24, + ), + }, + Token { + kind: Dollar, + span: ( + 18, + 26, + ), + }, + Token { + kind: LeftBrace, + span: ( + 18, + 27, + ), + }, + Token { + kind: LiteralString( + "foo", + ), + span: ( + 18, + 28, + ), + }, + Token { + kind: Dot, + span: ( + 18, + 34, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 18, + 36, + ), + }, + Token { + kind: Question, + span: ( + 18, + 39, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 18, + 41, + ), + }, + Token { + kind: Colon, + span: ( + 18, + 43, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 18, + 45, + ), + }, + Token { + kind: RightBrace, + span: ( + 18, + 46, + ), + }, + Token { + kind: SemiColon, + span: ( + 18, + 47, + ), + }, +] diff --git a/tests/0012/tokens.txt b/tests/0012/tokens.txt new file mode 100644 index 00000000..cb595aed --- /dev/null +++ b/tests/0012/tokens.txt @@ -0,0 +1,1300 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 3, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 14, + ), + }, + Token { + kind: DoubleEquals, + span: ( + 3, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 3, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 22, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 24, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 4, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 14, + ), + }, + Token { + kind: TripleEquals, + span: ( + 4, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 23, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 24, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 25, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 5, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 5, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 5, + 14, + ), + }, + Token { + kind: BangEquals, + span: ( + 5, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 5, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 22, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 24, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 6, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 6, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 6, + 14, + ), + }, + Token { + kind: BangDoubleEquals, + span: ( + 6, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 6, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 23, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 24, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 25, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 7, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 7, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 7, + 14, + ), + }, + Token { + kind: Plus, + span: ( + 7, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 7, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 23, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 8, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 8, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 8, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 8, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 8, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 8, + 14, + ), + }, + Token { + kind: Minus, + span: ( + 8, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 8, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 8, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 8, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 23, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 9, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 9, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 9, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 9, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 9, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 9, + 14, + ), + }, + Token { + kind: Slash, + span: ( + 9, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 9, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 9, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 9, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 23, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 10, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 10, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 10, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 10, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 10, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 10, + 14, + ), + }, + Token { + kind: Caret, + span: ( + 10, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 10, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 10, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 10, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 10, + 23, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 11, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 11, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 11, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 11, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 11, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 11, + 14, + ), + }, + Token { + kind: Asterisk, + span: ( + 11, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 11, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 11, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 11, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 23, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 12, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 12, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 12, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 12, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 12, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 12, + 14, + ), + }, + Token { + kind: RightShift, + span: ( + 12, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 12, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 12, + 22, + ), + }, + Token { + kind: RightParen, + span: ( + 12, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 12, + 24, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 13, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 13, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 13, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 13, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 13, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 13, + 14, + ), + }, + Token { + kind: LeftShift, + span: ( + 13, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 13, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 13, + 22, + ), + }, + Token { + kind: RightParen, + span: ( + 13, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 13, + 24, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 14, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 14, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 14, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 14, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 14, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 14, + 14, + ), + }, + Token { + kind: Pipe, + span: ( + 14, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 14, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 14, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 14, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 14, + 23, + ), + }, + Token { + kind: Identifier( + "define", + ), + span: ( + 15, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 15, + 7, + ), + }, + Token { + kind: LiteralString( + "a", + ), + span: ( + 15, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 15, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 15, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 15, + 14, + ), + }, + Token { + kind: Ampersand, + span: ( + 15, + 17, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 15, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 15, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 15, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 15, + 23, + ), + }, + Token { + kind: Echo, + span: ( + 17, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 17, + 6, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 17, + 7, + ), + }, + Token { + kind: Plus, + span: ( + 17, + 10, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 17, + 12, + ), + }, + Token { + kind: RightParen, + span: ( + 17, + 14, + ), + }, + Token { + kind: Asterisk, + span: ( + 17, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 17, + 18, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 17, + 19, + ), + }, + Token { + kind: Slash, + span: ( + 17, + 22, + ), + }, + Token { + kind: Variable( + "d", + ), + span: ( + 17, + 24, + ), + }, + Token { + kind: Minus, + span: ( + 17, + 27, + ), + }, + Token { + kind: Dollar, + span: ( + 17, + 29, + ), + }, + Token { + kind: LeftBrace, + span: ( + 17, + 30, + ), + }, + Token { + kind: LiteralString( + "foo", + ), + span: ( + 17, + 31, + ), + }, + Token { + kind: Dot, + span: ( + 17, + 37, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 17, + 39, + ), + }, + Token { + kind: Question, + span: ( + 17, + 42, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 17, + 44, + ), + }, + Token { + kind: Colon, + span: ( + 17, + 46, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 17, + 48, + ), + }, + Token { + kind: RightBrace, + span: ( + 17, + 49, + ), + }, + Token { + kind: RightParen, + span: ( + 17, + 50, + ), + }, + Token { + kind: SemiColon, + span: ( + 17, + 51, + ), + }, +] diff --git a/tests/0013/tokens.txt b/tests/0013/tokens.txt new file mode 100644 index 00000000..c78b5eff --- /dev/null +++ b/tests/0013/tokens.txt @@ -0,0 +1,420 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Use, + span: ( + 3, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\", + ), + span: ( + 3, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 9, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: Comma, + span: ( + 3, + 13, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 3, + 15, + ), + }, + Token { + kind: As, + span: ( + 3, + 19, + ), + }, + Token { + kind: Identifier( + "Bob", + ), + span: ( + 3, + 22, + ), + }, + Token { + kind: Comma, + span: ( + 3, + 25, + ), + }, + Token { + kind: Identifier( + "Car", + ), + span: ( + 3, + 27, + ), + }, + Token { + kind: RightBrace, + span: ( + 3, + 30, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 31, + ), + }, + Token { + kind: Use, + span: ( + 4, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Bar\", + ), + span: ( + 4, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 9, + ), + }, + Token { + kind: Identifier( + "Bar0", + ), + span: ( + 4, + 10, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 14, + ), + }, + Token { + kind: Identifier( + "Baz0", + ), + span: ( + 4, + 16, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 20, + ), + }, + Token { + kind: Identifier( + "Car0", + ), + span: ( + 4, + 22, + ), + }, + Token { + kind: RightBrace, + span: ( + 4, + 26, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 27, + ), + }, + Token { + kind: Use, + span: ( + 5, + 1, + ), + }, + Token { + kind: Identifier( + "Foo1", + ), + span: ( + 5, + 5, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "Bar1", + ), + span: ( + 5, + 11, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 15, + ), + }, + Token { + kind: Identifier( + "Baz1", + ), + span: ( + 5, + 17, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 21, + ), + }, + Token { + kind: Use, + span: ( + 6, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 6, + 5, + ), + }, + Token { + kind: As, + span: ( + 6, + 9, + ), + }, + Token { + kind: Identifier( + "Qux", + ), + span: ( + 6, + 12, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 15, + ), + }, + Token { + kind: Use, + span: ( + 7, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 7, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 8, + ), + }, + Token { + kind: Use, + span: ( + 9, + 1, + ), + }, + Token { + kind: Const, + span: ( + 9, + 5, + ), + }, + Token { + kind: Identifier( + "FOO", + ), + span: ( + 9, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 14, + ), + }, + Token { + kind: Use, + span: ( + 10, + 1, + ), + }, + Token { + kind: Const, + span: ( + 10, + 5, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\Bar\Baz\QUX", + ), + span: ( + 10, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 10, + 26, + ), + }, + Token { + kind: Use, + span: ( + 12, + 1, + ), + }, + Token { + kind: Function, + span: ( + 12, + 5, + ), + }, + Token { + kind: Identifier( + "f", + ), + span: ( + 12, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 12, + 15, + ), + }, + Token { + kind: Use, + span: ( + 13, + 1, + ), + }, + Token { + kind: Const, + span: ( + 13, + 5, + ), + }, + Token { + kind: QualifiedIdentifier( + "Pop\Bar\f", + ), + span: ( + 13, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 13, + 20, + ), + }, +] diff --git a/tests/0014/tokens.txt b/tests/0014/tokens.txt new file mode 100644 index 00000000..9cb194fd --- /dev/null +++ b/tests/0014/tokens.txt @@ -0,0 +1,434 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo2", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 12, + ), + }, + Token { + kind: Use, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "B", + ), + span: ( + 4, + 9, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 11, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: As, + span: ( + 5, + 13, + ), + }, + Token { + kind: Protected, + span: ( + 5, + 16, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 5, + 26, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 29, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, + Token { + kind: Class, + span: ( + 9, + 1, + ), + }, + Token { + kind: Identifier( + "Bar2", + ), + span: ( + 9, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 9, + 12, + ), + }, + Token { + kind: Use, + span: ( + 10, + 5, + ), + }, + Token { + kind: Identifier( + "B", + ), + span: ( + 10, + 9, + ), + }, + Token { + kind: Comma, + span: ( + 10, + 10, + ), + }, + Token { + kind: Identifier( + "C", + ), + span: ( + 11, + 9, + ), + }, + Token { + kind: LeftBrace, + span: ( + 11, + 11, + ), + }, + Token { + kind: Identifier( + "B", + ), + span: ( + 12, + 13, + ), + }, + Token { + kind: DoubleColon, + span: ( + 12, + 14, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 12, + 16, + ), + }, + Token { + kind: Insteadof, + span: ( + 12, + 20, + ), + }, + Token { + kind: Identifier( + "C", + ), + span: ( + 12, + 30, + ), + }, + Token { + kind: SemiColon, + span: ( + 12, + 31, + ), + }, + Token { + kind: RightBrace, + span: ( + 13, + 9, + ), + }, + Token { + kind: RightBrace, + span: ( + 14, + 1, + ), + }, + Token { + kind: Class, + span: ( + 16, + 1, + ), + }, + Token { + kind: Identifier( + "Bar3", + ), + span: ( + 16, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 16, + 12, + ), + }, + Token { + kind: Use, + span: ( + 17, + 5, + ), + }, + Token { + kind: Identifier( + "B", + ), + span: ( + 17, + 9, + ), + }, + Token { + kind: LeftBrace, + span: ( + 17, + 11, + ), + }, + Token { + kind: Identifier( + "B", + ), + span: ( + 17, + 13, + ), + }, + Token { + kind: DoubleColon, + span: ( + 17, + 14, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 17, + 16, + ), + }, + Token { + kind: As, + span: ( + 17, + 20, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 17, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 17, + 26, + ), + }, + Token { + kind: RightBrace, + span: ( + 17, + 28, + ), + }, + Token { + kind: RightBrace, + span: ( + 18, + 1, + ), + }, + Token { + kind: Class, + span: ( + 20, + 1, + ), + }, + Token { + kind: Identifier( + "Bar4", + ), + span: ( + 20, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 20, + 12, + ), + }, + Token { + kind: Use, + span: ( + 21, + 5, + ), + }, + Token { + kind: Identifier( + "B", + ), + span: ( + 21, + 9, + ), + }, + Token { + kind: LeftBrace, + span: ( + 21, + 11, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 21, + 13, + ), + }, + Token { + kind: As, + span: ( + 21, + 17, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 21, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 21, + 23, + ), + }, + Token { + kind: RightBrace, + span: ( + 21, + 25, + ), + }, + Token { + kind: RightBrace, + span: ( + 22, + 1, + ), + }, +] diff --git a/tests/0015/tokens.txt b/tests/0015/tokens.txt new file mode 100644 index 00000000..da4378b6 --- /dev/null +++ b/tests/0015/tokens.txt @@ -0,0 +1,230 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Declare, + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 8, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 3, + 9, + ), + }, + Token { + kind: Equals, + span: ( + 3, + 10, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 12, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 7, + ), + }, + Token { + kind: EndDeclare, + span: ( + 5, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 11, + ), + }, + Token { + kind: Declare, + span: ( + 7, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 8, + ), + }, + Token { + kind: Identifier( + "b", + ), + span: ( + 7, + 9, + ), + }, + Token { + kind: Equals, + span: ( + 7, + 10, + ), + }, + Token { + kind: LiteralString( + "9", + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 14, + ), + }, + Token { + kind: Colon, + span: ( + 7, + 15, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 8, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 7, + ), + }, + Token { + kind: EndDeclare, + span: ( + 9, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 11, + ), + }, + Token { + kind: Declare, + span: ( + 11, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 11, + 8, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 11, + 9, + ), + }, + Token { + kind: Equals, + span: ( + 11, + 12, + ), + }, + Token { + kind: LiteralFloat( + 1.42, + ), + span: ( + 11, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 11, + 17, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 18, + ), + }, +] diff --git a/tests/0016/tokens.txt b/tests/0016/tokens.txt new file mode 100644 index 00000000..7e058e9a --- /dev/null +++ b/tests/0016/tokens.txt @@ -0,0 +1,78 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Declare, + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 8, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: Equals, + span: ( + 3, + 14, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 3, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 23, + ), + }, +] diff --git a/tests/0017/tokens.txt b/tests/0017/tokens.txt new file mode 100644 index 00000000..dff611e7 --- /dev/null +++ b/tests/0017/tokens.txt @@ -0,0 +1,220 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 3, + 4, + ), + }, + Token { + kind: New, + span: ( + 3, + 6, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 15, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 4, + ), + }, + Token { + kind: Plus, + span: ( + 4, + 6, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 4, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 8, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 5, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 5, + 4, + ), + }, + Token { + kind: BitwiseNot, + span: ( + 5, + 6, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 5, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 8, + ), + }, + Token { + kind: Variable( + "d", + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 6, + 4, + ), + }, + Token { + kind: Decrement, + span: ( + 6, + 6, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 6, + 8, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 10, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 7, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 7, + 4, + ), + }, + Token { + kind: Increment, + span: ( + 7, + 6, + ), + }, + Token { + kind: Variable( + "d", + ), + span: ( + 7, + 8, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 10, + ), + }, +] diff --git a/tests/0018/tokens.txt b/tests/0018/tokens.txt new file mode 100644 index 00000000..7445a347 --- /dev/null +++ b/tests/0018/tokens.txt @@ -0,0 +1,235 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 11, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 12, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 13, + ), + }, + Token { + kind: Null, + span: ( + 3, + 15, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 20, + ), + }, + Token { + kind: Echo, + span: ( + 4, + 5, + ), + }, + Token { + kind: LiteralString( + "looping..\n", + ), + span: ( + 4, + 10, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 23, + ), + }, + Token { + kind: Return, + span: ( + 6, + 5, + ), + }, + Token { + kind: Null, + span: ( + 6, + 12, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 9, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 9, + 6, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 9, + 8, + ), + }, + Token { + kind: LeftParen, + span: ( + 9, + 9, + ), + }, + Token { + kind: Ellipsis, + span: ( + 9, + 10, + ), + }, + Token { + kind: RightParen, + span: ( + 9, + 13, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 14, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 11, + 1, + ), + }, + Token { + kind: Colon, + span: ( + 11, + 4, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 12, + 5, + ), + }, + Token { + kind: LeftParen, + span: ( + 12, + 9, + ), + }, + Token { + kind: RightParen, + span: ( + 12, + 10, + ), + }, + Token { + kind: SemiColon, + span: ( + 12, + 11, + ), + }, + Token { + kind: Goto, + span: ( + 13, + 5, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 13, + 10, + ), + }, + Token { + kind: SemiColon, + span: ( + 13, + 13, + ), + }, +] diff --git a/tests/0019/tokens.txt b/tests/0019/tokens.txt new file mode 100644 index 00000000..674e68ef --- /dev/null +++ b/tests/0019/tokens.txt @@ -0,0 +1,1601 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Function, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "globalFunc", + ), + span: ( + 4, + 14, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 24, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 25, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 27, + ), + }, + Token { + kind: RightBrace, + span: ( + 4, + 28, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 7, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 7, + 15, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 8, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 8, + 8, + ), + }, + Token { + kind: Function, + span: ( + 8, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 8, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 8, + 20, + ), + }, + Token { + kind: LeftBrace, + span: ( + 8, + 22, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 24, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 9, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 9, + 8, + ), + }, + Token { + kind: Function, + span: ( + 9, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 9, + 19, + ), + }, + Token { + kind: Ampersand, + span: ( + 9, + 20, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 9, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 9, + 23, + ), + }, + Token { + kind: LeftBrace, + span: ( + 9, + 25, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 26, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 27, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 10, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 10, + 8, + ), + }, + Token { + kind: Function, + span: ( + 10, + 10, + ), + }, + Token { + kind: Ampersand, + span: ( + 10, + 19, + ), + }, + Token { + kind: LeftParen, + span: ( + 10, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 10, + 21, + ), + }, + Token { + kind: LeftBrace, + span: ( + 10, + 23, + ), + }, + Token { + kind: RightBrace, + span: ( + 10, + 24, + ), + }, + Token { + kind: SemiColon, + span: ( + 10, + 25, + ), + }, + Token { + kind: Variable( + "d", + ), + span: ( + 11, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 11, + 8, + ), + }, + Token { + kind: Function, + span: ( + 11, + 10, + ), + }, + Token { + kind: Ampersand, + span: ( + 11, + 19, + ), + }, + Token { + kind: LeftParen, + span: ( + 11, + 20, + ), + }, + Token { + kind: Ampersand, + span: ( + 11, + 21, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 11, + 22, + ), + }, + Token { + kind: RightParen, + span: ( + 11, + 24, + ), + }, + Token { + kind: LeftBrace, + span: ( + 11, + 26, + ), + }, + Token { + kind: Return, + span: ( + 11, + 28, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 11, + 35, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 37, + ), + }, + Token { + kind: RightBrace, + span: ( + 11, + 39, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 40, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 12, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 12, + 8, + ), + }, + Token { + kind: Fn, + span: ( + 12, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 12, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 12, + 14, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 12, + 16, + ), + }, + Token { + kind: Null, + span: ( + 12, + 19, + ), + }, + Token { + kind: SemiColon, + span: ( + 12, + 23, + ), + }, + Token { + kind: Variable( + "f", + ), + span: ( + 13, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 13, + 8, + ), + }, + Token { + kind: Fn, + span: ( + 13, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 13, + 13, + ), + }, + Token { + kind: Ampersand, + span: ( + 13, + 14, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 13, + 15, + ), + }, + Token { + kind: RightParen, + span: ( + 13, + 17, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 13, + 19, + ), + }, + Token { + kind: Null, + span: ( + 13, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 13, + 26, + ), + }, + Token { + kind: Variable( + "g", + ), + span: ( + 14, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 14, + 8, + ), + }, + Token { + kind: Fn, + span: ( + 14, + 10, + ), + }, + Token { + kind: Ampersand, + span: ( + 14, + 13, + ), + }, + Token { + kind: LeftParen, + span: ( + 14, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 14, + 15, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 14, + 17, + ), + }, + Token { + kind: Null, + span: ( + 14, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 14, + 24, + ), + }, + Token { + kind: Variable( + "h", + ), + span: ( + 15, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 15, + 8, + ), + }, + Token { + kind: Fn, + span: ( + 15, + 10, + ), + }, + Token { + kind: Ampersand, + span: ( + 15, + 13, + ), + }, + Token { + kind: LeftParen, + span: ( + 15, + 14, + ), + }, + Token { + kind: Ampersand, + span: ( + 15, + 15, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 15, + 16, + ), + }, + Token { + kind: RightParen, + span: ( + 15, + 18, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 15, + 20, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 15, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 15, + 25, + ), + }, + Token { + kind: RightBrace, + span: ( + 16, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 18, + 1, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 18, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 18, + 15, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 19, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 19, + 8, + ), + }, + Token { + kind: Static, + span: ( + 19, + 10, + ), + }, + Token { + kind: Function, + span: ( + 19, + 17, + ), + }, + Token { + kind: LeftParen, + span: ( + 19, + 26, + ), + }, + Token { + kind: RightParen, + span: ( + 19, + 27, + ), + }, + Token { + kind: LeftBrace, + span: ( + 19, + 29, + ), + }, + Token { + kind: RightBrace, + span: ( + 19, + 30, + ), + }, + Token { + kind: SemiColon, + span: ( + 19, + 31, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 20, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 20, + 8, + ), + }, + Token { + kind: Static, + span: ( + 20, + 10, + ), + }, + Token { + kind: Function, + span: ( + 20, + 17, + ), + }, + Token { + kind: LeftParen, + span: ( + 20, + 26, + ), + }, + Token { + kind: Ampersand, + span: ( + 20, + 27, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 20, + 28, + ), + }, + Token { + kind: RightParen, + span: ( + 20, + 30, + ), + }, + Token { + kind: LeftBrace, + span: ( + 20, + 32, + ), + }, + Token { + kind: RightBrace, + span: ( + 20, + 33, + ), + }, + Token { + kind: SemiColon, + span: ( + 20, + 34, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 21, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 21, + 8, + ), + }, + Token { + kind: Static, + span: ( + 21, + 10, + ), + }, + Token { + kind: Function, + span: ( + 21, + 17, + ), + }, + Token { + kind: Ampersand, + span: ( + 21, + 26, + ), + }, + Token { + kind: LeftParen, + span: ( + 21, + 27, + ), + }, + Token { + kind: RightParen, + span: ( + 21, + 28, + ), + }, + Token { + kind: LeftBrace, + span: ( + 21, + 30, + ), + }, + Token { + kind: RightBrace, + span: ( + 21, + 31, + ), + }, + Token { + kind: SemiColon, + span: ( + 21, + 32, + ), + }, + Token { + kind: Variable( + "d", + ), + span: ( + 22, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 22, + 8, + ), + }, + Token { + kind: Static, + span: ( + 22, + 10, + ), + }, + Token { + kind: Function, + span: ( + 22, + 17, + ), + }, + Token { + kind: Ampersand, + span: ( + 22, + 26, + ), + }, + Token { + kind: LeftParen, + span: ( + 22, + 27, + ), + }, + Token { + kind: Ampersand, + span: ( + 22, + 28, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 22, + 29, + ), + }, + Token { + kind: RightParen, + span: ( + 22, + 31, + ), + }, + Token { + kind: LeftBrace, + span: ( + 22, + 33, + ), + }, + Token { + kind: Return, + span: ( + 22, + 35, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 22, + 42, + ), + }, + Token { + kind: SemiColon, + span: ( + 22, + 44, + ), + }, + Token { + kind: RightBrace, + span: ( + 22, + 46, + ), + }, + Token { + kind: SemiColon, + span: ( + 22, + 47, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 23, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 23, + 8, + ), + }, + Token { + kind: Static, + span: ( + 23, + 10, + ), + }, + Token { + kind: Fn, + span: ( + 23, + 17, + ), + }, + Token { + kind: LeftParen, + span: ( + 23, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 23, + 21, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 23, + 23, + ), + }, + Token { + kind: Null, + span: ( + 23, + 26, + ), + }, + Token { + kind: SemiColon, + span: ( + 23, + 30, + ), + }, + Token { + kind: Variable( + "f", + ), + span: ( + 24, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 24, + 8, + ), + }, + Token { + kind: Static, + span: ( + 24, + 10, + ), + }, + Token { + kind: Fn, + span: ( + 24, + 17, + ), + }, + Token { + kind: LeftParen, + span: ( + 24, + 20, + ), + }, + Token { + kind: Ampersand, + span: ( + 24, + 21, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 24, + 22, + ), + }, + Token { + kind: RightParen, + span: ( + 24, + 24, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 24, + 26, + ), + }, + Token { + kind: Null, + span: ( + 24, + 29, + ), + }, + Token { + kind: SemiColon, + span: ( + 24, + 33, + ), + }, + Token { + kind: Variable( + "g", + ), + span: ( + 25, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 25, + 8, + ), + }, + Token { + kind: Static, + span: ( + 25, + 10, + ), + }, + Token { + kind: Fn, + span: ( + 25, + 17, + ), + }, + Token { + kind: Ampersand, + span: ( + 25, + 20, + ), + }, + Token { + kind: LeftParen, + span: ( + 25, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 25, + 22, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 25, + 24, + ), + }, + Token { + kind: Null, + span: ( + 25, + 27, + ), + }, + Token { + kind: SemiColon, + span: ( + 25, + 31, + ), + }, + Token { + kind: Variable( + "h", + ), + span: ( + 26, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 26, + 8, + ), + }, + Token { + kind: Static, + span: ( + 26, + 10, + ), + }, + Token { + kind: Fn, + span: ( + 26, + 17, + ), + }, + Token { + kind: Ampersand, + span: ( + 26, + 20, + ), + }, + Token { + kind: LeftParen, + span: ( + 26, + 21, + ), + }, + Token { + kind: Ampersand, + span: ( + 26, + 22, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 26, + 23, + ), + }, + Token { + kind: RightParen, + span: ( + 26, + 25, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 26, + 27, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 26, + 30, + ), + }, + Token { + kind: SemiColon, + span: ( + 26, + 32, + ), + }, + Token { + kind: RightBrace, + span: ( + 27, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 29, + 1, + ), + }, + Token { + kind: Identifier( + "baz", + ), + span: ( + 29, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 29, + 15, + ), + }, + Token { + kind: Function, + span: ( + 30, + 5, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 30, + 14, + ), + }, + Token { + kind: LeftParen, + span: ( + 30, + 15, + ), + }, + Token { + kind: Ampersand, + span: ( + 30, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 30, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 30, + 19, + ), + }, + Token { + kind: LeftBrace, + span: ( + 30, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 30, + 22, + ), + }, + Token { + kind: Function, + span: ( + 31, + 5, + ), + }, + Token { + kind: Ampersand, + span: ( + 31, + 14, + ), + }, + Token { + kind: Identifier( + "b", + ), + span: ( + 31, + 15, + ), + }, + Token { + kind: LeftParen, + span: ( + 31, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 31, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 31, + 19, + ), + }, + Token { + kind: LeftBrace, + span: ( + 31, + 21, + ), + }, + Token { + kind: Return, + span: ( + 31, + 23, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 31, + 30, + ), + }, + Token { + kind: SemiColon, + span: ( + 31, + 32, + ), + }, + Token { + kind: RightBrace, + span: ( + 31, + 34, + ), + }, + Token { + kind: Function, + span: ( + 32, + 5, + ), + }, + Token { + kind: Ampersand, + span: ( + 32, + 14, + ), + }, + Token { + kind: Identifier( + "c", + ), + span: ( + 32, + 15, + ), + }, + Token { + kind: LeftParen, + span: ( + 32, + 16, + ), + }, + Token { + kind: RightParen, + span: ( + 32, + 17, + ), + }, + Token { + kind: LeftBrace, + span: ( + 32, + 19, + ), + }, + Token { + kind: Return, + span: ( + 32, + 21, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 32, + 28, + ), + }, + Token { + kind: SemiColon, + span: ( + 32, + 30, + ), + }, + Token { + kind: RightBrace, + span: ( + 32, + 32, + ), + }, + Token { + kind: RightBrace, + span: ( + 33, + 1, + ), + }, +] diff --git a/tests/0020/tokens.txt b/tests/0020/tokens.txt new file mode 100644 index 00000000..cd5bd21f --- /dev/null +++ b/tests/0020/tokens.txt @@ -0,0 +1,811 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: Percent, + span: ( + 3, + 3, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 3, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 6, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: LeftShift, + span: ( + 4, + 3, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 4, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 7, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 5, + 1, + ), + }, + Token { + kind: RightShift, + span: ( + 5, + 3, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 5, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 7, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: Ampersand, + span: ( + 6, + 3, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 6, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 6, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 7, + 1, + ), + }, + Token { + kind: Pipe, + span: ( + 7, + 3, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 7, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 6, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 8, + 1, + ), + }, + Token { + kind: Caret, + span: ( + 8, + 3, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 8, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 6, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 9, + 1, + ), + }, + Token { + kind: AngledLeftRight, + span: ( + 9, + 3, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 9, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 7, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 10, + 1, + ), + }, + Token { + kind: Spaceship, + span: ( + 10, + 3, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 10, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 10, + 8, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 11, + 1, + ), + }, + Token { + kind: LogicalAnd, + span: ( + 11, + 3, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 11, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 8, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 12, + 1, + ), + }, + Token { + kind: LogicalOr, + span: ( + 12, + 3, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 12, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 12, + 7, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 13, + 1, + ), + }, + Token { + kind: LogicalXor, + span: ( + 13, + 3, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 13, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 13, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 14, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 14, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 14, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 14, + 7, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 15, + 1, + ), + }, + Token { + kind: PlusEquals, + span: ( + 15, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 15, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 15, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 16, + 1, + ), + }, + Token { + kind: MinusEquals, + span: ( + 16, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 16, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 16, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 17, + 1, + ), + }, + Token { + kind: AsteriskEqual, + span: ( + 17, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 17, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 17, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 18, + 1, + ), + }, + Token { + kind: PowEquals, + span: ( + 18, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 18, + 8, + ), + }, + Token { + kind: SemiColon, + span: ( + 18, + 9, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 19, + 1, + ), + }, + Token { + kind: SlashEquals, + span: ( + 19, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 19, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 19, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 20, + 1, + ), + }, + Token { + kind: DotEquals, + span: ( + 20, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 20, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 20, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 21, + 1, + ), + }, + Token { + kind: PercentEquals, + span: ( + 21, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 21, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 21, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 22, + 1, + ), + }, + Token { + kind: AmpersandEquals, + span: ( + 22, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 22, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 22, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 23, + 1, + ), + }, + Token { + kind: PipeEquals, + span: ( + 23, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 23, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 23, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 24, + 1, + ), + }, + Token { + kind: CaretEquals, + span: ( + 24, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 24, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 24, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 25, + 1, + ), + }, + Token { + kind: LeftShiftEquals, + span: ( + 25, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 25, + 8, + ), + }, + Token { + kind: SemiColon, + span: ( + 25, + 9, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 26, + 1, + ), + }, + Token { + kind: RightShiftEquals, + span: ( + 26, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 26, + 8, + ), + }, + Token { + kind: SemiColon, + span: ( + 26, + 9, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 27, + 1, + ), + }, + Token { + kind: CoalesceEqual, + span: ( + 27, + 4, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 27, + 8, + ), + }, + Token { + kind: SemiColon, + span: ( + 27, + 9, + ), + }, +] diff --git a/tests/0021/tokens.txt b/tests/0021/tokens.txt new file mode 100644 index 00000000..5399aed8 --- /dev/null +++ b/tests/0021/tokens.txt @@ -0,0 +1,552 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: If, + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 4, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 5, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 7, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 7, + ), + }, + Token { + kind: EndIf, + span: ( + 5, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 6, + ), + }, + Token { + kind: If, + span: ( + 7, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 4, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 7, + 5, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 7, + ), + }, + Token { + kind: Colon, + span: ( + 7, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 8, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 7, + ), + }, + Token { + kind: Else, + span: ( + 9, + 1, + ), + }, + Token { + kind: Colon, + span: ( + 9, + 5, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 10, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 10, + 7, + ), + }, + Token { + kind: EndIf, + span: ( + 11, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 6, + ), + }, + Token { + kind: If, + span: ( + 13, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 13, + 4, + ), + }, + Token { + kind: True, + span: ( + 13, + 5, + ), + }, + Token { + kind: RightParen, + span: ( + 13, + 9, + ), + }, + Token { + kind: Colon, + span: ( + 13, + 10, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 14, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 14, + 7, + ), + }, + Token { + kind: ElseIf, + span: ( + 15, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 15, + 8, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 15, + 9, + ), + }, + Token { + kind: Arrow, + span: ( + 15, + 13, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 15, + 15, + ), + }, + Token { + kind: LeftParen, + span: ( + 15, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 15, + 19, + ), + }, + Token { + kind: BooleanAnd, + span: ( + 15, + 21, + ), + }, + Token { + kind: Variable( + "baz", + ), + span: ( + 15, + 24, + ), + }, + Token { + kind: Arrow, + span: ( + 15, + 28, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 15, + 30, + ), + }, + Token { + kind: NullsafeArrow, + span: ( + 15, + 33, + ), + }, + Token { + kind: Identifier( + "qux", + ), + span: ( + 15, + 36, + ), + }, + Token { + kind: LeftParen, + span: ( + 15, + 39, + ), + }, + Token { + kind: RightParen, + span: ( + 15, + 40, + ), + }, + Token { + kind: RightParen, + span: ( + 15, + 41, + ), + }, + Token { + kind: Colon, + span: ( + 15, + 42, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 16, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 16, + 7, + ), + }, + Token { + kind: EndIf, + span: ( + 17, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 17, + 6, + ), + }, + Token { + kind: If, + span: ( + 19, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 19, + 4, + ), + }, + Token { + kind: True, + span: ( + 19, + 5, + ), + }, + Token { + kind: RightParen, + span: ( + 19, + 9, + ), + }, + Token { + kind: Colon, + span: ( + 19, + 10, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 20, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 20, + 7, + ), + }, + Token { + kind: ElseIf, + span: ( + 21, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 21, + 8, + ), + }, + Token { + kind: True, + span: ( + 21, + 9, + ), + }, + Token { + kind: RightParen, + span: ( + 21, + 13, + ), + }, + Token { + kind: Colon, + span: ( + 21, + 14, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 22, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 22, + 7, + ), + }, + Token { + kind: ElseIf, + span: ( + 23, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 23, + 8, + ), + }, + Token { + kind: True, + span: ( + 23, + 9, + ), + }, + Token { + kind: RightParen, + span: ( + 23, + 13, + ), + }, + Token { + kind: Colon, + span: ( + 23, + 14, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 24, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 24, + 7, + ), + }, + Token { + kind: EndIf, + span: ( + 25, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 25, + 6, + ), + }, +] diff --git a/tests/0022/tokens.txt b/tests/0022/tokens.txt new file mode 100644 index 00000000..34bd8b78 --- /dev/null +++ b/tests/0022/tokens.txt @@ -0,0 +1,265 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Foreach, + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 9, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: As, + span: ( + 3, + 13, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 3, + 16, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 18, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 19, + ), + }, + Token { + kind: Echo, + span: ( + 4, + 5, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 10, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 12, + ), + }, + Token { + kind: EndForeach, + span: ( + 5, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 11, + ), + }, + Token { + kind: While, + span: ( + 7, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 7, + ), + }, + Token { + kind: True, + span: ( + 7, + 8, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 12, + ), + }, + Token { + kind: Colon, + span: ( + 7, + 13, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 8, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 7, + ), + }, + Token { + kind: EndWhile, + span: ( + 9, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 9, + ), + }, + Token { + kind: For, + span: ( + 11, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 11, + 5, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 11, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 8, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 11, + 10, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 12, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 11, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 11, + 16, + ), + }, + Token { + kind: Colon, + span: ( + 11, + 17, + ), + }, + Token { + kind: Variable( + "d", + ), + span: ( + 12, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 12, + 7, + ), + }, + Token { + kind: EndFor, + span: ( + 13, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 13, + 7, + ), + }, +] diff --git a/tests/0023/tokens.txt b/tests/0023/tokens.txt new file mode 100644 index 00000000..ede73488 --- /dev/null +++ b/tests/0023/tokens.txt @@ -0,0 +1,505 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Foreach, + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 9, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: As, + span: ( + 3, + 13, + ), + }, + Token { + kind: Ampersand, + span: ( + 3, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 3, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 19, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 3, + 22, + ), + }, + Token { + kind: Foreach, + span: ( + 5, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 9, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 5, + 10, + ), + }, + Token { + kind: As, + span: ( + 5, + 13, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 5, + 19, + ), + }, + Token { + kind: Ampersand, + span: ( + 5, + 22, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 5, + 23, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 25, + ), + }, + Token { + kind: LeftBrace, + span: ( + 5, + 27, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 28, + ), + }, + Token { + kind: Switch, + span: ( + 7, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 8, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 7, + 9, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 7, + 13, + ), + }, + Token { + kind: Case, + span: ( + 8, + 5, + ), + }, + Token { + kind: LiteralInteger( + 0, + ), + span: ( + 8, + 10, + ), + }, + Token { + kind: Colon, + span: ( + 8, + 11, + ), + }, + Token { + kind: Break, + span: ( + 9, + 9, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 14, + ), + }, + Token { + kind: Case, + span: ( + 10, + 5, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 10, + 10, + ), + }, + Token { + kind: SemiColon, + span: ( + 10, + 11, + ), + }, + Token { + kind: Default, + span: ( + 11, + 5, + ), + }, + Token { + kind: Colon, + span: ( + 11, + 12, + ), + }, + Token { + kind: RightBrace, + span: ( + 12, + 1, + ), + }, + Token { + kind: Foreach, + span: ( + 14, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 14, + 9, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 14, + 10, + ), + }, + Token { + kind: As, + span: ( + 14, + 15, + ), + }, + Token { + kind: LeftBracket, + span: ( + 14, + 18, + ), + }, + Token { + kind: Variable( + "baz", + ), + span: ( + 14, + 19, + ), + }, + Token { + kind: Comma, + span: ( + 14, + 23, + ), + }, + Token { + kind: Variable( + "car", + ), + span: ( + 14, + 25, + ), + }, + Token { + kind: RightBracket, + span: ( + 14, + 29, + ), + }, + Token { + kind: RightParen, + span: ( + 14, + 30, + ), + }, + Token { + kind: LeftBrace, + span: ( + 14, + 32, + ), + }, + Token { + kind: RightBrace, + span: ( + 14, + 33, + ), + }, + Token { + kind: Foreach, + span: ( + 16, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 16, + 9, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 16, + 10, + ), + }, + Token { + kind: As, + span: ( + 16, + 15, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 16, + 18, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 16, + 23, + ), + }, + Token { + kind: Variable( + "baz", + ), + span: ( + 16, + 26, + ), + }, + Token { + kind: RightParen, + span: ( + 16, + 30, + ), + }, + Token { + kind: LeftBrace, + span: ( + 16, + 32, + ), + }, + Token { + kind: RightBrace, + span: ( + 16, + 33, + ), + }, + Token { + kind: Foreach, + span: ( + 18, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 18, + 9, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 18, + 10, + ), + }, + Token { + kind: As, + span: ( + 18, + 15, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 18, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 18, + 22, + ), + }, + Token { + kind: LeftBrace, + span: ( + 18, + 24, + ), + }, + Token { + kind: RightBrace, + span: ( + 18, + 25, + ), + }, +] diff --git a/tests/0024/tokens.txt b/tests/0024/tokens.txt new file mode 100644 index 00000000..1cdb16c0 --- /dev/null +++ b/tests/0024/tokens.txt @@ -0,0 +1,414 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Try, + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, + Token { + kind: Catch, + span: ( + 5, + 3, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "Exception", + ), + span: ( + 5, + 10, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 5, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 22, + ), + }, + Token { + kind: LeftBrace, + span: ( + 5, + 24, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, + Token { + kind: Catch, + span: ( + 7, + 3, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 9, + ), + }, + Token { + kind: Identifier( + "CustomException", + ), + span: ( + 7, + 10, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 7, + 26, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 28, + ), + }, + Token { + kind: LeftBrace, + span: ( + 7, + 30, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, + Token { + kind: Try, + span: ( + 11, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 11, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 13, + 1, + ), + }, + Token { + kind: Catch, + span: ( + 13, + 3, + ), + }, + Token { + kind: LeftParen, + span: ( + 13, + 9, + ), + }, + Token { + kind: Identifier( + "Exception", + ), + span: ( + 13, + 10, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 13, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 13, + 22, + ), + }, + Token { + kind: LeftBrace, + span: ( + 13, + 24, + ), + }, + Token { + kind: RightBrace, + span: ( + 15, + 1, + ), + }, + Token { + kind: Finally, + span: ( + 15, + 3, + ), + }, + Token { + kind: LeftBrace, + span: ( + 15, + 11, + ), + }, + Token { + kind: RightBrace, + span: ( + 17, + 1, + ), + }, + Token { + kind: Try, + span: ( + 19, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 19, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 21, + 1, + ), + }, + Token { + kind: Finally, + span: ( + 21, + 3, + ), + }, + Token { + kind: LeftBrace, + span: ( + 21, + 11, + ), + }, + Token { + kind: RightBrace, + span: ( + 21, + 12, + ), + }, + Token { + kind: Try, + span: ( + 23, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 23, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 25, + 1, + ), + }, + Token { + kind: Catch, + span: ( + 25, + 3, + ), + }, + Token { + kind: LeftParen, + span: ( + 25, + 9, + ), + }, + Token { + kind: Identifier( + "Exception", + ), + span: ( + 25, + 10, + ), + }, + Token { + kind: RightParen, + span: ( + 25, + 19, + ), + }, + Token { + kind: LeftBrace, + span: ( + 25, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 27, + 1, + ), + }, + Token { + kind: Try, + span: ( + 29, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 29, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 31, + 1, + ), + }, + Token { + kind: Catch, + span: ( + 31, + 3, + ), + }, + Token { + kind: LeftParen, + span: ( + 31, + 9, + ), + }, + Token { + kind: Identifier( + "Exception", + ), + span: ( + 31, + 10, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 31, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 31, + 22, + ), + }, + Token { + kind: LeftBrace, + span: ( + 31, + 24, + ), + }, + Token { + kind: RightBrace, + span: ( + 33, + 1, + ), + }, +] diff --git a/tests/0025/tokens.txt b/tests/0025/tokens.txt new file mode 100644 index 00000000..1370bdb2 --- /dev/null +++ b/tests/0025/tokens.txt @@ -0,0 +1,25 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Break, + span: ( + 1, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 12, + ), + }, +] diff --git a/tests/0026/tokens.txt b/tests/0026/tokens.txt new file mode 100644 index 00000000..f7b2e071 --- /dev/null +++ b/tests/0026/tokens.txt @@ -0,0 +1,34 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Break, + span: ( + 1, + 7, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 14, + ), + }, +] diff --git a/tests/0027/tokens.txt b/tests/0027/tokens.txt new file mode 100644 index 00000000..c5c03edd --- /dev/null +++ b/tests/0027/tokens.txt @@ -0,0 +1,25 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Continue, + span: ( + 1, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 15, + ), + }, +] diff --git a/tests/0028/tokens.txt b/tests/0028/tokens.txt new file mode 100644 index 00000000..dfd6b4b2 --- /dev/null +++ b/tests/0028/tokens.txt @@ -0,0 +1,34 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Continue, + span: ( + 1, + 7, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 17, + ), + }, +] diff --git a/tests/0029/tokens.txt b/tests/0029/tokens.txt new file mode 100644 index 00000000..17cb705a --- /dev/null +++ b/tests/0029/tokens.txt @@ -0,0 +1,91 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 11, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 16, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 18, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 22, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 1, + 24, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 27, + ), + }, + Token { + kind: Identifier( + "baz", + ), + span: ( + 1, + 29, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 32, + ), + }, +] diff --git a/tests/0030/tokens.txt b/tests/0030/tokens.txt new file mode 100644 index 00000000..09f6ae56 --- /dev/null +++ b/tests/0030/tokens.txt @@ -0,0 +1,193 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 11, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 16, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 17, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 18, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 24, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 1, + 26, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 29, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 30, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 31, + ), + }, + Token { + kind: Identifier( + "baz", + ), + span: ( + 1, + 33, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 36, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 37, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 38, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 40, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 44, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 1, + 46, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 49, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 50, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 51, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 52, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 53, + ), + }, +] diff --git a/tests/0031/tokens.txt b/tests/0031/tokens.txt new file mode 100644 index 00000000..ae701cbb --- /dev/null +++ b/tests/0031/tokens.txt @@ -0,0 +1,159 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: StringPart( + "", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 8, + ), + }, + Token { + kind: StringPart( + " abc ", + ), + span: ( + 1, + 12, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 1, + 12, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 21, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 1, + 23, + ), + }, + Token { + kind: StringPart( + " def ", + ), + span: ( + 1, + 24, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 1, + 24, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 33, + ), + }, + Token { + kind: LiteralInteger( + 0, + ), + span: ( + 1, + 34, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 35, + ), + }, + Token { + kind: StringPart( + " ghi ", + ), + span: ( + 1, + 36, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 1, + 36, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 45, + ), + }, + Token { + kind: Identifier( + "baz", + ), + span: ( + 1, + 46, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 49, + ), + }, + Token { + kind: DoubleQuote, + span: ( + 1, + 50, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 51, + ), + }, +] diff --git a/tests/0032/tokens.txt b/tests/0032/tokens.txt new file mode 100644 index 00000000..52e759fd --- /dev/null +++ b/tests/0032/tokens.txt @@ -0,0 +1,186 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: StringPart( + "", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: DollarLeftBrace, + span: ( + 1, + 8, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 10, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 13, + ), + }, + Token { + kind: DollarLeftBrace, + span: ( + 1, + 14, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 19, + ), + }, + Token { + kind: LiteralInteger( + 0, + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 22, + ), + }, + Token { + kind: DollarLeftBrace, + span: ( + 1, + 23, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 25, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 28, + ), + }, + Token { + kind: LiteralString( + "bar", + ), + span: ( + 1, + 29, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 34, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 35, + ), + }, + Token { + kind: DollarLeftBrace, + span: ( + 1, + 36, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 38, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 39, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 43, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 44, + ), + }, + Token { + kind: DoubleQuote, + span: ( + 1, + 45, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 46, + ), + }, +] diff --git a/tests/0033/tokens.txt b/tests/0033/tokens.txt new file mode 100644 index 00000000..17e5ae2a --- /dev/null +++ b/tests/0033/tokens.txt @@ -0,0 +1,241 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: StringPart( + "", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 8, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 9, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 13, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 14, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 15, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 19, + ), + }, + Token { + kind: LiteralInteger( + 0, + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 22, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 23, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 24, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 28, + ), + }, + Token { + kind: LiteralString( + "bar", + ), + span: ( + 1, + 29, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 34, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 35, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 36, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 37, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 41, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 1, + 43, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 46, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 47, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 48, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 52, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 1, + 54, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 57, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 58, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 59, + ), + }, + Token { + kind: DoubleQuote, + span: ( + 1, + 60, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 61, + ), + }, +] diff --git a/tests/0034/tokens.txt b/tests/0034/tokens.txt new file mode 100644 index 00000000..210443e2 --- /dev/null +++ b/tests/0034/tokens.txt @@ -0,0 +1,59 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: LiteralString( + "foo", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: Dot, + span: ( + 1, + 13, + ), + }, + Token { + kind: LiteralString( + "bar", + ), + span: ( + 1, + 15, + ), + }, + Token { + kind: Dot, + span: ( + 1, + 21, + ), + }, + Token { + kind: LiteralString( + "baz", + ), + span: ( + 1, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 28, + ), + }, +] diff --git a/tests/0035/tokens.txt b/tests/0035/tokens.txt new file mode 100644 index 00000000..d605f2f9 --- /dev/null +++ b/tests/0035/tokens.txt @@ -0,0 +1,55 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 20, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 22, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 23, + ), + }, +] diff --git a/tests/0036/tokens.txt b/tests/0036/tokens.txt new file mode 100644 index 00000000..abcb15ad --- /dev/null +++ b/tests/0036/tokens.txt @@ -0,0 +1,64 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Variable( + "n", + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 22, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 24, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 25, + ), + }, +] diff --git a/tests/0037/tokens.txt b/tests/0037/tokens.txt new file mode 100644 index 00000000..267408aa --- /dev/null +++ b/tests/0037/tokens.txt @@ -0,0 +1,80 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Variable( + "n", + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 22, + ), + }, + Token { + kind: Variable( + "m", + ), + span: ( + 1, + 24, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 26, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 28, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 29, + ), + }, +] diff --git a/tests/0038/tokens.txt b/tests/0038/tokens.txt new file mode 100644 index 00000000..60b47224 --- /dev/null +++ b/tests/0038/tokens.txt @@ -0,0 +1,264 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "fib", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: Variable( + "n", + ), + span: ( + 3, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 16, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 18, + ), + }, + Token { + kind: If, + span: ( + 4, + 5, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 8, + ), + }, + Token { + kind: Variable( + "n", + ), + span: ( + 4, + 9, + ), + }, + Token { + kind: LessThan, + span: ( + 4, + 12, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 4, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 15, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 17, + ), + }, + Token { + kind: Return, + span: ( + 5, + 9, + ), + }, + Token { + kind: Variable( + "n", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 18, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: Return, + span: ( + 8, + 5, + ), + }, + Token { + kind: Identifier( + "fib", + ), + span: ( + 8, + 12, + ), + }, + Token { + kind: LeftParen, + span: ( + 8, + 15, + ), + }, + Token { + kind: Variable( + "n", + ), + span: ( + 8, + 16, + ), + }, + Token { + kind: Minus, + span: ( + 8, + 19, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 8, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 8, + 22, + ), + }, + Token { + kind: Plus, + span: ( + 8, + 24, + ), + }, + Token { + kind: Identifier( + "fib", + ), + span: ( + 8, + 26, + ), + }, + Token { + kind: LeftParen, + span: ( + 8, + 29, + ), + }, + Token { + kind: Variable( + "n", + ), + span: ( + 8, + 30, + ), + }, + Token { + kind: Minus, + span: ( + 8, + 33, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 8, + 35, + ), + }, + Token { + kind: RightParen, + span: ( + 8, + 36, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 37, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, +] diff --git a/tests/0039/tokens.txt b/tests/0039/tokens.txt new file mode 100644 index 00000000..e7d8868a --- /dev/null +++ b/tests/0039/tokens.txt @@ -0,0 +1,64 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: If, + span: ( + 1, + 7, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 9, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 10, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 14, + ), + }, + Token { + kind: Return, + span: ( + 1, + 16, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 27, + ), + }, +] diff --git a/tests/0040/tokens.txt b/tests/0040/tokens.txt new file mode 100644 index 00000000..92b0e474 --- /dev/null +++ b/tests/0040/tokens.txt @@ -0,0 +1,122 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: If, + span: ( + 1, + 7, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 9, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 10, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 14, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 16, + ), + }, + Token { + kind: Return, + span: ( + 1, + 18, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 25, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 29, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 31, + ), + }, + Token { + kind: Else, + span: ( + 1, + 33, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 38, + ), + }, + Token { + kind: Return, + span: ( + 1, + 40, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 47, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 51, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 53, + ), + }, +] diff --git a/tests/0041/tokens.txt b/tests/0041/tokens.txt new file mode 100644 index 00000000..a53585cd --- /dev/null +++ b/tests/0041/tokens.txt @@ -0,0 +1,189 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: If, + span: ( + 1, + 7, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 9, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 10, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 14, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 16, + ), + }, + Token { + kind: Return, + span: ( + 1, + 18, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 25, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 29, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 31, + ), + }, + Token { + kind: ElseIf, + span: ( + 1, + 33, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 39, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 40, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 44, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 46, + ), + }, + Token { + kind: Return, + span: ( + 1, + 48, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 55, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 59, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 61, + ), + }, + Token { + kind: Else, + span: ( + 1, + 63, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 68, + ), + }, + Token { + kind: Return, + span: ( + 1, + 70, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 77, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 81, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 83, + ), + }, +] diff --git a/tests/0042/tokens.txt b/tests/0042/tokens.txt new file mode 100644 index 00000000..37a9effe --- /dev/null +++ b/tests/0042/tokens.txt @@ -0,0 +1,34 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Echo, + span: ( + 1, + 7, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 1, + 12, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 13, + ), + }, +] diff --git a/tests/0043/tokens.txt b/tests/0043/tokens.txt new file mode 100644 index 00000000..0773dadf --- /dev/null +++ b/tests/0043/tokens.txt @@ -0,0 +1,41 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 17, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 18, + ), + }, +] diff --git a/tests/0044/tokens.txt b/tests/0044/tokens.txt new file mode 100644 index 00000000..31248f1b --- /dev/null +++ b/tests/0044/tokens.txt @@ -0,0 +1,90 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 17, + ), + }, + Token { + kind: Public, + span: ( + 1, + 19, + ), + }, + Token { + kind: Fn, + span: ( + 1, + 26, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 28, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 29, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 31, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 32, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 33, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 35, + ), + }, +] diff --git a/tests/0045/tokens.txt b/tests/0045/tokens.txt new file mode 100644 index 00000000..8b5e07fe --- /dev/null +++ b/tests/0045/tokens.txt @@ -0,0 +1,108 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Function, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 4, + 14, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 18, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 20, + ), + }, + Token { + kind: Echo, + span: ( + 5, + 9, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 5, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 15, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0046/tokens.txt b/tests/0046/tokens.txt new file mode 100644 index 00000000..e1d9a5cb --- /dev/null +++ b/tests/0046/tokens.txt @@ -0,0 +1,57 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: Extends, + span: ( + 3, + 11, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 3, + 19, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 23, + ), + }, + Token { + kind: RightBrace, + span: ( + 3, + 24, + ), + }, +] diff --git a/tests/0047/tokens.txt b/tests/0047/tokens.txt new file mode 100644 index 00000000..9751c619 --- /dev/null +++ b/tests/0047/tokens.txt @@ -0,0 +1,73 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: Implements, + span: ( + 3, + 11, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 3, + 22, + ), + }, + Token { + kind: Comma, + span: ( + 3, + 25, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 3, + 27, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 31, + ), + }, + Token { + kind: RightBrace, + span: ( + 3, + 32, + ), + }, +] diff --git a/tests/0048/tokens.txt b/tests/0048/tokens.txt new file mode 100644 index 00000000..52490450 --- /dev/null +++ b/tests/0048/tokens.txt @@ -0,0 +1,73 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 1, + 27, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 29, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 31, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 32, + ), + }, +] diff --git a/tests/0049/tokens.txt b/tests/0049/tokens.txt new file mode 100644 index 00000000..3ff44d8f --- /dev/null +++ b/tests/0049/tokens.txt @@ -0,0 +1,121 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: Pipe, + span: ( + 1, + 26, + ), + }, + Token { + kind: Identifier( + "ArrAy", + ), + span: ( + 1, + 27, + ), + }, + Token { + kind: Pipe, + span: ( + 1, + 32, + ), + }, + Token { + kind: Identifier( + "iterable", + ), + span: ( + 1, + 33, + ), + }, + Token { + kind: Pipe, + span: ( + 1, + 41, + ), + }, + Token { + kind: Identifier( + "CALLABLE", + ), + span: ( + 1, + 42, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 1, + 51, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 53, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 55, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 56, + ), + }, +] diff --git a/tests/0050/tokens.txt b/tests/0050/tokens.txt new file mode 100644 index 00000000..4f6c0904 --- /dev/null +++ b/tests/0050/tokens.txt @@ -0,0 +1,71 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Ellipsis, + span: ( + 1, + 20, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 1, + 23, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 27, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 29, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 30, + ), + }, +] diff --git a/tests/0051/tokens.txt b/tests/0051/tokens.txt new file mode 100644 index 00000000..48af358e --- /dev/null +++ b/tests/0051/tokens.txt @@ -0,0 +1,80 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: Ellipsis, + span: ( + 1, + 27, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 1, + 30, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 34, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 36, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 37, + ), + }, +] diff --git a/tests/0052/tokens.txt b/tests/0052/tokens.txt new file mode 100644 index 00000000..80ae5fee --- /dev/null +++ b/tests/0052/tokens.txt @@ -0,0 +1,103 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 24, + ), + }, + Token { + kind: Variable( + "baz", + ), + span: ( + 1, + 26, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 30, + ), + }, + Token { + kind: Ellipsis, + span: ( + 1, + 32, + ), + }, + Token { + kind: Variable( + "car", + ), + span: ( + 1, + 35, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 39, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 41, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 42, + ), + }, +] diff --git a/tests/0053/tokens.txt b/tests/0053/tokens.txt new file mode 100644 index 00000000..a52e218c --- /dev/null +++ b/tests/0053/tokens.txt @@ -0,0 +1,80 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Question, + span: ( + 1, + 20, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 1, + 21, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 1, + 28, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 30, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 32, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 33, + ), + }, +] diff --git a/tests/0054/tokens.txt b/tests/0054/tokens.txt new file mode 100644 index 00000000..1cbe0b82 --- /dev/null +++ b/tests/0054/tokens.txt @@ -0,0 +1,89 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: Pipe, + span: ( + 1, + 23, + ), + }, + Token { + kind: Identifier( + "float", + ), + span: ( + 1, + 24, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 1, + 30, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 32, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 34, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 35, + ), + }, +] diff --git a/tests/0055/tokens.txt b/tests/0055/tokens.txt new file mode 100644 index 00000000..2578d445 --- /dev/null +++ b/tests/0055/tokens.txt @@ -0,0 +1,105 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: Pipe, + span: ( + 1, + 26, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 1, + 27, + ), + }, + Token { + kind: Pipe, + span: ( + 1, + 30, + ), + }, + Token { + kind: Identifier( + "float", + ), + span: ( + 1, + 31, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 1, + 37, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 39, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 41, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 42, + ), + }, +] diff --git a/tests/0056/tokens.txt b/tests/0056/tokens.txt new file mode 100644 index 00000000..b7534301 --- /dev/null +++ b/tests/0056/tokens.txt @@ -0,0 +1,89 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: Ampersand, + span: ( + 1, + 23, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 1, + 24, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 1, + 28, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 30, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 32, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 33, + ), + }, +] diff --git a/tests/0057/tokens.txt b/tests/0057/tokens.txt new file mode 100644 index 00000000..83249100 --- /dev/null +++ b/tests/0057/tokens.txt @@ -0,0 +1,105 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: Ampersand, + span: ( + 1, + 23, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 1, + 24, + ), + }, + Token { + kind: Ampersand, + span: ( + 1, + 27, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 1, + 28, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 1, + 32, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 34, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 36, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 37, + ), + }, +] diff --git a/tests/0058/tokens.txt b/tests/0058/tokens.txt new file mode 100644 index 00000000..ba3ca39b --- /dev/null +++ b/tests/0058/tokens.txt @@ -0,0 +1,71 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 20, + ), + }, + Token { + kind: Colon, + span: ( + 1, + 21, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 1, + 23, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 30, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 31, + ), + }, +] diff --git a/tests/0059/tokens.txt b/tests/0059/tokens.txt new file mode 100644 index 00000000..57b1fc98 --- /dev/null +++ b/tests/0059/tokens.txt @@ -0,0 +1,71 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 20, + ), + }, + Token { + kind: Colon, + span: ( + 1, + 21, + ), + }, + Token { + kind: Identifier( + "void", + ), + span: ( + 1, + 23, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 28, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 29, + ), + }, +] diff --git a/tests/0060/tokens.txt b/tests/0060/tokens.txt new file mode 100644 index 00000000..989e2cb5 --- /dev/null +++ b/tests/0060/tokens.txt @@ -0,0 +1,46 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: New, + span: ( + 1, + 7, + ), + }, + Token { + kind: Class, + span: ( + 1, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 17, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 18, + ), + }, +] diff --git a/tests/0061/tokens.txt b/tests/0061/tokens.txt new file mode 100644 index 00000000..9a34b3f4 --- /dev/null +++ b/tests/0061/tokens.txt @@ -0,0 +1,85 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: New, + span: ( + 1, + 7, + ), + }, + Token { + kind: Class, + span: ( + 1, + 11, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 16, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 1, + 17, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 18, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 21, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 23, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 24, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 25, + ), + }, +] diff --git a/tests/0062/tokens.txt b/tests/0062/tokens.txt new file mode 100644 index 00000000..023916d1 --- /dev/null +++ b/tests/0062/tokens.txt @@ -0,0 +1,62 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: New, + span: ( + 1, + 7, + ), + }, + Token { + kind: Class, + span: ( + 1, + 11, + ), + }, + Token { + kind: Extends, + span: ( + 1, + 17, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 1, + 25, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 29, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 30, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 31, + ), + }, +] diff --git a/tests/0063/tokens.txt b/tests/0063/tokens.txt new file mode 100644 index 00000000..ea04e7e4 --- /dev/null +++ b/tests/0063/tokens.txt @@ -0,0 +1,78 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: New, + span: ( + 1, + 7, + ), + }, + Token { + kind: Class, + span: ( + 1, + 11, + ), + }, + Token { + kind: Implements, + span: ( + 1, + 17, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 1, + 28, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 31, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 1, + 33, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 37, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 38, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 39, + ), + }, +] diff --git a/tests/0064/tokens.txt b/tests/0064/tokens.txt new file mode 100644 index 00000000..9431c68e --- /dev/null +++ b/tests/0064/tokens.txt @@ -0,0 +1,97 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: New, + span: ( + 3, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 24, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 25, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 27, + ), + }, + Token { + kind: RightBrace, + span: ( + 4, + 28, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 2, + ), + }, +] diff --git a/tests/0065/tokens.txt b/tests/0065/tokens.txt new file mode 100644 index 00000000..fc9a7389 --- /dev/null +++ b/tests/0065/tokens.txt @@ -0,0 +1,25 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 8, + ), + }, +] diff --git a/tests/0066/tokens.txt b/tests/0066/tokens.txt new file mode 100644 index 00000000..85220962 --- /dev/null +++ b/tests/0066/tokens.txt @@ -0,0 +1,41 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 7, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 9, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 11, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 13, + ), + }, +] diff --git a/tests/0067/tokens.txt b/tests/0067/tokens.txt new file mode 100644 index 00000000..ae245f7a --- /dev/null +++ b/tests/0067/tokens.txt @@ -0,0 +1,18 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 7, + ), + }, +] diff --git a/tests/0068/tokens.txt b/tests/0068/tokens.txt new file mode 100644 index 00000000..ef6ca6b1 --- /dev/null +++ b/tests/0068/tokens.txt @@ -0,0 +1,73 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 2, + 1, + ), + }, + Token { + kind: Identifier( + "MyClass", + ), + span: ( + 2, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 2, + 15, + ), + }, + Token { + kind: Protected, + span: ( + 3, + 5, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 15, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 17, + ), + }, + Token { + kind: Comment( + "// my comment", + ), + span: ( + 4, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0069/tokens.txt b/tests/0069/tokens.txt new file mode 100644 index 00000000..86fffe49 --- /dev/null +++ b/tests/0069/tokens.txt @@ -0,0 +1,69 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Do, + span: ( + 1, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 10, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 12, + ), + }, + Token { + kind: While, + span: ( + 1, + 14, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 20, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 21, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 24, + ), + }, +] diff --git a/tests/0070/tokens.txt b/tests/0070/tokens.txt new file mode 100644 index 00000000..b73101eb --- /dev/null +++ b/tests/0070/tokens.txt @@ -0,0 +1,90 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Do, + span: ( + 2, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 2, + 4, + ), + }, + Token { + kind: Echo, + span: ( + 3, + 5, + ), + }, + Token { + kind: LiteralString( + "Hi!", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 15, + ), + }, + Token { + kind: RightBrace, + span: ( + 4, + 1, + ), + }, + Token { + kind: While, + span: ( + 4, + 3, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 9, + ), + }, + Token { + kind: True, + span: ( + 4, + 10, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 15, + ), + }, +] diff --git a/tests/0071/tokens.txt b/tests/0071/tokens.txt new file mode 100644 index 00000000..9144537b --- /dev/null +++ b/tests/0071/tokens.txt @@ -0,0 +1,27 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: CloseTag, + span: ( + 1, + 7, + ), + }, + Token { + kind: InlineHtml( + " ", + ), + span: ( + 1, + 9, + ), + }, +] diff --git a/tests/0072/tokens.txt b/tests/0072/tokens.txt new file mode 100644 index 00000000..a9e39d83 --- /dev/null +++ b/tests/0072/tokens.txt @@ -0,0 +1,48 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: At, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "hello", + ), + span: ( + 1, + 8, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 15, + ), + }, +] diff --git a/tests/0073/tokens.txt b/tests/0073/tokens.txt new file mode 100644 index 00000000..0b4cdb01 --- /dev/null +++ b/tests/0073/tokens.txt @@ -0,0 +1,43 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: NullsafeArrow, + span: ( + 1, + 9, + ), + }, + Token { + kind: Identifier( + "b", + ), + span: ( + 1, + 12, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 13, + ), + }, +] diff --git a/tests/0074/tokens.txt b/tests/0074/tokens.txt new file mode 100644 index 00000000..fda48f29 --- /dev/null +++ b/tests/0074/tokens.txt @@ -0,0 +1,57 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: NullsafeArrow, + span: ( + 1, + 9, + ), + }, + Token { + kind: Identifier( + "b", + ), + span: ( + 1, + 12, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 15, + ), + }, +] diff --git a/tests/0075/tokens.txt b/tests/0075/tokens.txt new file mode 100644 index 00000000..e1d59e28 --- /dev/null +++ b/tests/0075/tokens.txt @@ -0,0 +1,66 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: NullsafeArrow, + span: ( + 1, + 9, + ), + }, + Token { + kind: Identifier( + "b", + ), + span: ( + 1, + 12, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 13, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 1, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 17, + ), + }, +] diff --git a/tests/0076/tokens.txt b/tests/0076/tokens.txt new file mode 100644 index 00000000..08ee306f --- /dev/null +++ b/tests/0076/tokens.txt @@ -0,0 +1,73 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: NullsafeArrow, + span: ( + 1, + 9, + ), + }, + Token { + kind: Identifier( + "b", + ), + span: ( + 1, + 12, + ), + }, + Token { + kind: NullsafeArrow, + span: ( + 1, + 13, + ), + }, + Token { + kind: Identifier( + "c", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 19, + ), + }, +] diff --git a/tests/0077/tokens.txt b/tests/0077/tokens.txt new file mode 100644 index 00000000..46af1005 --- /dev/null +++ b/tests/0077/tokens.txt @@ -0,0 +1,50 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Const, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "FOO", + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: Equals, + span: ( + 1, + 17, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 1, + 19, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 20, + ), + }, +] diff --git a/tests/0078/tokens.txt b/tests/0078/tokens.txt new file mode 100644 index 00000000..327d6ca7 --- /dev/null +++ b/tests/0078/tokens.txt @@ -0,0 +1,82 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Const, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "FOO", + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: Equals, + span: ( + 1, + 17, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 1, + 19, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 20, + ), + }, + Token { + kind: Identifier( + "BAR", + ), + span: ( + 1, + 22, + ), + }, + Token { + kind: Equals, + span: ( + 1, + 26, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 1, + 28, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 29, + ), + }, +] diff --git a/tests/0079/tokens.txt b/tests/0079/tokens.txt new file mode 100644 index 00000000..eee50ba0 --- /dev/null +++ b/tests/0079/tokens.txt @@ -0,0 +1,34 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Global, + span: ( + 1, + 7, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 16, + ), + }, +] diff --git a/tests/0080/tokens.txt b/tests/0080/tokens.txt new file mode 100644 index 00000000..d0b2bd43 --- /dev/null +++ b/tests/0080/tokens.txt @@ -0,0 +1,50 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Global, + span: ( + 1, + 7, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 14, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 16, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 1, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 20, + ), + }, +] diff --git a/tests/0081/tokens.txt b/tests/0081/tokens.txt new file mode 100644 index 00000000..18131e11 --- /dev/null +++ b/tests/0081/tokens.txt @@ -0,0 +1,64 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Declare, + span: ( + 1, + 7, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 14, + ), + }, + Token { + kind: Identifier( + "A", + ), + span: ( + 1, + 15, + ), + }, + Token { + kind: Equals, + span: ( + 1, + 16, + ), + }, + Token { + kind: LiteralString( + "B", + ), + span: ( + 1, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 21, + ), + }, +] diff --git a/tests/0082/tokens.txt b/tests/0082/tokens.txt new file mode 100644 index 00000000..e3060d08 --- /dev/null +++ b/tests/0082/tokens.txt @@ -0,0 +1,96 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Declare, + span: ( + 1, + 7, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 14, + ), + }, + Token { + kind: Identifier( + "A", + ), + span: ( + 1, + 15, + ), + }, + Token { + kind: Equals, + span: ( + 1, + 16, + ), + }, + Token { + kind: LiteralString( + "B", + ), + span: ( + 1, + 17, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 20, + ), + }, + Token { + kind: Identifier( + "C", + ), + span: ( + 1, + 22, + ), + }, + Token { + kind: Equals, + span: ( + 1, + 23, + ), + }, + Token { + kind: LiteralString( + "D", + ), + span: ( + 1, + 24, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 27, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 28, + ), + }, +] diff --git a/tests/0083/tokens.txt b/tests/0083/tokens.txt new file mode 100644 index 00000000..d098fff8 --- /dev/null +++ b/tests/0083/tokens.txt @@ -0,0 +1,94 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Declare, + span: ( + 1, + 7, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 14, + ), + }, + Token { + kind: Identifier( + "A", + ), + span: ( + 1, + 15, + ), + }, + Token { + kind: Equals, + span: ( + 1, + 16, + ), + }, + Token { + kind: LiteralString( + "B", + ), + span: ( + 1, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 20, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 22, + ), + }, + Token { + kind: Echo, + span: ( + 1, + 24, + ), + }, + Token { + kind: LiteralString( + "Hello, world!", + ), + span: ( + 1, + 29, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 44, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 46, + ), + }, +] diff --git a/tests/0084/tokens.txt b/tests/0084/tokens.txt new file mode 100644 index 00000000..d3610797 --- /dev/null +++ b/tests/0084/tokens.txt @@ -0,0 +1,80 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 7, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 1, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 9, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 1, + 11, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 12, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 14, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 17, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 18, + ), + }, +] diff --git a/tests/0085/tokens.txt b/tests/0085/tokens.txt new file mode 100644 index 00000000..57db28e4 --- /dev/null +++ b/tests/0085/tokens.txt @@ -0,0 +1,48 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Readonly, + span: ( + 1, + 7, + ), + }, + Token { + kind: Class, + span: ( + 1, + 16, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 1, + 22, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 26, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 27, + ), + }, +] diff --git a/tests/0086/tokens.txt b/tests/0086/tokens.txt new file mode 100644 index 00000000..b870fd57 --- /dev/null +++ b/tests/0086/tokens.txt @@ -0,0 +1,71 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 17, + ), + }, + Token { + kind: Public, + span: ( + 1, + 19, + ), + }, + Token { + kind: Readonly, + span: ( + 1, + 26, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 1, + 35, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 39, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 41, + ), + }, +] diff --git a/tests/0087/tokens.txt b/tests/0087/tokens.txt new file mode 100644 index 00000000..3a2ca180 --- /dev/null +++ b/tests/0087/tokens.txt @@ -0,0 +1,119 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Enum, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 6, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 9, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 1, + ), + }, + Token { + kind: Case, + span: ( + 5, + 5, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 5, + 10, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 13, + ), + }, + Token { + kind: Case, + span: ( + 6, + 5, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 6, + 10, + ), + }, + Token { + kind: Equals, + span: ( + 6, + 14, + ), + }, + Token { + kind: LiteralString( + "Baz", + ), + span: ( + 6, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0088/tokens.txt b/tests/0088/tokens.txt new file mode 100644 index 00000000..9d0cfec4 --- /dev/null +++ b/tests/0088/tokens.txt @@ -0,0 +1,55 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 10, + ), + }, + Token { + kind: Ellipsis, + span: ( + 1, + 11, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 14, + ), + }, + Token { + kind: Class, + span: ( + 1, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 21, + ), + }, +] diff --git a/tests/0089/tokens.txt b/tests/0089/tokens.txt new file mode 100644 index 00000000..1fd1cb63 --- /dev/null +++ b/tests/0089/tokens.txt @@ -0,0 +1,64 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "this", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 12, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 14, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 17, + ), + }, + Token { + kind: Ellipsis, + span: ( + 1, + 18, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 21, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 22, + ), + }, +] diff --git a/tests/0090/tokens.txt b/tests/0090/tokens.txt new file mode 100644 index 00000000..f4594c57 --- /dev/null +++ b/tests/0090/tokens.txt @@ -0,0 +1,64 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Identifier( + "A", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: DoubleColon, + span: ( + 1, + 8, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 13, + ), + }, + Token { + kind: Ellipsis, + span: ( + 1, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 17, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 18, + ), + }, +] diff --git a/tests/0091/tokens.txt b/tests/0091/tokens.txt new file mode 100644 index 00000000..bd5167d3 --- /dev/null +++ b/tests/0091/tokens.txt @@ -0,0 +1,69 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 18, + ), + }, + Token { + kind: Colon, + span: ( + 1, + 19, + ), + }, + Token { + kind: True, + span: ( + 1, + 21, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 26, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 27, + ), + }, +] diff --git a/tests/0092/tokens.txt b/tests/0092/tokens.txt new file mode 100644 index 00000000..bcd39ae8 --- /dev/null +++ b/tests/0092/tokens.txt @@ -0,0 +1,69 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 1, + 7, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 18, + ), + }, + Token { + kind: Colon, + span: ( + 1, + 19, + ), + }, + Token { + kind: False, + span: ( + 1, + 21, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 27, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 28, + ), + }, +] diff --git a/tests/0093/tokens.txt b/tests/0093/tokens.txt new file mode 100644 index 00000000..dcebfe01 --- /dev/null +++ b/tests/0093/tokens.txt @@ -0,0 +1,48 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Dollar, + span: ( + 1, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 8, + ), + }, + Token { + kind: LiteralString( + "foo", + ), + span: ( + 1, + 9, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 15, + ), + }, +] diff --git a/tests/0094/tokens.txt b/tests/0094/tokens.txt new file mode 100644 index 00000000..30159a6f --- /dev/null +++ b/tests/0094/tokens.txt @@ -0,0 +1,62 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Dollar, + span: ( + 1, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 8, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 1, + 9, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 12, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 13, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 15, + ), + }, +] diff --git a/tests/0095/tokens.txt b/tests/0095/tokens.txt new file mode 100644 index 00000000..7264fce3 --- /dev/null +++ b/tests/0095/tokens.txt @@ -0,0 +1,34 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Dollar, + span: ( + 1, + 7, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 8, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 10, + ), + }, +] diff --git a/tests/0096/tokens.txt b/tests/0096/tokens.txt new file mode 100644 index 00000000..942caef7 --- /dev/null +++ b/tests/0096/tokens.txt @@ -0,0 +1,50 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 9, + ), + }, + Token { + kind: Dollar, + span: ( + 1, + 11, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 1, + 12, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 14, + ), + }, +] diff --git a/tests/0097/tokens.txt b/tests/0097/tokens.txt new file mode 100644 index 00000000..19319fd5 --- /dev/null +++ b/tests/0097/tokens.txt @@ -0,0 +1,64 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: Arrow, + span: ( + 1, + 9, + ), + }, + Token { + kind: Dollar, + span: ( + 1, + 11, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 1, + 12, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 15, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 16, + ), + }, +] diff --git a/tests/0098/tokens.txt b/tests/0098/tokens.txt new file mode 100644 index 00000000..81fe0d2b --- /dev/null +++ b/tests/0098/tokens.txt @@ -0,0 +1,50 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: DoubleColon, + span: ( + 1, + 10, + ), + }, + Token { + kind: Dollar, + span: ( + 1, + 12, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 15, + ), + }, +] diff --git a/tests/0099/tokens.txt b/tests/0099/tokens.txt new file mode 100644 index 00000000..217fd2c0 --- /dev/null +++ b/tests/0099/tokens.txt @@ -0,0 +1,64 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: DoubleColon, + span: ( + 1, + 10, + ), + }, + Token { + kind: Dollar, + span: ( + 1, + 12, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 15, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 17, + ), + }, +] diff --git a/tests/0100/tokens.txt b/tests/0100/tokens.txt new file mode 100644 index 00000000..97f8979b --- /dev/null +++ b/tests/0100/tokens.txt @@ -0,0 +1,71 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 1, + 7, + ), + }, + Token { + kind: DoubleColon, + span: ( + 1, + 10, + ), + }, + Token { + kind: LeftBrace, + span: ( + 1, + 12, + ), + }, + Token { + kind: LiteralString( + "foo", + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: RightBrace, + span: ( + 1, + 18, + ), + }, + Token { + kind: LeftParen, + span: ( + 1, + 19, + ), + }, + Token { + kind: RightParen, + span: ( + 1, + 20, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 21, + ), + }, +] diff --git a/tests/0101/tokens.txt b/tests/0101/tokens.txt new file mode 100644 index 00000000..d0c96f1b --- /dev/null +++ b/tests/0101/tokens.txt @@ -0,0 +1,53 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 7, + ), + }, + Token { + kind: Ellipsis, + span: ( + 1, + 8, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 11, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 12, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 13, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 14, + ), + }, +] diff --git a/tests/0102/tokens.txt b/tests/0102/tokens.txt new file mode 100644 index 00000000..78934905 --- /dev/null +++ b/tests/0102/tokens.txt @@ -0,0 +1,99 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 7, + ), + }, + Token { + kind: Ellipsis, + span: ( + 1, + 8, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 11, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 1, + 12, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 13, + ), + }, + Token { + kind: Comma, + span: ( + 1, + 14, + ), + }, + Token { + kind: Ellipsis, + span: ( + 1, + 16, + ), + }, + Token { + kind: LeftBracket, + span: ( + 1, + 19, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 1, + 20, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 21, + ), + }, + Token { + kind: RightBracket, + span: ( + 1, + 22, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 23, + ), + }, +] diff --git a/tests/0103/tokens.txt b/tests/0103/tokens.txt new file mode 100644 index 00000000..6a9e54f6 --- /dev/null +++ b/tests/0103/tokens.txt @@ -0,0 +1,34 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Print, + span: ( + 1, + 7, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 17, + ), + }, +] diff --git a/tests/0105/tokens.txt b/tests/0105/tokens.txt new file mode 100644 index 00000000..74885706 --- /dev/null +++ b/tests/0105/tokens.txt @@ -0,0 +1,25 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Yield, + span: ( + 1, + 7, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 12, + ), + }, +] diff --git a/tests/0106/tokens.txt b/tests/0106/tokens.txt new file mode 100644 index 00000000..82468553 --- /dev/null +++ b/tests/0106/tokens.txt @@ -0,0 +1,34 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Yield, + span: ( + 1, + 7, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 14, + ), + }, +] diff --git a/tests/0107/tokens.txt b/tests/0107/tokens.txt new file mode 100644 index 00000000..a8ccac0d --- /dev/null +++ b/tests/0107/tokens.txt @@ -0,0 +1,50 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Yield, + span: ( + 1, + 7, + ), + }, + Token { + kind: LiteralInteger( + 0, + ), + span: ( + 1, + 13, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 1, + 15, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 1, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 19, + ), + }, +] diff --git a/tests/0108/tokens.txt b/tests/0108/tokens.txt new file mode 100644 index 00000000..3a6e77dd --- /dev/null +++ b/tests/0108/tokens.txt @@ -0,0 +1,41 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Yield, + span: ( + 1, + 7, + ), + }, + Token { + kind: From, + span: ( + 1, + 13, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 1, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 1, + 19, + ), + }, +] diff --git a/tests/0109/tokens.txt b/tests/0109/tokens.txt new file mode 100644 index 00000000..fd1aea2e --- /dev/null +++ b/tests/0109/tokens.txt @@ -0,0 +1,239 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 5, + 23, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 25, + ), + }, + Token { + kind: Public, + span: ( + 6, + 9, + ), + }, + Token { + kind: Readonly, + span: ( + 6, + 16, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 6, + 25, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 6, + 29, + ), + }, + Token { + kind: Comma, + span: ( + 6, + 31, + ), + }, + Token { + kind: Public, + span: ( + 7, + 9, + ), + }, + Token { + kind: Readonly, + span: ( + 7, + 16, + ), + }, + Token { + kind: Identifier( + "float", + ), + span: ( + 7, + 25, + ), + }, + Token { + kind: Ampersand, + span: ( + 7, + 31, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 7, + 32, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 34, + ), + }, + Token { + kind: Ampersand, + span: ( + 8, + 9, + ), + }, + Token { + kind: Ellipsis, + span: ( + 8, + 10, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 8, + 13, + ), + }, + Token { + kind: Comma, + span: ( + 8, + 15, + ), + }, + Token { + kind: RightParen, + span: ( + 9, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 9, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 10, + 1, + ), + }, +] diff --git a/tests/0110/tokens.txt b/tests/0110/tokens.txt new file mode 100644 index 00000000..b6701aaa --- /dev/null +++ b/tests/0110/tokens.txt @@ -0,0 +1,232 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 5, + 23, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 25, + ), + }, + Token { + kind: Public, + span: ( + 6, + 9, + ), + }, + Token { + kind: Readonly, + span: ( + 6, + 16, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 6, + 25, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 6, + 29, + ), + }, + Token { + kind: Comma, + span: ( + 6, + 31, + ), + }, + Token { + kind: Public, + span: ( + 7, + 9, + ), + }, + Token { + kind: Readonly, + span: ( + 7, + 16, + ), + }, + Token { + kind: Identifier( + "float", + ), + span: ( + 7, + 25, + ), + }, + Token { + kind: Ampersand, + span: ( + 7, + 31, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 7, + 32, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 34, + ), + }, + Token { + kind: Ellipsis, + span: ( + 8, + 9, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 8, + 12, + ), + }, + Token { + kind: Comma, + span: ( + 8, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 9, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 9, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 10, + 1, + ), + }, +] diff --git a/tests/0111/tokens.txt b/tests/0111/tokens.txt new file mode 100644 index 00000000..874980b2 --- /dev/null +++ b/tests/0111/tokens.txt @@ -0,0 +1,129 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Readonly, + span: ( + 5, + 16, + ), + }, + Token { + kind: Ellipsis, + span: ( + 5, + 25, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 5, + 28, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 30, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0112/tokens.txt b/tests/0112/tokens.txt new file mode 100644 index 00000000..2421bdcc --- /dev/null +++ b/tests/0112/tokens.txt @@ -0,0 +1,129 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Readonly, + span: ( + 5, + 16, + ), + }, + Token { + kind: Ampersand, + span: ( + 5, + 25, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 5, + 26, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 28, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0113/tokens.txt b/tests/0113/tokens.txt new file mode 100644 index 00000000..9266f2a5 --- /dev/null +++ b/tests/0113/tokens.txt @@ -0,0 +1,124 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 24, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 5, + 23, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 25, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0114/tokens.txt b/tests/0114/tokens.txt new file mode 100644 index 00000000..1ac51b4e --- /dev/null +++ b/tests/0114/tokens.txt @@ -0,0 +1,117 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Interface, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 15, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 5, + 23, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 25, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 6, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0115/tokens.txt b/tests/0115/tokens.txt new file mode 100644 index 00000000..7bbeec35 --- /dev/null +++ b/tests/0115/tokens.txt @@ -0,0 +1,131 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Abstract, + span: ( + 3, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 10, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 16, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 20, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Abstract, + span: ( + 4, + 12, + ), + }, + Token { + kind: Function, + span: ( + 4, + 21, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 30, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 41, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 5, + 23, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 25, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 6, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0116/tokens.txt b/tests/0116/tokens.txt new file mode 100644 index 00000000..6a5140a4 --- /dev/null +++ b/tests/0116/tokens.txt @@ -0,0 +1,124 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Trait, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Abstract, + span: ( + 4, + 12, + ), + }, + Token { + kind: Function, + span: ( + 4, + 21, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 30, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 41, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 5, + 23, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 25, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 6, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0117/tokens.txt b/tests/0117/tokens.txt new file mode 100644 index 00000000..8d58559a --- /dev/null +++ b/tests/0117/tokens.txt @@ -0,0 +1,135 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Enum, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 6, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 9, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 15, + ), + }, + Token { + kind: Case, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 4, + 10, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 14, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 4, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 17, + ), + }, + Token { + kind: Case, + span: ( + 5, + 5, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 5, + 10, + ), + }, + Token { + kind: Equals, + span: ( + 5, + 14, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 17, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 1, + ), + }, +] diff --git a/tests/0118/tokens.txt b/tests/0118/tokens.txt new file mode 100644 index 00000000..25378b80 --- /dev/null +++ b/tests/0118/tokens.txt @@ -0,0 +1,87 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Enum, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 6, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 10, + ), + }, + Token { + kind: Case, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 4, + 10, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 13, + ), + }, + Token { + kind: Case, + span: ( + 5, + 5, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 5, + 10, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 13, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 1, + ), + }, +] diff --git a/tests/0119/tokens.txt b/tests/0119/tokens.txt new file mode 100644 index 00000000..048a48e4 --- /dev/null +++ b/tests/0119/tokens.txt @@ -0,0 +1,135 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Enum, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 6, + ), + }, + Token { + kind: Colon, + span: ( + 3, + 9, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 18, + ), + }, + Token { + kind: Case, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 4, + 10, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 14, + ), + }, + Token { + kind: LiteralString( + "3", + ), + span: ( + 4, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 19, + ), + }, + Token { + kind: Case, + span: ( + 5, + 5, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 5, + 10, + ), + }, + Token { + kind: Equals, + span: ( + 5, + 14, + ), + }, + Token { + kind: LiteralString( + "g", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 19, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 1, + ), + }, +] diff --git a/tests/0120/tokens.txt b/tests/0120/tokens.txt new file mode 100644 index 00000000..92f74f19 --- /dev/null +++ b/tests/0120/tokens.txt @@ -0,0 +1,85 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Abstract, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 14, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 4, + 23, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 26, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 27, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 28, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0121/tokens.txt b/tests/0121/tokens.txt new file mode 100644 index 00000000..4a180c1d --- /dev/null +++ b/tests/0121/tokens.txt @@ -0,0 +1,87 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Static, + span: ( + 4, + 5, + ), + }, + Token { + kind: Const, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "BAR", + ), + span: ( + 4, + 18, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 22, + ), + }, + Token { + kind: LiteralInteger( + 34, + ), + span: ( + 4, + 24, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 26, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0122/tokens.txt b/tests/0122/tokens.txt new file mode 100644 index 00000000..b04ae41c --- /dev/null +++ b/tests/0122/tokens.txt @@ -0,0 +1,87 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Const, + span: ( + 4, + 5, + ), + }, + Token { + kind: Static, + span: ( + 4, + 11, + ), + }, + Token { + kind: Identifier( + "BAR", + ), + span: ( + 4, + 18, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 22, + ), + }, + Token { + kind: LiteralInteger( + 34, + ), + span: ( + 4, + 24, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 26, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0123/tokens.txt b/tests/0123/tokens.txt new file mode 100644 index 00000000..9746f121 --- /dev/null +++ b/tests/0123/tokens.txt @@ -0,0 +1,87 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Readonly, + span: ( + 4, + 5, + ), + }, + Token { + kind: Const, + span: ( + 4, + 14, + ), + }, + Token { + kind: Identifier( + "BAR", + ), + span: ( + 4, + 20, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 24, + ), + }, + Token { + kind: LiteralInteger( + 34, + ), + span: ( + 4, + 26, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 28, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0124/tokens.txt b/tests/0124/tokens.txt new file mode 100644 index 00000000..f45fc289 --- /dev/null +++ b/tests/0124/tokens.txt @@ -0,0 +1,99 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Final, + span: ( + 4, + 5, + ), + }, + Token { + kind: Abstract, + span: ( + 4, + 11, + ), + }, + Token { + kind: Function, + span: ( + 4, + 20, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 4, + 29, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 33, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 35, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0125/tokens.txt b/tests/0125/tokens.txt new file mode 100644 index 00000000..df0510c1 --- /dev/null +++ b/tests/0125/tokens.txt @@ -0,0 +1,99 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Final, + span: ( + 3, + 1, + ), + }, + Token { + kind: Abstract, + span: ( + 3, + 7, + ), + }, + Token { + kind: Class, + span: ( + 3, + 16, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 22, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 26, + ), + }, + Token { + kind: Function, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 4, + 14, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 18, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 20, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0126/tokens.txt b/tests/0126/tokens.txt new file mode 100644 index 00000000..20991cea --- /dev/null +++ b/tests/0126/tokens.txt @@ -0,0 +1,94 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Final, + span: ( + 4, + 5, + ), + }, + Token { + kind: Private, + span: ( + 4, + 11, + ), + }, + Token { + kind: Const, + span: ( + 4, + 19, + ), + }, + Token { + kind: Identifier( + "BAR", + ), + span: ( + 4, + 25, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 29, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 4, + 31, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 32, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0127/tokens.txt b/tests/0127/tokens.txt new file mode 100644 index 00000000..3722780a --- /dev/null +++ b/tests/0127/tokens.txt @@ -0,0 +1,147 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Readonly, + span: ( + 5, + 9, + ), + }, + Token { + kind: Public, + span: ( + 5, + 18, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 5, + 25, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 5, + 32, + ), + }, + Token { + kind: Equals, + span: ( + 5, + 35, + ), + }, + Token { + kind: LiteralString( + "h", + ), + span: ( + 5, + 37, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 40, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0128/tokens.txt b/tests/0128/tokens.txt new file mode 100644 index 00000000..e468c251 --- /dev/null +++ b/tests/0128/tokens.txt @@ -0,0 +1,138 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Readonly, + span: ( + 5, + 9, + ), + }, + Token { + kind: Public, + span: ( + 5, + 18, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 5, + 25, + ), + }, + Token { + kind: Equals, + span: ( + 5, + 28, + ), + }, + Token { + kind: LiteralString( + "h", + ), + span: ( + 5, + 30, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 33, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0129/tokens.txt b/tests/0129/tokens.txt new file mode 100644 index 00000000..2bef352d --- /dev/null +++ b/tests/0129/tokens.txt @@ -0,0 +1,161 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Readonly, + span: ( + 5, + 9, + ), + }, + Token { + kind: Public, + span: ( + 5, + 18, + ), + }, + Token { + kind: Protected, + span: ( + 5, + 25, + ), + }, + Token { + kind: Private, + span: ( + 5, + 35, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 5, + 43, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 5, + 50, + ), + }, + Token { + kind: Equals, + span: ( + 5, + 53, + ), + }, + Token { + kind: LiteralString( + "h", + ), + span: ( + 5, + 55, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 58, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0130/tokens.txt b/tests/0130/tokens.txt new file mode 100644 index 00000000..618d48f6 --- /dev/null +++ b/tests/0130/tokens.txt @@ -0,0 +1,99 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Foreach, + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 28, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 29, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 31, + ), + }, + Token { + kind: Comment( + "//", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0131/tokens.txt b/tests/0131/tokens.txt new file mode 100644 index 00000000..cbbb8979 --- /dev/null +++ b/tests/0131/tokens.txt @@ -0,0 +1,62 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Foreach, + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 18, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 20, + ), + }, + Token { + kind: Comment( + "//", + ), + span: ( + 4, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0133/code.php b/tests/0133/code.php new file mode 100644 index 00000000..4e450cc5 --- /dev/null +++ b/tests/0133/code.php @@ -0,0 +1,5 @@ + Syntax Error: invalid octal escape on line 5 column 11 diff --git a/tests/0134/code.php b/tests/0134/code.php new file mode 100644 index 00000000..8723c612 --- /dev/null +++ b/tests/0134/code.php @@ -0,0 +1,4 @@ + Syntax Error: invalid unicode escape on line 4 column 10 diff --git a/tests/0135/code.php b/tests/0135/code.php new file mode 100644 index 00000000..6c300cd9 --- /dev/null +++ b/tests/0135/code.php @@ -0,0 +1,4 @@ + Syntax Error: invalid unicode escape on line 4 column 10 diff --git a/tests/0136/code.php b/tests/0136/code.php new file mode 100644 index 00000000..cf559e58 --- /dev/null +++ b/tests/0136/code.php @@ -0,0 +1,4 @@ + Syntax Error: invalid unicode escape on line 4 column 12 diff --git a/tests/0137/code.php b/tests/0137/code.php new file mode 100644 index 00000000..0d7d6b3e --- /dev/null +++ b/tests/0137/code.php @@ -0,0 +1,4 @@ + Syntax Error: invalid unicode escape on line 4 column 17 diff --git a/tests/0138/code.php b/tests/0138/code.php new file mode 100644 index 00000000..745e22ca --- /dev/null +++ b/tests/0138/code.php @@ -0,0 +1,3 @@ + Syntax Error: unexpected end of file on line 4 column 1 diff --git a/tests/0139/code.php b/tests/0139/code.php new file mode 100644 index 00000000..a81511aa --- /dev/null +++ b/tests/0139/code.php @@ -0,0 +1,3 @@ + Syntax Error: unexpected end of file on line 4 column 1 diff --git a/tests/0140/code.php b/tests/0140/code.php new file mode 100644 index 00000000..323a509c --- /dev/null +++ b/tests/0140/code.php @@ -0,0 +1,3 @@ + Syntax Error: invalid octal literal on line 3 column 8 diff --git a/tests/third_party_tests.rs b/tests/third_party_tests.rs index ca3d098b..6cc27983 100644 --- a/tests/third_party_tests.rs +++ b/tests/third_party_tests.rs @@ -3,8 +3,8 @@ use std::fs; use std::path::PathBuf; use std::process::Command; -use php_parser_rs::Lexer; -use php_parser_rs::Parser; +use php_parser_rs::prelude::Lexer; +use php_parser_rs::prelude::Parser; #[test] fn third_party_php_standard_library() { @@ -73,7 +73,7 @@ fn test_directory(root: PathBuf, directory: PathBuf) { fn test_file(name: &str, filename: PathBuf) { let code = std::fs::read_to_string(&filename).unwrap(); - Lexer::new(None) + Lexer::new() .tokenize(code.as_bytes()) .map(|tokens| { Parser::new(None) From 9bde28a3c8d9b66fc4b1be9bc33e4a2325338e96 Mon Sep 17 00:00:00 2001 From: Saif Eddin Gmati <29315886+azjezz@users.noreply.github.com> Date: Thu, 1 Dec 2022 01:45:28 +0100 Subject: [PATCH 02/13] chore: don't use class member flags for interface member flags Signed-off-by: azjezz --- src/parser/classish_statement.rs | 17 +++-- src/parser/flags.rs | 61 ++++++++++------- src/parser/macros.rs | 38 ++++++----- tests/0132/code.php | 5 ++ tests/0132/parser-error.txt | 1 + tests/0132/tokens.txt | 108 +++++++++++++++++++++++++++++++ tests/0141/code.php | 5 ++ tests/0141/parser-error.txt | 1 + tests/0141/tokens.txt | 85 ++++++++++++++++++++++++ tests/0142/code.php | 5 ++ tests/0142/parser-error.txt | 1 + tests/0142/tokens.txt | 85 ++++++++++++++++++++++++ tests/0143/ast.txt | 21 ++++++ tests/0143/code.php | 5 ++ tests/0143/tokens.txt | 85 ++++++++++++++++++++++++ tests/0144/code.php | 5 ++ tests/0144/parser-error.txt | 1 + tests/0144/tokens.txt | 92 ++++++++++++++++++++++++++ tests/0145/ast.txt | 22 +++++++ tests/0145/code.php | 5 ++ tests/0145/tokens.txt | 92 ++++++++++++++++++++++++++ 21 files changed, 690 insertions(+), 50 deletions(-) create mode 100644 tests/0132/code.php create mode 100644 tests/0132/parser-error.txt create mode 100644 tests/0132/tokens.txt create mode 100644 tests/0141/code.php create mode 100644 tests/0141/parser-error.txt create mode 100644 tests/0141/tokens.txt create mode 100644 tests/0142/code.php create mode 100644 tests/0142/parser-error.txt create mode 100644 tests/0142/tokens.txt create mode 100644 tests/0143/ast.txt create mode 100644 tests/0143/code.php create mode 100644 tests/0143/tokens.txt create mode 100644 tests/0144/code.php create mode 100644 tests/0144/parser-error.txt create mode 100644 tests/0144/tokens.txt create mode 100644 tests/0145/ast.txt create mode 100644 tests/0145/code.php create mode 100644 tests/0145/tokens.txt diff --git a/src/parser/classish_statement.rs b/src/parser/classish_statement.rs index 96d086e7..0f382281 100644 --- a/src/parser/classish_statement.rs +++ b/src/parser/classish_statement.rs @@ -11,6 +11,7 @@ use crate::parser::Parser; use crate::expect_token; use crate::expected_token_err; +use crate::peek_token; #[derive(Debug)] pub enum ClassishDefinitionType { @@ -38,16 +39,15 @@ impl Parser { return self.method(ClassishDefinitionType::Interface, vec![]); } - let member_flags = self.class_members_flags()?; + let member_flags = self.interface_members_flags()?; - match &self.current.kind { + peek_token!([ TokenKind::Const => self.parse_classish_const(member_flags), TokenKind::Function => self.method( ClassishDefinitionType::Interface, member_flags.iter().map(|t| t.clone().into()).collect(), - ), - _ => expected_token_err!(["`const`", "`function`"], self), - } + ) + ], self, ["`const`", "`function`"]) } pub(in crate::parser) fn trait_statement(&mut self) -> ParseResult { @@ -91,14 +91,13 @@ impl Parser { let member_flags = self.enum_members_flags()?; - match &self.current.kind { + peek_token!([ TokenKind::Const => self.parse_classish_const(member_flags), TokenKind::Function => self.method( ClassishDefinitionType::Enum, member_flags.iter().map(|t| t.clone().into()).collect(), - ), - _ => expected_token_err!(["`const`", "`function`"], self), - } + ) + ], self, ["`const`", "`function`"]) } fn complete_class_statement( diff --git a/src/parser/flags.rs b/src/parser/flags.rs index 278b8094..847827ee 100644 --- a/src/parser/flags.rs +++ b/src/parser/flags.rs @@ -3,10 +3,12 @@ use crate::parser::error::ParseError; use crate::parser::error::ParseResult; use crate::parser::Parser; +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] enum FlagTarget { Class, EnumMember, ClassMember, + InterfaceMember, PromotedProperty, } @@ -18,6 +20,13 @@ impl Parser { ) } + pub(in crate::parser) fn interface_members_flags(&mut self) -> ParseResult> { + self.collect( + vec![TokenKind::Public, TokenKind::Static], + FlagTarget::InterfaceMember, + ) + } + pub(in crate::parser) fn class_members_flags(&mut self) -> ParseResult> { self.collect( vec![ @@ -92,34 +101,36 @@ impl Parser { { return Err(ParseError::MultipleAccessModifiers(self.current.span)); } - TokenKind::Final if collected.contains(&TokenKind::Abstract) => match target { - FlagTarget::Class => { - return Err(ParseError::FinalModifierOnAbstractClass( - self.current.span, - )); - } - FlagTarget::ClassMember => { - return Err(ParseError::FinalModifierOnAbstractClassMember( - self.current.span, - )); - } - _ => {} - }, - TokenKind::Abstract if collected.contains(&TokenKind::Final) => match target { - FlagTarget::Class => { - return Err(ParseError::FinalModifierOnAbstractClass( - self.current.span, - )); + _ => {} + }; + + if matches!(target, FlagTarget::ClassMember | FlagTarget::Class) { + match self.current.kind { + TokenKind::Final if collected.contains(&TokenKind::Abstract) => { + if target == FlagTarget::Class { + return Err(ParseError::FinalModifierOnAbstractClass( + self.current.span, + )); + } else { + return Err(ParseError::FinalModifierOnAbstractClassMember( + self.current.span, + )); + } } - FlagTarget::ClassMember => { - return Err(ParseError::FinalModifierOnAbstractClassMember( - self.current.span, - )); + TokenKind::Abstract if collected.contains(&TokenKind::Final) => { + if target == FlagTarget::Class { + return Err(ParseError::FinalModifierOnAbstractClass( + self.current.span, + )); + } else { + return Err(ParseError::FinalModifierOnAbstractClassMember( + self.current.span, + )); + } } _ => {} - }, - _ => {} - }; + }; + } collected.push(self.current.kind.clone()); self.next(); diff --git a/src/parser/macros.rs b/src/parser/macros.rs index f869a91b..1bc4f279 100644 --- a/src/parser/macros.rs +++ b/src/parser/macros.rs @@ -1,13 +1,10 @@ #[macro_export] -macro_rules! expect_token { +macro_rules! peek_token { ([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => {{ $parser.skip_comments(); match $parser.current.kind.clone() { $( - $expected => { - $parser.next(); - $out - } + $expected => $out, )+ _ => { return $crate::expected_token_err!([ $($message,)+ ], $parser); @@ -16,22 +13,31 @@ macro_rules! expect_token { }}; ([ $($expected:pat),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => {{ $parser.skip_comments(); - match $parser.current.kind.clone() { - $( - $expected => { - $parser.next(); - } - )+ - _ => { - return $crate::expected_token_err!([ $($message,)+ ], $parser); - } + if !matches!($parser.current.kind, $(| $expected )+) { + return $crate::expected_token_err!([ $($message,)+ ], $parser); } }}; ([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, $message:literal) => { - $crate::expect_token!([ $($expected => $out,)+ ], $parser, [$message]) + $crate::peek_token!([ $($expected => $out,)+ ], $parser, [$message]) + }; + ([ $($expected:pat),+ $(,)? ], $parser:expr, $message:literal) => { + $crate::peek_token!([ $($expected,)+ ], $parser, [$message]) + }; +} + +#[macro_export] +macro_rules! expect_token { + ([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => { + $crate::peek_token!([ $($expected => { $parser.next(); $out },)+ ], $parser, [$($message,)+]) + }; + ([ $($expected:pat),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => { + $crate::peek_token!([ $($expected => { $parser.next(); },)+ ], $parser, [$($message,)+]) + }; + ([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, $message:literal) => { + $crate::peek_token!([ $($expected => { $parser.next(); $out },)+ ], $parser, [$message]) }; ([ $($expected:pat),+ $(,)? ], $parser:expr, $message:literal) => { - $crate::expect_token!([ $($expected,)+ ], $parser, [$message]) + $crate::peek_token!([ $($expected => { $parser.next(); },)+ ], $parser, [$message]) }; } diff --git a/tests/0132/code.php b/tests/0132/code.php new file mode 100644 index 00000000..a47a8b00 --- /dev/null +++ b/tests/0132/code.php @@ -0,0 +1,5 @@ + Parse error: unexpected token `abstract`, expecting `const`, or `function` on line 4 column 12 diff --git a/tests/0132/tokens.txt b/tests/0132/tokens.txt new file mode 100644 index 00000000..9004779d --- /dev/null +++ b/tests/0132/tokens.txt @@ -0,0 +1,108 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Interface, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 15, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Abstract, + span: ( + 4, + 12, + ), + }, + Token { + kind: Function, + span: ( + 4, + 21, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 4, + 30, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 33, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 34, + ), + }, + Token { + kind: Colon, + span: ( + 4, + 35, + ), + }, + Token { + kind: Identifier( + "void", + ), + span: ( + 4, + 37, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 41, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0141/code.php b/tests/0141/code.php new file mode 100644 index 00000000..5395fb09 --- /dev/null +++ b/tests/0141/code.php @@ -0,0 +1,5 @@ + Parse error: unexpected token `private`, expecting `const`, or `function` on line 4 column 5 diff --git a/tests/0141/tokens.txt b/tests/0141/tokens.txt new file mode 100644 index 00000000..6cf60816 --- /dev/null +++ b/tests/0141/tokens.txt @@ -0,0 +1,85 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Interface, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 15, + ), + }, + Token { + kind: Private, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 13, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 4, + 22, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 25, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 26, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 27, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0142/code.php b/tests/0142/code.php new file mode 100644 index 00000000..2b8eda87 --- /dev/null +++ b/tests/0142/code.php @@ -0,0 +1,5 @@ + Parse error: unexpected token `protected`, expecting `const`, or `function` on line 4 column 5 diff --git a/tests/0142/tokens.txt b/tests/0142/tokens.txt new file mode 100644 index 00000000..b313fe6e --- /dev/null +++ b/tests/0142/tokens.txt @@ -0,0 +1,85 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Interface, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 15, + ), + }, + Token { + kind: Protected, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 15, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 4, + 24, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 27, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 28, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 29, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0143/ast.txt b/tests/0143/ast.txt new file mode 100644 index 00000000..fde9fd17 --- /dev/null +++ b/tests/0143/ast.txt @@ -0,0 +1,21 @@ +[ + Interface { + name: Identifier { + name: "foo", + }, + extends: [], + body: [ + AbstractMethod { + name: Identifier { + name: "bar", + }, + params: [], + flags: [ + Public, + ], + return_type: None, + by_ref: false, + }, + ], + }, +] diff --git a/tests/0143/code.php b/tests/0143/code.php new file mode 100644 index 00000000..ed3e83ae --- /dev/null +++ b/tests/0143/code.php @@ -0,0 +1,5 @@ + Parse error: unexpected token `final`, expecting `const`, or `function` on line 4 column 5 diff --git a/tests/0144/tokens.txt b/tests/0144/tokens.txt new file mode 100644 index 00000000..31f09dc1 --- /dev/null +++ b/tests/0144/tokens.txt @@ -0,0 +1,92 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Interface, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 15, + ), + }, + Token { + kind: Final, + span: ( + 4, + 5, + ), + }, + Token { + kind: Public, + span: ( + 4, + 11, + ), + }, + Token { + kind: Function, + span: ( + 4, + 18, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 4, + 27, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 30, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 31, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 32, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0145/ast.txt b/tests/0145/ast.txt new file mode 100644 index 00000000..89d413b0 --- /dev/null +++ b/tests/0145/ast.txt @@ -0,0 +1,22 @@ +[ + Interface { + name: Identifier { + name: "foo", + }, + extends: [], + body: [ + AbstractMethod { + name: Identifier { + name: "bar", + }, + params: [], + flags: [ + Public, + Static, + ], + return_type: None, + by_ref: false, + }, + ], + }, +] diff --git a/tests/0145/code.php b/tests/0145/code.php new file mode 100644 index 00000000..71fb9f7e --- /dev/null +++ b/tests/0145/code.php @@ -0,0 +1,5 @@ + Date: Thu, 1 Dec 2022 03:49:51 +0100 Subject: [PATCH 03/13] feat: make parser, and lexer immutable --- bin/snapshot.rs | 13 +- build.rs | 32 +- src/lexer/mod.rs | 657 ++++++++--------- src/lexer/state.rs | 80 +++ src/main.rs | 4 +- src/parser/block.rs | 15 +- src/parser/classish.rs | 176 ++--- src/parser/classish_statement.rs | 225 +++--- src/parser/comments.rs | 30 - src/parser/flags.rs | 61 +- src/parser/functions.rs | 62 +- src/parser/ident.rs | 55 +- src/parser/macros.rs | 68 +- src/parser/mod.rs | 1132 +++++++++++++++--------------- src/parser/params.rs | 63 +- src/parser/punc.rs | 35 +- src/parser/state.rs | 59 ++ src/parser/vars.rs | 19 +- tests/third_party_tests.rs | 2 +- 19 files changed, 1428 insertions(+), 1360 deletions(-) create mode 100644 src/lexer/state.rs delete mode 100644 src/parser/comments.rs create mode 100644 src/parser/state.rs diff --git a/bin/snapshot.rs b/bin/snapshot.rs index 378af610..a738c5d0 100644 --- a/bin/snapshot.rs +++ b/bin/snapshot.rs @@ -3,6 +3,9 @@ use std::env; use std::fs::read_dir; use std::path::PathBuf; +static PARSER: Parser = Parser::new(); +static LEXER: Lexer = Lexer::new(); + fn main() { let manifest = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); let mut entries = read_dir(manifest.join("tests")) @@ -14,10 +17,6 @@ fn main() { entries.sort(); - let mut content = String::new(); - content.push_str("/// this file is auto-generated by the build script.\n"); - content.push_str("/// you should never manually change it.\n\n\n"); - for entry in entries { let code_filename = entry.join("code.php"); let ast_filename = entry.join("ast.txt"); @@ -46,8 +45,7 @@ fn main() { } let code = std::fs::read_to_string(&code_filename).unwrap(); - let mut lexer = Lexer::new(); - let tokens = lexer.tokenize(code.as_bytes()); + let tokens = LEXER.tokenize(code.as_bytes()); match tokens { Ok(tokens) => { @@ -57,8 +55,7 @@ fn main() { entry.to_string_lossy() ); - let mut parser = Parser::new(None); - let ast = parser.parse(tokens); + let ast = PARSER.parse(tokens); match ast { Ok(ast) => { std::fs::write(ast_filename, format!("{:#?}\n", ast)).unwrap(); diff --git a/build.rs b/build.rs index a4d3e13e..2e173c09 100644 --- a/build.rs +++ b/build.rs @@ -26,7 +26,11 @@ fn main() { let mut content = String::new(); content.push_str("/// this file is auto-generated by the build script.\n"); - content.push_str("/// you should never manually change it.\n\n\n"); + content.push_str("/// you should never manually change it.\n\n"); + content.push_str("use php_parser_rs::prelude::{Lexer, Parser};\n"); + content.push_str("use pretty_assertions::assert_str_eq;\n\n"); + content.push_str("static PARSER: Parser = Parser::new();\n"); + content.push_str("static LEXER: Lexer = Lexer::new();\n\n"); for entry in entries { let code_filename = entry.join("code.php"); @@ -98,10 +102,6 @@ fn build_success_test( format!( r#"#[test] fn test_success_{}() {{ - use php_parser_rs::prelude::Parser; - use php_parser_rs::prelude::Lexer; - use pretty_assertions::assert_str_eq; - let code_filename = "{}"; let ast_filename = "{}"; let tokens_filename = "{}"; @@ -110,13 +110,11 @@ fn test_success_{}() {{ let expected_ast = std::fs::read_to_string(&ast_filename).unwrap(); let expected_tokens = std::fs::read_to_string(&tokens_filename).unwrap(); - let mut lexer = Lexer::new(); - let tokens = lexer.tokenize(code.as_bytes()).unwrap(); + let tokens = LEXER.tokenize(code.as_bytes()).unwrap(); assert_str_eq!(expected_tokens.trim(), format!("{{:#?}}", tokens)); - let mut parser = Parser::new(None); - let ast = parser.parse(tokens).unwrap(); + let ast = PARSER.parse(tokens).unwrap(); assert_str_eq!(expected_ast.trim(), format!("{{:#?}}", ast)); }} @@ -137,17 +135,13 @@ fn build_lexer_error_test( format!( r#"#[test] fn test_lexer_error_{}() {{ - use php_parser_rs::prelude::Lexer; - use pretty_assertions::assert_str_eq; - let code_filename = "{}"; let lexer_error_filename = "{}"; let code = std::fs::read_to_string(&code_filename).unwrap(); let expected_error = std::fs::read_to_string(&lexer_error_filename).unwrap(); - let mut lexer = Lexer::new(); - let error = lexer.tokenize(code.as_bytes()).err().unwrap(); + let error = LEXER.tokenize(code.as_bytes()).err().unwrap(); assert_str_eq!( expected_error.trim(), @@ -171,10 +165,6 @@ fn build_parser_error_test( format!( r#"#[test] fn test_paser_error_{}() {{ - use php_parser_rs::prelude::Parser; - use php_parser_rs::prelude::Lexer; - use pretty_assertions::assert_str_eq; - let code_filename = "{}"; let tokens_filename = "{}"; let parser_error_filename = "{}"; @@ -183,13 +173,11 @@ fn test_paser_error_{}() {{ let expected_tokens = std::fs::read_to_string(&tokens_filename).unwrap(); let expected_error = std::fs::read_to_string(&parser_error_filename).unwrap(); - let mut lexer = Lexer::new(); - let tokens = lexer.tokenize(code.as_bytes()).unwrap(); + let tokens = LEXER.tokenize(code.as_bytes()).unwrap(); assert_str_eq!(expected_tokens.trim(), format!("{{:#?}}", tokens)); - let mut parser = Parser::new(None); - let error = parser.parse(tokens).err().unwrap(); + let error = PARSER.parse(tokens).err().unwrap(); assert_str_eq!( expected_error.trim(), diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index 46dc993b..de47b286 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -1,12 +1,16 @@ pub mod byte_string; pub mod error; -mod macros; pub mod token; +mod macros; +mod state; + use std::num::IntErrorKind; use crate::lexer::byte_string::ByteString; use crate::lexer::error::SyntaxError; +use crate::lexer::state::StackState; +use crate::lexer::state::State; use crate::lexer::token::OpenTagKind; use crate::lexer::token::Span; use crate::lexer::token::Token; @@ -15,104 +19,71 @@ use crate::lexer::token::TokenKind; use crate::ident; use crate::ident_start; -#[derive(Debug, PartialEq, Eq)] -pub enum LexerState { - Initial, - Scripting, - Halted, - DoubleQuote, - LookingForVarname, - LookingForProperty, - VarOffset, -} - -pub struct Lexer { - state_stack: Vec, - chars: Vec, - cursor: usize, - current: Option, - span: Span, -} - -impl Default for Lexer { - fn default() -> Self { - Self::new() - } -} +#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)] +pub struct Lexer; impl Lexer { - pub fn new() -> Self { - Self { - state_stack: vec![LexerState::Initial], - chars: Vec::new(), - cursor: 0, - current: None, - span: (1, 1), - } + pub const fn new() -> Self { + Self {} } - pub fn tokenize>( - &mut self, - input: &B, - ) -> Result, SyntaxError> { + pub fn tokenize>(&self, input: &B) -> Result, SyntaxError> { + let mut state = State::new(input); let mut tokens = Vec::new(); - self.chars = input.as_ref().to_vec(); - self.current = self.chars.first().copied(); - - while self.current.is_some() { - match self.state_stack.last().unwrap() { + while state.current.is_some() { + match state.stack.last().unwrap() { // The "Initial" state is used to parse inline HTML. It is essentially a catch-all // state that will build up a single token buffer until it encounters an open tag // of some description. - LexerState::Initial => { - tokens.append(&mut self.initial()?); + StackState::Initial => { + tokens.append(&mut self.initial(&mut state)?); } // The scripting state is entered when an open tag is encountered in the source code. // This tells the lexer to start analysing characters at PHP tokens instead of inline HTML. - LexerState::Scripting => { - self.skip_whitespace(); + StackState::Scripting => { + self.skip_whitespace(&mut state); // If we have consumed whitespace and then reached the end of the file, we should break. - if self.current.is_none() { + if state.current.is_none() { break; } - tokens.push(self.scripting()?); + tokens.push(self.scripting(&mut state)?); } // The "Halted" state is entered when the `__halt_compiler` token is encountered. // In this state, all the text that follows is no longer parsed as PHP as is collected // into a single "InlineHtml" token (kind of cheating, oh well). - LexerState::Halted => { + StackState::Halted => { tokens.push(Token { - kind: TokenKind::InlineHtml(self.chars[self.cursor..].into()), - span: self.span, + kind: TokenKind::InlineHtml(state.chars[state.cursor..].into()), + span: state.span, }); break; } // The double quote state is entered when inside a double-quoted string that // contains variables. - LexerState::DoubleQuote => tokens.extend(self.double_quote()?), + StackState::DoubleQuote => tokens.extend(self.double_quote(&mut state)?), // LookingForProperty is entered inside double quotes, // backticks, or a heredoc, expecting a variable name. // If one isn't found, it switches to scripting. - LexerState::LookingForVarname => { - if let Some(token) = self.looking_for_varname() { + StackState::LookingForVarname => { + if let Some(token) = self.looking_for_varname(&mut state) { tokens.push(token); } } // LookingForProperty is entered inside double quotes, // backticks, or a heredoc, expecting an arrow followed by a // property name. - LexerState::LookingForProperty => { - tokens.push(self.looking_for_property()?); + StackState::LookingForProperty => { + tokens.push(self.looking_for_property(&mut state)?); } - LexerState::VarOffset => { - if self.current.is_none() { + StackState::VarOffset => { + if state.current.is_none() { break; } - tokens.push(self.var_offset()?); + tokens.push(self.var_offset(&mut state)?); } } } @@ -120,21 +91,21 @@ impl Lexer { Ok(tokens) } - fn skip_whitespace(&mut self) { - while let Some(b' ' | b'\n' | b'\r' | b'\t') = self.current { - self.next(); + fn skip_whitespace(&self, state: &mut State) { + while let Some(b' ' | b'\n' | b'\r' | b'\t') = state.current { + state.next(); } } - fn initial(&mut self) -> Result, SyntaxError> { - let inline_span = self.span; + fn initial(&self, state: &mut State) -> Result, SyntaxError> { + let inline_span = state.span; let mut buffer = Vec::new(); - while let Some(char) = self.current { - if self.try_read(b" Result { - let span = self.span; - let kind = match self.peek_buf() { + fn scripting(&self, state: &mut State) -> Result { + let span = state.span; + let kind = match state.peek_buf() { [b'@', ..] => { - self.next(); + state.next(); TokenKind::At } [b'!', b'=', b'=', ..] => { - self.skip(3); + state.skip(3); TokenKind::BangDoubleEquals } [b'!', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::BangEquals } [b'!', ..] => { - self.next(); + state.next(); TokenKind::Bang } [b'&', b'&', ..] => { - self.skip(2); + state.skip(2); TokenKind::BooleanAnd } [b'&', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::AmpersandEquals } [b'&', ..] => { - self.next(); + state.next(); TokenKind::Ampersand } [b'?', b'>', ..] => { // This is a close tag, we can enter "Initial" mode again. - self.skip(2); + state.skip(2); - self.enter_state(LexerState::Initial); + state.enter_state(StackState::Initial); TokenKind::CloseTag } [b'?', b'?', b'=', ..] => { - self.skip(3); + state.skip(3); TokenKind::CoalesceEqual } [b'?', b'?', ..] => { - self.skip(2); + state.skip(2); TokenKind::Coalesce } [b'?', b':', ..] => { - self.skip(2); + state.skip(2); TokenKind::QuestionColon } [b'?', b'-', b'>', ..] => { - self.skip(3); + state.skip(3); TokenKind::NullsafeArrow } [b'?', ..] => { - self.next(); + state.next(); TokenKind::Question } [b'=', b'>', ..] => { - self.skip(2); + state.skip(2); TokenKind::DoubleArrow } [b'=', b'=', b'=', ..] => { - self.skip(3); + state.skip(3); TokenKind::TripleEquals } [b'=', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::DoubleEquals } [b'=', ..] => { - self.next(); + state.next(); TokenKind::Equals } // Single quoted string. [b'\'', ..] => { - self.next(); - self.tokenize_single_quote_string()? + state.next(); + self.tokenize_single_quote_string(state)? } [b'b' | b'B', b'\'', ..] => { - self.skip(2); - self.tokenize_single_quote_string()? + state.skip(2); + self.tokenize_single_quote_string(state)? } [b'"', ..] => { - self.next(); - self.tokenize_double_quote_string()? + state.next(); + self.tokenize_double_quote_string(state)? } [b'b' | b'B', b'"', ..] => { - self.skip(2); - self.tokenize_double_quote_string()? + state.skip(2); + self.tokenize_double_quote_string(state)? } [b'$', ident_start!(), ..] => { - self.next(); - self.tokenize_variable() + state.next(); + self.tokenize_variable(state) } [b'$', ..] => { - self.next(); + state.next(); TokenKind::Dollar } [b'.', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::DotEquals } - [b'.', b'0'..=b'9', ..] => self.tokenize_number()?, + [b'.', b'0'..=b'9', ..] => self.tokenize_number(state)?, [b'.', b'.', b'.', ..] => { - self.skip(3); + state.skip(3); TokenKind::Ellipsis } [b'.', ..] => { - self.next(); + state.next(); TokenKind::Dot } - &[b'0'..=b'9', ..] => self.tokenize_number()?, + &[b'0'..=b'9', ..] => self.tokenize_number(state)?, &[b'\\', ident_start!(), ..] => { - self.next(); + state.next(); - match self.scripting()? { + match self.scripting(state)? { Token { kind: TokenKind::Identifier(ByteString(mut i)) @@ -295,19 +266,19 @@ impl Lexer { } } [b'\\', ..] => { - self.next(); + state.next(); TokenKind::NamespaceSeparator } &[b @ ident_start!(), ..] => { - self.next(); + state.next(); let mut qualified = false; let mut last_was_slash = false; let mut buffer = vec![b]; - while let Some(next) = self.current { + while let Some(next) = state.current { if next.is_ascii_alphanumeric() || next == b'_' { buffer.push(next); - self.next(); + state.next(); last_was_slash = false; continue; } @@ -316,7 +287,7 @@ impl Lexer { qualified = true; last_was_slash = true; buffer.push(next); - self.next(); + state.next(); continue; } @@ -330,12 +301,12 @@ impl Lexer { .unwrap_or_else(|| TokenKind::Identifier(buffer.into())); if kind == TokenKind::HaltCompiler { - match self.peek_buf() { + match state.peek_buf() { [b'(', b')', b';', ..] => { - self.skip(3); - self.enter_state(LexerState::Halted); + state.skip(3); + state.enter_state(StackState::Halted); } - _ => return Err(SyntaxError::InvalidHaltCompiler(self.span)), + _ => return Err(SyntaxError::InvalidHaltCompiler(state.span)), } } @@ -343,24 +314,24 @@ impl Lexer { } } [b'/', b'*', ..] => { - self.next(); + state.next(); let mut buffer = vec![b'/']; - while self.current.is_some() { - match self.peek_buf() { + while state.current.is_some() { + match state.peek_buf() { [b'*', b'/', ..] => { - self.skip(2); + state.skip(2); buffer.extend_from_slice(b"*/"); break; } &[t, ..] => { - self.next(); + state.next(); buffer.push(t); } [] => unreachable!(), } } - self.next(); + state.next(); if buffer.starts_with(b"/**") { TokenKind::DocComment(buffer.into()) @@ -369,287 +340,287 @@ impl Lexer { } } [b'#', b'[', ..] => { - self.skip(2); + state.skip(2); TokenKind::Attribute } &[ch @ b'/', b'/', ..] | &[ch @ b'#', ..] => { let mut buffer = if ch == b'/' { - self.skip(2); + state.skip(2); b"//".to_vec() } else { - self.next(); + state.next(); b"#".to_vec() }; - while let Some(c) = self.current { + while let Some(c) = state.current { if c == b'\n' { break; } buffer.push(c); - self.next(); + state.next(); } - self.next(); + state.next(); TokenKind::Comment(buffer.into()) } [b'/', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::SlashEquals } [b'/', ..] => { - self.next(); + state.next(); TokenKind::Slash } [b'*', b'*', b'=', ..] => { - self.skip(3); + state.skip(3); TokenKind::PowEquals } [b'*', b'*', ..] => { - self.skip(2); + state.skip(2); TokenKind::Pow } [b'*', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::AsteriskEqual } [b'*', ..] => { - self.next(); + state.next(); TokenKind::Asterisk } [b'|', b'|', ..] => { - self.skip(2); + state.skip(2); TokenKind::Pipe } [b'|', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::PipeEquals } [b'|', ..] => { - self.next(); + state.next(); TokenKind::Pipe } [b'^', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::CaretEquals } [b'^', ..] => { - self.next(); + state.next(); TokenKind::Caret } [b'{', ..] => { - self.next(); - self.push_state(LexerState::Scripting); + state.next(); + state.push_state(StackState::Scripting); TokenKind::LeftBrace } [b'}', ..] => { - self.next(); - self.pop_state(); + state.next(); + state.pop_state(); TokenKind::RightBrace } [b'(', ..] => { - self.next(); + state.next(); - if self.try_read(b"int)") { - self.skip(4); + if state.try_read(b"int)") { + state.skip(4); TokenKind::IntCast - } else if self.try_read(b"integer)") { - self.skip(8); + } else if state.try_read(b"integer)") { + state.skip(8); TokenKind::IntegerCast - } else if self.try_read(b"bool)") { - self.skip(5); + } else if state.try_read(b"bool)") { + state.skip(5); TokenKind::BoolCast - } else if self.try_read(b"boolean)") { - self.skip(8); + } else if state.try_read(b"boolean)") { + state.skip(8); TokenKind::BooleanCast - } else if self.try_read(b"float)") { - self.skip(6); + } else if state.try_read(b"float)") { + state.skip(6); TokenKind::FloatCast - } else if self.try_read(b"double)") { - self.skip(7); + } else if state.try_read(b"double)") { + state.skip(7); TokenKind::DoubleCast - } else if self.try_read(b"real)") { - self.skip(5); + } else if state.try_read(b"real)") { + state.skip(5); TokenKind::RealCast - } else if self.try_read(b"string)") { - self.skip(7); + } else if state.try_read(b"string)") { + state.skip(7); TokenKind::StringCast - } else if self.try_read(b"binary)") { - self.skip(7); + } else if state.try_read(b"binary)") { + state.skip(7); TokenKind::BinaryCast - } else if self.try_read(b"array)") { - self.skip(6); + } else if state.try_read(b"array)") { + state.skip(6); TokenKind::ArrayCast - } else if self.try_read(b"object)") { - self.skip(7); + } else if state.try_read(b"object)") { + state.skip(7); TokenKind::ObjectCast - } else if self.try_read(b"unset)") { - self.skip(6); + } else if state.try_read(b"unset)") { + state.skip(6); TokenKind::UnsetCast } else { TokenKind::LeftParen } } [b')', ..] => { - self.next(); + state.next(); TokenKind::RightParen } [b';', ..] => { - self.next(); + state.next(); TokenKind::SemiColon } [b'+', b'+', ..] => { - self.skip(2); + state.skip(2); TokenKind::Increment } [b'+', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::PlusEquals } [b'+', ..] => { - self.next(); + state.next(); TokenKind::Plus } [b'%', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::PercentEquals } [b'%', ..] => { - self.next(); + state.next(); TokenKind::Percent } [b'-', b'-', ..] => { - self.skip(2); + state.skip(2); TokenKind::Decrement } [b'-', b'>', ..] => { - self.skip(2); + state.skip(2); TokenKind::Arrow } [b'-', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::MinusEquals } [b'-', ..] => { - self.next(); + state.next(); TokenKind::Minus } [b'<', b'<', b'<', ..] => { // TODO: Handle both heredocs and nowdocs. - self.skip(3); + state.skip(3); todo!("heredocs & nowdocs"); } [b'<', b'<', b'=', ..] => { - self.skip(3); + state.skip(3); TokenKind::LeftShiftEquals } [b'<', b'<', ..] => { - self.skip(2); + state.skip(2); TokenKind::LeftShift } [b'<', b'=', b'>', ..] => { - self.skip(3); + state.skip(3); TokenKind::Spaceship } [b'<', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::LessThanEquals } [b'<', b'>', ..] => { - self.skip(2); + state.skip(2); TokenKind::AngledLeftRight } [b'<', ..] => { - self.next(); + state.next(); TokenKind::LessThan } [b'>', b'>', b'=', ..] => { - self.skip(3); + state.skip(3); TokenKind::RightShiftEquals } [b'>', b'>', ..] => { - self.skip(2); + state.skip(2); TokenKind::RightShift } [b'>', b'=', ..] => { - self.skip(2); + state.skip(2); TokenKind::GreaterThanEquals } [b'>', ..] => { - self.next(); + state.next(); TokenKind::GreaterThan } [b',', ..] => { - self.next(); + state.next(); TokenKind::Comma } [b'[', ..] => { - self.next(); + state.next(); TokenKind::LeftBracket } [b']', ..] => { - self.next(); + state.next(); TokenKind::RightBracket } [b':', b':', ..] => { - self.skip(2); + state.skip(2); TokenKind::DoubleColon } [b':', ..] => { - self.next(); + state.next(); TokenKind::Colon } &[b'~', ..] => { - self.next(); + state.next(); TokenKind::BitwiseNot } &[b, ..] => unimplemented!( " char: {}, line: {}, col: {}", b as char, - self.span.0, - self.span.1 + state.span.0, + state.span.1 ), // We should never reach this point since we have the empty checks surrounding // the call to this function, but it's better to be safe than sorry. - [] => return Err(SyntaxError::UnexpectedEndOfFile(self.span)), + [] => return Err(SyntaxError::UnexpectedEndOfFile(state.span)), }; Ok(Token { kind, span }) } - fn double_quote(&mut self) -> Result, SyntaxError> { - let span = self.span; + fn double_quote(&self, state: &mut State) -> Result, SyntaxError> { + let span = state.span; let mut buffer = Vec::new(); let kind = loop { - match self.peek_buf() { + match state.peek_buf() { [b'$', b'{', ..] => { - self.skip(2); - self.push_state(LexerState::LookingForVarname); + state.skip(2); + state.push_state(StackState::LookingForVarname); break TokenKind::DollarLeftBrace; } [b'{', b'$', ..] => { // Intentionally only consume the left brace. - self.next(); - self.push_state(LexerState::Scripting); + state.next(); + state.push_state(StackState::Scripting); break TokenKind::LeftBrace; } [b'"', ..] => { - self.next(); - self.enter_state(LexerState::Scripting); + state.next(); + state.enter_state(StackState::Scripting); break TokenKind::DoubleQuote; } [b'$', ident_start!(), ..] => { - self.next(); - let ident = self.consume_identifier(); + state.next(); + let ident = self.consume_identifier(state); - match self.peek_buf() { - [b'[', ..] => self.push_state(LexerState::VarOffset), + match state.peek_buf() { + [b'[', ..] => state.push_state(StackState::VarOffset), [b'-', b'>', ident_start!(), ..] | [b'?', b'-', b'>', ident_start!(), ..] => { - self.push_state(LexerState::LookingForProperty) + state.push_state(StackState::LookingForProperty) } _ => {} } @@ -657,10 +628,10 @@ impl Lexer { break TokenKind::Variable(ident.into()); } &[b, ..] => { - self.next(); + state.next(); buffer.push(b); } - [] => return Err(SyntaxError::UnexpectedEndOfFile(self.span)), + [] => return Err(SyntaxError::UnexpectedEndOfFile(state.span)), } }; @@ -676,13 +647,15 @@ impl Lexer { Ok(tokens) } - fn looking_for_varname(&mut self) -> Option { - if let Some(ident) = self.peek_identifier() { - if let Some(b'[' | b'}') = self.peek_byte(ident.len()) { + fn looking_for_varname(&self, state: &mut State) -> Option { + let identifier = self.peek_identifier(state); + + if let Some(ident) = identifier { + if let Some(b'[' | b'}') = state.peek_byte(ident.len()) { let ident = ident.to_vec(); - let span = self.span; - self.skip(ident.len()); - self.enter_state(LexerState::Scripting); + let span = state.span; + state.skip(ident.len()); + state.enter_state(StackState::Scripting); return Some(Token { kind: TokenKind::Identifier(ident.into()), span, @@ -690,24 +663,24 @@ impl Lexer { } } - self.enter_state(LexerState::Scripting); + state.enter_state(StackState::Scripting); None } - fn looking_for_property(&mut self) -> Result { - let span = self.span; - let kind = match self.peek_buf() { + fn looking_for_property(&self, state: &mut State) -> Result { + let span = state.span; + let kind = match state.peek_buf() { [b'-', b'>', ..] => { - self.skip(2); + state.skip(2); TokenKind::Arrow } [b'?', b'-', b'>', ..] => { - self.skip(3); + state.skip(3); TokenKind::NullsafeArrow } &[ident_start!(), ..] => { - let buffer = self.consume_identifier(); - self.pop_state(); + let buffer = self.consume_identifier(state); + state.pop_state(); TokenKind::Identifier(buffer.into()) } // Should be impossible as we already looked ahead this far inside double_quote. @@ -716,114 +689,114 @@ impl Lexer { Ok(Token { kind, span }) } - fn var_offset(&mut self) -> Result { - let span = self.span; - let kind = match self.peek_buf() { + fn var_offset(&self, state: &mut State) -> Result { + let span = state.span; + let kind = match state.peek_buf() { [b'$', ident_start!(), ..] => { - self.next(); - self.tokenize_variable() + state.next(); + self.tokenize_variable(state) } &[b'0'..=b'9', ..] => { // TODO: all integer literals are allowed, but only decimal integers with no underscores // are actually treated as numbers. Others are treated as strings. // Float literals are not allowed, but that could be handled in the parser. - self.tokenize_number()? + self.tokenize_number(state)? } [b'[', ..] => { - self.next(); + state.next(); TokenKind::LeftBracket } [b'-', ..] => { - self.next(); + state.next(); TokenKind::Minus } [b']', ..] => { - self.next(); - self.pop_state(); + state.next(); + state.pop_state(); TokenKind::RightBracket } &[ident_start!(), ..] => { - let label = self.consume_identifier(); + let label = self.consume_identifier(state); TokenKind::Identifier(label.into()) } &[b, ..] => unimplemented!( " char: {}, line: {}, col: {}", b as char, - self.span.0, - self.span.1 + state.span.0, + state.span.1 ), - [] => return Err(SyntaxError::UnexpectedEndOfFile(self.span)), + [] => return Err(SyntaxError::UnexpectedEndOfFile(state.span)), }; Ok(Token { kind, span }) } - fn tokenize_single_quote_string(&mut self) -> Result { + fn tokenize_single_quote_string(&self, state: &mut State) -> Result { let mut buffer = Vec::new(); loop { - match self.peek_buf() { + match state.peek_buf() { [b'\'', ..] => { - self.next(); + state.next(); break; } &[b'\\', b @ b'\'' | b @ b'\\', ..] => { - self.skip(2); + state.skip(2); buffer.push(b); } &[b, ..] => { - self.next(); + state.next(); buffer.push(b); } - [] => return Err(SyntaxError::UnexpectedEndOfFile(self.span)), + [] => return Err(SyntaxError::UnexpectedEndOfFile(state.span)), } } Ok(TokenKind::LiteralString(buffer.into())) } - fn tokenize_double_quote_string(&mut self) -> Result { + fn tokenize_double_quote_string(&self, state: &mut State) -> Result { let mut buffer = Vec::new(); let constant = loop { - match self.peek_buf() { + match state.peek_buf() { [b'"', ..] => { - self.next(); + state.next(); break true; } &[b'\\', b @ (b'"' | b'\\' | b'$'), ..] => { - self.skip(2); + state.skip(2); buffer.push(b); } &[b'\\', b'n', ..] => { - self.skip(2); + state.skip(2); buffer.push(b'\n'); } &[b'\\', b'r', ..] => { - self.skip(2); + state.skip(2); buffer.push(b'\r'); } &[b'\\', b't', ..] => { - self.skip(2); + state.skip(2); buffer.push(b'\t'); } &[b'\\', b'v', ..] => { - self.skip(2); + state.skip(2); buffer.push(b'\x0b'); } &[b'\\', b'e', ..] => { - self.skip(2); + state.skip(2); buffer.push(b'\x1b'); } &[b'\\', b'f', ..] => { - self.skip(2); + state.skip(2); buffer.push(b'\x0c'); } &[b'\\', b'x', b @ (b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'), ..] => { - self.skip(3); + state.skip(3); let mut hex = String::from(b as char); - if let Some(b @ (b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F')) = self.current { - self.next(); + if let Some(b @ (b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F')) = state.current { + state.next(); hex.push(b as char); } @@ -831,23 +804,23 @@ impl Lexer { buffer.push(b); } &[b'\\', b'u', b'{', ..] => { - self.skip(3); + state.skip(3); let mut code_point = String::new(); - while let Some(b @ (b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F')) = self.current { - self.next(); + while let Some(b @ (b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F')) = state.current { + state.next(); code_point.push(b as char); } - if code_point.is_empty() || self.current != Some(b'}') { - return Err(SyntaxError::InvalidUnicodeEscape(self.span)); + if code_point.is_empty() || state.current != Some(b'}') { + return Err(SyntaxError::InvalidUnicodeEscape(state.span)); } - self.next(); + state.next(); let c = if let Ok(c) = u32::from_str_radix(&code_point, 16) { c } else { - return Err(SyntaxError::InvalidUnicodeEscape(self.span)); + return Err(SyntaxError::InvalidUnicodeEscape(state.span)); }; if let Some(c) = char::from_u32(c) { @@ -855,85 +828,85 @@ impl Lexer { let bytes = c.encode_utf8(&mut tmp); buffer.extend(bytes.as_bytes()); } else { - return Err(SyntaxError::InvalidUnicodeEscape(self.span)); + return Err(SyntaxError::InvalidUnicodeEscape(state.span)); } } &[b'\\', b @ b'0'..=b'7', ..] => { - self.skip(2); + state.skip(2); let mut octal = String::from(b as char); - if let Some(b @ b'0'..=b'7') = self.current { - self.next(); + if let Some(b @ b'0'..=b'7') = state.current { + state.next(); octal.push(b as char); } - if let Some(b @ b'0'..=b'7') = self.current { - self.next(); + if let Some(b @ b'0'..=b'7') = state.current { + state.next(); octal.push(b as char); } if let Ok(b) = u8::from_str_radix(&octal, 8) { buffer.push(b); } else { - return Err(SyntaxError::InvalidOctalEscape(self.span)); + return Err(SyntaxError::InvalidOctalEscape(state.span)); } } [b'$', ident_start!(), ..] | [b'{', b'$', ..] | [b'$', b'{', ..] => { break false; } &[b, ..] => { - self.next(); + state.next(); buffer.push(b); } - [] => return Err(SyntaxError::UnexpectedEndOfFile(self.span)), + [] => return Err(SyntaxError::UnexpectedEndOfFile(state.span)), } }; Ok(if constant { TokenKind::LiteralString(buffer.into()) } else { - self.enter_state(LexerState::DoubleQuote); + state.enter_state(StackState::DoubleQuote); TokenKind::StringPart(buffer.into()) }) } - fn peek_identifier(&self) -> Option<&[u8]> { - let mut cursor = self.cursor; - if let Some(ident_start!()) = self.chars.get(cursor) { + fn peek_identifier<'a>(&'a self, state: &'a State) -> Option<&[u8]> { + let mut cursor = state.cursor; + if let Some(ident_start!()) = state.chars.get(cursor) { cursor += 1; - while let Some(ident!()) = self.chars.get(cursor) { + while let Some(ident!()) = state.chars.get(cursor) { cursor += 1; } - Some(&self.chars[self.cursor..cursor]) + Some(&state.chars[state.cursor..cursor]) } else { None } } - fn consume_identifier(&mut self) -> Vec { - let ident = self.peek_identifier().unwrap().to_vec(); - self.skip(ident.len()); + fn consume_identifier(&self, state: &mut State) -> Vec { + let ident = self.peek_identifier(state).unwrap().to_vec(); + state.skip(ident.len()); ident } - fn tokenize_variable(&mut self) -> TokenKind { - TokenKind::Variable(self.consume_identifier().into()) + fn tokenize_variable(&self, state: &mut State) -> TokenKind { + TokenKind::Variable(self.consume_identifier(state).into()) } - fn tokenize_number(&mut self) -> Result { + fn tokenize_number(&self, state: &mut State) -> Result { let mut buffer = String::new(); - let (base, kind) = match self.peek_buf() { + let (base, kind) = match state.peek_buf() { [b'0', b'B' | b'b', ..] => { - self.skip(2); + state.skip(2); (2, NumberKind::Int) } [b'0', b'O' | b'o', ..] => { - self.skip(2); + state.skip(2); (8, NumberKind::Int) } [b'0', b'X' | b'x', ..] => { - self.skip(2); + state.skip(2); (16, NumberKind::Int) } [b'0', ..] => (10, NumberKind::OctalOrFloat), @@ -942,15 +915,15 @@ impl Lexer { }; if kind != NumberKind::Float { - self.read_digits(&mut buffer, base); + self.read_digits(state, &mut buffer, base); if kind == NumberKind::Int { - return parse_int(&buffer, base as u32, self.span); + return parse_int(&buffer, base as u32, state.span); } } // Remaining cases: decimal integer, legacy octal integer, or float. let is_float = matches!( - self.peek_buf(), + state.peek_buf(), [b'.', ..] | [b'e' | b'E', b'-' | b'+', b'0'..=b'9', ..] | [b'e' | b'E', b'0'..=b'9', ..] @@ -961,55 +934,60 @@ impl Lexer { } else { 10 }; - return parse_int(&buffer, base as u32, self.span); + return parse_int(&buffer, base as u32, state.span); } - if self.current == Some(b'.') { + if state.current == Some(b'.') { buffer.push('.'); - self.next(); - self.read_digits(&mut buffer, 10); + state.next(); + self.read_digits(state, &mut buffer, 10); } - if let Some(b'e' | b'E') = self.current { + if let Some(b'e' | b'E') = state.current { buffer.push('e'); - self.next(); - if let Some(b @ (b'-' | b'+')) = self.current { + state.next(); + if let Some(b @ (b'-' | b'+')) = state.current { buffer.push(b as char); - self.next(); + state.next(); } - self.read_digits(&mut buffer, 10); + self.read_digits(state, &mut buffer, 10); } Ok(TokenKind::LiteralFloat(buffer.parse().unwrap())) } - fn read_digits(&mut self, buffer: &mut String, base: usize) { + fn read_digits(&self, state: &mut State, buffer: &mut String, base: usize) { if base == 16 { - self.read_digits_fn(buffer, u8::is_ascii_hexdigit); + self.read_digits_fn(state, buffer, u8::is_ascii_hexdigit); } else { let max = b'0' + base as u8; - self.read_digits_fn(buffer, |b| (b'0'..max).contains(b)); + self.read_digits_fn(state, buffer, |b| (b'0'..max).contains(b)); }; } - fn read_digits_fn bool>(&mut self, buffer: &mut String, is_digit: F) { - if let Some(b) = self.current { + fn read_digits_fn bool>( + &self, + state: &mut State, + buffer: &mut String, + is_digit: F, + ) { + if let Some(b) = state.current { if is_digit(&b) { - self.next(); + state.next(); buffer.push(b as char); } else { return; } } loop { - match *self.peek_buf() { + match *state.peek_buf() { [b, ..] if is_digit(&b) => { - self.next(); + state.next(); buffer.push(b as char); } [b'_', b, ..] if is_digit(&b) => { - self.next(); - self.next(); + state.next(); + state.next(); buffer.push(b as char); } _ => { @@ -1018,49 +996,6 @@ impl Lexer { } } } - - fn enter_state(&mut self, state: LexerState) { - *self.state_stack.last_mut().unwrap() = state; - } - - fn push_state(&mut self, state: LexerState) { - self.state_stack.push(state); - } - - fn pop_state(&mut self) { - self.state_stack.pop(); - } - - fn peek_buf(&self) -> &[u8] { - &self.chars[self.cursor..] - } - - fn peek_byte(&self, delta: usize) -> Option { - self.chars.get(self.cursor + delta).copied() - } - - fn try_read(&self, search: &'static [u8]) -> bool { - self.peek_buf().starts_with(search) - } - - fn skip(&mut self, count: usize) { - for _ in 0..count { - self.next(); - } - } - - fn next(&mut self) { - match self.current { - Some(b'\n') => { - self.span.0 += 1; - self.span.1 = 1; - } - Some(_) => self.span.1 += 1, - _ => {} - } - self.cursor += 1; - self.current = self.chars.get(self.cursor).copied(); - } } // Parses an integer literal in the given base and converts errors to SyntaxError. diff --git a/src/lexer/state.rs b/src/lexer/state.rs new file mode 100644 index 00000000..59a0ba80 --- /dev/null +++ b/src/lexer/state.rs @@ -0,0 +1,80 @@ +use crate::lexer::token::Span; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum StackState { + Initial, + Scripting, + Halted, + DoubleQuote, + LookingForVarname, + LookingForProperty, + VarOffset, +} + +// TODO(azjezz): make `chars` a `[u8, N]`, and `State`, `State` +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct State { + pub stack: Vec, + pub chars: Vec, + pub cursor: usize, + pub current: Option, + pub span: Span, +} + +impl State { + pub fn new>(input: &B) -> Self { + let chars = input.as_ref().to_vec(); + let current = chars.first().copied(); + + Self { + stack: vec![StackState::Initial], + chars, + current, + cursor: 0, + span: (1, 1), + } + } + + pub fn enter_state(&mut self, state: StackState) { + *self.stack.last_mut().unwrap() = state; + } + + pub fn push_state(&mut self, state: StackState) { + self.stack.push(state); + } + + pub fn pop_state(&mut self) { + self.stack.pop(); + } + + pub fn peek_buf(&self) -> &[u8] { + &self.chars[self.cursor..] + } + + pub fn peek_byte(&self, delta: usize) -> Option { + self.chars.get(self.cursor + delta).copied() + } + + pub fn try_read(&self, search: &'static [u8]) -> bool { + self.peek_buf().starts_with(search) + } + + pub fn skip(&mut self, count: usize) { + for _ in 0..count { + self.next(); + } + } + + pub fn next(&mut self) { + match self.current { + Some(b'\n') => { + self.span.0 += 1; + self.span.1 = 1; + } + Some(_) => self.span.1 += 1, + _ => {} + } + self.cursor += 1; + self.current = self.chars.get(self.cursor).copied(); + } +} diff --git a/src/main.rs b/src/main.rs index 676b7437..bd54b47d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,7 +20,7 @@ fn main() { } }; - let mut lexer = Lexer::new(); + let lexer = Lexer::new(); let tokens = match lexer.tokenize(contents.as_bytes()) { Ok(tokens) => tokens, Err(error) => { @@ -30,7 +30,7 @@ fn main() { } }; - let mut parser = Parser::new(None); + let parser = Parser::new(); let ast = match parser.parse(tokens) { Ok(ast) => ast, Err(error) => { diff --git a/src/parser/block.rs b/src/parser/block.rs index d1d4a3db..af388651 100644 --- a/src/parser/block.rs +++ b/src/parser/block.rs @@ -1,17 +1,22 @@ use crate::lexer::token::TokenKind; use crate::parser::ast::Block; use crate::parser::error::ParseResult; +use crate::parser::state::State; use crate::parser::Parser; impl Parser { - pub(in crate::parser) fn block(&mut self, until: &TokenKind) -> ParseResult { - self.skip_comments(); + pub(in crate::parser) fn block( + &self, + state: &mut State, + until: &TokenKind, + ) -> ParseResult { + state.skip_comments(); let mut block = Block::new(); - while !self.is_eof() && &self.current.kind != until { - block.push(self.statement()?); - self.skip_comments(); + while !state.is_eof() && &state.current.kind != until { + block.push(self.statement(state)?); + state.skip_comments(); } Ok(block) diff --git a/src/parser/classish.rs b/src/parser/classish.rs index a7c774e8..7dd5f033 100644 --- a/src/parser/classish.rs +++ b/src/parser/classish.rs @@ -6,49 +6,50 @@ use crate::parser::ast::Expression; use crate::parser::ast::Identifier; use crate::parser::ast::Statement; use crate::parser::error::ParseResult; +use crate::parser::state::State; use crate::parser::Parser; use crate::expect_token; use crate::expected_token_err; impl Parser { - pub(in crate::parser) fn class_definition(&mut self) -> ParseResult { - let flags: Vec = self.class_flags()?.iter().map(|f| f.into()).collect(); + pub(in crate::parser) fn class_definition(&self, state: &mut State) -> ParseResult { + let flags: Vec = self.class_flags(state)?.iter().map(|f| f.into()).collect(); - expect_token!([TokenKind::Class], self, ["`class`"]); + expect_token!([TokenKind::Class], state, ["`class`"]); - let name = self.ident()?; + let name = self.ident(state)?; let mut extends: Option = None; - if self.current.kind == TokenKind::Extends { - self.next(); - extends = Some(self.full_name()?.into()); + if state.current.kind == TokenKind::Extends { + state.next(); + extends = Some(self.full_name(state)?.into()); } - let implements = if self.current.kind == TokenKind::Implements { - self.next(); + let implements = if state.current.kind == TokenKind::Implements { + state.next(); - self.at_least_one_comma_separated::(&|parser| { - Ok(parser.full_name()?.into()) + self.at_least_one_comma_separated::(state, &|parser, state| { + Ok(parser.full_name(state)?.into()) })? } else { Vec::new() }; - self.lbrace()?; + self.lbrace(state)?; let mut body = Vec::new(); - while self.current.kind != TokenKind::RightBrace { - self.gather_comments(); + while state.current.kind != TokenKind::RightBrace { + state.gather_comments(); - if self.current.kind == TokenKind::RightBrace { - self.clear_comments(); + if state.current.kind == TokenKind::RightBrace { + state.clear_comments(); break; } - body.push(self.class_statement(flags.clone())?); + body.push(self.class_statement(state, flags.clone())?); } - self.rbrace()?; + self.rbrace(state)?; Ok(Statement::Class { name: name.into(), @@ -59,35 +60,38 @@ impl Parser { }) } - pub(in crate::parser) fn interface_definition(&mut self) -> ParseResult { - expect_token!([TokenKind::Interface], self, ["`interface`"]); + pub(in crate::parser) fn interface_definition( + &self, + state: &mut State, + ) -> ParseResult { + expect_token!([TokenKind::Interface], state, ["`interface`"]); - let name = self.ident()?; + let name = self.ident(state)?; - let extends = if self.current.kind == TokenKind::Extends { - self.next(); + let extends = if state.current.kind == TokenKind::Extends { + state.next(); - self.at_least_one_comma_separated::(&|parser| { - Ok(parser.full_name()?.into()) + self.at_least_one_comma_separated::(state, &|parser, state| { + Ok(parser.full_name(state)?.into()) })? } else { Vec::new() }; - self.lbrace()?; + self.lbrace(state)?; let mut body = Vec::new(); - while self.current.kind != TokenKind::RightBrace && !self.is_eof() { - self.gather_comments(); + while state.current.kind != TokenKind::RightBrace && !state.is_eof() { + state.gather_comments(); - if self.current.kind == TokenKind::RightBrace { - self.clear_comments(); + if state.current.kind == TokenKind::RightBrace { + state.clear_comments(); break; } - body.push(self.interface_statement()?); + body.push(self.interface_statement(state)?); } - self.rbrace()?; + self.rbrace(state)?; Ok(Statement::Interface { name: name.into(), @@ -96,25 +100,25 @@ impl Parser { }) } - pub(in crate::parser) fn trait_definition(&mut self) -> ParseResult { - expect_token!([TokenKind::Trait], self, ["`trait`"]); + pub(in crate::parser) fn trait_definition(&self, state: &mut State) -> ParseResult { + expect_token!([TokenKind::Trait], state, ["`trait`"]); - let name = self.ident()?; + let name = self.ident(state)?; - self.lbrace()?; + self.lbrace(state)?; let mut body = Vec::new(); - while self.current.kind != TokenKind::RightBrace && !self.is_eof() { - self.gather_comments(); + while state.current.kind != TokenKind::RightBrace && !state.is_eof() { + state.gather_comments(); - if self.current.kind == TokenKind::RightBrace { - self.clear_comments(); + if state.current.kind == TokenKind::RightBrace { + state.clear_comments(); break; } - body.push(self.trait_statement()?); + body.push(self.trait_statement(state)?); } - self.rbrace()?; + self.rbrace(state)?; Ok(Statement::Trait { name: name.into(), @@ -122,47 +126,50 @@ impl Parser { }) } - pub(in crate::parser) fn anonymous_class_definition(&mut self) -> ParseResult { - self.next(); + pub(in crate::parser) fn anonymous_class_definition( + &self, + state: &mut State, + ) -> ParseResult { + state.next(); - expect_token!([TokenKind::Class], self, ["`class`"]); + expect_token!([TokenKind::Class], state, ["`class`"]); let mut args = vec![]; - if self.current.kind == TokenKind::LeftParen { - self.lparen()?; + if state.current.kind == TokenKind::LeftParen { + self.lparen(state)?; - args = self.args_list()?; + args = self.args_list(state)?; - self.rparen()?; + self.rparen(state)?; } let mut extends: Option = None; - if self.current.kind == TokenKind::Extends { - self.next(); - extends = Some(self.full_name()?.into()); + if state.current.kind == TokenKind::Extends { + state.next(); + extends = Some(self.full_name(state)?.into()); } let mut implements = Vec::new(); - if self.current.kind == TokenKind::Implements { - self.next(); + if state.current.kind == TokenKind::Implements { + state.next(); - while self.current.kind != TokenKind::LeftBrace { - self.optional_comma()?; + while state.current.kind != TokenKind::LeftBrace { + self.optional_comma(state)?; - implements.push(self.full_name()?.into()); + implements.push(self.full_name(state)?.into()); } } - self.lbrace()?; + self.lbrace(state)?; let mut body = Vec::new(); - while self.current.kind != TokenKind::RightBrace && !self.is_eof() { - body.push(self.anonymous_class_statement()?); + while state.current.kind != TokenKind::RightBrace && !state.is_eof() { + body.push(self.anonymous_class_statement(state)?); } - self.rbrace()?; + self.rbrace(state)?; Ok(Expression::New { target: Box::new(Expression::AnonymousClass { @@ -174,17 +181,17 @@ impl Parser { }) } - pub(in crate::parser) fn enum_definition(&mut self) -> ParseResult { - self.next(); + pub(in crate::parser) fn enum_definition(&self, state: &mut State) -> ParseResult { + state.next(); - let name = self.ident()?; + let name = self.ident(state)?; - let backed_type: Option = if self.current.kind == TokenKind::Colon { - self.colon()?; + let backed_type: Option = if state.current.kind == TokenKind::Colon { + self.colon(state)?; - match self.current.kind.clone() { + match state.current.kind.clone() { TokenKind::Identifier(s) if s == b"string" || s == b"int" => { - self.next(); + state.next(); Some(match &s[..] { b"string" => BackedEnumType::String, @@ -193,7 +200,7 @@ impl Parser { }) } _ => { - return expected_token_err!(["`string`", "`int`"], self); + return expected_token_err!(["`string`", "`int`"], state); } } } else { @@ -201,25 +208,25 @@ impl Parser { }; let mut implements = Vec::new(); - if self.current.kind == TokenKind::Implements { - self.next(); + if state.current.kind == TokenKind::Implements { + state.next(); - while self.current.kind != TokenKind::LeftBrace { - implements.push(self.full_name()?.into()); + while state.current.kind != TokenKind::LeftBrace { + implements.push(self.full_name(state)?.into()); - self.optional_comma()?; + self.optional_comma(state)?; } } - self.lbrace()?; + self.lbrace(state)?; let mut body = Block::new(); - while self.current.kind != TokenKind::RightBrace { - self.skip_comments(); - body.push(self.enum_statement(backed_type.is_some())?); + while state.current.kind != TokenKind::RightBrace { + state.skip_comments(); + body.push(self.enum_statement(state, backed_type.is_some())?); } - self.rbrace()?; + self.rbrace(state)?; match backed_type { Some(backed_type) => Ok(Statement::BackedEnum { @@ -237,17 +244,18 @@ impl Parser { } fn at_least_one_comma_separated( - &mut self, - func: &(dyn Fn(&mut Parser) -> ParseResult), + &self, + state: &mut State, + func: &(dyn Fn(&Parser, &mut State) -> ParseResult), ) -> ParseResult> { let mut result: Vec = vec![]; loop { - result.push(func(self)?); - if self.current.kind != TokenKind::Comma { + result.push(func(self, state)?); + if state.current.kind != TokenKind::Comma { break; } - self.next(); + state.next(); } Ok(result) diff --git a/src/parser/classish_statement.rs b/src/parser/classish_statement.rs index 0f382281..f8f3966c 100644 --- a/src/parser/classish_statement.rs +++ b/src/parser/classish_statement.rs @@ -7,6 +7,7 @@ use crate::parser::ast::TraitAdaptation; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; use crate::parser::precedence::Precedence; +use crate::parser::state::State; use crate::parser::Parser; use crate::expect_token; @@ -24,121 +25,136 @@ pub enum ClassishDefinitionType { impl Parser { pub(in crate::parser) fn class_statement( - &mut self, + &self, + state: &mut State, flags: Vec, ) -> ParseResult { - self.complete_class_statement(ClassishDefinitionType::Class(flags)) + self.complete_class_statement(state, ClassishDefinitionType::Class(flags)) } - pub(in crate::parser) fn interface_statement(&mut self) -> ParseResult { - if self.current.kind == TokenKind::Const { - return self.parse_classish_const(vec![]); + pub(in crate::parser) fn interface_statement( + &self, + state: &mut State, + ) -> ParseResult { + if state.current.kind == TokenKind::Const { + return self.parse_classish_const(state, vec![]); } - if self.current.kind == TokenKind::Function { - return self.method(ClassishDefinitionType::Interface, vec![]); + if state.current.kind == TokenKind::Function { + return self.method(state, ClassishDefinitionType::Interface, vec![]); } - let member_flags = self.interface_members_flags()?; + let member_flags = self.interface_members_flags(state)?; peek_token!([ - TokenKind::Const => self.parse_classish_const(member_flags), + TokenKind::Const => self.parse_classish_const(state, member_flags), TokenKind::Function => self.method( + state, ClassishDefinitionType::Interface, member_flags.iter().map(|t| t.clone().into()).collect(), ) - ], self, ["`const`", "`function`"]) + ], state, ["`const`", "`function`"]) } - pub(in crate::parser) fn trait_statement(&mut self) -> ParseResult { - self.complete_class_statement(ClassishDefinitionType::Trait) + pub(in crate::parser) fn trait_statement(&self, state: &mut State) -> ParseResult { + self.complete_class_statement(state, ClassishDefinitionType::Trait) } - pub(in crate::parser) fn anonymous_class_statement(&mut self) -> ParseResult { - self.complete_class_statement(ClassishDefinitionType::AnonymousClass) + pub(in crate::parser) fn anonymous_class_statement( + &self, + state: &mut State, + ) -> ParseResult { + self.complete_class_statement(state, ClassishDefinitionType::AnonymousClass) } - pub(in crate::parser) fn enum_statement(&mut self, backed: bool) -> ParseResult { - if self.current.kind == TokenKind::Case { - self.next(); + pub(in crate::parser) fn enum_statement( + &self, + state: &mut State, + backed: bool, + ) -> ParseResult { + if state.current.kind == TokenKind::Case { + state.next(); - let name = self.ident()?; + let name = self.ident(state)?; if backed { - expect_token!([TokenKind::Equals], self, "`=`"); + expect_token!([TokenKind::Equals], state, "`=`"); - let value = self.expression(Precedence::Lowest)?; - self.semi()?; + let value = self.expression(state, Precedence::Lowest)?; + self.semi(state)?; return Ok(Statement::BackedEnumCase { name: name.into(), value, }); } else { - self.semi()?; + self.semi(state)?; return Ok(Statement::UnitEnumCase { name: name.into() }); } } - if self.current.kind == TokenKind::Const { - return self.parse_classish_const(vec![]); + if state.current.kind == TokenKind::Const { + return self.parse_classish_const(state, vec![]); } - if self.current.kind == TokenKind::Function { - return self.method(ClassishDefinitionType::Enum, vec![]); + if state.current.kind == TokenKind::Function { + return self.method(state, ClassishDefinitionType::Enum, vec![]); } - let member_flags = self.enum_members_flags()?; + let member_flags = self.enum_members_flags(state)?; peek_token!([ - TokenKind::Const => self.parse_classish_const(member_flags), + TokenKind::Const => self.parse_classish_const(state, member_flags), TokenKind::Function => self.method( + state, ClassishDefinitionType::Enum, member_flags.iter().map(|t| t.clone().into()).collect(), ) - ], self, ["`const`", "`function`"]) + ], state, ["`const`", "`function`"]) } fn complete_class_statement( - &mut self, + &self, + state: &mut State, class_type: ClassishDefinitionType, ) -> ParseResult { - if self.current.kind == TokenKind::Use { - return self.parse_classish_uses(); + if state.current.kind == TokenKind::Use { + return self.parse_classish_uses(state); } - if self.current.kind == TokenKind::Var { - return self.parse_classish_var(); + if state.current.kind == TokenKind::Var { + return self.parse_classish_var(state); } - if self.current.kind == TokenKind::Const { - return self.parse_classish_const(vec![]); + if state.current.kind == TokenKind::Const { + return self.parse_classish_const(state, vec![]); } - if self.current.kind == TokenKind::Function { - return self.method(class_type, vec![]); + if state.current.kind == TokenKind::Function { + return self.method(state, class_type, vec![]); } - let member_flags = self.class_members_flags()?; + let member_flags = self.class_members_flags(state)?; - match &self.current.kind { - TokenKind::Const => self.parse_classish_const(member_flags), + match &state.current.kind { + TokenKind::Const => self.parse_classish_const(state, member_flags), TokenKind::Function => self.method( + state, class_type, member_flags.iter().map(|t| t.clone().into()).collect(), ), // TODO TokenKind::Variable(_) => { - let var = self.var()?; + let var = self.var(state)?; let mut value = None; - if self.current.kind == TokenKind::Equals { - self.next(); - value = Some(self.expression(Precedence::Lowest)?); + if state.current.kind == TokenKind::Equals { + state.next(); + value = Some(self.expression(state, Precedence::Lowest)?); } - self.semi()?; + self.semi(state)?; Ok(Statement::Property { var, @@ -153,19 +169,19 @@ impl Parser { | TokenKind::FullyQualifiedIdentifier(_) | TokenKind::Array | TokenKind::Null => { - let prop_type = self.type_string()?; - let var = self.var()?; + let prop_type = self.type_string(state)?; + let var = self.var(state)?; let mut value = None; - if self.current.kind == TokenKind::Equals { - self.next(); - value = Some(self.expression(Precedence::Lowest)?); + if state.current.kind == TokenKind::Equals { + state.next(); + value = Some(self.expression(state, Precedence::Lowest)?); } // TODO: Support comma-separated property declarations. // nikic/php-parser does this with a single Property statement // that is capable of holding multiple property declarations. - self.semi()?; + self.semi(state)?; Ok(Statement::Property { var, @@ -176,30 +192,30 @@ impl Parser { } _ => expected_token_err!( ["`const`", "`function`", "an identifier", "a varaible"], - self + state ), } } - fn parse_classish_var(&mut self) -> ParseResult { - self.next(); + fn parse_classish_var(&self, state: &mut State) -> ParseResult { + state.next(); let mut var_type = None; - if !matches!(self.current.kind, TokenKind::Variable(_)) || self.config.force_type_strings { - var_type = Some(self.type_string()?); + if !matches!(state.current.kind, TokenKind::Variable(_)) { + var_type = Some(self.type_string(state)?); } - let var = self.var()?; + let var = self.var(state)?; let mut value = None; - if self.current.kind == TokenKind::Equals { - self.next(); + if state.current.kind == TokenKind::Equals { + state.next(); - value = Some(self.expression(Precedence::Lowest)?); + value = Some(self.expression(state, Precedence::Lowest)?); } - self.semi()?; + self.semi(state)?; Ok(Statement::Var { var, @@ -208,51 +224,52 @@ impl Parser { }) } - fn parse_classish_uses(&mut self) -> ParseResult { - self.next(); + fn parse_classish_uses(&self, state: &mut State) -> ParseResult { + state.next(); let mut traits = Vec::new(); - while self.current.kind != TokenKind::SemiColon && self.current.kind != TokenKind::LeftBrace + while state.current.kind != TokenKind::SemiColon + && state.current.kind != TokenKind::LeftBrace { - self.optional_comma()?; + self.optional_comma(state)?; - let t = self.full_name()?; + let t = self.full_name(state)?; traits.push(t.into()); } let mut adaptations = Vec::new(); - if self.current.kind == TokenKind::LeftBrace { - self.lbrace()?; + if state.current.kind == TokenKind::LeftBrace { + self.lbrace(state)?; - while self.current.kind != TokenKind::RightBrace { - let (r#trait, method): (Option, Identifier) = match self.peek.kind { + while state.current.kind != TokenKind::RightBrace { + let (r#trait, method): (Option, Identifier) = match state.peek.kind { TokenKind::DoubleColon => { - let r#trait = self.full_name()?; - self.next(); - let method = self.ident()?; + let r#trait = self.full_name(state)?; + state.next(); + let method = self.ident(state)?; (Some(r#trait.into()), method.into()) } - _ => (None, self.ident()?.into()), + _ => (None, self.ident(state)?.into()), }; - match self.current.kind { + match state.current.kind { TokenKind::As => { - self.next(); + state.next(); - match self.current.kind { + match state.current.kind { TokenKind::Public | TokenKind::Protected | TokenKind::Private => { - let visibility: MethodFlag = self.current.kind.clone().into(); - self.next(); + let visibility: MethodFlag = state.current.kind.clone().into(); + state.next(); - if self.current.kind == TokenKind::SemiColon { + if state.current.kind == TokenKind::SemiColon { adaptations.push(TraitAdaptation::Visibility { r#trait, method, visibility, }); } else { - let alias: Identifier = self.name()?.into(); + let alias: Identifier = self.name(state)?.into(); adaptations.push(TraitAdaptation::Alias { r#trait, method, @@ -262,7 +279,7 @@ impl Parser { } } _ => { - let alias: Identifier = self.name()?.into(); + let alias: Identifier = self.name(state)?.into(); adaptations.push(TraitAdaptation::Alias { r#trait, method, @@ -273,13 +290,13 @@ impl Parser { } } TokenKind::Insteadof => { - self.next(); + state.next(); let mut insteadof = Vec::new(); - insteadof.push(self.full_name()?.into()); - while self.current.kind != TokenKind::SemiColon { - self.optional_comma()?; - insteadof.push(self.full_name()?.into()); + insteadof.push(self.full_name(state)?.into()); + while state.current.kind != TokenKind::SemiColon { + self.optional_comma(state)?; + insteadof.push(self.full_name(state)?.into()); } adaptations.push(TraitAdaptation::Precedence { @@ -290,18 +307,18 @@ impl Parser { } _ => { return Err(ParseError::UnexpectedToken( - self.current.kind.to_string(), - self.current.span, + state.current.kind.to_string(), + state.current.span, )) } }; - self.semi()?; + self.semi(state)?; } - self.rbrace()?; + self.rbrace(state)?; } else { - self.semi()?; + self.semi(state)?; } Ok(Statement::TraitUse { @@ -310,30 +327,34 @@ impl Parser { }) } - fn parse_classish_const(&mut self, const_flags: Vec) -> ParseResult { + fn parse_classish_const( + &self, + state: &mut State, + const_flags: Vec, + ) -> ParseResult { if const_flags.contains(&TokenKind::Static) { - return Err(ParseError::StaticModifierOnConstant(self.current.span)); + return Err(ParseError::StaticModifierOnConstant(state.current.span)); } if const_flags.contains(&TokenKind::Readonly) { - return Err(ParseError::ReadonlyModifierOnConstant(self.current.span)); + return Err(ParseError::ReadonlyModifierOnConstant(state.current.span)); } if const_flags.contains(&TokenKind::Final) && const_flags.contains(&TokenKind::Private) { return Err(ParseError::FinalModifierOnPrivateConstant( - self.current.span, + state.current.span, )); } - self.next(); + state.next(); - let name = self.ident()?; + let name = self.ident(state)?; - expect_token!([TokenKind::Equals], self, "`=`"); + expect_token!([TokenKind::Equals], state, "`=`"); - let value = self.expression(Precedence::Lowest)?; + let value = self.expression(state, Precedence::Lowest)?; - self.semi()?; + self.semi(state)?; Ok(Statement::ClassishConstant { name: name.into(), diff --git a/src/parser/comments.rs b/src/parser/comments.rs deleted file mode 100644 index 29cdc313..00000000 --- a/src/parser/comments.rs +++ /dev/null @@ -1,30 +0,0 @@ -use crate::lexer::token::Token; -use crate::lexer::token::TokenKind; -use crate::parser::Parser; - -impl Parser { - pub(in crate::parser) fn skip_comments(&mut self) { - while matches!( - self.current.kind, - TokenKind::Comment(_) | TokenKind::DocComment(_) - ) { - self.next(); - } - } - - pub(in crate::parser) fn gather_comments(&mut self) { - while matches!( - self.current.kind, - TokenKind::Comment(_) | TokenKind::DocComment(_) - ) { - self.comments.push(self.current.clone()); - self.next(); - } - } - - pub(in crate::parser) fn clear_comments(&mut self) -> Vec { - let c = self.comments.clone(); - self.comments = vec![]; - c - } -} diff --git a/src/parser/flags.rs b/src/parser/flags.rs index 847827ee..df1c3cd3 100644 --- a/src/parser/flags.rs +++ b/src/parser/flags.rs @@ -1,6 +1,7 @@ use crate::lexer::token::TokenKind; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; +use crate::parser::state::State; use crate::parser::Parser; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -13,22 +14,31 @@ enum FlagTarget { } impl Parser { - pub(in crate::parser) fn class_flags(&mut self) -> ParseResult> { + pub(in crate::parser) fn class_flags(&self, state: &mut State) -> ParseResult> { self.collect( + state, vec![TokenKind::Final, TokenKind::Abstract, TokenKind::Readonly], FlagTarget::Class, ) } - pub(in crate::parser) fn interface_members_flags(&mut self) -> ParseResult> { + pub(in crate::parser) fn interface_members_flags( + &self, + state: &mut State, + ) -> ParseResult> { self.collect( + state, vec![TokenKind::Public, TokenKind::Static], FlagTarget::InterfaceMember, ) } - pub(in crate::parser) fn class_members_flags(&mut self) -> ParseResult> { + pub(in crate::parser) fn class_members_flags( + &self, + state: &mut State, + ) -> ParseResult> { self.collect( + state, vec![ TokenKind::Final, TokenKind::Abstract, @@ -42,8 +52,12 @@ impl Parser { ) } - pub(in crate::parser) fn enum_members_flags(&mut self) -> ParseResult> { + pub(in crate::parser) fn enum_members_flags( + &self, + state: &mut State, + ) -> ParseResult> { self.collect( + state, vec![ TokenKind::Final, TokenKind::Private, @@ -55,8 +69,12 @@ impl Parser { ) } - pub(in crate::parser) fn promoted_property_flags(&mut self) -> ParseResult> { + pub(in crate::parser) fn promoted_property_flags( + &self, + state: &mut State, + ) -> ParseResult> { self.collect( + state, vec![ TokenKind::Private, TokenKind::Protected, @@ -68,63 +86,64 @@ impl Parser { } fn collect( - &mut self, + &self, + state: &mut State, flags: Vec, target: FlagTarget, ) -> ParseResult> { let mut collected: Vec = vec![]; loop { - if flags.contains(&self.current.kind) { - if collected.contains(&self.current.kind) { + if flags.contains(&state.current.kind) { + if collected.contains(&state.current.kind) { return Err(ParseError::MultipleModifiers( - self.current.kind.to_string(), - self.current.span, + state.current.kind.to_string(), + state.current.span, )); } - match self.current.kind { + match state.current.kind { TokenKind::Private if collected.contains(&TokenKind::Protected) || collected.contains(&TokenKind::Public) => { - return Err(ParseError::MultipleAccessModifiers(self.current.span)); + return Err(ParseError::MultipleAccessModifiers(state.current.span)); } TokenKind::Protected if collected.contains(&TokenKind::Private) || collected.contains(&TokenKind::Public) => { - return Err(ParseError::MultipleAccessModifiers(self.current.span)); + return Err(ParseError::MultipleAccessModifiers(state.current.span)); } TokenKind::Public if collected.contains(&TokenKind::Private) || collected.contains(&TokenKind::Protected) => { - return Err(ParseError::MultipleAccessModifiers(self.current.span)); + return Err(ParseError::MultipleAccessModifiers(state.current.span)); } _ => {} }; if matches!(target, FlagTarget::ClassMember | FlagTarget::Class) { - match self.current.kind { + match state.current.kind { TokenKind::Final if collected.contains(&TokenKind::Abstract) => { if target == FlagTarget::Class { return Err(ParseError::FinalModifierOnAbstractClass( - self.current.span, + state.current.span, )); } else { return Err(ParseError::FinalModifierOnAbstractClassMember( - self.current.span, + state.current.span, )); } } TokenKind::Abstract if collected.contains(&TokenKind::Final) => { if target == FlagTarget::Class { return Err(ParseError::FinalModifierOnAbstractClass( - self.current.span, + state.current.span, )); } else { return Err(ParseError::FinalModifierOnAbstractClassMember( - self.current.span, + state.current.span, )); } } @@ -132,8 +151,8 @@ impl Parser { }; } - collected.push(self.current.kind.clone()); - self.next(); + collected.push(state.current.kind.clone()); + state.next(); } else { break; } diff --git a/src/parser/functions.rs b/src/parser/functions.rs index eac96e29..a2093fda 100644 --- a/src/parser/functions.rs +++ b/src/parser/functions.rs @@ -7,40 +7,41 @@ use crate::parser::classish_statement::ClassishDefinitionType; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; use crate::parser::params::ParamPosition; +use crate::parser::state::State; use crate::parser::Parser; impl Parser { - pub(in crate::parser) fn function(&mut self) -> ParseResult { - self.next(); + pub(in crate::parser) fn function(&self, state: &mut State) -> ParseResult { + state.next(); - let by_ref = if self.current.kind == TokenKind::Ampersand { - self.next(); + let by_ref = if state.current.kind == TokenKind::Ampersand { + state.next(); true } else { false }; - let name = self.ident()?; + let name = self.ident(state)?; - self.lparen()?; + self.lparen(state)?; - let params = self.param_list(ParamPosition::Function)?; + let params = self.param_list(state, ParamPosition::Function)?; - self.rparen()?; + self.rparen(state)?; let mut return_type = None; - if self.current.kind == TokenKind::Colon || self.config.force_type_strings { - self.colon()?; + if state.current.kind == TokenKind::Colon { + self.colon(state)?; - return_type = Some(self.type_string()?); + return_type = Some(self.type_string(state)?); } - self.lbrace()?; + self.lbrace(state)?; - let body = self.block(&TokenKind::RightBrace)?; + let body = self.block(state, &TokenKind::RightBrace)?; - self.rbrace()?; + self.rbrace(state)?; Ok(Statement::Function { name: name.into(), @@ -52,7 +53,8 @@ impl Parser { } pub(in crate::parser) fn method( - &mut self, + &self, + state: &mut State, class_type: ClassishDefinitionType, flags: Vec, ) -> ParseResult { @@ -62,13 +64,13 @@ impl Parser { if !cf.contains(&ClassFlag::Abstract) && flags.contains(&MethodFlag::Abstract) => { return Err(ParseError::AbstractModifierOnNonAbstractClassMethod( - self.current.span, + state.current.span, )); } _ => (), } - self.next(); + state.next(); let has_body = match &class_type { ClassishDefinitionType::Class(_) | ClassishDefinitionType::Trait => { @@ -78,33 +80,33 @@ impl Parser { ClassishDefinitionType::Enum | ClassishDefinitionType::AnonymousClass => true, }; - let by_ref = if self.current.kind == TokenKind::Ampersand { - self.next(); + let by_ref = if state.current.kind == TokenKind::Ampersand { + state.next(); true } else { false }; - let name = self.ident_maybe_reserved()?; + let name = self.ident_maybe_reserved(state)?; - self.lparen()?; + self.lparen(state)?; let position = position_from_flags_and_name(class_type, flags.clone(), name.clone()); - let params = self.param_list(position)?; + let params = self.param_list(state, position)?; - self.rparen()?; + self.rparen(state)?; let mut return_type = None; - if self.current.kind == TokenKind::Colon || self.config.force_type_strings { - self.colon()?; + if state.current.kind == TokenKind::Colon { + self.colon(state)?; - return_type = Some(self.type_string()?); + return_type = Some(self.type_string(state)?); } if !has_body { - self.semi()?; + self.semi(state)?; Ok(Statement::AbstractMethod { name: name.into(), @@ -114,11 +116,11 @@ impl Parser { by_ref, }) } else { - self.lbrace()?; + self.lbrace(state)?; - let body = self.block(&TokenKind::RightBrace)?; + let body = self.block(state, &TokenKind::RightBrace)?; - self.rbrace()?; + self.rbrace(state)?; Ok(Statement::Method { name: name.into(), diff --git a/src/parser/ident.rs b/src/parser/ident.rs index 21aa8f50..3000caf1 100644 --- a/src/parser/ident.rs +++ b/src/parser/ident.rs @@ -1,71 +1,78 @@ use crate::lexer::byte_string::ByteString; use crate::lexer::token::TokenKind; use crate::parser::error::ParseResult; +use crate::parser::state::State; use crate::parser::Parser; use crate::expect_token; impl Parser { /// Expect an unqualified identifier such as Foo or Bar. - pub(in crate::parser) fn ident(&mut self) -> ParseResult { + pub(in crate::parser) fn ident(&self, state: &mut State) -> ParseResult { Ok(expect_token!([ TokenKind::Identifier(identifier) => identifier, - ], self, "an identifier")) + ], state, "an identifier")) } /// Expect an unqualified or qualified identifier such as Foo, Bar or Foo\Bar. - pub(in crate::parser) fn name(&mut self) -> ParseResult { + pub(in crate::parser) fn name(&self, state: &mut State) -> ParseResult { Ok(expect_token!([ TokenKind::Identifier(identifier) => identifier, TokenKind::QualifiedIdentifier(qualified) => qualified, - ], self, "an identifier")) + ], state, "an identifier")) } /// Expect an unqualified, qualified or fully qualified identifier such as Foo, Foo\Bar or \Foo\Bar. - pub(in crate::parser) fn full_name(&mut self) -> ParseResult { + pub(in crate::parser) fn full_name(&self, state: &mut State) -> ParseResult { Ok(expect_token!([ TokenKind::Identifier(identifier) => identifier, TokenKind::QualifiedIdentifier(qualified) => qualified, TokenKind::FullyQualifiedIdentifier(fully_qualified) => fully_qualified, - ], self, "an identifier")) + ], state, "an identifier")) } - pub(in crate::parser) fn var(&mut self) -> ParseResult { + pub(in crate::parser) fn var(&self, state: &mut State) -> ParseResult { Ok(expect_token!([ TokenKind::Variable(v) => v, - ], self, "a variable")) + ], state, "a variable")) } - pub(in crate::parser) fn full_name_maybe_type_keyword(&mut self) -> ParseResult { - match self.current.kind { + pub(in crate::parser) fn full_name_maybe_type_keyword( + &self, + state: &mut State, + ) -> ParseResult { + match state.current.kind { TokenKind::Array | TokenKind::Callable => { - let r = Ok(self.current.kind.to_string().into()); - self.next(); + let r = Ok(state.current.kind.to_string().into()); + state.next(); r } - _ => self.full_name(), + _ => self.full_name(state), } } - pub(in crate::parser) fn type_with_static(&mut self) -> ParseResult { - Ok(match self.current.kind { + pub(in crate::parser) fn type_with_static(&self, state: &mut State) -> ParseResult { + Ok(match state.current.kind { TokenKind::Static | TokenKind::Null | TokenKind::True | TokenKind::False => { - let str = self.current.kind.to_string(); - self.next(); + let str = state.current.kind.to_string(); + state.next(); str.into() } - _ => self.full_name_maybe_type_keyword()?, + _ => self.full_name_maybe_type_keyword(state)?, }) } - pub(in crate::parser) fn ident_maybe_reserved(&mut self) -> ParseResult { - match self.current.kind { - _ if is_reserved_ident(&self.current.kind) => { - let string = self.current.kind.to_string().into(); - self.next(); + pub(in crate::parser) fn ident_maybe_reserved( + &self, + state: &mut State, + ) -> ParseResult { + match state.current.kind { + _ if is_reserved_ident(&state.current.kind) => { + let string = state.current.kind.to_string().into(); + state.next(); Ok(string) } - _ => self.ident(), + _ => self.ident(state), } } } diff --git a/src/parser/macros.rs b/src/parser/macros.rs index 1bc4f279..54bb0779 100644 --- a/src/parser/macros.rs +++ b/src/parser/macros.rs @@ -1,68 +1,68 @@ #[macro_export] macro_rules! peek_token { - ([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => {{ - $parser.skip_comments(); - match $parser.current.kind.clone() { + ([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{ + $state.skip_comments(); + match $state.current.kind.clone() { $( $expected => $out, )+ _ => { - return $crate::expected_token_err!([ $($message,)+ ], $parser); + return $crate::expected_token_err!([ $($message,)+ ], $state); } } }}; - ([ $($expected:pat),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => {{ - $parser.skip_comments(); - if !matches!($parser.current.kind, $(| $expected )+) { - return $crate::expected_token_err!([ $($message,)+ ], $parser); + ([ $($expected:pat),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{ + $state.skip_comments(); + if !matches!($state.current.kind, $(| $expected )+) { + return $crate::expected_token_err!([ $($message,)+ ], $state); } }}; - ([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, $message:literal) => { - $crate::peek_token!([ $($expected => $out,)+ ], $parser, [$message]) + ([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, $message:literal) => { + $crate::peek_token!([ $($expected => $out,)+ ], $state, [$message]) }; - ([ $($expected:pat),+ $(,)? ], $parser:expr, $message:literal) => { - $crate::peek_token!([ $($expected,)+ ], $parser, [$message]) + ([ $($expected:pat),+ $(,)? ], $state:expr, $message:literal) => { + $crate::peek_token!([ $($expected,)+ ], $state, [$message]) }; } #[macro_export] macro_rules! expect_token { - ([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => { - $crate::peek_token!([ $($expected => { $parser.next(); $out },)+ ], $parser, [$($message,)+]) + ([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => { + $crate::peek_token!([ $($expected => { $state.next(); $out },)+ ], $state, [$($message,)+]) }; - ([ $($expected:pat),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => { - $crate::peek_token!([ $($expected => { $parser.next(); },)+ ], $parser, [$($message,)+]) + ([ $($expected:pat),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => { + $crate::peek_token!([ $($expected => { $state.next(); },)+ ], $state, [$($message,)+]) }; - ([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, $message:literal) => { - $crate::peek_token!([ $($expected => { $parser.next(); $out },)+ ], $parser, [$message]) + ([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, $message:literal) => { + $crate::peek_token!([ $($expected => { $state.next(); $out },)+ ], $state, [$message]) }; - ([ $($expected:pat),+ $(,)? ], $parser:expr, $message:literal) => { - $crate::peek_token!([ $($expected => { $parser.next(); },)+ ], $parser, [$message]) + ([ $($expected:pat),+ $(,)? ], $state:expr, $message:literal) => { + $crate::peek_token!([ $($expected => { $state.next(); },)+ ], $state, [$message]) }; } #[macro_export] macro_rules! expect_literal { - ($parser:expr) => {{ - $parser.skip_comments(); - match $parser.current.kind.clone() { + ($state:expr) => {{ + $state.skip_comments(); + match $state.current.kind.clone() { TokenKind::LiteralInteger(i) => { let e = Expression::LiteralInteger { i }; - $parser.next(); + $state.next(); e } TokenKind::LiteralFloat(f) => { let e = Expression::LiteralFloat { f }; - $parser.next(); + $state.next(); e } TokenKind::LiteralString(s) => { let e = Expression::LiteralString { value: s.clone() }; - $parser.next(); + $state.next(); e } _ => { - return $crate::expected_token_err!(["a literal"], $parser); + return $crate::expected_token_err!(["a literal"], $state); } } }}; @@ -70,26 +70,26 @@ macro_rules! expect_literal { #[macro_export] macro_rules! expected_token_err { - ([ $($expected:literal),+ $(,)? ], $parser:expr $(,)?) => {{ - match &$parser.current.kind { + ([ $($expected:literal),+ $(,)? ], $state:expr $(,)?) => {{ + match &$state.current.kind { TokenKind::Eof => { Err($crate::parser::error::ParseError::ExpectedToken( vec![$($expected.into()),+], None, - $parser.current.span, + $state.current.span, )) }, _ => { Err($crate::parser::error::ParseError::ExpectedToken( vec![$($expected.into()),+], - Some($parser.current.kind.to_string()), - $parser.current.span, + Some($state.current.kind.to_string()), + $state.current.span, )) } } }}; - ($expected:literal, $parser:expr $(,)?) => { - $crate::expected_token_err!([$expected], $parser) + ($expected:literal, $state:expr $(,)?) => { + $crate::expected_token_err!([$expected], $state) }; } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 54b4d660..66884d50 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1,5 +1,3 @@ -use std::vec::IntoIter; - use crate::expect_literal; use crate::expect_token; use crate::expected_token_err; @@ -16,6 +14,7 @@ use crate::parser::error::ParseResult; use crate::parser::ident::is_reserved_ident; use crate::parser::params::ParamPosition; use crate::parser::precedence::{Associativity, Precedence}; +use crate::parser::state::State; pub mod ast; pub mod error; @@ -23,7 +22,6 @@ pub mod error; mod block; mod classish; mod classish_statement; -mod comments; mod flags; mod functions; mod ident; @@ -31,88 +29,62 @@ mod macros; mod params; mod precedence; mod punc; +mod state; mod vars; -pub struct ParserConfig { - force_type_strings: bool, -} +#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)] +pub struct Parser; -#[allow(clippy::derivable_impls)] -impl Default for ParserConfig { - fn default() -> Self { - Self { - force_type_strings: false, - } - } -} - -pub struct Parser { - config: ParserConfig, - pub current: Token, - pub peek: Token, - iter: IntoIter, - comments: Vec, -} - -#[allow(dead_code)] impl Parser { - pub fn new(config: Option) -> Self { - Self { - config: config.unwrap_or_default(), - current: Token::default(), - peek: Token::default(), - iter: vec![].into_iter(), - comments: vec![], - } + pub const fn new() -> Self { + Self {} } - pub fn parse(&mut self, tokens: Vec) -> ParseResult { - self.iter = tokens.into_iter(); - self.next(); - self.next(); + pub fn parse(&self, tokens: Vec) -> ParseResult { + let mut state = State::new(tokens); let mut ast = Program::new(); - while self.current.kind != TokenKind::Eof { + while state.current.kind != TokenKind::Eof { if matches!( - self.current.kind, + state.current.kind, TokenKind::OpenTag(_) | TokenKind::CloseTag ) { - self.next(); + state.next(); continue; } - self.gather_comments(); + state.gather_comments(); - if self.is_eof() { + if state.is_eof() { break; } - ast.push(self.top_level_statement()?); + ast.push(self.top_level_statement(&mut state)?); - self.clear_comments(); + state.clear_comments(); } Ok(ast.to_vec()) } - fn try_block_caught_type_string(&mut self) -> ParseResult { - let id = self.full_name()?; + fn try_block_caught_type_string(&self, state: &mut State) -> ParseResult { + let id = self.full_name(state)?; - if self.current.kind == TokenKind::Pipe { - self.next(); + if state.current.kind == TokenKind::Pipe { + state.next(); let mut types = vec![id.into()]; - while !self.is_eof() { - let id = self.full_name()?; + while !state.is_eof() { + let id = self.full_name(state)?; types.push(id.into()); - if self.current.kind != TokenKind::Pipe { + if state.current.kind != TokenKind::Pipe { break; } - self.next(); + state.next(); } return Ok(TryBlockCaughtType::Union(types)); @@ -121,81 +93,81 @@ impl Parser { Ok(TryBlockCaughtType::Identifier(id.into())) } - fn type_string(&mut self) -> ParseResult { - if self.current.kind == TokenKind::Question { - self.next(); - let t = self.type_with_static()?; + fn type_string(&self, state: &mut State) -> ParseResult { + if state.current.kind == TokenKind::Question { + state.next(); + let t = self.type_with_static(state)?; return Ok(Type::Nullable(Box::new(parse_simple_type(t)))); } - let id = self.type_with_static()?; + let id = self.type_with_static(state)?; - if self.current.kind == TokenKind::Pipe { - self.next(); + if state.current.kind == TokenKind::Pipe { + state.next(); let r#type = parse_simple_type(id); if r#type.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( r#type, - self.current.span, + state.current.span, )); } let mut types = vec![r#type]; - while !self.is_eof() { - let id = self.type_with_static()?; + while !state.is_eof() { + let id = self.type_with_static(state)?; let r#type = parse_simple_type(id); if r#type.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( r#type, - self.current.span, + state.current.span, )); } types.push(r#type); - if self.current.kind != TokenKind::Pipe { + if state.current.kind != TokenKind::Pipe { break; } else { - self.next(); + state.next(); } } return Ok(Type::Union(types)); } - if self.current.kind == TokenKind::Ampersand - && !matches!(self.peek.kind, TokenKind::Variable(_)) + if state.current.kind == TokenKind::Ampersand + && !matches!(state.peek.kind, TokenKind::Variable(_)) { - self.next(); + state.next(); let r#type = parse_simple_type(id); if r#type.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( r#type, - self.current.span, + state.current.span, )); } let mut types = vec![r#type]; - while !self.is_eof() { - let id = self.type_with_static()?; + while !state.is_eof() { + let id = self.type_with_static(state)?; let r#type = parse_simple_type(id); if r#type.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( r#type, - self.current.span, + state.current.span, )); } types.push(r#type); - if self.current.kind != TokenKind::Ampersand { + if state.current.kind != TokenKind::Ampersand { break; } else { - self.next(); + state.next(); } } @@ -205,75 +177,75 @@ impl Parser { Ok(parse_simple_type(id)) } - fn top_level_statement(&mut self) -> ParseResult { - self.skip_comments(); + fn top_level_statement(&self, state: &mut State) -> ParseResult { + state.skip_comments(); - let statement = match &self.current.kind { + let statement = match &state.current.kind { TokenKind::Namespace => { - self.next(); + state.next(); let mut braced = false; - let name = if self.current.kind == TokenKind::LeftBrace { + let name = if state.current.kind == TokenKind::LeftBrace { braced = true; - self.lbrace()?; + self.lbrace(state)?; None } else { - Some(self.name()?) + Some(self.name(state)?) }; if name.is_some() { - if self.current.kind == TokenKind::LeftBrace { + if state.current.kind == TokenKind::LeftBrace { braced = true; - self.next(); + state.next(); } else { - self.semi()?; + self.semi(state)?; } } let body = if braced { - self.block(&TokenKind::RightBrace)? + self.block(state, &TokenKind::RightBrace)? } else { let mut body = Block::new(); - while !self.is_eof() { - body.push(self.top_level_statement()?); + while !state.is_eof() { + body.push(self.top_level_statement(state)?); } body }; if braced { - self.rbrace()?; + self.rbrace(state)?; } Statement::Namespace { name, body } } TokenKind::Use => { - self.next(); + state.next(); - let kind = match self.current.kind { + let kind = match state.current.kind { TokenKind::Function => { - self.next(); + state.next(); UseKind::Function } TokenKind::Const => { - self.next(); + state.next(); UseKind::Const } _ => UseKind::Normal, }; - if self.peek.kind == TokenKind::LeftBrace { - let prefix = self.full_name()?; - self.next(); + if state.peek.kind == TokenKind::LeftBrace { + let prefix = self.full_name(state)?; + state.next(); let mut uses = Vec::new(); - while self.current.kind != TokenKind::RightBrace { - let name = self.full_name()?; + while state.current.kind != TokenKind::RightBrace { + let name = self.full_name(state)?; let mut alias = None; - if self.current.kind == TokenKind::As { - self.next(); - alias = Some(self.ident()?.into()); + if state.current.kind == TokenKind::As { + state.next(); + alias = Some(self.ident(state)?.into()); } uses.push(Use { @@ -281,14 +253,14 @@ impl Parser { alias, }); - if self.current.kind == TokenKind::Comma { - self.next(); + if state.current.kind == TokenKind::Comma { + state.next(); continue; } } - self.rbrace()?; - self.semi()?; + self.rbrace(state)?; + self.semi(state)?; Statement::GroupUse { prefix: prefix.into(), @@ -297,13 +269,13 @@ impl Parser { } } else { let mut uses = Vec::new(); - while !self.is_eof() { - let name = self.full_name()?; + while !state.is_eof() { + let name = self.full_name(state)?; let mut alias = None; - if self.current.kind == TokenKind::As { - self.next(); - alias = Some(self.ident()?.into()); + if state.current.kind == TokenKind::As { + state.next(); + alias = Some(self.ident(state)?.into()); } uses.push(Use { @@ -311,12 +283,12 @@ impl Parser { alias, }); - if self.current.kind == TokenKind::Comma { - self.next(); + if state.current.kind == TokenKind::Comma { + state.next(); continue; } - self.semi()?; + self.semi(state)?; break; } @@ -324,34 +296,34 @@ impl Parser { } } TokenKind::Const => { - self.next(); + state.next(); let mut constants = vec![]; - while self.current.kind != TokenKind::SemiColon { - let name = self.ident()?; + while state.current.kind != TokenKind::SemiColon { + let name = self.ident(state)?; - expect_token!([TokenKind::Equals], self, "`=`"); + expect_token!([TokenKind::Equals], state, "`=`"); - let value = self.expression(Precedence::Lowest)?; + let value = self.expression(state, Precedence::Lowest)?; constants.push(Constant { name: name.into(), value, }); - self.optional_comma()?; + self.optional_comma(state)?; } - self.semi()?; + self.semi(state)?; Statement::Constant { constants } } TokenKind::HaltCompiler => { - self.next(); + state.next(); - let content = if let TokenKind::InlineHtml(content) = self.current.kind.clone() { - self.next(); + let content = if let TokenKind::InlineHtml(content) = state.current.kind.clone() { + state.next(); Some(content) } else { None @@ -359,47 +331,47 @@ impl Parser { Statement::HaltCompiler { content } } - _ => self.statement()?, + _ => self.statement(state)?, }; - self.clear_comments(); + state.clear_comments(); Ok(statement) } - fn statement(&mut self) -> ParseResult { - self.skip_comments(); + fn statement(&self, state: &mut State) -> ParseResult { + state.skip_comments(); - let statement = match &self.current.kind { + let statement = match &state.current.kind { TokenKind::Goto => { - self.next(); + state.next(); - let label = self.ident()?.into(); + let label = self.ident(state)?.into(); - self.semi()?; + self.semi(state)?; Statement::Goto { label } } - TokenKind::Identifier(_) if self.peek.kind == TokenKind::Colon => { - let label = self.ident()?.into(); + TokenKind::Identifier(_) if state.peek.kind == TokenKind::Colon => { + let label = self.ident(state)?.into(); - self.colon()?; + self.colon(state)?; Statement::Label { label } } TokenKind::Declare => { - self.next(); - self.lparen()?; + state.next(); + self.lparen(state)?; let mut declares = Vec::new(); - while self.current.kind != TokenKind::RightParen { - let key = self.ident()?; + while state.current.kind != TokenKind::RightParen { + let key = self.ident(state)?; - expect_token!([TokenKind::Equals], self, "`=`"); + expect_token!([TokenKind::Equals], state, "`=`"); - let value = expect_literal!(self); + let value = expect_literal!(state); - self.optional_comma()?; + self.optional_comma(state)?; declares.push(DeclareItem { key: key.into(), @@ -407,113 +379,115 @@ impl Parser { }); } - self.rparen()?; + self.rparen(state)?; - let body = if self.current.kind == TokenKind::LeftBrace { - self.next(); - let b = self.block(&TokenKind::RightBrace)?; - self.rbrace()?; + let body = if state.current.kind == TokenKind::LeftBrace { + state.next(); + let b = self.block(state, &TokenKind::RightBrace)?; + self.rbrace(state)?; b - } else if self.current.kind == TokenKind::Colon { - self.colon()?; - let b = self.block(&TokenKind::EndDeclare)?; - expect_token!([TokenKind::EndDeclare], self, "`enddeclare`"); - self.semi()?; + } else if state.current.kind == TokenKind::Colon { + self.colon(state)?; + let b = self.block(state, &TokenKind::EndDeclare)?; + expect_token!([TokenKind::EndDeclare], state, "`enddeclare`"); + self.semi(state)?; b } else { - self.semi()?; + self.semi(state)?; vec![] }; Statement::Declare { declares, body } } TokenKind::Global => { - self.next(); + state.next(); let mut vars = vec![]; - while self.current.kind != TokenKind::SemiColon { - vars.push(self.var()?.into()); + while state.current.kind != TokenKind::SemiColon { + vars.push(self.var(state)?.into()); - self.optional_comma()?; + self.optional_comma(state)?; } - self.semi()?; + self.semi(state)?; Statement::Global { vars } } - TokenKind::Static if matches!(self.peek.kind, TokenKind::Variable(_)) => { - self.next(); + TokenKind::Static if matches!(state.peek.kind, TokenKind::Variable(_)) => { + state.next(); let mut vars = vec![]; - while self.current.kind != TokenKind::SemiColon { - let var = Expression::Variable { name: self.var()? }; + while state.current.kind != TokenKind::SemiColon { + let var = Expression::Variable { + name: self.var(state)?, + }; let mut default = None; - if self.current.kind == TokenKind::Equals { - expect_token!([TokenKind::Equals], self, "`=`"); - default = Some(self.expression(Precedence::Lowest)?); + if state.current.kind == TokenKind::Equals { + expect_token!([TokenKind::Equals], state, "`=`"); + default = Some(self.expression(state, Precedence::Lowest)?); } - self.optional_comma()?; + self.optional_comma(state)?; vars.push(StaticVar { var, default }) } - self.semi()?; + self.semi(state)?; Statement::Static { vars } } TokenKind::InlineHtml(html) => { let s = Statement::InlineHtml(html.clone()); - self.next(); + state.next(); s } TokenKind::Comment(comment) => { let s = Statement::Comment { comment: comment.clone(), }; - self.next(); + state.next(); s } TokenKind::Do => { - self.next(); + state.next(); - self.lbrace()?; - let body = self.block(&TokenKind::RightBrace)?; - self.rbrace()?; + self.lbrace(state)?; + let body = self.block(state, &TokenKind::RightBrace)?; + self.rbrace(state)?; - expect_token!([TokenKind::While], self, "`while`"); + expect_token!([TokenKind::While], state, "`while`"); - self.lparen()?; - let condition = self.expression(Precedence::Lowest)?; - self.rparen()?; - self.semi()?; + self.lparen(state)?; + let condition = self.expression(state, Precedence::Lowest)?; + self.rparen(state)?; + self.semi(state)?; Statement::DoWhile { condition, body } } TokenKind::While => { - self.next(); - self.lparen()?; + state.next(); + self.lparen(state)?; - let condition = self.expression(Precedence::Lowest)?; + let condition = self.expression(state, Precedence::Lowest)?; - self.rparen()?; + self.rparen(state)?; - let end_token = if self.current.kind == TokenKind::Colon { - self.colon()?; + let end_token = if state.current.kind == TokenKind::Colon { + self.colon(state)?; TokenKind::EndWhile } else { - self.lbrace()?; + self.lbrace(state)?; TokenKind::RightBrace }; - let body = self.block(&end_token)?; + let body = self.block(state, &end_token)?; if end_token == TokenKind::RightBrace { - self.rbrace()?; + self.rbrace(state)?; } else { - expect_token!([TokenKind::EndWhile], self, "`endwhile`"); - self.semi()?; + expect_token!([TokenKind::EndWhile], state, "`endwhile`"); + self.semi(state)?; } Statement::While { condition, body } @@ -522,54 +496,54 @@ impl Parser { | TokenKind::IncludeOnce | TokenKind::Require | TokenKind::RequireOnce => { - let kind: IncludeKind = (&self.current.kind).into(); - self.next(); + let kind: IncludeKind = (&state.current.kind).into(); + state.next(); - let path = self.expression(Precedence::Lowest)?; + let path = self.expression(state, Precedence::Lowest)?; - self.semi()?; + self.semi(state)?; Statement::Include { kind, path } } TokenKind::For => { - self.next(); + state.next(); - self.lparen()?; + self.lparen(state)?; let mut init = None; - if self.current.kind != TokenKind::SemiColon { - init = Some(self.expression(Precedence::Lowest)?); + if state.current.kind != TokenKind::SemiColon { + init = Some(self.expression(state, Precedence::Lowest)?); } - self.semi()?; + self.semi(state)?; let mut condition = None; - if self.current.kind != TokenKind::SemiColon { - condition = Some(self.expression(Precedence::Lowest)?); + if state.current.kind != TokenKind::SemiColon { + condition = Some(self.expression(state, Precedence::Lowest)?); } - self.semi()?; + self.semi(state)?; let mut r#loop = None; - if self.current.kind != TokenKind::RightParen { - r#loop = Some(self.expression(Precedence::Lowest)?); + if state.current.kind != TokenKind::RightParen { + r#loop = Some(self.expression(state, Precedence::Lowest)?); } - self.rparen()?; + self.rparen(state)?; - let end_token = if self.current.kind == TokenKind::Colon { - self.colon()?; + let end_token = if state.current.kind == TokenKind::Colon { + self.colon(state)?; TokenKind::EndFor } else { - self.lbrace()?; + self.lbrace(state)?; TokenKind::RightBrace }; - let then = self.block(&end_token)?; + let then = self.block(state, &end_token)?; if end_token == TokenKind::EndFor { - expect_token!([TokenKind::EndFor], self, "`endfor`"); - self.semi()?; + expect_token!([TokenKind::EndFor], state, "`endfor`"); + self.semi(state)?; } else { - self.rbrace()?; + self.rbrace(state)?; }; Statement::For { @@ -580,52 +554,52 @@ impl Parser { } } TokenKind::Foreach => { - self.next(); + state.next(); - self.lparen()?; + self.lparen(state)?; - let expr = self.expression(Precedence::Lowest)?; + let expr = self.expression(state, Precedence::Lowest)?; - expect_token!([TokenKind::As], self, ["`as`"]); + expect_token!([TokenKind::As], state, ["`as`"]); - let mut by_ref = self.current.kind == TokenKind::Ampersand; + let mut by_ref = state.current.kind == TokenKind::Ampersand; if by_ref { - self.next(); + state.next(); } let mut key_var = None; - let mut value_var = self.expression(Precedence::Lowest)?; + let mut value_var = self.expression(state, Precedence::Lowest)?; - if self.current.kind == TokenKind::DoubleArrow { - self.next(); + if state.current.kind == TokenKind::DoubleArrow { + state.next(); key_var = Some(value_var.clone()); - by_ref = self.current.kind == TokenKind::Ampersand; + by_ref = state.current.kind == TokenKind::Ampersand; if by_ref { - self.next(); + state.next(); } - value_var = self.expression(Precedence::Lowest)?; + value_var = self.expression(state, Precedence::Lowest)?; } - self.rparen()?; + self.rparen(state)?; - let end_token = if self.current.kind == TokenKind::Colon { - self.colon()?; + let end_token = if state.current.kind == TokenKind::Colon { + self.colon(state)?; TokenKind::EndForeach } else { - self.lbrace()?; + self.lbrace(state)?; TokenKind::RightBrace }; - let body = self.block(&end_token)?; + let body = self.block(state, &end_token)?; if end_token == TokenKind::EndForeach { - expect_token!([TokenKind::EndForeach], self, "`endforeach`"); - self.semi()?; + expect_token!([TokenKind::EndForeach], state, "`endforeach`"); + self.semi(state)?; } else { - self.rbrace()?; + self.rbrace(state)?; } Statement::Foreach { @@ -636,54 +610,54 @@ impl Parser { body, } } - TokenKind::Abstract => self.class_definition()?, - TokenKind::Readonly => self.class_definition()?, - TokenKind::Final => self.class_definition()?, - TokenKind::Class => self.class_definition()?, - TokenKind::Interface => self.interface_definition()?, - TokenKind::Trait => self.trait_definition()?, - TokenKind::Enum => self.enum_definition()?, + TokenKind::Abstract => self.class_definition(state)?, + TokenKind::Readonly => self.class_definition(state)?, + TokenKind::Final => self.class_definition(state)?, + TokenKind::Class => self.class_definition(state)?, + TokenKind::Interface => self.interface_definition(state)?, + TokenKind::Trait => self.trait_definition(state)?, + TokenKind::Enum => self.enum_definition(state)?, TokenKind::Switch => { - self.next(); + state.next(); - self.lparen()?; + self.lparen(state)?; - let condition = self.expression(Precedence::Lowest)?; + let condition = self.expression(state, Precedence::Lowest)?; - self.rparen()?; + self.rparen(state)?; - let end_token = if self.current.kind == TokenKind::Colon { - self.colon()?; + let end_token = if state.current.kind == TokenKind::Colon { + self.colon(state)?; TokenKind::EndSwitch } else { - self.lbrace()?; + self.lbrace(state)?; TokenKind::RightBrace }; let mut cases = Vec::new(); loop { - if self.current.kind == end_token { + if state.current.kind == end_token { break; } - match self.current.kind { + match state.current.kind { TokenKind::Case => { - self.next(); + state.next(); - let condition = self.expression(Precedence::Lowest)?; + let condition = self.expression(state, Precedence::Lowest)?; expect_token!( [TokenKind::Colon, TokenKind::SemiColon], - self, + state, ["`:`", "`;`"] ); let mut body = Block::new(); - while self.current.kind != TokenKind::Case - && self.current.kind != TokenKind::Default - && self.current.kind != TokenKind::RightBrace + while state.current.kind != TokenKind::Case + && state.current.kind != TokenKind::Default + && state.current.kind != TokenKind::RightBrace { - body.push(self.statement()?); + body.push(self.statement(state)?); } cases.push(Case { @@ -692,21 +666,21 @@ impl Parser { }); } TokenKind::Default => { - self.next(); + state.next(); expect_token!( [TokenKind::Colon, TokenKind::SemiColon], - self, + state, ["`:`", "`;`"] ); let mut body = Block::new(); - while self.current.kind != TokenKind::Case - && self.current.kind != TokenKind::Default - && self.current.kind != TokenKind::RightBrace + while state.current.kind != TokenKind::Case + && state.current.kind != TokenKind::Default + && state.current.kind != TokenKind::RightBrace { - body.push(self.statement()?); + body.push(self.statement(state)?); } cases.push(Case { @@ -715,81 +689,81 @@ impl Parser { }); } _ => { - return expected_token_err!(["`case`", "`default`"], self); + return expected_token_err!(["`case`", "`default`"], state); } } } if end_token == TokenKind::EndSwitch { - expect_token!([TokenKind::EndSwitch], self, ["`endswitch`"]); - self.semi()?; + expect_token!([TokenKind::EndSwitch], state, ["`endswitch`"]); + self.semi(state)?; } else { - self.rbrace()?; + self.rbrace(state)?; } Statement::Switch { condition, cases } } TokenKind::If => { - self.next(); + state.next(); - self.lparen()?; + self.lparen(state)?; - let condition = self.expression(Precedence::Lowest)?; + let condition = self.expression(state, Precedence::Lowest)?; - self.rparen()?; + self.rparen(state)?; // FIXME: Tidy up duplication and make the intent a bit clearer. - match self.current.kind { + match state.current.kind { TokenKind::Colon => { - self.next(); + state.next(); let mut then = vec![]; while !matches!( - self.current.kind, + state.current.kind, TokenKind::ElseIf | TokenKind::Else | TokenKind::EndIf ) { - then.push(self.statement()?); + then.push(self.statement(state)?); } let mut else_ifs = vec![]; loop { - if self.current.kind != TokenKind::ElseIf { + if state.current.kind != TokenKind::ElseIf { break; } - self.next(); + state.next(); - self.lparen()?; - let condition = self.expression(Precedence::Lowest)?; - self.rparen()?; + self.lparen(state)?; + let condition = self.expression(state, Precedence::Lowest)?; + self.rparen(state)?; - self.colon()?; + self.colon(state)?; let mut body = vec![]; while !matches!( - self.current.kind, + state.current.kind, TokenKind::ElseIf | TokenKind::Else | TokenKind::EndIf ) { - body.push(self.statement()?); + body.push(self.statement(state)?); } else_ifs.push(ElseIf { condition, body }); } let mut r#else = None; - if self.current.kind == TokenKind::Else { - self.next(); - self.colon()?; + if state.current.kind == TokenKind::Else { + state.next(); + self.colon(state)?; let mut body = vec![]; - while self.current.kind != TokenKind::EndIf { - body.push(self.statement()?); + while state.current.kind != TokenKind::EndIf { + body.push(self.statement(state)?); } r#else = Some(body); } - expect_token!([TokenKind::EndIf], self, ["`endif`"]); - self.semi()?; + expect_token!([TokenKind::EndIf], state, ["`endif`"]); + self.semi(state)?; Statement::If { condition, @@ -799,36 +773,36 @@ impl Parser { } } _ => { - let body_end_token = if self.current.kind == TokenKind::LeftBrace { - self.next(); + let body_end_token = if state.current.kind == TokenKind::LeftBrace { + state.next(); TokenKind::RightBrace } else { TokenKind::SemiColon }; - let then = self.block(&body_end_token)?; + let then = self.block(state, &body_end_token)?; if body_end_token == TokenKind::RightBrace { - self.rbrace()?; + self.rbrace(state)?; } let mut else_ifs: Vec = Vec::new(); loop { - if self.current.kind == TokenKind::ElseIf { - self.next(); + if state.current.kind == TokenKind::ElseIf { + state.next(); - self.lparen()?; + self.lparen(state)?; - let condition = self.expression(Precedence::Lowest)?; + let condition = self.expression(state, Precedence::Lowest)?; - self.rparen()?; + self.rparen(state)?; - self.lbrace()?; + self.lbrace(state)?; - let body = self.block(&TokenKind::RightBrace)?; + let body = self.block(state, &TokenKind::RightBrace)?; - self.rbrace()?; + self.rbrace(state)?; else_ifs.push(ElseIf { condition, body }); } else { @@ -836,7 +810,7 @@ impl Parser { } } - if self.current.kind != TokenKind::Else { + if state.current.kind != TokenKind::Else { return Ok(Statement::If { condition, then, @@ -845,13 +819,13 @@ impl Parser { }); } - expect_token!([TokenKind::Else], self, ["`else`"]); + expect_token!([TokenKind::Else], state, ["`else`"]); - self.lbrace()?; + self.lbrace(state)?; - let r#else = self.block(&TokenKind::RightBrace)?; + let r#else = self.block(state, &TokenKind::RightBrace)?; - self.rbrace()?; + self.rbrace(state)?; Statement::If { condition, @@ -863,72 +837,72 @@ impl Parser { } } TokenKind::Echo => { - self.next(); + state.next(); let mut values = Vec::new(); - while !self.is_eof() && self.current.kind != TokenKind::SemiColon { - values.push(self.expression(Precedence::Lowest)?); + while !state.is_eof() && state.current.kind != TokenKind::SemiColon { + values.push(self.expression(state, Precedence::Lowest)?); - self.optional_comma()?; + self.optional_comma(state)?; } - self.semi()?; + self.semi(state)?; Statement::Echo { values } } TokenKind::Continue => { - self.next(); + state.next(); let mut num = None; - if self.current.kind != TokenKind::SemiColon { - num = Some(self.expression(Precedence::Lowest)?); + if state.current.kind != TokenKind::SemiColon { + num = Some(self.expression(state, Precedence::Lowest)?); } - self.semi()?; + self.semi(state)?; Statement::Continue { num } } TokenKind::Break => { - self.next(); + state.next(); let mut num = None; - if self.current.kind != TokenKind::SemiColon { - num = Some(self.expression(Precedence::Lowest)?); + if state.current.kind != TokenKind::SemiColon { + num = Some(self.expression(state, Precedence::Lowest)?); } - self.semi()?; + self.semi(state)?; Statement::Break { num } } TokenKind::Return => { - self.next(); + state.next(); if let Token { kind: TokenKind::SemiColon, .. - } = self.current + } = state.current { let ret = Statement::Return { value: None }; - self.semi()?; + self.semi(state)?; ret } else { let ret = Statement::Return { - value: self.expression(Precedence::Lowest).ok(), + value: self.expression(state, Precedence::Lowest).ok(), }; - self.semi()?; + self.semi(state)?; ret } } TokenKind::Function if matches!( - self.peek.kind, + state.peek.kind, TokenKind::Identifier(_) | TokenKind::Ampersand ) => { // FIXME: This is incredibly hacky but we don't have a way to look at // the next N tokens right now. We could probably do with a `peek_buf()` // method like the Lexer has. - if self.peek.kind == TokenKind::Ampersand { - let mut cloned = self.iter.clone(); - if let Some((index, _)) = self.iter.clone().enumerate().next() { + if state.peek.kind == TokenKind::Ampersand { + let mut cloned = state.iter.clone(); + if let Some((index, _)) = state.iter.clone().enumerate().next() { if !matches!( cloned.nth(index), Some(Token { @@ -936,68 +910,68 @@ impl Parser { .. }) ) { - let expr = self.expression(Precedence::Lowest)?; + let expr = self.expression(state, Precedence::Lowest)?; - self.semi()?; + self.semi(state)?; return Ok(Statement::Expression { expr }); } } - self.function()? + self.function(state)? } else { - self.function()? + self.function(state)? } } TokenKind::SemiColon => { - self.next(); + state.next(); Statement::Noop } TokenKind::Try => { - let start_span = self.current.span; + let start_span = state.current.span; - self.next(); - self.lbrace()?; + state.next(); + self.lbrace(state)?; - let body = self.block(&TokenKind::RightBrace)?; + let body = self.block(state, &TokenKind::RightBrace)?; - self.rbrace()?; + self.rbrace(state)?; let mut catches = Vec::new(); loop { - if self.current.kind != TokenKind::Catch { + if state.current.kind != TokenKind::Catch { break; } - self.next(); - self.lparen()?; + state.next(); + self.lparen(state)?; - let types = self.try_block_caught_type_string()?; - let var = if self.current.kind == TokenKind::RightParen { + let types = self.try_block_caught_type_string(state)?; + let var = if state.current.kind == TokenKind::RightParen { None } else { - Some(self.expression(Precedence::Lowest)?) + Some(self.expression(state, Precedence::Lowest)?) }; - self.rparen()?; - self.lbrace()?; + self.rparen(state)?; + self.lbrace(state)?; - let body = self.block(&TokenKind::RightBrace)?; + let body = self.block(state, &TokenKind::RightBrace)?; - self.rbrace()?; + self.rbrace(state)?; catches.push(Catch { types, var, body }) } let mut finally = None; - if self.current.kind == TokenKind::Finally { - self.next(); - self.lbrace()?; + if state.current.kind == TokenKind::Finally { + state.next(); + self.lbrace(state)?; - finally = Some(self.block(&TokenKind::RightBrace)?); + finally = Some(self.block(state, &TokenKind::RightBrace)?); - self.rbrace()?; + self.rbrace(state)?; } if catches.is_empty() && finally.is_none() { @@ -1011,46 +985,46 @@ impl Parser { } } TokenKind::LeftBrace => { - self.next(); - let body = self.block(&TokenKind::RightBrace)?; - self.rbrace()?; + state.next(); + let body = self.block(state, &TokenKind::RightBrace)?; + self.rbrace(state)?; Statement::Block { body } } _ => { - let expr = self.expression(Precedence::Lowest)?; + let expr = self.expression(state, Precedence::Lowest)?; - self.semi()?; + self.semi(state)?; Statement::Expression { expr } } }; - self.skip_comments(); + state.skip_comments(); Ok(statement) } - fn expression(&mut self, precedence: Precedence) -> ParseResult { - if self.is_eof() { + fn expression(&self, state: &mut State, precedence: Precedence) -> ParseResult { + if state.is_eof() { return Err(ParseError::UnexpectedEndOfFile); } - self.skip_comments(); + state.skip_comments(); - let mut left = match &self.current.kind { + let mut left = match &state.current.kind { TokenKind::Throw => { - self.next(); + state.next(); - let value = self.expression(Precedence::Lowest)?; + let value = self.expression(state, Precedence::Lowest)?; Expression::Throw { value: Box::new(value), } } TokenKind::Yield => { - self.next(); + state.next(); - if self.current.kind == TokenKind::SemiColon { + if state.current.kind == TokenKind::SemiColon { Expression::Yield { key: None, value: None, @@ -1058,22 +1032,25 @@ impl Parser { } else { let mut from = false; - if self.current.kind == TokenKind::From { - self.next(); + if state.current.kind == TokenKind::From { + state.next(); from = true; } let mut key = None; - let mut value = Box::new(self.expression(if from { - Precedence::YieldFrom - } else { - Precedence::Yield - })?); + let mut value = Box::new(self.expression( + state, + if from { + Precedence::YieldFrom + } else { + Precedence::Yield + }, + )?); - if self.current.kind == TokenKind::DoubleArrow && !from { - self.next(); + if state.current.kind == TokenKind::DoubleArrow && !from { + state.next(); key = Some(value.clone()); - value = Box::new(self.expression(Precedence::Yield)?); + value = Box::new(self.expression(state, Precedence::Yield)?); } if from { @@ -1087,9 +1064,9 @@ impl Parser { } } TokenKind::Clone => { - self.next(); + state.next(); - let target = self.expression(Precedence::CloneOrNew)?; + let target = self.expression(state, Precedence::CloneOrNew)?; Expression::Clone { target: Box::new(target), @@ -1097,88 +1074,88 @@ impl Parser { } TokenKind::Variable(v) => { let e = Expression::Variable { name: v.clone() }; - self.next(); + state.next(); e } TokenKind::LiteralInteger(i) => { let e = Expression::LiteralInteger { i: *i }; - self.next(); + state.next(); e } TokenKind::LiteralFloat(f) => { let f = Expression::LiteralFloat { f: *f }; - self.next(); + state.next(); f } TokenKind::Identifier(i) | TokenKind::QualifiedIdentifier(i) | TokenKind::FullyQualifiedIdentifier(i) => { let e = Expression::Identifier { name: i.clone() }; - self.next(); + state.next(); e } - TokenKind::Static if matches!(self.peek.kind, TokenKind::DoubleColon) => { - self.next(); + TokenKind::Static if matches!(state.peek.kind, TokenKind::DoubleColon) => { + state.next(); Expression::Static } TokenKind::LiteralString(s) => { let e = Expression::LiteralString { value: s.clone() }; - self.next(); + state.next(); e } - TokenKind::StringPart(_) => self.interpolated_string()?, + TokenKind::StringPart(_) => self.interpolated_string(state)?, TokenKind::True => { let e = Expression::Bool { value: true }; - self.next(); + state.next(); e } TokenKind::False => { let e = Expression::Bool { value: false }; - self.next(); + state.next(); e } TokenKind::Null => { - self.next(); + state.next(); Expression::Null } TokenKind::LeftParen => { - self.next(); + state.next(); - let e = self.expression(Precedence::Lowest)?; + let e = self.expression(state, Precedence::Lowest)?; - self.rparen()?; + self.rparen(state)?; e } TokenKind::Match => { - self.next(); - self.lparen()?; + state.next(); + self.lparen(state)?; - let condition = Box::new(self.expression(Precedence::Lowest)?); + let condition = Box::new(self.expression(state, Precedence::Lowest)?); - self.rparen()?; - self.lbrace()?; + self.rparen(state)?; + self.lbrace(state)?; let mut arms = Vec::new(); - while self.current.kind != TokenKind::RightBrace { + while state.current.kind != TokenKind::RightBrace { let mut conditions = Vec::new(); - while self.current.kind != TokenKind::DoubleArrow { - if self.current.kind == TokenKind::Default { - self.next(); + while state.current.kind != TokenKind::DoubleArrow { + if state.current.kind == TokenKind::Default { + state.next(); break; } - conditions.push(self.expression(Precedence::Lowest)?); + conditions.push(self.expression(state, Precedence::Lowest)?); - self.optional_comma()?; + self.optional_comma(state)?; } - expect_token!([TokenKind::DoubleArrow], self, "`=>`"); + expect_token!([TokenKind::DoubleArrow], state, "`=>`"); - let body = self.expression(Precedence::Lowest)?; + let body = self.expression(state, Precedence::Lowest)?; - self.optional_comma()?; + self.optional_comma(state)?; arms.push(MatchArm { conditions: if conditions.is_empty() { @@ -1190,94 +1167,94 @@ impl Parser { }) } - self.rbrace()?; + self.rbrace(state)?; Expression::Match { condition, arms } } TokenKind::Array => { let mut items = vec![]; - self.next(); + state.next(); - self.lparen()?; + self.lparen(state)?; - while self.current.kind != TokenKind::RightParen { + while state.current.kind != TokenKind::RightParen { let mut key = None; - let unpack = if self.current.kind == TokenKind::Ellipsis { - self.next(); + let unpack = if state.current.kind == TokenKind::Ellipsis { + state.next(); true } else { false }; - let mut value = self.expression(Precedence::Lowest)?; + let mut value = self.expression(state, Precedence::Lowest)?; - if self.current.kind == TokenKind::DoubleArrow { - self.next(); + if state.current.kind == TokenKind::DoubleArrow { + state.next(); key = Some(value); - value = self.expression(Precedence::Lowest)?; + value = self.expression(state, Precedence::Lowest)?; } items.push(ArrayItem { key, value, unpack }); - self.optional_comma()?; + self.optional_comma(state)?; - self.skip_comments(); + state.skip_comments(); } - self.rparen()?; + self.rparen(state)?; Expression::Array { items } } TokenKind::LeftBracket => { let mut items = Vec::new(); - self.next(); + state.next(); - self.skip_comments(); + state.skip_comments(); - while self.current.kind != TokenKind::RightBracket { - if self.current.kind == TokenKind::Comma { + while state.current.kind != TokenKind::RightBracket { + if state.current.kind == TokenKind::Comma { items.push(ArrayItem { key: None, value: Expression::Empty, unpack: false, }); - self.next(); + state.next(); continue; } let mut key = None; - let unpack = if self.current.kind == TokenKind::Ellipsis { - self.next(); + let unpack = if state.current.kind == TokenKind::Ellipsis { + state.next(); true } else { false }; - let mut value = self.expression(Precedence::Lowest)?; + let mut value = self.expression(state, Precedence::Lowest)?; - if self.current.kind == TokenKind::DoubleArrow { - self.next(); + if state.current.kind == TokenKind::DoubleArrow { + state.next(); key = Some(value); - value = self.expression(Precedence::Lowest)?; + value = self.expression(state, Precedence::Lowest)?; } items.push(ArrayItem { key, value, unpack }); - self.optional_comma()?; + self.optional_comma(state)?; - self.skip_comments(); + state.skip_comments(); } - self.rbracket()?; + self.rbracket(state)?; Expression::Array { items } } - TokenKind::Static if matches!(self.peek.kind, TokenKind::Function | TokenKind::Fn) => { - self.next(); + TokenKind::Static if matches!(state.peek.kind, TokenKind::Function | TokenKind::Fn) => { + state.next(); - match self.expression(Precedence::Lowest)? { + match self.expression(state, Precedence::Lowest)? { Expression::Closure { params, uses, @@ -1310,33 +1287,33 @@ impl Parser { } } TokenKind::Function => { - self.next(); + state.next(); - let by_ref = if self.current.kind == TokenKind::Ampersand { - self.next(); + let by_ref = if state.current.kind == TokenKind::Ampersand { + state.next(); true } else { false }; - self.lparen()?; + self.lparen(state)?; - let params = self.param_list(ParamPosition::Function)?; + let params = self.param_list(state, ParamPosition::Function)?; - self.rparen()?; + self.rparen(state)?; let mut uses = vec![]; - if self.current.kind == TokenKind::Use { - self.next(); + if state.current.kind == TokenKind::Use { + state.next(); - self.lparen()?; + self.lparen(state)?; - while self.current.kind != TokenKind::RightParen { - let var = match self.current.kind { + while state.current.kind != TokenKind::RightParen { + let var = match state.current.kind { TokenKind::Ampersand => { - self.next(); + state.next(); - match self.expression(Precedence::Lowest)? { + match self.expression(state, Precedence::Lowest)? { s @ Expression::Variable { .. } => ClosureUse { var: s, by_ref: true, @@ -1344,12 +1321,12 @@ impl Parser { _ => { return Err(ParseError::UnexpectedToken( "expected variable".into(), - self.current.span, + state.current.span, )) } } } - _ => match self.expression(Precedence::Lowest)? { + _ => match self.expression(state, Precedence::Lowest)? { s @ Expression::Variable { .. } => ClosureUse { var: s, by_ref: false, @@ -1357,7 +1334,7 @@ impl Parser { _ => { return Err(ParseError::UnexpectedToken( "expected variable".into(), - self.current.span, + state.current.span, )) } }, @@ -1365,24 +1342,24 @@ impl Parser { uses.push(var); - self.optional_comma()?; + self.optional_comma(state)?; } - self.rparen()?; + self.rparen(state)?; } let mut return_type = None; - if self.current.kind == TokenKind::Colon || self.config.force_type_strings { - self.colon()?; + if state.current.kind == TokenKind::Colon { + self.colon(state)?; - return_type = Some(self.type_string()?); + return_type = Some(self.type_string(state)?); } - self.lbrace()?; + self.lbrace(state)?; - let body = self.block(&TokenKind::RightBrace)?; + let body = self.block(state, &TokenKind::RightBrace)?; - self.rbrace()?; + self.rbrace(state)?; Expression::Closure { params, @@ -1394,32 +1371,32 @@ impl Parser { } } TokenKind::Fn => { - self.next(); + state.next(); - let by_ref = if self.current.kind == TokenKind::Ampersand { - self.next(); + let by_ref = if state.current.kind == TokenKind::Ampersand { + state.next(); true } else { false }; - self.lparen()?; + self.lparen(state)?; - let params = self.param_list(ParamPosition::Function)?; + let params = self.param_list(state, ParamPosition::Function)?; - self.rparen()?; + self.rparen(state)?; let mut return_type = None; - if self.current.kind == TokenKind::Colon || self.config.force_type_strings { - self.colon()?; + if state.current.kind == TokenKind::Colon { + self.colon(state)?; - return_type = Some(self.type_string()?); + return_type = Some(self.type_string(state)?); } - expect_token!([TokenKind::DoubleArrow], self, ["`=>`"]); + expect_token!([TokenKind::DoubleArrow], state, ["`=>`"]); - let value = self.expression(Precedence::Lowest)?; + let value = self.expression(state, Precedence::Lowest)?; Expression::ArrowFunction { params, @@ -1429,21 +1406,21 @@ impl Parser { r#static: false, } } - TokenKind::New if self.peek.kind == TokenKind::Class => { - self.anonymous_class_definition()? + TokenKind::New if state.peek.kind == TokenKind::Class => { + self.anonymous_class_definition(state)? } TokenKind::New => { - self.next(); + state.next(); let mut args = vec![]; - let target = self.expression(Precedence::CloneOrNew)?; + let target = self.expression(state, Precedence::CloneOrNew)?; - if self.current.kind == TokenKind::LeftParen { - self.lparen()?; + if state.current.kind == TokenKind::LeftParen { + self.lparen(state)?; - args = self.args_list()?; + args = self.args_list(state)?; - self.rparen()?; + self.rparen(state)?; } Expression::New { @@ -1452,45 +1429,45 @@ impl Parser { } } TokenKind::DirConstant => { - self.next(); + state.next(); Expression::MagicConst { constant: MagicConst::Dir, } } - _ if is_prefix(&self.current.kind) => { - let op = self.current.kind.clone(); + _ if is_prefix(&state.current.kind) => { + let op = state.current.kind.clone(); - self.next(); + state.next(); let rpred = Precedence::prefix(&op); - let rhs = self.expression(rpred)?; + let rhs = self.expression(state, rpred)?; prefix(&op, rhs) } - TokenKind::Dollar => self.dynamic_variable()?, + TokenKind::Dollar => self.dynamic_variable(state)?, _ => { return Err(ParseError::UnexpectedToken( - self.current.kind.to_string(), - self.current.span, + state.current.kind.to_string(), + state.current.span, )) } }; - if self.current.kind == TokenKind::SemiColon { + if state.current.kind == TokenKind::SemiColon { return Ok(left); } - self.skip_comments(); + state.skip_comments(); loop { - self.skip_comments(); + state.skip_comments(); - if matches!(self.current.kind, TokenKind::SemiColon | TokenKind::Eof) { + if matches!(state.current.kind, TokenKind::SemiColon | TokenKind::Eof) { break; } - let span = self.current.span; - let kind = self.current.kind.clone(); + let span = state.current.span; + let kind = state.current.kind.clone(); if is_postfix(&kind) { let lpred = Precedence::postfix(&kind); @@ -1499,9 +1476,9 @@ impl Parser { break; } - self.next(); + state.next(); - left = self.postfix(left, &kind)?; + left = self.postfix(state, left, &kind)?; continue; } @@ -1522,13 +1499,13 @@ impl Parser { return Err(ParseError::UnexpectedToken(kind.to_string(), span)); } - self.next(); + state.next(); match kind { TokenKind::Question => { - let then = self.expression(Precedence::Lowest)?; - self.colon()?; - let otherwise = self.expression(rpred)?; + let then = self.expression(state, Precedence::Lowest)?; + self.colon(state)?; + let otherwise = self.expression(state, rpred)?; left = Expression::Ternary { condition: Box::new(left), then: Some(Box::new(then)), @@ -1536,7 +1513,7 @@ impl Parser { } } TokenKind::QuestionColon => { - let r#else = self.expression(Precedence::Lowest)?; + let r#else = self.expression(state, Precedence::Lowest)?; left = Expression::Ternary { condition: Box::new(left), then: None, @@ -1544,7 +1521,7 @@ impl Parser { } } _ => { - let rhs = self.expression(rpred)?; + let rhs = self.expression(state, rpred)?; left = infix(left, kind, rhs); } } @@ -1555,15 +1532,20 @@ impl Parser { break; } - self.skip_comments(); + state.skip_comments(); Ok(left) } - fn postfix(&mut self, lhs: Expression, op: &TokenKind) -> Result { + fn postfix( + &self, + state: &mut State, + lhs: Expression, + op: &TokenKind, + ) -> Result { Ok(match op { TokenKind::Coalesce => { - let rhs = self.expression(Precedence::NullCoalesce)?; + let rhs = self.expression(state, Precedence::NullCoalesce)?; Expression::Coalesce { lhs: Box::new(lhs), @@ -1571,9 +1553,9 @@ impl Parser { } } TokenKind::LeftParen => { - let args = self.args_list()?; + let args = self.args_list(state)?; - self.rparen()?; + self.rparen(state)?; Expression::Call { target: Box::new(lhs), @@ -1581,17 +1563,17 @@ impl Parser { } } TokenKind::LeftBracket => { - if self.current.kind == TokenKind::RightBracket { - self.next(); + if state.current.kind == TokenKind::RightBracket { + state.next(); Expression::ArrayIndex { array: Box::new(lhs), index: None, } } else { - let index = self.expression(Precedence::Lowest)?; + let index = self.expression(state, Precedence::Lowest)?; - expect_token!([TokenKind::RightBracket], self, ["`]`"]); + expect_token!([TokenKind::RightBracket], state, ["`]`"]); Expression::ArrayIndex { array: Box::new(lhs), @@ -1602,40 +1584,40 @@ impl Parser { TokenKind::DoubleColon => { let mut must_be_method_call = false; - let property = match self.current.kind.clone() { - TokenKind::Dollar => self.dynamic_variable()?, + let property = match state.current.kind.clone() { + TokenKind::Dollar => self.dynamic_variable(state)?, TokenKind::Variable(var) => { - self.next(); + state.next(); Expression::Variable { name: var } } TokenKind::LeftBrace => { must_be_method_call = true; - self.next(); + state.next(); - let name = self.expression(Precedence::Lowest)?; + let name = self.expression(state, Precedence::Lowest)?; - self.rbrace()?; + self.rbrace(state)?; Expression::DynamicVariable { name: Box::new(name), } } TokenKind::Identifier(ident) => { - self.next(); + state.next(); Expression::Identifier { name: ident } } TokenKind::Class => { - self.next(); + state.next(); // FIXME: Can this be represented in a nicer way? Kind of hacky. Expression::Identifier { name: "class".into(), } } - _ if is_reserved_ident(&self.current.kind) => Expression::Identifier { - name: self.ident_maybe_reserved()?, + _ if is_reserved_ident(&state.current.kind) => Expression::Identifier { + name: self.ident_maybe_reserved(state)?, }, _ => { - return expected_token_err!(["`{`", "`$`", "an identifier"], self); + return expected_token_err!(["`{`", "`$`", "an identifier"], state); } }; @@ -1645,7 +1627,7 @@ impl Parser { // 1. If we have an identifier and the current token is not a left paren, // the resulting expression must be a constant fetch. Expression::Identifier { name } - if self.current.kind != TokenKind::LeftParen => + if state.current.kind != TokenKind::LeftParen => { Expression::ConstFetch { target: lhs, @@ -1655,12 +1637,12 @@ impl Parser { // 2. If the current token is a left paren, or if we know the property expression // is only valid a method call context, we can assume we're parsing a static // method call. - _ if self.current.kind == TokenKind::LeftParen || must_be_method_call => { - self.lparen()?; + _ if state.current.kind == TokenKind::LeftParen || must_be_method_call => { + self.lparen(state)?; - let args = self.args_list()?; + let args = self.args_list(state)?; - self.rparen()?; + self.rparen(state)?; Expression::StaticMethodCall { target: lhs, @@ -1677,30 +1659,30 @@ impl Parser { } } TokenKind::Arrow | TokenKind::NullsafeArrow => { - let property = match self.current.kind { + let property = match state.current.kind { TokenKind::LeftBrace => { - self.lbrace()?; - let expr = self.expression(Precedence::Lowest)?; - self.rbrace()?; + self.lbrace(state)?; + let expr = self.expression(state, Precedence::Lowest)?; + self.rbrace(state)?; expr } TokenKind::Variable(ref var) => { let var = Expression::Variable { name: var.clone() }; - self.next(); + state.next(); var } - TokenKind::Dollar => self.dynamic_variable()?, + TokenKind::Dollar => self.dynamic_variable(state)?, _ => Expression::Identifier { - name: self.ident_maybe_reserved()?, + name: self.ident_maybe_reserved(state)?, }, }; - if self.current.kind == TokenKind::LeftParen { - self.next(); + if state.current.kind == TokenKind::LeftParen { + state.next(); - let args = self.args_list()?; + let args = self.args_list(state)?; - self.rparen()?; + self.rparen(state)?; if op == &TokenKind::NullsafeArrow { Expression::NullsafeMethodCall { @@ -1737,35 +1719,35 @@ impl Parser { }) } - fn interpolated_string(&mut self) -> ParseResult { + fn interpolated_string(&self, state: &mut State) -> ParseResult { let mut parts = Vec::new(); - while self.current.kind != TokenKind::DoubleQuote { - match &self.current.kind { + while state.current.kind != TokenKind::DoubleQuote { + match &state.current.kind { TokenKind::StringPart(s) => { if s.len() > 0 { parts.push(StringPart::Const(s.clone())); } - self.next(); + state.next(); } TokenKind::DollarLeftBrace => { - self.next(); - let e = match (&self.current.kind, &self.peek.kind) { + state.next(); + let e = match (&state.current.kind, &state.peek.kind) { (TokenKind::Identifier(var), TokenKind::RightBrace) => { // "${var}" let e = Expression::Variable { name: var.clone() }; - self.next(); - self.next(); + state.next(); + state.next(); e } (TokenKind::Identifier(var), TokenKind::LeftBracket) => { // "${var[e]}" let var = Expression::Variable { name: var.clone() }; - self.next(); - self.next(); - let e = self.expression(Precedence::Lowest)?; - expect_token!([TokenKind::RightBracket], self, "`]`"); - expect_token!([TokenKind::RightBrace], self, "`}`"); + state.next(); + state.next(); + let e = self.expression(state, Precedence::Lowest)?; + expect_token!([TokenKind::RightBracket], state, "`]`"); + expect_token!([TokenKind::RightBrace], state, "`}`"); Expression::ArrayIndex { array: Box::new(var), index: Some(Box::new(e)), @@ -1773,8 +1755,8 @@ impl Parser { } _ => { // Arbitrary expressions are allowed, but are treated as variable variables. - let e = self.expression(Precedence::Lowest)?; - expect_token!([TokenKind::RightBrace], self, "`}`"); + let e = self.expression(state, Precedence::Lowest)?; + expect_token!([TokenKind::RightBrace], state, "`}`"); Expression::DynamicVariable { name: Box::new(e) } } @@ -1783,77 +1765,77 @@ impl Parser { } TokenKind::LeftBrace => { // "{$expr}" - self.next(); - let e = self.expression(Precedence::Lowest)?; - expect_token!([TokenKind::RightBrace], self, "`}`"); + state.next(); + let e = self.expression(state, Precedence::Lowest)?; + expect_token!([TokenKind::RightBrace], state, "`}`"); parts.push(StringPart::Expr(Box::new(e))); } TokenKind::Variable(var) => { // "$expr", "$expr[0]", "$expr[name]", "$expr->a" let var = Expression::Variable { name: var.clone() }; - self.next(); - let e = match self.current.kind { + state.next(); + let e = match state.current.kind { TokenKind::LeftBracket => { - self.next(); + state.next(); // Full expression syntax is not allowed here, // so we can't call self.expression. - let index = match &self.current.kind { + let index = match &state.current.kind { &TokenKind::LiteralInteger(i) => { - self.next(); + state.next(); Expression::LiteralInteger { i } } TokenKind::Minus => { - self.next(); - if let TokenKind::LiteralInteger(i) = self.current.kind { - self.next(); + state.next(); + if let TokenKind::LiteralInteger(i) = state.current.kind { + state.next(); Expression::Negate { value: Box::new(Expression::LiteralInteger { i }), } } else { - return expected_token_err!("an integer", self); + return expected_token_err!("an integer", state); } } TokenKind::Identifier(ident) => { let e = Expression::LiteralString { value: ident.clone(), }; - self.next(); + state.next(); e } TokenKind::Variable(var) => { let e = Expression::Variable { name: var.clone() }; - self.next(); + state.next(); e } _ => { return expected_token_err!( ["`-`", "an integer", "an identifier", "a variable"], - self + state ); } }; - expect_token!([TokenKind::RightBracket], self, "`]`"); + expect_token!([TokenKind::RightBracket], state, "`]`"); Expression::ArrayIndex { array: Box::new(var), index: Some(Box::new(index)), } } TokenKind::Arrow => { - self.next(); + state.next(); Expression::PropertyFetch { target: Box::new(var), property: Box::new(Expression::Identifier { - name: self.ident_maybe_reserved()?, + name: self.ident_maybe_reserved(state)?, }), } } TokenKind::NullsafeArrow => { - self.next(); + state.next(); Expression::NullsafePropertyFetch { target: Box::new(var), property: Box::new(Expression::Identifier { - name: self.ident_maybe_reserved()?, + name: self.ident_maybe_reserved(state)?, }), } } @@ -1862,22 +1844,14 @@ impl Parser { parts.push(StringPart::Expr(Box::new(e))); } _ => { - return expected_token_err!(["`${`", "`{$", "`\"`", "a variable"], self); + return expected_token_err!(["`${`", "`{$", "`\"`", "a variable"], state); } } } - self.next(); - Ok(Expression::InterpolatedString { parts }) - } - - fn is_eof(&self) -> bool { - self.current.kind == TokenKind::Eof - } + state.next(); - pub fn next(&mut self) { - self.current = self.peek.clone(); - self.peek = self.iter.next().unwrap_or_default() + Ok(Expression::InterpolatedString { parts }) } } diff --git a/src/parser/params.rs b/src/parser/params.rs index 1d1a7dba..d09820a4 100644 --- a/src/parser/params.rs +++ b/src/parser/params.rs @@ -7,6 +7,7 @@ use crate::parser::ast::PropertyFlag; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; use crate::parser::precedence::Precedence; +use crate::parser::state::State; use crate::parser::Parser; use crate::expect_token; @@ -20,16 +21,17 @@ pub enum ParamPosition { impl Parser { pub(in crate::parser) fn param_list( - &mut self, + &self, + state: &mut State, position: ParamPosition, ) -> Result { let mut params = ParamList::new(); - while !self.is_eof() && self.current.kind != TokenKind::RightParen { + while !state.is_eof() && state.current.kind != TokenKind::RightParen { let mut param_type = None; let flags: Vec = self - .promoted_property_flags()? + .promoted_property_flags(state)? .iter() .map(|f| f.into()) .collect(); @@ -38,17 +40,17 @@ impl Parser { match position { ParamPosition::Method(name) if name != "__construct" => { return Err(ParseError::PromotedPropertyOutsideConstructor( - self.current.span, + state.current.span, )); } ParamPosition::AbstractMethod(name) => { if name == "__construct" { return Err(ParseError::PromotedPropertyOnAbstractConstructor( - self.current.span, + state.current.span, )); } else { return Err(ParseError::PromotedPropertyOutsideConstructor( - self.current.span, + state.current.span, )); } } @@ -57,29 +59,28 @@ impl Parser { } // If this is a readonly promoted property, or we don't see a variable - if self.config.force_type_strings - || flags.contains(&PropertyFlag::Readonly) + if flags.contains(&PropertyFlag::Readonly) || !matches!( - self.current.kind, + state.current.kind, TokenKind::Variable(_) | TokenKind::Ellipsis | TokenKind::Ampersand ) { // Try to parse the type. - param_type = Some(self.type_string()?); + param_type = Some(self.type_string(state)?); } let mut variadic = false; let mut by_ref = false; - if matches!(self.current.kind, TokenKind::Ampersand) { - self.next(); + if matches!(state.current.kind, TokenKind::Ampersand) { + state.next(); by_ref = true; } - if matches!(self.current.kind, TokenKind::Ellipsis) { - self.next(); + if matches!(state.current.kind, TokenKind::Ellipsis) { + state.next(); if !flags.is_empty() { - return Err(ParseError::VariadicPromotedProperty(self.current.span)); + return Err(ParseError::VariadicPromotedProperty(state.current.span)); } variadic = true; @@ -88,12 +89,12 @@ impl Parser { // 2. Then expect a variable. let var = expect_token!([ TokenKind::Variable(v) => v - ], self, "a varaible"); + ], state, "a varaible"); let mut default = None; - if self.current.kind == TokenKind::Equals { - self.next(); - default = Some(self.expression(Precedence::Lowest)?); + if state.current.kind == TokenKind::Equals { + state.next(); + default = Some(self.expression(state, Precedence::Lowest)?); } params.push(Param { @@ -105,29 +106,29 @@ impl Parser { by_ref, }); - self.optional_comma()?; + self.optional_comma(state)?; } Ok(params) } - pub(in crate::parser) fn args_list(&mut self) -> ParseResult> { + pub(in crate::parser) fn args_list(&self, state: &mut State) -> ParseResult> { let mut args = Vec::new(); - while !self.is_eof() && self.current.kind != TokenKind::RightParen { + while !state.is_eof() && state.current.kind != TokenKind::RightParen { let mut name = None; let mut unpack = false; - if matches!(self.current.kind, TokenKind::Identifier(_)) - && self.peek.kind == TokenKind::Colon + if matches!(state.current.kind, TokenKind::Identifier(_)) + && state.peek.kind == TokenKind::Colon { - name = Some(self.ident_maybe_reserved()?); - self.next(); - } else if self.current.kind == TokenKind::Ellipsis { - self.next(); + name = Some(self.ident_maybe_reserved(state)?); + state.next(); + } else if state.current.kind == TokenKind::Ellipsis { + state.next(); unpack = true; } - if unpack && self.current.kind == TokenKind::RightParen { + if unpack && state.current.kind == TokenKind::RightParen { args.push(Arg { name: None, unpack: false, @@ -137,7 +138,7 @@ impl Parser { break; } - let value = self.expression(Precedence::Lowest)?; + let value = self.expression(state, Precedence::Lowest)?; args.push(Arg { name, @@ -145,7 +146,7 @@ impl Parser { value, }); - self.optional_comma()?; + self.optional_comma(state)?; } Ok(args) diff --git a/src/parser/punc.rs b/src/parser/punc.rs index 1eb5b9dc..e8a060d5 100644 --- a/src/parser/punc.rs +++ b/src/parser/punc.rs @@ -1,50 +1,51 @@ use crate::lexer::token::TokenKind; use crate::parser::error::ParseResult; +use crate::parser::state::State; use crate::parser::Parser; use crate::expect_token; impl Parser { - pub(in crate::parser) fn semi(&mut self) -> ParseResult<()> { - expect_token!([TokenKind::SemiColon], self, "`;`"); + pub(in crate::parser) fn semi(&self, state: &mut State) -> ParseResult<()> { + expect_token!([TokenKind::SemiColon], state, "`;`"); Ok(()) } - pub(in crate::parser) fn lbrace(&mut self) -> ParseResult<()> { - expect_token!([TokenKind::LeftBrace], self, "`{`"); + pub(in crate::parser) fn lbrace(&self, state: &mut State) -> ParseResult<()> { + expect_token!([TokenKind::LeftBrace], state, "`{`"); Ok(()) } - pub(in crate::parser) fn rbrace(&mut self) -> ParseResult<()> { - expect_token!([TokenKind::RightBrace], self, "`}`"); + pub(in crate::parser) fn rbrace(&self, state: &mut State) -> ParseResult<()> { + expect_token!([TokenKind::RightBrace], state, "`}`"); Ok(()) } - pub(in crate::parser) fn lparen(&mut self) -> ParseResult<()> { - expect_token!([TokenKind::LeftParen], self, "`(`"); + pub(in crate::parser) fn lparen(&self, state: &mut State) -> ParseResult<()> { + expect_token!([TokenKind::LeftParen], state, "`(`"); Ok(()) } - pub(in crate::parser) fn rparen(&mut self) -> ParseResult<()> { - expect_token!([TokenKind::RightParen], self, "`)`"); + pub(in crate::parser) fn rparen(&self, state: &mut State) -> ParseResult<()> { + expect_token!([TokenKind::RightParen], state, "`)`"); Ok(()) } - pub(in crate::parser) fn rbracket(&mut self) -> ParseResult<()> { - expect_token!([TokenKind::RightBracket], self, "`]`"); + pub(in crate::parser) fn rbracket(&self, state: &mut State) -> ParseResult<()> { + expect_token!([TokenKind::RightBracket], state, "`]`"); Ok(()) } - pub(in crate::parser) fn optional_comma(&mut self) -> ParseResult<()> { - if self.current.kind == TokenKind::Comma { - expect_token!([TokenKind::Comma], self, "`,`"); + pub(in crate::parser) fn optional_comma(&self, state: &mut State) -> ParseResult<()> { + if state.current.kind == TokenKind::Comma { + expect_token!([TokenKind::Comma], state, "`,`"); } Ok(()) } - pub(in crate::parser) fn colon(&mut self) -> ParseResult<()> { - expect_token!([TokenKind::Colon], self, "`:`"); + pub(in crate::parser) fn colon(&self, state: &mut State) -> ParseResult<()> { + expect_token!([TokenKind::Colon], state, "`:`"); Ok(()) } diff --git a/src/parser/state.rs b/src/parser/state.rs new file mode 100644 index 00000000..2eb25d82 --- /dev/null +++ b/src/parser/state.rs @@ -0,0 +1,59 @@ +use std::vec::IntoIter; + +use crate::lexer::token::Token; +use crate::lexer::token::TokenKind; + +#[derive(Debug, Clone)] +pub struct State { + pub current: Token, + pub peek: Token, + pub iter: IntoIter, + pub comments: Vec, +} + +impl State { + pub fn new(tokens: Vec) -> Self { + let mut iter = tokens.into_iter(); + + Self { + current: iter.next().unwrap_or_default(), + peek: iter.next().unwrap_or_default(), + iter, + comments: vec![], + } + } + + pub fn skip_comments(&mut self) { + while matches!( + self.current.kind, + TokenKind::Comment(_) | TokenKind::DocComment(_) + ) { + self.next(); + } + } + + pub fn gather_comments(&mut self) { + while matches!( + self.current.kind, + TokenKind::Comment(_) | TokenKind::DocComment(_) + ) { + self.comments.push(self.current.clone()); + self.next(); + } + } + + pub fn clear_comments(&mut self) -> Vec { + let c = self.comments.clone(); + self.comments = vec![]; + c + } + + pub fn is_eof(&mut self) -> bool { + self.current.kind == TokenKind::Eof + } + + pub fn next(&mut self) { + self.current = self.peek.clone(); + self.peek = self.iter.next().unwrap_or_default() + } +} diff --git a/src/parser/vars.rs b/src/parser/vars.rs index 29da2f78..58d5b5be 100644 --- a/src/parser/vars.rs +++ b/src/parser/vars.rs @@ -3,19 +3,20 @@ use crate::parser::ast::Expression; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; use crate::parser::precedence::Precedence; +use crate::parser::state::State; use crate::parser::Parser; impl Parser { - pub(in crate::parser) fn dynamic_variable(&mut self) -> ParseResult { - self.next(); + pub(in crate::parser) fn dynamic_variable(&self, state: &mut State) -> ParseResult { + state.next(); - Ok(match &self.current.kind { + Ok(match &state.current.kind { TokenKind::LeftBrace => { - self.next(); + state.next(); - let name = self.expression(Precedence::Lowest)?; + let name = self.expression(state, Precedence::Lowest)?; - self.rbrace()?; + self.rbrace(state)?; Expression::DynamicVariable { name: Box::new(name), @@ -24,7 +25,7 @@ impl Parser { TokenKind::Variable(variable) => { let variable = variable.clone(); - self.next(); + state.next(); Expression::DynamicVariable { name: Box::new(Expression::Variable { name: variable }), @@ -32,8 +33,8 @@ impl Parser { } _ => { return Err(ParseError::UnexpectedToken( - self.current.kind.to_string(), - self.current.span, + state.current.kind.to_string(), + state.current.span, )) } }) diff --git a/tests/third_party_tests.rs b/tests/third_party_tests.rs index 6cc27983..4522bfaa 100644 --- a/tests/third_party_tests.rs +++ b/tests/third_party_tests.rs @@ -76,7 +76,7 @@ fn test_file(name: &str, filename: PathBuf) { Lexer::new() .tokenize(code.as_bytes()) .map(|tokens| { - Parser::new(None) + Parser::new() .parse(tokens) .map(|_| { println!("✅ successfully parsed file: `\"{}\"`.", name); From 66bfa1b77eb4961a64c40ee3fa3a259a528a6b50 Mon Sep 17 00:00:00 2001 From: Saif Eddin Gmati <29315886+azjezz@users.noreply.github.com> Date: Thu, 1 Dec 2022 13:55:04 +0100 Subject: [PATCH 04/13] feat: add parser state scope --- src/lexer/error.rs | 8 +- src/lexer/mod.rs | 80 ++-- src/lexer/state.rs | 34 +- src/parser/ast.rs | 4 + src/parser/classish.rs | 263 ------------- src/parser/error.rs | 34 +- src/parser/functions.rs | 155 -------- src/parser/{ => internal}/block.rs | 0 src/parser/internal/classish.rs | 274 +++++++++++++ .../{ => internal}/classish_statement.rs | 43 +- src/parser/{ => internal}/flags.rs | 0 src/parser/internal/functions.rs | 274 +++++++++++++ src/parser/{ => internal}/ident.rs | 0 src/parser/internal/mod.rs | 11 + src/parser/{ => internal}/params.rs | 62 +-- src/parser/{ => internal}/precedence.rs | 0 src/parser/{ => internal}/punc.rs | 0 src/parser/internal/types.rs | 144 +++++++ src/parser/{ => internal}/vars.rs | 2 +- src/parser/macros.rs | 14 + src/parser/mod.rs | 371 ++---------------- src/parser/state.rs | 69 ++++ src/prelude.rs | 1 - tests/0002/parser-error.txt | 2 +- tests/0003/parser-error.txt | 2 +- tests/0004/parser-error.txt | 2 +- tests/0005/parser-error.txt | 2 +- tests/0016/parser-error.txt | 2 +- tests/0019/ast.txt | 8 +- tests/0044/parser-error.txt | 2 +- tests/0087/parser-error.txt | 2 +- tests/0088/parser-error.txt | 2 +- tests/0111/parser-error.txt | 2 +- tests/0112/parser-error.txt | 2 +- tests/0113/parser-error.txt | 2 +- tests/0114/parser-error.txt | 2 +- tests/0115/parser-error.txt | 2 +- tests/0116/parser-error.txt | 2 +- tests/0120/parser-error.txt | 2 +- tests/0121/parser-error.txt | 2 +- tests/0122/parser-error.txt | 2 +- tests/0123/parser-error.txt | 2 +- tests/0124/parser-error.txt | 2 +- tests/0125/parser-error.txt | 2 +- tests/0126/parser-error.txt | 2 +- tests/0128/parser-error.txt | 2 +- tests/0131/parser-error.txt | 2 +- tests/0132/parser-error.txt | 2 +- tests/0141/parser-error.txt | 2 +- tests/0142/parser-error.txt | 2 +- tests/0144/parser-error.txt | 2 +- tests/0146/code.php | 9 + tests/0146/parser-error.txt | 1 + tests/0146/tokens.txt | 115 ++++++ 54 files changed, 1123 insertions(+), 907 deletions(-) delete mode 100644 src/parser/classish.rs delete mode 100644 src/parser/functions.rs rename src/parser/{ => internal}/block.rs (100%) create mode 100644 src/parser/internal/classish.rs rename src/parser/{ => internal}/classish_statement.rs (89%) rename src/parser/{ => internal}/flags.rs (100%) create mode 100644 src/parser/internal/functions.rs rename src/parser/{ => internal}/ident.rs (100%) create mode 100644 src/parser/internal/mod.rs rename src/parser/{ => internal}/params.rs (71%) rename src/parser/{ => internal}/precedence.rs (100%) rename src/parser/{ => internal}/punc.rs (100%) create mode 100644 src/parser/internal/types.rs rename src/parser/{ => internal}/vars.rs (95%) create mode 100644 tests/0146/code.php create mode 100644 tests/0146/parser-error.txt create mode 100644 tests/0146/tokens.txt diff --git a/src/lexer/error.rs b/src/lexer/error.rs index 5576a58b..9b48eb78 100644 --- a/src/lexer/error.rs +++ b/src/lexer/error.rs @@ -2,7 +2,7 @@ use std::fmt::Display; use crate::lexer::token::Span; -pub type LexResult = Result; +pub type SyntaxResult = Result; #[derive(Debug, Eq, PartialEq)] pub enum SyntaxError { @@ -13,6 +13,7 @@ pub enum SyntaxError { InvalidOctalEscape(Span), InvalidOctalLiteral(Span), InvalidUnicodeEscape(Span), + UnpredictableState(Span), } impl Display for SyntaxError { @@ -53,6 +54,11 @@ impl Display for SyntaxError { "Syntax Error: invalid unicode escape on line {} column {}", span.0, span.1 ), + Self::UnpredictableState(span) => write!( + f, + "Syntax Error: Reached an unpredictable state on line {} column {}", + span.0, span.1 + ), } } } diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index de47b286..d0cf09eb 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -9,7 +9,8 @@ use std::num::IntErrorKind; use crate::lexer::byte_string::ByteString; use crate::lexer::error::SyntaxError; -use crate::lexer::state::StackState; +use crate::lexer::error::SyntaxResult; +use crate::lexer::state::StackFrame; use crate::lexer::state::State; use crate::lexer::token::OpenTagKind; use crate::lexer::token::Span; @@ -27,21 +28,21 @@ impl Lexer { Self {} } - pub fn tokenize>(&self, input: &B) -> Result, SyntaxError> { + pub fn tokenize>(&self, input: &B) -> SyntaxResult> { let mut state = State::new(input); let mut tokens = Vec::new(); while state.current.is_some() { - match state.stack.last().unwrap() { + match state.frame()? { // The "Initial" state is used to parse inline HTML. It is essentially a catch-all // state that will build up a single token buffer until it encounters an open tag // of some description. - StackState::Initial => { + StackFrame::Initial => { tokens.append(&mut self.initial(&mut state)?); } // The scripting state is entered when an open tag is encountered in the source code. // This tells the lexer to start analysing characters at PHP tokens instead of inline HTML. - StackState::Scripting => { + StackFrame::Scripting => { self.skip_whitespace(&mut state); // If we have consumed whitespace and then reached the end of the file, we should break. @@ -54,7 +55,7 @@ impl Lexer { // The "Halted" state is entered when the `__halt_compiler` token is encountered. // In this state, all the text that follows is no longer parsed as PHP as is collected // into a single "InlineHtml" token (kind of cheating, oh well). - StackState::Halted => { + StackFrame::Halted => { tokens.push(Token { kind: TokenKind::InlineHtml(state.chars[state.cursor..].into()), span: state.span, @@ -63,22 +64,22 @@ impl Lexer { } // The double quote state is entered when inside a double-quoted string that // contains variables. - StackState::DoubleQuote => tokens.extend(self.double_quote(&mut state)?), + StackFrame::DoubleQuote => tokens.extend(self.double_quote(&mut state)?), // LookingForProperty is entered inside double quotes, // backticks, or a heredoc, expecting a variable name. // If one isn't found, it switches to scripting. - StackState::LookingForVarname => { - if let Some(token) = self.looking_for_varname(&mut state) { + StackFrame::LookingForVarname => { + if let Some(token) = self.looking_for_varname(&mut state)? { tokens.push(token); } } // LookingForProperty is entered inside double quotes, // backticks, or a heredoc, expecting an arrow followed by a // property name. - StackState::LookingForProperty => { + StackFrame::LookingForProperty => { tokens.push(self.looking_for_property(&mut state)?); } - StackState::VarOffset => { + StackFrame::VarOffset => { if state.current.is_none() { break; } @@ -97,7 +98,7 @@ impl Lexer { } } - fn initial(&self, state: &mut State) -> Result, SyntaxError> { + fn initial(&self, state: &mut State) -> SyntaxResult> { let inline_span = state.span; let mut buffer = Vec::new(); while let Some(char) = state.current { @@ -105,7 +106,7 @@ impl Lexer { let tag_span = state.span; state.skip(5); - state.enter_state(StackState::Scripting); + state.set(StackFrame::Scripting)?; let mut tokens = vec![]; @@ -134,7 +135,7 @@ impl Lexer { }]) } - fn scripting(&self, state: &mut State) -> Result { + fn scripting(&self, state: &mut State) -> SyntaxResult { let span = state.span; let kind = match state.peek_buf() { [b'@', ..] => { @@ -170,7 +171,7 @@ impl Lexer { // This is a close tag, we can enter "Initial" mode again. state.skip(2); - state.enter_state(StackState::Initial); + state.set(StackFrame::Initial)?; TokenKind::CloseTag } @@ -304,7 +305,7 @@ impl Lexer { match state.peek_buf() { [b'(', b')', b';', ..] => { state.skip(3); - state.enter_state(StackState::Halted); + state.set(StackFrame::Halted)?; } _ => return Err(SyntaxError::InvalidHaltCompiler(state.span)), } @@ -411,12 +412,12 @@ impl Lexer { } [b'{', ..] => { state.next(); - state.push_state(StackState::Scripting); + state.enter(StackFrame::Scripting); TokenKind::LeftBrace } [b'}', ..] => { state.next(); - state.pop_state(); + state.exit(); TokenKind::RightBrace } [b'(', ..] => { @@ -591,25 +592,25 @@ impl Lexer { Ok(Token { kind, span }) } - fn double_quote(&self, state: &mut State) -> Result, SyntaxError> { + fn double_quote(&self, state: &mut State) -> SyntaxResult> { let span = state.span; let mut buffer = Vec::new(); let kind = loop { match state.peek_buf() { [b'$', b'{', ..] => { state.skip(2); - state.push_state(StackState::LookingForVarname); + state.enter(StackFrame::LookingForVarname); break TokenKind::DollarLeftBrace; } [b'{', b'$', ..] => { // Intentionally only consume the left brace. state.next(); - state.push_state(StackState::Scripting); + state.enter(StackFrame::Scripting); break TokenKind::LeftBrace; } [b'"', ..] => { state.next(); - state.enter_state(StackState::Scripting); + state.set(StackFrame::Scripting)?; break TokenKind::DoubleQuote; } [b'$', ident_start!(), ..] => { @@ -617,10 +618,10 @@ impl Lexer { let ident = self.consume_identifier(state); match state.peek_buf() { - [b'[', ..] => state.push_state(StackState::VarOffset), + [b'[', ..] => state.enter(StackFrame::VarOffset), [b'-', b'>', ident_start!(), ..] | [b'?', b'-', b'>', ident_start!(), ..] => { - state.push_state(StackState::LookingForProperty) + state.enter(StackFrame::LookingForProperty) } _ => {} } @@ -647,7 +648,7 @@ impl Lexer { Ok(tokens) } - fn looking_for_varname(&self, state: &mut State) -> Option { + fn looking_for_varname(&self, state: &mut State) -> SyntaxResult> { let identifier = self.peek_identifier(state); if let Some(ident) = identifier { @@ -655,19 +656,20 @@ impl Lexer { let ident = ident.to_vec(); let span = state.span; state.skip(ident.len()); - state.enter_state(StackState::Scripting); - return Some(Token { + state.set(StackFrame::Scripting)?; + return Ok(Some(Token { kind: TokenKind::Identifier(ident.into()), span, - }); + })); } } - state.enter_state(StackState::Scripting); - None + state.set(StackFrame::Scripting)?; + + Ok(None) } - fn looking_for_property(&self, state: &mut State) -> Result { + fn looking_for_property(&self, state: &mut State) -> SyntaxResult { let span = state.span; let kind = match state.peek_buf() { [b'-', b'>', ..] => { @@ -680,7 +682,7 @@ impl Lexer { } &[ident_start!(), ..] => { let buffer = self.consume_identifier(state); - state.pop_state(); + state.exit(); TokenKind::Identifier(buffer.into()) } // Should be impossible as we already looked ahead this far inside double_quote. @@ -689,7 +691,7 @@ impl Lexer { Ok(Token { kind, span }) } - fn var_offset(&self, state: &mut State) -> Result { + fn var_offset(&self, state: &mut State) -> SyntaxResult { let span = state.span; let kind = match state.peek_buf() { [b'$', ident_start!(), ..] => { @@ -712,7 +714,7 @@ impl Lexer { } [b']', ..] => { state.next(); - state.pop_state(); + state.exit(); TokenKind::RightBracket } &[ident_start!(), ..] => { @@ -730,7 +732,7 @@ impl Lexer { Ok(Token { kind, span }) } - fn tokenize_single_quote_string(&self, state: &mut State) -> Result { + fn tokenize_single_quote_string(&self, state: &mut State) -> SyntaxResult { let mut buffer = Vec::new(); loop { @@ -754,7 +756,7 @@ impl Lexer { Ok(TokenKind::LiteralString(buffer.into())) } - fn tokenize_double_quote_string(&self, state: &mut State) -> Result { + fn tokenize_double_quote_string(&self, state: &mut State) -> SyntaxResult { let mut buffer = Vec::new(); let constant = loop { @@ -864,7 +866,7 @@ impl Lexer { Ok(if constant { TokenKind::LiteralString(buffer.into()) } else { - state.enter_state(StackState::DoubleQuote); + state.set(StackFrame::DoubleQuote)?; TokenKind::StringPart(buffer.into()) }) } @@ -893,7 +895,7 @@ impl Lexer { TokenKind::Variable(self.consume_identifier(state).into()) } - fn tokenize_number(&self, state: &mut State) -> Result { + fn tokenize_number(&self, state: &mut State) -> SyntaxResult { let mut buffer = String::new(); let (base, kind) = match state.peek_buf() { @@ -1000,7 +1002,7 @@ impl Lexer { // Parses an integer literal in the given base and converts errors to SyntaxError. // It returns a float token instead on overflow. -fn parse_int(buffer: &str, base: u32, span: Span) -> Result { +fn parse_int(buffer: &str, base: u32, span: Span) -> SyntaxResult { match i64::from_str_radix(buffer, base) { Ok(i) => Ok(TokenKind::LiteralInteger(i)), Err(err) if err.kind() == &IntErrorKind::InvalidDigit => { diff --git a/src/lexer/state.rs b/src/lexer/state.rs index 59a0ba80..2815b7af 100644 --- a/src/lexer/state.rs +++ b/src/lexer/state.rs @@ -1,7 +1,11 @@ +use std::collections::VecDeque; + +use crate::lexer::error::SyntaxError; +use crate::lexer::error::SyntaxResult; use crate::lexer::token::Span; #[derive(Debug, PartialEq, Eq, Clone)] -pub enum StackState { +pub enum StackFrame { Initial, Scripting, Halted, @@ -11,10 +15,9 @@ pub enum StackState { VarOffset, } -// TODO(azjezz): make `chars` a `[u8, N]`, and `State`, `State` #[derive(Debug, PartialEq, Eq, Clone)] pub struct State { - pub stack: Vec, + pub stack: VecDeque, pub chars: Vec, pub cursor: usize, pub current: Option, @@ -27,7 +30,7 @@ impl State { let current = chars.first().copied(); Self { - stack: vec![StackState::Initial], + stack: VecDeque::from([StackFrame::Initial]), chars, current, cursor: 0, @@ -35,16 +38,27 @@ impl State { } } - pub fn enter_state(&mut self, state: StackState) { - *self.stack.last_mut().unwrap() = state; + pub fn set(&mut self, state: StackFrame) -> SyntaxResult<()> { + *self + .stack + .back_mut() + .ok_or(SyntaxError::UnpredictableState(self.span))? = state; + + Ok(()) + } + + pub fn frame(&self) -> SyntaxResult<&StackFrame> { + self.stack + .back() + .ok_or(SyntaxError::UnpredictableState(self.span)) } - pub fn push_state(&mut self, state: StackState) { - self.stack.push(state); + pub fn enter(&mut self, state: StackFrame) { + self.stack.push_back(state); } - pub fn pop_state(&mut self) { - self.stack.pop(); + pub fn exit(&mut self) { + self.stack.pop_back(); } pub fn peek_buf(&self) -> &[u8] { diff --git a/src/parser/ast.rs b/src/parser/ast.rs index f61ad1c5..3c53862d 100644 --- a/src/parser/ast.rs +++ b/src/parser/ast.rs @@ -397,6 +397,10 @@ pub enum Statement { expr: Expression, }, Namespace { + name: ByteString, + body: Block, + }, + BracedNamespace { name: Option, body: Block, }, diff --git a/src/parser/classish.rs b/src/parser/classish.rs deleted file mode 100644 index 7dd5f033..00000000 --- a/src/parser/classish.rs +++ /dev/null @@ -1,263 +0,0 @@ -use crate::lexer::token::TokenKind; -use crate::parser::ast::BackedEnumType; -use crate::parser::ast::Block; -use crate::parser::ast::ClassFlag; -use crate::parser::ast::Expression; -use crate::parser::ast::Identifier; -use crate::parser::ast::Statement; -use crate::parser::error::ParseResult; -use crate::parser::state::State; -use crate::parser::Parser; - -use crate::expect_token; -use crate::expected_token_err; - -impl Parser { - pub(in crate::parser) fn class_definition(&self, state: &mut State) -> ParseResult { - let flags: Vec = self.class_flags(state)?.iter().map(|f| f.into()).collect(); - - expect_token!([TokenKind::Class], state, ["`class`"]); - - let name = self.ident(state)?; - let mut extends: Option = None; - - if state.current.kind == TokenKind::Extends { - state.next(); - extends = Some(self.full_name(state)?.into()); - } - - let implements = if state.current.kind == TokenKind::Implements { - state.next(); - - self.at_least_one_comma_separated::(state, &|parser, state| { - Ok(parser.full_name(state)?.into()) - })? - } else { - Vec::new() - }; - - self.lbrace(state)?; - - let mut body = Vec::new(); - while state.current.kind != TokenKind::RightBrace { - state.gather_comments(); - - if state.current.kind == TokenKind::RightBrace { - state.clear_comments(); - break; - } - - body.push(self.class_statement(state, flags.clone())?); - } - self.rbrace(state)?; - - Ok(Statement::Class { - name: name.into(), - extends, - implements, - body, - flags, - }) - } - - pub(in crate::parser) fn interface_definition( - &self, - state: &mut State, - ) -> ParseResult { - expect_token!([TokenKind::Interface], state, ["`interface`"]); - - let name = self.ident(state)?; - - let extends = if state.current.kind == TokenKind::Extends { - state.next(); - - self.at_least_one_comma_separated::(state, &|parser, state| { - Ok(parser.full_name(state)?.into()) - })? - } else { - Vec::new() - }; - - self.lbrace(state)?; - - let mut body = Vec::new(); - while state.current.kind != TokenKind::RightBrace && !state.is_eof() { - state.gather_comments(); - - if state.current.kind == TokenKind::RightBrace { - state.clear_comments(); - break; - } - - body.push(self.interface_statement(state)?); - } - self.rbrace(state)?; - - Ok(Statement::Interface { - name: name.into(), - extends, - body, - }) - } - - pub(in crate::parser) fn trait_definition(&self, state: &mut State) -> ParseResult { - expect_token!([TokenKind::Trait], state, ["`trait`"]); - - let name = self.ident(state)?; - - self.lbrace(state)?; - - let mut body = Vec::new(); - while state.current.kind != TokenKind::RightBrace && !state.is_eof() { - state.gather_comments(); - - if state.current.kind == TokenKind::RightBrace { - state.clear_comments(); - break; - } - - body.push(self.trait_statement(state)?); - } - self.rbrace(state)?; - - Ok(Statement::Trait { - name: name.into(), - body, - }) - } - - pub(in crate::parser) fn anonymous_class_definition( - &self, - state: &mut State, - ) -> ParseResult { - state.next(); - - expect_token!([TokenKind::Class], state, ["`class`"]); - - let mut args = vec![]; - - if state.current.kind == TokenKind::LeftParen { - self.lparen(state)?; - - args = self.args_list(state)?; - - self.rparen(state)?; - } - - let mut extends: Option = None; - - if state.current.kind == TokenKind::Extends { - state.next(); - extends = Some(self.full_name(state)?.into()); - } - - let mut implements = Vec::new(); - if state.current.kind == TokenKind::Implements { - state.next(); - - while state.current.kind != TokenKind::LeftBrace { - self.optional_comma(state)?; - - implements.push(self.full_name(state)?.into()); - } - } - - self.lbrace(state)?; - - let mut body = Vec::new(); - while state.current.kind != TokenKind::RightBrace && !state.is_eof() { - body.push(self.anonymous_class_statement(state)?); - } - - self.rbrace(state)?; - - Ok(Expression::New { - target: Box::new(Expression::AnonymousClass { - extends, - implements, - body, - }), - args, - }) - } - - pub(in crate::parser) fn enum_definition(&self, state: &mut State) -> ParseResult { - state.next(); - - let name = self.ident(state)?; - - let backed_type: Option = if state.current.kind == TokenKind::Colon { - self.colon(state)?; - - match state.current.kind.clone() { - TokenKind::Identifier(s) if s == b"string" || s == b"int" => { - state.next(); - - Some(match &s[..] { - b"string" => BackedEnumType::String, - b"int" => BackedEnumType::Int, - _ => unreachable!(), - }) - } - _ => { - return expected_token_err!(["`string`", "`int`"], state); - } - } - } else { - None - }; - - let mut implements = Vec::new(); - if state.current.kind == TokenKind::Implements { - state.next(); - - while state.current.kind != TokenKind::LeftBrace { - implements.push(self.full_name(state)?.into()); - - self.optional_comma(state)?; - } - } - - self.lbrace(state)?; - - let mut body = Block::new(); - while state.current.kind != TokenKind::RightBrace { - state.skip_comments(); - body.push(self.enum_statement(state, backed_type.is_some())?); - } - - self.rbrace(state)?; - - match backed_type { - Some(backed_type) => Ok(Statement::BackedEnum { - name: name.into(), - backed_type, - implements, - body, - }), - None => Ok(Statement::UnitEnum { - name: name.into(), - implements, - body, - }), - } - } - - fn at_least_one_comma_separated( - &self, - state: &mut State, - func: &(dyn Fn(&Parser, &mut State) -> ParseResult), - ) -> ParseResult> { - let mut result: Vec = vec![]; - loop { - result.push(func(self, state)?); - if state.current.kind != TokenKind::Comma { - break; - } - - state.next(); - } - - Ok(result) - } -} diff --git a/src/parser/error.rs b/src/parser/error.rs index 8dd99607..adb963f8 100644 --- a/src/parser/error.rs +++ b/src/parser/error.rs @@ -18,11 +18,13 @@ pub enum ParseError { PromotedPropertyOutsideConstructor(Span), PromotedPropertyOnAbstractConstructor(Span), AbstractModifierOnNonAbstractClassMethod(Span), + ConstructorInEnum(String, Span), StaticModifierOnConstant(Span), ReadonlyModifierOnConstant(Span), FinalModifierOnAbstractClassMember(Span), FinalModifierOnPrivateConstant(Span), FinalModifierOnAbstractClass(Span), + UnpredictableState(Span), } impl Display for ParseError { @@ -39,25 +41,27 @@ impl Display for ParseError { }; match found { - Some(token) => write!(f, "Parse error: unexpected token `{}`, expecting {} on line {} column {}", token, expected, span.0, span.1), - None => write!(f, "Parse error: unexpected end of file, expecting {} on line {} column {}", expected, span.0, span.1), + Some(token) => write!(f, "Parse Error: unexpected token `{}`, expecting {} on line {} column {}", token, expected, span.0, span.1), + None => write!(f, "Parse Error: unexpected end of file, expecting {} on line {} column {}", expected, span.0, span.1), } }, Self::MultipleModifiers(modifier, span) => write!(f, "Parse Error: Multiple {} modifiers are not allowed on line {} column {}", modifier, span.0, span.1), Self::MultipleAccessModifiers( span) => write!(f, "Parse Error: Multiple access type modifiers are not allowed on line {} column {}", span.0, span.1), - Self::UnexpectedToken(message, span) => write!(f, "Parse error: unexpected token {} on line {} column {}", message, span.0, span.1), - Self::UnexpectedEndOfFile => write!(f, "Parse error: unexpected end of file."), - Self::FinalModifierOnAbstractClassMember(span) => write!(f, "Parse error: Cannot use the final modifier on an abstract class member on line {} column {}", span.0, span.1), - Self::StaticModifierOnConstant(span) => write!(f, "Parse error: Cannot use 'static' as constant modifier on line {} column {}", span.0, span.1), - Self::ReadonlyModifierOnConstant(span) => write!(f, "Parse error: Cannot use 'readonly' as constant modifier on line {} column {}", span.0, span.1), - Self::FinalModifierOnPrivateConstant(span) => write!(f, "Parse error: Private constant cannot be final as it is not visible to other classes on line {} column {}", span.0, span.1), - Self::TryWithoutCatchOrFinally(span) => write!(f, "Parse error: cannot use try without catch or finally on line {} column {}", span.0, span.1), - Self::StandaloneTypeUsedInCombination(r#type, span) => write!(f, "Parse error: {} can only be used as a standalone type on line {} column {}", r#type, span.0, span.1), - Self::VariadicPromotedProperty(span) => write!(f, "Parse error: Cannot declare variadic promoted property on line {} column {}", span.0, span.1), - Self::PromotedPropertyOutsideConstructor(span) => write!(f, "Parse error: Cannot declare promoted property outside a constructor on line {} column {}", span.0, span.1), - Self::PromotedPropertyOnAbstractConstructor(span) => write!(f, "Parse error: Cannot declare promoted property in an abstract constructor on line {} column {}", span.0, span.1), - Self::AbstractModifierOnNonAbstractClassMethod(span) => write!(f, "Parse error: Cannot declare abstract methods on a non-abstract class on line {} column {}", span.0, span.1), - Self::FinalModifierOnAbstractClass(span) => write!(f, "Parse error: Cannot use the final modifier on an abstract class on line {} column {}", span.0, span.1), + Self::UnexpectedToken(message, span) => write!(f, "Parse Error: Unexpected token {} on line {} column {}", message, span.0, span.1), + Self::UnexpectedEndOfFile => write!(f, "Parse Error: unexpected end of file."), + Self::FinalModifierOnAbstractClassMember(span) => write!(f, "Parse Error: Cannot use the final modifier on an abstract class member on line {} column {}", span.0, span.1), + Self::StaticModifierOnConstant(span) => write!(f, "Parse Error: Cannot use 'static' as constant modifier on line {} column {}", span.0, span.1), + Self::ReadonlyModifierOnConstant(span) => write!(f, "Parse Error: Cannot use 'readonly' as constant modifier on line {} column {}", span.0, span.1), + Self::FinalModifierOnPrivateConstant(span) => write!(f, "Parse Error: Private constant cannot be final as it is not visible to other classes on line {} column {}", span.0, span.1), + Self::TryWithoutCatchOrFinally(span) => write!(f, "Parse Error: Cannot use try without catch or finally on line {} column {}", span.0, span.1), + Self::StandaloneTypeUsedInCombination(r#type, span) => write!(f, "Parse error: '{}' can only be used as a standalone type on line {} column {}", r#type, span.0, span.1), + Self::VariadicPromotedProperty(span) => write!(f, "Parse Error: Cannot declare variadic promoted property on line {} column {}", span.0, span.1), + Self::PromotedPropertyOutsideConstructor(span) => write!(f, "Parse Error: Cannot declare promoted property outside a constructor on line {} column {}", span.0, span.1), + Self::PromotedPropertyOnAbstractConstructor(span) => write!(f, "Parse Error: Cannot declare promoted property in an abstract constructor on line {} column {}", span.0, span.1), + Self::AbstractModifierOnNonAbstractClassMethod(span) => write!(f, "Parse Error: Cannot declare abstract methods on a non-abstract class on line {} column {}", span.0, span.1), + Self::FinalModifierOnAbstractClass(span) => write!(f, "Parse Error: Cannot use the final modifier on an abstract class on line {} column {}", span.0, span.1), + Self::ConstructorInEnum(name, span) => write!(f, "Parse Error: Enum '{}' cannot have a constructor on line {} column {}", name, span.0, span.1), + Self::UnpredictableState(span) => write!(f, "Parse Error: Reached an unpredictable state on line {} column {}", span.0, span.1) } } } diff --git a/src/parser/functions.rs b/src/parser/functions.rs deleted file mode 100644 index a2093fda..00000000 --- a/src/parser/functions.rs +++ /dev/null @@ -1,155 +0,0 @@ -use crate::lexer::byte_string::ByteString; -use crate::lexer::token::TokenKind; -use crate::parser::ast::ClassFlag; -use crate::parser::ast::MethodFlag; -use crate::parser::ast::Statement; -use crate::parser::classish_statement::ClassishDefinitionType; -use crate::parser::error::ParseError; -use crate::parser::error::ParseResult; -use crate::parser::params::ParamPosition; -use crate::parser::state::State; -use crate::parser::Parser; - -impl Parser { - pub(in crate::parser) fn function(&self, state: &mut State) -> ParseResult { - state.next(); - - let by_ref = if state.current.kind == TokenKind::Ampersand { - state.next(); - true - } else { - false - }; - - let name = self.ident(state)?; - - self.lparen(state)?; - - let params = self.param_list(state, ParamPosition::Function)?; - - self.rparen(state)?; - - let mut return_type = None; - - if state.current.kind == TokenKind::Colon { - self.colon(state)?; - - return_type = Some(self.type_string(state)?); - } - - self.lbrace(state)?; - - let body = self.block(state, &TokenKind::RightBrace)?; - - self.rbrace(state)?; - - Ok(Statement::Function { - name: name.into(), - params, - body, - return_type, - by_ref, - }) - } - - pub(in crate::parser) fn method( - &self, - state: &mut State, - class_type: ClassishDefinitionType, - flags: Vec, - ) -> ParseResult { - // TODO: more verification goes here, we know what type of class and what method flags there are. - match class_type { - ClassishDefinitionType::Class(cf) - if !cf.contains(&ClassFlag::Abstract) && flags.contains(&MethodFlag::Abstract) => - { - return Err(ParseError::AbstractModifierOnNonAbstractClassMethod( - state.current.span, - )); - } - _ => (), - } - - state.next(); - - let has_body = match &class_type { - ClassishDefinitionType::Class(_) | ClassishDefinitionType::Trait => { - !flags.contains(&MethodFlag::Abstract) - } - ClassishDefinitionType::Interface => false, - ClassishDefinitionType::Enum | ClassishDefinitionType::AnonymousClass => true, - }; - - let by_ref = if state.current.kind == TokenKind::Ampersand { - state.next(); - true - } else { - false - }; - - let name = self.ident_maybe_reserved(state)?; - - self.lparen(state)?; - - let position = position_from_flags_and_name(class_type, flags.clone(), name.clone()); - - let params = self.param_list(state, position)?; - - self.rparen(state)?; - - let mut return_type = None; - - if state.current.kind == TokenKind::Colon { - self.colon(state)?; - - return_type = Some(self.type_string(state)?); - } - - if !has_body { - self.semi(state)?; - - Ok(Statement::AbstractMethod { - name: name.into(), - params, - return_type, - flags: flags.to_vec(), - by_ref, - }) - } else { - self.lbrace(state)?; - - let body = self.block(state, &TokenKind::RightBrace)?; - - self.rbrace(state)?; - - Ok(Statement::Method { - name: name.into(), - params, - body, - return_type, - by_ref, - flags, - }) - } - } -} - -fn position_from_flags_and_name( - class_type: ClassishDefinitionType, - flags: Vec, - name: ByteString, -) -> ParamPosition { - match class_type { - ClassishDefinitionType::Enum - | ClassishDefinitionType::Class(_) - | ClassishDefinitionType::Trait - | ClassishDefinitionType::AnonymousClass => { - if !flags.contains(&MethodFlag::Abstract) { - ParamPosition::Method(name.to_string()) - } else { - ParamPosition::AbstractMethod(name.to_string()) - } - } - ClassishDefinitionType::Interface => ParamPosition::AbstractMethod(name.to_string()), - } -} diff --git a/src/parser/block.rs b/src/parser/internal/block.rs similarity index 100% rename from src/parser/block.rs rename to src/parser/internal/block.rs diff --git a/src/parser/internal/classish.rs b/src/parser/internal/classish.rs new file mode 100644 index 00000000..0dd32a28 --- /dev/null +++ b/src/parser/internal/classish.rs @@ -0,0 +1,274 @@ +use crate::lexer::token::TokenKind; +use crate::parser::ast::BackedEnumType; +use crate::parser::ast::Block; +use crate::parser::ast::ClassFlag; +use crate::parser::ast::Expression; +use crate::parser::ast::Identifier; +use crate::parser::ast::Statement; +use crate::parser::error::ParseResult; +use crate::parser::state::Scope; +use crate::parser::state::State; +use crate::parser::Parser; + +use crate::expect_token; +use crate::expected_token_err; +use crate::scoped; + +impl Parser { + pub(in crate::parser) fn class_definition(&self, state: &mut State) -> ParseResult { + let flags: Vec = self.class_flags(state)?.iter().map(|f| f.into()).collect(); + + expect_token!([TokenKind::Class], state, ["`class`"]); + + let name = self.ident(state)?; + + scoped!(state, Scope::Class(name.clone(), flags.clone()), { + let mut extends: Option = None; + + if state.current.kind == TokenKind::Extends { + state.next(); + extends = Some(self.full_name(state)?.into()); + } + + let implements = if state.current.kind == TokenKind::Implements { + state.next(); + + self.at_least_one_comma_separated::(state, &|parser, state| { + Ok(parser.full_name(state)?.into()) + })? + } else { + Vec::new() + }; + + self.lbrace(state)?; + + let mut body = Vec::new(); + while state.current.kind != TokenKind::RightBrace { + state.gather_comments(); + + if state.current.kind == TokenKind::RightBrace { + state.clear_comments(); + break; + } + + body.push(self.class_like_statement(state)?); + } + self.rbrace(state)?; + + Ok(Statement::Class { + name: name.into(), + extends, + implements, + body, + flags, + }) + }) + } + + pub(in crate::parser) fn interface_definition( + &self, + state: &mut State, + ) -> ParseResult { + expect_token!([TokenKind::Interface], state, ["`interface`"]); + let name = self.ident(state)?; + + scoped!(state, Scope::Interface(name.clone()), { + let extends = if state.current.kind == TokenKind::Extends { + state.next(); + + self.at_least_one_comma_separated::(state, &|parser, state| { + Ok(parser.full_name(state)?.into()) + })? + } else { + Vec::new() + }; + + self.lbrace(state)?; + + let mut body = Vec::new(); + while state.current.kind != TokenKind::RightBrace && !state.is_eof() { + state.gather_comments(); + + if state.current.kind == TokenKind::RightBrace { + state.clear_comments(); + break; + } + + body.push(self.interface_statement(state)?); + } + self.rbrace(state)?; + + Ok(Statement::Interface { + name: name.into(), + extends, + body, + }) + }) + } + + pub(in crate::parser) fn trait_definition(&self, state: &mut State) -> ParseResult { + expect_token!([TokenKind::Trait], state, ["`trait`"]); + + let name = self.ident(state)?; + + scoped!(state, Scope::Trait(name.clone()), { + self.lbrace(state)?; + + let mut body = Vec::new(); + while state.current.kind != TokenKind::RightBrace && !state.is_eof() { + state.gather_comments(); + + if state.current.kind == TokenKind::RightBrace { + state.clear_comments(); + break; + } + + body.push(self.class_like_statement(state)?); + } + self.rbrace(state)?; + + Ok(Statement::Trait { + name: name.into(), + body, + }) + }) + } + + pub(in crate::parser) fn anonymous_class_definition( + &self, + state: &mut State, + ) -> ParseResult { + expect_token!([TokenKind::New], state, ["`new`"]); + expect_token!([TokenKind::Class], state, ["`class`"]); + + scoped!(state, Scope::AnonymousClass, { + let mut args = vec![]; + + if state.current.kind == TokenKind::LeftParen { + self.lparen(state)?; + + args = self.args_list(state)?; + + self.rparen(state)?; + } + + let mut extends: Option = None; + + if state.current.kind == TokenKind::Extends { + state.next(); + extends = Some(self.full_name(state)?.into()); + } + + let mut implements = Vec::new(); + if state.current.kind == TokenKind::Implements { + state.next(); + + while state.current.kind != TokenKind::LeftBrace { + self.optional_comma(state)?; + + implements.push(self.full_name(state)?.into()); + } + } + + self.lbrace(state)?; + + let mut body = Vec::new(); + while state.current.kind != TokenKind::RightBrace && !state.is_eof() { + body.push(self.class_like_statement(state)?); + } + + self.rbrace(state)?; + + Ok(Expression::New { + target: Box::new(Expression::AnonymousClass { + extends, + implements, + body, + }), + args, + }) + }) + } + + pub(in crate::parser) fn enum_definition(&self, state: &mut State) -> ParseResult { + expect_token!([TokenKind::Enum], state, ["`enum`"]); + + let name = self.ident(state)?; + + scoped!(state, Scope::Enum(name.clone()), { + let backed_type: Option = if state.current.kind == TokenKind::Colon { + self.colon(state)?; + + match state.current.kind.clone() { + TokenKind::Identifier(s) if s == b"string" || s == b"int" => { + state.next(); + + Some(match &s[..] { + b"string" => BackedEnumType::String, + b"int" => BackedEnumType::Int, + _ => unreachable!(), + }) + } + _ => { + return expected_token_err!(["`string`", "`int`"], state); + } + } + } else { + None + }; + + let mut implements = Vec::new(); + if state.current.kind == TokenKind::Implements { + state.next(); + + while state.current.kind != TokenKind::LeftBrace { + implements.push(self.full_name(state)?.into()); + + self.optional_comma(state)?; + } + } + + self.lbrace(state)?; + + let mut body = Block::new(); + while state.current.kind != TokenKind::RightBrace { + state.skip_comments(); + body.push(self.enum_statement(state, backed_type.is_some())?); + } + + self.rbrace(state)?; + + match backed_type { + Some(backed_type) => Ok(Statement::BackedEnum { + name: name.into(), + backed_type, + implements, + body, + }), + None => Ok(Statement::UnitEnum { + name: name.into(), + implements, + body, + }), + } + }) + } + + fn at_least_one_comma_separated( + &self, + state: &mut State, + func: &(dyn Fn(&Parser, &mut State) -> ParseResult), + ) -> ParseResult> { + let mut result: Vec = vec![]; + loop { + result.push(func(self, state)?); + if state.current.kind != TokenKind::Comma { + break; + } + + state.next(); + } + + Ok(result) + } +} diff --git a/src/parser/classish_statement.rs b/src/parser/internal/classish_statement.rs similarity index 89% rename from src/parser/classish_statement.rs rename to src/parser/internal/classish_statement.rs index f8f3966c..7d846ea3 100644 --- a/src/parser/classish_statement.rs +++ b/src/parser/internal/classish_statement.rs @@ -1,12 +1,11 @@ use crate::lexer::token::TokenKind; -use crate::parser::ast::ClassFlag; use crate::parser::ast::Identifier; use crate::parser::ast::MethodFlag; use crate::parser::ast::Statement; use crate::parser::ast::TraitAdaptation; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; -use crate::parser::precedence::Precedence; +use crate::parser::internal::precedence::Precedence; use crate::parser::state::State; use crate::parser::Parser; @@ -14,24 +13,7 @@ use crate::expect_token; use crate::expected_token_err; use crate::peek_token; -#[derive(Debug)] -pub enum ClassishDefinitionType { - Class(Vec), - AnonymousClass, - Trait, - Interface, - Enum, -} - impl Parser { - pub(in crate::parser) fn class_statement( - &self, - state: &mut State, - flags: Vec, - ) -> ParseResult { - self.complete_class_statement(state, ClassishDefinitionType::Class(flags)) - } - pub(in crate::parser) fn interface_statement( &self, state: &mut State, @@ -41,7 +23,7 @@ impl Parser { } if state.current.kind == TokenKind::Function { - return self.method(state, ClassishDefinitionType::Interface, vec![]); + return self.method(state, vec![]); } let member_flags = self.interface_members_flags(state)?; @@ -50,23 +32,11 @@ impl Parser { TokenKind::Const => self.parse_classish_const(state, member_flags), TokenKind::Function => self.method( state, - ClassishDefinitionType::Interface, member_flags.iter().map(|t| t.clone().into()).collect(), ) ], state, ["`const`", "`function`"]) } - pub(in crate::parser) fn trait_statement(&self, state: &mut State) -> ParseResult { - self.complete_class_statement(state, ClassishDefinitionType::Trait) - } - - pub(in crate::parser) fn anonymous_class_statement( - &self, - state: &mut State, - ) -> ParseResult { - self.complete_class_statement(state, ClassishDefinitionType::AnonymousClass) - } - pub(in crate::parser) fn enum_statement( &self, state: &mut State, @@ -99,7 +69,7 @@ impl Parser { } if state.current.kind == TokenKind::Function { - return self.method(state, ClassishDefinitionType::Enum, vec![]); + return self.method(state, vec![]); } let member_flags = self.enum_members_flags(state)?; @@ -108,16 +78,14 @@ impl Parser { TokenKind::Const => self.parse_classish_const(state, member_flags), TokenKind::Function => self.method( state, - ClassishDefinitionType::Enum, member_flags.iter().map(|t| t.clone().into()).collect(), ) ], state, ["`const`", "`function`"]) } - fn complete_class_statement( + pub(in crate::parser) fn class_like_statement( &self, state: &mut State, - class_type: ClassishDefinitionType, ) -> ParseResult { if state.current.kind == TokenKind::Use { return self.parse_classish_uses(state); @@ -132,7 +100,7 @@ impl Parser { } if state.current.kind == TokenKind::Function { - return self.method(state, class_type, vec![]); + return self.method(state, vec![]); } let member_flags = self.class_members_flags(state)?; @@ -141,7 +109,6 @@ impl Parser { TokenKind::Const => self.parse_classish_const(state, member_flags), TokenKind::Function => self.method( state, - class_type, member_flags.iter().map(|t| t.clone().into()).collect(), ), // TODO diff --git a/src/parser/flags.rs b/src/parser/internal/flags.rs similarity index 100% rename from src/parser/flags.rs rename to src/parser/internal/flags.rs diff --git a/src/parser/internal/functions.rs b/src/parser/internal/functions.rs new file mode 100644 index 00000000..5b07840d --- /dev/null +++ b/src/parser/internal/functions.rs @@ -0,0 +1,274 @@ +use crate::expect_token; +use crate::lexer::token::TokenKind; +use crate::parser::ast::ClassFlag; +use crate::parser::ast::ClosureUse; +use crate::parser::ast::Expression; +use crate::parser::ast::MethodFlag; +use crate::parser::ast::Statement; +use crate::parser::error::ParseError; +use crate::parser::error::ParseResult; +use crate::parser::internal::precedence::Precedence; +use crate::parser::state::Scope; +use crate::parser::state::State; +use crate::parser::Parser; +use crate::scoped; + +impl Parser { + pub(in crate::parser) fn anonymous_function( + &self, + state: &mut State, + ) -> ParseResult { + let is_static = if state.current.kind == TokenKind::Static { + state.next(); + + true + } else { + false + }; + + expect_token!([TokenKind::Function], state, ["`function`"]); + + let by_ref = if state.current.kind == TokenKind::Ampersand { + state.next(); + true + } else { + false + }; + + scoped!(state, Scope::AnonymousFunction(is_static), { + self.lparen(state)?; + + let params = self.param_list(state)?; + + self.rparen(state)?; + + let mut uses = vec![]; + if state.current.kind == TokenKind::Use { + state.next(); + + self.lparen(state)?; + + while state.current.kind != TokenKind::RightParen { + let mut by_ref = false; + if state.current.kind == TokenKind::Ampersand { + by_ref = true; + } + + // TODO(azjezz): this shouldn't call expr, we should have a function + // just for variables, so we don't have to go through the whole `match` in `expression(...)` + let var = match self.expression(state, Precedence::Lowest)? { + s @ Expression::Variable { .. } => ClosureUse { var: s, by_ref }, + _ => { + return Err(ParseError::UnexpectedToken( + "expected variable".into(), + state.current.span, + )) + } + }; + + uses.push(var); + + self.optional_comma(state)?; + } + + self.rparen(state)?; + } + + let mut return_type = None; + if state.current.kind == TokenKind::Colon { + self.colon(state)?; + + return_type = Some(self.type_string(state)?); + } + + self.lbrace(state)?; + + let body = self.block(state, &TokenKind::RightBrace)?; + + self.rbrace(state)?; + + Ok(Expression::Closure { + params, + uses, + return_type, + body, + r#static: is_static, + by_ref, + }) + }) + } + + pub(in crate::parser) fn arrow_function(&self, state: &mut State) -> ParseResult { + let is_static = if state.current.kind == TokenKind::Static { + state.next(); + + true + } else { + false + }; + + expect_token!([TokenKind::Fn], state, ["`fn`"]); + + let by_ref = if state.current.kind == TokenKind::Ampersand { + state.next(); + true + } else { + false + }; + + scoped!(state, Scope::ArrowFunction(is_static), { + self.lparen(state)?; + + let params = self.param_list(state)?; + + self.rparen(state)?; + + let mut return_type = None; + if state.current.kind == TokenKind::Colon { + self.colon(state)?; + + return_type = Some(self.type_string(state)?); + } + + expect_token!([TokenKind::DoubleArrow], state, ["`=>`"]); + + let value = self.expression(state, Precedence::Lowest)?; + + Ok(Expression::ArrowFunction { + params, + return_type, + expr: Box::new(value), + by_ref, + r#static: is_static, + }) + }) + } + + pub(in crate::parser) fn function(&self, state: &mut State) -> ParseResult { + expect_token!([TokenKind::Function], state, ["`function`"]); + + let by_ref = if state.current.kind == TokenKind::Ampersand { + state.next(); + true + } else { + false + }; + + let name = self.ident(state)?; + + scoped!(state, Scope::Function(name.clone()), { + self.lparen(state)?; + + let params = self.param_list(state)?; + + self.rparen(state)?; + + let mut return_type = None; + + if state.current.kind == TokenKind::Colon { + self.colon(state)?; + + return_type = Some(self.type_string(state)?); + } + + self.lbrace(state)?; + + let body = self.block(state, &TokenKind::RightBrace)?; + + self.rbrace(state)?; + + Ok(Statement::Function { + name: name.into(), + params, + body, + return_type, + by_ref, + }) + }) + } + + pub(in crate::parser) fn method( + &self, + state: &mut State, + flags: Vec, + ) -> ParseResult { + expect_token!([TokenKind::Function], state, ["`function`"]); + + let by_ref = if state.current.kind == TokenKind::Ampersand { + state.next(); + true + } else { + false + }; + + let name = self.ident_maybe_reserved(state)?; + + scoped!(state, Scope::Method(name.clone(), flags.clone()), { + let has_body = match state.parent()? { + Scope::Class(_, cf) => { + if !cf.contains(&ClassFlag::Abstract) && flags.contains(&MethodFlag::Abstract) { + return Err(ParseError::AbstractModifierOnNonAbstractClassMethod( + state.current.span, + )); + } + + !flags.contains(&MethodFlag::Abstract) + } + Scope::Trait(_) => !flags.contains(&MethodFlag::Abstract), + Scope::Interface(_) => false, + Scope::Enum(enum_name) => { + if name.to_string() == "__construct" { + return Err(ParseError::ConstructorInEnum( + state.named(enum_name), + state.current.span, + )); + } + + true + } + _ => true, + }; + + self.lparen(state)?; + + let params = self.param_list(state)?; + + self.rparen(state)?; + + let mut return_type = None; + + if state.current.kind == TokenKind::Colon { + self.colon(state)?; + + return_type = Some(self.type_string(state)?); + } + + if !has_body { + self.semi(state)?; + + Ok(Statement::AbstractMethod { + name: name.into(), + params, + return_type, + flags: flags.to_vec(), + by_ref, + }) + } else { + self.lbrace(state)?; + + let body = self.block(state, &TokenKind::RightBrace)?; + + self.rbrace(state)?; + + Ok(Statement::Method { + name: name.into(), + params, + body, + return_type, + by_ref, + flags, + }) + } + }) + } +} diff --git a/src/parser/ident.rs b/src/parser/internal/ident.rs similarity index 100% rename from src/parser/ident.rs rename to src/parser/internal/ident.rs diff --git a/src/parser/internal/mod.rs b/src/parser/internal/mod.rs new file mode 100644 index 00000000..27d965dd --- /dev/null +++ b/src/parser/internal/mod.rs @@ -0,0 +1,11 @@ +pub mod block; +pub mod classish; +pub mod classish_statement; +pub mod flags; +pub mod functions; +pub mod ident; +pub mod params; +pub mod precedence; +pub mod punc; +pub mod types; +pub mod vars; diff --git a/src/parser/params.rs b/src/parser/internal/params.rs similarity index 71% rename from src/parser/params.rs rename to src/parser/internal/params.rs index d09820a4..d62a9da6 100644 --- a/src/parser/params.rs +++ b/src/parser/internal/params.rs @@ -1,32 +1,50 @@ use crate::lexer::token::TokenKind; use crate::parser::ast::Arg; use crate::parser::ast::Expression; +use crate::parser::ast::MethodFlag; use crate::parser::ast::Param; use crate::parser::ast::ParamList; use crate::parser::ast::PropertyFlag; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; -use crate::parser::precedence::Precedence; +use crate::parser::internal::precedence::Precedence; +use crate::parser::state::Scope; use crate::parser::state::State; use crate::parser::Parser; use crate::expect_token; -#[derive(Debug)] -pub enum ParamPosition { - Function, - Method(String), - AbstractMethod(String), -} - impl Parser { - pub(in crate::parser) fn param_list( - &self, - state: &mut State, - position: ParamPosition, - ) -> Result { + pub(in crate::parser) fn param_list(&self, state: &mut State) -> Result { let mut params = ParamList::new(); + let construct: i8 = match state.scope()? { + Scope::Function(_) | Scope::AnonymousFunction(_) | Scope::ArrowFunction(_) => 0, + Scope::Method(name, flags) => { + if name.to_string() != "__construct" { + 0 + } else { + match state.parent()? { + // can only have abstract ctor + Scope::Interface(_) => 1, + // can only have concret ctor + Scope::AnonymousClass => 2, + // can have either abstract or concret ctor, + // depens on method flag. + Scope::Class(_, _) | Scope::Trait(_) => { + if flags.contains(&MethodFlag::Abstract) { + 1 + } else { + 2 + } + } + _ => unreachable!(), + } + } + } + _ => unreachable!(), + }; + while !state.is_eof() && state.current.kind != TokenKind::RightParen { let mut param_type = None; @@ -37,22 +55,16 @@ impl Parser { .collect(); if !flags.is_empty() { - match position { - ParamPosition::Method(name) if name != "__construct" => { + match construct { + 0 => { return Err(ParseError::PromotedPropertyOutsideConstructor( state.current.span, )); } - ParamPosition::AbstractMethod(name) => { - if name == "__construct" { - return Err(ParseError::PromotedPropertyOnAbstractConstructor( - state.current.span, - )); - } else { - return Err(ParseError::PromotedPropertyOutsideConstructor( - state.current.span, - )); - } + 1 => { + return Err(ParseError::PromotedPropertyOnAbstractConstructor( + state.current.span, + )); } _ => {} } diff --git a/src/parser/precedence.rs b/src/parser/internal/precedence.rs similarity index 100% rename from src/parser/precedence.rs rename to src/parser/internal/precedence.rs diff --git a/src/parser/punc.rs b/src/parser/internal/punc.rs similarity index 100% rename from src/parser/punc.rs rename to src/parser/internal/punc.rs diff --git a/src/parser/internal/types.rs b/src/parser/internal/types.rs new file mode 100644 index 00000000..68d82262 --- /dev/null +++ b/src/parser/internal/types.rs @@ -0,0 +1,144 @@ +use crate::lexer::byte_string::ByteString; +use crate::lexer::token::TokenKind; +use crate::parser::ast::TryBlockCaughtType; +use crate::parser::ast::Type; +use crate::parser::error::ParseError; +use crate::parser::error::ParseResult; +use crate::parser::state::State; +use crate::parser::Parser; + +impl Parser { + pub(in crate::parser) fn try_block_caught_type_string( + &self, + state: &mut State, + ) -> ParseResult { + let id = self.full_name(state)?; + + if state.current.kind == TokenKind::Pipe { + state.next(); + + let mut types = vec![id.into()]; + + while !state.is_eof() { + let id = self.full_name(state)?; + types.push(id.into()); + + if state.current.kind != TokenKind::Pipe { + break; + } + + state.next(); + } + + return Ok(TryBlockCaughtType::Union(types)); + } + + Ok(TryBlockCaughtType::Identifier(id.into())) + } + + pub(in crate::parser) fn type_string(&self, state: &mut State) -> ParseResult { + if state.current.kind == TokenKind::Question { + state.next(); + let t = self.type_with_static(state)?; + return Ok(Type::Nullable(Box::new(parse_simple_type(t)))); + } + + let id = self.type_with_static(state)?; + + if state.current.kind == TokenKind::Pipe { + state.next(); + + let r#type = parse_simple_type(id); + if r#type.standalone() { + return Err(ParseError::StandaloneTypeUsedInCombination( + r#type, + state.current.span, + )); + } + + let mut types = vec![r#type]; + + while !state.is_eof() { + let id = self.type_with_static(state)?; + let r#type = parse_simple_type(id); + if r#type.standalone() { + return Err(ParseError::StandaloneTypeUsedInCombination( + r#type, + state.current.span, + )); + } + + types.push(r#type); + + if state.current.kind != TokenKind::Pipe { + break; + } else { + state.next(); + } + } + + return Ok(Type::Union(types)); + } + + if state.current.kind == TokenKind::Ampersand + && !matches!(state.peek.kind, TokenKind::Variable(_)) + { + state.next(); + + let r#type = parse_simple_type(id); + if r#type.standalone() { + return Err(ParseError::StandaloneTypeUsedInCombination( + r#type, + state.current.span, + )); + } + + let mut types = vec![r#type]; + + while !state.is_eof() { + let id = self.type_with_static(state)?; + let r#type = parse_simple_type(id); + if r#type.standalone() { + return Err(ParseError::StandaloneTypeUsedInCombination( + r#type, + state.current.span, + )); + } + + types.push(r#type); + + if state.current.kind != TokenKind::Ampersand { + break; + } else { + state.next(); + } + } + + return Ok(Type::Intersection(types)); + } + + Ok(parse_simple_type(id)) + } +} + +fn parse_simple_type(id: ByteString) -> Type { + let name = &id[..]; + let lowered_name = name.to_ascii_lowercase(); + match lowered_name.as_slice() { + b"void" => Type::Void, + b"never" => Type::Never, + b"null" => Type::Null, + b"true" => Type::True, + b"false" => Type::False, + b"float" => Type::Float, + b"bool" => Type::Boolean, + b"int" => Type::Integer, + b"string" => Type::String, + b"array" => Type::Array, + b"object" => Type::Object, + b"mixed" => Type::Mixed, + b"iterable" => Type::Iterable, + b"callable" => Type::Callable, + _ => Type::Identifier(id.into()), + } +} diff --git a/src/parser/vars.rs b/src/parser/internal/vars.rs similarity index 95% rename from src/parser/vars.rs rename to src/parser/internal/vars.rs index 58d5b5be..19920ca5 100644 --- a/src/parser/vars.rs +++ b/src/parser/internal/vars.rs @@ -2,7 +2,7 @@ use crate::lexer::token::TokenKind; use crate::parser::ast::Expression; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; -use crate::parser::precedence::Precedence; +use crate::parser::internal::precedence::Precedence; use crate::parser::state::State; use crate::parser::Parser; diff --git a/src/parser/macros.rs b/src/parser/macros.rs index 54bb0779..e816fcc9 100644 --- a/src/parser/macros.rs +++ b/src/parser/macros.rs @@ -93,3 +93,17 @@ macro_rules! expected_token_err { $crate::expected_token_err!([$expected], $state) }; } + +#[macro_export] +macro_rules! scoped { + ($state:expr, $scope:expr, $block:block) => {{ + let scope = $scope; + $state.enter(scope.clone()); + + let result = $block?; + + $state.exit(); + + Ok(result) + }}; +} diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 66884d50..63adba7b 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1,36 +1,26 @@ use crate::expect_literal; use crate::expect_token; use crate::expected_token_err; -use crate::lexer::byte_string::ByteString; use crate::lexer::token::Token; use crate::lexer::token::TokenKind; use crate::parser::ast::{ - ArrayItem, Block, Case, Catch, ClosureUse, Constant, DeclareItem, ElseIf, Expression, - IncludeKind, MagicConst, MatchArm, Program, Statement, StaticVar, StringPart, - TryBlockCaughtType, Type, Use, UseKind, + ArrayItem, Block, Case, Catch, Constant, DeclareItem, ElseIf, Expression, IncludeKind, + MagicConst, MatchArm, Program, Statement, StaticVar, StringPart, Use, UseKind, }; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; -use crate::parser::ident::is_reserved_ident; -use crate::parser::params::ParamPosition; -use crate::parser::precedence::{Associativity, Precedence}; +use crate::parser::internal::ident::is_reserved_ident; +use crate::parser::internal::precedence::{Associativity, Precedence}; +use crate::parser::state::Scope; use crate::parser::state::State; +use crate::scoped; pub mod ast; pub mod error; -mod block; -mod classish; -mod classish_statement; -mod flags; -mod functions; -mod ident; +mod internal; mod macros; -mod params; -mod precedence; -mod punc; mod state; -mod vars; #[derive(Debug, PartialEq, Eq, Clone, Copy, Default)] pub struct Parser; @@ -68,115 +58,6 @@ impl Parser { Ok(ast.to_vec()) } - fn try_block_caught_type_string(&self, state: &mut State) -> ParseResult { - let id = self.full_name(state)?; - - if state.current.kind == TokenKind::Pipe { - state.next(); - - let mut types = vec![id.into()]; - - while !state.is_eof() { - let id = self.full_name(state)?; - types.push(id.into()); - - if state.current.kind != TokenKind::Pipe { - break; - } - - state.next(); - } - - return Ok(TryBlockCaughtType::Union(types)); - } - - Ok(TryBlockCaughtType::Identifier(id.into())) - } - - fn type_string(&self, state: &mut State) -> ParseResult { - if state.current.kind == TokenKind::Question { - state.next(); - let t = self.type_with_static(state)?; - return Ok(Type::Nullable(Box::new(parse_simple_type(t)))); - } - - let id = self.type_with_static(state)?; - - if state.current.kind == TokenKind::Pipe { - state.next(); - - let r#type = parse_simple_type(id); - if r#type.standalone() { - return Err(ParseError::StandaloneTypeUsedInCombination( - r#type, - state.current.span, - )); - } - - let mut types = vec![r#type]; - - while !state.is_eof() { - let id = self.type_with_static(state)?; - let r#type = parse_simple_type(id); - if r#type.standalone() { - return Err(ParseError::StandaloneTypeUsedInCombination( - r#type, - state.current.span, - )); - } - - types.push(r#type); - - if state.current.kind != TokenKind::Pipe { - break; - } else { - state.next(); - } - } - - return Ok(Type::Union(types)); - } - - if state.current.kind == TokenKind::Ampersand - && !matches!(state.peek.kind, TokenKind::Variable(_)) - { - state.next(); - - let r#type = parse_simple_type(id); - if r#type.standalone() { - return Err(ParseError::StandaloneTypeUsedInCombination( - r#type, - state.current.span, - )); - } - - let mut types = vec![r#type]; - - while !state.is_eof() { - let id = self.type_with_static(state)?; - let r#type = parse_simple_type(id); - if r#type.standalone() { - return Err(ParseError::StandaloneTypeUsedInCombination( - r#type, - state.current.span, - )); - } - - types.push(r#type); - - if state.current.kind != TokenKind::Ampersand { - break; - } else { - state.next(); - } - } - - return Ok(Type::Intersection(types)); - } - - Ok(parse_simple_type(id)) - } - fn top_level_statement(&self, state: &mut State) -> ParseResult { state.skip_comments(); @@ -184,40 +65,45 @@ impl Parser { TokenKind::Namespace => { state.next(); - let mut braced = false; - - let name = if state.current.kind == TokenKind::LeftBrace { - braced = true; - self.lbrace(state)?; - None - } else { - Some(self.name(state)?) - }; + if state.current.kind != TokenKind::LeftBrace { + let name = self.name(state)?; - if name.is_some() { if state.current.kind == TokenKind::LeftBrace { - braced = true; - state.next(); + self.lbrace(state)?; + + let body = scoped!(state, Scope::BracedNamespace(Some(name.clone())), { + self.block(state, &TokenKind::RightBrace) + })?; + + self.rbrace(state)?; + + Statement::BracedNamespace { + name: Some(name), + body, + } } else { - self.semi(state)?; - } - } + let body = scoped!(state, Scope::Namespace(name.clone()), { + let mut body = Block::new(); + while !state.is_eof() { + body.push(self.top_level_statement(state)?); + } - let body = if braced { - self.block(state, &TokenKind::RightBrace)? - } else { - let mut body = Block::new(); - while !state.is_eof() { - body.push(self.top_level_statement(state)?); + Ok(body) + })?; + + Statement::Namespace { name, body } } - body - }; + } else { + self.lbrace(state)?; + + let body = scoped!(state, Scope::BracedNamespace(None), { + self.block(state, &TokenKind::RightBrace) + })?; - if braced { self.rbrace(state)?; - } - Statement::Namespace { name, body } + Statement::BracedNamespace { name: None, body } + } } TokenKind::Use => { state.next(); @@ -1251,161 +1137,12 @@ impl Parser { Expression::Array { items } } - TokenKind::Static if matches!(state.peek.kind, TokenKind::Function | TokenKind::Fn) => { - state.next(); - - match self.expression(state, Precedence::Lowest)? { - Expression::Closure { - params, - uses, - return_type, - body, - by_ref, - .. - } => Expression::Closure { - params, - uses, - return_type, - body, - by_ref, - r#static: true, - }, - Expression::ArrowFunction { - params, - return_type, - expr, - by_ref, - .. - } => Expression::ArrowFunction { - params, - return_type, - expr, - by_ref, - r#static: true, - }, - _ => unreachable!(), - } - } - TokenKind::Function => { - state.next(); - - let by_ref = if state.current.kind == TokenKind::Ampersand { - state.next(); - true - } else { - false - }; - - self.lparen(state)?; - - let params = self.param_list(state, ParamPosition::Function)?; - - self.rparen(state)?; - - let mut uses = vec![]; - if state.current.kind == TokenKind::Use { - state.next(); - - self.lparen(state)?; - - while state.current.kind != TokenKind::RightParen { - let var = match state.current.kind { - TokenKind::Ampersand => { - state.next(); - - match self.expression(state, Precedence::Lowest)? { - s @ Expression::Variable { .. } => ClosureUse { - var: s, - by_ref: true, - }, - _ => { - return Err(ParseError::UnexpectedToken( - "expected variable".into(), - state.current.span, - )) - } - } - } - _ => match self.expression(state, Precedence::Lowest)? { - s @ Expression::Variable { .. } => ClosureUse { - var: s, - by_ref: false, - }, - _ => { - return Err(ParseError::UnexpectedToken( - "expected variable".into(), - state.current.span, - )) - } - }, - }; - - uses.push(var); - - self.optional_comma(state)?; - } - - self.rparen(state)?; - } - - let mut return_type = None; - if state.current.kind == TokenKind::Colon { - self.colon(state)?; - - return_type = Some(self.type_string(state)?); - } - - self.lbrace(state)?; - - let body = self.block(state, &TokenKind::RightBrace)?; - - self.rbrace(state)?; - - Expression::Closure { - params, - uses, - return_type, - body, - r#static: false, - by_ref, - } - } - TokenKind::Fn => { - state.next(); - - let by_ref = if state.current.kind == TokenKind::Ampersand { - state.next(); - true - } else { - false - }; - - self.lparen(state)?; - - let params = self.param_list(state, ParamPosition::Function)?; - - self.rparen(state)?; - - let mut return_type = None; - - if state.current.kind == TokenKind::Colon { - self.colon(state)?; - - return_type = Some(self.type_string(state)?); - } - - expect_token!([TokenKind::DoubleArrow], state, ["`=>`"]); - - let value = self.expression(state, Precedence::Lowest)?; - - Expression::ArrowFunction { - params, - return_type, - expr: Box::new(value), - by_ref, - r#static: false, - } + TokenKind::Static if state.peek.kind == TokenKind::Function => { + self.anonymous_function(state)? } + TokenKind::Static if state.peek.kind == TokenKind::Fn => self.arrow_function(state)?, + TokenKind::Function => self.anonymous_function(state)?, + TokenKind::Fn => self.arrow_function(state)?, TokenKind::New if state.peek.kind == TokenKind::Class => { self.anonymous_class_definition(state)? } @@ -1855,28 +1592,6 @@ impl Parser { } } -fn parse_simple_type(id: ByteString) -> Type { - let name = &id[..]; - let lowered_name = name.to_ascii_lowercase(); - match lowered_name.as_slice() { - b"void" => Type::Void, - b"never" => Type::Never, - b"null" => Type::Null, - b"true" => Type::True, - b"false" => Type::False, - b"float" => Type::Float, - b"bool" => Type::Boolean, - b"int" => Type::Integer, - b"string" => Type::String, - b"array" => Type::Array, - b"object" => Type::Object, - b"mixed" => Type::Mixed, - b"iterable" => Type::Iterable, - b"callable" => Type::Callable, - _ => Type::Identifier(id.into()), - } -} - fn is_prefix(op: &TokenKind) -> bool { matches!( op, diff --git a/src/parser/state.rs b/src/parser/state.rs index 2eb25d82..173ca892 100644 --- a/src/parser/state.rs +++ b/src/parser/state.rs @@ -1,10 +1,34 @@ +use std::collections::VecDeque; use std::vec::IntoIter; +use crate::lexer::byte_string::ByteString; use crate::lexer::token::Token; use crate::lexer::token::TokenKind; +use crate::parser::ast::ClassFlag; +use crate::parser::ast::MethodFlag; +use crate::parser::error::ParseError; +use crate::parser::error::ParseResult; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum Scope { + Namespace(ByteString), + BracedNamespace(Option), + + Interface(ByteString), + Class(ByteString, Vec), + Trait(ByteString), + Enum(ByteString), + AnonymousClass, + + Function(ByteString), + Method(ByteString, Vec), + AnonymousFunction(bool), + ArrowFunction(bool), +} #[derive(Debug, Clone)] pub struct State { + pub stack: VecDeque, pub current: Token, pub peek: Token, pub iter: IntoIter, @@ -16,6 +40,7 @@ impl State { let mut iter = tokens.into_iter(); Self { + stack: VecDeque::new(), current: iter.next().unwrap_or_default(), peek: iter.next().unwrap_or_default(), iter, @@ -23,6 +48,50 @@ impl State { } } + pub fn named(&self, name: &ByteString) -> String { + let mut namespace = None; + for scope in &self.stack { + match scope { + Scope::Namespace(n) => { + namespace = Some(n.to_string()); + + break; + } + Scope::BracedNamespace(n) => { + namespace = n.as_ref().map(|s| s.to_string()); + + break; + } + _ => {} + } + } + + match namespace { + Some(v) => format!("{}\\{}", v, name), + None => name.to_string(), + } + } + + pub fn scope(&self) -> ParseResult<&Scope> { + self.stack + .back() + .ok_or(ParseError::UnpredictableState(self.current.span)) + } + + pub fn parent(&self) -> ParseResult<&Scope> { + self.stack + .get(self.stack.len() - 2) + .ok_or(ParseError::UnpredictableState(self.current.span)) + } + + pub fn enter(&mut self, state: Scope) { + self.stack.push_back(state); + } + + pub fn exit(&mut self) { + self.stack.pop_back(); + } + pub fn skip_comments(&mut self) { while matches!( self.current.kind, diff --git a/src/prelude.rs b/src/prelude.rs index 8298ab6a..a456ce2a 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -2,7 +2,6 @@ pub use crate::lexer::byte_string::*; pub use crate::lexer::error::*; pub use crate::lexer::token::*; pub use crate::lexer::*; - pub use crate::parser::ast::*; pub use crate::parser::error::*; pub use crate::parser::*; diff --git a/tests/0002/parser-error.txt b/tests/0002/parser-error.txt index 5168e7d8..229da3de 100644 --- a/tests/0002/parser-error.txt +++ b/tests/0002/parser-error.txt @@ -1 +1 @@ -StandaloneTypeUsedInCombination(Never, (3, 23)) -> Parse error: never can only be used as a standalone type on line 3 column 23 +StandaloneTypeUsedInCombination(Never, (3, 23)) -> Parse error: 'never' can only be used as a standalone type on line 3 column 23 diff --git a/tests/0003/parser-error.txt b/tests/0003/parser-error.txt index af6dff75..1f44f522 100644 --- a/tests/0003/parser-error.txt +++ b/tests/0003/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["an identifier"], Some(")"), (6, 14)) -> Parse error: unexpected token `)`, expecting an identifier on line 6 column 14 +ExpectedToken(["an identifier"], Some(")"), (6, 14)) -> Parse Error: unexpected token `)`, expecting an identifier on line 6 column 14 diff --git a/tests/0004/parser-error.txt b/tests/0004/parser-error.txt index 2a15c0e9..242969bd 100644 --- a/tests/0004/parser-error.txt +++ b/tests/0004/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["an identifier"], Some("e"), (6, 14)) -> Parse error: unexpected token `e`, expecting an identifier on line 6 column 14 +ExpectedToken(["an identifier"], Some("e"), (6, 14)) -> Parse Error: unexpected token `e`, expecting an identifier on line 6 column 14 diff --git a/tests/0005/parser-error.txt b/tests/0005/parser-error.txt index d137201c..649f8191 100644 --- a/tests/0005/parser-error.txt +++ b/tests/0005/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["`(`"], Some("{"), (6, 13)) -> Parse error: unexpected token `{`, expecting `(` on line 6 column 13 +ExpectedToken(["`(`"], Some("{"), (6, 13)) -> Parse Error: unexpected token `{`, expecting `(` on line 6 column 13 diff --git a/tests/0016/parser-error.txt b/tests/0016/parser-error.txt index 32dab20d..81b48573 100644 --- a/tests/0016/parser-error.txt +++ b/tests/0016/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["a literal"], Some("bar"), (3, 16)) -> Parse error: unexpected token `bar`, expecting a literal on line 3 column 16 +ExpectedToken(["a literal"], Some("bar"), (3, 16)) -> Parse Error: unexpected token `bar`, expecting a literal on line 3 column 16 diff --git a/tests/0019/ast.txt b/tests/0019/ast.txt index 428a671a..531fe195 100644 --- a/tests/0019/ast.txt +++ b/tests/0019/ast.txt @@ -1,5 +1,5 @@ [ - Namespace { + BracedNamespace { name: None, body: [ Function { @@ -13,7 +13,7 @@ }, ], }, - Namespace { + BracedNamespace { name: Some( "foo", ), @@ -198,7 +198,7 @@ }, ], }, - Namespace { + BracedNamespace { name: Some( "bar", ), @@ -383,7 +383,7 @@ }, ], }, - Namespace { + BracedNamespace { name: Some( "baz", ), diff --git a/tests/0044/parser-error.txt b/tests/0044/parser-error.txt index 750c5681..950193f0 100644 --- a/tests/0044/parser-error.txt +++ b/tests/0044/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["`const`", "`function`", "an identifier", "a varaible"], Some("fn"), (1, 26)) -> Parse error: unexpected token `fn`, expecting `const`, `function`, an identifier, or a varaible on line 1 column 26 +ExpectedToken(["`const`", "`function`", "an identifier", "a varaible"], Some("fn"), (1, 26)) -> Parse Error: unexpected token `fn`, expecting `const`, `function`, an identifier, or a varaible on line 1 column 26 diff --git a/tests/0087/parser-error.txt b/tests/0087/parser-error.txt index 7a80e776..410594e8 100644 --- a/tests/0087/parser-error.txt +++ b/tests/0087/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["`=`"], Some(";"), (5, 13)) -> Parse error: unexpected token `;`, expecting `=` on line 5 column 13 +ExpectedToken(["`=`"], Some(";"), (5, 13)) -> Parse Error: unexpected token `;`, expecting `=` on line 5 column 13 diff --git a/tests/0088/parser-error.txt b/tests/0088/parser-error.txt index 3fc6797e..d1c90531 100644 --- a/tests/0088/parser-error.txt +++ b/tests/0088/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["`;`"], Some("class"), (1, 16)) -> Parse error: unexpected token `class`, expecting `;` on line 1 column 16 +ExpectedToken(["`;`"], Some("class"), (1, 16)) -> Parse Error: unexpected token `class`, expecting `;` on line 1 column 16 diff --git a/tests/0111/parser-error.txt b/tests/0111/parser-error.txt index 2a105e31..5369330f 100644 --- a/tests/0111/parser-error.txt +++ b/tests/0111/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["an identifier"], Some("..."), (5, 25)) -> Parse error: unexpected token `...`, expecting an identifier on line 5 column 25 +ExpectedToken(["an identifier"], Some("..."), (5, 25)) -> Parse Error: unexpected token `...`, expecting an identifier on line 5 column 25 diff --git a/tests/0112/parser-error.txt b/tests/0112/parser-error.txt index 4ae69a31..872d6967 100644 --- a/tests/0112/parser-error.txt +++ b/tests/0112/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["an identifier"], Some("&"), (5, 25)) -> Parse error: unexpected token `&`, expecting an identifier on line 5 column 25 +ExpectedToken(["an identifier"], Some("&"), (5, 25)) -> Parse Error: unexpected token `&`, expecting an identifier on line 5 column 25 diff --git a/tests/0113/parser-error.txt b/tests/0113/parser-error.txt index 0b0956c4..da1607de 100644 --- a/tests/0113/parser-error.txt +++ b/tests/0113/parser-error.txt @@ -1 +1 @@ -PromotedPropertyOutsideConstructor((5, 16)) -> Parse error: Cannot declare promoted property outside a constructor on line 5 column 16 +PromotedPropertyOutsideConstructor((5, 16)) -> Parse Error: Cannot declare promoted property outside a constructor on line 5 column 16 diff --git a/tests/0114/parser-error.txt b/tests/0114/parser-error.txt index 78046f26..9eec84bf 100644 --- a/tests/0114/parser-error.txt +++ b/tests/0114/parser-error.txt @@ -1 +1 @@ -PromotedPropertyOnAbstractConstructor((5, 16)) -> Parse error: Cannot declare promoted property in an abstract constructor on line 5 column 16 +PromotedPropertyOnAbstractConstructor((5, 16)) -> Parse Error: Cannot declare promoted property in an abstract constructor on line 5 column 16 diff --git a/tests/0115/parser-error.txt b/tests/0115/parser-error.txt index 78046f26..9eec84bf 100644 --- a/tests/0115/parser-error.txt +++ b/tests/0115/parser-error.txt @@ -1 +1 @@ -PromotedPropertyOnAbstractConstructor((5, 16)) -> Parse error: Cannot declare promoted property in an abstract constructor on line 5 column 16 +PromotedPropertyOnAbstractConstructor((5, 16)) -> Parse Error: Cannot declare promoted property in an abstract constructor on line 5 column 16 diff --git a/tests/0116/parser-error.txt b/tests/0116/parser-error.txt index 78046f26..9eec84bf 100644 --- a/tests/0116/parser-error.txt +++ b/tests/0116/parser-error.txt @@ -1 +1 @@ -PromotedPropertyOnAbstractConstructor((5, 16)) -> Parse error: Cannot declare promoted property in an abstract constructor on line 5 column 16 +PromotedPropertyOnAbstractConstructor((5, 16)) -> Parse Error: Cannot declare promoted property in an abstract constructor on line 5 column 16 diff --git a/tests/0120/parser-error.txt b/tests/0120/parser-error.txt index f1ab1dd6..64c0675f 100644 --- a/tests/0120/parser-error.txt +++ b/tests/0120/parser-error.txt @@ -1 +1 @@ -AbstractModifierOnNonAbstractClassMethod((4, 14)) -> Parse error: Cannot declare abstract methods on a non-abstract class on line 4 column 14 +AbstractModifierOnNonAbstractClassMethod((4, 26)) -> Parse Error: Cannot declare abstract methods on a non-abstract class on line 4 column 26 diff --git a/tests/0121/parser-error.txt b/tests/0121/parser-error.txt index 18bec76e..51d3026f 100644 --- a/tests/0121/parser-error.txt +++ b/tests/0121/parser-error.txt @@ -1 +1 @@ -StaticModifierOnConstant((4, 12)) -> Parse error: Cannot use 'static' as constant modifier on line 4 column 12 +StaticModifierOnConstant((4, 12)) -> Parse Error: Cannot use 'static' as constant modifier on line 4 column 12 diff --git a/tests/0122/parser-error.txt b/tests/0122/parser-error.txt index 3421ec02..0e928e36 100644 --- a/tests/0122/parser-error.txt +++ b/tests/0122/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["an identifier"], Some("static"), (4, 11)) -> Parse error: unexpected token `static`, expecting an identifier on line 4 column 11 +ExpectedToken(["an identifier"], Some("static"), (4, 11)) -> Parse Error: unexpected token `static`, expecting an identifier on line 4 column 11 diff --git a/tests/0123/parser-error.txt b/tests/0123/parser-error.txt index e1698a5f..9d479a47 100644 --- a/tests/0123/parser-error.txt +++ b/tests/0123/parser-error.txt @@ -1 +1 @@ -ReadonlyModifierOnConstant((4, 14)) -> Parse error: Cannot use 'readonly' as constant modifier on line 4 column 14 +ReadonlyModifierOnConstant((4, 14)) -> Parse Error: Cannot use 'readonly' as constant modifier on line 4 column 14 diff --git a/tests/0124/parser-error.txt b/tests/0124/parser-error.txt index a59f86e0..a3fdb76c 100644 --- a/tests/0124/parser-error.txt +++ b/tests/0124/parser-error.txt @@ -1 +1 @@ -FinalModifierOnAbstractClassMember((4, 11)) -> Parse error: Cannot use the final modifier on an abstract class member on line 4 column 11 +FinalModifierOnAbstractClassMember((4, 11)) -> Parse Error: Cannot use the final modifier on an abstract class member on line 4 column 11 diff --git a/tests/0125/parser-error.txt b/tests/0125/parser-error.txt index 122e038f..94347b1a 100644 --- a/tests/0125/parser-error.txt +++ b/tests/0125/parser-error.txt @@ -1 +1 @@ -FinalModifierOnAbstractClass((3, 7)) -> Parse error: Cannot use the final modifier on an abstract class on line 3 column 7 +FinalModifierOnAbstractClass((3, 7)) -> Parse Error: Cannot use the final modifier on an abstract class on line 3 column 7 diff --git a/tests/0126/parser-error.txt b/tests/0126/parser-error.txt index a7979b63..28e82485 100644 --- a/tests/0126/parser-error.txt +++ b/tests/0126/parser-error.txt @@ -1 +1 @@ -FinalModifierOnPrivateConstant((4, 19)) -> Parse error: Private constant cannot be final as it is not visible to other classes on line 4 column 19 +FinalModifierOnPrivateConstant((4, 19)) -> Parse Error: Private constant cannot be final as it is not visible to other classes on line 4 column 19 diff --git a/tests/0128/parser-error.txt b/tests/0128/parser-error.txt index 92eb11ea..c9c94267 100644 --- a/tests/0128/parser-error.txt +++ b/tests/0128/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["an identifier"], Some("s"), (5, 25)) -> Parse error: unexpected token `s`, expecting an identifier on line 5 column 25 +ExpectedToken(["an identifier"], Some("s"), (5, 25)) -> Parse Error: unexpected token `s`, expecting an identifier on line 5 column 25 diff --git a/tests/0131/parser-error.txt b/tests/0131/parser-error.txt index a662eb6a..2d34187e 100644 --- a/tests/0131/parser-error.txt +++ b/tests/0131/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["`(`"], Some("foreach"), (3, 10)) -> Parse error: unexpected token `foreach`, expecting `(` on line 3 column 10 +ExpectedToken(["`(`"], Some("foreach"), (3, 10)) -> Parse Error: unexpected token `foreach`, expecting `(` on line 3 column 10 diff --git a/tests/0132/parser-error.txt b/tests/0132/parser-error.txt index 892352d9..ac975017 100644 --- a/tests/0132/parser-error.txt +++ b/tests/0132/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["`const`", "`function`"], Some("abstract"), (4, 12)) -> Parse error: unexpected token `abstract`, expecting `const`, or `function` on line 4 column 12 +ExpectedToken(["`const`", "`function`"], Some("abstract"), (4, 12)) -> Parse Error: unexpected token `abstract`, expecting `const`, or `function` on line 4 column 12 diff --git a/tests/0141/parser-error.txt b/tests/0141/parser-error.txt index ab5056d8..e4d28706 100644 --- a/tests/0141/parser-error.txt +++ b/tests/0141/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["`const`", "`function`"], Some("private"), (4, 5)) -> Parse error: unexpected token `private`, expecting `const`, or `function` on line 4 column 5 +ExpectedToken(["`const`", "`function`"], Some("private"), (4, 5)) -> Parse Error: unexpected token `private`, expecting `const`, or `function` on line 4 column 5 diff --git a/tests/0142/parser-error.txt b/tests/0142/parser-error.txt index b293fc06..f1ee98c7 100644 --- a/tests/0142/parser-error.txt +++ b/tests/0142/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["`const`", "`function`"], Some("protected"), (4, 5)) -> Parse error: unexpected token `protected`, expecting `const`, or `function` on line 4 column 5 +ExpectedToken(["`const`", "`function`"], Some("protected"), (4, 5)) -> Parse Error: unexpected token `protected`, expecting `const`, or `function` on line 4 column 5 diff --git a/tests/0144/parser-error.txt b/tests/0144/parser-error.txt index 24ca7203..f3f2a17b 100644 --- a/tests/0144/parser-error.txt +++ b/tests/0144/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["`const`", "`function`"], Some("final"), (4, 5)) -> Parse error: unexpected token `final`, expecting `const`, or `function` on line 4 column 5 +ExpectedToken(["`const`", "`function`"], Some("final"), (4, 5)) -> Parse Error: unexpected token `final`, expecting `const`, or `function` on line 4 column 5 diff --git a/tests/0146/code.php b/tests/0146/code.php new file mode 100644 index 00000000..b3048f5b --- /dev/null +++ b/tests/0146/code.php @@ -0,0 +1,9 @@ + Parse Error: Enum 'Foo\Bar' cannot have a constructor on line 6 column 33 diff --git a/tests/0146/tokens.txt b/tests/0146/tokens.txt new file mode 100644 index 00000000..1259c99c --- /dev/null +++ b/tests/0146/tokens.txt @@ -0,0 +1,115 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 14, + ), + }, + Token { + kind: Enum, + span: ( + 5, + 1, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 5, + 6, + ), + }, + Token { + kind: LeftBrace, + span: ( + 5, + 10, + ), + }, + Token { + kind: Public, + span: ( + 6, + 6, + ), + }, + Token { + kind: Function, + span: ( + 6, + 13, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 6, + 22, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 33, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 34, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 36, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 6, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, +] From acce902c9eb474791d68e6895a0962bd88b193f2 Mon Sep 17 00:00:00 2001 From: Saif Eddin Gmati <29315886+azjezz@users.noreply.github.com> Date: Thu, 1 Dec 2022 15:20:25 +0100 Subject: [PATCH 05/13] fix: enum parser Signed-off-by: azjezz --- src/parser/error.rs | 4 + src/parser/internal/classish.rs | 42 +++---- src/parser/internal/classish_statement.rs | 28 ++++- src/parser/internal/functions.rs | 45 +++---- src/parser/macros.rs | 46 ++++--- src/parser/state.rs | 2 +- tests/0087/parser-error.txt | 2 +- tests/0147/code.php | 8 ++ tests/0147/parser-error.txt | 1 + tests/0147/tokens.txt | 142 ++++++++++++++++++++++ tests/0148/code.php | 8 ++ tests/0148/parser-error.txt | 1 + tests/0148/tokens.txt | 126 +++++++++++++++++++ 13 files changed, 386 insertions(+), 69 deletions(-) create mode 100644 tests/0147/code.php create mode 100644 tests/0147/parser-error.txt create mode 100644 tests/0147/tokens.txt create mode 100644 tests/0148/code.php create mode 100644 tests/0148/parser-error.txt create mode 100644 tests/0148/tokens.txt diff --git a/src/parser/error.rs b/src/parser/error.rs index adb963f8..bebb0da6 100644 --- a/src/parser/error.rs +++ b/src/parser/error.rs @@ -19,6 +19,8 @@ pub enum ParseError { PromotedPropertyOnAbstractConstructor(Span), AbstractModifierOnNonAbstractClassMethod(Span), ConstructorInEnum(String, Span), + MissingCaseValueForBackedEnum(String, String, Span), + CaseValueForUnitEnum(String, String, Span), StaticModifierOnConstant(Span), ReadonlyModifierOnConstant(Span), FinalModifierOnAbstractClassMember(Span), @@ -61,6 +63,8 @@ impl Display for ParseError { Self::AbstractModifierOnNonAbstractClassMethod(span) => write!(f, "Parse Error: Cannot declare abstract methods on a non-abstract class on line {} column {}", span.0, span.1), Self::FinalModifierOnAbstractClass(span) => write!(f, "Parse Error: Cannot use the final modifier on an abstract class on line {} column {}", span.0, span.1), Self::ConstructorInEnum(name, span) => write!(f, "Parse Error: Enum '{}' cannot have a constructor on line {} column {}", name, span.0, span.1), + Self::MissingCaseValueForBackedEnum(case, name, span) => write!(f, "Parse Error: Case `{}` of backed enum `{}` must have a value on line {} column {}", case, name, span.0, span.1), + Self::CaseValueForUnitEnum(case, name, span) => write!(f, "Parse Error: Case `{}` of unit enum `{}` must not have a value on line {} column {}", case, name, span.0, span.1), Self::UnpredictableState(span) => write!(f, "Parse Error: Reached an unpredictable state on line {} column {}", span.0, span.1) } } diff --git a/src/parser/internal/classish.rs b/src/parser/internal/classish.rs index 0dd32a28..98402606 100644 --- a/src/parser/internal/classish.rs +++ b/src/parser/internal/classish.rs @@ -11,7 +11,6 @@ use crate::parser::state::State; use crate::parser::Parser; use crate::expect_token; -use crate::expected_token_err; use crate::scoped; impl Parser { @@ -195,28 +194,23 @@ impl Parser { let name = self.ident(state)?; - scoped!(state, Scope::Enum(name.clone()), { - let backed_type: Option = if state.current.kind == TokenKind::Colon { - self.colon(state)?; - - match state.current.kind.clone() { - TokenKind::Identifier(s) if s == b"string" || s == b"int" => { - state.next(); - - Some(match &s[..] { - b"string" => BackedEnumType::String, - b"int" => BackedEnumType::Int, - _ => unreachable!(), - }) - } - _ => { - return expected_token_err!(["`string`", "`int`"], state); - } - } - } else { - None - }; - + let backed_type: Option = if state.current.kind == TokenKind::Colon { + self.colon(state)?; + + expect_token!([ + TokenKind::Identifier(s) if s == b"string" || s == b"int" => { + Some(match &s[..] { + b"string" => BackedEnumType::String, + b"int" => BackedEnumType::Int, + _ => unreachable!(), + }) + }, + ], state, ["`string`", "`int`",]) + } else { + None + }; + + scoped!(state, Scope::Enum(name.clone(), backed_type.is_some()), { let mut implements = Vec::new(); if state.current.kind == TokenKind::Implements { state.next(); @@ -233,7 +227,7 @@ impl Parser { let mut body = Block::new(); while state.current.kind != TokenKind::RightBrace { state.skip_comments(); - body.push(self.enum_statement(state, backed_type.is_some())?); + body.push(self.enum_statement(state)?); } self.rbrace(state)?; diff --git a/src/parser/internal/classish_statement.rs b/src/parser/internal/classish_statement.rs index 7d846ea3..0f6eac1c 100644 --- a/src/parser/internal/classish_statement.rs +++ b/src/parser/internal/classish_statement.rs @@ -1,3 +1,4 @@ +use crate::expected_scope; use crate::lexer::token::TokenKind; use crate::parser::ast::Identifier; use crate::parser::ast::MethodFlag; @@ -6,6 +7,7 @@ use crate::parser::ast::TraitAdaptation; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; use crate::parser::internal::precedence::Precedence; +use crate::parser::state::Scope; use crate::parser::state::State; use crate::parser::Parser; @@ -37,17 +39,25 @@ impl Parser { ], state, ["`const`", "`function`"]) } - pub(in crate::parser) fn enum_statement( - &self, - state: &mut State, - backed: bool, - ) -> ParseResult { + pub(in crate::parser) fn enum_statement(&self, state: &mut State) -> ParseResult { + let (enum_name, backed) = expected_scope!([ + Scope::Enum(enum_name, backed) => (enum_name, backed), + ], state); + if state.current.kind == TokenKind::Case { state.next(); let name = self.ident(state)?; if backed { + if state.current.kind == TokenKind::SemiColon { + return Err(ParseError::MissingCaseValueForBackedEnum( + name.to_string(), + state.named(&enum_name), + state.current.span, + )); + } + expect_token!([TokenKind::Equals], state, "`=`"); let value = self.expression(state, Precedence::Lowest)?; @@ -58,6 +68,14 @@ impl Parser { value, }); } else { + if state.current.kind == TokenKind::Equals { + return Err(ParseError::CaseValueForUnitEnum( + name.to_string(), + state.named(&enum_name), + state.current.span, + )); + } + self.semi(state)?; return Ok(Statement::UnitEnumCase { name: name.into() }); diff --git a/src/parser/internal/functions.rs b/src/parser/internal/functions.rs index 5b07840d..b393726e 100644 --- a/src/parser/internal/functions.rs +++ b/src/parser/internal/functions.rs @@ -1,4 +1,5 @@ use crate::expect_token; +use crate::expected_scope; use crate::lexer::token::TokenKind; use crate::parser::ast::ClassFlag; use crate::parser::ast::ClosureUse; @@ -203,32 +204,32 @@ impl Parser { let name = self.ident_maybe_reserved(state)?; - scoped!(state, Scope::Method(name.clone(), flags.clone()), { - let has_body = match state.parent()? { - Scope::Class(_, cf) => { - if !cf.contains(&ClassFlag::Abstract) && flags.contains(&MethodFlag::Abstract) { - return Err(ParseError::AbstractModifierOnNonAbstractClassMethod( - state.current.span, - )); - } - - !flags.contains(&MethodFlag::Abstract) + let has_body = expected_scope!([ + Scope::Class(_, cf) => { + if !cf.contains(&ClassFlag::Abstract) && flags.contains(&MethodFlag::Abstract) { + return Err(ParseError::AbstractModifierOnNonAbstractClassMethod( + state.current.span, + )); } - Scope::Trait(_) => !flags.contains(&MethodFlag::Abstract), - Scope::Interface(_) => false, - Scope::Enum(enum_name) => { - if name.to_string() == "__construct" { - return Err(ParseError::ConstructorInEnum( - state.named(enum_name), - state.current.span, - )); - } - true + !flags.contains(&MethodFlag::Abstract) + }, + Scope::Trait(_) => !flags.contains(&MethodFlag::Abstract), + Scope::Interface(_) => false, + Scope::Enum(enum_name, _) => { + if name.to_string() == "__construct" { + return Err(ParseError::ConstructorInEnum( + state.named(&enum_name), + state.current.span, + )); } - _ => true, - }; + true + }, + Scope::AnonymousClass => true, + ], state); + + scoped!(state, Scope::Method(name.clone(), flags.clone()), { self.lparen(state)?; let params = self.param_list(state)?; diff --git a/src/parser/macros.rs b/src/parser/macros.rs index e816fcc9..b49e7176 100644 --- a/src/parser/macros.rs +++ b/src/parser/macros.rs @@ -1,43 +1,43 @@ #[macro_export] macro_rules! peek_token { - ([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{ + ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{ $state.skip_comments(); match $state.current.kind.clone() { $( - $expected => $out, + $( $pattern )|+ $( if $guard )? => $out, )+ _ => { return $crate::expected_token_err!([ $($message,)+ ], $state); } } }}; - ([ $($expected:pat),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{ + ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )?),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{ $state.skip_comments(); - if !matches!($state.current.kind, $(| $expected )+) { + if !matches!($state.current.kind, $( $pattern )|+ $( if $guard )?) { return $crate::expected_token_err!([ $($message,)+ ], $state); } }}; - ([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, $message:literal) => { - $crate::peek_token!([ $($expected => $out,)+ ], $state, [$message]) + ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, $message:literal) => { + $crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => $out,)+ ], $state, [$message]) }; - ([ $($expected:pat),+ $(,)? ], $state:expr, $message:literal) => { - $crate::peek_token!([ $($expected,)+ ], $state, [$message]) + ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )?),+ $(,)? ], $state:expr, $message:literal) => { + $crate::peek_token!([ $($( $pattern )|+ $( if $guard )?,)+ ], $state, [$message]) }; } #[macro_export] macro_rules! expect_token { - ([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => { - $crate::peek_token!([ $($expected => { $state.next(); $out },)+ ], $state, [$($message,)+]) + ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => { + $crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); $out },)+ ], $state, [$($message,)+]) }; - ([ $($expected:pat),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => { - $crate::peek_token!([ $($expected => { $state.next(); },)+ ], $state, [$($message,)+]) + ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )?),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => { + $crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); },)+ ], $state, [$($message,)+]) }; - ([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, $message:literal) => { - $crate::peek_token!([ $($expected => { $state.next(); $out },)+ ], $state, [$message]) + ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, $message:literal) => { + $crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); $out },)+ ], $state, [$message]) }; - ([ $($expected:pat),+ $(,)? ], $state:expr, $message:literal) => { - $crate::peek_token!([ $($expected => { $state.next(); },)+ ], $state, [$message]) + ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )?),+ $(,)? ], $state:expr, $message:literal) => { + $crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); },)+ ], $state, [$message]) }; } @@ -94,6 +94,20 @@ macro_rules! expected_token_err { }; } +#[macro_export] +macro_rules! expected_scope { + ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr) => {{ + match $state.scope().cloned()? { + $( + $( $pattern )|+ $( if $guard )? => $out, + )+ + _ => { + return Err($crate::parser::error::ParseError::UnpredictableState($state.current.span)); + } + } + }}; +} + #[macro_export] macro_rules! scoped { ($state:expr, $scope:expr, $block:block) => {{ diff --git a/src/parser/state.rs b/src/parser/state.rs index 173ca892..d9aace41 100644 --- a/src/parser/state.rs +++ b/src/parser/state.rs @@ -17,7 +17,7 @@ pub enum Scope { Interface(ByteString), Class(ByteString, Vec), Trait(ByteString), - Enum(ByteString), + Enum(ByteString, bool), AnonymousClass, Function(ByteString), diff --git a/tests/0087/parser-error.txt b/tests/0087/parser-error.txt index 410594e8..2ac64645 100644 --- a/tests/0087/parser-error.txt +++ b/tests/0087/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["`=`"], Some(";"), (5, 13)) -> Parse Error: unexpected token `;`, expecting `=` on line 5 column 13 +MissingCaseValueForBackedEnum("Bar", "Foo", (5, 13)) -> Parse Error: Case `Bar` of backed enum `Foo` must have a value on line 5 column 13 diff --git a/tests/0147/code.php b/tests/0147/code.php new file mode 100644 index 00000000..b293e450 --- /dev/null +++ b/tests/0147/code.php @@ -0,0 +1,8 @@ + Parse Error: Case `Baz` of backed enum `A\B\C\D\E\Foo` must have a value on line 7 column 14 diff --git a/tests/0147/tokens.txt b/tests/0147/tokens.txt new file mode 100644 index 00000000..88dae872 --- /dev/null +++ b/tests/0147/tokens.txt @@ -0,0 +1,142 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "A\B\C\D\E", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 20, + ), + }, + Token { + kind: Enum, + span: ( + 5, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 5, + 6, + ), + }, + Token { + kind: Colon, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 5, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 5, + 15, + ), + }, + Token { + kind: Case, + span: ( + 6, + 6, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 6, + 11, + ), + }, + Token { + kind: Equals, + span: ( + 6, + 15, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 6, + 17, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 18, + ), + }, + Token { + kind: Case, + span: ( + 7, + 6, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 14, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 1, + ), + }, +] diff --git a/tests/0148/code.php b/tests/0148/code.php new file mode 100644 index 00000000..884ae634 --- /dev/null +++ b/tests/0148/code.php @@ -0,0 +1,8 @@ + Parse Error: Case `Baz` of unit enum `A\B\C\D\E\Foo` must not have a value on line 7 column 15 diff --git a/tests/0148/tokens.txt b/tests/0148/tokens.txt new file mode 100644 index 00000000..562b9e59 --- /dev/null +++ b/tests/0148/tokens.txt @@ -0,0 +1,126 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "A\B\C\D\E", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 20, + ), + }, + Token { + kind: Enum, + span: ( + 5, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 5, + 6, + ), + }, + Token { + kind: LeftBrace, + span: ( + 5, + 10, + ), + }, + Token { + kind: Case, + span: ( + 6, + 6, + ), + }, + Token { + kind: Identifier( + "Bar", + ), + span: ( + 6, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 14, + ), + }, + Token { + kind: Case, + span: ( + 7, + 6, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: Equals, + span: ( + 7, + 15, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 7, + 17, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 18, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 1, + ), + }, +] From 2a6652012816bd0f9aaaa7c2783c05d45c2adc7c Mon Sep 17 00:00:00 2001 From: Saif Eddin Gmati <29315886+azjezz@users.noreply.github.com> Date: Fri, 2 Dec 2022 16:52:16 +0100 Subject: [PATCH 06/13] fix: properties parser Signed-off-by: azjezz --- src/lexer/token.rs | 6 +- src/parser/ast.rs | 30 +++ src/parser/error.rs | 10 +- src/parser/internal/classish_statement.rs | 101 +++---- src/parser/internal/functions.rs | 8 +- src/parser/internal/ident.rs | 25 -- src/parser/internal/params.rs | 32 ++- src/parser/internal/types.rs | 312 ++++++++++++++++++---- src/parser/internal/vars.rs | 18 +- src/parser/macros.rs | 21 +- tests/0004/parser-error.txt | 2 +- tests/0044/parser-error.txt | 2 +- tests/0111/parser-error.txt | 2 +- tests/0112/parser-error.txt | 2 +- tests/0124/parser-error.txt | 2 +- tests/0128/parser-error.txt | 2 +- tests/0149/ast.txt | 49 ++++ tests/0149/code.php | 7 + tests/0149/tokens.txt | 151 +++++++++++ tests/0150/ast.txt | 49 ++++ tests/0150/code.php | 7 + tests/0150/tokens.txt | 149 +++++++++++ tests/0151/ast.txt | 49 ++++ tests/0151/code.php | 7 + tests/0151/tokens.txt | 151 +++++++++++ tests/0152/code.php | 9 + tests/0152/parser-error.txt | 1 + tests/0152/tokens.txt | 152 +++++++++++ tests/0153/ast.txt | 52 ++++ tests/0153/code.php | 9 + tests/0153/tokens.txt | 177 ++++++++++++ tests/0154/code.php | 7 + tests/0154/parser-error.txt | 1 + tests/0154/tokens.txt | 117 ++++++++ tests/0155/ast.txt | 31 +++ tests/0155/code.php | 7 + tests/0155/tokens.txt | 110 ++++++++ tests/0156/code.php | 7 + tests/0156/parser-error.txt | 1 + tests/0156/tokens.txt | 126 +++++++++ 40 files changed, 1828 insertions(+), 173 deletions(-) create mode 100644 tests/0149/ast.txt create mode 100644 tests/0149/code.php create mode 100644 tests/0149/tokens.txt create mode 100644 tests/0150/ast.txt create mode 100644 tests/0150/code.php create mode 100644 tests/0150/tokens.txt create mode 100644 tests/0151/ast.txt create mode 100644 tests/0151/code.php create mode 100644 tests/0151/tokens.txt create mode 100644 tests/0152/code.php create mode 100644 tests/0152/parser-error.txt create mode 100644 tests/0152/tokens.txt create mode 100644 tests/0153/ast.txt create mode 100644 tests/0153/code.php create mode 100644 tests/0153/tokens.txt create mode 100644 tests/0154/code.php create mode 100644 tests/0154/parser-error.txt create mode 100644 tests/0154/tokens.txt create mode 100644 tests/0155/ast.txt create mode 100644 tests/0155/code.php create mode 100644 tests/0155/tokens.txt create mode 100644 tests/0156/code.php create mode 100644 tests/0156/parser-error.txt create mode 100644 tests/0156/tokens.txt diff --git a/src/lexer/token.rs b/src/lexer/token.rs index fe55a8c1..026c6644 100644 --- a/src/lexer/token.rs +++ b/src/lexer/token.rs @@ -282,7 +282,7 @@ impl Display for TokenKind { Self::EndSwitch => "endswitch", Self::EndWhile => "endwhile", Self::Enum => "enum", - Self::Eof => "", + Self::Eof => "[end of file]", Self::Equals => "=", Self::Extends => "extends", Self::False => "false", @@ -382,8 +382,10 @@ impl Display for TokenKind { Self::Interface => "interface", Self::NamespaceConstant => "__NAMESPACE__", Self::PowEquals => "**=", + Self::Variable(v) => { + return write!(f, "${}", v); + } Self::StringPart(v) - | Self::Variable(v) | Self::QualifiedIdentifier(v) | Self::Identifier(v) | Self::FullyQualifiedIdentifier(v) diff --git a/src/parser/ast.rs b/src/parser/ast.rs index 3c53862d..f52d8ca9 100644 --- a/src/parser/ast.rs +++ b/src/parser/ast.rs @@ -32,12 +32,39 @@ pub enum Type { Mixed, Callable, Iterable, + StaticReference, + SelfReference, + ParentReference, } impl Type { pub fn standalone(&self) -> bool { matches!(self, Type::Mixed | Type::Never | Type::Void) } + + pub fn nullable(&self) -> bool { + matches!(self, Type::Nullable(_)) + } + + pub fn includes_callable(&self) -> bool { + match &self { + Self::Callable => true, + Self::Union(types) | Self::Intersection(types) => { + types.iter().any(|x| x.includes_callable()) + } + _ => false, + } + } + + pub fn includes_class_scoped(&self) -> bool { + match &self { + Self::StaticReference | Self::SelfReference | Self::ParentReference => true, + Self::Union(types) | Self::Intersection(types) => { + types.iter().any(|x| x.includes_class_scoped()) + } + _ => false, + } + } } impl Display for Type { @@ -77,6 +104,9 @@ impl Display for Type { Type::Mixed => write!(f, "mixed"), Type::Callable => write!(f, "callable"), Type::Iterable => write!(f, "iterable"), + Type::StaticReference => write!(f, "static"), + Type::SelfReference => write!(f, "self"), + Type::ParentReference => write!(f, "parent"), } } } diff --git a/src/parser/error.rs b/src/parser/error.rs index bebb0da6..870ca937 100644 --- a/src/parser/error.rs +++ b/src/parser/error.rs @@ -15,6 +15,7 @@ pub enum ParseError { StandaloneTypeUsedInCombination(Type, Span), TryWithoutCatchOrFinally(Span), VariadicPromotedProperty(Span), + MissingTypeForReadonlyProperty(String, String, Span), PromotedPropertyOutsideConstructor(Span), PromotedPropertyOnAbstractConstructor(Span), AbstractModifierOnNonAbstractClassMethod(Span), @@ -27,6 +28,8 @@ pub enum ParseError { FinalModifierOnPrivateConstant(Span), FinalModifierOnAbstractClass(Span), UnpredictableState(Span), + StaticPropertyUsingReadonlyModifier(String, String, Span), + ReadonlyPropertyHasDefaultValue(String, String, Span), } impl Display for ParseError { @@ -47,11 +50,12 @@ impl Display for ParseError { None => write!(f, "Parse Error: unexpected end of file, expecting {} on line {} column {}", expected, span.0, span.1), } }, + Self::MissingTypeForReadonlyProperty(class, prop, span) => write!(f, "Parse Error: Readonly property {}::${} must have type on line {} column {}", class, prop, span.0, span.1), Self::MultipleModifiers(modifier, span) => write!(f, "Parse Error: Multiple {} modifiers are not allowed on line {} column {}", modifier, span.0, span.1), Self::MultipleAccessModifiers( span) => write!(f, "Parse Error: Multiple access type modifiers are not allowed on line {} column {}", span.0, span.1), Self::UnexpectedToken(message, span) => write!(f, "Parse Error: Unexpected token {} on line {} column {}", message, span.0, span.1), Self::UnexpectedEndOfFile => write!(f, "Parse Error: unexpected end of file."), - Self::FinalModifierOnAbstractClassMember(span) => write!(f, "Parse Error: Cannot use the final modifier on an abstract class member on line {} column {}", span.0, span.1), + Self::FinalModifierOnAbstractClassMember(span) => write!(f, "Parse Error: Cannot use 'final' as an abstract class member modifier on line {} column {}", span.0, span.1), Self::StaticModifierOnConstant(span) => write!(f, "Parse Error: Cannot use 'static' as constant modifier on line {} column {}", span.0, span.1), Self::ReadonlyModifierOnConstant(span) => write!(f, "Parse Error: Cannot use 'readonly' as constant modifier on line {} column {}", span.0, span.1), Self::FinalModifierOnPrivateConstant(span) => write!(f, "Parse Error: Private constant cannot be final as it is not visible to other classes on line {} column {}", span.0, span.1), @@ -65,7 +69,9 @@ impl Display for ParseError { Self::ConstructorInEnum(name, span) => write!(f, "Parse Error: Enum '{}' cannot have a constructor on line {} column {}", name, span.0, span.1), Self::MissingCaseValueForBackedEnum(case, name, span) => write!(f, "Parse Error: Case `{}` of backed enum `{}` must have a value on line {} column {}", case, name, span.0, span.1), Self::CaseValueForUnitEnum(case, name, span) => write!(f, "Parse Error: Case `{}` of unit enum `{}` must not have a value on line {} column {}", case, name, span.0, span.1), - Self::UnpredictableState(span) => write!(f, "Parse Error: Reached an unpredictable state on line {} column {}", span.0, span.1) + Self::UnpredictableState(span) => write!(f, "Parse Error: Reached an unpredictable state on line {} column {}", span.0, span.1), + Self::StaticPropertyUsingReadonlyModifier(class, prop, span) => write!(f, "Parse Error: Static property {}:${} cannot be readonly on line {} column {}", class, prop, span.0, span.1), + Self::ReadonlyPropertyHasDefaultValue(class, prop, span) => write!(f, "Parse Error: Readonly property {}:${} cannot have a default value on line {} column {}", class, prop, span.0, span.1), } } } diff --git a/src/parser/internal/classish_statement.rs b/src/parser/internal/classish_statement.rs index 0f6eac1c..cd5985a2 100644 --- a/src/parser/internal/classish_statement.rs +++ b/src/parser/internal/classish_statement.rs @@ -2,6 +2,7 @@ use crate::expected_scope; use crate::lexer::token::TokenKind; use crate::parser::ast::Identifier; use crate::parser::ast::MethodFlag; +use crate::parser::ast::PropertyFlag; use crate::parser::ast::Statement; use crate::parser::ast::TraitAdaptation; use crate::parser::error::ParseError; @@ -12,7 +13,6 @@ use crate::parser::state::State; use crate::parser::Parser; use crate::expect_token; -use crate::expected_token_err; use crate::peek_token; impl Parser { @@ -123,15 +123,22 @@ impl Parser { let member_flags = self.class_members_flags(state)?; - match &state.current.kind { - TokenKind::Const => self.parse_classish_const(state, member_flags), - TokenKind::Function => self.method( + if state.current.kind == TokenKind::Const { + return self.parse_classish_const(state, member_flags); + } + + if state.current.kind == TokenKind::Function { + return self.method( state, member_flags.iter().map(|t| t.clone().into()).collect(), - ), - // TODO - TokenKind::Variable(_) => { - let var = self.var(state)?; + ); + } + + let ty = self.get_optional_type(state)?; + + expect_token!([ + TokenKind::Variable(var) => { + let flags: Vec = member_flags.into_iter().map(|f| f.into()).collect(); let mut value = None; if state.current.kind == TokenKind::Equals { @@ -141,56 +148,42 @@ impl Parser { self.semi(state)?; - Ok(Statement::Property { - var, - value, - r#type: None, - flags: member_flags.into_iter().map(|f| f.into()).collect(), - }) - } - TokenKind::Question - | TokenKind::Identifier(_) - | TokenKind::QualifiedIdentifier(_) - | TokenKind::FullyQualifiedIdentifier(_) - | TokenKind::Array - | TokenKind::Null => { - let prop_type = self.type_string(state)?; - let var = self.var(state)?; - let mut value = None; + if flags.contains(&PropertyFlag::Readonly) { + if flags.contains(&PropertyFlag::Static) { + let class_name: String = expected_scope!([ + Scope::Class(name, _) => state.named(&name), + Scope::Trait(name) => state.named(&name), + Scope::AnonymousClass => state.named(&"class@anonymous".into()), + ], state); - if state.current.kind == TokenKind::Equals { - state.next(); - value = Some(self.expression(state, Precedence::Lowest)?); - } + return Err(ParseError::StaticPropertyUsingReadonlyModifier(class_name, var.to_string(), state.current.span)); + } - // TODO: Support comma-separated property declarations. - // nikic/php-parser does this with a single Property statement - // that is capable of holding multiple property declarations. - self.semi(state)?; + if value.is_some() { + let class_name: String = expected_scope!([ + Scope::Class(name, _) => state.named(&name), + Scope::Trait(name) => state.named(&name), + Scope::AnonymousClass => state.named(&"class@anonymous".into()), + ], state); + + return Err(ParseError::ReadonlyPropertyHasDefaultValue(class_name, var.to_string(), state.current.span)); + } + } Ok(Statement::Property { var, value, - r#type: Some(prop_type), - flags: member_flags.into_iter().map(|f| f.into()).collect(), + r#type: ty, + flags, }) } - _ => expected_token_err!( - ["`const`", "`function`", "an identifier", "a varaible"], - state - ), - } + ], state, ["a varaible"]) } fn parse_classish_var(&self, state: &mut State) -> ParseResult { state.next(); - let mut var_type = None; - - if !matches!(state.current.kind, TokenKind::Variable(_)) { - var_type = Some(self.type_string(state)?); - } - + let ty = self.get_optional_type(state)?; let var = self.var(state)?; let mut value = None; @@ -205,7 +198,7 @@ impl Parser { Ok(Statement::Var { var, value, - r#type: var_type, + r#type: ty, }) } @@ -238,10 +231,8 @@ impl Parser { _ => (None, self.ident(state)?.into()), }; - match state.current.kind { + expect_token!([ TokenKind::As => { - state.next(); - match state.current.kind { TokenKind::Public | TokenKind::Protected | TokenKind::Private => { let visibility: MethodFlag = state.current.kind.clone().into(); @@ -273,10 +264,8 @@ impl Parser { }); } } - } + }, TokenKind::Insteadof => { - state.next(); - let mut insteadof = Vec::new(); insteadof.push(self.full_name(state)?.into()); while state.current.kind != TokenKind::SemiColon { @@ -290,13 +279,7 @@ impl Parser { insteadof, }); } - _ => { - return Err(ParseError::UnexpectedToken( - state.current.kind.to_string(), - state.current.span, - )) - } - }; + ], state, ["`as`", "`insteadof`"]); self.semi(state)?; } diff --git a/src/parser/internal/functions.rs b/src/parser/internal/functions.rs index b393726e..186f8fbb 100644 --- a/src/parser/internal/functions.rs +++ b/src/parser/internal/functions.rs @@ -79,7 +79,7 @@ impl Parser { if state.current.kind == TokenKind::Colon { self.colon(state)?; - return_type = Some(self.type_string(state)?); + return_type = Some(self.get_type(state)?); } self.lbrace(state)?; @@ -128,7 +128,7 @@ impl Parser { if state.current.kind == TokenKind::Colon { self.colon(state)?; - return_type = Some(self.type_string(state)?); + return_type = Some(self.get_type(state)?); } expect_token!([TokenKind::DoubleArrow], state, ["`=>`"]); @@ -169,7 +169,7 @@ impl Parser { if state.current.kind == TokenKind::Colon { self.colon(state)?; - return_type = Some(self.type_string(state)?); + return_type = Some(self.get_type(state)?); } self.lbrace(state)?; @@ -241,7 +241,7 @@ impl Parser { if state.current.kind == TokenKind::Colon { self.colon(state)?; - return_type = Some(self.type_string(state)?); + return_type = Some(self.get_type(state)?); } if !has_body { diff --git a/src/parser/internal/ident.rs b/src/parser/internal/ident.rs index 3000caf1..c5c8faf1 100644 --- a/src/parser/internal/ident.rs +++ b/src/parser/internal/ident.rs @@ -37,31 +37,6 @@ impl Parser { ], state, "a variable")) } - pub(in crate::parser) fn full_name_maybe_type_keyword( - &self, - state: &mut State, - ) -> ParseResult { - match state.current.kind { - TokenKind::Array | TokenKind::Callable => { - let r = Ok(state.current.kind.to_string().into()); - state.next(); - r - } - _ => self.full_name(state), - } - } - - pub(in crate::parser) fn type_with_static(&self, state: &mut State) -> ParseResult { - Ok(match state.current.kind { - TokenKind::Static | TokenKind::Null | TokenKind::True | TokenKind::False => { - let str = state.current.kind.to_string(); - state.next(); - str.into() - } - _ => self.full_name_maybe_type_keyword(state)?, - }) - } - pub(in crate::parser) fn ident_maybe_reserved( &self, state: &mut State, diff --git a/src/parser/internal/params.rs b/src/parser/internal/params.rs index d62a9da6..ec59c1b4 100644 --- a/src/parser/internal/params.rs +++ b/src/parser/internal/params.rs @@ -18,6 +18,7 @@ impl Parser { pub(in crate::parser) fn param_list(&self, state: &mut State) -> Result { let mut params = ParamList::new(); + let mut class_name = String::new(); let construct: i8 = match state.scope()? { Scope::Function(_) | Scope::AnonymousFunction(_) | Scope::ArrowFunction(_) => 0, Scope::Method(name, flags) => { @@ -28,13 +29,19 @@ impl Parser { // can only have abstract ctor Scope::Interface(_) => 1, // can only have concret ctor - Scope::AnonymousClass => 2, + Scope::AnonymousClass => { + class_name = state.named(&"class@anonymous".into()); + + 2 + } // can have either abstract or concret ctor, // depens on method flag. - Scope::Class(_, _) | Scope::Trait(_) => { + Scope::Class(name, _) | Scope::Trait(name) => { if flags.contains(&MethodFlag::Abstract) { 1 } else { + class_name = state.named(name); + 2 } } @@ -70,15 +77,12 @@ impl Parser { } } - // If this is a readonly promoted property, or we don't see a variable - if flags.contains(&PropertyFlag::Readonly) - || !matches!( - state.current.kind, - TokenKind::Variable(_) | TokenKind::Ellipsis | TokenKind::Ampersand - ) - { + if !matches!( + state.current.kind, + TokenKind::Variable(_) | TokenKind::Ellipsis | TokenKind::Ampersand + ) { // Try to parse the type. - param_type = Some(self.type_string(state)?); + param_type = Some(self.get_type(state)?); } let mut variadic = false; @@ -103,6 +107,14 @@ impl Parser { TokenKind::Variable(v) => v ], state, "a varaible"); + if flags.contains(&PropertyFlag::Readonly) && param_type.is_none() { + return Err(ParseError::MissingTypeForReadonlyProperty( + class_name, + var.to_string(), + state.current.span, + )); + } + let mut default = None; if state.current.kind == TokenKind::Equals { state.next(); diff --git a/src/parser/internal/types.rs b/src/parser/internal/types.rs index 68d82262..d4071e02 100644 --- a/src/parser/internal/types.rs +++ b/src/parser/internal/types.rs @@ -1,4 +1,4 @@ -use crate::lexer::byte_string::ByteString; +use crate::expected_token; use crate::lexer::token::TokenKind; use crate::parser::ast::TryBlockCaughtType; use crate::parser::ast::Type; @@ -36,39 +36,36 @@ impl Parser { Ok(TryBlockCaughtType::Identifier(id.into())) } - pub(in crate::parser) fn type_string(&self, state: &mut State) -> ParseResult { - if state.current.kind == TokenKind::Question { - state.next(); - let t = self.type_with_static(state)?; - return Ok(Type::Nullable(Box::new(parse_simple_type(t)))); - } + pub(in crate::parser) fn get_type(&self, state: &mut State) -> ParseResult { + let ty = self.maybe_nullable(state, &|state| { + self.maybe_static(state, &|state| self.get_simple_type(state)) + })?; - let id = self.type_with_static(state)?; + if ty.nullable() { + return Ok(ty); + } if state.current.kind == TokenKind::Pipe { state.next(); - let r#type = parse_simple_type(id); - if r#type.standalone() { + if ty.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( - r#type, + ty, state.current.span, )); } - let mut types = vec![r#type]; - + let mut types = vec![ty]; while !state.is_eof() { - let id = self.type_with_static(state)?; - let r#type = parse_simple_type(id); - if r#type.standalone() { + let ty = self.maybe_static(state, &|state| self.get_simple_type(state))?; + if ty.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( - r#type, + ty, state.current.span, )); } - types.push(r#type); + types.push(ty); if state.current.kind != TokenKind::Pipe { break; @@ -85,27 +82,24 @@ impl Parser { { state.next(); - let r#type = parse_simple_type(id); - if r#type.standalone() { + if ty.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( - r#type, + ty, state.current.span, )); } - let mut types = vec![r#type]; - + let mut types = vec![ty]; while !state.is_eof() { - let id = self.type_with_static(state)?; - let r#type = parse_simple_type(id); - if r#type.standalone() { + let ty = self.maybe_static(state, &|state| self.get_simple_type(state))?; + if ty.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( - r#type, + ty, state.current.span, )); } - types.push(r#type); + types.push(ty); if state.current.kind != TokenKind::Ampersand { break; @@ -117,28 +111,248 @@ impl Parser { return Ok(Type::Intersection(types)); } - Ok(parse_simple_type(id)) + Ok(ty) } -} -fn parse_simple_type(id: ByteString) -> Type { - let name = &id[..]; - let lowered_name = name.to_ascii_lowercase(); - match lowered_name.as_slice() { - b"void" => Type::Void, - b"never" => Type::Never, - b"null" => Type::Null, - b"true" => Type::True, - b"false" => Type::False, - b"float" => Type::Float, - b"bool" => Type::Boolean, - b"int" => Type::Integer, - b"string" => Type::String, - b"array" => Type::Array, - b"object" => Type::Object, - b"mixed" => Type::Mixed, - b"iterable" => Type::Iterable, - b"callable" => Type::Callable, - _ => Type::Identifier(id.into()), + pub(in crate::parser) fn get_optional_type( + &self, + state: &mut State, + ) -> ParseResult> { + let ty = self.maybe_optional_nullable(state, &|state| { + self.maybe_optional_static(state, &|state| self.get_optional_simple_type(state)) + }); + + match ty { + Some(ty) => { + if ty.nullable() { + return Ok(Some(ty)); + } + + if state.current.kind == TokenKind::Pipe { + state.next(); + + if ty.standalone() { + return Err(ParseError::StandaloneTypeUsedInCombination( + ty, + state.current.span, + )); + } + + let mut types = vec![ty]; + while !state.is_eof() { + let ty = self.maybe_static(state, &|state| self.get_simple_type(state))?; + if ty.standalone() { + return Err(ParseError::StandaloneTypeUsedInCombination( + ty, + state.current.span, + )); + } + + types.push(ty); + + if state.current.kind != TokenKind::Pipe { + break; + } else { + state.next(); + } + } + + return Ok(Some(Type::Union(types))); + } + + if state.current.kind == TokenKind::Ampersand + && !matches!(state.peek.kind, TokenKind::Variable(_)) + { + state.next(); + + if ty.standalone() { + return Err(ParseError::StandaloneTypeUsedInCombination( + ty, + state.current.span, + )); + } + + let mut types = vec![ty]; + while !state.is_eof() { + let ty = self.maybe_static(state, &|state| self.get_simple_type(state))?; + if ty.standalone() { + return Err(ParseError::StandaloneTypeUsedInCombination( + ty, + state.current.span, + )); + } + + types.push(ty); + + if state.current.kind != TokenKind::Ampersand { + break; + } else { + state.next(); + } + } + + return Ok(Some(Type::Intersection(types))); + } + + Ok(Some(ty)) + } + None => Ok(None), + } + } + + fn get_optional_simple_type(&self, state: &mut State) -> Option { + match state.current.kind.clone() { + TokenKind::Array => { + state.next(); + + Some(Type::Array) + } + TokenKind::Callable => { + state.next(); + + Some(Type::Callable) + } + TokenKind::Null => { + state.next(); + + Some(Type::Null) + } + TokenKind::True => { + state.next(); + + Some(Type::True) + } + TokenKind::False => { + state.next(); + + Some(Type::False) + } + TokenKind::Identifier(id) => { + state.next(); + + let name = &id[..]; + let lowered_name = name.to_ascii_lowercase(); + match lowered_name.as_slice() { + b"void" => Some(Type::Void), + b"never" => Some(Type::Never), + b"float" => Some(Type::Float), + b"bool" => Some(Type::Boolean), + b"int" => Some(Type::Integer), + b"string" => Some(Type::String), + b"object" => Some(Type::Object), + b"mixed" => Some(Type::Mixed), + b"iterable" => Some(Type::Iterable), + b"null" => Some(Type::Null), + b"true" => Some(Type::True), + b"false" => Some(Type::False), + b"array" => Some(Type::Array), + b"callable" => Some(Type::Callable), + _ => Some(Type::Identifier(id.into())), + } + } + TokenKind::QualifiedIdentifier(id) | TokenKind::FullyQualifiedIdentifier(id) => { + state.next(); + + Some(Type::Identifier(id.into())) + } + _ => None, + } + } + + fn get_simple_type(&self, state: &mut State) -> ParseResult { + self.get_optional_simple_type(state) + .ok_or_else(|| expected_token!(["a type"], state)) + } + + fn maybe_nullable( + &self, + state: &mut State, + otherwise: &(dyn Fn(&mut State) -> ParseResult), + ) -> ParseResult { + if state.current.kind == TokenKind::Question { + state.next(); + + Ok(Type::Nullable(Box::new(otherwise(state)?))) + } else { + otherwise(state) + } + } + + fn maybe_static( + &self, + state: &mut State, + otherwise: &(dyn Fn(&mut State) -> ParseResult), + ) -> ParseResult { + if TokenKind::Static == state.current.kind { + state.next(); + + return Ok(Type::StaticReference); + } + + if let TokenKind::Identifier(id) = &state.current.kind { + let name = &id[..]; + let lowered_name = name.to_ascii_lowercase(); + match lowered_name.as_slice() { + b"self" => { + state.next(); + + return Ok(Type::SelfReference); + } + b"parent" => { + state.next(); + + return Ok(Type::ParentReference); + } + _ => {} + }; + } + + otherwise(state) + } + + fn maybe_optional_nullable( + &self, + state: &mut State, + otherwise: &(dyn Fn(&mut State) -> Option), + ) -> Option { + if state.current.kind == TokenKind::Question { + state.next(); + + Some(Type::Nullable(Box::new(otherwise(state)?))) + } else { + otherwise(state) + } + } + + fn maybe_optional_static( + &self, + state: &mut State, + otherwise: &(dyn Fn(&mut State) -> Option), + ) -> Option { + if TokenKind::Static == state.current.kind { + state.next(); + + return Some(Type::StaticReference); + } + + if let TokenKind::Identifier(id) = &state.current.kind { + let name = &id[..]; + let lowered_name = name.to_ascii_lowercase(); + match lowered_name.as_slice() { + b"self" => { + state.next(); + + return Some(Type::SelfReference); + } + b"parent" => { + state.next(); + + return Some(Type::ParentReference); + } + _ => {} + }; + } + + otherwise(state) } } diff --git a/src/parser/internal/vars.rs b/src/parser/internal/vars.rs index 19920ca5..976f9e7b 100644 --- a/src/parser/internal/vars.rs +++ b/src/parser/internal/vars.rs @@ -1,16 +1,16 @@ use crate::lexer::token::TokenKind; use crate::parser::ast::Expression; -use crate::parser::error::ParseError; use crate::parser::error::ParseResult; use crate::parser::internal::precedence::Precedence; use crate::parser::state::State; use crate::parser::Parser; +use crate::peek_token; impl Parser { pub(in crate::parser) fn dynamic_variable(&self, state: &mut State) -> ParseResult { state.next(); - Ok(match &state.current.kind { + let expr = peek_token!([ TokenKind::LeftBrace => { state.next(); @@ -21,9 +21,9 @@ impl Parser { Expression::DynamicVariable { name: Box::new(name), } - } + }, TokenKind::Variable(variable) => { - let variable = variable.clone(); + let variable = variable; state.next(); @@ -31,12 +31,8 @@ impl Parser { name: Box::new(Expression::Variable { name: variable }), } } - _ => { - return Err(ParseError::UnexpectedToken( - state.current.kind.to_string(), - state.current.span, - )) - } - }) + ], state, ["`{`", "a variable"]); + + Ok(expr) } } diff --git a/src/parser/macros.rs b/src/parser/macros.rs index b49e7176..23041926 100644 --- a/src/parser/macros.rs +++ b/src/parser/macros.rs @@ -70,27 +70,38 @@ macro_rules! expect_literal { #[macro_export] macro_rules! expected_token_err { + ([ $($expected:literal),+ $(,)? ], $state:expr $(,)?) => {{ + Err($crate::expected_token!([$($expected),+], $state)) + }}; + + ($expected:literal, $state:expr $(,)?) => { + $crate::expected_token_err!([$expected], $state) + }; +} + +#[macro_export] +macro_rules! expected_token { ([ $($expected:literal),+ $(,)? ], $state:expr $(,)?) => {{ match &$state.current.kind { TokenKind::Eof => { - Err($crate::parser::error::ParseError::ExpectedToken( + $crate::parser::error::ParseError::ExpectedToken( vec![$($expected.into()),+], None, $state.current.span, - )) + ) }, _ => { - Err($crate::parser::error::ParseError::ExpectedToken( + $crate::parser::error::ParseError::ExpectedToken( vec![$($expected.into()),+], Some($state.current.kind.to_string()), $state.current.span, - )) + ) } } }}; ($expected:literal, $state:expr $(,)?) => { - $crate::expected_token_err!([$expected], $state) + $crate::expected_token!([$expected], $state) }; } diff --git a/tests/0004/parser-error.txt b/tests/0004/parser-error.txt index 242969bd..c0e3cf84 100644 --- a/tests/0004/parser-error.txt +++ b/tests/0004/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["an identifier"], Some("e"), (6, 14)) -> Parse Error: unexpected token `e`, expecting an identifier on line 6 column 14 +ExpectedToken(["an identifier"], Some("$e"), (6, 14)) -> Parse Error: unexpected token `$e`, expecting an identifier on line 6 column 14 diff --git a/tests/0044/parser-error.txt b/tests/0044/parser-error.txt index 950193f0..83ebbfe2 100644 --- a/tests/0044/parser-error.txt +++ b/tests/0044/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["`const`", "`function`", "an identifier", "a varaible"], Some("fn"), (1, 26)) -> Parse Error: unexpected token `fn`, expecting `const`, `function`, an identifier, or a varaible on line 1 column 26 +ExpectedToken(["a varaible"], Some("fn"), (1, 26)) -> Parse Error: unexpected token `fn`, expecting a varaible on line 1 column 26 diff --git a/tests/0111/parser-error.txt b/tests/0111/parser-error.txt index 5369330f..3b913366 100644 --- a/tests/0111/parser-error.txt +++ b/tests/0111/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["an identifier"], Some("..."), (5, 25)) -> Parse Error: unexpected token `...`, expecting an identifier on line 5 column 25 +VariadicPromotedProperty((5, 28)) -> Parse Error: Cannot declare variadic promoted property on line 5 column 28 diff --git a/tests/0112/parser-error.txt b/tests/0112/parser-error.txt index 872d6967..1e796a73 100644 --- a/tests/0112/parser-error.txt +++ b/tests/0112/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["an identifier"], Some("&"), (5, 25)) -> Parse Error: unexpected token `&`, expecting an identifier on line 5 column 25 +MissingTypeForReadonlyProperty("foo", "e", (5, 28)) -> Parse Error: Readonly property foo::$e must have type on line 5 column 28 diff --git a/tests/0124/parser-error.txt b/tests/0124/parser-error.txt index a3fdb76c..2f3898af 100644 --- a/tests/0124/parser-error.txt +++ b/tests/0124/parser-error.txt @@ -1 +1 @@ -FinalModifierOnAbstractClassMember((4, 11)) -> Parse Error: Cannot use the final modifier on an abstract class member on line 4 column 11 +FinalModifierOnAbstractClassMember((4, 11)) -> Parse Error: Cannot use 'final' as an abstract class member modifier on line 4 column 11 diff --git a/tests/0128/parser-error.txt b/tests/0128/parser-error.txt index c9c94267..53dc2568 100644 --- a/tests/0128/parser-error.txt +++ b/tests/0128/parser-error.txt @@ -1 +1 @@ -ExpectedToken(["an identifier"], Some("s"), (5, 25)) -> Parse Error: unexpected token `s`, expecting an identifier on line 5 column 25 +MissingTypeForReadonlyProperty("foo", "s", (5, 28)) -> Parse Error: Readonly property foo::$s must have type on line 5 column 28 diff --git a/tests/0149/ast.txt b/tests/0149/ast.txt new file mode 100644 index 00000000..9a41f989 --- /dev/null +++ b/tests/0149/ast.txt @@ -0,0 +1,49 @@ +[ + Namespace { + name: "A\B\C\D\E", + body: [ + Noop, + Function { + name: Identifier { + name: "foo", + }, + params: [ + Param { + name: Variable { + name: "s", + }, + type: Some( + String, + ), + variadic: false, + default: None, + flags: [], + by_ref: false, + }, + ], + body: [ + Expression { + expr: Call { + target: Identifier { + name: "exit", + }, + args: [ + Arg { + name: None, + value: LiteralInteger { + i: 0, + }, + unpack: false, + }, + ], + }, + }, + ], + return_type: Some( + SelfReference, + ), + by_ref: false, + }, + ], + }, +] diff --git a/tests/0149/code.php b/tests/0149/code.php new file mode 100644 index 00000000..c93b8784 --- /dev/null +++ b/tests/0149/code.php @@ -0,0 +1,7 @@ + Parse Error: Readonly property Foo\Bar\Baz::$name must have type on line 7 column 32 diff --git a/tests/0152/tokens.txt b/tests/0152/tokens.txt new file mode 100644 index 00000000..b2fb3286 --- /dev/null +++ b/tests/0152/tokens.txt @@ -0,0 +1,152 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\Bar", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 18, + ), + }, + Token { + kind: Final, + span: ( + 5, + 1, + ), + }, + Token { + kind: Class, + span: ( + 5, + 7, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 5, + 13, + ), + }, + Token { + kind: LeftBrace, + span: ( + 5, + 17, + ), + }, + Token { + kind: Public, + span: ( + 6, + 6, + ), + }, + Token { + kind: Function, + span: ( + 6, + 13, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 6, + 22, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 33, + ), + }, + Token { + kind: Public, + span: ( + 7, + 11, + ), + }, + Token { + kind: Readonly, + span: ( + 7, + 18, + ), + }, + Token { + kind: Variable( + "name", + ), + span: ( + 7, + 27, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 32, + ), + }, + Token { + kind: RightParen, + span: ( + 8, + 6, + ), + }, + Token { + kind: LeftBrace, + span: ( + 8, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 9, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, +] diff --git a/tests/0153/ast.txt b/tests/0153/ast.txt new file mode 100644 index 00000000..3505def4 --- /dev/null +++ b/tests/0153/ast.txt @@ -0,0 +1,52 @@ +[ + Namespace { + name: "Foo\Bar", + body: [ + Noop, + Class { + name: Identifier { + name: "Baz", + }, + extends: None, + implements: [], + body: [ + Method { + name: Identifier { + name: "__construct", + }, + params: [ + Param { + name: Variable { + name: "name", + }, + type: Some( + String, + ), + variadic: false, + default: Some( + LiteralString { + value: "foo", + }, + ), + flags: [ + Public, + Readonly, + ], + by_ref: false, + }, + ], + body: [], + flags: [ + Public, + ], + return_type: None, + by_ref: false, + }, + ], + flags: [ + Final, + ], + }, + ], + }, +] diff --git a/tests/0153/code.php b/tests/0153/code.php new file mode 100644 index 00000000..0e033676 --- /dev/null +++ b/tests/0153/code.php @@ -0,0 +1,9 @@ + Parse Error: Static property Foo\Bar\Baz:$foo cannot be readonly on line 7 column 1 diff --git a/tests/0154/tokens.txt b/tests/0154/tokens.txt new file mode 100644 index 00000000..07a5b0b2 --- /dev/null +++ b/tests/0154/tokens.txt @@ -0,0 +1,117 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\Bar", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 18, + ), + }, + Token { + kind: Final, + span: ( + 5, + 1, + ), + }, + Token { + kind: Class, + span: ( + 5, + 7, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 5, + 13, + ), + }, + Token { + kind: LeftBrace, + span: ( + 5, + 17, + ), + }, + Token { + kind: Public, + span: ( + 6, + 6, + ), + }, + Token { + kind: Readonly, + span: ( + 6, + 13, + ), + }, + Token { + kind: Static, + span: ( + 6, + 22, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 6, + 29, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 6, + 36, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 40, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0155/ast.txt b/tests/0155/ast.txt new file mode 100644 index 00000000..4f0362a7 --- /dev/null +++ b/tests/0155/ast.txt @@ -0,0 +1,31 @@ +[ + Namespace { + name: "Foo\Bar", + body: [ + Noop, + Class { + name: Identifier { + name: "Baz", + }, + extends: None, + implements: [], + body: [ + Property { + var: "foo", + value: None, + type: Some( + String, + ), + flags: [ + Public, + Readonly, + ], + }, + ], + flags: [ + Final, + ], + }, + ], + }, +] diff --git a/tests/0155/code.php b/tests/0155/code.php new file mode 100644 index 00000000..1ac23bb3 --- /dev/null +++ b/tests/0155/code.php @@ -0,0 +1,7 @@ + Parse Error: Readonly property Foo\Bar\Baz:$foo cannot have a default value on line 7 column 1 diff --git a/tests/0156/tokens.txt b/tests/0156/tokens.txt new file mode 100644 index 00000000..94dac44f --- /dev/null +++ b/tests/0156/tokens.txt @@ -0,0 +1,126 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\Bar", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 18, + ), + }, + Token { + kind: Final, + span: ( + 5, + 1, + ), + }, + Token { + kind: Class, + span: ( + 5, + 7, + ), + }, + Token { + kind: Identifier( + "Baz", + ), + span: ( + 5, + 13, + ), + }, + Token { + kind: LeftBrace, + span: ( + 5, + 17, + ), + }, + Token { + kind: Public, + span: ( + 6, + 6, + ), + }, + Token { + kind: Readonly, + span: ( + 6, + 13, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 6, + 22, + ), + }, + Token { + kind: Variable( + "foo", + ), + span: ( + 6, + 29, + ), + }, + Token { + kind: Equals, + span: ( + 6, + 34, + ), + }, + Token { + kind: LiteralString( + "foo", + ), + span: ( + 6, + 36, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 41, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] From 9f87dae3768184284aca3e222fd8f44644e6b74f Mon Sep 17 00:00:00 2001 From: Saif Eddin Gmati <29315886+azjezz@users.noreply.github.com> Date: Fri, 2 Dec 2022 19:00:16 +0100 Subject: [PATCH 07/13] fix: namespace parser Signed-off-by: azjezz --- src/parser/error.rs | 4 + src/parser/internal/ident.rs | 19 +++- src/parser/internal/mod.rs | 1 + src/parser/internal/namespace.rs | 86 +++++++++++++++++ src/parser/mod.rs | 52 +---------- src/parser/state.rs | 59 ++++++++---- tests/0157/ast.txt | 18 ++++ tests/0157/code.php | 5 + tests/0157/tokens.txt | 85 +++++++++++++++++ tests/0158/ast.txt | 32 +++++++ tests/0158/code.php | 9 ++ tests/0158/tokens.txt | 145 +++++++++++++++++++++++++++++ tests/0159/code.php | 9 ++ tests/0159/parser-error.txt | 1 + tests/0159/tokens.txt | 152 +++++++++++++++++++++++++++++++ tests/0160/code.php | 9 ++ tests/0160/parser-error.txt | 1 + tests/0160/tokens.txt | 152 +++++++++++++++++++++++++++++++ tests/0161/code.php | 9 ++ tests/0161/parser-error.txt | 1 + tests/0161/tokens.txt | 152 +++++++++++++++++++++++++++++++ tests/0162/code.php | 9 ++ tests/0162/parser-error.txt | 1 + tests/0162/tokens.txt | 143 +++++++++++++++++++++++++++++ tests/0163/code.php | 5 + tests/0163/parser-error.txt | 1 + tests/0163/tokens.txt | 69 ++++++++++++++ tests/0164/ast.txt | 32 +++++++ tests/0164/code.php | 9 ++ tests/0164/tokens.txt | 150 ++++++++++++++++++++++++++++++ 30 files changed, 1350 insertions(+), 70 deletions(-) create mode 100644 src/parser/internal/namespace.rs create mode 100644 tests/0157/ast.txt create mode 100644 tests/0157/code.php create mode 100644 tests/0157/tokens.txt create mode 100644 tests/0158/ast.txt create mode 100644 tests/0158/code.php create mode 100644 tests/0158/tokens.txt create mode 100644 tests/0159/code.php create mode 100644 tests/0159/parser-error.txt create mode 100644 tests/0159/tokens.txt create mode 100644 tests/0160/code.php create mode 100644 tests/0160/parser-error.txt create mode 100644 tests/0160/tokens.txt create mode 100644 tests/0161/code.php create mode 100644 tests/0161/parser-error.txt create mode 100644 tests/0161/tokens.txt create mode 100644 tests/0162/code.php create mode 100644 tests/0162/parser-error.txt create mode 100644 tests/0162/tokens.txt create mode 100644 tests/0163/code.php create mode 100644 tests/0163/parser-error.txt create mode 100644 tests/0163/tokens.txt create mode 100644 tests/0164/ast.txt create mode 100644 tests/0164/code.php create mode 100644 tests/0164/tokens.txt diff --git a/src/parser/error.rs b/src/parser/error.rs index 870ca937..00e9edb5 100644 --- a/src/parser/error.rs +++ b/src/parser/error.rs @@ -30,6 +30,8 @@ pub enum ParseError { UnpredictableState(Span), StaticPropertyUsingReadonlyModifier(String, String, Span), ReadonlyPropertyHasDefaultValue(String, String, Span), + MixingBracedAndUnBracedNamespaceDeclarations(Span), + NestedNamespaceDeclarations(Span), } impl Display for ParseError { @@ -72,6 +74,8 @@ impl Display for ParseError { Self::UnpredictableState(span) => write!(f, "Parse Error: Reached an unpredictable state on line {} column {}", span.0, span.1), Self::StaticPropertyUsingReadonlyModifier(class, prop, span) => write!(f, "Parse Error: Static property {}:${} cannot be readonly on line {} column {}", class, prop, span.0, span.1), Self::ReadonlyPropertyHasDefaultValue(class, prop, span) => write!(f, "Parse Error: Readonly property {}:${} cannot have a default value on line {} column {}", class, prop, span.0, span.1), + Self::MixingBracedAndUnBracedNamespaceDeclarations(span) => write!(f, "Parse Error: Cannot mix braced namespace declarations with unbraced namespace declarations on line {} column {}", span.0, span.1), + Self::NestedNamespaceDeclarations(span) => write!(f, "Parse Error: Namespace declarations cannot be mixed on line {} column {}", span.0, span.1), } } } diff --git a/src/parser/internal/ident.rs b/src/parser/internal/ident.rs index c5c8faf1..9c0a0036 100644 --- a/src/parser/internal/ident.rs +++ b/src/parser/internal/ident.rs @@ -16,10 +16,21 @@ impl Parser { /// Expect an unqualified or qualified identifier such as Foo, Bar or Foo\Bar. pub(in crate::parser) fn name(&self, state: &mut State) -> ParseResult { - Ok(expect_token!([ - TokenKind::Identifier(identifier) => identifier, - TokenKind::QualifiedIdentifier(qualified) => qualified, - ], state, "an identifier")) + expect_token!([ + TokenKind::Identifier(name) | TokenKind::QualifiedIdentifier(name) => Ok(name), + ], state, "an identifier") + } + + /// Expect an optional unqualified or qualified identifier such as Foo, Bar or Foo\Bar. + pub(in crate::parser) fn optional_name(&self, state: &mut State) -> Option { + match state.current.kind.clone() { + TokenKind::Identifier(name) | TokenKind::QualifiedIdentifier(name) => { + state.next(); + + Some(name) + } + _ => None, + } } /// Expect an unqualified, qualified or fully qualified identifier such as Foo, Foo\Bar or \Foo\Bar. diff --git a/src/parser/internal/mod.rs b/src/parser/internal/mod.rs index 27d965dd..88a216bf 100644 --- a/src/parser/internal/mod.rs +++ b/src/parser/internal/mod.rs @@ -4,6 +4,7 @@ pub mod classish_statement; pub mod flags; pub mod functions; pub mod ident; +pub mod namespace; pub mod params; pub mod precedence; pub mod punc; diff --git a/src/parser/internal/namespace.rs b/src/parser/internal/namespace.rs new file mode 100644 index 00000000..ffdc7b5d --- /dev/null +++ b/src/parser/internal/namespace.rs @@ -0,0 +1,86 @@ +use crate::lexer::token::TokenKind; +use crate::parser::ast::Block; +use crate::parser::ast::Statement; +use crate::parser::error::ParseError; +use crate::parser::error::ParseResult; +use crate::parser::state::NamespaceType; +use crate::parser::state::Scope; +use crate::parser::state::State; +use crate::parser::Parser; +use crate::prelude::ByteString; +use crate::scoped; + +impl Parser { + pub(in crate::parser) fn namespace(&self, state: &mut State) -> ParseResult { + state.next(); + + let name = self.optional_name(state); + + if let Some(name) = &name { + if state.current.kind != TokenKind::LeftBrace { + match state.namespace_type() { + Some(NamespaceType::Braced) => { + return Err(ParseError::MixingBracedAndUnBracedNamespaceDeclarations( + state.current.span, + )); + } + Some(NamespaceType::Unbraced) => { + // exit the current namespace scope. + // we don't need to check if the current scope is a namespace + // because we know it is, it is not possible for it to be anything else. + // as using `namespace` anywhere aside from a top-level stmt would result + // in a parse error. + state.exit(); + } + _ => {} + } + + return self.unbraced_namespace(state, name.clone()); + } + } + + match state.namespace_type() { + Some(NamespaceType::Unbraced) => Err( + ParseError::MixingBracedAndUnBracedNamespaceDeclarations(state.current.span), + ), + Some(NamespaceType::Braced) if state.namespace().is_some() => { + Err(ParseError::NestedNamespaceDeclarations(state.current.span)) + } + _ => self.braced_namespace(state, name), + } + } + + fn unbraced_namespace(&self, state: &mut State, name: ByteString) -> ParseResult { + let body = scoped!(state, Scope::Namespace(name.clone()), { + let mut body = Block::new(); + while !state.is_eof() { + body.push(self.top_level_statement(state)?); + } + + Ok(body) + })?; + + Ok(Statement::Namespace { name, body }) + } + + fn braced_namespace( + &self, + state: &mut State, + name: Option, + ) -> ParseResult { + self.lbrace(state)?; + + let body = scoped!(state, Scope::BracedNamespace(name.clone()), { + let mut body = Block::new(); + while state.current.kind != TokenKind::RightBrace && !state.is_eof() { + body.push(self.top_level_statement(state)?); + } + + Ok(body) + })?; + + self.rbrace(state)?; + + Ok(Statement::BracedNamespace { name, body }) + } +} diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 63adba7b..ae0952b5 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -11,9 +11,7 @@ use crate::parser::error::ParseError; use crate::parser::error::ParseResult; use crate::parser::internal::ident::is_reserved_ident; use crate::parser::internal::precedence::{Associativity, Precedence}; -use crate::parser::state::Scope; use crate::parser::state::State; -use crate::scoped; pub mod ast; pub mod error; @@ -62,49 +60,7 @@ impl Parser { state.skip_comments(); let statement = match &state.current.kind { - TokenKind::Namespace => { - state.next(); - - if state.current.kind != TokenKind::LeftBrace { - let name = self.name(state)?; - - if state.current.kind == TokenKind::LeftBrace { - self.lbrace(state)?; - - let body = scoped!(state, Scope::BracedNamespace(Some(name.clone())), { - self.block(state, &TokenKind::RightBrace) - })?; - - self.rbrace(state)?; - - Statement::BracedNamespace { - name: Some(name), - body, - } - } else { - let body = scoped!(state, Scope::Namespace(name.clone()), { - let mut body = Block::new(); - while !state.is_eof() { - body.push(self.top_level_statement(state)?); - } - - Ok(body) - })?; - - Statement::Namespace { name, body } - } - } else { - self.lbrace(state)?; - - let body = scoped!(state, Scope::BracedNamespace(None), { - self.block(state, &TokenKind::RightBrace) - })?; - - self.rbrace(state)?; - - Statement::BracedNamespace { name: None, body } - } - } + TokenKind::Namespace => self.namespace(state)?, TokenKind::Use => { state.next(); @@ -521,11 +477,7 @@ impl Parser { }; let mut cases = Vec::new(); - loop { - if state.current.kind == end_token { - break; - } - + while state.current.kind != end_token { match state.current.kind { TokenKind::Case => { state.next(); diff --git a/src/parser/state.rs b/src/parser/state.rs index d9aace41..9b7a1ed4 100644 --- a/src/parser/state.rs +++ b/src/parser/state.rs @@ -9,6 +9,12 @@ use crate::parser::ast::MethodFlag; use crate::parser::error::ParseError; use crate::parser::error::ParseResult; +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum NamespaceType { + Braced, + Unbraced, +} + #[derive(Debug, PartialEq, Eq, Clone)] pub enum Scope { Namespace(ByteString), @@ -33,6 +39,7 @@ pub struct State { pub peek: Token, pub iter: IntoIter, pub comments: Vec, + pub namespace_type: Option, } impl State { @@ -45,30 +52,40 @@ impl State { peek: iter.next().unwrap_or_default(), iter, comments: vec![], + namespace_type: None, } } - pub fn named(&self, name: &ByteString) -> String { - let mut namespace = None; + /// Return the namespace type used in the current state + /// + /// The namespace type is retrieve from the last entered + /// namespace scope. + /// + /// Note: even when a namespace scope is exited, the namespace type + /// is retained, until the next namespace scope is entered. + pub fn namespace_type(&self) -> Option { + self.namespace_type.clone() + } + + pub fn namespace(&self) -> Option<&Scope> { for scope in &self.stack { match scope { - Scope::Namespace(n) => { - namespace = Some(n.to_string()); - - break; - } - Scope::BracedNamespace(n) => { - namespace = n.as_ref().map(|s| s.to_string()); - - break; + Scope::Namespace(_) | Scope::BracedNamespace(_) => { + return Some(scope); } _ => {} } } - match namespace { - Some(v) => format!("{}\\{}", v, name), - None => name.to_string(), + None + } + + pub fn named(&self, name: &ByteString) -> String { + match self.namespace() { + Some(Scope::Namespace(n)) | Some(Scope::BracedNamespace(Some(n))) => { + format!("{}\\{}", n, name) + } + _ => name.to_string(), } } @@ -84,8 +101,18 @@ impl State { .ok_or(ParseError::UnpredictableState(self.current.span)) } - pub fn enter(&mut self, state: Scope) { - self.stack.push_back(state); + pub fn enter(&mut self, scope: Scope) { + match &scope { + Scope::Namespace(_) => { + self.namespace_type = Some(NamespaceType::Unbraced); + } + Scope::BracedNamespace(_) => { + self.namespace_type = Some(NamespaceType::Braced); + } + _ => {} + } + + self.stack.push_back(scope); } pub fn exit(&mut self) { diff --git a/tests/0157/ast.txt b/tests/0157/ast.txt new file mode 100644 index 00000000..03d23e79 --- /dev/null +++ b/tests/0157/ast.txt @@ -0,0 +1,18 @@ +[ + BracedNamespace { + name: Some( + "Foo\Bar", + ), + body: [ + Function { + name: Identifier { + name: "foo", + }, + params: [], + body: [], + return_type: None, + by_ref: false, + }, + ], + }, +] diff --git a/tests/0157/code.php b/tests/0157/code.php new file mode 100644 index 00000000..9c107665 --- /dev/null +++ b/tests/0157/code.php @@ -0,0 +1,5 @@ + Parse Error: Cannot mix braced namespace declarations with unbraced namespace declarations on line 7 column 19 diff --git a/tests/0159/tokens.txt b/tests/0159/tokens.txt new file mode 100644 index 00000000..89a7dc9c --- /dev/null +++ b/tests/0159/tokens.txt @@ -0,0 +1,152 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\Bar", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 18, + ), + }, + Token { + kind: Function, + span: ( + 5, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 5, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 14, + ), + }, + Token { + kind: LeftBrace, + span: ( + 5, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 17, + ), + }, + Token { + kind: Namespace, + span: ( + 7, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\Baz", + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 7, + 19, + ), + }, + Token { + kind: Function, + span: ( + 8, + 5, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 8, + 14, + ), + }, + Token { + kind: LeftParen, + span: ( + 8, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 8, + 18, + ), + }, + Token { + kind: LeftBrace, + span: ( + 8, + 20, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, +] diff --git a/tests/0160/code.php b/tests/0160/code.php new file mode 100644 index 00000000..5687cf6b --- /dev/null +++ b/tests/0160/code.php @@ -0,0 +1,9 @@ + Parse Error: Cannot mix braced namespace declarations with unbraced namespace declarations on line 6 column 22 diff --git a/tests/0160/tokens.txt b/tests/0160/tokens.txt new file mode 100644 index 00000000..c4a787cb --- /dev/null +++ b/tests/0160/tokens.txt @@ -0,0 +1,152 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\Baz", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 19, + ), + }, + Token { + kind: Function, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 4, + 14, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 18, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 20, + ), + }, + Token { + kind: RightBrace, + span: ( + 4, + 21, + ), + }, + Token { + kind: Namespace, + span: ( + 6, + 5, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\Bar", + ), + span: ( + 6, + 15, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 22, + ), + }, + Token { + kind: Function, + span: ( + 8, + 5, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 8, + 14, + ), + }, + Token { + kind: LeftParen, + span: ( + 8, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 8, + 18, + ), + }, + Token { + kind: LeftBrace, + span: ( + 8, + 20, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, +] diff --git a/tests/0161/code.php b/tests/0161/code.php new file mode 100644 index 00000000..81e51d37 --- /dev/null +++ b/tests/0161/code.php @@ -0,0 +1,9 @@ + Parse Error: Cannot mix braced namespace declarations with unbraced namespace declarations on line 7 column 18 diff --git a/tests/0161/tokens.txt b/tests/0161/tokens.txt new file mode 100644 index 00000000..6b272e2d --- /dev/null +++ b/tests/0161/tokens.txt @@ -0,0 +1,152 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\Baz", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 19, + ), + }, + Token { + kind: Function, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 4, + 14, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 18, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 20, + ), + }, + Token { + kind: RightBrace, + span: ( + 4, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 7, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\Bar", + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 18, + ), + }, + Token { + kind: Function, + span: ( + 9, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 9, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 9, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 9, + 14, + ), + }, + Token { + kind: LeftBrace, + span: ( + 9, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 17, + ), + }, +] diff --git a/tests/0162/code.php b/tests/0162/code.php new file mode 100644 index 00000000..0fe97207 --- /dev/null +++ b/tests/0162/code.php @@ -0,0 +1,9 @@ + Parse Error: Cannot mix braced namespace declarations with unbraced namespace declarations on line 7 column 18 diff --git a/tests/0162/tokens.txt b/tests/0162/tokens.txt new file mode 100644 index 00000000..d2a89a40 --- /dev/null +++ b/tests/0162/tokens.txt @@ -0,0 +1,143 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Function, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 4, + 14, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 18, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 20, + ), + }, + Token { + kind: RightBrace, + span: ( + 4, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 7, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Foo\Bar", + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 18, + ), + }, + Token { + kind: Function, + span: ( + 9, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 9, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 9, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 9, + 14, + ), + }, + Token { + kind: LeftBrace, + span: ( + 9, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 17, + ), + }, +] diff --git a/tests/0163/code.php b/tests/0163/code.php new file mode 100644 index 00000000..ff0eca25 --- /dev/null +++ b/tests/0163/code.php @@ -0,0 +1,5 @@ + Parse Error: unexpected token `;`, expecting `{` on line 3 column 10 diff --git a/tests/0163/tokens.txt b/tests/0163/tokens.txt new file mode 100644 index 00000000..46fc5c52 --- /dev/null +++ b/tests/0163/tokens.txt @@ -0,0 +1,69 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Namespace, + span: ( + 3, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 10, + ), + }, + Token { + kind: Function, + span: ( + 5, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 5, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 14, + ), + }, + Token { + kind: LeftBrace, + span: ( + 5, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 17, + ), + }, +] diff --git a/tests/0164/ast.txt b/tests/0164/ast.txt new file mode 100644 index 00000000..a1699387 --- /dev/null +++ b/tests/0164/ast.txt @@ -0,0 +1,32 @@ +[ + BracedNamespace { + name: None, + body: [ + Function { + name: Identifier { + name: "foo", + }, + params: [], + body: [], + return_type: None, + by_ref: false, + }, + ], + }, + BracedNamespace { + name: Some( + "a", + ), + body: [ + Function { + name: Identifier { + name: "foo", + }, + params: [], + body: [], + return_type: None, + by_ref: false, + }, + ], + }, +] diff --git a/tests/0164/code.php b/tests/0164/code.php new file mode 100644 index 00000000..f1378308 --- /dev/null +++ b/tests/0164/code.php @@ -0,0 +1,9 @@ + Date: Fri, 2 Dec 2022 22:38:43 +0100 Subject: [PATCH 08/13] fix: make sure that properties have the correct type --- src/parser/ast.rs | 4 + src/parser/error.rs | 4 +- src/parser/internal/classish_statement.rs | 41 ++++-- src/parser/internal/params.rs | 72 +++++----- src/parser/internal/types.rs | 35 ++--- tests/0086/ast.txt | 21 --- tests/0086/parser-error.txt | 1 + tests/0113/parser-error.txt | 2 +- tests/0114/parser-error.txt | 2 +- tests/0115/parser-error.txt | 2 +- tests/0116/parser-error.txt | 2 +- tests/0154/parser-error.txt | 2 +- tests/0156/parser-error.txt | 2 +- tests/0165/0166/code.php | 7 + tests/0165/0166/parser-error.txt | 1 + tests/0165/0166/tokens.txt | 124 +++++++++++++++++ tests/0165/0167/code.php | 7 + tests/0165/0167/parser-error.txt | 1 + tests/0165/0167/tokens.txt | 124 +++++++++++++++++ tests/0165/0168/code.php | 7 + tests/0165/0168/parser-error.txt | 1 + tests/0165/0168/tokens.txt | 156 ++++++++++++++++++++++ tests/0165/code.php | 7 + tests/0165/parser-error.txt | 1 + tests/0165/tokens.txt | 124 +++++++++++++++++ tests/0166/code.php | 7 + tests/0166/parser-error.txt | 1 + tests/0166/tokens.txt | 124 +++++++++++++++++ tests/0167/code.php | 7 + tests/0167/parser-error.txt | 1 + tests/0167/tokens.txt | 124 +++++++++++++++++ tests/0168/code.php | 7 + tests/0168/parser-error.txt | 1 + tests/0168/tokens.txt | 156 ++++++++++++++++++++++ tests/0169/code.php | 5 + tests/0169/parser-error.txt | 1 + tests/0169/tokens.txt | 73 ++++++++++ tests/0170/code.php | 5 + tests/0170/parser-error.txt | 1 + tests/0170/tokens.txt | 73 ++++++++++ tests/0171/code.php | 5 + tests/0171/parser-error.txt | 1 + tests/0171/tokens.txt | 105 +++++++++++++++ tests/0172/code.php | 5 + tests/0172/parser-error.txt | 1 + tests/0172/tokens.txt | 73 ++++++++++ tests/0173/code.php | 5 + tests/0173/parser-error.txt | 1 + tests/0173/tokens.txt | 80 +++++++++++ 49 files changed, 1518 insertions(+), 94 deletions(-) delete mode 100644 tests/0086/ast.txt create mode 100644 tests/0086/parser-error.txt create mode 100644 tests/0165/0166/code.php create mode 100644 tests/0165/0166/parser-error.txt create mode 100644 tests/0165/0166/tokens.txt create mode 100644 tests/0165/0167/code.php create mode 100644 tests/0165/0167/parser-error.txt create mode 100644 tests/0165/0167/tokens.txt create mode 100644 tests/0165/0168/code.php create mode 100644 tests/0165/0168/parser-error.txt create mode 100644 tests/0165/0168/tokens.txt create mode 100644 tests/0165/code.php create mode 100644 tests/0165/parser-error.txt create mode 100644 tests/0165/tokens.txt create mode 100644 tests/0166/code.php create mode 100644 tests/0166/parser-error.txt create mode 100644 tests/0166/tokens.txt create mode 100644 tests/0167/code.php create mode 100644 tests/0167/parser-error.txt create mode 100644 tests/0167/tokens.txt create mode 100644 tests/0168/code.php create mode 100644 tests/0168/parser-error.txt create mode 100644 tests/0168/tokens.txt create mode 100644 tests/0169/code.php create mode 100644 tests/0169/parser-error.txt create mode 100644 tests/0169/tokens.txt create mode 100644 tests/0170/code.php create mode 100644 tests/0170/parser-error.txt create mode 100644 tests/0170/tokens.txt create mode 100644 tests/0171/code.php create mode 100644 tests/0171/parser-error.txt create mode 100644 tests/0171/tokens.txt create mode 100644 tests/0172/code.php create mode 100644 tests/0172/parser-error.txt create mode 100644 tests/0172/tokens.txt create mode 100644 tests/0173/code.php create mode 100644 tests/0173/parser-error.txt create mode 100644 tests/0173/tokens.txt diff --git a/src/parser/ast.rs b/src/parser/ast.rs index f52d8ca9..7fa0307b 100644 --- a/src/parser/ast.rs +++ b/src/parser/ast.rs @@ -65,6 +65,10 @@ impl Type { _ => false, } } + + pub fn is_bottom(&self) -> bool { + matches!(self, Type::Never | Type::Void) + } } impl Display for Type { diff --git a/src/parser/error.rs b/src/parser/error.rs index 00e9edb5..a201f392 100644 --- a/src/parser/error.rs +++ b/src/parser/error.rs @@ -32,6 +32,7 @@ pub enum ParseError { ReadonlyPropertyHasDefaultValue(String, String, Span), MixingBracedAndUnBracedNamespaceDeclarations(Span), NestedNamespaceDeclarations(Span), + ForbiddenTypeUsedInProperty(String, String, Type, Span), } impl Display for ParseError { @@ -71,11 +72,12 @@ impl Display for ParseError { Self::ConstructorInEnum(name, span) => write!(f, "Parse Error: Enum '{}' cannot have a constructor on line {} column {}", name, span.0, span.1), Self::MissingCaseValueForBackedEnum(case, name, span) => write!(f, "Parse Error: Case `{}` of backed enum `{}` must have a value on line {} column {}", case, name, span.0, span.1), Self::CaseValueForUnitEnum(case, name, span) => write!(f, "Parse Error: Case `{}` of unit enum `{}` must not have a value on line {} column {}", case, name, span.0, span.1), - Self::UnpredictableState(span) => write!(f, "Parse Error: Reached an unpredictable state on line {} column {}", span.0, span.1), Self::StaticPropertyUsingReadonlyModifier(class, prop, span) => write!(f, "Parse Error: Static property {}:${} cannot be readonly on line {} column {}", class, prop, span.0, span.1), Self::ReadonlyPropertyHasDefaultValue(class, prop, span) => write!(f, "Parse Error: Readonly property {}:${} cannot have a default value on line {} column {}", class, prop, span.0, span.1), Self::MixingBracedAndUnBracedNamespaceDeclarations(span) => write!(f, "Parse Error: Cannot mix braced namespace declarations with unbraced namespace declarations on line {} column {}", span.0, span.1), Self::NestedNamespaceDeclarations(span) => write!(f, "Parse Error: Namespace declarations cannot be mixed on line {} column {}", span.0, span.1), + Self::UnpredictableState(span) => write!(f, "Parse Error: Reached an unpredictable state on line {} column {}", span.0, span.1), + Self::ForbiddenTypeUsedInProperty(class, prop, ty, span) => write!(f, "Parse Error: Property {}::${} cannot have type `{}` on line {} column {}", class, prop, ty, span.0, span.1), } } } diff --git a/src/parser/internal/classish_statement.rs b/src/parser/internal/classish_statement.rs index cd5985a2..c8e8ac48 100644 --- a/src/parser/internal/classish_statement.rs +++ b/src/parser/internal/classish_statement.rs @@ -146,30 +146,47 @@ impl Parser { value = Some(self.expression(state, Precedence::Lowest)?); } - self.semi(state)?; + let class_name: String = expected_scope!([ + Scope::Class(name, _) => state.named(&name), + Scope::Trait(name) => state.named(&name), + Scope::AnonymousClass => state.named(&"class@anonymous".into()), + ], state); if flags.contains(&PropertyFlag::Readonly) { if flags.contains(&PropertyFlag::Static) { - let class_name: String = expected_scope!([ - Scope::Class(name, _) => state.named(&name), - Scope::Trait(name) => state.named(&name), - Scope::AnonymousClass => state.named(&"class@anonymous".into()), - ], state); - return Err(ParseError::StaticPropertyUsingReadonlyModifier(class_name, var.to_string(), state.current.span)); } if value.is_some() { - let class_name: String = expected_scope!([ - Scope::Class(name, _) => state.named(&name), - Scope::Trait(name) => state.named(&name), - Scope::AnonymousClass => state.named(&"class@anonymous".into()), - ], state); return Err(ParseError::ReadonlyPropertyHasDefaultValue(class_name, var.to_string(), state.current.span)); } } + match &ty { + Some(ty) => { + if ty.includes_callable() || ty.is_bottom() { + return Err(ParseError::ForbiddenTypeUsedInProperty( + class_name, + var.to_string(), + ty.clone(), + state.current.span, + )); + } + } + None => { + if flags.contains(&PropertyFlag::Readonly) { + return Err(ParseError::MissingTypeForReadonlyProperty( + class_name, + var.to_string(), + state.current.span, + )); + } + } + } + + self.semi(state)?; + Ok(Statement::Property { var, value, diff --git a/src/parser/internal/params.rs b/src/parser/internal/params.rs index ec59c1b4..56ee1279 100644 --- a/src/parser/internal/params.rs +++ b/src/parser/internal/params.rs @@ -53,37 +53,13 @@ impl Parser { }; while !state.is_eof() && state.current.kind != TokenKind::RightParen { - let mut param_type = None; - let flags: Vec = self .promoted_property_flags(state)? .iter() .map(|f| f.into()) .collect(); - if !flags.is_empty() { - match construct { - 0 => { - return Err(ParseError::PromotedPropertyOutsideConstructor( - state.current.span, - )); - } - 1 => { - return Err(ParseError::PromotedPropertyOnAbstractConstructor( - state.current.span, - )); - } - _ => {} - } - } - - if !matches!( - state.current.kind, - TokenKind::Variable(_) | TokenKind::Ellipsis | TokenKind::Ampersand - ) { - // Try to parse the type. - param_type = Some(self.get_type(state)?); - } + let ty = self.get_optional_type(state)?; let mut variadic = false; let mut by_ref = false; @@ -107,12 +83,42 @@ impl Parser { TokenKind::Variable(v) => v ], state, "a varaible"); - if flags.contains(&PropertyFlag::Readonly) && param_type.is_none() { - return Err(ParseError::MissingTypeForReadonlyProperty( - class_name, - var.to_string(), - state.current.span, - )); + if !flags.is_empty() { + match construct { + 0 => { + return Err(ParseError::PromotedPropertyOutsideConstructor( + state.current.span, + )); + } + 1 => { + return Err(ParseError::PromotedPropertyOnAbstractConstructor( + state.current.span, + )); + } + _ => {} + } + + match &ty { + Some(ty) => { + if ty.includes_callable() || ty.is_bottom() { + return Err(ParseError::ForbiddenTypeUsedInProperty( + class_name, + var.to_string(), + ty.clone(), + state.current.span, + )); + } + } + None => { + if flags.contains(&PropertyFlag::Readonly) { + return Err(ParseError::MissingTypeForReadonlyProperty( + class_name, + var.to_string(), + state.current.span, + )); + } + } + } } let mut default = None; @@ -123,13 +129,15 @@ impl Parser { params.push(Param { name: Expression::Variable { name: var }, - r#type: param_type, + r#type: ty, variadic, default, flags, by_ref, }); + // TODO: bug! this allows `function foo(string $a ...$b &$c) {}` + // TODO: if `,` is found, look for next param, otherwise break out of the loop. self.optional_comma(state)?; } diff --git a/src/parser/internal/types.rs b/src/parser/internal/types.rs index d4071e02..8af34bf3 100644 --- a/src/parser/internal/types.rs +++ b/src/parser/internal/types.rs @@ -118,16 +118,14 @@ impl Parser { &self, state: &mut State, ) -> ParseResult> { - let ty = self.maybe_optional_nullable(state, &|state| { - self.maybe_optional_static(state, &|state| self.get_optional_simple_type(state)) - }); + if state.current.kind == TokenKind::Question { + return Ok(Some(self.get_type(state)?)); + } + + let ty = self.maybe_optional_static(state, &|state| self.get_optional_simple_type(state)); match ty { Some(ty) => { - if ty.nullable() { - return Ok(Some(ty)); - } - if state.current.kind == TokenKind::Pipe { state.next(); @@ -271,8 +269,15 @@ impl Parser { ) -> ParseResult { if state.current.kind == TokenKind::Question { state.next(); + let inner = otherwise(state)?; + if inner.standalone() { + return Err(ParseError::StandaloneTypeUsedInCombination( + inner, + state.current.span, + )); + } - Ok(Type::Nullable(Box::new(otherwise(state)?))) + Ok(Type::Nullable(Box::new(inner))) } else { otherwise(state) } @@ -310,20 +315,6 @@ impl Parser { otherwise(state) } - fn maybe_optional_nullable( - &self, - state: &mut State, - otherwise: &(dyn Fn(&mut State) -> Option), - ) -> Option { - if state.current.kind == TokenKind::Question { - state.next(); - - Some(Type::Nullable(Box::new(otherwise(state)?))) - } else { - otherwise(state) - } - } - fn maybe_optional_static( &self, state: &mut State, diff --git a/tests/0086/ast.txt b/tests/0086/ast.txt deleted file mode 100644 index 6ddd97b2..00000000 --- a/tests/0086/ast.txt +++ /dev/null @@ -1,21 +0,0 @@ -[ - Class { - name: Identifier { - name: "Foo", - }, - extends: None, - implements: [], - body: [ - Property { - var: "bar", - value: None, - type: None, - flags: [ - Public, - Readonly, - ], - }, - ], - flags: [], - }, -] diff --git a/tests/0086/parser-error.txt b/tests/0086/parser-error.txt new file mode 100644 index 00000000..43df73db --- /dev/null +++ b/tests/0086/parser-error.txt @@ -0,0 +1 @@ +MissingTypeForReadonlyProperty("Foo", "bar", (1, 39)) -> Parse Error: Readonly property Foo::$bar must have type on line 1 column 39 diff --git a/tests/0113/parser-error.txt b/tests/0113/parser-error.txt index da1607de..63d82ac1 100644 --- a/tests/0113/parser-error.txt +++ b/tests/0113/parser-error.txt @@ -1 +1 @@ -PromotedPropertyOutsideConstructor((5, 16)) -> Parse Error: Cannot declare promoted property outside a constructor on line 5 column 16 +PromotedPropertyOutsideConstructor((5, 25)) -> Parse Error: Cannot declare promoted property outside a constructor on line 5 column 25 diff --git a/tests/0114/parser-error.txt b/tests/0114/parser-error.txt index 9eec84bf..5dbd2f76 100644 --- a/tests/0114/parser-error.txt +++ b/tests/0114/parser-error.txt @@ -1 +1 @@ -PromotedPropertyOnAbstractConstructor((5, 16)) -> Parse Error: Cannot declare promoted property in an abstract constructor on line 5 column 16 +PromotedPropertyOnAbstractConstructor((5, 25)) -> Parse Error: Cannot declare promoted property in an abstract constructor on line 5 column 25 diff --git a/tests/0115/parser-error.txt b/tests/0115/parser-error.txt index 9eec84bf..5dbd2f76 100644 --- a/tests/0115/parser-error.txt +++ b/tests/0115/parser-error.txt @@ -1 +1 @@ -PromotedPropertyOnAbstractConstructor((5, 16)) -> Parse Error: Cannot declare promoted property in an abstract constructor on line 5 column 16 +PromotedPropertyOnAbstractConstructor((5, 25)) -> Parse Error: Cannot declare promoted property in an abstract constructor on line 5 column 25 diff --git a/tests/0116/parser-error.txt b/tests/0116/parser-error.txt index 9eec84bf..5dbd2f76 100644 --- a/tests/0116/parser-error.txt +++ b/tests/0116/parser-error.txt @@ -1 +1 @@ -PromotedPropertyOnAbstractConstructor((5, 16)) -> Parse Error: Cannot declare promoted property in an abstract constructor on line 5 column 16 +PromotedPropertyOnAbstractConstructor((5, 25)) -> Parse Error: Cannot declare promoted property in an abstract constructor on line 5 column 25 diff --git a/tests/0154/parser-error.txt b/tests/0154/parser-error.txt index e5ed2581..8e362edd 100644 --- a/tests/0154/parser-error.txt +++ b/tests/0154/parser-error.txt @@ -1 +1 @@ -StaticPropertyUsingReadonlyModifier("Foo\\Bar\\Baz", "foo", (7, 1)) -> Parse Error: Static property Foo\Bar\Baz:$foo cannot be readonly on line 7 column 1 +StaticPropertyUsingReadonlyModifier("Foo\\Bar\\Baz", "foo", (6, 40)) -> Parse Error: Static property Foo\Bar\Baz:$foo cannot be readonly on line 6 column 40 diff --git a/tests/0156/parser-error.txt b/tests/0156/parser-error.txt index 349560f4..b84d9493 100644 --- a/tests/0156/parser-error.txt +++ b/tests/0156/parser-error.txt @@ -1 +1 @@ -ReadonlyPropertyHasDefaultValue("Foo\\Bar\\Baz", "foo", (7, 1)) -> Parse Error: Readonly property Foo\Bar\Baz:$foo cannot have a default value on line 7 column 1 +ReadonlyPropertyHasDefaultValue("Foo\\Bar\\Baz", "foo", (6, 41)) -> Parse Error: Readonly property Foo\Bar\Baz:$foo cannot have a default value on line 6 column 41 diff --git a/tests/0165/0166/code.php b/tests/0165/0166/code.php new file mode 100644 index 00000000..01f0d3c2 --- /dev/null +++ b/tests/0165/0166/code.php @@ -0,0 +1,7 @@ + Parse Error: Property Foo::$s cannot have type `void` on line 5 column 23 diff --git a/tests/0165/0166/tokens.txt b/tests/0165/0166/tokens.txt new file mode 100644 index 00000000..d57a33d5 --- /dev/null +++ b/tests/0165/0166/tokens.txt @@ -0,0 +1,124 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "void", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 5, + 21, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 23, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0165/0167/code.php b/tests/0165/0167/code.php new file mode 100644 index 00000000..b9d53459 --- /dev/null +++ b/tests/0165/0167/code.php @@ -0,0 +1,7 @@ + Parse Error: Property Foo::$s cannot have type `never` on line 5 column 24 diff --git a/tests/0165/0167/tokens.txt b/tests/0165/0167/tokens.txt new file mode 100644 index 00000000..af824054 --- /dev/null +++ b/tests/0165/0167/tokens.txt @@ -0,0 +1,124 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "never", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 5, + 22, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 24, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0165/0168/code.php b/tests/0165/0168/code.php new file mode 100644 index 00000000..e926c5ea --- /dev/null +++ b/tests/0165/0168/code.php @@ -0,0 +1,7 @@ + Parse Error: Property Foo::$s cannot have type `string|int|callable` on line 5 column 38 diff --git a/tests/0165/0168/tokens.txt b/tests/0165/0168/tokens.txt new file mode 100644 index 00000000..889cabed --- /dev/null +++ b/tests/0165/0168/tokens.txt @@ -0,0 +1,156 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Pipe, + span: ( + 5, + 22, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 5, + 23, + ), + }, + Token { + kind: Pipe, + span: ( + 5, + 26, + ), + }, + Token { + kind: Identifier( + "callable", + ), + span: ( + 5, + 27, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 5, + 36, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 38, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0165/code.php b/tests/0165/code.php new file mode 100644 index 00000000..d6804c16 --- /dev/null +++ b/tests/0165/code.php @@ -0,0 +1,7 @@ + Parse Error: Property Foo::$s cannot have type `callable` on line 5 column 27 diff --git a/tests/0165/tokens.txt b/tests/0165/tokens.txt new file mode 100644 index 00000000..aea37dd8 --- /dev/null +++ b/tests/0165/tokens.txt @@ -0,0 +1,124 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "callable", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 5, + 25, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 27, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0166/code.php b/tests/0166/code.php new file mode 100644 index 00000000..01f0d3c2 --- /dev/null +++ b/tests/0166/code.php @@ -0,0 +1,7 @@ + Parse Error: Property Foo::$s cannot have type `void` on line 5 column 23 diff --git a/tests/0166/tokens.txt b/tests/0166/tokens.txt new file mode 100644 index 00000000..d57a33d5 --- /dev/null +++ b/tests/0166/tokens.txt @@ -0,0 +1,124 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "void", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 5, + 21, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 23, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0167/code.php b/tests/0167/code.php new file mode 100644 index 00000000..b9d53459 --- /dev/null +++ b/tests/0167/code.php @@ -0,0 +1,7 @@ + Parse Error: Property Foo::$s cannot have type `never` on line 5 column 24 diff --git a/tests/0167/tokens.txt b/tests/0167/tokens.txt new file mode 100644 index 00000000..af824054 --- /dev/null +++ b/tests/0167/tokens.txt @@ -0,0 +1,124 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "never", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 5, + 22, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 24, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0168/code.php b/tests/0168/code.php new file mode 100644 index 00000000..e926c5ea --- /dev/null +++ b/tests/0168/code.php @@ -0,0 +1,7 @@ + Parse Error: Property Foo::$s cannot have type `string|int|callable` on line 5 column 38 diff --git a/tests/0168/tokens.txt b/tests/0168/tokens.txt new file mode 100644 index 00000000..889cabed --- /dev/null +++ b/tests/0168/tokens.txt @@ -0,0 +1,156 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "__construct", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 32, + ), + }, + Token { + kind: Public, + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Pipe, + span: ( + 5, + 22, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 5, + 23, + ), + }, + Token { + kind: Pipe, + span: ( + 5, + 26, + ), + }, + Token { + kind: Identifier( + "callable", + ), + span: ( + 5, + 27, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 5, + 36, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 38, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 7, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 8, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0169/code.php b/tests/0169/code.php new file mode 100644 index 00000000..074db080 --- /dev/null +++ b/tests/0169/code.php @@ -0,0 +1,5 @@ + Parse Error: Property Foo::$s cannot have type `callable` on line 4 column 23 diff --git a/tests/0169/tokens.txt b/tests/0169/tokens.txt new file mode 100644 index 00000000..3e08aeb0 --- /dev/null +++ b/tests/0169/tokens.txt @@ -0,0 +1,73 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "callable", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 23, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0170/code.php b/tests/0170/code.php new file mode 100644 index 00000000..dec93cbe --- /dev/null +++ b/tests/0170/code.php @@ -0,0 +1,5 @@ + Parse Error: Property Foo::$s cannot have type `void` on line 4 column 19 diff --git a/tests/0170/tokens.txt b/tests/0170/tokens.txt new file mode 100644 index 00000000..9ed70d2c --- /dev/null +++ b/tests/0170/tokens.txt @@ -0,0 +1,73 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "void", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 4, + 17, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 19, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0171/code.php b/tests/0171/code.php new file mode 100644 index 00000000..110d7d43 --- /dev/null +++ b/tests/0171/code.php @@ -0,0 +1,5 @@ + Parse Error: Property Foo::$s cannot have type `string|int|callable` on line 4 column 34 diff --git a/tests/0171/tokens.txt b/tests/0171/tokens.txt new file mode 100644 index 00000000..f696c802 --- /dev/null +++ b/tests/0171/tokens.txt @@ -0,0 +1,105 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Pipe, + span: ( + 4, + 18, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 4, + 19, + ), + }, + Token { + kind: Pipe, + span: ( + 4, + 22, + ), + }, + Token { + kind: Identifier( + "callable", + ), + span: ( + 4, + 23, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 4, + 32, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 34, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0172/code.php b/tests/0172/code.php new file mode 100644 index 00000000..9ff2c7bb --- /dev/null +++ b/tests/0172/code.php @@ -0,0 +1,5 @@ + Parse Error: Property Foo::$s cannot have type `never` on line 4 column 20 diff --git a/tests/0172/tokens.txt b/tests/0172/tokens.txt new file mode 100644 index 00000000..ffc0ad49 --- /dev/null +++ b/tests/0172/tokens.txt @@ -0,0 +1,73 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "never", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 4, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 20, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0173/code.php b/tests/0173/code.php new file mode 100644 index 00000000..0d1b4833 --- /dev/null +++ b/tests/0173/code.php @@ -0,0 +1,5 @@ + Parse error: 'never' can only be used as a standalone type on line 4 column 19 diff --git a/tests/0173/tokens.txt b/tests/0173/tokens.txt new file mode 100644 index 00000000..73767ce0 --- /dev/null +++ b/tests/0173/tokens.txt @@ -0,0 +1,80 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "Foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Question, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "never", + ), + span: ( + 4, + 13, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 4, + 19, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] From 6a5c1ec6252f3dd6b1de078fac813151c509fcc9 Mon Sep 17 00:00:00 2001 From: Saif Eddin Gmati <29315886+azjezz@users.noreply.github.com> Date: Fri, 2 Dec 2022 23:16:27 +0100 Subject: [PATCH 09/13] fix: require comma between parameters --- src/parser/internal/functions.rs | 16 ---- src/parser/internal/params.rs | 12 ++- src/parser/internal/punc.rs | 26 +++---- tests/0174/code.php | 6 ++ tests/0174/parser-error.txt | 1 + tests/0174/tokens.txt | 89 +++++++++++++++++++++ tests/0175/code.php | 7 ++ tests/0175/parser-error.txt | 1 + tests/0175/tokens.txt | 116 +++++++++++++++++++++++++++ tests/0176/ast.txt | 48 ++++++++++++ tests/0176/code.php | 7 ++ tests/0176/tokens.txt | 130 +++++++++++++++++++++++++++++++ tests/0177/ast.txt | 48 ++++++++++++ tests/0177/code.php | 7 ++ tests/0177/tokens.txt | 123 +++++++++++++++++++++++++++++ 15 files changed, 603 insertions(+), 34 deletions(-) create mode 100644 tests/0174/code.php create mode 100644 tests/0174/parser-error.txt create mode 100644 tests/0174/tokens.txt create mode 100644 tests/0175/code.php create mode 100644 tests/0175/parser-error.txt create mode 100644 tests/0175/tokens.txt create mode 100644 tests/0176/ast.txt create mode 100644 tests/0176/code.php create mode 100644 tests/0176/tokens.txt create mode 100644 tests/0177/ast.txt create mode 100644 tests/0177/code.php create mode 100644 tests/0177/tokens.txt diff --git a/src/parser/internal/functions.rs b/src/parser/internal/functions.rs index 186f8fbb..d1224441 100644 --- a/src/parser/internal/functions.rs +++ b/src/parser/internal/functions.rs @@ -37,12 +37,8 @@ impl Parser { }; scoped!(state, Scope::AnonymousFunction(is_static), { - self.lparen(state)?; - let params = self.param_list(state)?; - self.rparen(state)?; - let mut uses = vec![]; if state.current.kind == TokenKind::Use { state.next(); @@ -118,12 +114,8 @@ impl Parser { }; scoped!(state, Scope::ArrowFunction(is_static), { - self.lparen(state)?; - let params = self.param_list(state)?; - self.rparen(state)?; - let mut return_type = None; if state.current.kind == TokenKind::Colon { self.colon(state)?; @@ -158,12 +150,8 @@ impl Parser { let name = self.ident(state)?; scoped!(state, Scope::Function(name.clone()), { - self.lparen(state)?; - let params = self.param_list(state)?; - self.rparen(state)?; - let mut return_type = None; if state.current.kind == TokenKind::Colon { @@ -230,12 +218,8 @@ impl Parser { ], state); scoped!(state, Scope::Method(name.clone(), flags.clone()), { - self.lparen(state)?; - let params = self.param_list(state)?; - self.rparen(state)?; - let mut return_type = None; if state.current.kind == TokenKind::Colon { diff --git a/src/parser/internal/params.rs b/src/parser/internal/params.rs index 56ee1279..232adcd7 100644 --- a/src/parser/internal/params.rs +++ b/src/parser/internal/params.rs @@ -52,6 +52,8 @@ impl Parser { _ => unreachable!(), }; + self.lparen(state)?; + while !state.is_eof() && state.current.kind != TokenKind::RightParen { let flags: Vec = self .promoted_property_flags(state)? @@ -136,11 +138,15 @@ impl Parser { by_ref, }); - // TODO: bug! this allows `function foo(string $a ...$b &$c) {}` - // TODO: if `,` is found, look for next param, otherwise break out of the loop. - self.optional_comma(state)?; + if state.current.kind == TokenKind::Comma { + self.comma(state)?; + } else { + break; + } } + self.rparen(state)?; + Ok(params) } diff --git a/src/parser/internal/punc.rs b/src/parser/internal/punc.rs index e8a060d5..da779596 100644 --- a/src/parser/internal/punc.rs +++ b/src/parser/internal/punc.rs @@ -7,33 +7,31 @@ use crate::expect_token; impl Parser { pub(in crate::parser) fn semi(&self, state: &mut State) -> ParseResult<()> { - expect_token!([TokenKind::SemiColon], state, "`;`"); - Ok(()) + expect_token!([TokenKind::SemiColon => Ok(())], state, "`;`") } pub(in crate::parser) fn lbrace(&self, state: &mut State) -> ParseResult<()> { - expect_token!([TokenKind::LeftBrace], state, "`{`"); - Ok(()) + expect_token!([TokenKind::LeftBrace => Ok(())], state, "`{`") } pub(in crate::parser) fn rbrace(&self, state: &mut State) -> ParseResult<()> { - expect_token!([TokenKind::RightBrace], state, "`}`"); - Ok(()) + expect_token!([TokenKind::RightBrace => Ok(())], state, "`}`") } pub(in crate::parser) fn lparen(&self, state: &mut State) -> ParseResult<()> { - expect_token!([TokenKind::LeftParen], state, "`(`"); - Ok(()) + expect_token!([TokenKind::LeftParen => Ok(())], state, "`(`") } pub(in crate::parser) fn rparen(&self, state: &mut State) -> ParseResult<()> { - expect_token!([TokenKind::RightParen], state, "`)`"); - Ok(()) + expect_token!([TokenKind::RightParen => Ok(())], state, "`)`") } pub(in crate::parser) fn rbracket(&self, state: &mut State) -> ParseResult<()> { - expect_token!([TokenKind::RightBracket], state, "`]`"); - Ok(()) + expect_token!([TokenKind::RightBracket => Ok(())], state, "`]`") + } + + pub(in crate::parser) fn comma(&self, state: &mut State) -> ParseResult<()> { + expect_token!([TokenKind::Comma => Ok(())], state, "`,`") } pub(in crate::parser) fn optional_comma(&self, state: &mut State) -> ParseResult<()> { @@ -45,8 +43,6 @@ impl Parser { } pub(in crate::parser) fn colon(&self, state: &mut State) -> ParseResult<()> { - expect_token!([TokenKind::Colon], state, "`:`"); - - Ok(()) + expect_token!([TokenKind::Colon => Ok(())], state, "`:`") } } diff --git a/tests/0174/code.php b/tests/0174/code.php new file mode 100644 index 00000000..1051bc62 --- /dev/null +++ b/tests/0174/code.php @@ -0,0 +1,6 @@ + Parse Error: unexpected token `...`, expecting `)` on line 5 column 5 diff --git a/tests/0174/tokens.txt b/tests/0174/tokens.txt new file mode 100644 index 00000000..209d72d6 --- /dev/null +++ b/tests/0174/tokens.txt @@ -0,0 +1,89 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 4, + 5, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Ellipsis, + span: ( + 5, + 5, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 5, + 8, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 3, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 4, + ), + }, +] diff --git a/tests/0175/code.php b/tests/0175/code.php new file mode 100644 index 00000000..64fa6148 --- /dev/null +++ b/tests/0175/code.php @@ -0,0 +1,7 @@ + Parse Error: unexpected token `float`, expecting `)` on line 6 column 5 diff --git a/tests/0175/tokens.txt b/tests/0175/tokens.txt new file mode 100644 index 00000000..6c65a0c9 --- /dev/null +++ b/tests/0175/tokens.txt @@ -0,0 +1,116 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 4, + 5, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 14, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 5, + 5, + ), + }, + Token { + kind: Variable( + "bar", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: Identifier( + "float", + ), + span: ( + 6, + 5, + ), + }, + Token { + kind: Variable( + "baz", + ), + span: ( + 6, + 11, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 1, + ), + }, + Token { + kind: LeftBrace, + span: ( + 7, + 3, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 4, + ), + }, +] diff --git a/tests/0176/ast.txt b/tests/0176/ast.txt new file mode 100644 index 00000000..4536cb23 --- /dev/null +++ b/tests/0176/ast.txt @@ -0,0 +1,48 @@ +[ + Function { + name: Identifier { + name: "foo", + }, + params: [ + Param { + name: Variable { + name: "a", + }, + type: Some( + String, + ), + variadic: false, + default: None, + flags: [], + by_ref: false, + }, + Param { + name: Variable { + name: "bar", + }, + type: Some( + Integer, + ), + variadic: false, + default: None, + flags: [], + by_ref: false, + }, + Param { + name: Variable { + name: "baz", + }, + type: Some( + Float, + ), + variadic: false, + default: None, + flags: [], + by_ref: false, + }, + ], + body: [], + return_type: None, + by_ref: false, + }, +] diff --git a/tests/0176/code.php b/tests/0176/code.php new file mode 100644 index 00000000..62c54f33 --- /dev/null +++ b/tests/0176/code.php @@ -0,0 +1,7 @@ + Date: Sat, 3 Dec 2022 00:20:09 +0100 Subject: [PATCH 10/13] fix: use by reference in anonymous functions (#10) --- src/parser/internal/functions.rs | 2 + src/parser/state.rs | 2 +- tests/0178/ast.txt | 309 +++++++++ tests/0178/code.php | 61 ++ tests/0178/tokens.txt | 1001 ++++++++++++++++++++++++++++++ 5 files changed, 1374 insertions(+), 1 deletion(-) create mode 100644 tests/0178/ast.txt create mode 100644 tests/0178/code.php create mode 100644 tests/0178/tokens.txt diff --git a/src/parser/internal/functions.rs b/src/parser/internal/functions.rs index d1224441..5900a938 100644 --- a/src/parser/internal/functions.rs +++ b/src/parser/internal/functions.rs @@ -48,6 +48,8 @@ impl Parser { while state.current.kind != TokenKind::RightParen { let mut by_ref = false; if state.current.kind == TokenKind::Ampersand { + state.next(); + by_ref = true; } diff --git a/src/parser/state.rs b/src/parser/state.rs index 9b7a1ed4..01f1f24c 100644 --- a/src/parser/state.rs +++ b/src/parser/state.rs @@ -47,7 +47,7 @@ impl State { let mut iter = tokens.into_iter(); Self { - stack: VecDeque::new(), + stack: VecDeque::with_capacity(3), current: iter.next().unwrap_or_default(), peek: iter.next().unwrap_or_default(), iter, diff --git a/tests/0178/ast.txt b/tests/0178/ast.txt new file mode 100644 index 00000000..a5038dc6 --- /dev/null +++ b/tests/0178/ast.txt @@ -0,0 +1,309 @@ +[ + Declare { + declares: [ + DeclareItem { + key: Identifier { + name: "strict_types", + }, + value: LiteralInteger { + i: 1, + }, + }, + ], + body: [], + }, + Namespace { + name: "Psl\Internal", + body: [ + Noop, + Use { + uses: [ + Use { + name: Identifier { + name: "Closure", + }, + alias: None, + }, + ], + kind: Normal, + }, + Use { + uses: [ + Use { + name: Identifier { + name: "Psl\Str", + }, + alias: None, + }, + ], + kind: Normal, + }, + Use { + uses: [ + Use { + name: Identifier { + name: "restore_error_handler", + }, + alias: None, + }, + ], + kind: Function, + }, + Use { + uses: [ + Use { + name: Identifier { + name: "set_error_handler", + }, + alias: None, + }, + ], + kind: Function, + }, + Function { + name: Identifier { + name: "box", + }, + params: [ + Param { + name: Variable { + name: "fun", + }, + type: Some( + Identifier( + Identifier { + name: "Closure", + }, + ), + ), + variadic: false, + default: None, + flags: [], + by_ref: false, + }, + ], + body: [ + Expression { + expr: Infix { + lhs: Variable { + name: "last_message", + }, + op: Assign, + rhs: Null, + }, + }, + Expression { + expr: Call { + target: Identifier { + name: "set_error_handler", + }, + args: [ + Arg { + name: None, + value: Closure { + params: [ + Param { + name: Variable { + name: "_type", + }, + type: Some( + Integer, + ), + variadic: false, + default: None, + flags: [], + by_ref: false, + }, + Param { + name: Variable { + name: "message", + }, + type: Some( + String, + ), + variadic: false, + default: None, + flags: [], + by_ref: false, + }, + ], + uses: [ + ClosureUse { + var: Variable { + name: "last_message", + }, + by_ref: true, + }, + ], + return_type: None, + body: [ + Expression { + expr: Infix { + lhs: Variable { + name: "last_message", + }, + op: Assign, + rhs: Variable { + name: "message", + }, + }, + }, + ], + static: true, + by_ref: false, + }, + unpack: false, + }, + ], + }, + }, + If { + condition: Infix { + lhs: Infix { + lhs: Null, + op: NotIdentical, + rhs: Variable { + name: "last_message", + }, + }, + op: And, + rhs: Call { + target: Identifier { + name: "Str\contains", + }, + args: [ + Arg { + name: None, + value: Variable { + name: "last_message", + }, + unpack: false, + }, + Arg { + name: None, + value: LiteralString { + value: "): ", + }, + unpack: false, + }, + ], + }, + }, + then: [ + Expression { + expr: Infix { + lhs: Variable { + name: "last_message", + }, + op: Assign, + rhs: Call { + target: Identifier { + name: "Str\after", + }, + args: [ + Arg { + name: None, + value: Call { + target: Identifier { + name: "Str\lowercase", + }, + args: [ + Arg { + name: None, + value: Variable { + name: "last_message", + }, + unpack: false, + }, + ], + }, + unpack: false, + }, + Arg { + name: None, + value: LiteralString { + value: "): ", + }, + unpack: false, + }, + ], + }, + }, + }, + ], + else_ifs: [], + else: None, + }, + Try { + body: [ + Expression { + expr: Infix { + lhs: Variable { + name: "value", + }, + op: Assign, + rhs: Call { + target: Variable { + name: "fun", + }, + args: [], + }, + }, + }, + Expression { + expr: Infix { + lhs: Variable { + name: "result", + }, + op: Assign, + rhs: Array { + items: [ + ArrayItem { + key: None, + value: Variable { + name: "value", + }, + unpack: false, + }, + ArrayItem { + key: None, + value: Variable { + name: "last_message", + }, + unpack: false, + }, + ], + }, + }, + }, + Return { + value: Some( + Variable { + name: "result", + }, + ), + }, + ], + catches: [], + finally: Some( + [ + Expression { + expr: Call { + target: Identifier { + name: "restore_error_handler", + }, + args: [], + }, + }, + ], + ), + }, + ], + return_type: Some( + Array, + ), + by_ref: false, + }, + ], + }, +] diff --git a/tests/0178/code.php b/tests/0178/code.php new file mode 100644 index 00000000..b8e2f109 --- /dev/null +++ b/tests/0178/code.php @@ -0,0 +1,61 @@ + + +declare(strict_types=1); + +namespace Psl\Internal; + +use Closure; +use Psl\Str; + +use function restore_error_handler; +use function set_error_handler; + +/** + * @template T + * + * @param (Closure(): T) $fun + * + * @return array{0: T, 1: ?string} + * + * @internal + * + * @psalm-suppress MissingThrowsDocblock + */ +function box(Closure $fun): array +{ + $last_message = null; + /** @psalm-suppress InvalidArgument */ + set_error_handler(static function (int $_type, string $message) use (&$last_message) { + $last_message = $message; + }); + + /** + * @var string|null $last_message + */ + if (null !== $last_message && Str\contains($last_message, '): ')) { + $last_message = Str\after( + Str\lowercase($last_message), + // how i feel toward PHP error handling: + '): ' + ); + } + + try { + $value = $fun(); + + /** @var array{0: T, 1: ?string} $result */ + $result = [$value, $last_message]; + + return $result; + } finally { + restore_error_handler(); + } +} diff --git a/tests/0178/tokens.txt b/tests/0178/tokens.txt new file mode 100644 index 00000000..11b4aae6 --- /dev/null +++ b/tests/0178/tokens.txt @@ -0,0 +1,1001 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Comment( + "// The following code was taken from of PSL.", + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: Comment( + "//", + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: Comment( + "// https://github.com/azjezz/psl/blob/657ce9888be47cee49418989420b83661f7cf1c4/src/Psl/Internal/box.php", + ), + span: ( + 5, + 1, + ), + }, + Token { + kind: Comment( + "//", + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: Comment( + "// Code subject to the MIT license (https://github.com/azjezz/psl/blob/657ce9888be47cee49418989420b83661f7cf1c4/LICENSE).", + ), + span: ( + 7, + 1, + ), + }, + Token { + kind: Comment( + "//", + ), + span: ( + 8, + 1, + ), + }, + Token { + kind: Comment( + "// Copyright (c) 2019-2022 Saif Eddin Gmati ", + ), + span: ( + 9, + 1, + ), + }, + Token { + kind: Declare, + span: ( + 11, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 11, + 8, + ), + }, + Token { + kind: Identifier( + "strict_types", + ), + span: ( + 11, + 9, + ), + }, + Token { + kind: Equals, + span: ( + 11, + 21, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 11, + 22, + ), + }, + Token { + kind: RightParen, + span: ( + 11, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 24, + ), + }, + Token { + kind: Namespace, + span: ( + 13, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Psl\Internal", + ), + span: ( + 13, + 11, + ), + }, + Token { + kind: SemiColon, + span: ( + 13, + 23, + ), + }, + Token { + kind: Use, + span: ( + 15, + 1, + ), + }, + Token { + kind: Identifier( + "Closure", + ), + span: ( + 15, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 15, + 12, + ), + }, + Token { + kind: Use, + span: ( + 16, + 1, + ), + }, + Token { + kind: QualifiedIdentifier( + "Psl\Str", + ), + span: ( + 16, + 5, + ), + }, + Token { + kind: SemiColon, + span: ( + 16, + 12, + ), + }, + Token { + kind: Use, + span: ( + 18, + 1, + ), + }, + Token { + kind: Function, + span: ( + 18, + 5, + ), + }, + Token { + kind: Identifier( + "restore_error_handler", + ), + span: ( + 18, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 18, + 35, + ), + }, + Token { + kind: Use, + span: ( + 19, + 1, + ), + }, + Token { + kind: Function, + span: ( + 19, + 5, + ), + }, + Token { + kind: Identifier( + "set_error_handler", + ), + span: ( + 19, + 14, + ), + }, + Token { + kind: SemiColon, + span: ( + 19, + 31, + ), + }, + Token { + kind: DocComment( + "/**\n * @template T\n *\n * @param (Closure(): T) $fun\n *\n * @return array{0: T, 1: ?string}\n *\n * @internal\n *\n * @psalm-suppress MissingThrowsDocblock\n */", + ), + span: ( + 21, + 1, + ), + }, + Token { + kind: Function, + span: ( + 32, + 1, + ), + }, + Token { + kind: Identifier( + "box", + ), + span: ( + 32, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 32, + 13, + ), + }, + Token { + kind: Identifier( + "Closure", + ), + span: ( + 32, + 14, + ), + }, + Token { + kind: Variable( + "fun", + ), + span: ( + 32, + 22, + ), + }, + Token { + kind: RightParen, + span: ( + 32, + 26, + ), + }, + Token { + kind: Colon, + span: ( + 32, + 27, + ), + }, + Token { + kind: Array, + span: ( + 32, + 29, + ), + }, + Token { + kind: LeftBrace, + span: ( + 33, + 1, + ), + }, + Token { + kind: Variable( + "last_message", + ), + span: ( + 34, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 34, + 19, + ), + }, + Token { + kind: Null, + span: ( + 34, + 21, + ), + }, + Token { + kind: SemiColon, + span: ( + 34, + 25, + ), + }, + Token { + kind: DocComment( + "/** @psalm-suppress InvalidArgument */", + ), + span: ( + 35, + 5, + ), + }, + Token { + kind: Identifier( + "set_error_handler", + ), + span: ( + 36, + 5, + ), + }, + Token { + kind: LeftParen, + span: ( + 36, + 22, + ), + }, + Token { + kind: Static, + span: ( + 36, + 23, + ), + }, + Token { + kind: Function, + span: ( + 36, + 30, + ), + }, + Token { + kind: LeftParen, + span: ( + 36, + 39, + ), + }, + Token { + kind: Identifier( + "int", + ), + span: ( + 36, + 40, + ), + }, + Token { + kind: Variable( + "_type", + ), + span: ( + 36, + 44, + ), + }, + Token { + kind: Comma, + span: ( + 36, + 50, + ), + }, + Token { + kind: Identifier( + "string", + ), + span: ( + 36, + 52, + ), + }, + Token { + kind: Variable( + "message", + ), + span: ( + 36, + 59, + ), + }, + Token { + kind: RightParen, + span: ( + 36, + 67, + ), + }, + Token { + kind: Use, + span: ( + 36, + 69, + ), + }, + Token { + kind: LeftParen, + span: ( + 36, + 73, + ), + }, + Token { + kind: Ampersand, + span: ( + 36, + 74, + ), + }, + Token { + kind: Variable( + "last_message", + ), + span: ( + 36, + 75, + ), + }, + Token { + kind: RightParen, + span: ( + 36, + 88, + ), + }, + Token { + kind: LeftBrace, + span: ( + 36, + 90, + ), + }, + Token { + kind: Variable( + "last_message", + ), + span: ( + 37, + 9, + ), + }, + Token { + kind: Equals, + span: ( + 37, + 23, + ), + }, + Token { + kind: Variable( + "message", + ), + span: ( + 37, + 25, + ), + }, + Token { + kind: SemiColon, + span: ( + 37, + 33, + ), + }, + Token { + kind: RightBrace, + span: ( + 38, + 5, + ), + }, + Token { + kind: RightParen, + span: ( + 38, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 38, + 7, + ), + }, + Token { + kind: DocComment( + "/**\n * @var string|null $last_message\n */", + ), + span: ( + 40, + 5, + ), + }, + Token { + kind: If, + span: ( + 43, + 5, + ), + }, + Token { + kind: LeftParen, + span: ( + 43, + 8, + ), + }, + Token { + kind: Null, + span: ( + 43, + 9, + ), + }, + Token { + kind: BangDoubleEquals, + span: ( + 43, + 14, + ), + }, + Token { + kind: Variable( + "last_message", + ), + span: ( + 43, + 18, + ), + }, + Token { + kind: BooleanAnd, + span: ( + 43, + 32, + ), + }, + Token { + kind: QualifiedIdentifier( + "Str\contains", + ), + span: ( + 43, + 35, + ), + }, + Token { + kind: LeftParen, + span: ( + 43, + 47, + ), + }, + Token { + kind: Variable( + "last_message", + ), + span: ( + 43, + 48, + ), + }, + Token { + kind: Comma, + span: ( + 43, + 61, + ), + }, + Token { + kind: LiteralString( + "): ", + ), + span: ( + 43, + 63, + ), + }, + Token { + kind: RightParen, + span: ( + 43, + 68, + ), + }, + Token { + kind: RightParen, + span: ( + 43, + 69, + ), + }, + Token { + kind: LeftBrace, + span: ( + 43, + 71, + ), + }, + Token { + kind: Variable( + "last_message", + ), + span: ( + 44, + 9, + ), + }, + Token { + kind: Equals, + span: ( + 44, + 23, + ), + }, + Token { + kind: QualifiedIdentifier( + "Str\after", + ), + span: ( + 44, + 25, + ), + }, + Token { + kind: LeftParen, + span: ( + 44, + 34, + ), + }, + Token { + kind: QualifiedIdentifier( + "Str\lowercase", + ), + span: ( + 45, + 13, + ), + }, + Token { + kind: LeftParen, + span: ( + 45, + 26, + ), + }, + Token { + kind: Variable( + "last_message", + ), + span: ( + 45, + 27, + ), + }, + Token { + kind: RightParen, + span: ( + 45, + 40, + ), + }, + Token { + kind: Comma, + span: ( + 45, + 41, + ), + }, + Token { + kind: Comment( + "// how i feel toward PHP error handling:", + ), + span: ( + 46, + 13, + ), + }, + Token { + kind: LiteralString( + "): ", + ), + span: ( + 47, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 48, + 9, + ), + }, + Token { + kind: SemiColon, + span: ( + 48, + 10, + ), + }, + Token { + kind: RightBrace, + span: ( + 49, + 5, + ), + }, + Token { + kind: Try, + span: ( + 51, + 5, + ), + }, + Token { + kind: LeftBrace, + span: ( + 51, + 9, + ), + }, + Token { + kind: Variable( + "value", + ), + span: ( + 52, + 9, + ), + }, + Token { + kind: Equals, + span: ( + 52, + 16, + ), + }, + Token { + kind: Variable( + "fun", + ), + span: ( + 52, + 18, + ), + }, + Token { + kind: LeftParen, + span: ( + 52, + 22, + ), + }, + Token { + kind: RightParen, + span: ( + 52, + 23, + ), + }, + Token { + kind: SemiColon, + span: ( + 52, + 24, + ), + }, + Token { + kind: DocComment( + "/** @var array{0: T, 1: ?string} $result */", + ), + span: ( + 54, + 9, + ), + }, + Token { + kind: Variable( + "result", + ), + span: ( + 55, + 9, + ), + }, + Token { + kind: Equals, + span: ( + 55, + 17, + ), + }, + Token { + kind: LeftBracket, + span: ( + 55, + 19, + ), + }, + Token { + kind: Variable( + "value", + ), + span: ( + 55, + 20, + ), + }, + Token { + kind: Comma, + span: ( + 55, + 26, + ), + }, + Token { + kind: Variable( + "last_message", + ), + span: ( + 55, + 28, + ), + }, + Token { + kind: RightBracket, + span: ( + 55, + 41, + ), + }, + Token { + kind: SemiColon, + span: ( + 55, + 42, + ), + }, + Token { + kind: Return, + span: ( + 57, + 9, + ), + }, + Token { + kind: Variable( + "result", + ), + span: ( + 57, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 57, + 23, + ), + }, + Token { + kind: RightBrace, + span: ( + 58, + 5, + ), + }, + Token { + kind: Finally, + span: ( + 58, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 58, + 15, + ), + }, + Token { + kind: Identifier( + "restore_error_handler", + ), + span: ( + 59, + 9, + ), + }, + Token { + kind: LeftParen, + span: ( + 59, + 30, + ), + }, + Token { + kind: RightParen, + span: ( + 59, + 31, + ), + }, + Token { + kind: SemiColon, + span: ( + 59, + 32, + ), + }, + Token { + kind: RightBrace, + span: ( + 60, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 61, + 1, + ), + }, +] From 1e487ea19b93eea78c7a0db39921c56ea7193d0e Mon Sep 17 00:00:00 2001 From: Saif Eddin Gmati <29315886+azjezz@users.noreply.github.com> Date: Sat, 3 Dec 2022 02:25:22 +0100 Subject: [PATCH 11/13] fix: multiple issues regarding optional comma --- src/parser/internal/classish.rs | 18 +- src/parser/internal/classish_statement.rs | 46 ++++- src/parser/internal/functions.rs | 6 +- src/parser/internal/params.rs | 12 +- src/parser/internal/punc.rs | 12 -- src/parser/mod.rs | 155 ++++++++++----- tests/0179/code.php | 3 + tests/0179/parser-error.txt | 1 + tests/0179/tokens.txt | 84 ++++++++ tests/0180/code.php | 3 + tests/0180/parser-error.txt | 1 + tests/0180/tokens.txt | 84 ++++++++ tests/0181/code.php | 4 + tests/0181/parser-error.txt | 1 + tests/0181/tokens.txt | 75 +++++++ tests/0182/ast.txt | 39 ++++ tests/0182/code.php | 3 + tests/0182/tokens.txt | 105 ++++++++++ tests/0183/ast.txt | 42 ++++ tests/0183/code.php | 3 + tests/0183/tokens.txt | 105 ++++++++++ tests/0184/ast.txt | 42 ++++ tests/0184/code.php | 3 + tests/0184/tokens.txt | 112 +++++++++++ tests/0185/ast.txt | 22 ++ tests/0185/code.php | 4 + tests/0185/tokens.txt | 82 ++++++++ tests/0186/code.php | 6 + tests/0186/parser-error.txt | 1 + tests/0186/tokens.txt | 89 +++++++++ tests/0187/ast.txt | 23 +++ tests/0187/code.php | 6 + tests/0187/tokens.txt | 96 +++++++++ tests/0188/code.php | 5 + tests/0188/parser-error.txt | 1 + tests/0188/tokens.txt | 87 ++++++++ tests/0189/ast.txt | 22 ++ tests/0189/code.php | 5 + tests/0189/tokens.txt | 94 +++++++++ tests/0190/ast.txt | 28 +++ tests/0190/code.php | 5 + tests/0190/tokens.txt | 94 +++++++++ tests/0191/code.php | 5 + tests/0191/parser-error.txt | 1 + tests/0191/tokens.txt | 87 ++++++++ tests/0192/code.php | 5 + tests/0192/parser-error.txt | 1 + tests/0192/tokens.txt | 87 ++++++++ tests/0193/ast.txt | 27 +++ tests/0193/code.php | 7 + tests/0193/tokens.txt | 110 ++++++++++ tests/0194/ast.txt | 47 +++++ tests/0194/code.php | 8 + tests/0194/tokens.txt | 188 ++++++++++++++++++ tests/0195/ast.txt | 47 +++++ tests/0195/code.php | 8 + tests/0195/tokens.txt | 195 ++++++++++++++++++ tests/0196/ast.txt | 47 +++++ tests/0196/code.php | 8 + tests/0196/tokens.txt | 181 +++++++++++++++++ tests/0197/ast.txt | 51 +++++ tests/0197/code.php | 10 + tests/0197/tokens.txt | 232 ++++++++++++++++++++++ tests/0198/code.php | 10 + tests/0198/parser-error.txt | 1 + tests/0198/tokens.txt | 186 +++++++++++++++++ tests/0199/code.php | 10 + tests/0199/parser-error.txt | 1 + tests/0199/tokens.txt | 179 +++++++++++++++++ tests/0200/code.php | 3 + tests/0200/parser-error.txt | 1 + tests/0200/tokens.txt | 66 ++++++ tests/0201/ast.txt | 18 ++ tests/0201/code.php | 3 + tests/0201/tokens.txt | 73 +++++++ tests/0202/code.php | 3 + tests/0202/parser-error.txt | 1 + tests/0202/tokens.txt | 66 ++++++ tests/0203/ast.txt | 16 ++ tests/0203/code.php | 3 + tests/0203/tokens.txt | 73 +++++++ tests/0204/ast.txt | 23 +++ tests/0204/code.php | 5 + tests/0204/tokens.txt | 80 ++++++++ tests/0205/ast.txt | 23 +++ tests/0205/code.php | 5 + tests/0205/tokens.txt | 87 ++++++++ tests/0206/code.php | 7 + tests/0206/parser-error.txt | 1 + tests/0206/tokens.txt | 174 ++++++++++++++++ tests/0207/ast.txt | 45 +++++ tests/0207/code.php | 7 + tests/0207/tokens.txt | 167 ++++++++++++++++ tests/0208/code.php | 7 + tests/0208/parser-error.txt | 1 + tests/0208/tokens.txt | 174 ++++++++++++++++ tests/0209/code.php | 5 + tests/0209/parser-error.txt | 1 + tests/0209/tokens.txt | 103 ++++++++++ 99 files changed, 4528 insertions(+), 81 deletions(-) create mode 100644 tests/0179/code.php create mode 100644 tests/0179/parser-error.txt create mode 100644 tests/0179/tokens.txt create mode 100644 tests/0180/code.php create mode 100644 tests/0180/parser-error.txt create mode 100644 tests/0180/tokens.txt create mode 100644 tests/0181/code.php create mode 100644 tests/0181/parser-error.txt create mode 100644 tests/0181/tokens.txt create mode 100644 tests/0182/ast.txt create mode 100644 tests/0182/code.php create mode 100644 tests/0182/tokens.txt create mode 100644 tests/0183/ast.txt create mode 100644 tests/0183/code.php create mode 100644 tests/0183/tokens.txt create mode 100644 tests/0184/ast.txt create mode 100644 tests/0184/code.php create mode 100644 tests/0184/tokens.txt create mode 100644 tests/0185/ast.txt create mode 100644 tests/0185/code.php create mode 100644 tests/0185/tokens.txt create mode 100644 tests/0186/code.php create mode 100644 tests/0186/parser-error.txt create mode 100644 tests/0186/tokens.txt create mode 100644 tests/0187/ast.txt create mode 100644 tests/0187/code.php create mode 100644 tests/0187/tokens.txt create mode 100644 tests/0188/code.php create mode 100644 tests/0188/parser-error.txt create mode 100644 tests/0188/tokens.txt create mode 100644 tests/0189/ast.txt create mode 100644 tests/0189/code.php create mode 100644 tests/0189/tokens.txt create mode 100644 tests/0190/ast.txt create mode 100644 tests/0190/code.php create mode 100644 tests/0190/tokens.txt create mode 100644 tests/0191/code.php create mode 100644 tests/0191/parser-error.txt create mode 100644 tests/0191/tokens.txt create mode 100644 tests/0192/code.php create mode 100644 tests/0192/parser-error.txt create mode 100644 tests/0192/tokens.txt create mode 100644 tests/0193/ast.txt create mode 100644 tests/0193/code.php create mode 100644 tests/0193/tokens.txt create mode 100644 tests/0194/ast.txt create mode 100644 tests/0194/code.php create mode 100644 tests/0194/tokens.txt create mode 100644 tests/0195/ast.txt create mode 100644 tests/0195/code.php create mode 100644 tests/0195/tokens.txt create mode 100644 tests/0196/ast.txt create mode 100644 tests/0196/code.php create mode 100644 tests/0196/tokens.txt create mode 100644 tests/0197/ast.txt create mode 100644 tests/0197/code.php create mode 100644 tests/0197/tokens.txt create mode 100644 tests/0198/code.php create mode 100644 tests/0198/parser-error.txt create mode 100644 tests/0198/tokens.txt create mode 100644 tests/0199/code.php create mode 100644 tests/0199/parser-error.txt create mode 100644 tests/0199/tokens.txt create mode 100644 tests/0200/code.php create mode 100644 tests/0200/parser-error.txt create mode 100644 tests/0200/tokens.txt create mode 100644 tests/0201/ast.txt create mode 100644 tests/0201/code.php create mode 100644 tests/0201/tokens.txt create mode 100644 tests/0202/code.php create mode 100644 tests/0202/parser-error.txt create mode 100644 tests/0202/tokens.txt create mode 100644 tests/0203/ast.txt create mode 100644 tests/0203/code.php create mode 100644 tests/0203/tokens.txt create mode 100644 tests/0204/ast.txt create mode 100644 tests/0204/code.php create mode 100644 tests/0204/tokens.txt create mode 100644 tests/0205/ast.txt create mode 100644 tests/0205/code.php create mode 100644 tests/0205/tokens.txt create mode 100644 tests/0206/code.php create mode 100644 tests/0206/parser-error.txt create mode 100644 tests/0206/tokens.txt create mode 100644 tests/0207/ast.txt create mode 100644 tests/0207/code.php create mode 100644 tests/0207/tokens.txt create mode 100644 tests/0208/code.php create mode 100644 tests/0208/parser-error.txt create mode 100644 tests/0208/tokens.txt create mode 100644 tests/0209/code.php create mode 100644 tests/0209/parser-error.txt create mode 100644 tests/0209/tokens.txt diff --git a/src/parser/internal/classish.rs b/src/parser/internal/classish.rs index 98402606..bc6143a8 100644 --- a/src/parser/internal/classish.rs +++ b/src/parser/internal/classish.rs @@ -144,11 +144,7 @@ impl Parser { let mut args = vec![]; if state.current.kind == TokenKind::LeftParen { - self.lparen(state)?; - args = self.args_list(state)?; - - self.rparen(state)?; } let mut extends: Option = None; @@ -163,9 +159,13 @@ impl Parser { state.next(); while state.current.kind != TokenKind::LeftBrace { - self.optional_comma(state)?; - implements.push(self.full_name(state)?.into()); + + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } } } @@ -218,7 +218,11 @@ impl Parser { while state.current.kind != TokenKind::LeftBrace { implements.push(self.full_name(state)?.into()); - self.optional_comma(state)?; + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } } } diff --git a/src/parser/internal/classish_statement.rs b/src/parser/internal/classish_statement.rs index c8e8ac48..2a27f943 100644 --- a/src/parser/internal/classish_statement.rs +++ b/src/parser/internal/classish_statement.rs @@ -227,10 +227,24 @@ impl Parser { while state.current.kind != TokenKind::SemiColon && state.current.kind != TokenKind::LeftBrace { - self.optional_comma(state)?; - let t = self.full_name(state)?; traits.push(t.into()); + + if state.current.kind == TokenKind::Comma { + if state.peek.kind == TokenKind::SemiColon { + // will fail with unexpected token `,` + // as `use` doesn't allow for trailing commas. + self.semi(state)?; + } else if state.peek.kind == TokenKind::LeftBrace { + // will fail with unexpected token `{` + // as `use` doesn't allow for trailing commas. + self.lbrace(state)?; + } else { + state.next(); + } + } else { + break; + } } let mut adaptations = Vec::new(); @@ -285,9 +299,31 @@ impl Parser { TokenKind::Insteadof => { let mut insteadof = Vec::new(); insteadof.push(self.full_name(state)?.into()); - while state.current.kind != TokenKind::SemiColon { - self.optional_comma(state)?; - insteadof.push(self.full_name(state)?.into()); + + if state.current.kind == TokenKind::Comma { + if state.peek.kind == TokenKind::SemiColon { + // will fail with unexpected token `,` + // as `insteadof` doesn't allow for trailing commas. + self.semi(state)?; + } + + state.next(); + + while state.current.kind != TokenKind::SemiColon { + insteadof.push(self.full_name(state)?.into()); + + if state.current.kind == TokenKind::Comma { + if state.peek.kind == TokenKind::SemiColon { + // will fail with unexpected token `,` + // as `insteadof` doesn't allow for trailing commas. + self.semi(state)?; + } else { + state.next(); + } + } else { + break; + } + } } adaptations.push(TraitAdaptation::Precedence { diff --git a/src/parser/internal/functions.rs b/src/parser/internal/functions.rs index 5900a938..1cc6e766 100644 --- a/src/parser/internal/functions.rs +++ b/src/parser/internal/functions.rs @@ -67,7 +67,11 @@ impl Parser { uses.push(var); - self.optional_comma(state)?; + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } } self.rparen(state)?; diff --git a/src/parser/internal/params.rs b/src/parser/internal/params.rs index 232adcd7..a639037d 100644 --- a/src/parser/internal/params.rs +++ b/src/parser/internal/params.rs @@ -139,7 +139,7 @@ impl Parser { }); if state.current.kind == TokenKind::Comma { - self.comma(state)?; + state.next(); } else { break; } @@ -151,6 +151,8 @@ impl Parser { } pub(in crate::parser) fn args_list(&self, state: &mut State) -> ParseResult> { + self.lparen(state)?; + let mut args = Vec::new(); while !state.is_eof() && state.current.kind != TokenKind::RightParen { @@ -184,9 +186,15 @@ impl Parser { value, }); - self.optional_comma(state)?; + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } } + self.rparen(state)?; + Ok(args) } } diff --git a/src/parser/internal/punc.rs b/src/parser/internal/punc.rs index da779596..ecf6b85e 100644 --- a/src/parser/internal/punc.rs +++ b/src/parser/internal/punc.rs @@ -30,18 +30,6 @@ impl Parser { expect_token!([TokenKind::RightBracket => Ok(())], state, "`]`") } - pub(in crate::parser) fn comma(&self, state: &mut State) -> ParseResult<()> { - expect_token!([TokenKind::Comma => Ok(())], state, "`,`") - } - - pub(in crate::parser) fn optional_comma(&self, state: &mut State) -> ParseResult<()> { - if state.current.kind == TokenKind::Comma { - expect_token!([TokenKind::Comma], state, "`,`"); - } - - Ok(()) - } - pub(in crate::parser) fn colon(&self, state: &mut State) -> ParseResult<()> { expect_token!([TokenKind::Colon => Ok(())], state, "`:`") } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index ae0952b5..1aad12a8 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -142,7 +142,7 @@ impl Parser { let mut constants = vec![]; - while state.current.kind != TokenKind::SemiColon { + loop { let name = self.ident(state)?; expect_token!([TokenKind::Equals], state, "`=`"); @@ -154,7 +154,11 @@ impl Parser { value, }); - self.optional_comma(state)?; + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } } self.semi(state)?; @@ -206,19 +210,23 @@ impl Parser { self.lparen(state)?; let mut declares = Vec::new(); - while state.current.kind != TokenKind::RightParen { + loop { let key = self.ident(state)?; expect_token!([TokenKind::Equals], state, "`=`"); let value = expect_literal!(state); - self.optional_comma(state)?; - declares.push(DeclareItem { key: key.into(), value, }); + + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } } self.rparen(state)?; @@ -245,10 +253,15 @@ impl Parser { state.next(); let mut vars = vec![]; - while state.current.kind != TokenKind::SemiColon { + // `loop` instead of `while` as we don't allow for extra commas. + loop { vars.push(self.var(state)?.into()); - self.optional_comma(state)?; + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } } self.semi(state)?; @@ -259,20 +272,26 @@ impl Parser { let mut vars = vec![]; - while state.current.kind != TokenKind::SemiColon { + // `loop` instead of `while` as we don't allow for extra commas. + loop { let var = Expression::Variable { name: self.var(state)?, }; let mut default = None; if state.current.kind == TokenKind::Equals { - expect_token!([TokenKind::Equals], state, "`=`"); + state.next(); + default = Some(self.expression(state, Precedence::Lowest)?); } - self.optional_comma(state)?; + vars.push(StaticVar { var, default }); - vars.push(StaticVar { var, default }) + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } } self.semi(state)?; @@ -678,11 +697,16 @@ impl Parser { state.next(); let mut values = Vec::new(); - while !state.is_eof() && state.current.kind != TokenKind::SemiColon { + loop { values.push(self.expression(state, Precedence::Lowest)?); - self.optional_comma(state)?; + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } } + self.semi(state)?; Statement::Echo { values } } @@ -976,33 +1000,43 @@ impl Parser { let mut arms = Vec::new(); while state.current.kind != TokenKind::RightBrace { - let mut conditions = Vec::new(); + state.skip_comments(); - while state.current.kind != TokenKind::DoubleArrow { - if state.current.kind == TokenKind::Default { + let conditions = if state.current.kind == TokenKind::Default { + state.next(); + + // match conditions can have an extra comma at the end, including `default`. + if state.current.kind == TokenKind::Comma { state.next(); - break; } - conditions.push(self.expression(state, Precedence::Lowest)?); + None + } else { + let mut conditions = Vec::new(); + while state.current.kind != TokenKind::DoubleArrow { + conditions.push(self.expression(state, Precedence::Lowest)?); - self.optional_comma(state)?; - } + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } + } + + Some(conditions) + }; expect_token!([TokenKind::DoubleArrow], state, "`=>`"); let body = self.expression(state, Precedence::Lowest)?; - self.optional_comma(state)?; + arms.push(MatchArm { conditions, body }); - arms.push(MatchArm { - conditions: if conditions.is_empty() { - None - } else { - Some(conditions) - }, - body, - }) + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } } self.rbrace(state)?; @@ -1027,6 +1061,7 @@ impl Parser { let mut value = self.expression(state, Precedence::Lowest)?; + // TODO: return error for `[...$a => $b]`. if state.current.kind == TokenKind::DoubleArrow { state.next(); @@ -1036,7 +1071,11 @@ impl Parser { items.push(ArrayItem { key, value, unpack }); - self.optional_comma(state)?; + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } state.skip_comments(); } @@ -1052,6 +1091,9 @@ impl Parser { state.skip_comments(); while state.current.kind != TokenKind::RightBracket { + // TODO: return an error here instead of + // an empty array element + // see: https://3v4l.org/uLTVA if state.current.kind == TokenKind::Comma { items.push(ArrayItem { key: None, @@ -1069,6 +1111,7 @@ impl Parser { } else { false }; + let mut value = self.expression(state, Precedence::Lowest)?; if state.current.kind == TokenKind::DoubleArrow { @@ -1080,11 +1123,16 @@ impl Parser { items.push(ArrayItem { key, value, unpack }); - self.optional_comma(state)?; - state.skip_comments(); + if state.current.kind == TokenKind::Comma { + state.next(); + } else { + break; + } } + state.skip_comments(); + self.rbracket(state)?; Expression::Array { items } @@ -1105,11 +1153,7 @@ impl Parser { let target = self.expression(state, Precedence::CloneOrNew)?; if state.current.kind == TokenKind::LeftParen { - self.lparen(state)?; - args = self.args_list(state)?; - - self.rparen(state)?; } Expression::New { @@ -1165,8 +1209,6 @@ impl Parser { break; } - state.next(); - left = self.postfix(state, left, &kind)?; continue; } @@ -1234,6 +1276,8 @@ impl Parser { ) -> Result { Ok(match op { TokenKind::Coalesce => { + state.next(); + let rhs = self.expression(state, Precedence::NullCoalesce)?; Expression::Coalesce { @@ -1244,14 +1288,14 @@ impl Parser { TokenKind::LeftParen => { let args = self.args_list(state)?; - self.rparen(state)?; - Expression::Call { target: Box::new(lhs), args, } } TokenKind::LeftBracket => { + state.next(); + if state.current.kind == TokenKind::RightBracket { state.next(); @@ -1271,6 +1315,8 @@ impl Parser { } } TokenKind::DoubleColon => { + state.next(); + let mut must_be_method_call = false; let property = match state.current.kind.clone() { @@ -1327,12 +1373,8 @@ impl Parser { // is only valid a method call context, we can assume we're parsing a static // method call. _ if state.current.kind == TokenKind::LeftParen || must_be_method_call => { - self.lparen(state)?; - let args = self.args_list(state)?; - self.rparen(state)?; - Expression::StaticMethodCall { target: lhs, method: Box::new(property), @@ -1348,6 +1390,8 @@ impl Parser { } } TokenKind::Arrow | TokenKind::NullsafeArrow => { + state.next(); + let property = match state.current.kind { TokenKind::LeftBrace => { self.lbrace(state)?; @@ -1367,12 +1411,8 @@ impl Parser { }; if state.current.kind == TokenKind::LeftParen { - state.next(); - let args = self.args_list(state)?; - self.rparen(state)?; - if op == &TokenKind::NullsafeArrow { Expression::NullsafeMethodCall { target: Box::new(lhs), @@ -1398,12 +1438,19 @@ impl Parser { } } } - TokenKind::Increment => Expression::Increment { - value: Box::new(lhs), - }, - TokenKind::Decrement => Expression::Decrement { - value: Box::new(lhs), - }, + TokenKind::Increment => { + state.next(); + Expression::Increment { + value: Box::new(lhs), + } + } + TokenKind::Decrement => { + state.next(); + + Expression::Decrement { + value: Box::new(lhs), + } + } _ => todo!("postfix: {:?}", op), }) } diff --git a/tests/0179/code.php b/tests/0179/code.php new file mode 100644 index 00000000..226b277c --- /dev/null +++ b/tests/0179/code.php @@ -0,0 +1,3 @@ + Parse Error: unexpected token `$b`, expecting `)` on line 3 column 8 diff --git a/tests/0179/tokens.txt b/tests/0179/tokens.txt new file mode 100644 index 00000000..de23afe9 --- /dev/null +++ b/tests/0179/tokens.txt @@ -0,0 +1,84 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 4, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 5, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 3, + 8, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: Ellipsis, + span: ( + 3, + 14, + ), + }, + Token { + kind: Variable( + "d", + ), + span: ( + 3, + 17, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 19, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 20, + ), + }, +] diff --git a/tests/0180/code.php b/tests/0180/code.php new file mode 100644 index 00000000..f631f005 --- /dev/null +++ b/tests/0180/code.php @@ -0,0 +1,3 @@ + Parse Error: unexpected token `$b`, expecting `]` on line 3 column 10 diff --git a/tests/0180/tokens.txt b/tests/0180/tokens.txt new file mode 100644 index 00000000..1c4151a5 --- /dev/null +++ b/tests/0180/tokens.txt @@ -0,0 +1,84 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 3, + 4, + ), + }, + Token { + kind: LeftBracket, + span: ( + 3, + 6, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: Variable( + "c", + ), + span: ( + 3, + 13, + ), + }, + Token { + kind: Variable( + "d", + ), + span: ( + 3, + 16, + ), + }, + Token { + kind: RightBracket, + span: ( + 3, + 18, + ), + }, + Token { + kind: SemiColon, + span: ( + 3, + 19, + ), + }, +] diff --git a/tests/0181/code.php b/tests/0181/code.php new file mode 100644 index 00000000..31be8725 --- /dev/null +++ b/tests/0181/code.php @@ -0,0 +1,4 @@ + Parse Error: unexpected token `B`, expecting `;` on line 4 column 9 diff --git a/tests/0181/tokens.txt b/tests/0181/tokens.txt new file mode 100644 index 00000000..0d40837f --- /dev/null +++ b/tests/0181/tokens.txt @@ -0,0 +1,75 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Const, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "A", + ), + span: ( + 3, + 9, + ), + }, + Token { + kind: Equals, + span: ( + 3, + 11, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 3, + 13, + ), + }, + Token { + kind: Identifier( + "B", + ), + span: ( + 4, + 9, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 11, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 4, + 13, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 14, + ), + }, +] diff --git a/tests/0182/ast.txt b/tests/0182/ast.txt new file mode 100644 index 00000000..bdc4fcf6 --- /dev/null +++ b/tests/0182/ast.txt @@ -0,0 +1,39 @@ +[ + Expression { + expr: Call { + target: Identifier { + name: "foo", + }, + args: [ + Arg { + name: None, + value: Variable { + name: "a", + }, + unpack: false, + }, + Arg { + name: None, + value: Variable { + name: "b", + }, + unpack: false, + }, + Arg { + name: None, + value: Variable { + name: "c", + }, + unpack: false, + }, + Arg { + name: None, + value: Variable { + name: "d", + }, + unpack: true, + }, + ], + }, + }, +] diff --git a/tests/0182/code.php b/tests/0182/code.php new file mode 100644 index 00000000..d325fe46 --- /dev/null +++ b/tests/0182/code.php @@ -0,0 +1,3 @@ + Parse Error: unexpected token `c`, expecting `)` on line 5 column 5 diff --git a/tests/0186/tokens.txt b/tests/0186/tokens.txt new file mode 100644 index 00000000..9ed9d3d0 --- /dev/null +++ b/tests/0186/tokens.txt @@ -0,0 +1,89 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Declare, + span: ( + 3, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 8, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 4, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 7, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 4, + 9, + ), + }, + Token { + kind: Identifier( + "c", + ), + span: ( + 5, + 5, + ), + }, + Token { + kind: Equals, + span: ( + 5, + 7, + ), + }, + Token { + kind: LiteralString( + "f", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 2, + ), + }, +] diff --git a/tests/0187/ast.txt b/tests/0187/ast.txt new file mode 100644 index 00000000..c805e952 --- /dev/null +++ b/tests/0187/ast.txt @@ -0,0 +1,23 @@ +[ + Declare { + declares: [ + DeclareItem { + key: Identifier { + name: "a", + }, + value: LiteralInteger { + i: 3, + }, + }, + DeclareItem { + key: Identifier { + name: "c", + }, + value: LiteralString { + value: "f", + }, + }, + ], + body: [], + }, +] diff --git a/tests/0187/code.php b/tests/0187/code.php new file mode 100644 index 00000000..b83b176c --- /dev/null +++ b/tests/0187/code.php @@ -0,0 +1,6 @@ + Parse Error: unexpected token `$b`, expecting `;` on line 4 column 15 diff --git a/tests/0188/tokens.txt b/tests/0188/tokens.txt new file mode 100644 index 00000000..e9b85415 --- /dev/null +++ b/tests/0188/tokens.txt @@ -0,0 +1,87 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 14, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 16, + ), + }, + Token { + kind: Global, + span: ( + 4, + 5, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 4, + 15, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 17, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0189/ast.txt b/tests/0189/ast.txt new file mode 100644 index 00000000..60c3d406 --- /dev/null +++ b/tests/0189/ast.txt @@ -0,0 +1,22 @@ +[ + Function { + name: Identifier { + name: "foo", + }, + params: [], + body: [ + Global { + vars: [ + Identifier { + name: "a", + }, + Identifier { + name: "b", + }, + ], + }, + ], + return_type: None, + by_ref: false, + }, +] diff --git a/tests/0189/code.php b/tests/0189/code.php new file mode 100644 index 00000000..72eaea62 --- /dev/null +++ b/tests/0189/code.php @@ -0,0 +1,5 @@ + Parse Error: unexpected token `$b`, expecting `;` on line 4 column 15 diff --git a/tests/0191/tokens.txt b/tests/0191/tokens.txt new file mode 100644 index 00000000..58252437 --- /dev/null +++ b/tests/0191/tokens.txt @@ -0,0 +1,87 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 14, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 16, + ), + }, + Token { + kind: Static, + span: ( + 4, + 5, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 4, + 15, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 17, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0192/code.php b/tests/0192/code.php new file mode 100644 index 00000000..c6cf3c68 --- /dev/null +++ b/tests/0192/code.php @@ -0,0 +1,5 @@ + Parse Error: unexpected token `$b`, expecting `;` on line 4 column 15 diff --git a/tests/0192/tokens.txt b/tests/0192/tokens.txt new file mode 100644 index 00000000..58252437 --- /dev/null +++ b/tests/0192/tokens.txt @@ -0,0 +1,87 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Function, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftParen, + span: ( + 3, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 3, + 14, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 16, + ), + }, + Token { + kind: Static, + span: ( + 4, + 5, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 4, + 15, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 17, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] diff --git a/tests/0193/ast.txt b/tests/0193/ast.txt new file mode 100644 index 00000000..72172f78 --- /dev/null +++ b/tests/0193/ast.txt @@ -0,0 +1,27 @@ +[ + Expression { + expr: Infix { + lhs: Variable { + name: "a", + }, + op: Assign, + rhs: LiteralInteger { + i: 4, + }, + }, + }, + Expression { + expr: Infix { + lhs: Variable { + name: "b", + }, + op: Assign, + rhs: Match { + condition: Variable { + name: "a", + }, + arms: [], + }, + }, + }, +] diff --git a/tests/0193/code.php b/tests/0193/code.php new file mode 100644 index 00000000..109e0689 --- /dev/null +++ b/tests/0193/code.php @@ -0,0 +1,7 @@ + null +}; diff --git a/tests/0194/tokens.txt b/tests/0194/tokens.txt new file mode 100644 index 00000000..41608e26 --- /dev/null +++ b/tests/0194/tokens.txt @@ -0,0 +1,188 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 4, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 4, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 7, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 6, + 4, + ), + }, + Token { + kind: Match, + span: ( + 6, + 6, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 12, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 6, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 15, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 17, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 7, + 5, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 6, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 7, + 7, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 8, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 7, + 9, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 10, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 12, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 7, + 14, + ), + }, + Token { + kind: Null, + span: ( + 7, + 17, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 2, + ), + }, +] diff --git a/tests/0195/ast.txt b/tests/0195/ast.txt new file mode 100644 index 00000000..f07faedc --- /dev/null +++ b/tests/0195/ast.txt @@ -0,0 +1,47 @@ +[ + Expression { + expr: Infix { + lhs: Variable { + name: "a", + }, + op: Assign, + rhs: LiteralInteger { + i: 4, + }, + }, + }, + Expression { + expr: Infix { + lhs: Variable { + name: "b", + }, + op: Assign, + rhs: Match { + condition: Variable { + name: "a", + }, + arms: [ + MatchArm { + conditions: Some( + [ + LiteralInteger { + i: 1, + }, + LiteralInteger { + i: 2, + }, + LiteralInteger { + i: 3, + }, + LiteralInteger { + i: 4, + }, + ], + ), + body: Null, + }, + ], + }, + }, + }, +] diff --git a/tests/0195/code.php b/tests/0195/code.php new file mode 100644 index 00000000..37ef74de --- /dev/null +++ b/tests/0195/code.php @@ -0,0 +1,8 @@ + null, +}; diff --git a/tests/0195/tokens.txt b/tests/0195/tokens.txt new file mode 100644 index 00000000..10da61dd --- /dev/null +++ b/tests/0195/tokens.txt @@ -0,0 +1,195 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 4, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 4, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 7, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 6, + 4, + ), + }, + Token { + kind: Match, + span: ( + 6, + 6, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 12, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 6, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 15, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 17, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 7, + 5, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 6, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 7, + 7, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 8, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 7, + 9, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 10, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 12, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 7, + 14, + ), + }, + Token { + kind: Null, + span: ( + 7, + 17, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 2, + ), + }, +] diff --git a/tests/0196/ast.txt b/tests/0196/ast.txt new file mode 100644 index 00000000..f07faedc --- /dev/null +++ b/tests/0196/ast.txt @@ -0,0 +1,47 @@ +[ + Expression { + expr: Infix { + lhs: Variable { + name: "a", + }, + op: Assign, + rhs: LiteralInteger { + i: 4, + }, + }, + }, + Expression { + expr: Infix { + lhs: Variable { + name: "b", + }, + op: Assign, + rhs: Match { + condition: Variable { + name: "a", + }, + arms: [ + MatchArm { + conditions: Some( + [ + LiteralInteger { + i: 1, + }, + LiteralInteger { + i: 2, + }, + LiteralInteger { + i: 3, + }, + LiteralInteger { + i: 4, + }, + ], + ), + body: Null, + }, + ], + }, + }, + }, +] diff --git a/tests/0196/code.php b/tests/0196/code.php new file mode 100644 index 00000000..4c10b534 --- /dev/null +++ b/tests/0196/code.php @@ -0,0 +1,8 @@ + null +}; diff --git a/tests/0196/tokens.txt b/tests/0196/tokens.txt new file mode 100644 index 00000000..0171e009 --- /dev/null +++ b/tests/0196/tokens.txt @@ -0,0 +1,181 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 4, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 4, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 7, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 6, + 4, + ), + }, + Token { + kind: Match, + span: ( + 6, + 6, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 12, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 6, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 15, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 17, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 7, + 5, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 6, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 7, + 7, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 8, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 7, + 9, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 10, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 7, + 13, + ), + }, + Token { + kind: Null, + span: ( + 7, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 8, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 8, + 2, + ), + }, +] diff --git a/tests/0197/ast.txt b/tests/0197/ast.txt new file mode 100644 index 00000000..12ec3baa --- /dev/null +++ b/tests/0197/ast.txt @@ -0,0 +1,51 @@ +[ + Expression { + expr: Infix { + lhs: Variable { + name: "a", + }, + op: Assign, + rhs: LiteralInteger { + i: 4, + }, + }, + }, + Expression { + expr: Infix { + lhs: Variable { + name: "b", + }, + op: Assign, + rhs: Match { + condition: Variable { + name: "a", + }, + arms: [ + MatchArm { + conditions: Some( + [ + LiteralInteger { + i: 1, + }, + LiteralInteger { + i: 2, + }, + LiteralInteger { + i: 3, + }, + LiteralInteger { + i: 4, + }, + ], + ), + body: Null, + }, + MatchArm { + conditions: None, + body: Null, + }, + ], + }, + }, + }, +] diff --git a/tests/0197/code.php b/tests/0197/code.php new file mode 100644 index 00000000..8fd624ee --- /dev/null +++ b/tests/0197/code.php @@ -0,0 +1,10 @@ + null, + // seems weird, but PHP considers this valid. + default, => null, +}; diff --git a/tests/0197/tokens.txt b/tests/0197/tokens.txt new file mode 100644 index 00000000..daaa2bf2 --- /dev/null +++ b/tests/0197/tokens.txt @@ -0,0 +1,232 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 4, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 4, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 7, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 6, + 4, + ), + }, + Token { + kind: Match, + span: ( + 6, + 6, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 12, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 6, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 15, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 17, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 7, + 5, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 6, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 7, + 7, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 8, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 7, + 9, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 10, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 7, + 13, + ), + }, + Token { + kind: Null, + span: ( + 7, + 16, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 20, + ), + }, + Token { + kind: Comment( + "// seems weird, but PHP considers this valid.", + ), + span: ( + 8, + 5, + ), + }, + Token { + kind: Default, + span: ( + 9, + 5, + ), + }, + Token { + kind: Comma, + span: ( + 9, + 12, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 9, + 14, + ), + }, + Token { + kind: Null, + span: ( + 9, + 17, + ), + }, + Token { + kind: Comma, + span: ( + 9, + 21, + ), + }, + Token { + kind: RightBrace, + span: ( + 10, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 10, + 2, + ), + }, +] diff --git a/tests/0198/code.php b/tests/0198/code.php new file mode 100644 index 00000000..b96f9f87 --- /dev/null +++ b/tests/0198/code.php @@ -0,0 +1,10 @@ + null + 2 => null +}; + diff --git a/tests/0198/parser-error.txt b/tests/0198/parser-error.txt new file mode 100644 index 00000000..3e2e3605 --- /dev/null +++ b/tests/0198/parser-error.txt @@ -0,0 +1 @@ +ExpectedToken(["`}`"], Some("int"), (8, 5)) -> Parse Error: unexpected token `int`, expecting `}` on line 8 column 5 diff --git a/tests/0198/tokens.txt b/tests/0198/tokens.txt new file mode 100644 index 00000000..f7383b12 --- /dev/null +++ b/tests/0198/tokens.txt @@ -0,0 +1,186 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 4, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 4, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 7, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 6, + 4, + ), + }, + Token { + kind: Match, + span: ( + 6, + 6, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 12, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 6, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 15, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 17, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 5, + ), + }, + Token { + kind: LiteralInteger( + 34, + ), + span: ( + 7, + 6, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 9, + ), + }, + Token { + kind: LiteralInteger( + 21, + ), + span: ( + 7, + 11, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 7, + 14, + ), + }, + Token { + kind: Null, + span: ( + 7, + 17, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 8, + 5, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 8, + 7, + ), + }, + Token { + kind: Null, + span: ( + 8, + 10, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 2, + ), + }, +] diff --git a/tests/0199/code.php b/tests/0199/code.php new file mode 100644 index 00000000..0d73131e --- /dev/null +++ b/tests/0199/code.php @@ -0,0 +1,10 @@ + null + 2 => null +}; + diff --git a/tests/0199/parser-error.txt b/tests/0199/parser-error.txt new file mode 100644 index 00000000..185bb2b3 --- /dev/null +++ b/tests/0199/parser-error.txt @@ -0,0 +1 @@ +ExpectedToken(["`=>`"], Some("int"), (7, 10)) -> Parse Error: unexpected token `int`, expecting `=>` on line 7 column 10 diff --git a/tests/0199/tokens.txt b/tests/0199/tokens.txt new file mode 100644 index 00000000..f0193b52 --- /dev/null +++ b/tests/0199/tokens.txt @@ -0,0 +1,179 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 4, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 4, + 4, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 4, + 6, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 7, + ), + }, + Token { + kind: Variable( + "b", + ), + span: ( + 6, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 6, + 4, + ), + }, + Token { + kind: Match, + span: ( + 6, + 6, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 12, + ), + }, + Token { + kind: Variable( + "a", + ), + span: ( + 6, + 13, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 15, + ), + }, + Token { + kind: LeftBrace, + span: ( + 6, + 17, + ), + }, + Token { + kind: LeftParen, + span: ( + 7, + 5, + ), + }, + Token { + kind: LiteralInteger( + 34, + ), + span: ( + 7, + 6, + ), + }, + Token { + kind: RightParen, + span: ( + 7, + 8, + ), + }, + Token { + kind: LiteralInteger( + 21, + ), + span: ( + 7, + 10, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 7, + 13, + ), + }, + Token { + kind: Null, + span: ( + 7, + 16, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 8, + 5, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 8, + 7, + ), + }, + Token { + kind: Null, + span: ( + 8, + 10, + ), + }, + Token { + kind: RightBrace, + span: ( + 9, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 9, + 2, + ), + }, +] diff --git a/tests/0200/code.php b/tests/0200/code.php new file mode 100644 index 00000000..94791731 --- /dev/null +++ b/tests/0200/code.php @@ -0,0 +1,3 @@ + Parse Error: unexpected token `C`, expecting `{` on line 3 column 22 diff --git a/tests/0200/tokens.txt b/tests/0200/tokens.txt new file mode 100644 index 00000000..676429d9 --- /dev/null +++ b/tests/0200/tokens.txt @@ -0,0 +1,66 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "A", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: Implements, + span: ( + 3, + 9, + ), + }, + Token { + kind: Identifier( + "B", + ), + span: ( + 3, + 20, + ), + }, + Token { + kind: Identifier( + "C", + ), + span: ( + 3, + 22, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 24, + ), + }, + Token { + kind: RightBrace, + span: ( + 3, + 25, + ), + }, +] diff --git a/tests/0201/ast.txt b/tests/0201/ast.txt new file mode 100644 index 00000000..d57ba24a --- /dev/null +++ b/tests/0201/ast.txt @@ -0,0 +1,18 @@ +[ + Class { + name: Identifier { + name: "A", + }, + extends: None, + implements: [ + Identifier { + name: "B", + }, + Identifier { + name: "C", + }, + ], + body: [], + flags: [], + }, +] diff --git a/tests/0201/code.php b/tests/0201/code.php new file mode 100644 index 00000000..22a148b1 --- /dev/null +++ b/tests/0201/code.php @@ -0,0 +1,3 @@ + Parse Error: unexpected token `C`, expecting `{` on line 3 column 21 diff --git a/tests/0202/tokens.txt b/tests/0202/tokens.txt new file mode 100644 index 00000000..b8be980f --- /dev/null +++ b/tests/0202/tokens.txt @@ -0,0 +1,66 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Enum, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "A", + ), + span: ( + 3, + 6, + ), + }, + Token { + kind: Implements, + span: ( + 3, + 8, + ), + }, + Token { + kind: Identifier( + "B", + ), + span: ( + 3, + 19, + ), + }, + Token { + kind: Identifier( + "C", + ), + span: ( + 3, + 21, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 23, + ), + }, + Token { + kind: RightBrace, + span: ( + 3, + 24, + ), + }, +] diff --git a/tests/0203/ast.txt b/tests/0203/ast.txt new file mode 100644 index 00000000..295fdfa9 --- /dev/null +++ b/tests/0203/ast.txt @@ -0,0 +1,16 @@ +[ + UnitEnum { + name: Identifier { + name: "A", + }, + implements: [ + Identifier { + name: "B", + }, + Identifier { + name: "C", + }, + ], + body: [], + }, +] diff --git a/tests/0203/code.php b/tests/0203/code.php new file mode 100644 index 00000000..1f2bdee0 --- /dev/null +++ b/tests/0203/code.php @@ -0,0 +1,3 @@ + Parse Error: unexpected token `,`, expecting `;` on line 5 column 28 diff --git a/tests/0206/tokens.txt b/tests/0206/tokens.txt new file mode 100644 index 00000000..ea5ea16b --- /dev/null +++ b/tests/0206/tokens.txt @@ -0,0 +1,174 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Use, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 4, + 9, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 10, + ), + }, + Token { + kind: Identifier( + "b", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 13, + ), + }, + Token { + kind: Identifier( + "c", + ), + span: ( + 4, + 15, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 17, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: DoubleColon, + span: ( + 5, + 10, + ), + }, + Token { + kind: Identifier( + "s", + ), + span: ( + 5, + 12, + ), + }, + Token { + kind: Insteadof, + span: ( + 5, + 14, + ), + }, + Token { + kind: Identifier( + "b", + ), + span: ( + 5, + 24, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 25, + ), + }, + Token { + kind: Identifier( + "c", + ), + span: ( + 5, + 27, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 28, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 29, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0207/ast.txt b/tests/0207/ast.txt new file mode 100644 index 00000000..d1bfd9dc --- /dev/null +++ b/tests/0207/ast.txt @@ -0,0 +1,45 @@ +[ + Class { + name: Identifier { + name: "foo", + }, + extends: None, + implements: [], + body: [ + TraitUse { + traits: [ + Identifier { + name: "a", + }, + Identifier { + name: "b", + }, + Identifier { + name: "c", + }, + ], + adaptations: [ + Precedence { + trait: Some( + Identifier { + name: "a", + }, + ), + method: Identifier { + name: "s", + }, + insteadof: [ + Identifier { + name: "b", + }, + Identifier { + name: "c", + }, + ], + }, + ], + }, + ], + flags: [], + }, +] diff --git a/tests/0207/code.php b/tests/0207/code.php new file mode 100644 index 00000000..84adf6df --- /dev/null +++ b/tests/0207/code.php @@ -0,0 +1,7 @@ + Parse Error: unexpected token `,`, expecting `{` on line 4 column 16 diff --git a/tests/0208/tokens.txt b/tests/0208/tokens.txt new file mode 100644 index 00000000..88b3d964 --- /dev/null +++ b/tests/0208/tokens.txt @@ -0,0 +1,174 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Use, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 4, + 9, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 10, + ), + }, + Token { + kind: Identifier( + "b", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 13, + ), + }, + Token { + kind: Identifier( + "c", + ), + span: ( + 4, + 15, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 16, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 18, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: DoubleColon, + span: ( + 5, + 10, + ), + }, + Token { + kind: Identifier( + "s", + ), + span: ( + 5, + 12, + ), + }, + Token { + kind: Insteadof, + span: ( + 5, + 14, + ), + }, + Token { + kind: Identifier( + "b", + ), + span: ( + 5, + 24, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 25, + ), + }, + Token { + kind: Identifier( + "c", + ), + span: ( + 5, + 27, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 28, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0209/code.php b/tests/0209/code.php new file mode 100644 index 00000000..9cb9a8fd --- /dev/null +++ b/tests/0209/code.php @@ -0,0 +1,5 @@ + Parse Error: unexpected token `,`, expecting `;` on line 4 column 16 diff --git a/tests/0209/tokens.txt b/tests/0209/tokens.txt new file mode 100644 index 00000000..6969a6b4 --- /dev/null +++ b/tests/0209/tokens.txt @@ -0,0 +1,103 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Use, + span: ( + 4, + 5, + ), + }, + Token { + kind: Identifier( + "a", + ), + span: ( + 4, + 9, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 10, + ), + }, + Token { + kind: Identifier( + "b", + ), + span: ( + 4, + 12, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 13, + ), + }, + Token { + kind: Identifier( + "c", + ), + span: ( + 4, + 15, + ), + }, + Token { + kind: Comma, + span: ( + 4, + 16, + ), + }, + Token { + kind: SemiColon, + span: ( + 4, + 17, + ), + }, + Token { + kind: RightBrace, + span: ( + 5, + 1, + ), + }, +] From 5888e3a69a80b4b5e84d1dee669b7d4a16c4b898 Mon Sep 17 00:00:00 2001 From: Saif Eddin Gmati <29315886+azjezz@users.noreply.github.com> Date: Sat, 3 Dec 2022 19:23:12 +0100 Subject: [PATCH 12/13] fix: don't allow multiple default arms for match expression --- src/parser/ast.rs | 8 +- src/parser/error.rs | 2 + src/parser/mod.rs | 36 +++-- tests/0193/ast.txt | 1 + tests/0194/ast.txt | 31 ++-- tests/0195/ast.txt | 31 ++-- tests/0196/ast.txt | 31 ++-- tests/0197/ast.txt | 37 +++-- tests/0210/code.php | 7 + tests/0210/parser-error.txt | 1 + tests/0210/tokens.txt | 122 ++++++++++++++ tests/0211/ast.txt | 74 +++++++++ tests/0211/code.php | 11 ++ tests/0211/tokens.txt | 307 ++++++++++++++++++++++++++++++++++++ tests/0212/code.php | 7 + tests/0212/parser-error.txt | 1 + tests/0212/tokens.txt | 117 ++++++++++++++ 17 files changed, 747 insertions(+), 77 deletions(-) create mode 100644 tests/0210/code.php create mode 100644 tests/0210/parser-error.txt create mode 100644 tests/0210/tokens.txt create mode 100644 tests/0211/ast.txt create mode 100644 tests/0211/code.php create mode 100644 tests/0211/tokens.txt create mode 100644 tests/0212/code.php create mode 100644 tests/0212/parser-error.txt create mode 100644 tests/0212/tokens.txt diff --git a/src/parser/ast.rs b/src/parser/ast.rs index 7fa0307b..f035e257 100644 --- a/src/parser/ast.rs +++ b/src/parser/ast.rs @@ -706,6 +706,7 @@ pub enum Expression { }, Match { condition: Box, + default: Option>, arms: Vec, }, Throw { @@ -755,9 +756,14 @@ pub struct ClosureUse { pub by_ref: bool, } +#[derive(Debug, PartialEq, Clone)] +pub struct DefaultMatchArm { + pub body: Expression, +} + #[derive(Debug, PartialEq, Clone)] pub struct MatchArm { - pub conditions: Option>, + pub conditions: Vec, pub body: Expression, } diff --git a/src/parser/error.rs b/src/parser/error.rs index a201f392..90e41d6c 100644 --- a/src/parser/error.rs +++ b/src/parser/error.rs @@ -33,6 +33,7 @@ pub enum ParseError { MixingBracedAndUnBracedNamespaceDeclarations(Span), NestedNamespaceDeclarations(Span), ForbiddenTypeUsedInProperty(String, String, Type, Span), + MatchExpressionWithMultipleDefaultArms(Span), } impl Display for ParseError { @@ -78,6 +79,7 @@ impl Display for ParseError { Self::NestedNamespaceDeclarations(span) => write!(f, "Parse Error: Namespace declarations cannot be mixed on line {} column {}", span.0, span.1), Self::UnpredictableState(span) => write!(f, "Parse Error: Reached an unpredictable state on line {} column {}", span.0, span.1), Self::ForbiddenTypeUsedInProperty(class, prop, ty, span) => write!(f, "Parse Error: Property {}::${} cannot have type `{}` on line {} column {}", class, prop, ty, span.0, span.1), + Self::MatchExpressionWithMultipleDefaultArms(span) => write!(f, "Parse Error: Match expressions may only contain one default arm on line {} column {}", span.0, span.1), } } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 1aad12a8..b5aeb950 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -12,6 +12,7 @@ use crate::parser::error::ParseResult; use crate::parser::internal::ident::is_reserved_ident; use crate::parser::internal::precedence::{Associativity, Precedence}; use crate::parser::state::State; +use crate::prelude::DefaultMatchArm; pub mod ast; pub mod error; @@ -998,11 +999,18 @@ impl Parser { self.rparen(state)?; self.lbrace(state)?; + let mut default = None; let mut arms = Vec::new(); while state.current.kind != TokenKind::RightBrace { state.skip_comments(); - let conditions = if state.current.kind == TokenKind::Default { + if state.current.kind == TokenKind::Default { + if default.is_some() { + return Err(ParseError::MatchExpressionWithMultipleDefaultArms( + state.current.span, + )); + } + state.next(); // match conditions can have an extra comma at the end, including `default`. @@ -1010,7 +1018,11 @@ impl Parser { state.next(); } - None + expect_token!([TokenKind::DoubleArrow], state, "`=>`"); + + let body = self.expression(state, Precedence::Lowest)?; + + default = Some(Box::new(DefaultMatchArm { body })); } else { let mut conditions = Vec::new(); while state.current.kind != TokenKind::DoubleArrow { @@ -1023,14 +1035,16 @@ impl Parser { } } - Some(conditions) - }; - - expect_token!([TokenKind::DoubleArrow], state, "`=>`"); + if !conditions.is_empty() { + expect_token!([TokenKind::DoubleArrow], state, "`=>`"); + } else { + break; + } - let body = self.expression(state, Precedence::Lowest)?; + let body = self.expression(state, Precedence::Lowest)?; - arms.push(MatchArm { conditions, body }); + arms.push(MatchArm { conditions, body }); + } if state.current.kind == TokenKind::Comma { state.next(); @@ -1041,7 +1055,11 @@ impl Parser { self.rbrace(state)?; - Expression::Match { condition, arms } + Expression::Match { + condition, + default, + arms, + } } TokenKind::Array => { let mut items = vec![]; diff --git a/tests/0193/ast.txt b/tests/0193/ast.txt index 72172f78..4928eb98 100644 --- a/tests/0193/ast.txt +++ b/tests/0193/ast.txt @@ -20,6 +20,7 @@ condition: Variable { name: "a", }, + default: None, arms: [], }, }, diff --git a/tests/0194/ast.txt b/tests/0194/ast.txt index f07faedc..63ccfc37 100644 --- a/tests/0194/ast.txt +++ b/tests/0194/ast.txt @@ -20,24 +20,23 @@ condition: Variable { name: "a", }, + default: None, arms: [ MatchArm { - conditions: Some( - [ - LiteralInteger { - i: 1, - }, - LiteralInteger { - i: 2, - }, - LiteralInteger { - i: 3, - }, - LiteralInteger { - i: 4, - }, - ], - ), + conditions: [ + LiteralInteger { + i: 1, + }, + LiteralInteger { + i: 2, + }, + LiteralInteger { + i: 3, + }, + LiteralInteger { + i: 4, + }, + ], body: Null, }, ], diff --git a/tests/0195/ast.txt b/tests/0195/ast.txt index f07faedc..63ccfc37 100644 --- a/tests/0195/ast.txt +++ b/tests/0195/ast.txt @@ -20,24 +20,23 @@ condition: Variable { name: "a", }, + default: None, arms: [ MatchArm { - conditions: Some( - [ - LiteralInteger { - i: 1, - }, - LiteralInteger { - i: 2, - }, - LiteralInteger { - i: 3, - }, - LiteralInteger { - i: 4, - }, - ], - ), + conditions: [ + LiteralInteger { + i: 1, + }, + LiteralInteger { + i: 2, + }, + LiteralInteger { + i: 3, + }, + LiteralInteger { + i: 4, + }, + ], body: Null, }, ], diff --git a/tests/0196/ast.txt b/tests/0196/ast.txt index f07faedc..63ccfc37 100644 --- a/tests/0196/ast.txt +++ b/tests/0196/ast.txt @@ -20,24 +20,23 @@ condition: Variable { name: "a", }, + default: None, arms: [ MatchArm { - conditions: Some( - [ - LiteralInteger { - i: 1, - }, - LiteralInteger { - i: 2, - }, - LiteralInteger { - i: 3, - }, - LiteralInteger { - i: 4, - }, - ], - ), + conditions: [ + LiteralInteger { + i: 1, + }, + LiteralInteger { + i: 2, + }, + LiteralInteger { + i: 3, + }, + LiteralInteger { + i: 4, + }, + ], body: Null, }, ], diff --git a/tests/0197/ast.txt b/tests/0197/ast.txt index 12ec3baa..ea309d0f 100644 --- a/tests/0197/ast.txt +++ b/tests/0197/ast.txt @@ -20,28 +20,27 @@ condition: Variable { name: "a", }, - arms: [ - MatchArm { - conditions: Some( - [ - LiteralInteger { - i: 1, - }, - LiteralInteger { - i: 2, - }, - LiteralInteger { - i: 3, - }, - LiteralInteger { - i: 4, - }, - ], - ), + default: Some( + DefaultMatchArm { body: Null, }, + ), + arms: [ MatchArm { - conditions: None, + conditions: [ + LiteralInteger { + i: 1, + }, + LiteralInteger { + i: 2, + }, + LiteralInteger { + i: 3, + }, + LiteralInteger { + i: 4, + }, + ], body: Null, }, ], diff --git a/tests/0210/code.php b/tests/0210/code.php new file mode 100644 index 00000000..82c213d4 --- /dev/null +++ b/tests/0210/code.php @@ -0,0 +1,7 @@ + 43, + default => 34, +}; diff --git a/tests/0210/parser-error.txt b/tests/0210/parser-error.txt new file mode 100644 index 00000000..68f81e7b --- /dev/null +++ b/tests/0210/parser-error.txt @@ -0,0 +1 @@ +MatchExpressionWithMultipleDefaultArms((6, 5)) -> Parse Error: Match expressions may only contain one default arm on line 6 column 5 diff --git a/tests/0210/tokens.txt b/tests/0210/tokens.txt new file mode 100644 index 00000000..bacc3c3e --- /dev/null +++ b/tests/0210/tokens.txt @@ -0,0 +1,122 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Match, + span: ( + 4, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 7, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 4, + 8, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 10, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 12, + ), + }, + Token { + kind: Default, + span: ( + 5, + 5, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 5, + 13, + ), + }, + Token { + kind: LiteralInteger( + 43, + ), + span: ( + 5, + 16, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 18, + ), + }, + Token { + kind: Default, + span: ( + 6, + 5, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 6, + 13, + ), + }, + Token { + kind: LiteralInteger( + 34, + ), + span: ( + 6, + 16, + ), + }, + Token { + kind: Comma, + span: ( + 6, + 18, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 2, + ), + }, +] diff --git a/tests/0211/ast.txt b/tests/0211/ast.txt new file mode 100644 index 00000000..9738efd9 --- /dev/null +++ b/tests/0211/ast.txt @@ -0,0 +1,74 @@ +[ + Expression { + expr: Match { + condition: Variable { + name: "s", + }, + default: Some( + DefaultMatchArm { + body: LiteralInteger { + i: 124, + }, + }, + ), + arms: [ + MatchArm { + conditions: [ + LiteralInteger { + i: 1, + }, + ], + body: LiteralInteger { + i: 2, + }, + }, + MatchArm { + conditions: [ + LiteralInteger { + i: 3, + }, + ], + body: LiteralInteger { + i: 4, + }, + }, + MatchArm { + conditions: [ + LiteralInteger { + i: 5, + }, + LiteralInteger { + i: 6, + }, + ], + body: LiteralInteger { + i: 4, + }, + }, + MatchArm { + conditions: [ + LiteralInteger { + i: 9, + }, + LiteralInteger { + i: 123, + }, + ], + body: LiteralInteger { + i: 4, + }, + }, + MatchArm { + conditions: [ + Identifier { + name: "_", + }, + ], + body: LiteralInteger { + i: 43, + }, + }, + ], + }, + }, +] diff --git a/tests/0211/code.php b/tests/0211/code.php new file mode 100644 index 00000000..3eaa2024 --- /dev/null +++ b/tests/0211/code.php @@ -0,0 +1,11 @@ + 2, + 3, => 4, + 5,6 => 4, + 9, 123, => 4, + _ => 43, // _ here is a constant + default => 124, +}; diff --git a/tests/0211/tokens.txt b/tests/0211/tokens.txt new file mode 100644 index 00000000..de817fae --- /dev/null +++ b/tests/0211/tokens.txt @@ -0,0 +1,307 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Match, + span: ( + 4, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 7, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 4, + 8, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 10, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 12, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 5, + 5, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 5, + 7, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 5, + 10, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 11, + ), + }, + Token { + kind: LiteralInteger( + 3, + ), + span: ( + 6, + 5, + ), + }, + Token { + kind: Comma, + span: ( + 6, + 6, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 6, + 8, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 6, + 11, + ), + }, + Token { + kind: Comma, + span: ( + 6, + 12, + ), + }, + Token { + kind: LiteralInteger( + 5, + ), + span: ( + 7, + 5, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 6, + ), + }, + Token { + kind: LiteralInteger( + 6, + ), + span: ( + 7, + 7, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 7, + 9, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 7, + 12, + ), + }, + Token { + kind: Comma, + span: ( + 7, + 13, + ), + }, + Token { + kind: LiteralInteger( + 9, + ), + span: ( + 8, + 5, + ), + }, + Token { + kind: Comma, + span: ( + 8, + 6, + ), + }, + Token { + kind: LiteralInteger( + 123, + ), + span: ( + 8, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 8, + 11, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 8, + 13, + ), + }, + Token { + kind: LiteralInteger( + 4, + ), + span: ( + 8, + 16, + ), + }, + Token { + kind: Comma, + span: ( + 8, + 17, + ), + }, + Token { + kind: Identifier( + "_", + ), + span: ( + 9, + 5, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 9, + 7, + ), + }, + Token { + kind: LiteralInteger( + 43, + ), + span: ( + 9, + 10, + ), + }, + Token { + kind: Comma, + span: ( + 9, + 12, + ), + }, + Token { + kind: Comment( + "// _ here is a constant", + ), + span: ( + 9, + 14, + ), + }, + Token { + kind: Default, + span: ( + 10, + 5, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 10, + 13, + ), + }, + Token { + kind: LiteralInteger( + 124, + ), + span: ( + 10, + 16, + ), + }, + Token { + kind: Comma, + span: ( + 10, + 19, + ), + }, + Token { + kind: RightBrace, + span: ( + 11, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 11, + 2, + ), + }, +] diff --git a/tests/0212/code.php b/tests/0212/code.php new file mode 100644 index 00000000..e2caf7da --- /dev/null +++ b/tests/0212/code.php @@ -0,0 +1,7 @@ + 2, + => 43, +}; diff --git a/tests/0212/parser-error.txt b/tests/0212/parser-error.txt new file mode 100644 index 00000000..bffaff62 --- /dev/null +++ b/tests/0212/parser-error.txt @@ -0,0 +1 @@ +ExpectedToken(["`}`"], Some("=>"), (6, 5)) -> Parse Error: unexpected token `=>`, expecting `}` on line 6 column 5 diff --git a/tests/0212/tokens.txt b/tests/0212/tokens.txt new file mode 100644 index 00000000..60817e88 --- /dev/null +++ b/tests/0212/tokens.txt @@ -0,0 +1,117 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Match, + span: ( + 4, + 1, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 7, + ), + }, + Token { + kind: Variable( + "s", + ), + span: ( + 4, + 8, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 10, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 12, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 5, + 5, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 5, + 7, + ), + }, + Token { + kind: LiteralInteger( + 2, + ), + span: ( + 5, + 10, + ), + }, + Token { + kind: Comma, + span: ( + 5, + 11, + ), + }, + Token { + kind: DoubleArrow, + span: ( + 6, + 5, + ), + }, + Token { + kind: LiteralInteger( + 43, + ), + span: ( + 6, + 8, + ), + }, + Token { + kind: Comma, + span: ( + 6, + 10, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 2, + ), + }, +] From 988b93e030d29234262cadf2255d9737086bd6f2 Mon Sep 17 00:00:00 2001 From: Saif Eddin Gmati <29315886+azjezz@users.noreply.github.com> Date: Sat, 3 Dec 2022 20:31:10 +0100 Subject: [PATCH 13/13] fix: support scoped types --- src/parser/error.rs | 2 + src/parser/internal/classish.rs | 92 +++++---- src/parser/internal/classish_statement.rs | 6 +- src/parser/internal/functions.rs | 4 +- src/parser/internal/params.rs | 4 +- src/parser/internal/types.rs | 158 ++++++--------- src/parser/state.rs | 73 ++++++- tests/0149/ast.txt | 49 ----- tests/0149/parser-error.txt | 1 + tests/0150/ast.txt | 49 ----- tests/0150/parser-error.txt | 1 + tests/0151/parser-error.txt | 1 + tests/0213/code.php | 6 + tests/0213/parser-error.txt | 1 + tests/0213/tokens.txt | 101 ++++++++++ tests/0214/code.php | 7 + tests/0214/parser-error.txt | 1 + tests/0214/tokens.txt | 147 ++++++++++++++ tests/{0151 => 0215}/ast.txt | 31 ++- tests/0215/code.php | 10 + tests/0215/tokens.txt | 174 +++++++++++++++++ tests/0216/code.php | 7 + tests/0216/parser-error.txt | 1 + tests/0216/tokens.txt | 147 ++++++++++++++ tests/0217/ast.txt | 50 +++++ tests/0217/code.php | 9 + tests/0217/tokens.txt | 198 +++++++++++++++++++ tests/0218/code.php | 8 + tests/0218/parser-error.txt | 1 + tests/0218/tokens.txt | 168 ++++++++++++++++ tests/0219/code.php | 7 + tests/0219/parser-error.txt | 1 + tests/0219/tokens.txt | 147 ++++++++++++++ tests/0220/ast.txt | 58 ++++++ tests/0220/code.php | 10 + tests/0220/tokens.txt | 226 ++++++++++++++++++++++ 36 files changed, 1690 insertions(+), 266 deletions(-) delete mode 100644 tests/0149/ast.txt create mode 100644 tests/0149/parser-error.txt delete mode 100644 tests/0150/ast.txt create mode 100644 tests/0150/parser-error.txt create mode 100644 tests/0151/parser-error.txt create mode 100644 tests/0213/code.php create mode 100644 tests/0213/parser-error.txt create mode 100644 tests/0213/tokens.txt create mode 100644 tests/0214/code.php create mode 100644 tests/0214/parser-error.txt create mode 100644 tests/0214/tokens.txt rename tests/{0151 => 0215}/ast.txt (59%) create mode 100644 tests/0215/code.php create mode 100644 tests/0215/tokens.txt create mode 100644 tests/0216/code.php create mode 100644 tests/0216/parser-error.txt create mode 100644 tests/0216/tokens.txt create mode 100644 tests/0217/ast.txt create mode 100644 tests/0217/code.php create mode 100644 tests/0217/tokens.txt create mode 100644 tests/0218/code.php create mode 100644 tests/0218/parser-error.txt create mode 100644 tests/0218/tokens.txt create mode 100644 tests/0219/code.php create mode 100644 tests/0219/parser-error.txt create mode 100644 tests/0219/tokens.txt create mode 100644 tests/0220/ast.txt create mode 100644 tests/0220/code.php create mode 100644 tests/0220/tokens.txt diff --git a/src/parser/error.rs b/src/parser/error.rs index 90e41d6c..2b8264ef 100644 --- a/src/parser/error.rs +++ b/src/parser/error.rs @@ -34,6 +34,7 @@ pub enum ParseError { NestedNamespaceDeclarations(Span), ForbiddenTypeUsedInProperty(String, String, Type, Span), MatchExpressionWithMultipleDefaultArms(Span), + CannotFindTypeInCurrentScope(String, Span), } impl Display for ParseError { @@ -80,6 +81,7 @@ impl Display for ParseError { Self::UnpredictableState(span) => write!(f, "Parse Error: Reached an unpredictable state on line {} column {}", span.0, span.1), Self::ForbiddenTypeUsedInProperty(class, prop, ty, span) => write!(f, "Parse Error: Property {}::${} cannot have type `{}` on line {} column {}", class, prop, ty, span.0, span.1), Self::MatchExpressionWithMultipleDefaultArms(span) => write!(f, "Parse Error: Match expressions may only contain one default arm on line {} column {}", span.0, span.1), + Self::CannotFindTypeInCurrentScope(ty, span) => write!(f, "Parse Error: Cannot find type `{}` in this scope on line {} on column {}", ty, span.0, span.1), } } } diff --git a/src/parser/internal/classish.rs b/src/parser/internal/classish.rs index bc6143a8..55622af7 100644 --- a/src/parser/internal/classish.rs +++ b/src/parser/internal/classish.rs @@ -21,47 +21,53 @@ impl Parser { let name = self.ident(state)?; - scoped!(state, Scope::Class(name.clone(), flags.clone()), { - let mut extends: Option = None; + let mut has_parent = false; + let mut extends: Option = None; - if state.current.kind == TokenKind::Extends { - state.next(); - extends = Some(self.full_name(state)?.into()); - } + if state.current.kind == TokenKind::Extends { + state.next(); + extends = Some(self.full_name(state)?.into()); + has_parent = true; + } - let implements = if state.current.kind == TokenKind::Implements { - state.next(); + scoped!( + state, + Scope::Class(name.clone(), flags.clone(), has_parent), + { + let implements = if state.current.kind == TokenKind::Implements { + state.next(); - self.at_least_one_comma_separated::(state, &|parser, state| { - Ok(parser.full_name(state)?.into()) - })? - } else { - Vec::new() - }; + self.at_least_one_comma_separated::(state, &|parser, state| { + Ok(parser.full_name(state)?.into()) + })? + } else { + Vec::new() + }; - self.lbrace(state)?; + self.lbrace(state)?; - let mut body = Vec::new(); - while state.current.kind != TokenKind::RightBrace { - state.gather_comments(); + let mut body = Vec::new(); + while state.current.kind != TokenKind::RightBrace { + state.gather_comments(); - if state.current.kind == TokenKind::RightBrace { - state.clear_comments(); - break; + if state.current.kind == TokenKind::RightBrace { + state.clear_comments(); + break; + } + + body.push(self.class_like_statement(state)?); } + self.rbrace(state)?; - body.push(self.class_like_statement(state)?); + Ok(Statement::Class { + name: name.into(), + extends, + implements, + body, + flags, + }) } - self.rbrace(state)?; - - Ok(Statement::Class { - name: name.into(), - extends, - implements, - body, - flags, - }) - }) + ) } pub(in crate::parser) fn interface_definition( @@ -140,20 +146,22 @@ impl Parser { expect_token!([TokenKind::New], state, ["`new`"]); expect_token!([TokenKind::Class], state, ["`class`"]); - scoped!(state, Scope::AnonymousClass, { - let mut args = vec![]; + let mut args = vec![]; - if state.current.kind == TokenKind::LeftParen { - args = self.args_list(state)?; - } + if state.current.kind == TokenKind::LeftParen { + args = self.args_list(state)?; + } - let mut extends: Option = None; + let mut has_parent = false; + let mut extends: Option = None; - if state.current.kind == TokenKind::Extends { - state.next(); - extends = Some(self.full_name(state)?.into()); - } + if state.current.kind == TokenKind::Extends { + state.next(); + extends = Some(self.full_name(state)?.into()); + has_parent = true; + } + scoped!(state, Scope::AnonymousClass(has_parent), { let mut implements = Vec::new(); if state.current.kind == TokenKind::Implements { state.next(); diff --git a/src/parser/internal/classish_statement.rs b/src/parser/internal/classish_statement.rs index 2a27f943..1805667e 100644 --- a/src/parser/internal/classish_statement.rs +++ b/src/parser/internal/classish_statement.rs @@ -147,9 +147,8 @@ impl Parser { } let class_name: String = expected_scope!([ - Scope::Class(name, _) => state.named(&name), - Scope::Trait(name) => state.named(&name), - Scope::AnonymousClass => state.named(&"class@anonymous".into()), + Scope::Trait(name) | Scope::Class(name, _, _) => state.named(&name), + Scope::AnonymousClass(_) => state.named(&"class@anonymous".into()), ], state); if flags.contains(&PropertyFlag::Readonly) { @@ -158,7 +157,6 @@ impl Parser { } if value.is_some() { - return Err(ParseError::ReadonlyPropertyHasDefaultValue(class_name, var.to_string(), state.current.span)); } } diff --git a/src/parser/internal/functions.rs b/src/parser/internal/functions.rs index 1cc6e766..7e4b9550 100644 --- a/src/parser/internal/functions.rs +++ b/src/parser/internal/functions.rs @@ -199,7 +199,7 @@ impl Parser { let name = self.ident_maybe_reserved(state)?; let has_body = expected_scope!([ - Scope::Class(_, cf) => { + Scope::Class(_, cf, _) => { if !cf.contains(&ClassFlag::Abstract) && flags.contains(&MethodFlag::Abstract) { return Err(ParseError::AbstractModifierOnNonAbstractClassMethod( state.current.span, @@ -220,7 +220,7 @@ impl Parser { true }, - Scope::AnonymousClass => true, + Scope::AnonymousClass(_) => true, ], state); scoped!(state, Scope::Method(name.clone(), flags.clone()), { diff --git a/src/parser/internal/params.rs b/src/parser/internal/params.rs index a639037d..f8d8bade 100644 --- a/src/parser/internal/params.rs +++ b/src/parser/internal/params.rs @@ -29,14 +29,14 @@ impl Parser { // can only have abstract ctor Scope::Interface(_) => 1, // can only have concret ctor - Scope::AnonymousClass => { + Scope::AnonymousClass(_) => { class_name = state.named(&"class@anonymous".into()); 2 } // can have either abstract or concret ctor, // depens on method flag. - Scope::Class(name, _) | Scope::Trait(name) => { + Scope::Class(name, _, _) | Scope::Trait(name) => { if flags.contains(&MethodFlag::Abstract) { 1 } else { diff --git a/src/parser/internal/types.rs b/src/parser/internal/types.rs index 8af34bf3..58aacd83 100644 --- a/src/parser/internal/types.rs +++ b/src/parser/internal/types.rs @@ -37,9 +37,7 @@ impl Parser { } pub(in crate::parser) fn get_type(&self, state: &mut State) -> ParseResult { - let ty = self.maybe_nullable(state, &|state| { - self.maybe_static(state, &|state| self.get_simple_type(state)) - })?; + let ty = self.maybe_nullable(state, &|state| self.get_simple_type(state))?; if ty.nullable() { return Ok(ty); @@ -57,7 +55,7 @@ impl Parser { let mut types = vec![ty]; while !state.is_eof() { - let ty = self.maybe_static(state, &|state| self.get_simple_type(state))?; + let ty = self.get_simple_type(state)?; if ty.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( ty, @@ -91,7 +89,7 @@ impl Parser { let mut types = vec![ty]; while !state.is_eof() { - let ty = self.maybe_static(state, &|state| self.get_simple_type(state))?; + let ty = self.get_simple_type(state)?; if ty.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( ty, @@ -122,7 +120,7 @@ impl Parser { return Ok(Some(self.get_type(state)?)); } - let ty = self.maybe_optional_static(state, &|state| self.get_optional_simple_type(state)); + let ty = self.get_optional_simple_type(state)?; match ty { Some(ty) => { @@ -138,7 +136,7 @@ impl Parser { let mut types = vec![ty]; while !state.is_eof() { - let ty = self.maybe_static(state, &|state| self.get_simple_type(state))?; + let ty = self.get_simple_type(state)?; if ty.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( ty, @@ -172,7 +170,7 @@ impl Parser { let mut types = vec![ty]; while !state.is_eof() { - let ty = self.maybe_static(state, &|state| self.get_simple_type(state))?; + let ty = self.get_simple_type(state)?; if ty.standalone() { return Err(ParseError::StandaloneTypeUsedInCombination( ty, @@ -198,32 +196,44 @@ impl Parser { } } - fn get_optional_simple_type(&self, state: &mut State) -> Option { + fn get_optional_simple_type(&self, state: &mut State) -> ParseResult> { match state.current.kind.clone() { TokenKind::Array => { state.next(); - Some(Type::Array) + Ok(Some(Type::Array)) } TokenKind::Callable => { state.next(); - Some(Type::Callable) + Ok(Some(Type::Callable)) } TokenKind::Null => { state.next(); - Some(Type::Null) + Ok(Some(Type::Null)) } TokenKind::True => { state.next(); - Some(Type::True) + Ok(Some(Type::True)) } TokenKind::False => { state.next(); - Some(Type::False) + Ok(Some(Type::False)) + } + TokenKind::Static => { + state.next(); + + if !state.has_class_scope { + return Err(ParseError::CannotFindTypeInCurrentScope( + "static".to_owned(), + state.current.span, + )); + } + + Ok(Some(Type::StaticReference)) } TokenKind::Identifier(id) => { state.next(); @@ -231,34 +241,54 @@ impl Parser { let name = &id[..]; let lowered_name = name.to_ascii_lowercase(); match lowered_name.as_slice() { - b"void" => Some(Type::Void), - b"never" => Some(Type::Never), - b"float" => Some(Type::Float), - b"bool" => Some(Type::Boolean), - b"int" => Some(Type::Integer), - b"string" => Some(Type::String), - b"object" => Some(Type::Object), - b"mixed" => Some(Type::Mixed), - b"iterable" => Some(Type::Iterable), - b"null" => Some(Type::Null), - b"true" => Some(Type::True), - b"false" => Some(Type::False), - b"array" => Some(Type::Array), - b"callable" => Some(Type::Callable), - _ => Some(Type::Identifier(id.into())), + b"void" => Ok(Some(Type::Void)), + b"never" => Ok(Some(Type::Never)), + b"float" => Ok(Some(Type::Float)), + b"bool" => Ok(Some(Type::Boolean)), + b"int" => Ok(Some(Type::Integer)), + b"string" => Ok(Some(Type::String)), + b"object" => Ok(Some(Type::Object)), + b"mixed" => Ok(Some(Type::Mixed)), + b"iterable" => Ok(Some(Type::Iterable)), + b"null" => Ok(Some(Type::Null)), + b"true" => Ok(Some(Type::True)), + b"false" => Ok(Some(Type::False)), + b"array" => Ok(Some(Type::Array)), + b"callable" => Ok(Some(Type::Callable)), + b"self" => { + if !state.has_class_scope { + return Err(ParseError::CannotFindTypeInCurrentScope( + "self".to_owned(), + state.current.span, + )); + } + + Ok(Some(Type::SelfReference)) + } + b"parent" => { + if !state.has_class_parent_scope { + return Err(ParseError::CannotFindTypeInCurrentScope( + "parent".to_owned(), + state.current.span, + )); + } + + Ok(Some(Type::ParentReference)) + } + _ => Ok(Some(Type::Identifier(id.into()))), } } TokenKind::QualifiedIdentifier(id) | TokenKind::FullyQualifiedIdentifier(id) => { state.next(); - Some(Type::Identifier(id.into())) + Ok(Some(Type::Identifier(id.into()))) } - _ => None, + _ => Ok(None), } } fn get_simple_type(&self, state: &mut State) -> ParseResult { - self.get_optional_simple_type(state) + self.get_optional_simple_type(state)? .ok_or_else(|| expected_token!(["a type"], state)) } @@ -282,68 +312,4 @@ impl Parser { otherwise(state) } } - - fn maybe_static( - &self, - state: &mut State, - otherwise: &(dyn Fn(&mut State) -> ParseResult), - ) -> ParseResult { - if TokenKind::Static == state.current.kind { - state.next(); - - return Ok(Type::StaticReference); - } - - if let TokenKind::Identifier(id) = &state.current.kind { - let name = &id[..]; - let lowered_name = name.to_ascii_lowercase(); - match lowered_name.as_slice() { - b"self" => { - state.next(); - - return Ok(Type::SelfReference); - } - b"parent" => { - state.next(); - - return Ok(Type::ParentReference); - } - _ => {} - }; - } - - otherwise(state) - } - - fn maybe_optional_static( - &self, - state: &mut State, - otherwise: &(dyn Fn(&mut State) -> Option), - ) -> Option { - if TokenKind::Static == state.current.kind { - state.next(); - - return Some(Type::StaticReference); - } - - if let TokenKind::Identifier(id) = &state.current.kind { - let name = &id[..]; - let lowered_name = name.to_ascii_lowercase(); - match lowered_name.as_slice() { - b"self" => { - state.next(); - - return Some(Type::SelfReference); - } - b"parent" => { - state.next(); - - return Some(Type::ParentReference); - } - _ => {} - }; - } - - otherwise(state) - } } diff --git a/src/parser/state.rs b/src/parser/state.rs index 01f1f24c..1ac9a4a6 100644 --- a/src/parser/state.rs +++ b/src/parser/state.rs @@ -21,10 +21,10 @@ pub enum Scope { BracedNamespace(Option), Interface(ByteString), - Class(ByteString, Vec), + Class(ByteString, Vec, bool), Trait(ByteString), Enum(ByteString, bool), - AnonymousClass, + AnonymousClass(bool), Function(ByteString), Method(ByteString, Vec), @@ -40,6 +40,8 @@ pub struct State { pub iter: IntoIter, pub comments: Vec, pub namespace_type: Option, + pub has_class_scope: bool, + pub has_class_parent_scope: bool, } impl State { @@ -53,6 +55,8 @@ impl State { iter, comments: vec![], namespace_type: None, + has_class_scope: false, + has_class_parent_scope: false, } } @@ -113,10 +117,12 @@ impl State { } self.stack.push_back(scope); + self.update_scope(); } pub fn exit(&mut self) { self.stack.pop_back(); + self.update_scope(); } pub fn skip_comments(&mut self) { @@ -152,4 +158,67 @@ impl State { self.current = self.peek.clone(); self.peek = self.iter.next().unwrap_or_default() } + + fn update_scope(&mut self) { + self.has_class_scope = self.has_class_scope(); + self.has_class_parent_scope = if self.has_class_scope { + self.has_class_parent_scope() + } else { + false + }; + } + + fn has_class_scope(&self) -> bool { + for scope in self.stack.iter().rev() { + match &scope { + Scope::ArrowFunction(s) | Scope::AnonymousFunction(s) => { + // if it's a static closure, don't allow `static` type. + if *s { + return false; + } + } + Scope::BracedNamespace(_) | Scope::Namespace(_) | Scope::Function(_) => { + return false; + } + _ => { + return true; + } + }; + } + + false + } + + fn has_class_parent_scope(&self) -> bool { + for scope in self.stack.iter().rev() { + match &scope { + Scope::ArrowFunction(s) | Scope::AnonymousFunction(s) => { + // static closures don't have a parent + if *s { + return false; + } + } + Scope::BracedNamespace(_) | Scope::Namespace(_) | Scope::Function(_) => { + return false; + } + // we don't know if the trait has a parent at this point + // the only time that we can determine if a trait has a parent + // is when it's used in a class. + Scope::Trait(_) => { + return true; + } + // interfaces and enums don't have a parent. + Scope::Interface(_) | Scope::Enum(_, _) => { + return false; + } + Scope::Class(_, _, has_parent) | Scope::AnonymousClass(has_parent) => { + return *has_parent; + } + // we can't determine this from method, wait until we reach the classish scope. + Scope::Method(_, _) => {} + }; + } + + false + } } diff --git a/tests/0149/ast.txt b/tests/0149/ast.txt deleted file mode 100644 index 9a41f989..00000000 --- a/tests/0149/ast.txt +++ /dev/null @@ -1,49 +0,0 @@ -[ - Namespace { - name: "A\B\C\D\E", - body: [ - Noop, - Function { - name: Identifier { - name: "foo", - }, - params: [ - Param { - name: Variable { - name: "s", - }, - type: Some( - String, - ), - variadic: false, - default: None, - flags: [], - by_ref: false, - }, - ], - body: [ - Expression { - expr: Call { - target: Identifier { - name: "exit", - }, - args: [ - Arg { - name: None, - value: LiteralInteger { - i: 0, - }, - unpack: false, - }, - ], - }, - }, - ], - return_type: Some( - SelfReference, - ), - by_ref: false, - }, - ], - }, -] diff --git a/tests/0149/parser-error.txt b/tests/0149/parser-error.txt new file mode 100644 index 00000000..22f92879 --- /dev/null +++ b/tests/0149/parser-error.txt @@ -0,0 +1 @@ +CannotFindTypeInCurrentScope("self", (5, 31)) -> Parse Error: Cannot find type `self` in this scope on line 5 on column 31 diff --git a/tests/0150/ast.txt b/tests/0150/ast.txt deleted file mode 100644 index 62b12c2c..00000000 --- a/tests/0150/ast.txt +++ /dev/null @@ -1,49 +0,0 @@ -[ - Namespace { - name: "A\B\C\D\E", - body: [ - Noop, - Function { - name: Identifier { - name: "foo", - }, - params: [ - Param { - name: Variable { - name: "s", - }, - type: Some( - String, - ), - variadic: false, - default: None, - flags: [], - by_ref: false, - }, - ], - body: [ - Expression { - expr: Call { - target: Identifier { - name: "exit", - }, - args: [ - Arg { - name: None, - value: LiteralInteger { - i: 0, - }, - unpack: false, - }, - ], - }, - }, - ], - return_type: Some( - StaticReference, - ), - by_ref: false, - }, - ], - }, -] diff --git a/tests/0150/parser-error.txt b/tests/0150/parser-error.txt new file mode 100644 index 00000000..0ccb7536 --- /dev/null +++ b/tests/0150/parser-error.txt @@ -0,0 +1 @@ +CannotFindTypeInCurrentScope("static", (5, 33)) -> Parse Error: Cannot find type `static` in this scope on line 5 on column 33 diff --git a/tests/0151/parser-error.txt b/tests/0151/parser-error.txt new file mode 100644 index 00000000..bfe4f2b9 --- /dev/null +++ b/tests/0151/parser-error.txt @@ -0,0 +1 @@ +CannotFindTypeInCurrentScope("parent", (5, 33)) -> Parse Error: Cannot find type `parent` in this scope on line 5 on column 33 diff --git a/tests/0213/code.php b/tests/0213/code.php new file mode 100644 index 00000000..c562e81e --- /dev/null +++ b/tests/0213/code.php @@ -0,0 +1,6 @@ + Parse Error: Cannot find type `parent` in this scope on line 5 on column 34 diff --git a/tests/0213/tokens.txt b/tests/0213/tokens.txt new file mode 100644 index 00000000..f0436c35 --- /dev/null +++ b/tests/0213/tokens.txt @@ -0,0 +1,101 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Interface, + span: ( + 4, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 4, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 15, + ), + }, + Token { + kind: Public, + span: ( + 5, + 5, + ), + }, + Token { + kind: Function, + span: ( + 5, + 12, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 5, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 24, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 25, + ), + }, + Token { + kind: Colon, + span: ( + 5, + 26, + ), + }, + Token { + kind: Identifier( + "parent", + ), + span: ( + 5, + 28, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 34, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 1, + ), + }, +] diff --git a/tests/0214/code.php b/tests/0214/code.php new file mode 100644 index 00000000..4d663a33 --- /dev/null +++ b/tests/0214/code.php @@ -0,0 +1,7 @@ + Parse Error: Cannot find type `parent` in this scope on line 6 on column 34 diff --git a/tests/0214/tokens.txt b/tests/0214/tokens.txt new file mode 100644 index 00000000..1209cf39 --- /dev/null +++ b/tests/0214/tokens.txt @@ -0,0 +1,147 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Interface, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "s", + ), + span: ( + 3, + 11, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 13, + ), + }, + Token { + kind: RightBrace, + span: ( + 3, + 14, + ), + }, + Token { + kind: Interface, + span: ( + 5, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 5, + 11, + ), + }, + Token { + kind: Extends, + span: ( + 5, + 15, + ), + }, + Token { + kind: Identifier( + "s", + ), + span: ( + 5, + 23, + ), + }, + Token { + kind: LeftBrace, + span: ( + 5, + 25, + ), + }, + Token { + kind: Public, + span: ( + 6, + 5, + ), + }, + Token { + kind: Function, + span: ( + 6, + 12, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 6, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 6, + 24, + ), + }, + Token { + kind: RightParen, + span: ( + 6, + 25, + ), + }, + Token { + kind: Colon, + span: ( + 6, + 26, + ), + }, + Token { + kind: Identifier( + "parent", + ), + span: ( + 6, + 28, + ), + }, + Token { + kind: SemiColon, + span: ( + 6, + 34, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0151/ast.txt b/tests/0215/ast.txt similarity index 59% rename from tests/0151/ast.txt rename to tests/0215/ast.txt index 01d50219..cb3547fc 100644 --- a/tests/0151/ast.txt +++ b/tests/0215/ast.txt @@ -1,26 +1,14 @@ [ - Namespace { - name: "A\B\C\D\E", + Trait { + name: Identifier { + name: "foo", + }, body: [ - Noop, - Function { + Method { name: Identifier { - name: "foo", + name: "bar", }, - params: [ - Param { - name: Variable { - name: "s", - }, - type: Some( - String, - ), - variadic: false, - default: None, - flags: [], - by_ref: false, - }, - ], + params: [], body: [ Expression { expr: Call { @@ -31,7 +19,7 @@ Arg { name: None, value: LiteralInteger { - i: 0, + i: 1, }, unpack: false, }, @@ -39,6 +27,9 @@ }, }, ], + flags: [ + Public, + ], return_type: Some( ParentReference, ), diff --git a/tests/0215/code.php b/tests/0215/code.php new file mode 100644 index 00000000..d3d0c80d --- /dev/null +++ b/tests/0215/code.php @@ -0,0 +1,10 @@ + Parse Error: Cannot find type `parent` in this scope on line 4 on column 35 diff --git a/tests/0216/tokens.txt b/tests/0216/tokens.txt new file mode 100644 index 00000000..17ec2143 --- /dev/null +++ b/tests/0216/tokens.txt @@ -0,0 +1,147 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Class, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 7, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 11, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 24, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 25, + ), + }, + Token { + kind: Colon, + span: ( + 4, + 26, + ), + }, + Token { + kind: Identifier( + "parent", + ), + span: ( + 4, + 28, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 35, + ), + }, + Token { + kind: Identifier( + "exit", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 13, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 5, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 15, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0217/ast.txt b/tests/0217/ast.txt new file mode 100644 index 00000000..6abe17fd --- /dev/null +++ b/tests/0217/ast.txt @@ -0,0 +1,50 @@ +[ + Class { + name: Identifier { + name: "s", + }, + extends: None, + implements: [], + body: [], + flags: [], + }, + Class { + name: Identifier { + name: "foo", + }, + extends: Some( + Identifier { + name: "s", + }, + ), + implements: [], + body: [ + Method { + name: Identifier { + name: "bar", + }, + params: [], + body: [ + Return { + value: Some( + New { + target: Identifier { + name: "s", + }, + args: [], + }, + ), + }, + ], + flags: [ + Public, + ], + return_type: Some( + ParentReference, + ), + by_ref: false, + }, + ], + flags: [], + }, +] diff --git a/tests/0217/code.php b/tests/0217/code.php new file mode 100644 index 00000000..75231022 --- /dev/null +++ b/tests/0217/code.php @@ -0,0 +1,9 @@ + Parse Error: Cannot find type `parent` in this scope on line 4 on column 35 diff --git a/tests/0218/tokens.txt b/tests/0218/tokens.txt new file mode 100644 index 00000000..73ed1a5c --- /dev/null +++ b/tests/0218/tokens.txt @@ -0,0 +1,168 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Variable( + "e", + ), + span: ( + 3, + 1, + ), + }, + Token { + kind: Equals, + span: ( + 3, + 4, + ), + }, + Token { + kind: New, + span: ( + 3, + 6, + ), + }, + Token { + kind: Class, + span: ( + 3, + 10, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 16, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 24, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 25, + ), + }, + Token { + kind: Colon, + span: ( + 4, + 26, + ), + }, + Token { + kind: Identifier( + "parent", + ), + span: ( + 4, + 28, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 35, + ), + }, + Token { + kind: Identifier( + "exit", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 13, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 5, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 15, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, + Token { + kind: SemiColon, + span: ( + 7, + 2, + ), + }, +] diff --git a/tests/0219/code.php b/tests/0219/code.php new file mode 100644 index 00000000..1e29e2ee --- /dev/null +++ b/tests/0219/code.php @@ -0,0 +1,7 @@ + Parse Error: Cannot find type `parent` in this scope on line 4 on column 35 diff --git a/tests/0219/tokens.txt b/tests/0219/tokens.txt new file mode 100644 index 00000000..481627fd --- /dev/null +++ b/tests/0219/tokens.txt @@ -0,0 +1,147 @@ +[ + Token { + kind: OpenTag( + Full, + ), + span: ( + 1, + 1, + ), + }, + Token { + kind: Enum, + span: ( + 3, + 1, + ), + }, + Token { + kind: Identifier( + "foo", + ), + span: ( + 3, + 6, + ), + }, + Token { + kind: LeftBrace, + span: ( + 3, + 10, + ), + }, + Token { + kind: Public, + span: ( + 4, + 5, + ), + }, + Token { + kind: Function, + span: ( + 4, + 12, + ), + }, + Token { + kind: Identifier( + "bar", + ), + span: ( + 4, + 21, + ), + }, + Token { + kind: LeftParen, + span: ( + 4, + 24, + ), + }, + Token { + kind: RightParen, + span: ( + 4, + 25, + ), + }, + Token { + kind: Colon, + span: ( + 4, + 26, + ), + }, + Token { + kind: Identifier( + "parent", + ), + span: ( + 4, + 28, + ), + }, + Token { + kind: LeftBrace, + span: ( + 4, + 35, + ), + }, + Token { + kind: Identifier( + "exit", + ), + span: ( + 5, + 9, + ), + }, + Token { + kind: LeftParen, + span: ( + 5, + 13, + ), + }, + Token { + kind: LiteralInteger( + 1, + ), + span: ( + 5, + 14, + ), + }, + Token { + kind: RightParen, + span: ( + 5, + 15, + ), + }, + Token { + kind: SemiColon, + span: ( + 5, + 16, + ), + }, + Token { + kind: RightBrace, + span: ( + 6, + 5, + ), + }, + Token { + kind: RightBrace, + span: ( + 7, + 1, + ), + }, +] diff --git a/tests/0220/ast.txt b/tests/0220/ast.txt new file mode 100644 index 00000000..dd5a826f --- /dev/null +++ b/tests/0220/ast.txt @@ -0,0 +1,58 @@ +[ + Class { + name: Identifier { + name: "bar", + }, + extends: None, + implements: [], + body: [], + flags: [], + }, + Noop, + Expression { + expr: Infix { + lhs: Variable { + name: "e", + }, + op: Assign, + rhs: New { + target: AnonymousClass { + extends: Some( + Identifier { + name: "bar", + }, + ), + implements: [], + body: [ + Method { + name: Identifier { + name: "bar", + }, + params: [], + body: [ + Return { + value: Some( + New { + target: Identifier { + name: "bar", + }, + args: [], + }, + ), + }, + ], + flags: [ + Public, + ], + return_type: Some( + ParentReference, + ), + by_ref: false, + }, + ], + }, + args: [], + }, + }, + }, +] diff --git a/tests/0220/code.php b/tests/0220/code.php new file mode 100644 index 00000000..a0ab3223 --- /dev/null +++ b/tests/0220/code.php @@ -0,0 +1,10 @@ +