diff --git a/src/Lexer/Lexer.php b/src/Lexer/Lexer.php index 68889642..40747d95 100644 --- a/src/Lexer/Lexer.php +++ b/src/Lexer/Lexer.php @@ -105,16 +105,11 @@ public function tokenize(string $s): array assert($this->regexp !== null); assert($this->types !== null); - preg_match_all($this->regexp, $s, $tokens, PREG_SET_ORDER); - - $count = count($this->types); - foreach ($tokens as &$match) { - for ($i = 1; $i <= $count; $i++) { - if ($match[$i] !== null && $match[$i] !== '') { - $match = [$match[0], $this->types[$i - 1]]; - break; - } - } + preg_match_all($this->regexp, $s, $matches, PREG_SET_ORDER); + + $tokens = []; + foreach ($matches as $match) { + $tokens[] = [$match[0], $this->types[count($match) - 2]]; } $tokens[] = ['', self::TOKEN_END];