From d9a485445dbb15d9b9b3e9154c43eb743b22f160 Mon Sep 17 00:00:00 2001
From: Richard van Velzen <richard@frank.nl>
Date: Fri, 10 Jun 2022 09:19:42 +0200
Subject: [PATCH] Optimize Lexer::tokenize()

---
 src/Lexer/Lexer.php | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/src/Lexer/Lexer.php b/src/Lexer/Lexer.php
index 68889642..40747d95 100644
--- a/src/Lexer/Lexer.php
+++ b/src/Lexer/Lexer.php
@@ -105,16 +105,11 @@ public function tokenize(string $s): array
 		assert($this->regexp !== null);
 		assert($this->types !== null);
 
-		preg_match_all($this->regexp, $s, $tokens, PREG_SET_ORDER);
-
-		$count = count($this->types);
-		foreach ($tokens as &$match) {
-			for ($i = 1; $i <= $count; $i++) {
-				if ($match[$i] !== null && $match[$i] !== '') {
-					$match = [$match[0], $this->types[$i - 1]];
-					break;
-				}
-			}
+		preg_match_all($this->regexp, $s, $matches, PREG_SET_ORDER);
+
+		$tokens = [];
+		foreach ($matches as $match) {
+			$tokens[] = [$match[0], $this->types[count($match) - 2]];
 		}
 
 		$tokens[] = ['', self::TOKEN_END];