Fix issue with regex limit getting hit

Since we added logic that does not rely on the matches from the regex we can simplify everything by just matching on html tags
zacharymarshal · May 19, 2017 · 46f7a5d · 46f7a5d
1 parent e9fa3e9
commit 46f7a5d
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 1 deletion.
diff --git a/src/Tokenizer.php b/src/Tokenizer.php
@@ -27,7 +27,7 @@ public function getToken($string)
         };
         $token_patterns = [
             "/^\s+/"                   => ['type' => 'whitespace', 'value_fn' => $returnMatches],
-            "/^<.+?[\s]*\/?[\s]*>/" => [
+            "/^<.+>/" => [
                 'type' => 'html-tag',
                 'value_fn' => function($matches, $string) {
                     $matched = '';

diff --git a/tests/large_image.html b/tests/large_image.html
diff --git a/tests/lexer-test.php b/tests/lexer-test.php
@@ -218,3 +218,62 @@
         ]
     );
 });
+
+Test::create('should tokenize crazy ass images', function (Test $test) {
+    $img_html = file_get_contents(__DIR__ . '/large_image.html');
+    $tokenizer = new Tokenizer;
+    $test->equals($tokenizer->tokenize($img_html), [
+        ['type' => 'html-tag', 'value' => trim($img_html)],
+        ['type' => 'whitespace', 'value' => "\n"],
+    ]);
+});
+
+Test::create('should tokenize multiple tags', function(Test $test) {
+    $tokenizer = new Tokenizer;
+    $tokens = $tokenizer->tokenize("<br><br /><p>test</p>");
+    $test->equals(
+        $tokens,
+        [
+            ['type' => 'html-tag', 'value' => '<br>'],
+            ['type' => 'html-tag', 'value' => '<br />'],
+            ['type' => 'html-tag', 'value' => '<p>'],
+            ['type' => 'word', 'value' => 'test'],
+            ['type' => 'html-tag', 'value' => '</p>'],
+        ]
+    );
+});
+
+Test::create('should tokenize spaces before/after tag', function(Test $test) {
+    $tokenizer = new Tokenizer;
+    $tokens = $tokenizer->tokenize("< br ><br / >< hr style='color: blue' >");
+    $test->equals(
+        $tokens,
+        [
+            ['type' => 'html-tag', 'value' => '< br >'],
+            ['type' => 'html-tag', 'value' => '<br / >'],
+            ['type' => 'html-tag', 'value' => '< hr style=\'color: blue\' >'],
+        ]
+    );
+});
+
+Test::create('should tokenize gt/lt in attributes', function(Test $test) {
+    $tokenizer = new Tokenizer;
+    $tokens = $tokenizer->tokenize("<hr style='color: >blue'>");
+    $test->equals($tokens, [
+        ['type' => 'html-tag', 'value' => '<hr style=\'color: >blue\'>'],
+    ]);
+});
+
+Test::create('nested html', function(Test $test) {
+    $tokenizer = new Tokenizer;
+    $tokens = $tokenizer->tokenize("<p><hr/><img/></p>");
+    $test->equals(
+        $tokens,
+        [
+            ['type' => 'html-tag', 'value' => '<p>'],
+            ['type' => 'html-tag', 'value' => '<hr/>'],
+            ['type' => 'html-tag', 'value' => '<img/>'],
+            ['type' => 'html-tag', 'value' => '</p>'],
+        ]
+    );
+});