Treat #> as operator

CONTRIBUTING.md

@@ -20,3 +20,7 @@ vendor/bin/phpunit --testdox
 echo '' | vendor/bin/phpcs
 vendor/bin/phpstan analyze
 ```
+
+## Regenerating expected output
+
+To regenerate expected tests output, run `bin/regenerate-expected-output`.

bin/regenerate-expected-output

@@ -0,0 +1,29 @@
+#!/usr/bin/env php
+<?php
+
+use Doctrine\SqlFormatter\CliHighlighter;
+use Doctrine\SqlFormatter\HtmlHighlighter;
+use Doctrine\SqlFormatter\NullHighlighter;
+use Doctrine\SqlFormatter\SqlFormatter;
+use Doctrine\SqlFormatter\Tests\SqlFormatterTest;
+
+require __DIR__ . '/../vendor/autoload.php';
+
+$formatter = new SqlFormatter(new HtmlHighlighter());
+
+updateExpected('format-highlight.html', [$formatter, 'format']);
+updateExpected('format.txt', [new SqlFormatter(new NullHighlighter()), 'format']);
+updateExpected('highlight.html', [$formatter, 'highlight']);
+updateExpected('clihighlight.txt', [new SqlFormatter(new CliHighlighter()), 'format']);
+updateExpected('compress.txt', [$formatter, 'compress']);
+
+function updateExpected(string $filename, callable $highlight): void
+{
+	$data = [];
+
+	foreach (SqlFormatterTest::fileSqlData() as $sql) {
+		$data[] = rtrim($highlight($sql), "\n");
+	}
+
+	file_put_contents(__DIR__ . '/../tests/' . $filename, implode("\n---\n", $data) . "\n");
+}

src/Tokenizer.php

@@ -842,7 +842,7 @@ private function makeTokenizeRegexes(): array
 
         return [
             Token::TOKEN_TYPE_WHITESPACE => '\s+',
-            Token::TOKEN_TYPE_COMMENT => '(?:--|#)[^\n]*+',
+            Token::TOKEN_TYPE_COMMENT => '(?:--|#(?!>))[^\n]*+', // #>, #>> and <#> are PostgreSQL operators
             Token::TOKEN_TYPE_BLOCK_COMMENT => '/\*(?:[^*]+|\*(?!/))*+(?:\*|$)(?:/|$)',
             // 1. backtick quoted string using `` to escape
             // 2. square bracket quoted string (SQL Server) using ]] to escape

tests/SqlFormatterTest.php

@@ -101,7 +101,7 @@ public function testUsePre(): void
     }
 
     /** @return string[] */
-    private static function fileSqlData(): array
+    public static function fileSqlData(): array
     {
         $contents = file_get_contents(__DIR__ . '/sql.sql');
         assert($contents !== false);

tests/TokenizerTest.php

@@ -1658,6 +1658,20 @@ public static function tokenizeData(): Generator
             ],
             '/* foo...',
         ];
+
+        yield 'PostgreSQL operator' => [
+            [
+                new Token(Token::TOKEN_TYPE_RESERVED_TOPLEVEL, 'select'),
+                new Token(Token::TOKEN_TYPE_WHITESPACE, ' '),
+                new Token(Token::TOKEN_TYPE_WORD, 'json'),
+                new Token(Token::TOKEN_TYPE_WHITESPACE, ' '),
+                new Token(Token::TOKEN_TYPE_BOUNDARY, '#'),
+                new Token(Token::TOKEN_TYPE_BOUNDARY, '>'),
+                new Token(Token::TOKEN_TYPE_WHITESPACE, ' '),
+                new Token(Token::TOKEN_TYPE_RESERVED, 'null'),
+            ],
+            'select json #> null',
+        ];
     }
 
     public function testTokenizeLongConcat(): void

tests/clihighlight.txt

@@ -1205,3 +1205,9 @@ MY_NON_TOP_LEVEL_KEYWORD_FX_3();
 CREATE TABLE t (
   c VARCHAR(20)
 ) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB
+---
+SELECT
+  '{}' :: json #> '{}'
+---
+SELECT
+  vector1 <#> vector2

tests/compress.txt

@@ -111,3 +111,7 @@ SELECT a FROM test STRAIGHT_JOIN test2 ON test.id = test2.id
 SELECT t.id, t.start, t.end, t.end AS e2, t.limit, t.begin, t.case, t.when, t.then, t.else FROM t WHERE t.start = t.end
 ---
 CREATE TABLE t (c VARCHAR(20)) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB
+---
+SELECT '{}'::json #> '{}'
+---
+SELECT vector1 <#> vector2

tests/format-highlight.html

@@ -1205,3 +1205,9 @@
 <pre style="color: black; background-color: white;"><span style="font-weight:bold;">CREATE</span> <span style="font-weight:bold;">TABLE</span> <span style="color: #333;">t</span> (
   <span style="color: #333;">c</span> <span style="font-weight:bold;">VARCHAR</span>(<span style="color: green;">20</span>)
 ) <span style="font-weight:bold;">DEFAULT</span> <span style="font-weight:bold;">CHARACTER</span> <span style="font-weight:bold;">SET</span> <span style="color: #333;">utf8mb4</span> <span style="font-weight:bold;">ENGINE</span> <span >=</span> <span style="color: #333;">InnoDB</span></pre>
+---
+<pre style="color: black; background-color: white;"><span style="font-weight:bold;">SELECT</span>
+  <span style="color: blue;">'{}'</span> <span >::</span> <span style="color: #333;">json</span> <span >#</span><span >&gt;</span> <span style="color: blue;">'{}'</span></pre>
+---
+<pre style="color: black; background-color: white;"><span style="font-weight:bold;">SELECT</span>
+  <span style="color: #333;">vector1</span> <span >&lt;</span><span >#</span><span >&gt;</span> <span style="color: #333;">vector2</span></pre>

tests/format.txt

@@ -1203,3 +1203,9 @@ WHERE
 CREATE TABLE t (
   c VARCHAR(20)
 ) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB
+---
+SELECT
+  '{}' :: json #> '{}'
+---
+SELECT
+  vector1 <#> vector2

tests/highlight.html

@@ -425,3 +425,7 @@
 <span style="font-weight:bold;">WHERE</span> <span style="color: #333;">t</span><span >.</span><span style="color: #333;">start</span> <span >=</span> <span style="color: #333;">t</span><span >.</span><span style="color: #333;">end</span></pre>
 ---
 <pre style="color: black; background-color: white;"><span style="font-weight:bold;">CREATE</span> <span style="font-weight:bold;">TABLE</span> <span style="color: #333;">t</span> (<span style="color: #333;">c</span> <span style="font-weight:bold;">VARCHAR</span>(<span style="color: green;">20</span>)) <span style="font-weight:bold;">DEFAULT</span> <span style="font-weight:bold;">CHARACTER</span> <span style="font-weight:bold;">SET</span> <span style="color: #333;">utf8mb4</span> <span style="font-weight:bold;">ENGINE</span> <span >=</span> <span style="color: #333;">InnoDB</span></pre>
+---
+<pre style="color: black; background-color: white;"><span style="font-weight:bold;">SELECT</span> <span style="color: blue;">'{}'</span><span >::</span><span style="color: #333;">json</span> <span >#</span><span >&gt;</span> <span style="color: blue;">'{}'</span></pre>
+---
+<pre style="color: black; background-color: white;"><span style="font-weight:bold;">SELECT</span> <span style="color: #333;">vector1</span> <span >&lt;</span><span >#</span><span >&gt;</span> <span style="color: #333;">vector2</span></pre>

tests/sql.sql

@@ -425,3 +425,7 @@ FROM t
 WHERE t.start = t.end
 ---
 CREATE TABLE t (c VARCHAR(20)) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB
+---
+SELECT '{}'::json #> '{}'
+---
+SELECT vector1 <#> vector2