Replace grammar by antlr/grammars-v4's

README.md

@@ -56,9 +56,5 @@ Please also have a look at our documentation sections on [usage](https://pytsql.
 
 ## Credits
 
-Inspiration drawn from [codebuff](https://github.com/antlr/codebuff/blob/master/grammars/org/antlr/codebuff/tsql.g4).
+Grammar is based on [antlr4/grammars-v4](https://github.com/antlr/grammars-v4/tree/master/sql/tsql).
 
-Original authors:
-[Alex Gonopolskiy](https://github.com/agonopol)
-[Kevin Klein](https://github.com/kklein)
-[Encho Mishinev](https://github.com/EnchoMishinevQC)

environment.yml

@@ -17,3 +17,4 @@ dependencies:
   - pyodbc
   - speedy-antlr-tool==1.4.1
   - antlr4-python3-runtime==4.11.1
+  - click

src/pytsql/grammar/TSqlParser.g4

@@ -6054,6 +6054,7 @@ id_
     | DOUBLE_QUOTE_BLANK
     | SQUARE_BRACKET_ID
     | keyword
+    | RAW
     ;
 
 simple_id

src/pytsql/grammar/adjust_antlrs_grammar.py

@@ -0,0 +1,76 @@
+import click
+from pathlib import Path
+import re
+
+PROTECTED_RULE_ELEMENT_LABELS = [
+    "from",
+    "id",
+    "property",
+    "str",
+    "format",
+    "input",
+    "with",
+    "type",
+    "file"
+]
+CONFLICTING_NAMES_ON_WINDOWS = [
+    "PLATFORM"
+]
+
+# CLI rename_protected_rule_element_labels
+@click.group()
+def rename_protected_rule_element_labels_cli():
+    pass
+
+
+@rename_protected_rule_element_labels_cli.command()
+@click.option(
+    "-f", "--filepath",
+    type=click.Path(path_type=Path),
+    help="Path to a .g4 grammar file.",
+)
+def rename_protected_rule_element_labels(filepath: str) -> None:
+    """
+    Some rule element labels have the same name as protected objects in C++. Hence, rename those labels.
+    """
+    rename_policy_suffix = "_label"
+
+    with open(filepath, 'r+') as file:
+        grammar = file.read()
+        for label in PROTECTED_RULE_ELEMENT_LABELS:
+            grammar = re.sub(fr"(?<=[\( |]){label}(?=[ ]?\=)", f"{label}{rename_policy_suffix}", grammar)
+
+        file.seek(0, 0)
+        file.write(grammar)
+
+# CLI: add_removal_of_specified_names_on_windows
+@click.group()
+def removal_of_specified_names_on_windows_cli():
+    pass
+
+@removal_of_specified_names_on_windows_cli.command()
+@click.option(
+    "-f", "--filepath",
+    type=click.Path(path_type=Path),
+    help="Path to an auto-generated file by `speedy-antlr-tool`",
+)
+def add_removal_of_specified_names_on_windows(filepath: str) -> None:
+    """
+    Remove certain protected names for C++ compilation on Windows.
+
+    For more see: https://learn.microsoft.com/en-us/windows/win32/menurc/-undef
+    """
+    undef_list = "\n".join([f"#undef {name}" for name in CONFLICTING_NAMES_ON_WINDOWS])
+
+    with open(filepath, "r+") as file:
+
+        content = file.read()
+        file.seek(0, 0)
+        file.write(undef_list + content)
+
+cli = click.CommandCollection(
+    sources=[rename_protected_rule_element_labels_cli, removal_of_specified_names_on_windows_cli]
+)
+
+if __name__ == "__main__":
+    cli()

src/pytsql/grammar/cpp_src/TSqlLexer.h

@@ -1,11 +1,11 @@
-
+#undef PLATFORM
 // Generated from TSqlLexer.g4 by ANTLR 4.11.1
 
 #pragma once
 
 
 #include "antlr4-runtime.h"
-#undef PLATFORM
+
 
 
 

src/pytsql/grammar/cpp_src/TSqlParser.h

@@ -19533,6 +19533,7 @@ class  TSqlParser : public antlr4::Parser {
     antlr4::tree::TerminalNode *DOUBLE_QUOTE_BLANK();
     antlr4::tree::TerminalNode *SQUARE_BRACKET_ID();
     KeywordContext *keyword();
+    antlr4::tree::TerminalNode *RAW();
 
 
     virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;

src/pytsql/grammar/helper_generate_parsers.sh

@@ -8,6 +8,13 @@ cd "$( dirname "${BASH_SOURCE[0]}" )"
 
 antlr4="java -jar $ANTLR4_JAR_FILEPATH"
 
+# Optionally: Download grammar
+# curl -o TSqlParser.g4 https://raw.githubusercontent.com/antlr/grammars-v4/master/sql/tsql/TSqlParser.g4
+# curl -o TSqlLexer.g4 https://raw.githubusercontent.com/antlr/grammars-v4/master/sql/tsql/TSqlLexer.g4
+
+# Adjust grammar for C++ target
+python adjust_antlrs_grammar.py rename-protected-rule-element-labels -f TSqlParser.g4
+
 # Generate C++ target with visitor
 $antlr4 -Dlanguage=Cpp -o cpp_src TSqlLexer.g4
 $antlr4 -Dlanguage=Cpp -visitor -no-listener -o cpp_src TSqlParser.g4
@@ -26,3 +33,6 @@ generate(
     entry_rule_names=["tsql_file"],
 )
 EOF
+
+# Adjust auto-generated files for protected names on Windows
+python adjust_antlrs_grammar.py add-removal-of-specified-names-on-windows --filepath cpp_src/TSqlLexer.h

tests/unit/test_split.py

@@ -336,3 +336,14 @@ def test_declare_as_defined_table_type():
     DECLARE @test dbo.test_type
     """
     assert len(_split(seed)) == 1
+
+
+def test_token_raw_as_id():
+    seed = """
+    CREATE SCHEMA raw;
+    CREATE TABLE raw.tbl (
+        a INT
+    );
+    SELECT * FROM raw.tbl;
+    """
+    assert len(_split(seed)) == 3