Make a full copy of the parser and apply our modifications to it

dev/updgrade-calcite-parser

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#--------------------------------------------------------------------
+
+# Adopts base Calcite parser changes 
+#
+# Establishes a git friendly merge situation:
+#
+# Creates a commit which matches the original state of the calcite parser;
+# To this point creates to alternates:
+#   * one with local customizations
+#   * another with all the upstream updates
+# merges the two branches to obtain the upgrade state
+#
+
+[ $# -ne 2 ] && echo -e "updates base parser sources.\n usage: $0 <old_calcite_version> <new_calcite_version>" && exit 1
+
+CALCITE_OLD=$1
+CALCITE_NEW=$2
+
+set -e
+set -x
+
+BRANCH=`git name-rev --name-only HEAD`
+
+REPO=.git/calcite-upgrade
+rm -rf "$REPO"
+git clone $PWD --reference $PWD --branch $BRANCH $REPO
+
+cd "$REPO"
+git checkout -b curr-changes
+
+mvn -q generate-sources -pl sql -Dcalcite.version=$CALCITE_OLD -Pskip-static-checks
+cp -r sql/target/calcite-base-parser/codegen/./  sql/src/main/codegen/./
+git commit -m 'current reverse' -a
+git revert --no-edit HEAD
+# HEAD is now at the same as before; but parent are the base calcite changes
+
+git branch base-changes curr-changes^
+git checkout base-changes
+git show|patch -p0 -R # undo temproarily to ensure maven runs
+
+mvn -q generate-sources -pl sql -Dcalcite.version=$CALCITE_NEW -Pskip-static-checks
+cp -r sql/target/calcite-base-parser/codegen/./  sql/src/main/codegen/./
+
+git commit --allow-empty -m base-changes -a
+git checkout -b new-state
+git merge --no-edit curr-changes
+
+echo ok
+cd -
+
+git remote remove calcite-upgrade &>/dev/null || echo -n
+git remote add -f calcite-upgrade "$REPO"
+
+
+echo "merge branch calcite-upgrade/curr-changes if satisfied with those changes"
+

pom.xml

@@ -84,9 +84,8 @@
         <scala.library.version>2.13.14</scala.library.version>
         <avatica.version>1.23.0</avatica.version>
         <avro.version>1.11.3</avro.version>
-        <!-- When updating Calcite, also propagate updates to these files which we've copied and modified:
-             default_config.fmpp
-             sql/src/main/codegen/includes/*
+        <!--
+            The base calcite parser was copied into the project; when updating Calcite run dev/updgrade-calcite-parser to adopt upstream changes
           -->
         <calcite.version>1.35.0</calcite.version>
         <confluent.version>6.2.12</confluent.version>

sql/pom.xml

@@ -333,8 +333,8 @@
         </executions>
       </plugin>
 
-      <!-- Following plugins and their configurations are used to generate the custom Calcite's SQL parser for Druid -->
-      <!-- Extracts the Parser.jj from Calcite to ${project.build.directory}, where all the Freemarker templates are -->
+      <!-- Extracts the base parser from the calcite-core artifcat. -->
+      <!-- This is provided to have the base available - to compare changes to -->
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
@@ -353,50 +353,16 @@
                   <version>${calcite.version}</version>
                   <type>jar</type>
                   <overWrite>true</overWrite>
-                  <outputDirectory>${project.build.directory}/</outputDirectory>
-                  <includes>**/Parser.jj</includes>
-                </artifactItem>
-                <artifactItem>
-                  <groupId>org.apache.calcite</groupId>
-                  <artifactId>calcite-core</artifactId>
-                  <version>${calcite.version}</version>
-                  <type>jar</type>
-                  <overWrite>true</overWrite>
-                  <outputDirectory>${project.build.directory}/</outputDirectory>
-                  <includes>**/default_config.fmpp</includes>
+                  <outputDirectory>${project.build.directory}/calcite-base-parser</outputDirectory>
+                  <includes>codegen/**</includes>
                 </artifactItem>
               </artifactItems>
             </configuration>
           </execution>
         </executions>
       </plugin>
 
-      <!-- Copy the templates present in the codegen directory of druid-sql containing custom SQL rules to
-        ${project.build.directory}/codegen -->
-      <plugin>
-        <artifactId>maven-resources-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-fmpp-resources</id>
-            <phase>generate-sources</phase>
-            <goals>
-              <goal>copy-resources</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/codegen</outputDirectory>
-              <resources>
-                <resource>
-                  <directory>src/main/codegen</directory>
-                  <filtering>false</filtering>
-                </resource>
-              </resources>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-
-      <!-- "Plugs in" the Calcite's Parser.jj with the variables present in config.fmpp. These contain the custom rules
-        as well as the class to which the custom implementation will get generated -->
+      <!-- Applies the fmpp templates to produce the final Parser.jj -->
       <plugin>
         <groupId>com.googlecode.fmpp-maven-plugin</groupId>
         <artifactId>fmpp-maven-plugin</artifactId>
@@ -408,9 +374,9 @@
               <goal>generate</goal>
             </goals>
             <configuration>
-              <cfgFile>${project.build.directory}/codegen/config.fmpp</cfgFile>
+              <cfgFile>src/main/codegen/config.fmpp</cfgFile>
               <outputDirectory>${project.build.directory}/generated-sources</outputDirectory>
-              <templateDirectory>${project.build.directory}/codegen/templates</templateDirectory>
+              <templateDirectory>src/main/codegen/templates</templateDirectory>
             </configuration>
           </execution>
         </executions>
@@ -440,38 +406,6 @@
         </executions>
       </plugin>
 
-      <!-- This plugin is used to replace the production rule for FROM clause in the calcite grammar.
-           It is done by a search and replace since calcite doesn't allow to override production rules generally
-           in its grammar (override is possible only if there's a hook for it in the grammar). And the FROM clause
-           doesn't contain any hook for extension. For the custom changes done in
-           extension, please check from.ftl file in sql module.
-      -->
-      <plugin>
-        <groupId>com.google.code.maven-replacer-plugin</groupId>
-        <artifactId>replacer</artifactId>
-        <version>1.5.3</version>
-        <executions>
-          <execution>
-            <phase>generate-sources</phase>
-            <goals>
-              <goal>replace</goal>
-            </goals>
-          </execution>
-        </executions>
-        <configuration>
-          <basedir>${project.build.directory}/generated-sources/org/apache/druid/sql/calcite/parser</basedir>
-          <includes>
-            <include>**/DruidSqlParserImpl.java</include>
-          </includes>
-          <replacements>
-            <replacement>
-              <token>fromClause = FromClause</token>
-              <value>fromClause = DruidFromClause</value>
-            </replacement>
-          </replacements>
-        </configuration>
-      </plugin>
-
       <!-- Adds the path of the generated parser to the build path -->
       <plugin>
         <groupId>org.codehaus.mojo</groupId>

sql/src/main/codegen/config.fmpp

@@ -100,10 +100,8 @@ data: {
     # "dataTypeParserMethods".
     implementationFiles: [
       "common.ftl"
-      "insert.ftl"
       "explain.ftl"
       "replace.ftl"
-      "from.ftl"
     ]
   }
 }