Skip to content

[SPARK-53753][BUILD][SQL] shade antlr4-runtime #52490

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
+159 −65
Open

[SPARK-53753][BUILD][SQL] shade antlr4-runtime #52490

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
26dd2fd
[SPARK-53753][BUILD][SQL] shade antlr4-runtime
vrozov Sep 26, 2025
58b11fe
rebase
vrozov Nov 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion dev/deps/spark-deps-hadoop-3-hive-2.3
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar
aliyun-sdk-oss/3.13.2//aliyun-sdk-oss-3.13.2.jar
analyticsaccelerator-s3/1.3.0//analyticsaccelerator-s3-1.3.0.jar
antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
antlr4-runtime/4.13.1//antlr4-runtime-4.13.1.jar
aopalliance-repackaged/3.0.6//aopalliance-repackaged-3.0.6.jar
arpack/3.0.4//arpack-3.0.4.jar
arpack_combined_all/0.1//arpack_combined_all-0.1.jar
Expand Down
7 changes: 2 additions & 5 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@
<module>mllib-local</module>
<module>tools</module>
<module>streaming</module>
<module>sql/parser</module>
Copy link
Member Author

@vrozov vrozov Sep 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Introduced new module and moved .g4 files to the new module where antlr4-runtime is shaded. sql-api and other modules that use antlr4 will now depend on the new module. This avoids apache/maven-shade-plugin#345.

<module>sql/api</module>
<module>sql/catalyst</module>
<module>sql/core</module>
Expand Down Expand Up @@ -2482,6 +2483,7 @@
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
<version>${antlr4.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.ibm.icu</groupId>
Expand Down Expand Up @@ -2819,11 +2821,6 @@
<skip>true</skip> <!-- skip testCompile -->
</configuration>
</plugin>
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr4-maven-plugin</artifactId>
<version>${antlr4.version}</version>
</plugin>
<!-- Surefire runs all Java tests -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
Expand Down
39 changes: 33 additions & 6 deletions project/SparkBuild.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ object BuildCommons {

private val buildLocation = file(".").getAbsoluteFile.getParentFile

val sqlProjects@Seq(sqlApi, catalyst, sql, hive, hiveThriftServer, tokenProviderKafka010, sqlKafka010, avro, protobuf) =
val sqlProjects@Seq(sqlApi, catalyst, sql, hive, hiveThriftServer, tokenProviderKafka010, sqlKafka010, avro, protobuf, sqlParser) =
Seq("sql-api", "catalyst", "sql", "hive", "hive-thriftserver", "token-provider-kafka-0-10",
"sql-kafka-0-10", "avro", "protobuf").map(ProjectRef(buildLocation, _))
"sql-kafka-0-10", "avro", "protobuf", "sql-parser").map(ProjectRef(buildLocation, _))

val streamingProjects@Seq(streaming, streamingKafka010) =
Seq("streaming", "streaming-kafka-0-10").map(ProjectRef(buildLocation, _))
Expand Down Expand Up @@ -403,7 +403,7 @@ object SparkBuild extends PomBuild {
Seq(
spark, hive, hiveThriftServer, repl, networkCommon, networkShuffle, networkYarn,
unsafe, tags, tokenProviderKafka010, sqlKafka010, pipelines, connectCommon, connect,
connectJdbc, connectClient, variant, connectShims, profiler, commonUtilsJava
connectJdbc, connectClient, variant, connectShims, profiler, commonUtilsJava, sqlParser
).contains(x)
}

Expand Down Expand Up @@ -437,7 +437,10 @@ object SparkBuild extends PomBuild {
/* Enable unidoc only for the root spark project */
enable(Unidoc.settings)(spark)

/* Sql-api ANTLR generation settings */
/* Spark SQL Parser ANTLR generation settings */
enable(SqlParser.settings)(sqlParser)

/* Spark API */
enable(SqlApi.settings)(sqlApi)

/* Spark SQL Core settings */
Expand Down Expand Up @@ -1274,7 +1277,7 @@ object OldDeps {
)
}

object SqlApi {
object SqlParser {
import com.simplytyped.Antlr4Plugin
import com.simplytyped.Antlr4Plugin.autoImport._

Expand All @@ -1283,7 +1286,31 @@ object SqlApi {
(Antlr4 / antlr4PackageName) := Some("org.apache.spark.sql.catalyst.parser"),
(Antlr4 / antlr4GenListener) := true,
(Antlr4 / antlr4GenVisitor) := true,
(Antlr4 / antlr4TreatWarningsAsErrors) := true
(Antlr4 / antlr4TreatWarningsAsErrors) := true,

excludeDependencies ++= Seq(
ExclusionRule("com.google.guava", "guava"),
ExclusionRule("org.jpmml", "pmml-model")
),

(assembly / logLevel) := Level.Info,
// Exclude `scala-library` from assembly.
(assembly / assemblyPackageScala / assembleArtifact) := false,
assembly / assemblyShadeRules := Seq(
ShadeRule.rename("org.antlr.v4.runtime.**" -> "org.sparkproject.antlr.v4.runtime.@1").inAll
),
(assembly / assemblyMergeStrategy) := {
case m if m.toLowerCase(Locale.ROOT).endsWith("manifest.mf") => MergeStrategy.discard
case _ => MergeStrategy.first
},
(assembly / assemblyJarName) := s"${moduleName.value}_${scalaBinaryVersion.value}-${version.value}.jar"
)
}

object SqlApi {
lazy val settings = Seq(
// Assembly sql-parser to shade antlr4-runtime classes
Compile / compile := ((Compile / compile) dependsOn LocalProject("sql-parser") / assembly).value,
)
}

Expand Down
31 changes: 5 additions & 26 deletions sql/api/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,11 @@
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-parser_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
Expand All @@ -72,10 +77,6 @@
<groupId>org.json4s</groupId>
<artifactId>json4s-jackson_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
Expand All @@ -85,26 +86,4 @@
<artifactId>arrow-memory-netty</artifactId>
</dependency>
</dependencies>
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
<plugins>
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr4-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>antlr4</goal>
</goals>
</execution>
</executions>
<configuration>
<visitor>true</visitor>
<sourceDirectory>../api/src/main/antlr4</sourceDirectory>
<treatWarningsAsErrors>true</treatWarningsAsErrors>
</configuration>
</plugin>
</plugins>
</build>
</project>
4 changes: 2 additions & 2 deletions sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/DataTypeAstBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ import java.util.Locale

import scala.jdk.CollectionConverters._

import org.antlr.v4.runtime.{ParserRuleContext, Token}
import org.antlr.v4.runtime.tree.ParseTree
import org.sparkproject.antlr.v4.runtime.{ParserRuleContext, Token}
import org.sparkproject.antlr.v4.runtime.tree.ParseTree

import org.apache.spark.SparkException
import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
Expand Down
4 changes: 2 additions & 2 deletions sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/SparkParserErrorStrategy.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@

package org.apache.spark.sql.catalyst.parser

import org.antlr.v4.runtime.{DefaultErrorStrategy, InputMismatchException, IntStream, NoViableAltException, Parser, ParserRuleContext, RecognitionException, Recognizer, Token}
import org.antlr.v4.runtime.misc.ParseCancellationException
import org.sparkproject.antlr.v4.runtime.{DefaultErrorStrategy, InputMismatchException, IntStream, NoViableAltException, Parser, ParserRuleContext, RecognitionException, Recognizer, Token}
import org.sparkproject.antlr.v4.runtime.misc.ParseCancellationException

/**
* A [[SparkRecognitionException]] extends the [[RecognitionException]] with more information
Expand Down
2 changes: 1 addition & 1 deletion sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/SubstituteParmsAstBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ package org.apache.spark.sql.catalyst.parser

import scala.collection.mutable

import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode}
import org.sparkproject.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode}

import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
import org.apache.spark.sql.catalyst.util.SparkParserUtils.withOrigin
Expand Down
12 changes: 6 additions & 6 deletions sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ import java.util.concurrent.atomic.AtomicReference

import scala.jdk.CollectionConverters._

import org.antlr.v4.runtime._
import org.antlr.v4.runtime.atn.{ATN, ParserATNSimulator, PredictionContextCache, PredictionMode}
import org.antlr.v4.runtime.dfa.DFA
import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException}
import org.antlr.v4.runtime.tree.TerminalNodeImpl
import org.sparkproject.antlr.v4.runtime._
import org.sparkproject.antlr.v4.runtime.atn.{ATN, ParserATNSimulator, PredictionContextCache, PredictionMode}
import org.sparkproject.antlr.v4.runtime.dfa.DFA
import org.sparkproject.antlr.v4.runtime.misc.{Interval, Pair, ParseCancellationException}
import org.sparkproject.antlr.v4.runtime.tree.TerminalNodeImpl

import org.apache.spark.{QueryContext, SparkException, SparkThrowable, SparkThrowableHelper}
import org.apache.spark.internal.{Logging, LogKeys}
Expand Down Expand Up @@ -343,7 +343,7 @@ case object PostProcessor extends SqlBaseParserBaseListener {
parent.removeLastChild()
val token = ctx.getChild(0).getPayload.asInstanceOf[Token]
val newToken = new CommonToken(
new org.antlr.v4.runtime.misc.Pair(token.getTokenSource, token.getInputStream),
new Pair(token.getTokenSource, token.getInputStream),
SqlBaseParser.IDENTIFIER,
token.getChannel,
token.getStartIndex + stripMargins,
Expand Down
6 changes: 3 additions & 3 deletions sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkParserUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ package org.apache.spark.sql.catalyst.util

import java.lang.{Long => JLong, StringBuilder => JStringBuilder}

import org.antlr.v4.runtime.{ParserRuleContext, Token}
import org.antlr.v4.runtime.misc.Interval
import org.antlr.v4.runtime.tree.TerminalNode
import org.sparkproject.antlr.v4.runtime.{ParserRuleContext, Token}
import org.sparkproject.antlr.v4.runtime.misc.Interval
import org.sparkproject.antlr.v4.runtime.tree.TerminalNode

import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin}

Expand Down
2 changes: 1 addition & 1 deletion sql/api/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ package org.apache.spark.sql.errors

import java.util.Locale

import org.antlr.v4.runtime.ParserRuleContext
import org.sparkproject.antlr.v4.runtime.ParserRuleContext

import org.apache.spark.SparkException
import org.apache.spark.sql.catalyst.parser.ParseException
Expand Down
2 changes: 1 addition & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AbstractSqlParser.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.catalyst.parser

import org.antlr.v4.runtime.ParserRuleContext
import org.sparkproject.antlr.v4.runtime.ParserRuleContext

import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
import org.apache.spark.sql.catalyst.expressions.Expression
Expand Down
4 changes: 2 additions & 2 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ import java.util.concurrent.TimeUnit
import scala.collection.mutable.{ArrayBuffer, HashMap, ListBuffer, Set}
import scala.jdk.CollectionConverters._

import org.antlr.v4.runtime.{ParserRuleContext, RuleContext, Token}
import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode}
import org.sparkproject.antlr.v4.runtime.{ParserRuleContext, RuleContext, Token}
import org.sparkproject.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode}

import org.apache.spark.{SparkArithmeticException, SparkException, SparkIllegalArgumentException, SparkThrowable, SparkThrowableHelper}
import org.apache.spark.internal.Logging
Expand Down
6 changes: 3 additions & 3 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ import java.util.Locale
import scala.collection.{immutable, mutable}
import scala.util.matching.Regex

import org.antlr.v4.runtime.{ParserRuleContext, Token}
import org.antlr.v4.runtime.misc.Interval
import org.antlr.v4.runtime.tree.{ParseTree, TerminalNodeImpl}
import org.sparkproject.antlr.v4.runtime.{ParserRuleContext, Token}
import org.sparkproject.antlr.v4.runtime.misc.Interval
import org.sparkproject.antlr.v4.runtime.tree.{ParseTree, TerminalNodeImpl}

import org.apache.spark.SparkException
import org.apache.spark.sql.catalyst.analysis.UnresolvedIdentifier
Expand Down
2 changes: 1 addition & 1 deletion ...catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SubstituteParamsParser.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.catalyst.parser

import org.antlr.v4.runtime.{CharStreams, CommonTokenStream}
import org.sparkproject.antlr.v4.runtime.{CharStreams, CommonTokenStream}

import org.apache.spark.internal.Logging
import org.apache.spark.sql.AnalysisException
Expand Down
4 changes: 2 additions & 2 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ trait SQLKeywordUtils extends SparkFunSuite with SQLHelper {

val sqlSyntaxDefs = {
val sqlBaseParserPath =
getWorkspaceFilePath("sql", "api", "src", "main", "antlr4", "org",
getWorkspaceFilePath("sql", "parser", "src", "main", "antlr4", "org",
"apache", "spark", "sql", "catalyst", "parser", "SqlBaseParser.g4").toFile

val sqlBaseLexerPath =
getWorkspaceFilePath("sql", "api", "src", "main", "antlr4", "org",
getWorkspaceFilePath("sql", "parser", "src", "main", "antlr4", "org",
"apache", "spark", "sql", "catalyst", "parser", "SqlBaseLexer.g4").toFile

(Files.readString(sqlBaseParserPath.toPath) +
Expand Down
2 changes: 1 addition & 1 deletion sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParserUtilsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ package org.apache.spark.sql.catalyst.parser

import scala.jdk.CollectionConverters._

import org.antlr.v4.runtime.{CharStreams, CommonTokenStream, ParserRuleContext}
import org.sparkproject.antlr.v4.runtime.{CharStreams, CommonTokenStream, ParserRuleContext}

import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
Expand Down
4 changes: 2 additions & 2 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ import java.util.{Locale, TimeZone}

import scala.jdk.CollectionConverters._

import org.antlr.v4.runtime.ParserRuleContext
import org.antlr.v4.runtime.tree.TerminalNode
import org.sparkproject.antlr.v4.runtime.ParserRuleContext
import org.sparkproject.antlr.v4.runtime.tree.TerminalNode

import org.apache.spark.SparkException
import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
Expand Down
Loading