Skip to content

[DNM] Support spark-3.0 #1221

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[DNM] Support spark-3.0 #1221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .ci/integration_test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ def call(ghprbActualCommit, ghprbCommentBody, ghprbPullId, ghprbPullTitle, ghprb
def TIDB_BRANCH = "master"
def TIKV_BRANCH = "master"
def PD_BRANCH = "master"
def MVN_PROFILE = "-Pjenkins"
def MVN_PROFILE = "-Pjenkins-spark3.0"
def TEST_MODE = "simple"
def PARALLEL_NUMBER = 18

Expand Down Expand Up @@ -122,7 +122,7 @@ def call(ghprbActualCommit, ghprbCommentBody, ghprbPullId, ghprbPullTitle, ghprb
cp .ci/log4j-ci.properties core/src/test/resources/log4j.properties
bash core/scripts/version.sh
bash core/scripts/fetch-test-data.sh
mv core/src/test core-test/src/
mv core/src/test core-test-spark3.0/src/
bash tikv-client/scripts/proto.sh
"""
}
Expand Down
9 changes: 9 additions & 0 deletions assembly/src/main/assembly/assembly.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,5 +37,14 @@
<include>**/*</include>
</includes>
</fileSet>
<fileSet>
<directory>
${project.parent.basedir}/spark-wrapper/spark-3.0/target/classes/
</directory>
<outputDirectory>resources/spark-wrapper-spark-3.0</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>
</fileSets>
</assembly>
18 changes: 10 additions & 8 deletions core-test/pom.xml → core-test-spark2.4/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>tispark-core-test</artifactId>
<artifactId>tispark-core-test-spark2.4</artifactId>
<packaging>jar</packaging>
<name>TiSpark Project Core Test</name>
<name>TiSpark Project Core Test - Spark-2.4</name>
<url>http://github.copm/pingcap/tispark</url>

<properties>
<scalaj.version>2.3.0</scalaj.version>
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.11.12</scala.version>
</properties>

<dependencies>
Expand Down Expand Up @@ -79,22 +81,22 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version.test}</version>
<version>${spark.version.test.spark2.4}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<version>${spark.version.test}</version>
<version>${spark.version.test.spark2.4}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version.test}</version>
<version>${spark.version.test.spark2.4}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version.test}</version>
<version>${spark.version.test.spark2.4}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
Expand All @@ -105,12 +107,12 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
<version>${spark.version.test}</version>
<version>${spark.version.test.spark2.4}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-unsafe_${scala.binary.version}</artifactId>
<version>${spark.version.test}</version>
<version>${spark.version.test.spark2.4}</version>
</dependency>
<!-- For tests -->
<dependency>
Expand Down
0 core-test/src/KEEPME → core-test-spark2.4/src/KEEPME
View file
Open in desktop
File renamed without changes.
256 changes: 256 additions & 0 deletions core-test-spark3.0/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,256 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.pingcap.tispark</groupId>
<artifactId>tispark-parent</artifactId>
<version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>tispark-core-test-spark3.0</artifactId>
<packaging>jar</packaging>
<name>TiSpark Project Core Test - Spark-3.0</name>
<url>http://github.copm/pingcap/tispark</url>

<properties>
<scalaj.version>2.3.0</scalaj.version>
<scala.binary.version>2.12</scala.binary.version>
<scala.version>2.12.10</scala.version>
</properties>

<dependencies>
<dependency>
<groupId>com.pingcap.tispark</groupId>
<artifactId>tispark-core-internal</artifactId>
<version>${project.parent.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-unsafe_${scala.binary.version}</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.pingcap.tikv</groupId>
<artifactId>tikv-client</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.scalaj</groupId>
<artifactId>scalaj-http_${scala.binary.version}</artifactId>
<version>${scalaj.version}</version>
<exclusions>
<exclusion>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version.test.spark3.0}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<version>${spark.version.test.spark3.0}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version.test.spark3.0}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version.test.spark3.0}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
<version>${spark.version.test.spark3.0}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-unsafe_${scala.binary.version}</artifactId>
<version>${spark.version.test.spark3.0}</version>
</dependency>
<!-- For tests -->
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<version>${scalatest.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql.connector.version}</version>
</dependency>
<!--for using tidb datanucleus's adapter-->
<dependency>
<groupId>com.pingcap.tidb</groupId>
<artifactId>tidb-datanucleus-adapter</artifactId>
<version>1.0</version>
<scope>runtime</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<id>compile-scala</id>
<phase>compile</phase>
<goals>
<goal>add-source</goal>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>test-compile-scala</id>
<phase>test-compile</phase>
<goals>
<goal>add-source</goal>
<goal>testCompile</goal>
</goals>
</execution>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>doc-jar</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
</configuration>
</plugin>
<!-- Compiler Plug-in -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.6.0</version>
<configuration>
<skip>${skipFetchTestData}</skip>
</configuration>
<executions>
<execution>
<id>version imprint</id>
<configuration>
<executable>${basedir}/scripts/version.sh</executable>
</configuration>
<phase>validate</phase>
<goals>
<goal>exec</goal>
</goals>
</execution>
<execution>
<id>test data fetch</id>
<phase>validate</phase>
<configuration>
<executable>${basedir}/scripts/fetch-test-data.sh</executable>
</configuration>
<goals>
<goal>exec</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
<showWarnings>true</showWarnings>
<showDeprecation>true</showDeprecation>
</configuration>
</plugin>
<!-- Clean Plug-in -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
<version>2.4.1</version>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>WDF TestSuite.txt</filereports>
<argLine>-Dfile.encoding=UTF-8 -Duser.timezone=GMT+8 -Dio.netty.leakDetection.level=paranoid</argLine>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
Empty file added core-test-spark3.0/src/KEEPME
View file
Open in desktop
Empty file.
2 changes: 1 addition & 1 deletion core/src/main/scala/com/pingcap/tispark/TiSparkInfo.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
package com.pingcap.tispark

object TiSparkInfo {
val SUPPORTED_SPARK_VERSION: List[String] = "2.3" :: "2.4" :: Nil
val SUPPORTED_SPARK_VERSION: List[String] = "3.0" :: "2.3" :: "2.4" :: Nil

val SPARK_VERSION: String = org.apache.spark.SPARK_VERSION

Expand Down
12 changes: 9 additions & 3 deletions core/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/SpecialSum.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,15 @@ case class SpecialSum(child: Expression, retType: DataType, initVal: Any)

override lazy val aggBufferAttributes: Seq[AttributeReference] = sum :: Nil

override lazy val initialValues: Seq[Expression] = Seq(
/* sum = */ Literal.create(initVal, sumDataType)
)
override lazy val initialValues: Seq[Expression] = {
val longVal = initVal match {
case i: Integer => i.toLong
case other => other
}
Seq(
/* sum = */ Literal.create(longVal, sumDataType)
)
}

override lazy val updateExpressions: Seq[Expression] = {
if (child.nullable) {
Expand Down
Loading