Skip to content

Commit d24c18d

Browse files
jiaoqingbopan3793
jiaoqingbo
authored and
pan3793
committed
[KYUUBI #2741] Add kyuubi-spark-connector-common module
### _Why are the changes needed?_ fix #2741 ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #2777 from jiaoqingbo/kyuubi-2741. Closes #2741 ec7185e [jiaoqingbo] code review 33114d1 [jiaoqingbo] code review df603c8 [jiaoqingbo] Merge branch 'master' into kyuubi-2741 767e436 [jiaoqingbo] [KYUUBI #2741]add kyuubi-spark-connector-common module to contain things that are commonly used by multiple spark connectors 8e79bd9 [jiaoqingbo] invoke ci c5c7b81 [jiaoqingbo] [KYUUBI #2741]add kyuubi-spark-connector-common module to contain things that are commonly used by multiple spark connectors b0196d6 [jiaoqingbo] invoke ci af3dab2 [jiaoqingbo] [KYUUBI #2741]add kyuubi-spark-connector-common module to contain things that are commonly used by multiple spark connectors Authored-by: jiaoqingbo <1178404354@qq.com> Signed-off-by: Cheng Pan <chengpan@apache.org>
1 parent d2ceb04 commit d24c18d

File tree

13 files changed

+186
-122
lines changed

13 files changed

+186
-122
lines changed

‎extensions/spark/kyuubi-spark-connector-common/pom.xml

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
19+
<project xmlns="http://maven.apache.org/POM/4.0.0"
20+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
21+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
22+
<parent>
23+
<groupId>org.apache.kyuubi</groupId>
24+
<artifactId>kyuubi-parent</artifactId>
25+
<version>1.6.0-SNAPSHOT</version>
26+
<relativePath>../../../pom.xml</relativePath>
27+
</parent>
28+
<modelVersion>4.0.0</modelVersion>
29+
30+
<artifactId>kyuubi-spark-connector-common_2.12</artifactId>
31+
<name>Kyuubi Spark Connector Common</name>
32+
<packaging>jar</packaging>
33+
<url>https://kyuubi.apache.org/</url>
34+
35+
<dependencies>
36+
<dependency>
37+
<groupId>org.scala-lang</groupId>
38+
<artifactId>scala-library</artifactId>
39+
<scope>provided</scope>
40+
</dependency>
41+
42+
<dependency>
43+
<groupId>org.slf4j</groupId>
44+
<artifactId>slf4j-api</artifactId>
45+
<scope>provided</scope>
46+
</dependency>
47+
48+
<dependency>
49+
<groupId>org.apache.spark</groupId>
50+
<artifactId>spark-sql_${scala.binary.version}</artifactId>
51+
<scope>provided</scope>
52+
</dependency>
53+
54+
<dependency>
55+
<groupId>org.apache.spark</groupId>
56+
<artifactId>spark-core_${scala.binary.version}</artifactId>
57+
<type>test-jar</type>
58+
<scope>test</scope>
59+
</dependency>
60+
61+
<dependency>
62+
<groupId>org.apache.spark</groupId>
63+
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
64+
<type>test-jar</type>
65+
<scope>test</scope>
66+
</dependency>
67+
68+
<dependency>
69+
<groupId>org.scalatestplus</groupId>
70+
<artifactId>scalacheck-1-15_${scala.binary.version}</artifactId>
71+
<scope>test</scope>
72+
</dependency>
73+
74+
<dependency>
75+
<groupId>org.apache.spark</groupId>
76+
<artifactId>spark-sql_${scala.binary.version}</artifactId>
77+
<version>${spark.version}</version>
78+
<type>test-jar</type>
79+
<scope>test</scope>
80+
</dependency>
81+
</dependencies>
82+
83+
<build>
84+
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
85+
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
86+
87+
<plugins>
88+
<plugin>
89+
<groupId>org.apache.maven.plugins</groupId>
90+
<artifactId>maven-jar-plugin</artifactId>
91+
<executions>
92+
<execution>
93+
<id>prepare-test-jar</id>
94+
<phase>test-compile</phase>
95+
<goals>
96+
<goal>test-jar</goal>
97+
</goals>
98+
</execution>
99+
</executions>
100+
</plugin>
101+
</plugins>
102+
</build>
103+
</project>

‎extensions/spark/kyuubi-spark-connector-tpcds/src/main/scala/org/apache/kyuubi/spark/connector/tpcds/SparkUtils.scala renamed to ‎extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SparkUtils.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* limitations under the License.
1616
*/
1717

18-
package org.apache.kyuubi.spark.connector.tpcds
18+
package org.apache.kyuubi.spark.connector.common
1919

2020
import org.apache.spark.SPARK_VERSION
2121

‎extensions/spark/kyuubi-spark-connector-common/src/test/resources/log4j.properties

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
# Set everything to be logged to the file target/unit-tests.log
19+
log4j.rootLogger=INFO, CA, FA
20+
21+
# Console Appender
22+
log4j.appender.CA=org.apache.log4j.ConsoleAppender
23+
log4j.appender.CA.layout=org.apache.log4j.PatternLayout
24+
log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n
25+
log4j.appender.CA.Threshold = FATAL
26+
27+
# File Appender
28+
log4j.appender.FA=org.apache.log4j.FileAppender
29+
log4j.appender.FA.append=false
30+
log4j.appender.FA.file=target/unit-tests.log
31+
log4j.appender.FA.layout=org.apache.log4j.PatternLayout
32+
log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n
33+
34+
# Set the logger level of File Appender to WARN
35+
log4j.appender.FA.Threshold = DEBUG
36+
37+
# SPARK-34128: Suppress undesirable TTransportException warnings involved in THRIFT-4805
38+
log4j.appender.CA.filter.1=org.apache.log4j.varia.StringMatchFilter
39+
log4j.appender.CA.filter.1.StringToMatch=Thrift error occurred during processing of message
40+
log4j.appender.CA.filter.1.AcceptOnMatch=false
41+
42+
log4j.appender.FA.filter.1=org.apache.log4j.varia.StringMatchFilter
43+
log4j.appender.FA.filter.1.StringToMatch=Thrift error occurred during processing of message
44+
log4j.appender.FA.filter.1.AcceptOnMatch=false

‎extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/LocalSparkSession.scala renamed to ‎extensions/spark/kyuubi-spark-connector-common/src/test/scala/org/apache/kyuubi/spark/connector/common/LocalSparkSession.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* limitations under the License.
1616
*/
1717

18-
package org.apache.kyuubi.spark.connector.tpcds
18+
package org.apache.kyuubi.spark.connector.common
1919

2020
import org.apache.spark.sql.SparkSession
2121

‎extensions/spark/kyuubi-spark-connector-tpcds/pom.xml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,21 @@
3333
<url>https://kyuubi.apache.org/</url>
3434

3535
<dependencies>
36+
37+
<dependency>
38+
<groupId>org.apache.kyuubi</groupId>
39+
<artifactId>kyuubi-spark-connector-common_${scala.binary.version}</artifactId>
40+
<version>${project.version}</version>
41+
</dependency>
42+
43+
<dependency>
44+
<groupId>org.apache.kyuubi</groupId>
45+
<artifactId>kyuubi-spark-connector-common_${scala.binary.version}</artifactId>
46+
<version>${project.version}</version>
47+
<type>test-jar</type>
48+
<scope>test</scope>
49+
</dependency>
50+
3651
<dependency>
3752
<groupId>org.scala-lang</groupId>
3853
<artifactId>scala-library</artifactId>
@@ -162,6 +177,7 @@
162177
<includes>
163178
<include>io.trino.tpcds:tpcds</include>
164179
<include>com.google.guava:guava</include>
180+
<include>org.apache.kyuubi:kyuubi-spark-connector-common_${scala.binary.version}</include>
165181
</includes>
166182
</artifactSet>
167183
<relocations>

‎extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSCatalogSuite.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ import org.apache.spark.sql.{AnalysisException, SparkSession}
2222
import org.apache.spark.sql.util.CaseInsensitiveStringMap
2323

2424
import org.apache.kyuubi.KyuubiFunSuite
25-
import org.apache.kyuubi.spark.connector.tpcds.LocalSparkSession.withSparkSession
25+
import org.apache.kyuubi.spark.connector.common.LocalSparkSession.withSparkSession
26+
import org.apache.kyuubi.spark.connector.common.SparkUtils
2627

2728
class TPCDSCatalogSuite extends KyuubiFunSuite {
2829

‎extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSQuerySuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import org.apache.spark.SparkConf
2323
import org.apache.spark.sql.SparkSession
2424

2525
import org.apache.kyuubi.KyuubiFunSuite
26-
import org.apache.kyuubi.spark.connector.tpcds.LocalSparkSession.withSparkSession
26+
import org.apache.kyuubi.spark.connector.common.LocalSparkSession.withSparkSession
2727

2828
class TPCDSQuerySuite extends KyuubiFunSuite {
2929

‎extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSTableSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import org.apache.spark.SparkConf
2323
import org.apache.spark.sql.SparkSession
2424

2525
import org.apache.kyuubi.KyuubiFunSuite
26-
import org.apache.kyuubi.spark.connector.tpcds.LocalSparkSession.withSparkSession
26+
import org.apache.kyuubi.spark.connector.common.LocalSparkSession.withSparkSession
2727

2828
class TPCDSTableSuite extends KyuubiFunSuite {
2929

‎extensions/spark/kyuubi-spark-connector-tpch/pom.xml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,20 @@
3333
<url>https://kyuubi.apache.org/</url>
3434

3535
<dependencies>
36+
<dependency>
37+
<groupId>org.apache.kyuubi</groupId>
38+
<artifactId>kyuubi-spark-connector-common_${scala.binary.version}</artifactId>
39+
<version>${project.version}</version>
40+
</dependency>
41+
42+
<dependency>
43+
<groupId>org.apache.kyuubi</groupId>
44+
<artifactId>kyuubi-spark-connector-common_${scala.binary.version}</artifactId>
45+
<version>${project.version}</version>
46+
<type>test-jar</type>
47+
<scope>test</scope>
48+
</dependency>
49+
3650
<dependency>
3751
<groupId>org.scala-lang</groupId>
3852
<artifactId>scala-library</artifactId>
@@ -162,6 +176,7 @@
162176
<includes>
163177
<include>io.trino.tpch:tpch</include>
164178
<include>com.google.guava:guava</include>
179+
<include>org.apache.kyuubi:kyuubi-spark-connector-common_${scala.binary.version}</include>
165180
</includes>
166181
</artifactSet>
167182
<relocations>

‎extensions/spark/kyuubi-spark-connector-tpch/src/main/scala/org/apache/kyuubi/spark/connector/tpch/SparkUtils.scala

Lines changed: 0 additions & 78 deletions
This file was deleted.

0 commit comments

Comments
 (0)