Skip to content

Commit a56e4b4

Browse files
yaooqinnyaooqinn
committed
[KYUUBI #2024][FOLLOWUP] Hive Backend Engine - ProcBuilder for HiveEngine
### _Why are the changes needed?_ Make the HiveProcBuilder actually work. - hive 2.3.9 has scala 2.11 deps, so not compatible with kyuubi common, thus, upgrade 3.1.x - hive 3.1 does not work with java 11 as https://issues.apache.org/jira/browse/HIVE-21237 bug ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #2290 from yaooqinn/2024. Closes #2024 83e07fa [Kent Yao] fix 1e7167f [Kent Yao] import ac7853a [Kent Yao] jdk11 ba9f6c3 [Kent Yao] temp 4404973 [Kent Yao] temp fd62430 [Kent Yao] temp a4a5e42 [Kent Yao] temp b01bb22 [Kent Yao] temp 298fc47 [Kent Yao] temp 5630857 [Kent Yao] temp 98457b6 [Kent Yao] temp 18d8f5c [Kent Yao] [KYUUBI #2024][FOLLOWUP] Hive Backend Engine - ProcBuilder for HiveEngine 246681d [Kent Yao] [KYUUBI #2024][FOLLOWUP] Hive Backend Engine - ProcBuilder for HiveEngine 890579b [Kent Yao] [KYUUBI #2084][FOLLOWUP] Support arbitrary parameters for KyuubiConf 105b70b [Kent Yao] tmp bcc4a0e [Kent Yao] tmp Authored-by: Kent Yao <yao@apache.org> Signed-off-by: Kent Yao <yao@apache.org>
1 parent 04c536b commit a56e4b4

File tree

25 files changed

+597
-461
lines changed

25 files changed

+597
-461
lines changed

‎build/dist

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -262,9 +262,7 @@ chmod a+x "$DISTDIR/externals/engines/trino/bin/trino-engine.sh"
262262
cp "$KYUUBI_HOME/externals/kyuubi-trino-engine/target/kyuubi-trino-engine_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/externals/engines/trino/jars"
263263

264264
# Copy hive engines
265-
cp -r "$KYUUBI_HOME/externals/kyuubi-hive-sql-engine/bin/" "$DISTDIR/externals/engines/hive/bin/"
266-
chmod a+x "$DISTDIR/externals/engines/hive/bin/hive-sql-engine.sh"
267-
cp "$KYUUBI_HOME/externals/kyuubi-hive-sql-engine/target/kyuubi-hive-sql-engine_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/externals/engines/hive/jars"
265+
cp "$KYUUBI_HOME/externals/kyuubi-hive-sql-engine/target/kyuubi-hive-sql-engine_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/externals/engines/hive"
268266

269267
# Copy kyuubi tools
270268
if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SCALA_VERSION}-${VERSION}.jar" ]]; then

‎externals/kyuubi-download/pom.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,13 @@
7979
<unpack>true</unpack>
8080
</configuration>
8181
</plugin>
82+
83+
<plugin>
84+
<artifactId>maven-remote-resources-plugin</artifactId>
85+
<configuration>
86+
<skip>true</skip>
87+
</configuration>
88+
</plugin>
8289
</plugins>
8390
</build>
8491
</project>

‎externals/kyuubi-hive-sql-engine/bin/hive-sql-engine.sh

Lines changed: 0 additions & 46 deletions
This file was deleted.

‎externals/kyuubi-hive-sql-engine/pom.xml

Lines changed: 42 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,45 @@
4646
<version>${project.version}</version>
4747
</dependency>
4848

49+
<dependency>
50+
<groupId>org.apache.hive</groupId>
51+
<artifactId>hive-service-rpc</artifactId>
52+
<scope>provided</scope>
53+
</dependency>
54+
55+
<dependency>
56+
<groupId>org.apache.thrift</groupId>
57+
<artifactId>libfb303</artifactId>
58+
<scope>provided</scope>
59+
</dependency>
60+
61+
<dependency>
62+
<groupId>com.google.code.findbugs</groupId>
63+
<artifactId>jsr305</artifactId>
64+
<scope>provided</scope>
65+
</dependency>
66+
67+
<dependency>
68+
<groupId>commons-collections</groupId>
69+
<artifactId>commons-collections</artifactId>
70+
</dependency>
71+
72+
<dependency>
73+
<groupId>org.apache.thrift</groupId>
74+
<artifactId>libthrift</artifactId>
75+
<scope>provided</scope>
76+
</dependency>
77+
78+
<dependency>
79+
<groupId>com.google.guava</groupId>
80+
<artifactId>failureaccess</artifactId>
81+
<scope>provided</scope>
82+
</dependency>
83+
4984
<dependency>
5085
<groupId>org.apache.hive</groupId>
5186
<artifactId>hive-service</artifactId>
52-
<version>${hive.version}</version>
87+
<version>${hive.engine.hive.version}</version>
5388
<scope>provided</scope>
5489
<exclusions>
5590
<exclusion>
@@ -74,7 +109,7 @@
74109
<dependency>
75110
<groupId>org.apache.hive</groupId>
76111
<artifactId>hive-metastore</artifactId>
77-
<version>${hive.version}</version>
112+
<version>${hive.engine.hive.version}</version>
78113
<scope>provided</scope>
79114
<exclusions>
80115
<exclusion>
@@ -125,13 +160,16 @@
125160
<shadedArtifactAttached>false</shadedArtifactAttached>
126161
<artifactSet>
127162
<includes>
163+
<include>org.scala-lang:scala-library</include>
164+
<include>com.fasterxml.jackson.core:jackson-annotations</include>
165+
<include>com.fasterxml.jackson.core:jackson-core</include>
166+
<include>com.fasterxml.jackson.core:jackson-databind</include>
167+
<include>com.fasterxml.jackson.module:jackson-module-scala_2.12</include>
128168
<include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
129169
<include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
130170
<include>org.apache.curator:curator-client</include>
131171
<include>org.apache.curator:curator-framework</include>
132172
<include>org.apache.curator:curator-recipes</include>
133-
<include>org.apache.hive:hive-service-rpc</include>
134-
<include>org.apache.thrift:*</include>
135173
</includes>
136174
</artifactSet>
137175
<filters>
@@ -160,27 +198,6 @@
160198
<include>org.apache.curator.**</include>
161199
</includes>
162200
</relocation>
163-
<relocation>
164-
<pattern>org.apache.hive.service.rpc.thrift</pattern>
165-
<shadedPattern>${kyuubi.shade.packageName}.org.apache.hive.service.rpc.thrift</shadedPattern>
166-
<includes>
167-
<include>org.apache.hive.service.rpc.thrift.**</include>
168-
</includes>
169-
</relocation>
170-
<relocation>
171-
<pattern>com.facebook.fb303</pattern>
172-
<shadedPattern>${kyuubi.shade.packageName}.com.facebook.fb303</shadedPattern>
173-
<includes>
174-
<include>com.facebook.fb303.**</include>
175-
</includes>
176-
</relocation>
177-
<relocation>
178-
<pattern>org.apache.thrift</pattern>
179-
<shadedPattern>${kyuubi.shade.packageName}.org.apache.thrift</shadedPattern>
180-
<includes>
181-
<include>org.apache.thrift.**</include>
182-
</includes>
183-
</relocation>
184201
</relocations>
185202
</configuration>
186203
<executions>

‎externals/kyuubi-hive-sql-engine/src/main/resources/META-INF/LICENSE

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,8 @@ Apache License Version 2.0
210210
org.apache.curator:curator-client
211211
org.apache.curator:curator-framework
212212
org.apache.curator:curator-recipes
213-
org.apache.hive:hive-service-rpc
214-
org.apache.thrift:fb303
215-
org.apache.thrift:libthrift
213+
org.scala-lang:scala-library
214+
com.fasterxml.jackson.core:jackson-annotations
215+
com.fasterxml.jackson.core:jackson-core
216+
com.fasterxml.jackson.core:jackson-databind
217+
com.fasterxml.jackson.module:jackson-module-scala_2.12

‎externals/kyuubi-hive-sql-engine/src/main/resources/META-INF/NOTICE

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@ Copyright 2011-2017 The Apache Software Foundation
1717
Curator Recipes
1818
Copyright 2011-2017 The Apache Software Foundation
1919

20-
Hive Service RPC
21-
Copyright 2019 The Apache Software Foundation
22-
23-
Apache Thrift
24-
Copyright 2006-2010 The Apache Software Foundation.
20+
org.scala-lang:scala-library
21+
Copyright (c) 2002-2022 EPFL
22+
Copyright (c) 2011-2022 Lightbend, Inc.

‎externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/HiveSQLEngine.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ object HiveSQLEngine extends Logging {
5858

5959
def startEngine(): HiveSQLEngine = {
6060
try {
61+
// TODO: hive 2.3.x has scala 2.11 deps.
6162
initLoggerEventHandler(kyuubiConf)
6263
} catch {
6364
case NonFatal(e) =>
@@ -82,6 +83,14 @@ object HiveSQLEngine extends Logging {
8283
hiveConf.set(
8384
"hive.metastore.warehouse.dir",
8485
Utils.createTempDir(namePrefix = "kyuubi_hive_warehouse").toString)
86+
hiveConf.set("hive.metastore.fastpath", "true")
87+
88+
// TODO: Move this to test phase after #2021 resolved
89+
val metastore = Utils.createTempDir(namePrefix = "hms_temp")
90+
metastore.toFile.delete()
91+
hiveConf.set(
92+
"javax.jdo.option.ConnectionURL",
93+
s"jdbc:derby:;databaseName=$metastore;create=true")
8594
}
8695

8796
val engine = new HiveSQLEngine()

‎externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/operation/HiveOperation.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ abstract class HiveOperation(opType: OperationType, session: Session)
5858
}
5959
override def runInternal(): Unit = {
6060
internalHiveOperation.run()
61+
val hasResultSet = internalHiveOperation.getStatus.getHasResultSet
62+
setHasResultSet(hasResultSet)
6163
}
6264

6365
override def getBackgroundHandle: Future[_] = {
@@ -73,7 +75,6 @@ abstract class HiveOperation(opType: OperationType, session: Session)
7375
}
7476

7577
override def getStatus: OperationStatus = {
76-
super.getStatus
7778
val status = internalHiveOperation.getStatus
7879
val state = OperationState.withName(status.getState.name().stripSuffix("_STATE"))
7980

@@ -83,7 +84,7 @@ abstract class HiveOperation(opType: OperationType, session: Session)
8384
status.getOperationStarted,
8485
lastAccessTime,
8586
status.getOperationCompleted,
86-
status.getHasResultSet,
87+
hasResultSet,
8788
Option(status.getOperationException).map(KyuubiSQLException(_)))
8889
}
8990

‎externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationManager.scala

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
3232
import org.apache.kyuubi.session.Session
3333

3434
class HiveOperationManager() extends OperationManager("HiveOperationManager") {
35+
// we use hive's operation log
36+
override protected def skipOperationLog: Boolean = true
37+
3538
override def newExecuteStatementOperation(
3639
session: Session,
3740
statement: String,
@@ -140,7 +143,9 @@ class HiveOperationManager() extends OperationManager("HiveOperationManager") {
140143
val rowSet = RowSetFactory.create(getLogSchema, operation.getProtocolVersion, false)
141144
val operationLog = internalHiveOperation.getOperationLog
142145
if (operationLog == null) {
143-
throw KyuubiSQLException("Couldn't find log associated with operation handle: " + opHandle)
146+
// TODO: #2029 Operation Log support: set and read hive one directly
147+
// throw KyuubiSQLException("Couldn't find log associated with operation handle: " + opHandle)
148+
return rowSet.toTRowSet
144149
}
145150

146151
try {

‎externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/session/HiveSessionImpl.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import java.util.HashMap
2121

2222
import scala.collection.JavaConverters._
2323

24-
import org.apache.hive.service.cli.HiveSQLException
2524
import org.apache.hive.service.cli.session.HiveSession
2625
import org.apache.hive.service.rpc.thrift.TProtocolVersion
2726

@@ -65,7 +64,7 @@ class HiveSessionImpl(
6564
try {
6665
hive.close()
6766
} catch {
68-
case e: HiveSQLException =>
67+
case e: Exception =>
6968
error(s"Failed to close hive runtime session: ${e.getMessage}")
7069
}
7170
}

0 commit comments

Comments
 (0)