From 23008b15b87c672e885b591f6d1a2272c03248eb Mon Sep 17 00:00:00 2001 From: fwang12 Date: Fri, 13 Jan 2023 21:15:18 +0800 Subject: [PATCH 001/288] [KYUUBI #4164] Accumulate the operation terminal state ### _Why are the changes needed?_ For the KyuubiOperation, its final state should be `CLOSED`, and now the `FINISHED`, `ERROR` states, which are important for kyuubi administer, are not accumulated. In this pr, we check whether the old state is terminal state, if it is, do not decrease it. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4164 from turboFei/terminate_acc. Closes #4164 16e559c6f [fwang12] comment c9d86f4b6 [fwang12] add ut 4fcf6accc [fwang12] Accumulate the terminate state Authored-by: fwang12 Signed-off-by: fwang12 --- .../kyuubi/operation/KyuubiOperation.scala | 4 +++- .../KyuubiOperationPerUserSuite.scala | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiOperation.scala index 638985ea12b..a6b960847bb 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiOperation.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiOperation.scala @@ -176,7 +176,9 @@ abstract class KyuubiOperation(session: Session) extends AbstractOperation(sessi override def setState(newState: OperationState): Unit = { MetricsSystem.tracing { ms => - ms.markMeter(MetricRegistry.name(OPERATION_STATE, opType, state.toString.toLowerCase), -1) + if (!OperationState.isTerminal(state)) { + ms.markMeter(MetricRegistry.name(OPERATION_STATE, opType, state.toString.toLowerCase), -1) + } ms.markMeter(MetricRegistry.name(OPERATION_STATE, opType, newState.toString.toLowerCase)) ms.markMeter(MetricRegistry.name(OPERATION_STATE, newState.toString.toLowerCase)) } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala index 9ed72307977..2a8ca3a02cc 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala @@ -28,6 +28,7 @@ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.KYUUBI_ENGINE_ENV_PREFIX import org.apache.kyuubi.engine.SemanticVersion import org.apache.kyuubi.jdbc.hive.KyuubiStatement +import org.apache.kyuubi.metrics.{MetricsConstants, MetricsSystem} import org.apache.kyuubi.session.{KyuubiSessionImpl, KyuubiSessionManager, SessionHandle} import org.apache.kyuubi.zookeeper.ZookeeperConf @@ -331,4 +332,21 @@ class KyuubiOperationPerUserSuite assert(!result.next()) } } + + test("accumulate the operation terminal state") { + val opType = classOf[ExecuteStatement].getSimpleName + val finishedMetric = s"${MetricsConstants.OPERATION_STATE}.$opType" + + s".${OperationState.FINISHED.toString.toLowerCase}" + val closedMetric = s"${MetricsConstants.OPERATION_STATE}.$opType" + + s".${OperationState.CLOSED.toString.toLowerCase}" + val finishedCount = MetricsSystem.meterValue(finishedMetric).getOrElse(0L) + val closedCount = MetricsSystem.meterValue(finishedMetric).getOrElse(0L) + withJdbcStatement() { statement => + statement.executeQuery("select engine_name()") + } + eventually(timeout(5.seconds), interval(100.milliseconds)) { + assert(MetricsSystem.meterValue(finishedMetric).getOrElse(0L) > finishedCount) + assert(MetricsSystem.meterValue(closedMetric).getOrElse(0L) > closedCount) + } + } } From 2613a1ba74951e75fd5dfb8f0d4e943ca9098e48 Mon Sep 17 00:00:00 2001 From: liangbowen Date: Sat, 14 Jan 2023 23:51:27 +0800 Subject: [PATCH 002/288] [KYUUBI #4148] Update class name of MySQL 8 JDBC driver used in kyuubi-server tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### _Why are the changes needed?_ Prefer MySQL JDBC driver's new class name `com.mysql.cj.jdbc.Driver` in testing. To eliminate this warning message in kyuubi-server‘s test runs: ``` [INFO] --- scalatest-maven-plugin:2.0.2:test (test) kyuubi-server_2.12 --- Discovery starting. Loading class `com.mysql.jdbc.Driver'. This is deprecated. The new driver class is `com.mysql.cj.jdbc.Driver'. The driver is automatically registered via the SPI and manual loading of the driver class is generally unnecessary. ``` ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4148 from bowenliang123/test-mysql-driver-class. Closes #4148 70b1907a [liangbowen] use `com.mysql.cj.jdbc.Driver` for driver class name in MySQLJDBCTestHelper Authored-by: liangbowen Signed-off-by: Cheng Pan --- .../org/apache/kyuubi/server/mysql/MySQLJDBCTestHelper.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/mysql/MySQLJDBCTestHelper.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/mysql/MySQLJDBCTestHelper.scala index c258b6e6924..a703b6b15f2 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/mysql/MySQLJDBCTestHelper.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/mysql/MySQLJDBCTestHelper.scala @@ -22,7 +22,7 @@ import org.apache.kyuubi.operation.JDBCTestHelper trait MySQLJDBCTestHelper extends JDBCTestHelper { - override def jdbcDriverClass: String = "com.mysql.jdbc.Driver" + override def jdbcDriverClass: String = "com.mysql.cj.jdbc.Driver" protected lazy val user: String = Utils.currentUser From b420243a1faef3c24a5c425eeac6b73ff59bd428 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Mon, 16 Jan 2023 07:44:15 +0800 Subject: [PATCH 003/288] [KYUUBI #4155] Reduce the application info call for batch ### _Why are the changes needed?_ Reduce the application info call for batch. - If terminated and applicationInfo is defined, return applicationInfo directly. - For batch report, return the existing applicationInfo directly. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4155 from turboFei/terminate_state. Closes #4155 9d7e16121 [fwang12] comment a0d70a633 [fwang12] Fix style d9814c5b4 [fwang12] get or fetch e547ff071 [fwang12] refine the variable f9130e30e [fwang12] refactor code 5913d2419 [fwang12] fix ut 3b2772672 [fwang12] reduce app info call a001dd9c4 [fwang12] do not call yarn for batch report beaa54b32 [fwang12] if terminated, do not call Authored-by: fwang12 Signed-off-by: fwang12 --- .../spark/SparkOnKubernetesTestsSuite.scala | 2 +- .../kyuubi/operation/BatchJobSubmission.scala | 59 ++++++++++--------- .../KyuubiApplicationOperation.scala | 2 +- .../kyuubi/operation/LaunchEngine.scala | 2 +- .../server/api/v1/BatchesResource.scala | 2 +- .../kyuubi/WithKyuubiServerOnYarn.scala | 10 ++-- 6 files changed, 42 insertions(+), 35 deletions(-) diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala index e63c3704599..798618e4ca8 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala @@ -206,7 +206,7 @@ class KyuubiOperationKubernetesClusterClusterModeSuite val batchJobSubmissionOp = session.batchJobSubmissionOp eventually(timeout(3.minutes), interval(50.milliseconds)) { - val appInfo = batchJobSubmissionOp.currentApplicationInfo + val appInfo = batchJobSubmissionOp.getOrFetchCurrentApplicationInfo assert(appInfo.nonEmpty) assert(appInfo.exists(_.state == RUNNING)) assert(appInfo.exists(_.name.startsWith(driverPodNamePrefix))) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/BatchJobSubmission.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/BatchJobSubmission.scala index e99b3292c36..c1bcf6cec8d 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/BatchJobSubmission.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/BatchJobSubmission.scala @@ -32,7 +32,7 @@ import org.apache.kyuubi.engine.spark.SparkBatchProcessBuilder import org.apache.kyuubi.metrics.MetricsConstants.OPERATION_OPEN import org.apache.kyuubi.metrics.MetricsSystem import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation -import org.apache.kyuubi.operation.OperationState.{CANCELED, OperationState, RUNNING} +import org.apache.kyuubi.operation.OperationState.{isTerminal, CANCELED, OperationState, RUNNING} import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.server.metadata.api.Metadata import org.apache.kyuubi.session.KyuubiBatchSessionImpl @@ -69,7 +69,11 @@ class BatchJobSubmission( private[kyuubi] val batchId: String = session.handle.identifier.toString - private var applicationInfo: Option[ApplicationInfo] = None + @volatile private var _applicationInfo: Option[ApplicationInfo] = None + def getOrFetchCurrentApplicationInfo: Option[ApplicationInfo] = _applicationInfo match { + case Some(_) => _applicationInfo + case None => currentApplicationInfo + } private var killMessage: KillResponse = (false, "UNKNOWN") def getKillMessage: KillResponse = killMessage @@ -97,7 +101,8 @@ class BatchJobSubmission( } } - override private[kyuubi] def currentApplicationInfo: Option[ApplicationInfo] = { + override protected def currentApplicationInfo: Option[ApplicationInfo] = { + if (isTerminal(state) && _applicationInfo.nonEmpty) return _applicationInfo // only the ApplicationInfo with non-empty id is valid for the operation val applicationInfo = applicationManager.getApplicationInfo(builder.clusterManager(), batchId).filter(_.id != null) @@ -127,13 +132,13 @@ class BatchJobSubmission( } if (isTerminalState(state)) { - if (applicationInfo.isEmpty) { - applicationInfo = + if (_applicationInfo.isEmpty) { + _applicationInfo = Option(ApplicationInfo(id = null, name = null, state = ApplicationState.NOT_FOUND)) } } - applicationInfo.foreach { status => + _applicationInfo.foreach { status => val metadataToUpdate = Metadata( identifier = batchId, state = state.toString, @@ -154,7 +159,7 @@ class BatchJobSubmission( private def setStateIfNotCanceled(newState: OperationState): Unit = state.synchronized { if (state != CANCELED) { setState(newState) - applicationInfo.filter(_.id != null).foreach { ai => + _applicationInfo.filter(_.id != null).foreach { ai => session.getSessionEvent.foreach(_.engineId = ai.id) } if (newState == RUNNING) { @@ -184,8 +189,8 @@ class BatchJobSubmission( // submitted batch application. recoveryMetadata.map { metadata => if (metadata.state == OperationState.PENDING.toString) { - applicationInfo = currentApplicationInfo - applicationInfo.map(_.id) match { + _applicationInfo = currentApplicationInfo + _applicationInfo.map(_.id) match { case Some(null) => submitAndMonitorBatchJob() case Some(appId) => @@ -226,10 +231,10 @@ class BatchJobSubmission( try { info(s"Submitting $batchType batch[$batchId] job:\n$builder") val process = builder.start - applicationInfo = currentApplicationInfo - while (!applicationFailed(applicationInfo) && process.isAlive) { + _applicationInfo = currentApplicationInfo + while (!applicationFailed(_applicationInfo) && process.isAlive) { if (!appStatusFirstUpdated) { - if (applicationInfo.isDefined) { + if (_applicationInfo.isDefined) { setStateIfNotCanceled(OperationState.RUNNING) updateBatchMetadata() appStatusFirstUpdated = true @@ -243,19 +248,19 @@ class BatchJobSubmission( } } process.waitFor(applicationCheckInterval, TimeUnit.MILLISECONDS) - applicationInfo = currentApplicationInfo + _applicationInfo = currentApplicationInfo } - if (applicationFailed(applicationInfo)) { + if (applicationFailed(_applicationInfo)) { process.destroyForcibly() - throw new RuntimeException(s"Batch job failed: $applicationInfo") + throw new RuntimeException(s"Batch job failed: ${_applicationInfo}") } else { process.waitFor() if (process.exitValue() != 0) { throw new KyuubiException(s"Process exit with value ${process.exitValue()}") } - Option(applicationInfo.map(_.id)).foreach { + Option(_applicationInfo.map(_.id)).foreach { case Some(appId) => monitorBatchJob(appId) case _ => } @@ -267,30 +272,30 @@ class BatchJobSubmission( private def monitorBatchJob(appId: String): Unit = { info(s"Monitoring submitted $batchType batch[$batchId] job: $appId") - if (applicationInfo.isEmpty) { - applicationInfo = currentApplicationInfo + if (_applicationInfo.isEmpty) { + _applicationInfo = currentApplicationInfo } if (state == OperationState.PENDING) { setStateIfNotCanceled(OperationState.RUNNING) } - if (applicationInfo.isEmpty) { + if (_applicationInfo.isEmpty) { info(s"The $batchType batch[$batchId] job: $appId not found, assume that it has finished.") - } else if (applicationFailed(applicationInfo)) { - throw new RuntimeException(s"$batchType batch[$batchId] job failed: $applicationInfo") + } else if (applicationFailed(_applicationInfo)) { + throw new RuntimeException(s"$batchType batch[$batchId] job failed: ${_applicationInfo}") } else { updateBatchMetadata() // TODO: add limit for max batch job submission lifetime - while (applicationInfo.isDefined && !applicationTerminated(applicationInfo)) { + while (_applicationInfo.isDefined && !applicationTerminated(_applicationInfo)) { Thread.sleep(applicationCheckInterval) val newApplicationStatus = currentApplicationInfo - if (newApplicationStatus.map(_.state) != applicationInfo.map(_.state)) { - applicationInfo = newApplicationStatus - info(s"Batch report for $batchId, $applicationInfo") + if (newApplicationStatus.map(_.state) != _applicationInfo.map(_.state)) { + _applicationInfo = newApplicationStatus + info(s"Batch report for $batchId, ${_applicationInfo}") } } - if (applicationFailed(applicationInfo)) { - throw new RuntimeException(s"$batchType batch[$batchId] job failed: $applicationInfo") + if (applicationFailed(_applicationInfo)) { + throw new RuntimeException(s"$batchType batch[$batchId] job failed: ${_applicationInfo}") } } } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiApplicationOperation.scala index cf10b2da41a..b864f0101ef 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiApplicationOperation.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiApplicationOperation.scala @@ -31,7 +31,7 @@ import org.apache.kyuubi.util.ThriftUtils abstract class KyuubiApplicationOperation(session: Session) extends KyuubiOperation(session) { - private[kyuubi] def currentApplicationInfo: Option[ApplicationInfo] + protected def currentApplicationInfo: Option[ApplicationInfo] override def getResultSetMetadata: TGetResultSetMetadataResp = { val schema = new TTableSchema() diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/LaunchEngine.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/LaunchEngine.scala index 0444b92fd81..3d9b4937fd5 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/LaunchEngine.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/LaunchEngine.scala @@ -33,7 +33,7 @@ class LaunchEngine(session: KyuubiSessionImpl, override val shouldRunAsync: Bool } override def getOperationLog: Option[OperationLog] = Option(_operationLog) - override private[kyuubi] def currentApplicationInfo: Option[ApplicationInfo] = { + override protected def currentApplicationInfo: Option[ApplicationInfo] = { Option(client).map { cli => ApplicationInfo( cli.engineId.orNull, diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala index 487362d96b1..c00fb95f6aa 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala @@ -68,7 +68,7 @@ private[v1] class BatchesResource extends ApiRequestContext with Logging { private def buildBatch(session: KyuubiBatchSessionImpl): Batch = { val batchOp = session.batchJobSubmissionOp val batchOpStatus = batchOp.getStatus - val batchAppStatus = batchOp.currentApplicationInfo + val batchAppStatus = batchOp.getOrFetchCurrentApplicationInfo val name = Option(batchOp.batchName).getOrElse(batchAppStatus.map(_.name).orNull) var appId: String = null diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerOnYarn.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerOnYarn.scala index 727c5545e9d..27e769d2908 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerOnYarn.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerOnYarn.scala @@ -23,8 +23,8 @@ import scala.concurrent.duration._ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf._ import org.apache.kyuubi.config.KyuubiConf.FrontendProtocols.FrontendProtocol +import org.apache.kyuubi.engine.{ApplicationState, YarnApplicationOperation} import org.apache.kyuubi.engine.ApplicationState._ -import org.apache.kyuubi.engine.YarnApplicationOperation import org.apache.kyuubi.operation.{FetchOrientation, HiveJDBCTestHelper, OperationState} import org.apache.kyuubi.operation.OperationState.ERROR import org.apache.kyuubi.server.MiniYarnService @@ -117,7 +117,7 @@ class KyuubiOperationYarnClusterSuite extends WithKyuubiServerOnYarn with HiveJD val batchJobSubmissionOp = session.batchJobSubmissionOp eventually(timeout(3.minutes), interval(50.milliseconds)) { - val appInfo = batchJobSubmissionOp.currentApplicationInfo + val appInfo = batchJobSubmissionOp.getOrFetchCurrentApplicationInfo assert(appInfo.nonEmpty) assert(appInfo.exists(_.id.startsWith("application_"))) } @@ -152,7 +152,7 @@ class KyuubiOperationYarnClusterSuite extends WithKyuubiServerOnYarn with HiveJD val appUrl = rows("url") val appError = rows("error") - val appInfo2 = batchJobSubmissionOp.currentApplicationInfo.get + val appInfo2 = batchJobSubmissionOp.getOrFetchCurrentApplicationInfo.get assert(appId === appInfo2.id) assert(appName === appInfo2.name) assert(appState === appInfo2.state.toString) @@ -175,7 +175,9 @@ class KyuubiOperationYarnClusterSuite extends WithKyuubiServerOnYarn with HiveJD val batchJobSubmissionOp = session.batchJobSubmissionOp eventually(timeout(3.minutes), interval(50.milliseconds)) { - assert(batchJobSubmissionOp.currentApplicationInfo.isEmpty) + assert(batchJobSubmissionOp.getOrFetchCurrentApplicationInfo.exists(_.id == null)) + assert(batchJobSubmissionOp.getOrFetchCurrentApplicationInfo.exists( + _.state == ApplicationState.NOT_FOUND)) assert(batchJobSubmissionOp.getStatus.state === OperationState.ERROR) } } From 3c5b0988912cdc0bf164288203091624c5bd002d Mon Sep 17 00:00:00 2001 From: liangbowen Date: Mon, 16 Jan 2023 10:50:27 +0800 Subject: [PATCH 004/288] [KYUUBI #4170] [Arrow] Fix `fallthrough` cases in type conversion and resultset handling in kyuubi-hive-jdbc ### _Why are the changes needed?_ Fix possible `fallthrough` cases: - in DECIMAL_TYPE case of `ArrowUtils#toArrowType` - in "arrow" case of `KyuubiStatement#executeAsync` The cases above are discovered from warning from javac output which will be eliminated after this PR: ``` /home/runner/work/kyuubi/kyuubi/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiStatement.java:283: warning: [fallthrough] possible fall-through into case default: ^ /home/runner/work/kyuubi/kyuubi/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/arrow/ArrowUtils.java:83: warning: [fallthrough] possible fall-through into case case DATE_TYPE: ^ ``` ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4170 from bowenliang123/arrow-fallthrough. Closes #4170 be3d342e [liangbowen] fix fallthrough in "arrow" case of KyuubiStatement#executeAsync 07365d80 [liangbowen] fix fallthrough in DECIMAL_TYPE case of ArrowUtils#toArrowType Authored-by: liangbowen Signed-off-by: ulysses-you --- .../main/java/org/apache/kyuubi/jdbc/hive/KyuubiStatement.java | 1 + .../main/java/org/apache/kyuubi/jdbc/hive/arrow/ArrowUtils.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiStatement.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiStatement.java index ee3dc71a97d..ab7c06a5589 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiStatement.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiStatement.java @@ -280,6 +280,7 @@ public boolean executeAsync(String sql) throws SQLException { .setScrollable(isScrollableResultset) .setSchema(columnNames, columnTypes, columnAttributes) .build(); + break; default: resultSet = new KyuubiQueryResultSet.Builder(this) diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/arrow/ArrowUtils.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/arrow/ArrowUtils.java index 1d1587b5444..9a777d4c240 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/arrow/ArrowUtils.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/arrow/ArrowUtils.java @@ -79,6 +79,8 @@ public static ArrowType toArrowType(TTypeId ttype, JdbcColumnAttributes jdbcColu if (jdbcColumnAttributes != null) { return ArrowType.Decimal.createDecimal( jdbcColumnAttributes.precision, jdbcColumnAttributes.scale, null); + } else { + throw new IllegalStateException("Missing precision and scale where it is mandatory."); } case DATE_TYPE: return new ArrowType.Date(DateUnit.DAY); From 87582a74580e346f44f5eece7ac973600f5c0a74 Mon Sep 17 00:00:00 2001 From: Ruguo Yu Date: Mon, 16 Jan 2023 10:52:01 +0800 Subject: [PATCH 005/288] [KYUUBI #4165] [DOCS] Update the docs link ### _Why are the changes needed?_ Some links of the website are not available. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4165 from yuruguo/update_doc-url. Closes #4165 f7a847c3 [Ruguo Yu] [DOCS] Update the docs link Authored-by: Ruguo Yu Signed-off-by: ulysses-you --- .github/PULL_REQUEST_TEMPLATE | 2 +- CONTRIBUTING.md | 2 +- README.md | 2 +- conf/kyuubi-defaults.conf.template | 2 +- docs/deployment/settings.md | 2 +- .../src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 6e37b26c46f..bdb71f30fb1 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -20,4 +20,4 @@ Please clarify why the changes are needed. For instance, - [ ] Add screenshots for manual tests if appropriate -- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request +- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b6348cd29a..aa3dc4167b3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,7 +45,7 @@ You can make various types of contributions to Kyuubi, including the following b - Answer questions in the [Mailing Lists](https://kyuubi.apache.org/mailing_lists.html) - [Share your success stories with us](https://github.com/apache/kyuubi/discussions/925) -- Improve Documentation - [![Documentation Status](https://readthedocs.org/projects/kyuubi/badge/?version=latest)](https://kyuubi.apache.org/docs/latest/) +- Improve Documentation - [![Documentation Status](https://readthedocs.org/projects/kyuubi/badge/?version=latest)](https://kyuubi.readthedocs.io/en/master/) - Test latest releases - [![Latest tag](https://img.shields.io/github/v/tag/apache/kyuubi?label=tag)](https://github.com/apache/kyuubi/tags) - Improve test coverage - [![codecov](https://codecov.io/gh/apache/kyuubi/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/kyuubi) - Report bugs and better help developers to reproduce diff --git a/README.md b/README.md index 6ac866c3071..b38d6933407 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ [![codecov](https://codecov.io/gh/apache/kyuubi/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/kyuubi) ![GitHub Workflow Status](https://img.shields.io/github/workflow/status/apache/kyuubi/Kyuubi/master?style=plastic) [![Travis](https://api.travis-ci.com/apache/kyuubi.svg?branch=master)](https://travis-ci.com/apache/kyuubi) -[![Documentation Status](https://readthedocs.org/projects/kyuubi/badge/?version=latest)](https://kyuubi.apache.org/docs/latest/) +[![Documentation Status](https://readthedocs.org/projects/kyuubi/badge/?version=latest)](https://kyuubi.readthedocs.io/en/master/) ![GitHub top language](https://img.shields.io/github/languages/top/apache/kyuubi) [![Commit activity](https://img.shields.io/github/commit-activity/m/apache/kyuubi)](https://github.com/apache/kyuubi/graphs/commit-activity) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/apache/kyuubi.svg)](http://isitmaintained.com/project/apache/kyuubi "Average time to resolve an issue") diff --git a/conf/kyuubi-defaults.conf.template b/conf/kyuubi-defaults.conf.template index 5a4b9b2a791..d3e6026d9fb 100644 --- a/conf/kyuubi-defaults.conf.template +++ b/conf/kyuubi-defaults.conf.template @@ -23,4 +23,4 @@ # kyuubi.frontend.bind.port 10009 # -# Details in https://kyuubi.apache.org/docs/latest/deployment/settings.html +# Details in https://kyuubi.readthedocs.io/en/master/deployment/settings.html diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md index 539ca823f6b..14815897101 100644 --- a/docs/deployment/settings.md +++ b/docs/deployment/settings.md @@ -129,7 +129,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co # kyuubi.frontend.bind.port 10009 # -# Details in https://kyuubi.apache.org/docs/latest/deployment/settings.html +# Details in https://kyuubi.readthedocs.io/en/master/deployment/settings.html ``` ### Authentication diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala index 5b69b02f54d..a119d971dba 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala @@ -335,7 +335,7 @@ trait ProcBuilder { protected def validateEnv(requiredEnv: String): Throwable = { KyuubiSQLException(s"$requiredEnv is not set! For more information on installing and " + - s"configuring $requiredEnv, please visit https://kyuubi.apache.org/docs/latest/" + + s"configuring $requiredEnv, please visit https://kyuubi.readthedocs.io/en/master/" + s"deployment/settings.html#environments") } From 149c7d208cdc30cf0ca8414572c1c5dc1fb2d2d5 Mon Sep 17 00:00:00 2001 From: liangbowen Date: Mon, 16 Jan 2023 11:11:31 +0800 Subject: [PATCH 006/288] [KYUUBI #4173] Eliminate `scalac` warnings for class link in comments ### _Why are the changes needed?_ Adding following rules to `globalFilters` for `silencer` plugin - `.*Could not find any member to link for.*` - `.*undefined in comment for class.*` To refine CI output in a more helpful, to eliminate over 68 class link warnings in Scala comments in single run , as outputs like: ``` /home/runner/work/kyuubi/kyuubi/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala:80: warning: Could not find any member to link for "Credentials#tokenMap". /** ^ /home/runner/work/kyuubi/kyuubi/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/JsonLoggingEventHandler.scala:37: warning: Variable eventType undefined in comment for class JsonLoggingEventHandler in class JsonLoggingEventHandler * ${ENGINE_EVENT_JSON_LOG_PATH}/${eventType}/day=${date}/${logName}.json ``` ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4173 from bowenliang123/scalac-link-warning. Closes #4173 5ddb7e44 [liangbowen] add `.*undefined in comment for class.*` to silencer 7053623c [liangbowen] add `.*Could not find any member to link for.*` to silencer Authored-by: liangbowen Signed-off-by: ulysses-you --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index b7da49088d2..697115bd6f1 100644 --- a/pom.xml +++ b/pom.xml @@ -1769,6 +1769,8 @@ -explaintypes -Yno-adapted-args -P:silencer:globalFilters=.*deprecated.* + -P:silencer:globalFilters=.*Could not find any member to link for.* + -P:silencer:globalFilters=.*undefined in comment for class.* -Xfatal-warnings -Ywarn-unused:imports From 6237a543c2624564df890f3391282b64bc76da12 Mon Sep 17 00:00:00 2001 From: liangbowen Date: Mon, 16 Jan 2023 12:08:05 +0800 Subject: [PATCH 007/288] [KYUUBI #4166] Eliminate unhelpful warning messages in Javac output of building and testing ### _Why are the changes needed?_ The output of building and testing is now brought with repeated unuseful warnings from `javac`. This PR reduces over 500 unhelpful warning message lines from `-Xlint` rules in `javac` compilation output in a single run, by eliminating : - 74 of `deprecation`, by adding `SuppressWarning` to 6 Java classes - 6 of `rawtypes`, by adding `SuppressWarning` to methods of Java class - 4 of `unchecked`, by adding `SuppressWarning` to methods of Java class - 1 of `fallthrough`, by adding `SuppressWarning` to method of Java class - 1 of `cast`, by removal of redundant type casting - repeated `processing` for, by adding `-processing` to disable annotation processor warning for annotations from testing framework : ``` - Warning: Unexpected javac output: warning: No processor claimed any of these annotations: /org.scalatest.TagAnnotation Warning: Unexpected javac output: warning: No processor claimed any of these annotations: /org.junit.Test,/org.junit.Before,/org.junit.After ``` As 1 [fallthrough] warning message remain, which will be reported in followup issues or PR to fix missing breaks for branches of `switch-case` in Java. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4166 from bowenliang123/java-warnings. Closes #4166 ec4cfc42 [liangbowen] add @SuppressWarnings("fallthrough") to KyuubiConnection#waitLaunchEngineToComplete c9a59522 [liangbowen] skip deprecation warnings for SQLResultSet and KyuubiBaseResultSet cab8c358 [liangbowen] remove redundant casting f2ce8963 [liangbowen] style bf6c5936 [liangbowen] add `-processing` to javac Xlint option for eliminating the missing annotation processer warnings 01a89a8c [liangbowen] eliminate warning outputs in java by adding @SuppressWarnings Authored-by: liangbowen Signed-off-by: ulysses-you --- .../java/org/apache/kyuubi/reflection/DynConstructors.java | 3 ++- .../src/main/java/org/apache/kyuubi/reflection/DynFields.java | 1 + .../apache/kyuubi/jdbc/hive/KyuubiArrowBasedResultSet.java | 1 + .../java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java | 1 + .../java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java | 2 ++ .../apache/kyuubi/jdbc/hive/adapter/SQLCallableStatement.java | 1 + .../apache/kyuubi/jdbc/hive/adapter/SQLPreparedStatement.java | 1 + .../org/apache/kyuubi/jdbc/hive/adapter/SQLResultSet.java | 1 + .../java/org/apache/kyuubi/jdbc/hive/cli/ColumnBuffer.java | 1 + .../org/apache/kyuubi/jdbc/hive/common/FastHiveDecimal.java | 1 + .../apache/kyuubi/jdbc/hive/common/FastHiveDecimalImpl.java | 2 +- .../java/org/apache/kyuubi/client/RetryableRestClient.java | 1 + pom.xml | 4 ++-- 13 files changed, 16 insertions(+), 4 deletions(-) diff --git a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynConstructors.java b/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynConstructors.java index 7495ce0ffb4..59c79b88502 100644 --- a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynConstructors.java +++ b/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynConstructors.java @@ -119,6 +119,7 @@ public static Builder builder(Class baseClass) { return new Builder(baseClass); } + @SuppressWarnings("rawtypes") public static class Builder { private final Class baseClass; private ClassLoader loader = Thread.currentThread().getContextClassLoader(); @@ -182,7 +183,7 @@ public Builder hiddenImpl(Class... types) { return this; } - @SuppressWarnings("unchecked") + @SuppressWarnings({"unchecked", "rawtypes"}) public Builder hiddenImpl(String className, Class... types) { // don't do any work if an implementation has been found if (ctor != null) { diff --git a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynFields.java b/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynFields.java index 39c83b1621a..9430d54e9bb 100644 --- a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynFields.java +++ b/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynFields.java @@ -300,6 +300,7 @@ public Builder hiddenImpl(String className, String fieldName) { * @see Class#forName(String) * @see Class#getField(String) */ + @SuppressWarnings("rawtypes") public Builder hiddenImpl(Class targetClass, String fieldName) { // don't do any work if an implementation has been found if (field != null || targetClass == null) { diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowBasedResultSet.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowBasedResultSet.java index e45b6545471..c3e75c0ea0e 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowBasedResultSet.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowBasedResultSet.java @@ -34,6 +34,7 @@ import org.apache.kyuubi.jdbc.hive.arrow.ArrowUtils; /** Data independent base class which implements the common part of all Kyuubi result sets. */ +@SuppressWarnings("deprecation") public abstract class KyuubiArrowBasedResultSet implements SQLResultSet { protected Statement statement = null; diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java index 5fe889346e6..a9d32e8cafb 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java @@ -32,6 +32,7 @@ import org.apache.kyuubi.jdbc.hive.common.TimestampTZUtil; /** Data independent base class which implements the common part of all Kyuubi result sets. */ +@SuppressWarnings("deprecation") public abstract class KyuubiBaseResultSet implements SQLResultSet { protected Statement statement = null; diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java index 1d7755b1ef9..9931dcec2d0 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java @@ -1248,6 +1248,7 @@ public TProtocolVersion getProtocol() { return protocol; } + @SuppressWarnings("rawtypes") public static TCLIService.Iface newSynchronizedClient(TCLIService.Iface client) { return (TCLIService.Iface) Proxy.newProxyInstance( @@ -1286,6 +1287,7 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl } } + @SuppressWarnings("fallthrough") public void waitLaunchEngineToComplete() throws SQLException { if (launchEngineOpHandle == null) return; diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLCallableStatement.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLCallableStatement.java index 9ebe07011f7..4e62a3b00a3 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLCallableStatement.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLCallableStatement.java @@ -25,6 +25,7 @@ import java.util.Calendar; import java.util.Map; +@SuppressWarnings("deprecation") public interface SQLCallableStatement extends CallableStatement { @Override diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLPreparedStatement.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLPreparedStatement.java index 6bc9d383a1b..cbcaf2788e1 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLPreparedStatement.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLPreparedStatement.java @@ -23,6 +23,7 @@ import java.sql.*; import java.util.Calendar; +@SuppressWarnings("deprecation") public interface SQLPreparedStatement extends PreparedStatement { @Override diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLResultSet.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLResultSet.java index 8523e4b8d64..70c8ff4fe57 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLResultSet.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/adapter/SQLResultSet.java @@ -25,6 +25,7 @@ import java.util.Calendar; import java.util.Map; +@SuppressWarnings("deprecation") public interface SQLResultSet extends ResultSet { @Override diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/cli/ColumnBuffer.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/cli/ColumnBuffer.java index 0cdbb9a5e5d..e703cb1f00c 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/cli/ColumnBuffer.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/cli/ColumnBuffer.java @@ -89,6 +89,7 @@ public ColumnBuffer(TColumn colValues) { } } + @SuppressWarnings("unchecked") public ColumnBuffer(TTypeId type, BitSet nulls, Object values) { this.type = type; this.nulls = nulls; diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimal.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimal.java index 09b46fe4911..41fb9f840b9 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimal.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimal.java @@ -457,6 +457,7 @@ protected boolean fastScaleByPowerOfTen(int n, FastHiveDecimal fastResult) { fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, n, fastResult); } + @SuppressWarnings("deprecation") protected static String fastRoundingModeToString(int roundingMode) { String roundingModeString; switch (roundingMode) { diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimalImpl.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimalImpl.java index 619371cbfb4..50c7daa8bf8 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimalImpl.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimalImpl.java @@ -9369,7 +9369,7 @@ public static String getStackTraceAsSingleLine(StackTraceElement[] stackTrace) { public static String displayBytes(byte[] bytes, int start, int length) { StringBuilder sb = new StringBuilder(); for (int i = start; i < start + length; i++) { - sb.append(String.format("\\%03d", (int) (bytes[i] & 0xff))); + sb.append(String.format("\\%03d", bytes[i] & 0xff)); } return sb.toString(); } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RetryableRestClient.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RetryableRestClient.java index 6dd378a9ab0..dcd052acae4 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RetryableRestClient.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RetryableRestClient.java @@ -48,6 +48,7 @@ private RetryableRestClient(List uris, RestClientConf conf) { newRestClient(); } + @SuppressWarnings("rawtypes") public static IRestClient getRestClient(List uris, RestClientConf conf) { RetryableRestClient client = new RetryableRestClient(uris, conf); return (IRestClient) diff --git a/pom.xml b/pom.xml index 697115bd6f1..064e3656613 100644 --- a/pom.xml +++ b/pom.xml @@ -1750,7 +1750,7 @@ 1024m true - -Xlint:all,-serial,-path + -Xlint:all,-serial,-path,-processing @@ -1780,7 +1780,7 @@ -XX:ReservedCodeCacheSize=512M - -Xlint:all,-serial,-path,-try + -Xlint:all,-serial,-path,-try,-processing From 89c7435dcab5deadcfec007feb7142fcc56622d6 Mon Sep 17 00:00:00 2001 From: liangbowen Date: Mon, 16 Jan 2023 18:34:01 +0800 Subject: [PATCH 008/288] [KYUUBI #4161] [DOCS] Refine settings page with correction in grammar and spelling mistakes of config descriptions ### _Why are the changes needed?_ As Kyuubi graduated as top level project, the setting page will be more often requested and should be increasingly reliable and readable with less grammar and spelling mistakes. This PR is to - correct mistakes in grammar, spelling, abbreviation and terminology - with no config name or essential meanings changed ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4161 from bowenliang123/conf-grammar. Closes #4161 038edfbea [liangbowen] nit 1ec073a4b [liangbowen] to JSON 4f5259a32 [liangbowen] to Prometheus 523855008 [liangbowen] to K8s fc7a3a81e [liangbowen] to AUTO-GENERATED da64f54fa [liangbowen] update d54f9a528 [liangbowen] fix `comma separated` to `comma-separated` f1d7cc1f1 [liangbowen] update d84208844 [liangbowen] update 1b75f011c [liangbowen] correction of grammar and spelling mistakes Authored-by: liangbowen Signed-off-by: Kent Yao --- docs/deployment/settings.md | 262 ++++++++--------- docs/extensions/engines/spark/functions.md | 2 +- docs/monitor/metrics.md | 6 +- .../udf/KyuubiDefinedFunctionSuite.scala | 2 +- .../org/apache/kyuubi/config/KyuubiConf.scala | 278 +++++++++--------- .../scala/org/apache/kyuubi/ctl/CtlConf.scala | 6 +- .../kyuubi/ha/HighAvailabilityConf.scala | 34 +-- .../apache/kyuubi/metrics/MetricsConf.scala | 8 +- .../metadata/jdbc/JDBCMetadataStoreConf.scala | 21 +- .../config/AllKyuubiConfiguration.scala | 10 +- .../kyuubi/zookeeper/ZookeeperConf.scala | 20 +- 11 files changed, 329 insertions(+), 320 deletions(-) diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md index 14815897101..4c7a3b66aca 100644 --- a/docs/deployment/settings.md +++ b/docs/deployment/settings.md @@ -15,7 +15,7 @@ - limitations under the License. --> - + # Introduction to the Kyuubi Configurations System @@ -98,7 +98,7 @@ You can configure the environment variables in `$KYUUBI_HOME/conf/kyuubi-env.sh` # export KYUUBI_BEELINE_OPTS="-Xmx2g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark" ``` -For the environment variables that only needed to be transferred into engine side, you can set it with a Kyuubi configuration item formatted `kyuubi.engineEnv.VAR_NAME`. For example, with `kyuubi.engineEnv.SPARK_DRIVER_MEMORY=4g`, the environment variable `SPARK_DRIVER_MEMORY` with value `4g` would be transferred into engine side. With `kyuubi.engineEnv.SPARK_CONF_DIR=/apache/confs/spark/conf`, the value of `SPARK_CONF_DIR` in engine side is set to `/apache/confs/spark/conf`. +For the environment variables that only needed to be transferred into engine side, you can set it with a Kyuubi configuration item formatted `kyuubi.engineEnv.VAR_NAME`. For example, with `kyuubi.engineEnv.SPARK_DRIVER_MEMORY=4g`, the environment variable `SPARK_DRIVER_MEMORY` with value `4g` would be transferred into engine side. With `kyuubi.engineEnv.SPARK_CONF_DIR=/apache/confs/spark/conf`, the value of `SPARK_CONF_DIR` on the engine side is set to `/apache/confs/spark/conf`. ## Kyuubi Configurations @@ -136,7 +136,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co Key | Default | Meaning | Type | Since --- | --- | --- | --- | --- -kyuubi.authentication|NONE|A comma separated list of client authentication types.
  • NOSASL: raw transport.
  • NONE: no authentication check.
  • KERBEROS: Kerberos/GSSAPI authentication.
  • CUSTOM: User-defined authentication.
  • JDBC: JDBC query authentication.
  • LDAP: Lightweight Directory Access Protocol authentication.
Note that: For KERBEROS, it is SASL/GSSAPI mechanism, and for NONE, CUSTOM and LDAP, they are all SASL/PLAIN mechanism. If only NOSASL is specified, the authentication will be NOSASL. For SASL authentication, KERBEROS and PLAIN auth type are supported at the same time, and only the first specified PLAIN auth type is valid.|seq|1.0.0 +kyuubi.authentication|NONE|A comma-separated list of client authentication types.
  • NOSASL: raw transport.
  • NONE: no authentication check.
  • KERBEROS: Kerberos/GSSAPI authentication.
  • CUSTOM: User-defined authentication.
  • JDBC: JDBC query authentication.
  • LDAP: Lightweight Directory Access Protocol authentication.
Note that: For KERBEROS, it is SASL/GSSAPI mechanism, and for NONE, CUSTOM and LDAP, they are all SASL/PLAIN mechanisms. If only NOSASL is specified, the authentication will be NOSASL. For SASL authentication, KERBEROS and PLAIN auth types are supported at the same time, and only the first specified PLAIN auth type is valid.|seq|1.0.0 kyuubi.authentication.custom.class|<undefined>|User-defined authentication implementation of org.apache.kyuubi.service.authentication.PasswdAuthenticationProvider|string|1.3.0 kyuubi.authentication.jdbc.driver.class|<undefined>|Driver class name for JDBC Authentication Provider.|string|1.6.0 kyuubi.authentication.jdbc.password|<undefined>|Database password for JDBC Authentication Provider.|string|1.6.0 @@ -158,8 +158,8 @@ kyuubi.backend.engine.exec.pool.keepalive.time|PT1M|Time(ms) that an idle async kyuubi.backend.engine.exec.pool.shutdown.timeout|PT10S|Timeout(ms) for the operation execution thread pool to terminate in SQL engine applications|duration|1.0.0 kyuubi.backend.engine.exec.pool.size|100|Number of threads in the operation execution thread pool of SQL engine applications|int|1.0.0 kyuubi.backend.engine.exec.pool.wait.queue.size|100|Size of the wait queue for the operation execution thread pool in SQL engine applications|int|1.0.0 -kyuubi.backend.server.event.json.log.path|file:///tmp/kyuubi/events|The location of server events go for the builtin JSON logger|string|1.4.0 -kyuubi.backend.server.event.loggers||A comma separated list of server history loggers, where session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.backend.server.event.json.log.path
  • JDBC: to be done
  • CUSTOM: User-defined event handlers.
Note that: Kyuubi supports custom event handlers with the Java SPI. To register a custom event handler, user need to implement a class which is a child of org.apache.kyuubi.events.handler.CustomEventHandlerProvider which has zero-arg constructor.|seq|1.4.0 +kyuubi.backend.server.event.json.log.path|file:///tmp/kyuubi/events|The location of server events go for the built-in JSON logger|string|1.4.0 +kyuubi.backend.server.event.loggers||A comma-separated list of server history loggers, where session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.backend.server.event.json.log.path
  • JDBC: to be done
  • CUSTOM: User-defined event handlers.
Note that: Kyuubi supports custom event handlers with the Java SPI. To register a custom event handler, the user needs to implement a class which is a child of org.apache.kyuubi.events.handler.CustomEventHandlerProvider which has a zero-arg constructor.|seq|1.4.0 kyuubi.backend.server.exec.pool.keepalive.time|PT1M|Time(ms) that an idle async thread of the operation execution thread pool will wait for a new task to arrive before terminating in Kyuubi server|duration|1.0.0 kyuubi.backend.server.exec.pool.shutdown.timeout|PT10S|Timeout(ms) for the operation execution thread pool to terminate in Kyuubi server|duration|1.0.0 kyuubi.backend.server.exec.pool.size|100|Number of threads in the operation execution thread pool of Kyuubi server|int|1.0.0 @@ -172,7 +172,7 @@ Key | Default | Meaning | Type | Since --- | --- | --- | --- | --- kyuubi.batch.application.check.interval|PT5S|The interval to check batch job application information.|duration|1.6.0 kyuubi.batch.application.starvation.timeout|PT3M|Threshold above which to warn batch application may be starved.|duration|1.7.0 -kyuubi.batch.conf.ignore.list||A comma separated list of ignored keys for batch conf. If the batch conf contains any of them, the key and the corresponding value will be removed silently during batch job submission. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering. You can also pre-define some config for batch job submission with prefix: kyuubi.batchConf.[batchType]. For example, you can pre-define `spark.master` for spark batch job with key `kyuubi.batchConf.spark.spark.master`.|seq|1.6.0 +kyuubi.batch.conf.ignore.list||A comma-separated list of ignored keys for batch conf. If the batch conf contains any of them, the key and the corresponding value will be removed silently during batch job submission. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering. You can also pre-define some config for batch job submission with the prefix: kyuubi.batchConf.[batchType]. For example, you can pre-define `spark.master` for the Spark batch job with key `kyuubi.batchConf.spark.spark.master`.|seq|1.6.0 kyuubi.batch.session.idle.timeout|PT6H|Batch session idle timeout, it will be closed when it's not accessed for this duration|duration|1.6.2 @@ -184,10 +184,10 @@ kyuubi.credentials.check.interval|PT5M|The interval to check the expiration of c kyuubi.credentials.hadoopfs.enabled|true|Whether to renew Hadoop filesystem delegation tokens|boolean|1.4.0 kyuubi.credentials.hadoopfs.uris||Extra Hadoop filesystem URIs for which to request delegation tokens. The filesystem that hosts fs.defaultFS does not need to be listed here.|seq|1.4.0 kyuubi.credentials.hive.enabled|true|Whether to renew Hive metastore delegation token|boolean|1.4.0 -kyuubi.credentials.idle.timeout|PT6H|inactive users' credentials will be expired after a configured timeout|duration|1.6.0 +kyuubi.credentials.idle.timeout|PT6H|The inactive users' credentials will be expired after a configured timeout|duration|1.6.0 kyuubi.credentials.renewal.interval|PT1H|How often Kyuubi renews one user's delegation tokens|duration|1.4.0 kyuubi.credentials.renewal.retry.wait|PT1M|How long to wait before retrying to fetch new credentials after a failure.|duration|1.4.0 -kyuubi.credentials.update.wait.timeout|PT1M|How long to wait until credentials are ready.|duration|1.5.0 +kyuubi.credentials.update.wait.timeout|PT1M|How long to wait until the credentials are ready.|duration|1.5.0 ### Ctl @@ -197,11 +197,11 @@ Key | Default | Meaning | Type | Since kyuubi.ctl.batch.log.on.failure.timeout|PT10S|The timeout for fetching remaining batch logs if the batch failed.|duration|1.6.1 kyuubi.ctl.batch.log.query.interval|PT3S|The interval for fetching batch logs.|duration|1.6.0 kyuubi.ctl.rest.auth.schema|basic|The authentication schema. Valid values are: basic, spnego.|string|1.6.0 -kyuubi.ctl.rest.base.url|<undefined>|The REST API base URL, which contains the scheme (http:// or https://), host name, port number|string|1.6.0 -kyuubi.ctl.rest.connect.timeout|PT30S|The timeout[ms] for establishing the connection with the kyuubi server.A timeout value of zero is interpreted as an infinite timeout.|duration|1.6.0 +kyuubi.ctl.rest.base.url|<undefined>|The REST API base URL, which contains the scheme (http:// or https://), hostname, port number|string|1.6.0 +kyuubi.ctl.rest.connect.timeout|PT30S|The timeout[ms] for establishing the connection with the kyuubi server. A timeout value of zero is interpreted as an infinite timeout.|duration|1.6.0 kyuubi.ctl.rest.request.attempt.wait|PT3S|How long to wait between attempts of ctl rest request.|duration|1.6.0 kyuubi.ctl.rest.request.max.attempts|3|The max attempts number for ctl rest request.|int|1.6.0 -kyuubi.ctl.rest.socket.timeout|PT2M|The timeout[ms] for waiting for data packets after connection is established.A timeout value of zero is interpreted as an infinite timeout.|duration|1.6.0 +kyuubi.ctl.rest.socket.timeout|PT2M|The timeout[ms] for waiting for data packets after connection is established. A timeout value of zero is interpreted as an infinite timeout.|duration|1.6.0 kyuubi.ctl.rest.spnego.host|<undefined>|When auth schema is spnego, need to config spnego host.|string|1.6.0 @@ -219,57 +219,57 @@ kyuubi.delegation.token.renew.interval|PT168H|unused yet|duration|1.0.0 Key | Default | Meaning | Type | Since --- | --- | --- | --- | --- -kyuubi.engine.connection.url.use.hostname|true|(deprecated) When true, engine register with hostname to zookeeper. When spark run on k8s with cluster mode, set to false to ensure that server can connect to engine|boolean|1.3.0 -kyuubi.engine.deregister.exception.classes||A comma separated list of exception classes. If there is any exception thrown, whose class matches the specified classes, the engine would deregister itself.|seq|1.2.0 -kyuubi.engine.deregister.exception.messages||A comma separated list of exception messages. If there is any exception thrown, whose message or stacktrace matches the specified message list, the engine would deregister itself.|seq|1.2.0 +kyuubi.engine.connection.url.use.hostname|true|(deprecated) When true, the engine registers with hostname to zookeeper. When Spark runs on K8s with cluster mode, set to false to ensure that server can connect to engine|boolean|1.3.0 +kyuubi.engine.deregister.exception.classes||A comma-separated list of exception classes. If there is any exception thrown, whose class matches the specified classes, the engine would deregister itself.|seq|1.2.0 +kyuubi.engine.deregister.exception.messages||A comma-separated list of exception messages. If there is any exception thrown, whose message or stacktrace matches the specified message list, the engine would deregister itself.|seq|1.2.0 kyuubi.engine.deregister.exception.ttl|PT30M|Time to live(TTL) for exceptions pattern specified in kyuubi.engine.deregister.exception.classes and kyuubi.engine.deregister.exception.messages to deregister engines. Once the total error count hits the kyuubi.engine.deregister.job.max.failures within the TTL, an engine will deregister itself and wait for self-terminated. Otherwise, we suppose that the engine has recovered from temporary failures.|duration|1.2.0 kyuubi.engine.deregister.job.max.failures|4|Number of failures of job before deregistering the engine.|int|1.2.0 -kyuubi.engine.event.json.log.path|file:///tmp/kyuubi/events|The location of all the engine events go for the builtin JSON logger.
  • Local Path: start with 'file://'
  • HDFS Path: start with 'hdfs://'
|string|1.3.0 -kyuubi.engine.event.loggers|SPARK|A comma separated list of engine history loggers, where engine/session/operation etc events go.
  • SPARK: the events will be written to the spark listener bus.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: User-defined event handlers.
Note that: Kyuubi supports custom event handlers with the Java SPI. To register a custom event handler, user need to implement a class which is a child of org.apache.kyuubi.events.handler.CustomEventHandlerProvider which has zero-arg constructor.|seq|1.3.0 -kyuubi.engine.flink.extra.classpath|<undefined>|The extra classpath for the flink sql engine, for configuring location of hadoop client jars, etc|string|1.6.0 -kyuubi.engine.flink.java.options|<undefined>|The extra java options for the flink sql engine|string|1.6.0 -kyuubi.engine.flink.memory|1g|The heap memory for the flink sql engine|string|1.6.0 -kyuubi.engine.hive.event.loggers|JSON|A comma separated list of engine history loggers, where engine/session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
|seq|1.7.0 -kyuubi.engine.hive.extra.classpath|<undefined>|The extra classpath for the hive query engine, for configuring location of hadoop client jars, etc|string|1.6.0 -kyuubi.engine.hive.java.options|<undefined>|The extra java options for the hive query engine|string|1.6.0 -kyuubi.engine.hive.memory|1g|The heap memory for the hive query engine|string|1.6.0 +kyuubi.engine.event.json.log.path|file:///tmp/kyuubi/events|The location where all the engine events go for the built-in JSON logger.
  • Local Path: start with 'file://'
  • HDFS Path: start with 'hdfs://'
|string|1.3.0 +kyuubi.engine.event.loggers|SPARK|A comma-separated list of engine history loggers, where engine/session/operation etc events go.
  • SPARK: the events will be written to the Spark listener bus.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: User-defined event handlers.
Note that: Kyuubi supports custom event handlers with the Java SPI. To register a custom event handler, the user needs to implement a subclass of `org.apache.kyuubi.events.handler.CustomEventHandlerProvider` which has a zero-arg constructor.|seq|1.3.0 +kyuubi.engine.flink.extra.classpath|<undefined>|The extra classpath for the Flink SQL engine, for configuring the location of hadoop client jars, etc|string|1.6.0 +kyuubi.engine.flink.java.options|<undefined>|The extra Java options for the Flink SQL engine|string|1.6.0 +kyuubi.engine.flink.memory|1g|The heap memory for the Flink SQL engine|string|1.6.0 +kyuubi.engine.hive.event.loggers|JSON|A comma-separated list of engine history loggers, where engine/session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
|seq|1.7.0 +kyuubi.engine.hive.extra.classpath|<undefined>|The extra classpath for the Hive query engine, for configuring location of the hadoop client jars and etc.|string|1.6.0 +kyuubi.engine.hive.java.options|<undefined>|The extra Java options for the Hive query engine|string|1.6.0 +kyuubi.engine.hive.memory|1g|The heap memory for the Hive query engine|string|1.6.0 kyuubi.engine.initialize.sql|SHOW DATABASES|SemiColon-separated list of SQL statements to be initialized in the newly created engine before queries. i.e. use `SHOW DATABASES` to eagerly active HiveClient. This configuration can not be used in JDBC url due to the limitation of Beeline/JDBC driver.|seq|1.2.0 kyuubi.engine.jdbc.connection.password|<undefined>|The password is used for connecting to server|string|1.6.0 kyuubi.engine.jdbc.connection.properties||The additional properties are used for connecting to server|seq|1.6.0 -kyuubi.engine.jdbc.connection.provider|<undefined>|The connection provider is used for getting a connection from server|string|1.6.0 +kyuubi.engine.jdbc.connection.provider|<undefined>|The connection provider is used for getting a connection from the server|string|1.6.0 kyuubi.engine.jdbc.connection.url|<undefined>|The server url that engine will connect to|string|1.6.0 kyuubi.engine.jdbc.connection.user|<undefined>|The user is used for connecting to server|string|1.6.0 -kyuubi.engine.jdbc.driver.class|<undefined>|The driver class for jdbc engine connection|string|1.6.0 -kyuubi.engine.jdbc.extra.classpath|<undefined>|The extra classpath for the jdbc query engine, for configuring location of jdbc driver, etc|string|1.6.0 -kyuubi.engine.jdbc.java.options|<undefined>|The extra java options for the jdbc query engine|string|1.6.0 -kyuubi.engine.jdbc.memory|1g|The heap memory for the jdbc query engine|string|1.6.0 -kyuubi.engine.jdbc.type|<undefined>|The short name of jdbc type|string|1.6.0 +kyuubi.engine.jdbc.driver.class|<undefined>|The driver class for JDBC engine connection|string|1.6.0 +kyuubi.engine.jdbc.extra.classpath|<undefined>|The extra classpath for the JDBC query engine, for configuring the location of the JDBC driver and etc.|string|1.6.0 +kyuubi.engine.jdbc.java.options|<undefined>|The extra Java options for the JDBC query engine|string|1.6.0 +kyuubi.engine.jdbc.memory|1g|The heap memory for the JDBC query engine|string|1.6.0 +kyuubi.engine.jdbc.type|<undefined>|The short name of JDBC type|string|1.6.0 kyuubi.engine.operation.convert.catalog.database.enabled|true|When set to true, The engine converts the JDBC methods of set/get Catalog and set/get Schema to the implementation of different engines|boolean|1.6.0 kyuubi.engine.operation.log.dir.root|engine_operation_logs|Root directory for query operation log at engine-side.|string|1.4.0 -kyuubi.engine.pool.name|engine-pool|The name of engine pool.|string|1.5.0 +kyuubi.engine.pool.name|engine-pool|The name of the engine pool.|string|1.5.0 kyuubi.engine.pool.selectPolicy|RANDOM|The select policy of an engine from the corresponding engine pool engine for a session.
  • RANDOM - Randomly use the engine in the pool
  • POLLING - Polling use the engine in the pool
|string|1.7.0 -kyuubi.engine.pool.size|-1|The size of engine pool. Note that, if the size is less than 1, the engine pool will not be enabled; otherwise, the size of the engine pool will be min(this, kyuubi.engine.pool.size.threshold).|int|1.4.0 -kyuubi.engine.pool.size.threshold|9|This parameter is introduced as a server-side parameter, and controls the upper limit of the engine pool.|int|1.4.0 +kyuubi.engine.pool.size|-1|The size of the engine pool. Note that, if the size is less than 1, the engine pool will not be enabled; otherwise, the size of the engine pool will be min(this, kyuubi.engine.pool.size.threshold).|int|1.4.0 +kyuubi.engine.pool.size.threshold|9|This parameter is introduced as a server-side parameter controlling the upper limit of the engine pool.|int|1.4.0 kyuubi.engine.session.initialize.sql||SemiColon-separated list of SQL statements to be initialized in the newly created engine session before queries. This configuration can not be used in JDBC url due to the limitation of Beeline/JDBC driver.|seq|1.3.0 -kyuubi.engine.share.level|USER|Engines will be shared in different levels, available configs are:
  • CONNECTION: engine will not be shared but only used by the current client connection
  • USER: engine will be shared by all sessions created by a unique username, see also kyuubi.engine.share.level.subdomain
  • GROUP: engine will be shared by all sessions created by all users belong to the same primary group name. The engine will be launched by the group name as the effective username, so here the group name is kind of special user who is able to visit the compute resources/data of a team. It follows the [Hadoop GroupsMapping](https://reurl.cc/xE61Y5) to map user to a primary group. If the primary group is not found, it fallback to the USER level.
  • SERVER: the App will be shared by Kyuubi servers
|string|1.2.0 +kyuubi.engine.share.level|USER|Engines will be shared in different levels, available configs are:
  • CONNECTION: engine will not be shared but only used by the current client connection
  • USER: engine will be shared by all sessions created by a unique username, see also kyuubi.engine.share.level.subdomain
  • GROUP: the engine will be shared by all sessions created by all users belong to the same primary group name. The engine will be launched by the group name as the effective username, so here the group name is in value of special user who is able to visit the computing resources/data of the team. It follows the [Hadoop GroupsMapping](https://reurl.cc/xE61Y5) to map user to a primary group. If the primary group is not found, it fallback to the USER level.
  • SERVER: the App will be shared by Kyuubi servers
|string|1.2.0 kyuubi.engine.share.level.sub.domain|<undefined>|(deprecated) - Using kyuubi.engine.share.level.subdomain instead|string|1.2.0 -kyuubi.engine.share.level.subdomain|<undefined>|Allow end-users to create a subdomain for the share level of an engine. A subdomain is a case-insensitive string values that must be a valid zookeeper sub path. For example, for `USER` share level, an end-user can share a certain engine within a subdomain, not for all of its clients. End-users are free to create multiple engines in the `USER` share level. When disable engine pool, use 'default' if absent.|string|1.4.0 +kyuubi.engine.share.level.subdomain|<undefined>|Allow end-users to create a subdomain for the share level of an engine. A subdomain is a case-insensitive string values that must be a valid zookeeper subpath. For example, for the `USER` share level, an end-user can share a certain engine within a subdomain, not for all of its clients. End-users are free to create multiple engines in the `USER` share level. When disable engine pool, use 'default' if absent.|string|1.4.0 kyuubi.engine.single.spark.session|false|When set to true, this engine is running in a single session mode. All the JDBC/ODBC connections share the temporary views, function registries, SQL configuration and the current database.|boolean|1.3.0 -kyuubi.engine.spark.event.loggers|SPARK|A comma separated list of engine loggers, where engine/session/operation etc events go.
  • SPARK: the events will be written to the spark listener bus.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
|seq|1.7.0 -kyuubi.engine.spark.python.env.archive|<undefined>|Portable python env archive used for Spark engine python language mode.|string|1.7.0 -kyuubi.engine.spark.python.env.archive.exec.path|bin/python|The python exec path under the python env archive.|string|1.7.0 -kyuubi.engine.spark.python.home.archive|<undefined>|Spark archive containing $SPARK_HOME/python directory, which is used to init session python worker for python language mode.|string|1.7.0 -kyuubi.engine.trino.event.loggers|JSON|A comma separated list of engine history loggers, where engine/session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
|seq|1.7.0 -kyuubi.engine.trino.extra.classpath|<undefined>|The extra classpath for the trino query engine, for configuring other libs which may need by the trino engine |string|1.6.0 -kyuubi.engine.trino.java.options|<undefined>|The extra java options for the trino query engine|string|1.6.0 -kyuubi.engine.trino.memory|1g|The heap memory for the trino query engine|string|1.6.0 -kyuubi.engine.type|SPARK_SQL|Specify the detailed engine that supported by the Kyuubi. The engine type bindings to SESSION scope. This configuration is experimental. Currently, available configs are:
  • SPARK_SQL: specify this engine type will launch a Spark engine which can provide all the capacity of the Apache Spark. Note, it's a default engine type.
  • FLINK_SQL: specify this engine type will launch a Flink engine which can provide all the capacity of the Apache Flink.
  • TRINO: specify this engine type will launch a Trino engine which can provide all the capacity of the Trino.
  • HIVE_SQL: specify this engine type will launch a Hive engine which can provide all the capacity of the Hive Server2.
  • JDBC: specify this engine type will launch a JDBC engine which can provide a mysql protocol connector, for now we only support Doris dialect.
|string|1.4.0 +kyuubi.engine.spark.event.loggers|SPARK|A comma-separated list of engine loggers, where engine/session/operation etc events go.
  • SPARK: the events will be written to the Spark listener bus.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
|seq|1.7.0 +kyuubi.engine.spark.python.env.archive|<undefined>|Portable Python env archive used for Spark engine Python language mode.|string|1.7.0 +kyuubi.engine.spark.python.env.archive.exec.path|bin/python|The Python exec path under the Python env archive.|string|1.7.0 +kyuubi.engine.spark.python.home.archive|<undefined>|Spark archive containing $SPARK_HOME/python directory, which is used to init session Python worker for Python language mode.|string|1.7.0 +kyuubi.engine.trino.event.loggers|JSON|A comma-separated list of engine history loggers, where engine/session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
|seq|1.7.0 +kyuubi.engine.trino.extra.classpath|<undefined>|The extra classpath for the Trino query engine, for configuring other libs which may need by the Trino engine |string|1.6.0 +kyuubi.engine.trino.java.options|<undefined>|The extra Java options for the Trino query engine|string|1.6.0 +kyuubi.engine.trino.memory|1g|The heap memory for the Trino query engine|string|1.6.0 +kyuubi.engine.type|SPARK_SQL|Specify the detailed engine supported by Kyuubi. The engine type bindings to SESSION scope. This configuration is experimental. Currently, available configs are:
  • SPARK_SQL: specify this engine type will launch a Spark engine which can provide all the capacity of the Apache Spark. Note, it's a default engine type.
  • FLINK_SQL: specify this engine type will launch a Flink engine which can provide all the capacity of the Apache Flink.
  • TRINO: specify this engine type will launch a Trino engine which can provide all the capacity of the Trino.
  • HIVE_SQL: specify this engine type will launch a Hive engine which can provide all the capacity of the Hive Server2.
  • JDBC: specify this engine type will launch a JDBC engine which can provide a MySQL protocol connector, for now we only support Doris dialect.
|string|1.4.0 kyuubi.engine.ui.retainedSessions|200|The number of SQL client sessions kept in the Kyuubi Query Engine web UI.|int|1.4.0 kyuubi.engine.ui.retainedStatements|200|The number of statements kept in the Kyuubi Query Engine web UI.|int|1.4.0 kyuubi.engine.ui.stop.enabled|true|When true, allows Kyuubi engine to be killed from the Spark Web UI.|boolean|1.3.0 -kyuubi.engine.user.isolated.spark.session|true|When set to false, if the engine is running in a group or server share level, all the JDBC/ODBC connections will be isolated against the user. Including: the temporary views, function registries, SQL configuration and the current database. Note that, it does not affect if the share level is connection or user.|boolean|1.6.0 -kyuubi.engine.user.isolated.spark.session.idle.interval|PT1M|The interval to check if the user isolated spark session is timeout.|duration|1.6.0 -kyuubi.engine.user.isolated.spark.session.idle.timeout|PT6H|If kyuubi.engine.user.isolated.spark.session is false, we will release the spark session if its corresponding user is inactive after this configured timeout.|duration|1.6.0 +kyuubi.engine.user.isolated.spark.session|true|When set to false, if the engine is running in a group or server share level, all the JDBC/ODBC connections will be isolated against the user. Including the temporary views, function registries, SQL configuration, and the current database. Note that, it does not affect if the share level is connection or user.|boolean|1.6.0 +kyuubi.engine.user.isolated.spark.session.idle.interval|PT1M|The interval to check if the user-isolated Spark session is timeout.|duration|1.6.0 +kyuubi.engine.user.isolated.spark.session.idle.timeout|PT6H|If kyuubi.engine.user.isolated.spark.session is false, we will release the Spark session if its corresponding user is inactive after this configured timeout.|duration|1.6.0 ### Event @@ -287,38 +287,38 @@ Key | Default | Meaning | Type | Since --- | --- | --- | --- | --- kyuubi.frontend.backoff.slot.length|PT0.1S|(deprecated) Time to back off during login to the thrift frontend service.|duration|1.0.0 kyuubi.frontend.bind.host|<undefined>|Hostname or IP of the machine on which to run the frontend services.|string|1.0.0 -kyuubi.frontend.bind.port|10009|(deprecated) Port of the machine on which to run the thrift frontend service via binary protocol.|int|1.0.0 -kyuubi.frontend.connection.url.use.hostname|true|When true, frontend services prefer hostname, otherwise, ip address. Note that, the default value is set to `false` when engine running on Kubernetes to prevent potential network issue.|boolean|1.5.0 +kyuubi.frontend.bind.port|10009|(deprecated) Port of the machine on which to run the thrift frontend service via the binary protocol.|int|1.0.0 +kyuubi.frontend.connection.url.use.hostname|true|When true, frontend services prefer hostname, otherwise, ip address. Note that, the default value is set to `false` when engine running on Kubernetes to prevent potential network issues.|boolean|1.5.0 kyuubi.frontend.login.timeout|PT20S|(deprecated) Timeout for Thrift clients during login to the thrift frontend service.|duration|1.0.0 kyuubi.frontend.max.message.size|104857600|(deprecated) Maximum message size in bytes a Kyuubi server will accept.|int|1.0.0 -kyuubi.frontend.max.worker.threads|999|(deprecated) Maximum number of threads in the of frontend worker thread pool for the thrift frontend service|int|1.0.0 -kyuubi.frontend.min.worker.threads|9|(deprecated) Minimum number of threads in the of frontend worker thread pool for the thrift frontend service|int|1.0.0 +kyuubi.frontend.max.worker.threads|999|(deprecated) Maximum number of threads in the frontend worker thread pool for the thrift frontend service|int|1.0.0 +kyuubi.frontend.min.worker.threads|9|(deprecated) Minimum number of threads in the frontend worker thread pool for the thrift frontend service|int|1.0.0 kyuubi.frontend.mysql.bind.host|<undefined>|Hostname or IP of the machine on which to run the MySQL frontend service.|string|1.4.0 kyuubi.frontend.mysql.bind.port|3309|Port of the machine on which to run the MySQL frontend service.|int|1.4.0 kyuubi.frontend.mysql.max.worker.threads|999|Maximum number of threads in the command execution thread pool for the MySQL frontend service|int|1.4.0 kyuubi.frontend.mysql.min.worker.threads|9|Minimum number of threads in the command execution thread pool for the MySQL frontend service|int|1.4.0 kyuubi.frontend.mysql.netty.worker.threads|<undefined>|Number of thread in the netty worker event loop of MySQL frontend service. Use min(cpu_cores, 8) in default.|int|1.4.0 kyuubi.frontend.mysql.worker.keepalive.time|PT1M|Time(ms) that an idle async thread of the command execution thread pool will wait for a new task to arrive before terminating in MySQL frontend service|duration|1.4.0 -kyuubi.frontend.protocols|THRIFT_BINARY|A comma separated list for all frontend protocols
  • THRIFT_BINARY - HiveServer2 compatible thrift binary protocol.
  • THRIFT_HTTP - HiveServer2 compatible thrift http protocol.
  • REST - Kyuubi defined REST API(experimental).
  • MYSQL - MySQL compatible text protocol(experimental).
  • TRINO - Trino compatible http protocol(experimental).
|seq|1.4.0 -kyuubi.frontend.proxy.http.client.ip.header|X-Real-IP|The http header to record the real client ip address. If your server is behind a load balancer or other proxy, the server will see this load balancer or proxy IP address as the client IP address, to get around this common issue, most load balancers or proxies offer the ability to record the real remote IP address in an HTTP header that will be added to the request for other devices to use. Note that, because the header value can be specified to any ip address, so it will not be used for authentication.|string|1.6.0 +kyuubi.frontend.protocols|THRIFT_BINARY|A comma-separated list for all frontend protocols
  • THRIFT_BINARY - HiveServer2 compatible thrift binary protocol.
  • THRIFT_HTTP - HiveServer2 compatible thrift http protocol.
  • REST - Kyuubi defined REST API(experimental).
  • MYSQL - MySQL compatible text protocol(experimental).
  • TRINO - Trino compatible http protocol(experimental).
|seq|1.4.0 +kyuubi.frontend.proxy.http.client.ip.header|X-Real-IP|The HTTP header to record the real client IP address. If your server is behind a load balancer or other proxy, the server will see this load balancer or proxy IP address as the client IP address, to get around this common issue, most load balancers or proxies offer the ability to record the real remote IP address in an HTTP header that will be added to the request for other devices to use. Note that, because the header value can be specified to any IP address, so it will not be used for authentication.|string|1.6.0 kyuubi.frontend.rest.bind.host|<undefined>|Hostname or IP of the machine on which to run the REST frontend service.|string|1.4.0 kyuubi.frontend.rest.bind.port|10099|Port of the machine on which to run the REST frontend service.|int|1.4.0 -kyuubi.frontend.rest.max.worker.threads|999|Maximum number of threads in the of frontend worker thread pool for the rest frontend service|int|1.6.2 +kyuubi.frontend.rest.max.worker.threads|999|Maximum number of threads in the frontend worker thread pool for the rest frontend service|int|1.6.2 kyuubi.frontend.ssl.keystore.algorithm|<undefined>|SSL certificate keystore algorithm.|string|1.7.0 kyuubi.frontend.ssl.keystore.password|<undefined>|SSL certificate keystore password.|string|1.7.0 kyuubi.frontend.ssl.keystore.path|<undefined>|SSL certificate keystore location.|string|1.7.0 kyuubi.frontend.ssl.keystore.type|<undefined>|SSL certificate keystore type.|string|1.7.0 kyuubi.frontend.thrift.backoff.slot.length|PT0.1S|Time to back off during login to the thrift frontend service.|duration|1.4.0 -kyuubi.frontend.thrift.binary.bind.host|<undefined>|Hostname or IP of the machine on which to run the thrift frontend service via binary protocol.|string|1.4.0 -kyuubi.frontend.thrift.binary.bind.port|10009|Port of the machine on which to run the thrift frontend service via binary protocol.|int|1.4.0 +kyuubi.frontend.thrift.binary.bind.host|<undefined>|Hostname or IP of the machine on which to run the thrift frontend service via the binary protocol.|string|1.4.0 +kyuubi.frontend.thrift.binary.bind.port|10009|Port of the machine on which to run the thrift frontend service via the binary protocol.|int|1.4.0 kyuubi.frontend.thrift.binary.ssl.disallowed.protocols|SSLv2,SSLv3|SSL versions to disallow for Kyuubi thrift binary frontend.|seq|1.7.0 kyuubi.frontend.thrift.binary.ssl.enabled|false|Set this to true for using SSL encryption in thrift binary frontend server.|boolean|1.7.0 -kyuubi.frontend.thrift.binary.ssl.include.ciphersuites||A comma separated list of include SSL cipher suite names for thrift binary frontend.|seq|1.7.0 +kyuubi.frontend.thrift.binary.ssl.include.ciphersuites||A comma-separated list of include SSL cipher suite names for thrift binary frontend.|seq|1.7.0 kyuubi.frontend.thrift.http.allow.user.substitution|true|Allow alternate user to be specified as part of open connection request when using HTTP transport mode.|boolean|1.6.0 kyuubi.frontend.thrift.http.bind.host|<undefined>|Hostname or IP of the machine on which to run the thrift frontend service via http protocol.|string|1.6.0 kyuubi.frontend.thrift.http.bind.port|10010|Port of the machine on which to run the thrift frontend service via http protocol.|int|1.6.0 kyuubi.frontend.thrift.http.compression.enabled|true|Enable thrift http compression via Jetty compression support|boolean|1.6.0 -kyuubi.frontend.thrift.http.cookie.auth.enabled|true|When true, Kyuubi in HTTP transport mode, will use cookie based authentication mechanism|boolean|1.6.0 +kyuubi.frontend.thrift.http.cookie.auth.enabled|true|When true, Kyuubi in HTTP transport mode, will use cookie-based authentication mechanism|boolean|1.6.0 kyuubi.frontend.thrift.http.cookie.domain|<undefined>|Domain for the Kyuubi generated cookies|string|1.6.0 kyuubi.frontend.thrift.http.cookie.is.httponly|true|HttpOnly attribute of the Kyuubi generated cookie.|boolean|1.6.0 kyuubi.frontend.thrift.http.cookie.max.age|86400|Maximum age in seconds for server side cookie used by Kyuubi in HTTP mode.|int|1.6.0 @@ -327,20 +327,20 @@ kyuubi.frontend.thrift.http.max.idle.time|PT30M|Maximum idle time for a connecti kyuubi.frontend.thrift.http.path|cliservice|Path component of URL endpoint when in HTTP mode.|string|1.6.0 kyuubi.frontend.thrift.http.request.header.size|6144|Request header size in bytes, when using HTTP transport mode. Jetty defaults used.|int|1.6.0 kyuubi.frontend.thrift.http.response.header.size|6144|Response header size in bytes, when using HTTP transport mode. Jetty defaults used.|int|1.6.0 -kyuubi.frontend.thrift.http.ssl.exclude.ciphersuites||A comma separated list of exclude SSL cipher suite names for thrift http frontend.|seq|1.7.0 +kyuubi.frontend.thrift.http.ssl.exclude.ciphersuites||A comma-separated list of exclude SSL cipher suite names for thrift http frontend.|seq|1.7.0 kyuubi.frontend.thrift.http.ssl.keystore.password|<undefined>|SSL certificate keystore password.|string|1.6.0 kyuubi.frontend.thrift.http.ssl.keystore.path|<undefined>|SSL certificate keystore location.|string|1.6.0 kyuubi.frontend.thrift.http.ssl.protocol.blacklist|SSLv2,SSLv3|SSL Versions to disable when using HTTP transport mode.|seq|1.6.0 kyuubi.frontend.thrift.http.use.SSL|false|Set this to true for using SSL encryption in http mode.|boolean|1.6.0 -kyuubi.frontend.thrift.http.xsrf.filter.enabled|false|If enabled, Kyuubi will block any requests made to it over http if an X-XSRF-HEADER header is not present|boolean|1.6.0 +kyuubi.frontend.thrift.http.xsrf.filter.enabled|false|If enabled, Kyuubi will block any requests made to it over HTTP if an X-XSRF-HEADER header is not present|boolean|1.6.0 kyuubi.frontend.thrift.login.timeout|PT20S|Timeout for Thrift clients during login to the thrift frontend service.|duration|1.4.0 kyuubi.frontend.thrift.max.message.size|104857600|Maximum message size in bytes a Kyuubi server will accept.|int|1.4.0 -kyuubi.frontend.thrift.max.worker.threads|999|Maximum number of threads in the of frontend worker thread pool for the thrift frontend service|int|1.4.0 -kyuubi.frontend.thrift.min.worker.threads|9|Minimum number of threads in the of frontend worker thread pool for the thrift frontend service|int|1.4.0 +kyuubi.frontend.thrift.max.worker.threads|999|Maximum number of threads in the frontend worker thread pool for the thrift frontend service|int|1.4.0 +kyuubi.frontend.thrift.min.worker.threads|9|Minimum number of threads in the frontend worker thread pool for the thrift frontend service|int|1.4.0 kyuubi.frontend.thrift.worker.keepalive.time|PT1M|Keep-alive time (in milliseconds) for an idle worker thread|duration|1.4.0 kyuubi.frontend.trino.bind.host|<undefined>|Hostname or IP of the machine on which to run the TRINO frontend service.|string|1.7.0 kyuubi.frontend.trino.bind.port|10999|Port of the machine on which to run the TRINO frontend service.|int|1.7.0 -kyuubi.frontend.trino.max.worker.threads|999|Maximum number of threads in the of frontend worker thread pool for the trino frontend service|int|1.7.0 +kyuubi.frontend.trino.max.worker.threads|999|Maximum number of threads in the frontend worker thread pool for the Trino frontend service|int|1.7.0 kyuubi.frontend.worker.keepalive.time|PT1M|(deprecated) Keep-alive time (in milliseconds) for an idle worker thread|duration|1.0.0 @@ -350,27 +350,27 @@ Key | Default | Meaning | Type | Since --- | --- | --- | --- | --- kyuubi.ha.addresses||The connection string for the discovery ensemble|string|1.6.0 kyuubi.ha.client.class|org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient|Class name for service discovery client.
  • Zookeeper: org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient
  • Etcd: org.apache.kyuubi.ha.client.etcd.EtcdDiscoveryClient
|string|1.6.0 -kyuubi.ha.etcd.lease.timeout|PT10S|Timeout for etcd keep alive lease. The kyuubi server will known unexpected loss of engine after up to this seconds.|duration|1.6.0 +kyuubi.ha.etcd.lease.timeout|PT10S|Timeout for etcd keep alive lease. The kyuubi server will know the unexpected loss of engine after up to this seconds.|duration|1.6.0 kyuubi.ha.etcd.ssl.ca.path|<undefined>|Where the etcd CA certificate file is stored.|string|1.6.0 kyuubi.ha.etcd.ssl.client.certificate.path|<undefined>|Where the etcd SSL certificate file is stored.|string|1.6.0 kyuubi.ha.etcd.ssl.client.key.path|<undefined>|Where the etcd SSL key file is stored.|string|1.6.0 -kyuubi.ha.etcd.ssl.enabled|false|When set to true, will build a ssl secured etcd client.|boolean|1.6.0 +kyuubi.ha.etcd.ssl.enabled|false|When set to true, will build an SSL secured etcd client.|boolean|1.6.0 kyuubi.ha.namespace|kyuubi|The root directory for the service to deploy its instance uri|string|1.6.0 -kyuubi.ha.zookeeper.acl.enabled|false|Set to true if the zookeeper ensemble is kerberized|boolean|1.0.0 -kyuubi.ha.zookeeper.auth.digest|<undefined>|The digest auth string is used for zookeeper authentication, like: username:password.|string|1.3.2 -kyuubi.ha.zookeeper.auth.keytab|<undefined>|Location of Kyuubi server's keytab is used for zookeeper authentication.|string|1.3.2 -kyuubi.ha.zookeeper.auth.principal|<undefined>|Name of the Kerberos principal is used for zookeeper authentication.|string|1.3.2 -kyuubi.ha.zookeeper.auth.type|NONE|The type of zookeeper authentication, all candidates are
  • NONE
  • KERBEROS
  • DIGEST
|string|1.3.2 -kyuubi.ha.zookeeper.connection.base.retry.wait|1000|Initial amount of time to wait between retries to the zookeeper ensemble|int|1.0.0 -kyuubi.ha.zookeeper.connection.max.retries|3|Max retry times for connecting to the zookeeper ensemble|int|1.0.0 +kyuubi.ha.zookeeper.acl.enabled|false|Set to true if the ZooKeeper ensemble is kerberized|boolean|1.0.0 +kyuubi.ha.zookeeper.auth.digest|<undefined>|The digest auth string is used for ZooKeeper authentication, like: username:password.|string|1.3.2 +kyuubi.ha.zookeeper.auth.keytab|<undefined>|Location of the Kyuubi server's keytab is used for ZooKeeper authentication.|string|1.3.2 +kyuubi.ha.zookeeper.auth.principal|<undefined>|Name of the Kerberos principal is used for ZooKeeper authentication.|string|1.3.2 +kyuubi.ha.zookeeper.auth.type|NONE|The type of ZooKeeper authentication, all candidates are
  • NONE
  • KERBEROS
  • DIGEST
|string|1.3.2 +kyuubi.ha.zookeeper.connection.base.retry.wait|1000|Initial amount of time to wait between retries to the ZooKeeper ensemble|int|1.0.0 +kyuubi.ha.zookeeper.connection.max.retries|3|Max retry times for connecting to the ZooKeeper ensemble|int|1.0.0 kyuubi.ha.zookeeper.connection.max.retry.wait|30000|Max amount of time to wait between retries for BOUNDED_EXPONENTIAL_BACKOFF policy can reach, or max time until elapsed for UNTIL_ELAPSED policy to connect the zookeeper ensemble|int|1.0.0 -kyuubi.ha.zookeeper.connection.retry.policy|EXPONENTIAL_BACKOFF|The retry policy for connecting to the zookeeper ensemble, all candidates are:
  • ONE_TIME
  • N_TIME
  • EXPONENTIAL_BACKOFF
  • BOUNDED_EXPONENTIAL_BACKOFF
  • UNTIL_ELAPSED
|string|1.0.0 -kyuubi.ha.zookeeper.connection.timeout|15000|The timeout(ms) of creating the connection to the zookeeper ensemble|int|1.0.0 -kyuubi.ha.zookeeper.engine.auth.type|NONE|The type of zookeeper authentication for engine, all candidates are
  • NONE
  • KERBEROS
  • DIGEST
|string|1.3.2 +kyuubi.ha.zookeeper.connection.retry.policy|EXPONENTIAL_BACKOFF|The retry policy for connecting to the ZooKeeper ensemble, all candidates are:
  • ONE_TIME
  • N_TIME
  • EXPONENTIAL_BACKOFF
  • BOUNDED_EXPONENTIAL_BACKOFF
  • UNTIL_ELAPSED
|string|1.0.0 +kyuubi.ha.zookeeper.connection.timeout|15000|The timeout(ms) of creating the connection to the ZooKeeper ensemble|int|1.0.0 +kyuubi.ha.zookeeper.engine.auth.type|NONE|The type of ZooKeeper authentication for the engine, all candidates are
  • NONE
  • KERBEROS
  • DIGEST
|string|1.3.2 kyuubi.ha.zookeeper.namespace|kyuubi|(deprecated) The root directory for the service to deploy its instance uri|string|1.0.0 -kyuubi.ha.zookeeper.node.creation.timeout|PT2M|Timeout for creating zookeeper node|duration|1.2.0 -kyuubi.ha.zookeeper.publish.configs|false|When set to true, publish Kerberos configs to Zookeeper.Note that the Hive driver needs to be greater than 1.3 or 2.0 or apply HIVE-11581 patch.|boolean|1.4.0 -kyuubi.ha.zookeeper.quorum||(deprecated) The connection string for the zookeeper ensemble|string|1.0.0 +kyuubi.ha.zookeeper.node.creation.timeout|PT2M|Timeout for creating ZooKeeper node|duration|1.2.0 +kyuubi.ha.zookeeper.publish.configs|false|When set to true, publish Kerberos configs to Zookeeper. Note that the Hive driver needs to be greater than 1.3 or 2.0 or apply HIVE-11581 patch.|boolean|1.4.0 +kyuubi.ha.zookeeper.quorum||(deprecated) The connection string for the ZooKeeper ensemble|string|1.0.0 kyuubi.ha.zookeeper.session.timeout|60000|The timeout(ms) of a connected session to be idled|int|1.0.0 @@ -378,7 +378,7 @@ kyuubi.ha.zookeeper.session.timeout|60000|The timeout(ms) of a connected session Key | Default | Meaning | Type | Since --- | --- | --- | --- | --- -kyuubi.kinit.interval|PT1H|How often will Kyuubi server run `kinit -kt [keytab] [principal]` to renew the local Kerberos credentials cache|duration|1.0.0 +kyuubi.kinit.interval|PT1H|How often will the Kyuubi server run `kinit -kt [keytab] [principal]` to renew the local Kerberos credentials cache|duration|1.0.0 kyuubi.kinit.keytab|<undefined>|Location of Kyuubi server's keytab.|string|1.0.0 kyuubi.kinit.max.attempts|10|How many times will `kinit` process retry|int|1.0.0 kyuubi.kinit.principal|<undefined>|Name of the Kerberos principal.|string|1.0.0 @@ -391,9 +391,9 @@ Key | Default | Meaning | Type | Since kyuubi.kubernetes.authenticate.caCertFile|<undefined>|Path to the CA cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme)|string|1.7.0 kyuubi.kubernetes.authenticate.clientCertFile|<undefined>|Path to the client cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme)|string|1.7.0 kyuubi.kubernetes.authenticate.clientKeyFile|<undefined>|Path to the client key file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme)|string|1.7.0 -kyuubi.kubernetes.authenticate.oauthToken|<undefined>|The OAuth token to use when authenticating against the Kubernetes API server. Note that unlike the other authentication options, this must be the exact string value of the token to use for the authentication.|string|1.7.0 +kyuubi.kubernetes.authenticate.oauthToken|<undefined>|The OAuth token to use when authenticating against the Kubernetes API server. Note that unlike, the other authentication options, this must be the exact string value of the token to use for the authentication.|string|1.7.0 kyuubi.kubernetes.authenticate.oauthTokenFile|<undefined>|Path to the file containing the OAuth token to use when authenticating against the Kubernetes API server. Specify this as a path as opposed to a URI (i.e. do not provide a scheme)|string|1.7.0 -kyuubi.kubernetes.context|<undefined>|The desired context from your kubernetes config file used to configure the K8S client for interacting with the cluster.|string|1.6.0 +kyuubi.kubernetes.context|<undefined>|The desired context from your kubernetes config file used to configure the K8s client for interacting with the cluster.|string|1.6.0 kyuubi.kubernetes.master.address|<undefined>|The internal Kubernetes master (API server) address to be used for kyuubi.|string|1.7.0 kyuubi.kubernetes.namespace|default|The namespace that will be used for running the kyuubi pods and find engines.|string|1.7.0 kyuubi.kubernetes.trust.certificates|false|If set to true then client can submit to kubernetes cluster only with token|boolean|1.7.0 @@ -403,20 +403,20 @@ kyuubi.kubernetes.trust.certificates|false|If set to true then client can submit Key | Default | Meaning | Type | Since --- | --- | --- | --- | --- -kyuubi.metadata.cleaner.enabled|true|Whether to clean the metadata periodically. If it is enabled, Kyuubi will clean the metadata that is in terminate state with max age limitation.|boolean|1.6.0 +kyuubi.metadata.cleaner.enabled|true|Whether to clean the metadata periodically. If it is enabled, Kyuubi will clean the metadata that is in the terminate state with max age limitation.|boolean|1.6.0 kyuubi.metadata.cleaner.interval|PT30M|The interval to check and clean expired metadata.|duration|1.6.0 -kyuubi.metadata.max.age|PT72H|The maximum age of metadata, the metadata that exceeds the age will be cleaned.|duration|1.6.0 -kyuubi.metadata.recovery.threads|10|The number of threads for recovery from metadata store when Kyuubi server restarting.|int|1.6.0 +kyuubi.metadata.max.age|PT72H|The maximum age of metadata, the metadata exceeding the age will be cleaned.|duration|1.6.0 +kyuubi.metadata.recovery.threads|10|The number of threads for recovery from the metadata store when the Kyuubi server restarts.|int|1.6.0 kyuubi.metadata.request.retry.interval|PT5S|The interval to check and trigger the metadata request retry tasks.|duration|1.6.0 kyuubi.metadata.request.retry.queue.size|65536|The maximum queue size for buffering metadata requests in memory when the external metadata storage is down. Requests will be dropped if the queue exceeds.|int|1.6.0 -kyuubi.metadata.request.retry.threads|10|Number of threads in the metadata request retry manager thread pool. The metadata store might be unavailable sometimes and the requests will fail, to tolerant for this case and unblock the main thread, we support to retry the failed requests in async way.|int|1.6.0 +kyuubi.metadata.request.retry.threads|10|Number of threads in the metadata request retry manager thread pool. The metadata store might be unavailable sometimes and the requests will fail, tolerant for this case and unblock the main thread, we support retrying the failed requests in an async way.|int|1.6.0 kyuubi.metadata.store.class|org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore|Fully qualified class name for server metadata store.|string|1.6.0 -kyuubi.metadata.store.jdbc.database.schema.init|true|Whether to init the jdbc metadata store database schema.|boolean|1.6.0 -kyuubi.metadata.store.jdbc.database.type|DERBY|The database type for server jdbc metadata store.