diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala index b0beb1b16d7..9c84feb62b5 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala @@ -111,6 +111,10 @@ private[ctl] object Render { private def buildBatchAppInfo(batch: Batch, showDiagnostic: Boolean = true): List[String] = { val batchAppInfo = ListBuffer[String]() + if (batch.getAppSubmissionTime > 0) { + batchAppInfo += s"App Submission Time:" + + s" ${millisToDateString(batch.getAppSubmissionTime, "yyyy-MM-dd HH:mm:ss")}" + } Option(batch.getAppId).foreach { _ => batchAppInfo += s"App Id: ${batch.getAppId}" } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Batch.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Batch.java index a078c9ac9ab..f17a8382398 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Batch.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Batch.java @@ -26,6 +26,7 @@ public class Batch { private String user; private String batchType; private String name; + private long appSubmissionTime; private String appId; private String appUrl; private String appState; @@ -42,6 +43,7 @@ public Batch( String user, String batchType, String name, + long appSubmissionTime, String appId, String appUrl, String appState, @@ -54,6 +56,7 @@ public Batch( this.user = user; this.batchType = batchType; this.name = name; + this.appSubmissionTime = appSubmissionTime; this.appId = appId; this.appUrl = appUrl; this.appState = appState; @@ -152,6 +155,14 @@ public void setCreateTime(long createTime) { this.createTime = createTime; } + public long getAppSubmissionTime() { + return appSubmissionTime; + } + + public void setAppSubmissionTime(long appSubmissionTime) { + this.appSubmissionTime = appSubmissionTime; + } + public long getEndTime() { return endTime; } diff --git a/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/BatchRestClientTest.java b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/BatchRestClientTest.java index 99700ffc761..304caad907e 100644 --- a/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/BatchRestClientTest.java +++ b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/BatchRestClientTest.java @@ -117,6 +117,7 @@ public void testNoPasswordBasicClient() { assertEquals(result.getUser(), expectedBatch.getUser()); assertEquals(result.getBatchType(), expectedBatch.getBatchType()); assertEquals(result.getName(), expectedBatch.getName()); + assertEquals(result.getAppSubmissionTime(), expectedBatch.getAppSubmissionTime()); assertEquals(result.getAppId(), expectedBatch.getAppId()); assertEquals(result.getAppUrl(), expectedBatch.getAppUrl()); assertEquals(result.getAppState(), expectedBatch.getAppState()); @@ -145,6 +146,7 @@ public void testAnonymousBasicClient() { assertEquals(result.getUser(), expectedBatch.getUser()); assertEquals(result.getBatchType(), expectedBatch.getBatchType()); assertEquals(result.getName(), expectedBatch.getName()); + assertEquals(result.getAppSubmissionTime(), expectedBatch.getAppSubmissionTime()); assertEquals(result.getAppId(), expectedBatch.getAppId()); assertEquals(result.getAppUrl(), expectedBatch.getAppUrl()); assertEquals(result.getAppState(), expectedBatch.getAppState()); @@ -176,6 +178,7 @@ public void createBatchTest() { assertEquals(result.getUser(), expectedBatch.getUser()); assertEquals(result.getBatchType(), expectedBatch.getBatchType()); assertEquals(result.getName(), expectedBatch.getName()); + assertEquals(result.getAppSubmissionTime(), expectedBatch.getAppSubmissionTime()); assertEquals(result.getAppId(), expectedBatch.getAppId()); assertEquals(result.getAppUrl(), expectedBatch.getAppUrl()); assertEquals(result.getAppState(), expectedBatch.getAppState()); @@ -207,6 +210,7 @@ public void getBatchByIdTest() { assertEquals(result.getUser(), expectedBatch.getUser()); assertEquals(result.getBatchType(), expectedBatch.getBatchType()); assertEquals(result.getName(), expectedBatch.getName()); + assertEquals(result.getAppSubmissionTime(), expectedBatch.getAppSubmissionTime()); assertEquals(result.getAppId(), expectedBatch.getAppId()); assertEquals(result.getAppUrl(), expectedBatch.getAppUrl()); assertEquals(result.getAppState(), expectedBatch.getAppState()); diff --git a/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/RestClientTestUtils.java b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/RestClientTestUtils.java index bc93b55dff5..82413e2a40e 100644 --- a/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/RestClientTestUtils.java +++ b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/RestClientTestUtils.java @@ -51,6 +51,7 @@ public static Batch generateTestBatch(String id) { TEST_USERNAME, "spark", "batch_name", + 0, id, null, "RUNNING", diff --git a/kyuubi-server/src/main/resources/sql/derby/002-KYUUBI-4119.derby.sql b/kyuubi-server/src/main/resources/sql/derby/002-KYUUBI-4119.derby.sql new file mode 100644 index 00000000000..0870785359b --- /dev/null +++ b/kyuubi-server/src/main/resources/sql/derby/002-KYUUBI-4119.derby.sql @@ -0,0 +1 @@ +ALTER TABLE metadata ADD COLUMN engine_open_time bigint; diff --git a/kyuubi-server/src/main/resources/sql/derby/metadata-store-schema-1.7.0.derby.sql b/kyuubi-server/src/main/resources/sql/derby/metadata-store-schema-1.7.0.derby.sql index d829824d87e..d61a8c38747 100644 --- a/kyuubi-server/src/main/resources/sql/derby/metadata-store-schema-1.7.0.derby.sql +++ b/kyuubi-server/src/main/resources/sql/derby/metadata-store-schema-1.7.0.derby.sql @@ -19,6 +19,7 @@ CREATE TABLE metadata( create_time BIGINT NOT NULL, -- the metadata create time engine_type varchar(32) NOT NULL, -- the engine type cluster_manager varchar(128), -- the engine cluster manager + engine_open_time bigint, -- the engine open time engine_id varchar(128), -- the engine application id engine_name clob, -- the engine application name engine_url varchar(1024), -- the engine tracking url diff --git a/kyuubi-server/src/main/resources/sql/derby/upgrade-1.6.0-to-1.7.0.derby.sql b/kyuubi-server/src/main/resources/sql/derby/upgrade-1.6.0-to-1.7.0.derby.sql index 34cf05f038d..3e58cb59b33 100644 --- a/kyuubi-server/src/main/resources/sql/derby/upgrade-1.6.0-to-1.7.0.derby.sql +++ b/kyuubi-server/src/main/resources/sql/derby/upgrade-1.6.0-to-1.7.0.derby.sql @@ -1 +1,2 @@ RUN '001-KYUUBI-3967.derby.sql'; +RUN '002-KYUUBI-4119.derby.sql'; diff --git a/kyuubi-server/src/main/resources/sql/mysql/002-KYUUBI-4119.mysql.sql b/kyuubi-server/src/main/resources/sql/mysql/002-KYUUBI-4119.mysql.sql new file mode 100644 index 00000000000..08298d7796f --- /dev/null +++ b/kyuubi-server/src/main/resources/sql/mysql/002-KYUUBI-4119.mysql.sql @@ -0,0 +1,3 @@ +SELECT '< KYUUBI-4119: Return app submission time for batch >' AS ' '; + +ALTER TABLE metadata ADD COLUMN engine_open_time bigint COMMENT 'the engine open time'; diff --git a/kyuubi-server/src/main/resources/sql/mysql/metadata-store-schema-1.7.0.mysql.sql b/kyuubi-server/src/main/resources/sql/mysql/metadata-store-schema-1.7.0.mysql.sql index 3ef9b59144f..6562f08d091 100644 --- a/kyuubi-server/src/main/resources/sql/mysql/metadata-store-schema-1.7.0.mysql.sql +++ b/kyuubi-server/src/main/resources/sql/mysql/metadata-store-schema-1.7.0.mysql.sql @@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS metadata( create_time BIGINT NOT NULL COMMENT 'the metadata create time', engine_type varchar(32) NOT NULL COMMENT 'the engine type', cluster_manager varchar(128) COMMENT 'the engine cluster manager', + engine_open_time bigint COMMENT 'the engine open time', engine_id varchar(128) COMMENT 'the engine application id', engine_name mediumtext COMMENT 'the engine application name', engine_url varchar(1024) COMMENT 'the engine tracking url', diff --git a/kyuubi-server/src/main/resources/sql/mysql/upgrade-1.6.0-to-1.7.0.mysql.sql b/kyuubi-server/src/main/resources/sql/mysql/upgrade-1.6.0-to-1.7.0.mysql.sql index a202364b258..21939daa86c 100644 --- a/kyuubi-server/src/main/resources/sql/mysql/upgrade-1.6.0-to-1.7.0.mysql.sql +++ b/kyuubi-server/src/main/resources/sql/mysql/upgrade-1.6.0-to-1.7.0.mysql.sql @@ -1,3 +1,4 @@ SELECT '< Upgrading MetaStore schema from 1.6.0 to 1.7.0 >' AS ' '; SOURCE 001-KYUUBI-3967.mysql.sql; +SOURCE 002-KYUUBI-4119.mysql.sql; SELECT '< Finished upgrading MetaStore schema from 1.6.0 to 1.7.0 >' AS ' '; diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/BatchJobSubmission.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/BatchJobSubmission.scala index 9e77fa5b8a7..1485f457fa8 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/BatchJobSubmission.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/BatchJobSubmission.scala @@ -74,6 +74,9 @@ class BatchJobSubmission( private var killMessage: KillResponse = (false, "UNKNOWN") def getKillMessage: KillResponse = killMessage + @volatile private var _appSubmissionTime = recoveryMetadata.map(_.engineOpenTime).getOrElse(0L) + def appSubmissionTime: Long = _appSubmissionTime + @VisibleForTesting private[kyuubi] val builder: ProcBuilder = { Option(batchType).map(_.toUpperCase(Locale.ROOT)) match { @@ -96,7 +99,14 @@ class BatchJobSubmission( override private[kyuubi] def currentApplicationInfo: Option[ApplicationInfo] = { // only the ApplicationInfo with non-empty id is valid for the operation - applicationManager.getApplicationInfo(builder.clusterManager(), batchId).filter(_.id != null) + val applicationInfo = + applicationManager.getApplicationInfo(builder.clusterManager(), batchId).filter(_.id != null) + applicationInfo.foreach { _ => + if (_appSubmissionTime <= 0) { + _appSubmissionTime = System.currentTimeMillis() + } + } + applicationInfo } private[kyuubi] def killBatchApplication(): KillResponse = { @@ -127,6 +137,7 @@ class BatchJobSubmission( val metadataToUpdate = Metadata( identifier = batchId, state = state.toString, + engineOpenTime = appSubmissionTime, engineId = status.id, engineName = status.name, engineUrl = status.url.orNull, diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala index 5c8d691f993..5ada9faf54c 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala @@ -94,6 +94,7 @@ private[v1] class BatchesResource extends ApiRequestContext with Logging { session.user, batchOp.batchType, name, + batchOp.appSubmissionTime, appId, appUrl, appState, @@ -130,6 +131,7 @@ private[v1] class BatchesResource extends ApiRequestContext with Logging { metadata.username, metadata.engineType, name, + metadata.engineOpenTime, appId, appUrl, appState, diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/MetadataManager.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/MetadataManager.scala index c7946950c20..5cecd2ab149 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/MetadataManager.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/MetadataManager.scala @@ -311,6 +311,7 @@ object MetadataManager extends Logging { batchMetadata.username, batchMetadata.engineType, name, + batchMetadata.engineOpenTime, batchMetadata.engineId, batchMetadata.engineUrl, batchMetadata.engineState, diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/api/Metadata.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/api/Metadata.scala index 0eed3861b2a..949e88abdf1 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/api/Metadata.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/api/Metadata.scala @@ -41,6 +41,7 @@ import org.apache.kyuubi.session.SessionType.SessionType * @param createTime the create time. * @param engineType the engine type. * @param clusterManager the engine cluster manager. + * @param engineOpenTime the engine open time * @param engineId the engine id. * @param engineName the engine name. * @param engineUrl the engine tracking url. @@ -65,6 +66,7 @@ case class Metadata( createTime: Long = 0L, engineType: String = null, clusterManager: Option[String] = None, + engineOpenTime: Long = 0L, engineId: String = null, engineName: String = null, engineUrl: String = null, diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala index 99ea4c1ebae..151d846d8ca 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala @@ -265,6 +265,10 @@ class JDBCMetadataStore(conf: KyuubiConf) extends MetadataStore with Logging { setClauses += " end_time = ? " params += metadata.endTime } + if (metadata.engineOpenTime > 0) { + setClauses += " engine_open_time = ? " + params += metadata.engineOpenTime + } Option(metadata.engineId).foreach { _ => setClauses += " engine_id = ? " params += metadata.engineId diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiBatchSessionImpl.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiBatchSessionImpl.scala index 8e583029339..dcf7fa116b5 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiBatchSessionImpl.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiBatchSessionImpl.scala @@ -161,7 +161,7 @@ class KyuubiBatchSessionImpl( private[kyuubi] def onEngineOpened(): Unit = { if (sessionEvent.openedTime <= 0) { - sessionEvent.openedTime = System.currentTimeMillis() + sessionEvent.openedTime = batchJobSubmissionOp.appSubmissionTime EventBus.post(sessionEvent) } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/BatchesResourceSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/BatchesResourceSuite.scala index 9a167085d6b..31d77deaaa4 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/BatchesResourceSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/BatchesResourceSuite.scala @@ -100,6 +100,9 @@ class BatchesResourceSuite extends KyuubiFunSuite with RestFrontendTestHelper wi assert(batch.getName === sparkBatchTestAppName) assert(batch.getCreateTime > 0) assert(batch.getEndTime === 0) + if (batch.getAppId != null) { + assert(batch.getAppSubmissionTime > 0) + } // invalid batchId getBatchResponse = webTarget.path(s"api/v1/batches/invalidBatchId")