From 7842b70157840e3d823b3ee4401eb437aa4b568c Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 11:13:58 +0800 Subject: [PATCH 01/17] Resolve the problem of fetching progress with no information. --- .../client/operator/impl/EngineConnProgressOperator.scala | 2 +- .../executor/operator/OperableOnceEngineConnOperator.scala | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnProgressOperator.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnProgressOperator.scala index 3205e13734..621516c2ee 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnProgressOperator.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnProgressOperator.scala @@ -35,7 +35,7 @@ class EngineConnProgressOperator extends OnceJobOperator[EngineConnProgressInfo] val progressInfo = progressInfoList.asScala.map(map => JobProgressInfo(map.get("id").asInstanceOf[String], map.get("totalTasks").asInstanceOf[Int], map.get("runningTasks").asInstanceOf[Int], map.get("failedTasks").asInstanceOf[Int], map.get("succeedTasks").asInstanceOf[Int])).toArray - EngineConnProgressInfo(result.getAs("progress"), progressInfo) + EngineConnProgressInfo(result.getAs[Double]("progress").toFloat, progressInfo) } } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala index 2d75b4449d..6d4166958f 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala @@ -35,7 +35,12 @@ class OperableOnceEngineConnOperator extends Operator { case operableOnceExecutor: OperableOnceExecutor => operatorName match { case PROGRESS_OPERATOR_NAME => - Map("progress" -> operableOnceExecutor.getProgress, "progressInfo" -> operableOnceExecutor.getProgressInfo) + val progressInfo = operableOnceExecutor.getProgressInfo + val progressInfoMap = if (progressInfo != null && progressInfo.nonEmpty) { + progressInfo.map(progressInfo => Map("id" -> progressInfo.id, "totalTasks" -> progressInfo.totalTasks, + "runningTasks" -> progressInfo.runningTasks, "failedTasks" -> progressInfo.failedTasks, "succeedTasks" -> progressInfo.succeedTasks)) + } else Array.empty[Map[String, Any]] + Map("progress" -> operableOnceExecutor.getProgress, "progressInfo" -> progressInfoMap) case METRICS_OPERATOR_NAME => Map("metrics" -> operableOnceExecutor.getMetrics) case DIAGNOSIS_OPERATOR_NAME => From f8d4271f47003d117aed3685995aee6cea936386 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 11:16:00 +0800 Subject: [PATCH 02/17] Resolve the problem of deploying flink application without yarn appId. --- .../deployment/ClusterDescriptorAdapter.java | 52 ++++++++++++++++--- ...rnApplicationClusterDescriptorAdapter.java | 2 +- .../YarnPerJobClusterDescriptorAdapter.java | 3 +- .../YarnSessionClusterDescriptorAdapter.scala | 1 + 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/ClusterDescriptorAdapter.java b/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/ClusterDescriptorAdapter.java index 008e70e2c7..b88f9b68b5 100644 --- a/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/ClusterDescriptorAdapter.java +++ b/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/ClusterDescriptorAdapter.java @@ -17,22 +17,29 @@ package org.apache.linkis.engineconnplugin.flink.client.deployment; -import org.apache.linkis.engineconnplugin.flink.client.context.ExecutionContext; -import org.apache.linkis.engineconnplugin.flink.config.FlinkEnvConfiguration; -import org.apache.linkis.engineconnplugin.flink.exception.JobExecutionException; -import java.io.Closeable; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; +import org.apache.commons.lang.StringUtils; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.JobStatus; import org.apache.flink.client.deployment.ClusterRetrieveException; import org.apache.flink.client.program.ClusterClient; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.yarn.YarnClusterDescriptor; +import org.apache.flink.yarn.configuration.YarnConfigOptions; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.linkis.engineconnplugin.flink.client.context.ExecutionContext; +import org.apache.linkis.engineconnplugin.flink.config.FlinkEnvConfiguration; +import org.apache.linkis.engineconnplugin.flink.exception.JobExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Closeable; +import java.lang.reflect.Method; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + /** * Cluster Descriptor Adapter, adaptable with datastream/sql tasks(集群交互适配器，适合datastream、sql方式作业) */ @@ -131,6 +138,37 @@ protected R bridgeClientRequest( } } + protected void bindApplicationId() throws JobExecutionException { + Method method = null; + try { + method = StreamExecutionEnvironment.class.getDeclaredMethod("getConfiguration"); + } catch (NoSuchMethodException e) { + throw new JobExecutionException("Not support flink version, StreamExecutionEnvironment.class is not exists getConfiguration method!", e); + } + method.setAccessible(true); + Configuration configuration; + try { + configuration = (Configuration) method.invoke(executionContext.getStreamExecutionEnvironment()); + } catch (Exception e) { + throw new JobExecutionException("StreamExecutionEnvironment.getConfiguration() execute failed!", e); + } + String applicationId = configuration.getString(YarnConfigOptions.APPLICATION_ID); + if(StringUtils.isNotBlank(applicationId)) { + LOG.info("The applicationId {} is exists in StreamExecutionEnvironment, ignore to bind applicationId to StreamExecutionEnvironment.", applicationId); + return; + } + applicationId = executionContext.getFlinkConfig().getString(YarnConfigOptions.APPLICATION_ID); + if(StringUtils.isBlank(applicationId) && this.clusterID == null) { + throw new JobExecutionException("No applicationId is exists!"); + } else if(StringUtils.isNotBlank(applicationId)) { + configuration.setString(YarnConfigOptions.APPLICATION_ID, applicationId); + LOG.info("Bind applicationId {} to StreamExecutionEnvironment.", applicationId); + } else { + configuration.setString(YarnConfigOptions.APPLICATION_ID, ConverterUtils.toString(clusterID)); + LOG.info("Bind applicationId {} to StreamExecutionEnvironment.", clusterID); + } + } + @Override public String toString() { return "ClusterDescriptorAdapter{" + diff --git a/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java b/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java index 775d26337e..c98217ac13 100644 --- a/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java +++ b/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java @@ -49,7 +49,7 @@ public void deployCluster(String[] programArguments, String applicationClassName } catch (Exception e) { throw new JobExecutionException(ExceptionUtils.getRootCauseMessage(e), e); } - + bindApplicationId(); } public boolean initJobId() throws Exception { diff --git a/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnPerJobClusterDescriptorAdapter.java b/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnPerJobClusterDescriptorAdapter.java index c3aaea83a1..15eeddc545 100644 --- a/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnPerJobClusterDescriptorAdapter.java +++ b/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnPerJobClusterDescriptorAdapter.java @@ -71,11 +71,12 @@ private boolean isYarnApplicationStopped(Throwable e) { return false; } - public void deployCluster(JobID jobId, ClusterClient clusterClient) { + public void deployCluster(JobID jobId, ClusterClient clusterClient) throws JobExecutionException { this.clusterClient = clusterClient; this.setJobId(jobId); this.clusterID = clusterClient.getClusterId(); webInterfaceUrl = clusterClient.getWebInterfaceURL(); + bindApplicationId(); } } diff --git a/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala b/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala index 238fe41ff2..a587bb0f71 100644 --- a/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala +++ b/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala @@ -30,6 +30,7 @@ class YarnSessionClusterDescriptorAdapter(executionContext: ExecutionContext) ex clusterClient = clusterClientProvider.getClusterClient clusterID = clusterClient.getClusterId webInterfaceUrl = clusterClient.getWebInterfaceURL + bindApplicationId() } override def setJobId(jobId: JobID): Unit = super.setJobId(jobId) From 756eb87d496866f153f7993217c433a8612eced7 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 11:17:37 +0800 Subject: [PATCH 03/17] Enhance the FlinkOnceJob, support to execute set, show grammar of flinKSQL. --- .../executor/FlinkCodeOnceExecutor.scala | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkCodeOnceExecutor.scala b/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkCodeOnceExecutor.scala index 2206fd9290..f0c6d8f32f 100644 --- a/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkCodeOnceExecutor.scala +++ b/linkis-engineconn-plugins/engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkCodeOnceExecutor.scala @@ -21,19 +21,25 @@ import java.util import java.util.concurrent.Future import java.util.function.Supplier +import org.apache.calcite.rel.metadata.{JaninoRelMetadataProvider, RelMetadataQueryBase} +import org.apache.commons.lang.StringUtils +import org.apache.flink.client.deployment.ClusterClientJobClientAdapter +import org.apache.flink.client.program.{ClusterClient, ClusterClientProvider} +import org.apache.flink.table.api.{ResultKind, TableResult} +import org.apache.flink.table.planner.plan.metadata.FlinkDefaultRelMetadataProvider +import org.apache.hadoop.yarn.api.records.ApplicationId import org.apache.linkis.common.utils.{ByteTimeUtils, Utils, VariableUtils} import org.apache.linkis.engineconn.once.executor.OnceExecutorExecutionContext import org.apache.linkis.engineconnplugin.flink.client.deployment.YarnPerJobClusterDescriptorAdapter +import org.apache.linkis.engineconnplugin.flink.client.sql.operation.OperationFactory +import org.apache.linkis.engineconnplugin.flink.client.sql.operation.result.ResultKind.SUCCESS_WITH_CONTENT +import org.apache.linkis.engineconnplugin.flink.client.sql.parser.{SqlCommand, SqlCommandParser} import org.apache.linkis.engineconnplugin.flink.context.FlinkEngineConnContext -import org.apache.linkis.engineconnplugin.flink.exception.FlinkInitFailedException +import org.apache.linkis.engineconnplugin.flink.exception.{FlinkInitFailedException, SqlParseException} import org.apache.linkis.governance.common.paser.{CodeParserFactory, CodeType} import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.scheduler.executer.ErrorExecuteResponse -import org.apache.commons.lang.StringUtils -import org.apache.flink.client.deployment.ClusterClientJobClientAdapter -import org.apache.flink.client.program.{ClusterClient, ClusterClientProvider} -import org.apache.flink.table.api.{ResultKind, TableResult} -import org.apache.hadoop.yarn.api.records.ApplicationId + class FlinkCodeOnceExecutor(override val id: Long, @@ -62,6 +68,7 @@ class FlinkCodeOnceExecutor(override val id: Long, future = Utils.defaultScheduler.submit(new Runnable { override def run(): Unit = { info("Try to execute codes.") + RelMetadataQueryBase.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(FlinkDefaultRelMetadataProvider.INSTANCE)) Utils.tryCatch(CodeParserFactory.getCodeParser(CodeType.SQL).parse(codes).filter(StringUtils.isNotBlank).foreach(runCode)){ t => error("Run code failed!", t) setResponse(ErrorExecuteResponse("Run code failed!", t)) @@ -83,6 +90,24 @@ class FlinkCodeOnceExecutor(override val id: Long, val trimmedCode = StringUtils.trim(code) info(s"$getId >> " + trimmedCode) val startTime = System.currentTimeMillis + val callOpt = SqlCommandParser.getSqlCommandParser.parse(code.trim, true) + if(!callOpt.isPresent) throw new SqlParseException("Unknown statement: " + code) + val resultSet = callOpt.get().command match { + case SqlCommand.SET | SqlCommand.USE_CATALOG | SqlCommand.USE | SqlCommand.SHOW_MODULES | + SqlCommand.DESCRIBE_TABLE | SqlCommand.EXPLAIN => + val operation = OperationFactory.getOperationFactory.createOperation(callOpt.get(), flinkEngineConnContext) + Some(operation.execute()) + case command if command.toString.startsWith("SHOW_") => + val operation = OperationFactory.getOperationFactory.createOperation(callOpt.get(), flinkEngineConnContext) + Some(operation.execute()) + case _ => None + } + resultSet.foreach(r => r.getResultKind match { + case SUCCESS_WITH_CONTENT => + info(r.toString) + return + case _ => return + }) val tableResult = flinkEngineConnContext.getExecutionContext.wrapClassLoader(new Supplier[TableResult]{ override def get(): TableResult = flinkEngineConnContext.getExecutionContext.getTableEnvironment.executeSql(trimmedCode) }) From f2bacc1c1c3e7aa4fbecbc4ed435f0f64d139200 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 11:21:31 +0800 Subject: [PATCH 04/17] Enhance the FlinkOnceJobTest, add the support of using onceJob operators. --- .../linkis/computation/client/FlinkOnceJobTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/FlinkOnceJobTest.java b/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/FlinkOnceJobTest.java index 0cdafa4d61..491f97842f 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/FlinkOnceJobTest.java +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/FlinkOnceJobTest.java @@ -19,6 +19,9 @@ import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.computation.client.once.simple.SubmittableSimpleOnceJob; +import org.apache.linkis.computation.client.operator.impl.EngineConnLogOperator; +import org.apache.linkis.computation.client.operator.impl.EngineConnLogs; +import org.apache.linkis.computation.client.operator.impl.EngineConnMetricsOperator; import org.apache.linkis.computation.client.utils.LabelKeyUtils; public class FlinkOnceJobTest { @@ -65,6 +68,13 @@ public static void main(String[] args) { .build(); onceJob.submit(); System.out.println(onceJob.getId()); + EngineConnLogOperator logOperator = (EngineConnLogOperator) onceJob.getOperator(EngineConnLogOperator.OPERATOR_NAME()); + EngineConnMetricsOperator metricsOperator = (EngineConnMetricsOperator) onceJob.getOperator(EngineConnMetricsOperator.OPERATOR_NAME()); + logOperator.setFromLine(1); + logOperator.setPageSize(100); + EngineConnLogs logs = logOperator.apply(); + System.out.println("logPath: " + logs.logPath() + ", logs: " + logs.logs()); + System.out.println("Metrics: " + metricsOperator.apply()); onceJob.waitForCompleted(); System.exit(0); } From 9b5e2d9bb5e52e7aee7609b2e0164a986cf86f26 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 15:12:52 +0800 Subject: [PATCH 05/17] Since both ECM and EC need Operator module, so the Operator module has been moved to linkis-manager-common. --- .../EngineConnApplicationInfoOperator.scala | 2 +- .../service/DefaultOperateService.scala | 9 ++-- .../manager/common/operator/Operator.scala | 43 +++++++++++++++++++ .../common}/operator/OperatorFactory.scala | 31 +++++++------ 4 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/operator/Operator.scala rename linkis-computation-governance/{linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor => linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common}/operator/OperatorFactory.scala (64%) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala index 668816ef00..58afb816ce 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala @@ -17,10 +17,10 @@ package org.apache.linkis.engineconn.acessible.executor.operator.impl -import org.apache.linkis.engineconn.acessible.executor.operator.Operator import org.apache.linkis.engineconn.common.exception.EngineConnException import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.executor.entity.YarnExecutor +import org.apache.linkis.manager.common.operator.Operator class EngineConnApplicationInfoOperator extends Operator { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala index 005632da69..35e723c81d 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala @@ -18,22 +18,25 @@ package org.apache.linkis.engineconn.acessible.executor.service import org.apache.commons.lang.exception.ExceptionUtils import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.engineconn.acessible.executor.operator.OperatorFactory +import org.apache.linkis.manager.common.operator.OperatorFactory import org.apache.linkis.manager.common.protocol.engine.{EngineOperateRequest, EngineOperateResponse} import org.apache.linkis.message.annotation.Receiver import org.springframework.stereotype.Service +import scala.collection.JavaConverters.mapAsScalaMapConverter + @Service class DefaultOperateService extends OperateService with Logging { @Receiver override def executeOperation(engineOperateRequest: EngineOperateRequest): EngineOperateResponse = { - val operator = Utils.tryCatch(OperatorFactory().getOperatorRequest(engineOperateRequest)){ t => + val parameters = engineOperateRequest.parameters.asScala.toMap + val operator = Utils.tryCatch(OperatorFactory().getOperatorRequest(parameters)){ t => error(s"Get operator failed, parameters is ${engineOperateRequest.parameters}.", t) return EngineOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) } info(s"Try to execute operator ${operator.getClass.getSimpleName} with parameters ${engineOperateRequest.parameters}.") - val result = Utils.tryCatch(operator(engineOperateRequest.parameters)) {t => + val result = Utils.tryCatch(operator(parameters)) {t => error(s"Execute ${operator.getClass.getSimpleName} failed.", t) return EngineOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/operator/Operator.scala b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/operator/Operator.scala new file mode 100644 index 0000000000..35edca8d93 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/operator/Operator.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.common.operator + +import org.apache.linkis.governance.common.exception.GovernanceErrorException + + +trait Operator { + + def getNames: Array[String] + + def apply(implicit parameters: Map[String, Any]): Map[String, Any] + + protected def getAs[T](key: String, defaultVal: => T)(implicit parameters: Map[String, Any]): T = + parameters.getOrElse(key, defaultVal) match { + case t: T => t + case null => null.asInstanceOf[T] + case v => throw new GovernanceErrorException(20305, s"Unknown $v for key $key.") + } + + protected def getAsThrow[T](key: String)(implicit parameters: Map[String, Any]): T = + parameters.get(key) match { + case Some(t: T) => t + case Some(t: Any) => throw new GovernanceErrorException(20305, s"Unknown class type, cannot cast $t.") + case None => throw new GovernanceErrorException(20305, s"The parameter of $key is not exists.") + } + +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/OperatorFactory.scala b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/operator/OperatorFactory.scala similarity index 64% rename from linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/OperatorFactory.scala rename to linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/operator/OperatorFactory.scala index 0c0f3bdf8d..9a1281085e 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/OperatorFactory.scala +++ b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/operator/OperatorFactory.scala @@ -5,27 +5,27 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.apache.linkis.engineconn.acessible.executor.operator -import org.apache.linkis.common.exception.WarnException +package org.apache.linkis.manager.common.operator + import org.apache.linkis.common.utils.{ClassUtils, Logging} -import org.apache.linkis.manager.common.protocol.engine.EngineOperateRequest +import org.apache.linkis.governance.common.exception.GovernanceErrorException +import org.apache.linkis.manager.common.protocol.OperateRequest trait OperatorFactory { def getOperatorName(parameters: Map[String, Any]): String - def getOperatorRequest(request: EngineOperateRequest): Operator + def getOperatorRequest(parameters: Map[String, Any]): Operator } @@ -42,19 +42,18 @@ class OperatorFactoryImpl extends OperatorFactory with Logging { private val operators: Map[String, _ <: Operator] = ClassUtils.reflections.getSubTypesOf(classOf[Operator]) .filterNot(ClassUtils.isInterfaceOrAbstract).flatMap { clazz => - val operator = clazz.newInstance() - operator.getNames.map(name => name -> operator) - }.toMap + val operator = clazz.newInstance() + operator.getNames.map(name => name -> operator) + }.toMap info("Launched operators list => " + operators) override def getOperatorName(parameters: Map[String, Any]): String = - parameters.getOrElse(EngineOperateRequest.OPERATOR_NAME_KEY, - throw new WarnException(20031, s"${EngineOperateRequest.OPERATOR_NAME_KEY} is not exists.")).asInstanceOf[String] + OperateRequest.getOperationName(parameters) - override def getOperatorRequest(request: EngineOperateRequest): Operator = { - val operatorName = getOperatorName(request.parameters) + override def getOperatorRequest(parameters: Map[String, Any]): Operator = { + val operatorName = getOperatorName(parameters) if (operators.contains(operatorName)) operators(operatorName) - else throw new WarnException(20030, s"Cannot find operator named $operatorName.") + else throw new GovernanceErrorException(20030, s"Cannot find operator named $operatorName.") } } From 2b13129abca88e6b4e0cff22be3fbd7d127e10cc Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 15:13:38 +0800 Subject: [PATCH 06/17] Since both ECM and EC need Operator module, so the Operator module has been moved to linkis-manager-common. --- .../once/executor/operator/OperableOnceEngineConnOperator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala index 6d4166958f..29a90054a7 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala @@ -17,10 +17,10 @@ package org.apache.linkis.engineconn.once.executor.operator -import org.apache.linkis.engineconn.acessible.executor.operator.{Operator, OperatorFactory} import org.apache.linkis.engineconn.common.exception.EngineConnException import org.apache.linkis.engineconn.once.executor.OperableOnceExecutor import org.apache.linkis.engineconn.once.executor.creation.OnceExecutorManager +import org.apache.linkis.manager.common.operator.{Operator, OperatorFactory} class OperableOnceEngineConnOperator extends Operator { From dd465084656e8eaa087f388afa70cbca5c8a77eb Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 15:15:40 +0800 Subject: [PATCH 07/17] Add the capability of Operator to ECM, so LinkisManager can request to execute ECM Operators by RPC. --- .../ecm/server/exception/ECMErrorCode.scala | 26 ++++++++++ .../ecm/server/service/OperateService.scala | 27 +++++++++++ .../service/impl/DefaultOperateService.scala | 48 +++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala create mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/OperateService.scala create mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala new file mode 100644 index 0000000000..a25ca2ddee --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.server.exception + +object ECMErrorCode { + + val EC_START_TIME_OUT = 11100 + + val EC_INTERRUPT_TIME_OUT = 11101 + + val EC_START_FAILED = 11102 + + val EC_FETCH_LOG_FAILED = 11110 + +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/OperateService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/OperateService.scala new file mode 100644 index 0000000000..14bfbe3792 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/OperateService.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.server.service + +import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} + + +trait OperateService { + + def executeOperation(ecmOperateRequest: ECMOperateRequest): ECMOperateResponse + +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala new file mode 100644 index 0000000000..bae6a26334 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.server.service.impl + +import org.apache.commons.lang.exception.ExceptionUtils +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.ecm.server.service.OperateService +import org.apache.linkis.manager.common.operator.OperatorFactory +import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} +import org.apache.linkis.message.annotation.Receiver +import org.springframework.stereotype.Service + +import scala.collection.JavaConverters.mapAsScalaMapConverter + + +@Service +class DefaultOperateService extends OperateService with Logging { + + @Receiver + override def executeOperation(ecmOperateRequest: ECMOperateRequest): ECMOperateResponse = { + val parameters = ecmOperateRequest.parameters.asScala.toMap + val operator = Utils.tryCatch(OperatorFactory().getOperatorRequest(parameters)) { t => + error(s"Get operator failed, parameters is ${ecmOperateRequest.parameters}.", t) + return ECMOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) + } + info(s"Try to execute operator ${operator.getClass.getSimpleName} with parameters ${ecmOperateRequest.parameters}.") + val result = Utils.tryCatch(operator(parameters)) {t => + error(s"Execute ${operator.getClass.getSimpleName} failed.", t) + return ECMOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) + } + ECMOperateResponse(result) + } +} From 7eab04b8601718a5264cbc559f85d62652d01c92 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 15:16:58 +0800 Subject: [PATCH 08/17] Add the capability of fetch EC logs from ECM, now we can get all logs of all ECs whenever the EC is running or stopped. --- .../operator/EngineConnLogOperator.scala | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala new file mode 100644 index 0000000000..bd1e3c65f8 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.server.operator + +import java.io.{File, RandomAccessFile} +import java.util + +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.ecm.server.exception.{ECMErrorCode, ECMErrorException} +import org.apache.linkis.ecm.server.service.{EngineConnListService, LocalDirsHandleService} +import org.apache.linkis.manager.common.operator.Operator +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest +import org.apache.commons.io.IOUtils +import org.apache.commons.lang.StringUtils + +import scala.collection.JavaConverters.asScalaBufferConverter +import scala.util.matching.Regex + +class EngineConnLogOperator extends Operator with Logging { + + private var engineConnListService: EngineConnListService = _ + private var localDirsHandleService: LocalDirsHandleService = _ + + override def getNames: Array[String] = Array(EngineConnLogOperator.OPERATOR_NAME) + + override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { + val logPath = getLogPath + val lastRows = getAs("lastRows", 0) + if (lastRows > EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue) { + throw new ECMErrorException(ECMErrorCode.EC_FETCH_LOG_FAILED, s"Cannot fetch more than ${EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue} lines of logs.") + } else if (lastRows > 0) { + val logs = Utils.exec(Array("tail", "-f", logPath.getPath), 5000).split("\n") + return Map("logs" -> logs, "rows" -> logs.length) + } + val pageSize = getAs("pageSize", 100) + val fromLine = getAs("fromLine", 1) + val ignoreKeywords = getAs("ignoreKeywords", "") + val ignoreKeywordList = if (StringUtils.isNotEmpty(ignoreKeywords)) ignoreKeywords.split(",") else Array.empty[String] + val onlyKeywords = getAs("onlyKeywords", "") + val onlyKeywordList = if (StringUtils.isNotEmpty(onlyKeywords)) onlyKeywords.split(",") else Array.empty[String] + val reader = new RandomAccessFile(logPath, "r") + val logs = new util.ArrayList[String](pageSize) + var readLine, skippedLine, lineNum = 0 + var rowIgnore = false + var ignoreLine = 0 + val linePattern = Option(EngineConnLogOperator.MULTILINE_PATTERN.getValue) match { + case Some(pattern) => pattern.r + case _ => null + } + val maxMultiline = EngineConnLogOperator.MULTILINE_MAX.getValue + Utils.tryFinally { + var line = reader.readLine() + while (readLine < pageSize && line != null) { + lineNum += 1 + if (skippedLine < fromLine - 1) { + skippedLine += 1 + } else { + if (rowIgnore) { + linePattern match { + case reg: Regex => + if (reg.findFirstIn(line).isDefined) { + ignoreLine = 0 + rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList) + } else { + ignoreLine += 1 + if (ignoreLine >= maxMultiline) { + rowIgnore = false + } + } + case _ => rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList) + } + } else { + rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList) + } + if (!rowIgnore) { + logs.add(line) + readLine += 1 + } + } + line = reader.readLine + } + }(IOUtils.closeQuietly(reader)) + Map("logPath" -> logPath.getPath, "logs" -> logs, "endLine" -> lineNum, "rows" -> readLine) + } + + private def includeLine(line: String, + onlyKeywordList: Array[String], ignoreKeywordList: Array[String]): Boolean = { + if (onlyKeywordList.nonEmpty && onlyKeywordList.exists(line.contains)) { + true + } else if (ignoreKeywordList.nonEmpty && !ignoreKeywordList.exists(line.contains)) { + true + } else if (onlyKeywordList.isEmpty && ignoreKeywordList.isEmpty) { + true + } else { + false + } + } + private def getLogPath(implicit parameters: Map[String, Any]): File = { + if (engineConnListService == null) { + engineConnListService = DataWorkCloudApplication.getApplicationContext.getBean(classOf[EngineConnListService]) + localDirsHandleService = DataWorkCloudApplication.getApplicationContext.getBean(classOf[LocalDirsHandleService]) + } + val engineConnInstance = getAs(ECMOperateRequest.ENGINE_CONN_INSTANCE_KEY, getAs[String]("engineConnInstance", null)) + val (engineConnLogDir, ticketId) = Option(engineConnInstance).flatMap { instance => + engineConnListService.getEngineConns.asScala.find(_.getServiceInstance.getInstance == instance) + }.map(engineConn => (engineConn.getEngineConnManagerEnv.engineConnLogDirs, engineConn.getTickedId)).getOrElse { + val ticketId = getAs("ticketId", "") + if (StringUtils.isBlank(ticketId)) { + throw new ECMErrorException(ECMErrorCode.EC_FETCH_LOG_FAILED, s"the parameters of ${ECMOperateRequest.ENGINE_CONN_INSTANCE_KEY}, engineConnInstance and ticketId are both not exists.") + } + val logDir = engineConnListService.getEngineConn(ticketId).map(_.getEngineConnManagerEnv.engineConnLogDirs) + .getOrElse { + val creator = getAsThrow[String]("creator") + val engineConnType = getAsThrow[String]("engineConnType") + localDirsHandleService.getEngineConnLogDir(creator, ticketId) + } + (logDir, ticketId) + } + val logPath = new File(engineConnLogDir, EngineConnLogOperator.LOG_FILE_NAME.getValue) + if(!logPath.exists() || !logPath.isFile) { + throw new ECMErrorException(ECMErrorCode.EC_FETCH_LOG_FAILED, s"LogFile $logPath is not exists or is not a file.") + } + info(s"Try to fetch EngineConn(id: $ticketId, instance: $engineConnInstance) logs from ${logPath.getPath}.") + logPath + } +} + +object EngineConnLogOperator { + val OPERATOR_NAME = "engineConnLog" + val LOG_FILE_NAME = CommonVars("linkis.engineconn.log.filename", "stdout") + val MAX_LOG_FETCH_SIZE = CommonVars("linkis.engineconn.log.fetch.lines.max", 5000) + // yyyy-MM-dd HH:mm:ss.SSS + val MULTILINE_PATTERN = CommonVars("linkis.engineconn.log.multiline.pattern", "^\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\.\\d{3}") + val MULTILINE_MAX = CommonVars("linkis.engineconn.log.multiline.max", 500) +} \ No newline at end of file From bead7e1a5867189c011cf185d2b2049f82bc7442 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 15:19:23 +0800 Subject: [PATCH 09/17] Optimize the architecture of LinkisManager, so linkis-computation-client can request LinkisManager to route operator request to ECM. --- .../manager/am/restful/EMRestfulApi.java | 106 ++++++++++++++++++ .../manager/am/conf/AMConfiguration.scala | 6 + .../am/manager/DefaultEMNodeManager.scala | 4 + .../manager/am/manager/EMNodeManager.scala | 3 + .../am/pointer/DefaultEMNodPointer.scala | 23 ++-- .../service/em/DefaultECMOperateService.scala | 35 ++++++ .../am/service/em/DefaultEMInfoService.scala | 21 ++-- .../am/service/em/ECMOperateService.scala | 28 +++++ .../common/protocol/OperateRequest.scala | 40 +++++++ .../common/protocol/OperateResponse.scala | 26 +++++ .../protocol/em/ECMOperateRequest.scala | 35 ++++++ .../protocol/em/ECMOperateResponse.scala | 30 +++++ .../engine/EngineOperateRequest.scala | 26 ++--- .../engine/EngineOperateResponse.scala | 18 ++- .../service/common/pointer/EMNodPointer.scala | 3 + 15 files changed, 363 insertions(+), 41 deletions(-) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultECMOperateService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/ECMOperateService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/OperateRequest.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/OperateResponse.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/ECMOperateRequest.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/ECMOperateResponse.scala diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java index 7eec5f6899..da9df4e116 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java @@ -17,17 +17,29 @@ package org.apache.linkis.manager.am.restful; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.common.utils.JsonUtils; import org.apache.linkis.manager.am.conf.AMConfiguration; import org.apache.linkis.manager.am.converter.DefaultMetricsConverter; import org.apache.linkis.manager.am.exception.AMErrorCode; import org.apache.linkis.manager.am.exception.AMErrorException; +import org.apache.linkis.manager.am.service.em.ECMOperateService; import org.apache.linkis.manager.am.service.em.EMInfoService; +import org.apache.linkis.manager.am.service.engine.EngineCreateService; import org.apache.linkis.manager.am.utils.AMUtils; import org.apache.linkis.manager.am.vo.EMNodeVo; import org.apache.linkis.manager.common.entity.enumeration.NodeHealthy; import org.apache.linkis.manager.common.entity.metrics.NodeHealthyInfo; import org.apache.linkis.manager.common.entity.node.EMNode; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.protocol.OperateRequest$; +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest$; +import org.apache.linkis.manager.common.protocol.em.ECMOperateResponse; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; import org.apache.linkis.manager.label.entity.Label; @@ -67,10 +79,18 @@ public class EMRestfulApi { @Autowired private DefaultMetricsConverter defaultMetricsConverter; + @Autowired + private EngineCreateService engineCreateService; + + @Autowired + private ECMOperateService ecmOperateService; + private LabelBuilderFactory stdLabelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory(); private Logger logger = LoggerFactory.getLogger(EMRestfulApi.class); + private String[] adminOperations = AMConfiguration.ECM_ADMIN_OPERATIONS().getValue().split(","); + //todo add healthInfo @RequestMapping(path = "/listAllEMs", method = RequestMethod.GET) @@ -162,4 +182,90 @@ public Message modifyEMInfo( HttpServletRequest req, @RequestBody JsonNode jsonN return Message.ok("修改EM信息成功"); } + @RequestMapping(path = "/executeECMOperationByEC", method = RequestMethod.POST) + public Message executeECMOperationByEC(HttpServletRequest req, @RequestBody JsonNode jsonNode) throws AMErrorException { + String userName = SecurityFilter.getLoginUsername(req); + ServiceInstance serviceInstance = EngineRestfulApi.getServiceInstance(jsonNode); + logger.info("User {} try to execute ECM Operation by EngineConn {}.", userName, serviceInstance); + EngineNode engineNode = engineCreateService.getEngineNode(serviceInstance); + Map parameters; + try { + parameters = JsonUtils.jackson().readValue(jsonNode.get("parameters").toString(), + new TypeReference>() { + }); + } catch (JsonProcessingException e){ + logger.error("Fail to process the operation parameters: [{}] in request", jsonNode.get("parameters").toString(), e); + return Message.error("Fail to process the operation parameters, cased by " + ExceptionUtils.getRootCauseMessage(e)); + } + parameters.put(ECMOperateRequest.ENGINE_CONN_INSTANCE_KEY(), serviceInstance.getInstance()); + if(!userName.equals(engineNode.getOwner()) && !AMConfiguration.isAdmin(userName)) { + return Message.error("You have no permission to execute ECM Operation by this EngineConn " + serviceInstance); + } + return executeECMOperation(engineNode.getEMNode(), new ECMOperateRequest(userName, parameters)); + } + + + @RequestMapping(path = "/executeECMOperation", method = RequestMethod.POST) + public Message executeECMOperation(HttpServletRequest req, @RequestBody JsonNode jsonNode) throws AMErrorException { + String userName = SecurityFilter.getLoginUsername(req); + ServiceInstance serviceInstance = EngineRestfulApi.getServiceInstance(jsonNode); + logger.info("User {} try to execute ECM Operation with {}.", userName, serviceInstance); + EMNode ecmNode = this.emInfoService.getEM(serviceInstance); + Map parameters; + try { + parameters = JsonUtils.jackson().readValue(jsonNode.get("parameters").toString(), + new TypeReference>() { + }); + } catch (JsonProcessingException e) { + logger.error("Fail to process the operation parameters: [{}] in request", jsonNode.get("parameters").toString(), e); + return Message.error("Fail to process the operation parameters, cased by " + ExceptionUtils.getRootCauseMessage(e)); + } + return executeECMOperation(ecmNode, new ECMOperateRequest(userName, parameters)); + } + + @RequestMapping(path = "/openEngineLog", method = RequestMethod.POST) + public Message openEngineLog(HttpServletRequest req, @RequestBody JsonNode jsonNode) throws AMErrorException { + String userName = SecurityFilter.getLoginUsername(req); + EMNode ecmNode; + Map parameters; + try { + String emInstance = jsonNode.get("emInstance").asText(); + String engineInstance = jsonNode.get("instance").asText(); + ServiceInstance serviceInstance = EngineRestfulApi.getServiceInstance(jsonNode); + logger.info("User {} try to open engine: {} log.", userName, serviceInstance); + ecmNode = this.emInfoService.getEM(ServiceInstance.apply("linkis-cg-engineconnmanager", emInstance)); + logger.info("ecm node info:{}", ecmNode); + parameters = JsonUtils.jackson().readValue(jsonNode.get("parameters").toString(), + new TypeReference>() { + }); + String logType = (String) parameters.get("logType"); + if (!logType.equals("stdout") && !logType.equals("stderr")) { + throw new AMErrorException(AMErrorCode.PARAM_ERROR.getCode(), AMErrorCode.PARAM_ERROR.getMessage()); + } + parameters.put(OperateRequest$.MODULE$.OPERATOR_NAME_KEY(), "engineConnLog"); + parameters.put(ECMOperateRequest$.MODULE$.ENGINE_CONN_INSTANCE_KEY(), engineInstance); + } catch (JsonProcessingException e) { + logger.error("Fail to process the operation parameters: [{}] in request", jsonNode.get("parameters").toString(), e); + return Message.error("Fail to process the operation parameters, cased by " + ExceptionUtils.getRootCauseMessage(e)); + } catch (Exception e) { + logger.error("Failed to open engine log, error:", e); + return Message.error(e.getMessage()); + } + return executeECMOperation(ecmNode, new ECMOperateRequest(userName, parameters)); + } + + private Message executeECMOperation(EMNode ecmNode, ECMOperateRequest ecmOperateRequest) { + String operationName = OperateRequest$.MODULE$.getOperationName(ecmOperateRequest.parameters()); + if(ArrayUtils.contains(adminOperations, operationName) && !AMConfiguration.isAdmin(ecmOperateRequest.user())) { + logger.warn("User {} has no permission to execute {} admin Operation in ECM {}.", ecmOperateRequest.user(), operationName, ecmNode.getServiceInstance()); + return Message.error("You have no permission to execute " + operationName + " admin Operation in ECM " + ecmNode.getServiceInstance()); + } + ECMOperateResponse engineOperateResponse = ecmOperateService.executeOperation(ecmNode, ecmOperateRequest); + + return Message.ok() + .data("result", engineOperateResponse.getResult()) + .data("errorMsg", engineOperateResponse.errorMsg()) + .data("isError", engineOperateResponse.isError()); + } + } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala index 4209b0594b..ed33e7e2b9 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala @@ -25,6 +25,8 @@ object AMConfiguration { val GOVERNANCE_STATION_ADMIN = CommonVars("wds.linkis.governance.station.admin", "hadoop") + val ECM_ADMIN_OPERATIONS = CommonVars("wds.linkis.governance.admin.operations", "") + val ENGINE_START_MAX_TIME = CommonVars("wds.linkis.manager.am.engine.start.max.time", new TimeType("10m")) val ENGINE_CONN_START_REST_MAX_WAIT_TIME = CommonVars("wds.linkis.manager.am.engine.rest.start.max.time", new TimeType("40s")) @@ -64,4 +66,8 @@ object AMConfiguration { s""" {jdbc:"$jvmUser", es: "$jvmUser", presto:"$jvmUser",appconn:"$jvmUser", io_file:"root"}""" } + def isAdmin(userName: String): Boolean = { + GOVERNANCE_STATION_ADMIN.getValue.split(",").contains(userName) + } + } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.scala index 9981c76d3b..2051669990 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.scala @@ -23,6 +23,7 @@ import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.manager.common.entity.node._ import org.apache.linkis.manager.common.entity.persistence.PersistenceNodeEntity +import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest import org.apache.linkis.manager.exception.NodeInstanceDuplicateException @@ -174,5 +175,8 @@ class DefaultEMNodeManager extends EMNodeManager with Logging { nodePointerBuilder.buildEMNodePointer(emNode).stopEngine(engineStopRequest) } + override def executeOperation(ecmNode: EMNode, request: ECMOperateRequest): ECMOperateResponse = { + nodePointerBuilder.buildEMNodePointer(ecmNode).executeOperation(request) + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EMNodeManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EMNodeManager.scala index a5d4e25ef4..1a1b1b5528 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EMNodeManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EMNodeManager.scala @@ -19,6 +19,7 @@ package org.apache.linkis.manager.am.manager import org.apache.linkis.common.ServiceInstance import org.apache.linkis.manager.common.entity.node.{EMNode, EngineNode, Node, ScoreServiceInstance} +import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest @@ -65,4 +66,6 @@ trait EMNodeManager { def addEMNodeInstance(emNode: EMNode): Unit def initEMNodeMetrics(emNode: EMNode): Unit + + def executeOperation(ecmNode: EMNode, request: ECMOperateRequest): ECMOperateResponse } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala index c7241de173..2eb5673cbd 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala @@ -5,25 +5,25 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - + package org.apache.linkis.manager.am.pointer import org.apache.linkis.common.utils.Utils import org.apache.linkis.manager.am.exception.AMErrorException -import org.apache.linkis.manager.am.service.engine.EngineStopService import org.apache.linkis.manager.am.utils.AMUtils import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.entity.node.{EngineNode, Node} -import org.apache.linkis.manager.common.protocol.engine.{EngineStopRequest, EngineStopResponse, EngineSuicideRequest} +import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} +import org.apache.linkis.manager.common.protocol.engine.{EngineStopRequest, EngineStopResponse} import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest import org.apache.linkis.manager.service.common.pointer.EMNodPointer @@ -54,8 +54,8 @@ class DefaultEMNodPointer(val node: Node) extends AbstractNodePointer with EMNod case engineStopResponse: EngineStopResponse => if (!engineStopResponse.getStopStatus) { info(s"Kill engine : ${engineStopRequest.getServiceInstance.toString} failed, because ${engineStopResponse.getMsg} . Will ask engine to suicide.") - val engineSuicideRequest = new EngineSuicideRequest(engineStopRequest.getServiceInstance, engineStopRequest.getUser) - EngineStopService.askEngineToSuicide(engineSuicideRequest) + /* val engineSuicideRequest = new EngineSuicideRequest(engineStopRequest.getServiceInstance, engineStopRequest.getUser) + EngineStopService.askEngineToSuicide(engineSuicideRequest)*/ } else { info(s"Succeed to kill engine ${engineStopRequest.getServiceInstance.toString}.") } @@ -65,5 +65,10 @@ class DefaultEMNodPointer(val node: Node) extends AbstractNodePointer with EMNod } } - + override def executeOperation(request: ECMOperateRequest): ECMOperateResponse = { + getSender.ask(request) match { + case response: ECMOperateResponse => response + case _ => throw new AMErrorException(AMConstant.ENGINE_ERROR_CODE, "Failed to execute ECM operation.") + } + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultECMOperateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultECMOperateService.scala new file mode 100644 index 0000000000..fd19b82117 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultECMOperateService.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.service.em + +import org.apache.linkis.manager.am.manager.EMNodeManager +import org.apache.linkis.manager.common.entity.node.EMNode +import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + + +@Service +class DefaultECMOperateService extends ECMOperateService { + + @Autowired + private var emNodeManager: EMNodeManager = _ + + override def executeOperation(ecmNode: EMNode, request: ECMOperateRequest): ECMOperateResponse = + emNodeManager.executeOperation(ecmNode, request) +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala index 0092d79184..ae38013a3b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala @@ -5,30 +5,24 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.apache.linkis.manager.am.service.em -import java.util +package org.apache.linkis.manager.am.service.em import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.Logging import org.apache.linkis.governance.common.conf.GovernanceCommonConf -import org.apache.linkis.manager.am.exception.{AMErrorCode, AMErrorException} import org.apache.linkis.manager.am.manager.EMNodeManager -import org.apache.linkis.manager.am.vo.EMNodeVo -import org.apache.linkis.manager.common.entity.enumeration.NodeHealthy import org.apache.linkis.manager.common.entity.node.{AMEMNode, EMNode} import org.apache.linkis.manager.common.protocol.em.GetEMInfoRequest -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext import org.apache.linkis.manager.label.entity.node.AliasServiceInstanceLabel import org.apache.linkis.manager.label.service.NodeLabelService import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence @@ -40,6 +34,7 @@ import org.springframework.stereotype.Service import scala.collection.JavaConversions._ + @Service class DefaultEMInfoService extends EMInfoService with Logging { @@ -81,9 +76,11 @@ class DefaultEMInfoService extends EMInfoService with Logging { } override def getEM(serviceInstance: ServiceInstance): EMNode = { - if(serviceInstance != null){ + if (serviceInstance != null) { emNodeManager.getEM(serviceInstance) - }else null + } else { + null + } } /** diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/ECMOperateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/ECMOperateService.scala new file mode 100644 index 0000000000..2d4ab1ebbf --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/ECMOperateService.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.service.em + +import org.apache.linkis.manager.common.entity.node.EMNode +import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} + + +trait ECMOperateService { + + def executeOperation(ecmNode: EMNode, request: ECMOperateRequest): ECMOperateResponse + +} diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/OperateRequest.scala b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/OperateRequest.scala new file mode 100644 index 0000000000..a718dcb6a1 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/OperateRequest.scala @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.common.protocol + +import org.apache.linkis.governance.common.exception.GovernanceErrorException + + +trait OperateRequest { + val user: String + val parameters: java.util.Map[String, Object] +} + +object OperateRequest { + + val OPERATOR_NAME_KEY = "__operator_name__" + + def getOperationName(parameters: Map[String, Any]): String = + parameters.getOrElse(OperateRequest.OPERATOR_NAME_KEY, + throw new GovernanceErrorException(20031, s"$OPERATOR_NAME_KEY is not exists.")).asInstanceOf[String] + + def getOperationName(parameters: java.util.Map[String, Any]): String = parameters.get(OperateRequest.OPERATOR_NAME_KEY) match { + case v: String => v + case _ => throw new GovernanceErrorException(20031, s"$OPERATOR_NAME_KEY is not exists.") + } +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/OperateResponse.scala b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/OperateResponse.scala new file mode 100644 index 0000000000..d95d3fda54 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/OperateResponse.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.common.protocol + + +trait OperateResponse { + val result: Map[String, Any] + def getResult: java.util.Map[String, Any] + val isError: Boolean + val errorMsg: String +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/ECMOperateRequest.scala b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/ECMOperateRequest.scala new file mode 100644 index 0000000000..da033a6f57 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/ECMOperateRequest.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.common.protocol.em + +import java.util + +import org.apache.linkis.manager.common.protocol.OperateRequest +import org.apache.linkis.protocol.message.RequestProtocol + +import scala.collection.JavaConverters.mapAsJavaMapConverter + + +case class ECMOperateRequest(user: String, + parameters: util.Map[String, Object]) extends OperateRequest with RequestProtocol { + def this(user: String, parameters: Map[String, Any]) = this(user, new util.HashMap[String, Object](parameters.map{case (k, v: Object) => (k, v)}.asJava)) +} + +object ECMOperateRequest { + val ENGINE_CONN_INSTANCE_KEY = "__engine_conn_instance__" +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/ECMOperateResponse.scala b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/ECMOperateResponse.scala new file mode 100644 index 0000000000..6bef1e1ee3 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/ECMOperateResponse.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.common.protocol.em + +import org.apache.linkis.manager.common.protocol.OperateResponse +import org.apache.linkis.protocol.message.RequestProtocol + +import scala.collection.JavaConverters.mapAsJavaMapConverter + + +case class ECMOperateResponse(result: Map[String, Any], + isError: Boolean = false, + errorMsg: String = "") extends OperateResponse with RequestProtocol { + override def getResult: java.util.Map[String, Any] = result.asJava +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateRequest.scala b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateRequest.scala index 84186391c3..f31bc868c1 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateRequest.scala +++ b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateRequest.scala @@ -5,30 +5,26 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.apache.linkis.manager.common.protocol.engine - -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.protocol.message.RequestProtocol - -case class EngineOperateRequest(user: String, - parameters: Map[String, Any]) extends RequestProtocol +package org.apache.linkis.manager.common.protocol.engine -case class EngineOperateRequestWithService(user: String, serviceInstance: ServiceInstance, - parameters: Map[String, Any]) extends RequestProtocol +import java.util -object EngineOperateRequest { +import org.apache.linkis.manager.common.protocol.OperateRequest +import org.apache.linkis.protocol.message.RequestProtocol - val OPERATOR_NAME_KEY = "__operator_name__" +import scala.collection.JavaConverters.mapAsJavaMapConverter +case class EngineOperateRequest(user: String, + parameters: java.util.Map[String, Object]) extends OperateRequest with RequestProtocol { + def this(user: String, parameters: Map[String, Any]) = this(user, new util.HashMap[String, Object](parameters.map{case (k, v: Object) => (k, v)}.asJava)) } \ No newline at end of file diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateResponse.scala b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateResponse.scala index 967daabe83..169469b29c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateResponse.scala +++ b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateResponse.scala @@ -5,18 +5,26 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - + package org.apache.linkis.manager.common.protocol.engine +import org.apache.linkis.manager.common.protocol.OperateResponse import org.apache.linkis.protocol.message.RequestProtocol -case class EngineOperateResponse(result: Map[String, Any], isError: Boolean = false, errorMsg: String = "") extends RequestProtocol \ No newline at end of file +import scala.collection.JavaConverters.mapAsJavaMapConverter + + +case class EngineOperateResponse(result: Map[String, Any], + isError: Boolean = false, + errorMsg: String = "") extends OperateResponse with RequestProtocol { + override def getResult: java.util.Map[String, Any] = result.asJava +} diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-service-common/src/main/scala/org/apache/linkis/manager/service/common/pointer/EMNodPointer.scala b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-service-common/src/main/scala/org/apache/linkis/manager/service/common/pointer/EMNodPointer.scala index 91e35efe69..03a55f773e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-service-common/src/main/scala/org/apache/linkis/manager/service/common/pointer/EMNodPointer.scala +++ b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-service-common/src/main/scala/org/apache/linkis/manager/service/common/pointer/EMNodPointer.scala @@ -18,6 +18,7 @@ package org.apache.linkis.manager.service.common.pointer import org.apache.linkis.manager.common.entity.node.EngineNode +import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest @@ -28,4 +29,6 @@ trait EMNodPointer extends NodePointer { def stopEngine(engineStopRequest: EngineStopRequest): Unit + def executeOperation(request: ECMOperateRequest): ECMOperateResponse + } From e5bec148c2b1aba825b65ae90caf69c5604f1158 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 15:21:15 +0800 Subject: [PATCH 10/17] Optimize the logic of EC creation, and fill in more EC information when EC is creating. --- .../manager/am/restful/EngineRestfulApi.java | 53 ++++++++----------- .../engine/DefaultEngineCreateService.scala | 1 + .../common/entity/node/AMEngineNode.java | 12 +++++ .../common/entity/node/EngineNode.java | 2 + 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java index ac106de1ca..95ace8314e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java @@ -18,7 +18,11 @@ package org.apache.linkis.manager.am.restful; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.common.utils.ByteTimeUtils; import org.apache.linkis.manager.am.conf.AMConfiguration; @@ -47,30 +51,18 @@ import org.apache.linkis.resourcemanager.utils.RMUtils; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.util.*; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; -import javax.servlet.http.HttpServletRequest; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.exception.ExceptionUtils; -import com.fasterxml.jackson.databind.JsonNode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import scala.Predef; -import scala.Tuple2; -import scala.collection.JavaConverters; @RequestMapping(path = "/linkisManager", produces = {"application/json"}) @RestController @@ -128,7 +120,13 @@ public Message createEngineConn( HttpServletRequest req, @RequestBody JsonNode j //to transform to a map Map retEngineNode = new HashMap<>(); retEngineNode.put("serviceInstance", engineNode.getServiceInstance()); - retEngineNode.put("nodeStatus", engineNode.getNodeStatus().toString()); + if (null == engineNode.getNodeStatus()) { + engineNode.setNodeStatus(NodeStatus.Starting); + } else { + retEngineNode.put("nodeStatus", engineNode.getNodeStatus().toString()); + } + retEngineNode.put("ticketId", engineNode.getTicketId()); + retEngineNode.put("ecmServiceInstance", engineNode.getEMNode().getServiceInstance()); return Message.ok("create engineConn succeed.").data("engine", retEngineNode); } @@ -250,28 +248,19 @@ public Message executeEngineConnOperation(HttpServletRequest req, @RequestBody J Map parameters = objectMapper.convertValue(jsonNode.get("parameters") , new TypeReference>(){}); - EngineOperateRequest engineOperateRequest = new EngineOperateRequest(userName - , JavaConverters.mapAsScalaMapConverter(parameters).asScala().toMap(Predef.conforms())); - + EngineOperateRequest engineOperateRequest = new EngineOperateRequest(userName, parameters); EngineOperateResponse engineOperateResponse = engineOperateService.executeOperation(engineNode, engineOperateRequest); - - Map result = new HashMap<>(0); - if (engineOperateResponse != null && engineOperateResponse.result() != null) { - result = JavaConverters.mapAsJavaMapConverter(engineOperateResponse.result()).asJava(); - } - return Message.ok() - .data("result", result) + .data("result", engineOperateResponse.getResult()) .data("errorMsg", engineOperateResponse.errorMsg()) .data("isError", engineOperateResponse.isError()); } private boolean isAdmin(String user) { - String[] adminArray = AMConfiguration.GOVERNANCE_STATION_ADMIN().getValue().split(","); - return ArrayUtils.contains(adminArray, user); + return AMConfiguration.isAdmin(user); } - private ServiceInstance getServiceInstance(JsonNode jsonNode) throws AMErrorException { + static ServiceInstance getServiceInstance(JsonNode jsonNode) throws AMErrorException { String applicationName = jsonNode.get("applicationName").asText(); String instance = jsonNode.get("instance").asText(); if(StringUtils.isEmpty(applicationName)){ diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala index 9e057e5bb2..9e27e96600 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala @@ -171,6 +171,7 @@ class DefaultEngineCreateService extends AbstractEngineService with EngineCreate //6. 调用EM发送引擎启动请求调用ASK TODO 异常和等待时间处理 val engineNode = getEMService().createEngine(engineBuildRequest, emNode) info(s"Finished to create engineConn $engineNode. ticketId is $resourceTicketId") + engineNode.setTicketId(resourceTicketId); //7. 更新持久化信息：包括插入engine/metrics //AM会更新serviceInstance表 需要将ticketID进行替换,并更新 EngineConn的Label 需要修改EngineInstanceLabel 中的id为Instance信息 val oldServiceInstance = new ServiceInstance diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEngineNode.java b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEngineNode.java index e1254017a2..2bb15622f6 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEngineNode.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEngineNode.java @@ -58,6 +58,8 @@ public class AMEngineNode implements EngineNode, ScoreServiceInstance { private Date startTime; + private String ticketId; + public AMEngineNode() { } @@ -189,6 +191,16 @@ public void setLock(String lock) { this.lock = lock; } + @Override + public String getTicketId() { + return ticketId; + } + + @Override + public void setTicketId(String ticketId) { + this.ticketId = ticketId; + } + @Override public Date getStartTime() { return startTime; diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/EngineNode.java b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/EngineNode.java index 8dd1760faa..fc3ad25bda 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/EngineNode.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/EngineNode.java @@ -29,7 +29,9 @@ public interface EngineNode extends AMNode, RMNode, LabelNode { void setLock(String lock); + String getTicketId(); + void setTicketId(String ticketId); } From 8314db42ea54f07cf47a277a18048de472b222d5 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 15:23:30 +0800 Subject: [PATCH 11/17] Delete the EngineConnLogOperator in accessible-executor, since some errors will be throws when the EC is starting or stopped. The EC log fetch feature has been moved to ECM. --- .../operator/impl/EngineConnLogOperator.scala | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnLogOperator.scala diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnLogOperator.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnLogOperator.scala deleted file mode 100644 index 2556bfae53..0000000000 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnLogOperator.scala +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineconn.acessible.executor.operator.impl - -import java.io.{File, RandomAccessFile} -import java.util - -import org.apache.commons.io.IOUtils -import org.apache.commons.lang.StringUtils -import org.apache.linkis.common.conf.CommonVars -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.engineconn.acessible.executor.operator.Operator -import org.apache.linkis.engineconn.common.conf.EngineConnConf -import org.apache.linkis.engineconn.common.exception.EngineConnException - -class EngineConnLogOperator extends Operator with Logging { - - private val logPath = new File(EngineConnConf.getLogDir, EngineConnLogOperator.LOG_FILE_NAME.getValue) - info(s"Try to fetch EngineConn logs from ${logPath.getPath}.") - - override def getNames: Array[String] = Array(EngineConnLogOperator.OPERATOR_NAME) - - override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { - val lastRows = getAs("lastRows", 0) - if(lastRows > EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue) - throw EngineConnException(20305, s"Cannot fetch more than ${EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue} lines of logs.") - else if(lastRows > 0) { - val logs = Utils.exec(Array("tail", "-f", logPath.getPath), 5000).split("\n") - return Map("logs" -> logs, "rows" -> logs.length) - } - val pageSize = getAs("pageSize", 100) - val fromLine = getAs("fromLine", 1) - val ignoreKeywords = getAs("ignoreKeywords", "") - val ignoreKeywordList = if(StringUtils.isNotEmpty(ignoreKeywords)) ignoreKeywords.split(",") else Array.empty[String] - val onlyKeywords = getAs("onlyKeywords", "") - val onlyKeywordList = if(StringUtils.isNotEmpty(onlyKeywords)) onlyKeywords.split(",") else Array.empty[String] - val reader = new RandomAccessFile(logPath, "r") - val logs = new util.ArrayList[String](pageSize) - var readLine, skippedLine = 0 - Utils.tryFinally { - var line = reader.readLine() - while (readLine < pageSize && line != null) { - if(skippedLine < fromLine) { - skippedLine += 1 - } else if(onlyKeywordList.nonEmpty && onlyKeywordList.exists(line.contains)) { - logs.add(line) - readLine += 1 - } else if(ignoreKeywordList.nonEmpty && !ignoreKeywordList.exists(line.contains)) { - logs.add(line) - readLine += 1 - } else if(onlyKeywordList.isEmpty && ignoreKeywordList.isEmpty) { - logs.add(line) - readLine += 1 - } - line = reader.readLine - } - }(IOUtils.closeQuietly(reader)) - Map("logPath" -> logPath.getPath, "logs" -> logs, "rows" -> readLine) - } -} -object EngineConnLogOperator { - val OPERATOR_NAME = "engineConnLog" - val LOG_FILE_NAME = CommonVars("linkis.engineconn.log.filename", "stdout") - val MAX_LOG_FETCH_SIZE = CommonVars("linkis.engineconn.log.fetch.lines.max", 5000) -} \ No newline at end of file From 9b277404190c07ef3c083d39d2cdd8440e06f5f9 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 15:24:43 +0800 Subject: [PATCH 12/17] Optimize the logic of EC creation, and fill in more EC information when EC is creating. --- .../client/job/AbstractLinkisJob.scala | 1 + .../computation/client/once/OnceJob.scala | 12 ++++++-- .../once/action/EngineConnOperateAction.scala | 7 +++-- .../once/result/EngineConnOperateResult.scala | 6 ++-- .../client/once/simple/SimpleOnceJob.scala | 26 +++++++++++----- .../client/operator/OnceJobOperator.scala | 30 +++++++++++++------ .../client/operator/Operator.scala | 4 +++ 7 files changed, 62 insertions(+), 24 deletions(-) diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/job/AbstractLinkisJob.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/job/AbstractLinkisJob.scala index 230fe15fe2..aec1d71da7 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/job/AbstractLinkisJob.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/job/AbstractLinkisJob.scala @@ -77,6 +77,7 @@ trait AbstractLinkisJob extends LinkisJob with Logging { override final def getOperator(operatorName: String): Operator[_] = { var operator = OperatorFactory().createOperatorByName(operatorName) operatorActions.foreach(action => operator = action(operator)) + operator.initOperator(this) operator // operatorActions.foldLeft(operator)((operator, action) => action(operator)) } diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/OnceJob.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/OnceJob.scala index b474842e3c..ee62f398ef 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/OnceJob.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/OnceJob.scala @@ -32,6 +32,7 @@ trait OnceJob extends AbstractLinkisJob { protected var engineConnId: String = _ protected var serviceInstance: ServiceInstance = _ + protected var ticketId: String = _ protected def wrapperEC[T](op: => T): T = wrapperObj(serviceInstance, "Please submit job first.")(op) @@ -45,15 +46,20 @@ trait OnceJob extends AbstractLinkisJob { .setInstance(serviceInstance.getInstance).setUser(user).build()).getNodeInfo } - protected def getStatus(nodeInfo: util.Map[String, Any]): String = nodeInfo.get("nodeStatus") match { - case status: String => status - } + protected def getTicketId(nodeInfo: util.Map[String, Any]): String = getAs(nodeInfo, "ticketId") + + protected def getStatus(nodeInfo: util.Map[String, Any]): String = getAs(nodeInfo, "nodeStatus") protected def getServiceInstance(nodeInfo: util.Map[String, Any]): ServiceInstance = nodeInfo.get("serviceInstance") match { case serviceInstance: util.Map[String, Any] => ServiceInstance(getAs(serviceInstance, "applicationName"), getAs(serviceInstance, "instance")) } + protected def getECMServiceInstance(nodeInfo: util.Map[String, Any]): ServiceInstance = nodeInfo.get("ecmServiceInstance") match { + case serviceInstance: util.Map[String, Any] => + ServiceInstance(getAs(serviceInstance, "applicationName"), getAs(serviceInstance, "instance")) + } + protected def getAs[T](map: util.Map[String, Any], key: String): T = map.get(key).asInstanceOf[T] } diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/action/EngineConnOperateAction.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/action/EngineConnOperateAction.scala index cfd528ec20..23813a7397 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/action/EngineConnOperateAction.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/action/EngineConnOperateAction.scala @@ -37,7 +37,7 @@ object EngineConnOperateAction { private var parameters: util.Map[String, Any] = new util.HashMap[String, Any] - def operatorName(operatorName: String): this.type = { + def operatorName(operatorName: String): this.type = { this.operatorName = operatorName this } @@ -55,8 +55,11 @@ object EngineConnOperateAction { this } + protected def newEngineConnOperateAction(): EngineConnOperateAction = + new EngineConnOperateAction + override protected def createGetEngineConnAction(): EngineConnOperateAction = { - val action = new EngineConnOperateAction + val action = newEngineConnOperateAction() addParameter(OPERATOR_NAME_KEY, operatorName) action.addRequestPayload("parameters", parameters) action diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala index 6ce48c427e..0cdecb4aa1 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala @@ -43,7 +43,9 @@ class EngineConnOperateResult extends LinkisManagerResult { def getAsOption[T](key: String): Option[T] = Option(getAs(key)) - def getAs[T](key: String): T = if(getResult != null && result.containsKey(key)) result.get(key).asInstanceOf[T] - else null.asInstanceOf[T] + def getAs[T](key: String): T = getAs(key, null.asInstanceOf[T]) + + def getAs[T](key: String, defaultValue: T): T = if(getResult != null && result.containsKey(key)) result.get(key).asInstanceOf[T] + else defaultValue } diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJob.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJob.scala index 606904f4db..0b3992f9d3 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJob.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJob.scala @@ -68,31 +68,41 @@ trait SimpleOnceJob extends OnceJob { } protected def transformToId(): Unit = { - engineConnId = serviceInstance.getApplicationName.length + serviceInstance.getApplicationName + serviceInstance.getInstance + engineConnId = ticketId.length + "_" + serviceInstance.getApplicationName.length + + ticketId + serviceInstance.getApplicationName + serviceInstance.getInstance } protected def transformToServiceInstance(): Unit = engineConnId match { - case SimpleOnceJob.ENGINE_CONN_ID_REGEX(len, serviceInstanceStr) => - val length = len.toInt - serviceInstance = ServiceInstance(serviceInstanceStr.substring(0, length), serviceInstanceStr.substring(length)) + case SimpleOnceJob.ENGINE_CONN_ID_REGEX(ticketIdLen, appNameLen, serviceInstanceStr) => + val index1 = ticketIdLen.toInt + val index2 = index1 + appNameLen.toInt + ticketId = serviceInstanceStr.substring(0, index1) + serviceInstance = ServiceInstance(serviceInstanceStr.substring(index1, index2), serviceInstanceStr.substring(index2)) } protected def initOnceOperatorActions(): Unit = addOperatorAction { case onceJobOperator: OnceJobOperator[_] => - onceJobOperator.setUser(user).setServiceInstance(serviceInstance).setLinkisManagerClient(linkisManagerClient) + onceJobOperator.setUser(user).setTicketId(ticketId).setServiceInstance(serviceInstance).setLinkisManagerClient(linkisManagerClient) case operator => operator } } class SubmittableSimpleOnceJob(protected override val linkisManagerClient: LinkisManagerClient, - createEngineConnAction: CreateEngineConnAction) + val createEngineConnAction: CreateEngineConnAction) extends SimpleOnceJob with SubmittableOnceJob with AbstractSubmittableLinkisJob { + + private var ecmServiceInstance: ServiceInstance = _ + + def getECMServiceInstance: ServiceInstance = ecmServiceInstance + override protected def doSubmit(): Unit = { info(s"Ready to create a engineConn: ${createEngineConnAction.getRequestPayload}.") val nodeInfo = linkisManagerClient.createEngineConn(createEngineConnAction) lastNodeInfo = nodeInfo.getNodeInfo serviceInstance = getServiceInstance(lastNodeInfo) + ticketId = getTicketId(lastNodeInfo) + ecmServiceInstance = getECMServiceInstance(lastNodeInfo) lastEngineConnState = getStatus(lastNodeInfo) info(s"EngineConn created with status $lastEngineConnState, the nodeInfo is $lastNodeInfo.") initOnceOperatorActions() @@ -102,7 +112,7 @@ class SubmittableSimpleOnceJob(protected override val linkisManagerClient: Linki serviceInstance = getServiceInstance(lastNodeInfo) info(s"EngineConn of $serviceInstance is in $lastEngineConnState.") transformToId() - }else{ + } else { info(s"EngineConn $serviceInstance is aleady running, transform to id") transformToId() } @@ -122,7 +132,7 @@ class ExistingSimpleOnceJob(protected override val linkisManagerClient: LinkisMa object SimpleOnceJob { - private val ENGINE_CONN_ID_REGEX = "(\\d+)(.+)".r + private val ENGINE_CONN_ID_REGEX = "(\\d+)_(\\d+)(.+)".r def builder(): SimpleOnceJobBuilder = new SimpleOnceJobBuilder diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/OnceJobOperator.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/OnceJobOperator.scala index 4b08545e22..406fc43596 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/OnceJobOperator.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/OnceJobOperator.scala @@ -5,16 +5,16 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - + package org.apache.linkis.computation.client.operator import org.apache.linkis.common.ServiceInstance @@ -29,10 +29,12 @@ trait OnceJobOperator[T] extends Operator[T] with Logging { private var user: String = _ private var serviceInstance: ServiceInstance = _ + private var ticketId: String = _ private var linkisManagerClient: LinkisManagerClient = _ protected def getUser: String = user protected def getServiceInstance: ServiceInstance = serviceInstance + protected def getTicketId: String = ticketId protected def getLinkisManagerClient: LinkisManagerClient = linkisManagerClient def setUser(user: String): this.type = { @@ -45,25 +47,35 @@ trait OnceJobOperator[T] extends Operator[T] with Logging { this } + def setTicketId(ticketId: String): this.type = { + this.ticketId = ticketId + this + } + def setLinkisManagerClient(linkisManagerClient: LinkisManagerClient): this.type = { this.linkisManagerClient = linkisManagerClient this } + protected def createOperateActionBuilder(): EngineConnOperateAction.Builder = + EngineConnOperateAction.newBuilder() + override def apply(): T = { - val builder = EngineConnOperateAction.newBuilder() + val builder = createOperateActionBuilder() .operatorName(getName) .setUser(user) .setApplicationName(serviceInstance.getApplicationName) .setInstance(serviceInstance.getInstance) addParameters(builder) val engineConnOperateAction = builder.build() - if(OnceJobOperator.ONCE_JOB_OPERATOR_LOG_ENABLE.getValue) + if (OnceJobOperator.ONCE_JOB_OPERATOR_LOG_ENABLE.getValue) { info(s"$getUser try to ask EngineConn($serviceInstance) to execute $getName operation, parameters is ${engineConnOperateAction.getRequestPayload}.") + } val result = linkisManagerClient.executeEngineConnOperation(engineConnOperateAction) - val resultStr = String.valueOf(result.getResult) - if(OnceJobOperator.ONCE_JOB_OPERATOR_LOG_ENABLE.getValue) - info(s"$getUser asked EngineConn($serviceInstance) to execute $getName operation, results is ${if(resultStr.length <= 250) resultStr else resultStr.substring(0, 250) + "..."} .") + if (OnceJobOperator.ONCE_JOB_OPERATOR_LOG_ENABLE.getValue) { + val resultStr = String.valueOf(result.getResult) + info(s"$getUser asked EngineConn($serviceInstance) to execute $getName operation, results is ${if (resultStr.length <= 250) resultStr else resultStr.substring(0, 250) + "..."} .") + } resultToObject(result) } @@ -77,4 +89,4 @@ object OnceJobOperator { val ONCE_JOB_OPERATOR_LOG_ENABLE = CommonVars("linkis.client.operator.once.log.enable", true) -} \ No newline at end of file +} diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/Operator.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/Operator.scala index c7687b81c8..1ba97ff184 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/Operator.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/Operator.scala @@ -17,6 +17,8 @@ package org.apache.linkis.computation.client.operator +import org.apache.linkis.computation.client.LinkisJob + trait Operator[T] { @@ -24,4 +26,6 @@ trait Operator[T] { def apply(): T + def initOperator[U <: LinkisJob](job: U): Unit = {} + } \ No newline at end of file From 736520d5bab0b703df2421db1baf42b6ab3308c5 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 15:25:55 +0800 Subject: [PATCH 13/17] Add the capability of executing Operators to ECM in linkis-computation-client. --- .../client/once/action/ECMOperateAction.scala | 47 +++++++++++++++++++ .../once/result/ECMOperateResult.scala} | 30 +++++------- 2 files changed, 60 insertions(+), 17 deletions(-) create mode 100644 linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/action/ECMOperateAction.scala rename linkis-computation-governance/{linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/Operator.scala => linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/ECMOperateResult.scala} (55%) diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/action/ECMOperateAction.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/action/ECMOperateAction.scala new file mode 100644 index 0000000000..ce94786dac --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/action/ECMOperateAction.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.computation.client.once.action + +class ECMOperateAction extends EngineConnOperateAction { + + override def suffixURLs: Array[String] = Array("linkisManager", "executeECMOperation") + +} + +object ECMOperateAction { + + def newBuilder(): Builder = new Builder + + class Builder extends EngineConnOperateAction.Builder { + override protected def newEngineConnOperateAction(): EngineConnOperateAction = + new ECMOperateAction + } +} + +class ECMOperateByECAction extends ECMOperateAction { + override def suffixURLs: Array[String] = Array("linkisManager", "executeECMOperationByEC") +} + +object ECMOperateByECAction { + def newBuilder(): Builder = new Builder + + class Builder extends EngineConnOperateAction.Builder { + override protected def newEngineConnOperateAction(): EngineConnOperateAction = + new ECMOperateByECAction + } +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/Operator.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/ECMOperateResult.scala similarity index 55% rename from linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/Operator.scala rename to linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/ECMOperateResult.scala index b913238113..bd64397682 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/Operator.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/ECMOperateResult.scala @@ -5,31 +5,27 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.apache.linkis.engineconn.acessible.executor.operator -import org.apache.linkis.engineconn.common.exception.EngineConnException +package org.apache.linkis.computation.client.once.result -trait Operator { +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult - def getNames: Array[String] +/** + * Created by enjoyyin on 2021/12/18. + */ +@DWSHttpMessageResult("/api/rest_j/v\\d+/linkisManager/executeECMOperation") +class ECMOperateResult extends EngineConnOperateResult { - def apply(implicit parameters: Map[String, Any]): Map[String, Any] +} - protected def getAs[T](key: String, defaultVal: T)(implicit parameters: Map[String, Any]): T = - parameters.getOrElse(key, defaultVal) match { - case t: T => t - case null => null.asInstanceOf[T] - case v => throw EngineConnException(20305, s"Unknown $v for key $key.") - } - -} \ No newline at end of file +@DWSHttpMessageResult("/api/rest_j/v\\d+/linkisManager/executeECMOperationByEC") +class ECMOperateByECResult extends EngineConnOperateResult {} \ No newline at end of file From a1fefc289d721f193ad9d68de5b436b8665b85b7 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 15:27:28 +0800 Subject: [PATCH 14/17] Add the capability of fetching EC logs, this operator will ask LinkisManager to redirect log operator to ECM to fetch EC logs. --- .../operator/impl/EngineConnLogOperator.scala | 64 ++++++++++++++++--- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnLogOperator.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnLogOperator.scala index 738d37dcc8..f6a3047cd8 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnLogOperator.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnLogOperator.scala @@ -20,10 +20,15 @@ package org.apache.linkis.computation.client.operator.impl import java.util -import org.apache.commons.lang.StringUtils -import org.apache.linkis.computation.client.once.action.EngineConnOperateAction +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.computation.client.LinkisJob +import org.apache.linkis.computation.client.once.action.{ECMOperateAction, EngineConnOperateAction} import org.apache.linkis.computation.client.once.result.EngineConnOperateResult +import org.apache.linkis.computation.client.once.simple.SubmittableSimpleOnceJob import org.apache.linkis.computation.client.operator.OnceJobOperator +import org.apache.linkis.computation.client.utils.LabelKeyUtils +import org.apache.linkis.ujes.client.exception.UJESJobException +import org.apache.commons.lang.StringUtils class EngineConnLogOperator extends OnceJobOperator[EngineConnLogs] { @@ -32,7 +37,14 @@ class EngineConnLogOperator extends OnceJobOperator[EngineConnLogs] { private var fromLine = 1 private var ignoreKeywords: String = _ private var onlyKeywords: String = _ + private var engineConnType: String = _ private var lastRows = 0 + private var ecmServiceInstance: ServiceInstance = _ + private var ecInstance: String = _ + + def setEngineConnType(engineConnType: String): Unit = this.engineConnType = engineConnType + + def setECMServiceInstance(serviceInstance: ServiceInstance): Unit = ecmServiceInstance = serviceInstance def setPageSize(pageSize: Int): Unit = this.pageSize = pageSize @@ -44,21 +56,55 @@ class EngineConnLogOperator extends OnceJobOperator[EngineConnLogs] { def setLastRows(lastRows: Int): Unit = this.lastRows = lastRows + /** + * Try to fetch logs from ECM. + * @return + */ + override protected def createOperateActionBuilder(): EngineConnOperateAction.Builder = { + if (ecInstance == null) { + ecInstance = getServiceInstance.getInstance + if (ecmServiceInstance == null) { + throw new UJESJobException(20310, "ecmServiceInstance must be set!") + } + setServiceInstance(ecmServiceInstance) + } + ECMOperateAction.newBuilder() + } + + + override def initOperator[U <: LinkisJob](job: U): Unit = job match { + case submittableSimpleOnceJob: SubmittableSimpleOnceJob => + this.ecmServiceInstance = submittableSimpleOnceJob.getECMServiceInstance + this.engineConnType = submittableSimpleOnceJob.createEngineConnAction.getRequestPayloads.get("labels") match { + case labels: util.Map[String, String] => labels.get(LabelKeyUtils.ENGINE_TYPE_LABEL_KEY) + } + case _ => + } + override protected def addParameters(builder: EngineConnOperateAction.Builder): Unit = { builder.addParameter("pageSize", pageSize) builder.addParameter("fromLine", fromLine) - if(StringUtils.isNotEmpty(ignoreKeywords)) + builder.addParameter("ticketId", getTicketId) + builder.addParameter("creator", getUser) + builder.addParameter("engineConnInstance", ecInstance) + if (StringUtils.isNotEmpty(engineConnType)) { + builder.addParameter("engineConnType", engineConnType) + } + if (StringUtils.isNotEmpty(ignoreKeywords)) { builder.addParameter("ignoreKeywords", ignoreKeywords) - if(StringUtils.isNotEmpty(onlyKeywords)) + } + if (StringUtils.isNotEmpty(onlyKeywords)) { builder.addParameter("onlyKeywords", onlyKeywords) - if(lastRows > 0) + } + if (lastRows > 0) { builder.addParameter("lastRows", lastRows) + } } override protected def resultToObject(result: EngineConnOperateResult): EngineConnLogs = { - val rows: Int = result.getAs("rows") - fromLine += rows - EngineConnLogs(result.getAs("logPath"), result.getAs("logs"), rows) + val endLine: Int = result.getAs("endLine", 0) + fromLine = endLine + 1 + EngineConnLogs(result.getAs("logPath"), result.getAs("logs"), endLine) } override def getName: String = EngineConnLogOperator.OPERATOR_NAME @@ -69,4 +115,4 @@ object EngineConnLogOperator { val OPERATOR_NAME = "engineConnLog" } -case class EngineConnLogs(logPath: String, logs: util.ArrayList[String], rows: Int) \ No newline at end of file +case class EngineConnLogs(logPath: String, logs: util.ArrayList[String], endLine: Int) \ No newline at end of file From b8e7ba035e237924c49dc3f4efcb742145bd8ee7 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 20:24:10 +0800 Subject: [PATCH 15/17] Fixes some bugs to help compilation. --- .../apache/linkis/computation/client/FlinkOnceJobTest.java | 2 +- .../linkis/manager/am/service/em/DefaultEMInfoService.scala | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/FlinkOnceJobTest.java b/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/FlinkOnceJobTest.java index 491f97842f..0850c84d09 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/FlinkOnceJobTest.java +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/FlinkOnceJobTest.java @@ -72,7 +72,7 @@ public static void main(String[] args) { EngineConnMetricsOperator metricsOperator = (EngineConnMetricsOperator) onceJob.getOperator(EngineConnMetricsOperator.OPERATOR_NAME()); logOperator.setFromLine(1); logOperator.setPageSize(100); - EngineConnLogs logs = logOperator.apply(); + EngineConnLogs logs = (EngineConnLogs) logOperator.apply(); System.out.println("logPath: " + logs.logPath() + ", logs: " + logs.logs()); System.out.println("Metrics: " + metricsOperator.apply()); onceJob.waitForCompleted(); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala index 21213d3f7d..d9e0b5e7c0 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala @@ -28,7 +28,6 @@ import org.apache.linkis.manager.label.entity.node.AliasServiceInstanceLabel import org.apache.linkis.manager.label.service.NodeLabelService import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence import org.apache.linkis.manager.service.common.metrics.MetricsConverter -import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder import org.apache.linkis.message.annotation.Receiver import org.apache.linkis.resourcemanager.service.ResourceManager import org.springframework.beans.factory.annotation.Autowired @@ -53,9 +52,6 @@ class DefaultEMInfoService extends EMInfoService with Logging { @Autowired private var nodeMetricManagerPersistence: NodeMetricManagerPersistence = _ - @Autowired - private var nodePointerBuilder: NodePointerBuilder = _ - @Autowired private var defaultMetricsConverter: MetricsConverter = _ From 386a40b98aad07b789d3d219536a637a50079404 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 21:18:06 +0800 Subject: [PATCH 16/17] Remove author information and WeBank copyright. --- .../common/errorcode/LinkisErrorCodeSummary.java | 2 +- .../linkis/protocol/utils/ZuulEntranceUtils.scala | 2 +- .../linkis/rpc/exception/DWCURIException.scala | 3 --- .../org/apache/linkis/scheduler/queue/Job.scala | 1 - .../apache/linkis/scheduler/queue/JobInfo.scala | 1 - .../org/apache/linkis/storage/io/IOClient.scala | 1 - .../linkis/storage/script/VariableParser.scala | 2 +- .../application/constants/LinkisClientKeys.java | 2 +- .../client/once/result/ECMOperateResult.scala | 4 +--- .../governance/common/paser/CodeParser.scala | 2 +- .../linkis/ecm/server/exception/ECMErrorCode.scala | 1 - .../computation/executor/cs/CSResourceParser.scala | 2 +- .../executor/execute/EngineExecutionContext.scala | 6 +++--- .../entrance/conf/EntranceSpringConfiguration.java | 2 +- .../persistence/QueryPersistenceManager.java | 2 +- .../entrance/conf/EntranceConfiguration.scala | 2 +- .../entrance/execute/EntranceExecutorManager.scala | 2 +- .../org/apache/linkis/ujes/jdbc/JDBCSpiTest.java | 2 +- .../ujes/jdbc/UJESSQLPreparedStatementTest.java | 12 ++++++------ .../linkis/ujes/jdbc/UJESSQLStatementTest.java | 8 ++++---- .../src/test/java/PersistenceTest.java | 2 +- .../hive/creation/HiveEngineConnFactory.scala | 4 ++-- .../engineplugin/spark/cs/CSTableParser.scala | 2 +- .../engineplugin/spark/executor/SQLSession.scala | 2 +- .../org/apache/linkis/bml/common/Constant.java | 2 +- .../apache/linkis/bml/restful/BmlRestfulApi.java | 2 +- .../apache/linkis/bml/restful/RestfulUtils.java | 2 +- .../test/service/TestContextCacheService.java | 2 +- .../cs/client/test/bean/ClientTestContextKey.java | 2 +- .../test/ContextSerializationHelperTest.java | 14 +++++++------- .../apache/linkis/cs/persistence/AContextID.java | 2 +- .../linkis/cs/persistence/ContextIDTest.java | 2 +- .../linkis/cs/persistence/ContextMapTest.java | 4 ++-- .../linkis/cs/server/ContextIDRestfulApi.java | 4 ++-- .../apache/linkis/cs/server/ContextRestfulApi.java | 4 ++-- .../org/apache/linkis/cs/server/SchedulerTest.java | 2 +- .../server/src/main/resources/linkis.properties | 2 +- .../impl/JobHistoryDetailQueryServiceImpl.scala | 4 ++-- .../service/impl/JobHistoryQueryServiceImpl.scala | 2 +- .../filesystem/restful/api/FsRestfulApi.java | 5 +---- .../java/org/apache/linkis/udf/TestUDFTree.java | 2 +- .../eureka/SpringCloudEurekaApplication.java | 4 +--- .../linkis/eureka/conf/EurekaSecurityConfig.java | 5 +---- .../apache/linkis/gateway/http/GatewayRoute.scala | 4 +--- web/src/common/util/ras.js | 9 +++------ 45 files changed, 63 insertions(+), 85 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/errorcode/LinkisErrorCodeSummary.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/errorcode/LinkisErrorCodeSummary.java index c7657dab5c..32b42b925f 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/errorcode/LinkisErrorCodeSummary.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/errorcode/LinkisErrorCodeSummary.java @@ -30,7 +30,7 @@ public enum LinkisErrorCodeSummary { * 26000-29999 linkis-engineconn-plugins */ EngineManagerErrorException - (321, "引擎启动失败", "在某种情况下启动失败", "cooperyang", "EngineConnManager") + (321, "引擎启动失败", "在某种情况下启动失败", "hadoop", "EngineConnManager") ; /** * 错误码 diff --git a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala index 39f58a37e5..d19b15069f 100644 --- a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala +++ b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala @@ -70,7 +70,7 @@ object ZuulEntranceUtils { /** * - * @param shortExecID ExecID generated by the scheduler, such as IDE_neiljianliu_0(scheduler生成的ExecID, 如 IDE_neiljianliu_0) + * @param shortExecID ExecID generated by the scheduler, such as IDE_hadoop_0(scheduler生成的ExecID, 如 IDE_hadoop_0) * @param executeApplicationName {dd}{dd}${executeApplicationName}${instance}${shortExecID} * @return */ diff --git a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/exception/DWCURIException.scala b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/exception/DWCURIException.scala index 44ff62b89b..38ea2ea79d 100644 --- a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/exception/DWCURIException.scala +++ b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/exception/DWCURIException.scala @@ -19,7 +19,4 @@ package org.apache.linkis.rpc.exception import org.apache.linkis.common.exception.ErrorException -/** - * Created by enjoyyin on 2018/9/12. - */ private[rpc] class DWCURIException(errCode: Int, desc: String) extends ErrorException(errCode, desc) \ No newline at end of file diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Job.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Job.scala index 5951b146dd..d5c46821c3 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Job.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Job.scala @@ -122,7 +122,6 @@ abstract class Job extends Runnable with SchedulerEvent with Closeable with Logg } /** - * updated by enjoyyin * After some jobs call kill, they cannot be killed correctly, causing the state to not be flipped.（一些job调用kill之后，不能被正确kill，导致状态不能翻转） */ protected def forceCancel(t: Throwable): Unit = { diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/JobInfo.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/JobInfo.scala index 52d9c1beac..0c693b6441 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/JobInfo.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/JobInfo.scala @@ -18,7 +18,6 @@ package org.apache.linkis.scheduler.queue /** - * Created by enjoyyin on 2018/9/4. * JobInfo Is the job information that the server sends to the front end.（是服务端传给前端的job信息） * id: Job execution id（job执行id） * output: Result set（结果集） diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala index 0a5add2200..216b2feb31 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala @@ -24,7 +24,6 @@ import org.apache.linkis.storage.exception.StorageErrorException /** * IOClient is used to execute the proxy as the ujes code execution entry in io and get the return result. * IOClient用于在io进行代理作为ujes的代码执行入口，并获取返回结果 - * Created by johnnwang on 2018/11/2. */ trait IOClient { def execute(user: String, methodEntity: MethodEntity, params:java.util.Map[String,Any]):String diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala index b09069369a..0149467f70 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala @@ -48,7 +48,7 @@ object VariableParser { variables.toArray } - //TODO need delete 20200321 update by peaceWong + //TODO need delete def isContextIDINFO(key : String): Boolean ={ "contextID".equalsIgnoreCase(key) || "nodeName".equalsIgnoreCase(key) } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisClientKeys.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisClientKeys.java index 5410ba092b..39041c636f 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisClientKeys.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisClientKeys.java @@ -25,7 +25,7 @@ public class LinkisClientKeys { /** * User Not configurable */ - public static final String ADMIN_USERS = "hadoop,root,shangda"; + public static final String ADMIN_USERS = "hadoop,root"; public static final String LINKIS_CLIENT_NONCUSTOMIZABLE = "wds.linkis.client.noncustomizable"; public static final String LINKIS_CLIENT_NONCUSTOMIZABLE_ENABLE_USER_SPECIFICATION = LINKIS_CLIENT_NONCUSTOMIZABLE + ".enable.user.specification"; //allow user to specify submit user public static final String LINKIS_CLIENT_NONCUSTOMIZABLE_ENABLE_PROXY_USER = LINKIS_CLIENT_NONCUSTOMIZABLE + ".enable.proxy.user"; //allow user to specify proxy user diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/ECMOperateResult.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/ECMOperateResult.scala index bd64397682..50ac732e4c 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/ECMOperateResult.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/ECMOperateResult.scala @@ -19,9 +19,7 @@ package org.apache.linkis.computation.client.once.result import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult -/** - * Created by enjoyyin on 2021/12/18. - */ + @DWSHttpMessageResult("/api/rest_j/v\\d+/linkisManager/executeECMOperation") class ECMOperateResult extends EngineConnOperateResult { diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala index 51e8b90a6f..ff637d7af4 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala @@ -129,7 +129,7 @@ class PythonCodeParser extends SingleCodeParser { statementBuffer.append(l) recordBrackets(bracketStack, l) case l if quotationMarks => statementBuffer.append(l) - //shanhuang 用于修复python的引号问题 + //用于修复python的引号问题 //recordBrackets(bracketStack, l) case l if notDoc && l.startsWith("#") => case l if StringUtils.isNotBlank(statementBuffer.last) && statementBuffer.last.endsWith("""\""") => diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala index a25ca2ddee..6b5b4ddbda 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala @@ -1,5 +1,4 @@ /* - * Copyright 2019 WeBank * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala index 6f37091e92..820fbbde16 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala @@ -42,7 +42,7 @@ class CSResourceParser { def parse(props: util.Map[String, Object], code: String, contextIDValueStr: String, nodeNameStr: String): String = { - //TODO getBMLResource peaceWong + //TODO getBMLResource val bmlResourceList = CSResourceService.getInstance().getUpstreamBMLResource(contextIDValueStr, nodeNameStr) val parsedResources = new util.ArrayList[util.Map[String, Object]]() diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala index e5f6b5e79a..2fad052a0e 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala @@ -98,7 +98,7 @@ class EngineExecutionContext(executor: ComputationExecutor, executorUser: String if (resultSetFactory.isResultSetPath(output) || resultSetFactory.isResultSet(output)) { val listenerBus = getEngineSyncListenerBus getJobId.foreach(jId => { - //TODO peacewong executor.getEngineServerContext().getEngineAsyncListenerBus().post(ResultSetCreatedEvent(jId, output, alias)) + //TODO executor.getEngineServerContext().getEngineAsyncListenerBus().post(ResultSetCreatedEvent(jId, output, alias)) listenerBus.postToAll(TaskResultCreateEvent(jId, output, alias)) resultSize += 1 }) @@ -122,7 +122,7 @@ class EngineExecutionContext(executor: ComputationExecutor, executorUser: String def newResultSetWriter(resultSet: ResultSet[_ <: MetaData, _ <: Record], resultSetPath: FsPath, alias: String): ResultSetWriter[_ <: MetaData, _ <: Record] = { - //update by peaceWong 20200402 + //update by 20200402 resultSet match { case result: TableResultSet => val contextIDStr = ContextServiceUtils.getContextIDStrByMap(getProperties) @@ -136,7 +136,7 @@ class EngineExecutionContext(executor: ComputationExecutor, executorUser: String } case _ => ResultSetWriter.getResultSetWriter(resultSet, ComputationExecutorConf.ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath, executorUser) } - //update by peaceWong 20200402 end + //update by 20200402 end } def appendStdout(log: String): Unit = if (executor.isInternalExecute) { diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java index 8e2f7857a8..4664c7aad9 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java @@ -97,7 +97,7 @@ public EntranceEventListenerBus generateEn } /** - * Update by peaceWong add CSEntranceInterceptor + * add CSEntranceInterceptor * * @return */ diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java index 832188222e..8ef86c6c51 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java @@ -149,7 +149,7 @@ public void onJobWaitForRetry(Job job) { @Override public void onJobCompleted(Job job) { - //update by peaceWong to set jobID to CS + //to set jobID to CS try { if (job.isSucceed()) { CSEntranceHelper.registerCSRSData(job); diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index b496449215..de5e076b7a 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -93,7 +93,7 @@ object EntranceConfiguration { val ERROR_CODE_FILE = CommonVars("wds.linkis.errorcode.file", "") - //todo enjoyyin Simple processing first, this log is not filtered, and subsequent optimization is performed.(先简单处理 不过滤这个日志，后续进行优化) + //todo Simple processing first, this log is not filtered, and subsequent optimization is performed.(先简单处理 不过滤这个日志，后续进行优化) val HIVE_SPECIAL_LOG_INCLUDE = CommonVars("wds.linkis.hive.special.log.include", "org.apache.hadoop.hive.ql.exec.Task") val SHARE_FILE_PRE = CommonVars("wds.linkis.share.file.prefix", "") diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala index 50aeac7929..23b58553e8 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala @@ -109,7 +109,7 @@ abstract class EntranceExecutorManager(groupFactory: GroupFactory) extends Execu } override def getByGroup(groupName: String): Array[Executor] = { - //TODO by peaceWong + //TODO null } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java index 0884d5add3..530b46d2f2 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java @@ -33,7 +33,7 @@ public class JDBCSpiTest { private static UJESSQLConnection conn; public static UJESSQLConnection getConnection() throws ClassNotFoundException, SQLException { Class.forName("org.apache.linkis.ujes.jdbc.UJESSQLDriver"); - conn = (UJESSQLConnection) DriverManager.getConnection("jdbc:linkis://127.0.0.1:9001", "shanhuang", "Abcd1234@"); + conn = (UJESSQLConnection) DriverManager.getConnection("jdbc:linkis://127.0.0.1:9001", "root", "123456"); return conn; } @Test diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java index 930af12c20..edf499f5c8 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java @@ -53,13 +53,13 @@ public static void getConnection(){ @Test public void crud(){ preStatement = conn.prepareStatement(""); - preStatement.executeUpdate("CREATE TABLE if not exists johnnwang_qml.test1236 as select * from ai_fmi_ods.1000_10"); - preStatement.executeUpdate("insert into johnnwang_qml.test1236 select * from ai_fmi_ods.1000_10"); - // preStatement.executeUpdate("update johnnwang_qml.test1236 set label=6 where label=1"); - preStatement.executeUpdate("select * from johnnwang_qml.test1236"); + preStatement.executeUpdate("CREATE TABLE if not exists db.test1236 as select * from ai_fmi_ods.1000_10"); + preStatement.executeUpdate("insert into db.test1236 select * from ai_fmi_ods.1000_10"); + // preStatement.executeUpdate("update db.test1236 set label=6 where label=1"); + preStatement.executeUpdate("select * from db.test1236"); UJESSQLResultSet resultSet = preStatement.getResultSet(); showResult(resultSet); - preStatement.execute("drop table johnnwang_qml.test1236"); + preStatement.execute("drop table db.test1236"); Assert.assertTrue(resultSet.isAfterLast()); } @@ -85,7 +85,7 @@ public void execute() { @Test public void selectTest(){ - preStatement = conn.prepareStatement("select * from shanhuang_ind.testjohn1 limit 10"); + preStatement = conn.prepareStatement("select * from db.table limit 10"); UJESSQLResultSet resultSet = preStatement.executeQuery(); showResult(resultSet); Assert.assertTrue(resultSet.isAfterLast()); diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java index ead5069361..9eb20d027f 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java @@ -59,10 +59,10 @@ public static void createConnection(){ @Before public void setParams(){ sql = "show tables"; - sqlCreate = "CREATE TABLE if not exists johnnwang_qml.test1236 as select * from ai_fmi_ods.1000_10"; - sqlInsert = "insert into johnnwang_qml.test1236 select * from ai_fmi_ods.1000_10 limit 10"; - sqlSelect = "select * from johnnwang_qml.test1236"; - sqlDrop = "drop table johnnwang_qml.test1236"; + sqlCreate = "CREATE TABLE if not exists db.test1236 as select * from ai_fmi_ods.1000_10"; + sqlInsert = "insert into db.test1236 select * from ai_fmi_ods.1000_10 limit 10"; + sqlSelect = "select * from db.test1236"; + sqlDrop = "drop table db.test1236"; maxRows = 100; queryTimeout = 10000; } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/PersistenceTest.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/PersistenceTest.java index 10871f8969..a2d86266a3 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/PersistenceTest.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/PersistenceTest.java @@ -122,7 +122,7 @@ public void testListResourceLabelByValueList() throws InvocationTargetException, public void testListResourceLabelByKeyValuesMaps() throws InvocationTargetException, IllegalAccessException { HashMap> labelKeyValues = new HashMap<>(); HashMap key1 = new HashMap<>(); - key1.put("instance", "johnnwang:9026"); + key1.put("instance", "localhost:9026"); key1.put("serviceName", "linkis-engineManager"); labelKeyValues.put("emInstance", key1); HashMap key2 = new HashMap<>(); diff --git a/linkis-engineconn-plugins/engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/creation/HiveEngineConnFactory.scala b/linkis-engineconn-plugins/engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/creation/HiveEngineConnFactory.scala index afe3a06e8b..f54ffb0531 100644 --- a/linkis-engineconn-plugins/engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/creation/HiveEngineConnFactory.scala +++ b/linkis-engineconn-plugins/engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/creation/HiveEngineConnFactory.scala @@ -69,7 +69,7 @@ class HiveEngineConnFactory extends ComputationSingleExecutorEngineConnFactory w hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHOUTPUTSERDE, HiveEngineConfiguration.BASE64_SERDE_CLASS) hiveConf.set("enable_fetch_base64","true") } - /* //Update by peaceWong add hook to HiveDriver + /* //add hook to HiveDriver if (StringUtils.isNotBlank(EnvConfiguration.LINKIS_HIVE_POST_HOOKS)) { val hooks = if (StringUtils.isNotBlank(hiveConf.get("hive.exec.post.hooks"))) { hiveConf.get("hive.exec.post.hooks") + "," + EnvConfiguration.LINKIS_HIVE_POST_HOOKS @@ -78,7 +78,7 @@ class HiveEngineConnFactory extends ComputationSingleExecutorEngineConnFactory w } hiveConf.set("hive.exec.post.hooks", hooks) }*/ - //Update by peaceWong enable hive.stats.collect.scancols + //enable hive.stats.collect.scancols hiveConf.setBoolean("hive.stats.collect.scancols", true) val ugi = HDFSUtils.getUserGroupInformation(Utils.getJvmUser) val sessionState: SessionState = ugi.doAs(new PrivilegedExceptionAction[SessionState] { diff --git a/linkis-engineconn-plugins/engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/cs/CSTableParser.scala b/linkis-engineconn-plugins/engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/cs/CSTableParser.scala index c682331e90..47ebf40a9d 100644 --- a/linkis-engineconn-plugins/engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/cs/CSTableParser.scala +++ b/linkis-engineconn-plugins/engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/cs/CSTableParser.scala @@ -72,7 +72,7 @@ class CSTableParser extends Logging { } /** - * TODO peaceWong From cs to get csTable + * TODO From cs to get csTable * Exact Match * @param csTempTable * @return diff --git a/linkis-engineconn-plugins/engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala b/linkis-engineconn-plugins/engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala index b12d092191..9d03e14ee6 100644 --- a/linkis-engineconn-plugins/engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala +++ b/linkis-engineconn-plugins/engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala @@ -113,7 +113,7 @@ object SQLSession extends Logging { throw new SparkEngineException(40001, s"read record exception", t) } logger.warn(s"Time taken: ${System.currentTimeMillis() - startTime}, Fetched $index row(s).") - //Update by peaceWong to register TempTable + //to register TempTable //Utils.tryAndErrorMsg(CSTableRegister.registerTempTable(engineExecutorContext, writer, alias, columns))("Failed to register tmp table:") engineExecutionContext.appendStdout(s"${EngineUtils.getName} >> Time taken: ${System.currentTimeMillis() - startTime}, Fetched $index row(s).") engineExecutionContext.sendResultSet(writer) diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/Constant.java b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/Constant.java index a3e3403afb..6dbec8f062 100644 --- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/Constant.java +++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/Constant.java @@ -24,7 +24,7 @@ public interface Constant { - String FILE_SYSTEM_USER = "neiljianliu"; + String FILE_SYSTEM_USER = "hadoop"; List resourceTypes = Lists.newArrayList(Arrays.asList("hdfs", "share", "file")); diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlRestfulApi.java b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlRestfulApi.java index 76135d98c9..cf208afc1f 100644 --- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlRestfulApi.java +++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlRestfulApi.java @@ -514,7 +514,7 @@ public Message getBasic(@RequestParam(value="resourceId",required=false) String basic.setCreateTime(resource.getCreateTime()); basic.setDownloadedFileName(resource.getDownloadedFileName()); basic.setOwner(resource.getUser()); - //todo cooperyang 正确的版本信息 + //todo 正确的版本信息 basic.setNumberOfVerions(10); if (resource.isExpire()){ basic.setExpireTime(RestfulUtils.getExpireTime(resource.getCreateTime(),resource.getExpireType(), resource.getExpireTime())); diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/RestfulUtils.java b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/RestfulUtils.java index 2dda4775d0..0ecbe77ec5 100644 --- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/RestfulUtils.java +++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/RestfulUtils.java @@ -58,7 +58,7 @@ public static boolean canAccessResource(String user, String resourceId, ResourceService resourceService){ //String realUser = resourceService.getOwner(resourceId); //return user.equals(realUser); - //todo cooperyang + //todo return true; } diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-cache/src/test/java/org/apache/linkis/cs/contextcache/test/service/TestContextCacheService.java b/linkis-public-enhancements/linkis-context-service/linkis-cs-cache/src/test/java/org/apache/linkis/cs/contextcache/test/service/TestContextCacheService.java index a1006c09ae..a52cd39163 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-cache/src/test/java/org/apache/linkis/cs/contextcache/test/service/TestContextCacheService.java +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-cache/src/test/java/org/apache/linkis/cs/contextcache/test/service/TestContextCacheService.java @@ -50,7 +50,7 @@ public void generateData() throws CSErrorException { persistenceManager.getContextIDPersistence().deleteContextID(contextIDStr); PersistenceContextID persistenceContextID = new PersistenceContextID(); persistenceContextID.setContextId(String.valueOf(contextIDStr)); - persistenceContextID.setUser("johnnwang"); + persistenceContextID.setUser("hadoop"); persistenceContextID.setExpireTime(new Date()); persistenceContextID.setExpireType(ExpireType.TODAY); persistenceContextID.setInstance("updateInstance"); diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/test/java/org/apache/linkis/cs/client/test/bean/ClientTestContextKey.java b/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/test/java/org/apache/linkis/cs/client/test/bean/ClientTestContextKey.java index c470d9d426..26e9f37b83 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/test/java/org/apache/linkis/cs/client/test/bean/ClientTestContextKey.java +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/test/java/org/apache/linkis/cs/client/test/bean/ClientTestContextKey.java @@ -25,7 +25,7 @@ public class ClientTestContextKey implements ContextKey { - private final String key = "cooperyang.txt"; + private final String key = "hadoop.txt"; diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-common/src/test/java/org/apache/linkis/cs/common/serialize/test/ContextSerializationHelperTest.java b/linkis-public-enhancements/linkis-context-service/linkis-cs-common/src/test/java/org/apache/linkis/cs/common/serialize/test/ContextSerializationHelperTest.java index 5bff88a31f..04dba9725d 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-common/src/test/java/org/apache/linkis/cs/common/serialize/test/ContextSerializationHelperTest.java +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-common/src/test/java/org/apache/linkis/cs/common/serialize/test/ContextSerializationHelperTest.java @@ -42,11 +42,11 @@ public static void main(String[] args) throws Exception { ContextSerializationHelper contextSerializationHelper = ContextSerializationHelper.getInstance(); CSTable csTable = new CSTable(); - csTable.setCreator("neiljianliu"); + csTable.setCreator("hadoop"); csTable.setName("table1"); CSTable csTable1 = new CSTable(); - csTable1.setCreator("neiljianliu"); + csTable1.setCreator("hadoop"); csTable1.setName("table2"); CSTableMetadataContextHistory csTableMetadataContextHistory = new CSTableMetadataContextHistory(); csTableMetadataContextHistory.setTable(csTable); @@ -62,11 +62,11 @@ public static void testLineageHistory() throws Exception { ContextSerializationHelper contextSerializationHelper = ContextSerializationHelper.getInstance(); List tables = new ArrayList<>(); CSTable csTable = new CSTable(); - csTable.setCreator("neiljianliu"); + csTable.setCreator("hadoop"); csTable.setName("table1"); CSTable csTable1 = new CSTable(); - csTable1.setCreator("neiljianliu"); + csTable1.setCreator("hadoop"); csTable1.setName("table2"); tables.add(csTable); tables.add(csTable1); @@ -104,11 +104,11 @@ public static void testHAContextID() throws Exception { LinkisHAWorkFlowContextID haWorkFlowContextID = new LinkisHAWorkFlowContextID(); haWorkFlowContextID.setBackupInstance("test123"); haWorkFlowContextID.setInstance("test1234"); - haWorkFlowContextID.setUser("johnnwang"); + haWorkFlowContextID.setUser("hadoop"); haWorkFlowContextID.setContextId("hello"); haWorkFlowContextID.setFlow("hellof"); String json = contextSerializationHelper.serialize(haWorkFlowContextID); - String objstr = "{\"type\": \"HAWorkFlowContextID\",\"value\": \"{\\n \\\"instance\\\": null,\\n \\\"backupInstance\\\": null,\\n \\\"user\\\": \\\"neiljianliu\\\",\\n \\\"workspaceID\\\": null,\\n \\\"project\\\": \\\"test01_neiljianliu\\\",\\n \\\"flow\\\": \\\"dedede\\\",\\n \\\"contextId\\\": \\\"24-24--YmRwZHdzMTEwMDAxOjkxMTY\\\\u003dYmRwZHdzMTEwMDAxOjkxMTY\\\\u003d85197\\\",\\n \\\"version\\\": \\\"v000001\\\",\\n \\\"env\\\": null\\n}\"}\n"; + String objstr = "{\"type\": \"HAWorkFlowContextID\",\"value\": \"{\\n \\\"instance\\\": null,\\n \\\"backupInstance\\\": null,\\n \\\"user\\\": \\\"hadoop\\\",\\n \\\"workspaceID\\\": null,\\n \\\"project\\\": \\\"test01_neiljianliu\\\",\\n \\\"flow\\\": \\\"dedede\\\",\\n \\\"contextId\\\": \\\"24-24--YmRwZHdzMTEwMDAxOjkxMTY\\\\u003dYmRwZHdzMTEwMDAxOjkxMTY\\\\u003d85197\\\",\\n \\\"version\\\": \\\"v000001\\\",\\n \\\"env\\\": null\\n}\"}\n"; Object deserialize = contextSerializationHelper.deserialize(objstr); System.out.println(json); } @@ -148,7 +148,7 @@ private static void testSrialzer() throws Exception{ /* LinkisHAWorkFlowContextID haWorkFlowContextID = new LinkisHAWorkFlowContextID(); haWorkFlowContextID.setBackupInstance("test123"); haWorkFlowContextID.setInstance("test1234"); - haWorkFlowContextID.setUser("johnnwang"); + haWorkFlowContextID.setUser("hadoop"); haWorkFlowContextID.setContextId("hello"); haWorkFlowContextID.setFlow("hellof"); String json = contextSerializationHelper.serialize(haWorkFlowContextID); diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/AContextID.java b/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/AContextID.java index 78a1ef85a3..5ef619d5a0 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/AContextID.java +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/AContextID.java @@ -28,7 +28,7 @@ public class AContextID implements UserContextID, HAContextID { private String contextId; - private String user = "neiljianliu"; + private String user = "hadoop"; private String instance = "instance"; diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/ContextIDTest.java b/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/ContextIDTest.java index 428ff30e5f..3428d77ace 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/ContextIDTest.java +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/ContextIDTest.java @@ -58,7 +58,7 @@ public void testGetContextID() throws CSErrorException { public void testUpdateContextID() throws CSErrorException { AContextID aContextID = new AContextID(); aContextID.setContextId("84695"); - aContextID.setUser("johnnwang"); + aContextID.setUser("hadoop"); aContextID.setExpireTime(new Date()); aContextID.setExpireType(ExpireType.TODAY); aContextID.setInstance("updateInstance"); diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/ContextMapTest.java b/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/ContextMapTest.java index dc5e8849e5..173afaa90e 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/ContextMapTest.java +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-persistence/src/test/java/org/apache/linkis/cs/persistence/ContextMapTest.java @@ -47,7 +47,7 @@ public void testcreateContextMap() throws CSErrorException, JsonProcessingExcept persistenceContextID.setContextId(String.valueOf(new Random().nextInt(100000))); AContextValue aContextValue = new AContextValue(); CSTable csTable = new CSTable(); - csTable.setCreator("neiljianliu"); + csTable.setCreator("hadoop"); csTable.setName("table1"); aContextValue.setValue(csTable); aContextKeyValue.setContextValue(aContextValue); @@ -143,7 +143,7 @@ public void testUpdateContextMap() throws CSErrorException, JsonProcessingExcept persistenceContextID.setContextId("10067"); AContextValue aContextValue = new AContextValue(); CSTable csTable = new CSTable(); - csTable.setCreator("johnnwang"); + csTable.setCreator("hadoop"); csTable.setName("tableUpdate"); aContextValue.setValue(csTable); aContextKeyValue.setContextValue(aContextValue); diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/ContextIDRestfulApi.java b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/ContextIDRestfulApi.java index 0869b7cec3..eb572ae815 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/ContextIDRestfulApi.java +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/ContextIDRestfulApi.java @@ -49,7 +49,7 @@ public class ContextIDRestfulApi implements CsRestfulParent { public Message createContextID(HttpServletRequest req) throws InterruptedException { //contextID是client传过来的序列化的id PersistenceContextID contextID = new PersistenceContextID(); - contextID.setUser("neiljianliu"); + contextID.setUser("hadoop"); contextID.setExpireType(ExpireType.TODAY); contextID.setExpireTime(new Date()); contextID.setInstance("cs-server1"); @@ -71,7 +71,7 @@ public Message getContextID( HttpServletRequest req, @RequestParam(value="contex @RequestMapping(path = "updateContextID",method = RequestMethod.POST) public Message updateContextID( HttpServletRequest req) throws InterruptedException, CSErrorException { PersistenceContextID contextID = new PersistenceContextID(); - contextID.setUser("johnnwang"); + contextID.setUser("hadoop"); contextID.setExpireType(ExpireType.NEVER); contextID.setExpireTime(new Date()); contextID.setInstance("cs-server2"); diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/ContextRestfulApi.java b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/ContextRestfulApi.java index 226025f8e1..dfbdd9f0cc 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/ContextRestfulApi.java +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/ContextRestfulApi.java @@ -132,7 +132,7 @@ public Message setValueByKey( HttpServletRequest req) throws InterruptedExceptio ContextValue contextValue = new PersistenceContextValue(); CSTable csTable = new CSTable(); csTable.setName("table1"); - csTable.setCreator("neiljianliu"); + csTable.setCreator("hadoop"); String encode = jsonSerializer.encode(csTable); contextValue.setValue(encode); HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.SET, contextID, contextKey, contextValue); @@ -156,7 +156,7 @@ public Message setValue( HttpServletRequest req) throws InterruptedException, CS ContextValue contextValue = new PersistenceContextValue(); CSTable csTable = new CSTable(); csTable.setName("tableupdate"); - csTable.setCreator("neiljianliuupdate"); + csTable.setCreator("hadoopupdate"); String encode = jsonSerializer.encode(csTable); contextValue.setValue(encode); ContextKeyValue contextKeyValue = new PersistenceContextKeyValue(); diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/SchedulerTest.java b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/SchedulerTest.java index 0061cce37f..5dcc2fca8a 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/SchedulerTest.java +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/SchedulerTest.java @@ -31,7 +31,7 @@ public void test() throws InterruptedException { DefaultHttpAnswerJob job = new DefaultHttpAnswerJob(); ContextIDProtocol contextIDProtocol = new ContextIDProtocol(); RestResponseProtocol responseProtocol = new RestResponseProtocol(); - contextIDProtocol.setUsername("neiljianliu"); + contextIDProtocol.setUsername("hadoop"); job.setRequestProtocol(contextIDProtocol); job.setResponseProtocol(responseProtocol); scheduler.sumbit(job); diff --git a/linkis-public-enhancements/linkis-datasource/metadatamanager/server/src/main/resources/linkis.properties b/linkis-public-enhancements/linkis-datasource/metadatamanager/server/src/main/resources/linkis.properties index ac40d3dac0..9298ce38b2 100644 --- a/linkis-public-enhancements/linkis-datasource/metadatamanager/server/src/main/resources/linkis.properties +++ b/linkis-public-enhancements/linkis-datasource/metadatamanager/server/src/main/resources/linkis.properties @@ -22,4 +22,4 @@ wds.linkis.server.restful.scan.packages=org.apache.linkis.metadatamanager.server wds.linkis.server.version=v1 #test wds.linkis.test.mode=true -wds.linkis.test.user=davidhua +wds.linkis.test.user=hadoop diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryDetailQueryServiceImpl.scala b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryDetailQueryServiceImpl.scala index 40606d52dc..10cc478424 100644 --- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryDetailQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryDetailQueryServiceImpl.scala @@ -100,7 +100,7 @@ class JobHistoryDetailQueryServiceImpl extends JobHistoryDetailQueryService with jobDetailMapper.updateJobDetail(jobUpdate) // todo - /*//updated by shanhuang to write cache + /*// to write cache if (TaskStatus.Succeed.toString.equals(jobReq.getStatus) && queryCacheService.needCache(jobReq)) { info("Write cache for task: " + jobReq.getId) jobReq.setExecutionCode(executionCode) @@ -150,7 +150,7 @@ class JobHistoryDetailQueryServiceImpl extends JobHistoryDetailQueryService with jobDetailMapper.updateJobDetail(jobUpdate) // todo - /*//updated by shanhuang to write cache + /*//to write cache if (TaskStatus.Succeed.toString.equals(jobReq.getStatus) && queryCacheService.needCache(jobReq)) { info("Write cache for task: " + jobReq.getId) jobReq.setExecutionCode(executionCode) diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index a5480393c0..0194156602 100644 --- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -148,7 +148,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { jobHistoryMapper.updateJobHistory(jobUpdate) // todo - /*//updated by shanhuang to write cache + /*//to write cache if (TaskStatus.Succeed.toString.equals(jobReq.getStatus) && queryCacheService.needCache(jobReq)) { info("Write cache for task: " + jobReq.getId) jobReq.setExecutionCode(executionCode) diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index bf593716dd..05b3945bd4 100644 --- a/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -64,10 +64,7 @@ import static org.apache.linkis.filesystem.constant.WorkSpaceConstants.*; -/** - * johnnwang - * 2018/10/25 - */ + @RestController @RequestMapping(path = "/filesystem") public class FsRestfulApi { diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-udf/linkis-udf-service/src/test/java/org/apache/linkis/udf/TestUDFTree.java b/linkis-public-enhancements/linkis-publicservice/linkis-udf/linkis-udf-service/src/test/java/org/apache/linkis/udf/TestUDFTree.java index be801b3e57..573fa0a2e9 100644 --- a/linkis-public-enhancements/linkis-publicservice/linkis-udf/linkis-udf-service/src/test/java/org/apache/linkis/udf/TestUDFTree.java +++ b/linkis-public-enhancements/linkis-publicservice/linkis-udf/linkis-udf-service/src/test/java/org/apache/linkis/udf/TestUDFTree.java @@ -28,7 +28,7 @@ public class TestUDFTree { public static void main(String[] args) throws MalformedURLException, ClassNotFoundException, NoSuchMethodException { - File hiveDependency = new File("C:\\Users\\johnnwang\\.m2\\repository\\org\\apache\\hive\\hive-exec\\1.2.1\\hive-exec-1.2.1.jar"); + File hiveDependency = new File("repository\\org\\apache\\hive\\hive-exec\\1.2.1\\hive-exec-1.2.1.jar"); File jar = new File("E:\\tm_client_1.6.jar"); URL[] url = {new URL("file:" + jar.getAbsolutePath()), new URL("file:" + hiveDependency.getAbsolutePath())}; URLClassLoader loader = URLClassLoader.newInstance(url); diff --git a/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/src/main/java/org/apache/linkis/eureka/SpringCloudEurekaApplication.java b/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/src/main/java/org/apache/linkis/eureka/SpringCloudEurekaApplication.java index fbf1756cbf..a1fc37575f 100644 --- a/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/src/main/java/org/apache/linkis/eureka/SpringCloudEurekaApplication.java +++ b/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/src/main/java/org/apache/linkis/eureka/SpringCloudEurekaApplication.java @@ -21,9 +21,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cloud.netflix.eureka.server.EnableEurekaServer; -/** - * Created by allenlliu on 2018/7/31. - */ + @SpringBootApplication @EnableEurekaServer public class SpringCloudEurekaApplication { diff --git a/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/src/main/java/org/apache/linkis/eureka/conf/EurekaSecurityConfig.java b/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/src/main/java/org/apache/linkis/eureka/conf/EurekaSecurityConfig.java index 93cb2ab9ff..56cf13d7ee 100644 --- a/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/src/main/java/org/apache/linkis/eureka/conf/EurekaSecurityConfig.java +++ b/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/src/main/java/org/apache/linkis/eureka/conf/EurekaSecurityConfig.java @@ -21,10 +21,7 @@ //import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; //import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; -/** - * created by cooperyang on 2020/7/29 - * Description: - */ + /*@EnableWebSecurity public class EurekaSecurityConfig extends WebSecurityConfigurerAdapter { @Override diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/http/GatewayRoute.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/http/GatewayRoute.scala index 467cf8188d..7050623d45 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/http/GatewayRoute.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/http/GatewayRoute.scala @@ -23,9 +23,7 @@ import org.apache.linkis.common.ServiceInstance import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.server.JMap -/** - * created by cooperyang on 2019/1/9. - */ + class GatewayRoute { private var requestURI: String = _ diff --git a/web/src/common/util/ras.js b/web/src/common/util/ras.js index bc9fe38e88..4d9da9a70a 100644 --- a/web/src/common/util/ras.js +++ b/web/src/common/util/ras.js @@ -5,22 +5,19 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - + /* eslint-disable */ -/** - * Created by harrywan on 2016/5/11. - */ /** * jshashes - https://github.com/h2non/jshashes * Released under the "New BSD" license From adabaac1fd9790236cbf604bf530556ac529f7a2 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 28 Dec 2021 21:23:14 +0800 Subject: [PATCH 17/17] Update Apache license. --- .../linkis/ecm/server/exception/ECMErrorCode.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala index 6b5b4ddbda..8302e45b50 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala @@ -1,8 +1,13 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.