[Optimization-3215][admin] Optimize the configuration of dinky address (

#3216) Co-authored-by: wenmo <32723967+wenmo@users.noreply.github.com>
DataLinkDC · Mar 1, 2024 · 9d06354 · 9d06354
1 parent 9382416
commit 9d06354
Show file tree

Hide file tree

Showing 46 changed files with 73 additions and 54 deletions.
diff --git a/dinky-admin/src/main/java/org/dinky/data/dto/APIExplainSqlDTO.java b/dinky-admin/src/main/java/org/dinky/data/dto/APIExplainSqlDTO.java
@@ -19,7 +19,7 @@
 
 package org.dinky.data.dto;
 
-import org.dinky.gateway.enums.GatewayType;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.job.JobConfig;
 
 import java.util.Map;

diff --git a/dinky-admin/src/main/java/org/dinky/data/dto/StudioMetaStoreDTO.java b/dinky-admin/src/main/java/org/dinky/data/dto/StudioMetaStoreDTO.java
@@ -19,7 +19,7 @@
 
 package org.dinky.data.dto;
 
-import org.dinky.gateway.enums.GatewayType;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.job.JobConfig;
 
 import io.swagger.annotations.ApiModel;

diff --git a/dinky-admin/src/main/java/org/dinky/init/EnvInit.java b/dinky-admin/src/main/java/org/dinky/init/EnvInit.java
@@ -41,7 +41,7 @@ public void run(ApplicationArguments args) throws Exception {
         ApplicationContext application = SpringUtil.getApplicationContext();
         Environment env = application.getEnvironment();
         String port = env.getProperty("server.port");
-        System.setProperty("dinkyAddr", ipAddress + ":" + port);
+        System.setProperty("dinkyAddr", "http://" + ipAddress + ":" + port);
         log.info(
                 "\n----------------------------------------------------------\n\t"
                         + "Application 'Dinky' is running! Access URLs:\n\t"

diff --git a/dinky-admin/src/main/java/org/dinky/job/handler/Job2MysqlHandler.java b/dinky-admin/src/main/java/org/dinky/job/handler/Job2MysqlHandler.java
@@ -25,6 +25,7 @@
 import org.dinky.daemon.task.DaemonTask;
 import org.dinky.daemon.task.DaemonTaskConfig;
 import org.dinky.data.dto.ClusterInstanceDTO;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.enums.JobStatus;
 import org.dinky.data.model.ClusterInstance;
 import org.dinky.data.model.Task;
@@ -33,7 +34,6 @@
 import org.dinky.data.model.job.JobInstance;
 import org.dinky.data.model.mapping.ClusterConfigurationMapping;
 import org.dinky.data.model.mapping.ClusterInstanceMapping;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.job.FlinkJobTask;
 import org.dinky.job.Job;
 import org.dinky.service.ClusterConfigurationService;

diff --git a/dinky-admin/src/main/java/org/dinky/job/handler/JobRefreshHandler.java b/dinky-admin/src/main/java/org/dinky/job/handler/JobRefreshHandler.java
@@ -25,6 +25,7 @@
 import org.dinky.data.constant.FlinkRestResultConstant;
 import org.dinky.data.dto.ClusterConfigurationDTO;
 import org.dinky.data.dto.JobDataDto;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.enums.JobStatus;
 import org.dinky.data.flink.backpressure.FlinkJobNodeBackPressure;
 import org.dinky.data.flink.checkpoint.CheckPointOverView;
@@ -37,7 +38,6 @@
 import org.dinky.data.model.job.JobInstance;
 import org.dinky.gateway.Gateway;
 import org.dinky.gateway.config.GatewayConfig;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.gateway.exception.NotSupportGetStatusException;
 import org.dinky.gateway.model.FlinkClusterConfig;
 import org.dinky.job.JobConfig;

diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/CatalogueServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/CatalogueServiceImpl.java
@@ -24,6 +24,7 @@
 import org.dinky.assertion.Asserts;
 import org.dinky.config.Dialect;
 import org.dinky.data.dto.CatalogueTaskDTO;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.enums.JobLifeCycle;
 import org.dinky.data.enums.JobStatus;
 import org.dinky.data.enums.Status;
@@ -36,7 +37,6 @@
 import org.dinky.data.model.job.JobHistory;
 import org.dinky.data.model.job.JobInstance;
 import org.dinky.data.result.Result;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.mapper.CatalogueMapper;
 import org.dinky.mybatis.service.impl.SuperServiceImpl;
 import org.dinky.service.CatalogueService;

diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/ClusterConfigurationServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/ClusterConfigurationServiceImpl.java
@@ -21,12 +21,12 @@
 
 import org.dinky.assertion.DinkyAssert;
 import org.dinky.data.dto.ClusterConfigurationDTO;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.enums.Status;
 import org.dinky.data.exception.BusException;
 import org.dinky.data.model.ClusterConfiguration;
 import org.dinky.data.model.Task;
 import org.dinky.gateway.config.GatewayConfig;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.gateway.model.FlinkClusterConfig;
 import org.dinky.gateway.result.TestResult;
 import org.dinky.job.JobManager;

diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/ClusterInstanceServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/ClusterInstanceServiceImpl.java
@@ -24,14 +24,14 @@
 import org.dinky.cluster.FlinkCluster;
 import org.dinky.cluster.FlinkClusterInfo;
 import org.dinky.data.dto.ClusterInstanceDTO;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.enums.Status;
 import org.dinky.data.exception.BusException;
 import org.dinky.data.exception.DinkyException;
 import org.dinky.data.model.ClusterConfiguration;
 import org.dinky.data.model.ClusterInstance;
 import org.dinky.data.model.Task;
 import org.dinky.gateway.config.GatewayConfig;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.gateway.exception.GatewayException;
 import org.dinky.gateway.model.FlinkClusterConfig;
 import org.dinky.gateway.result.GatewayResult;

diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/TaskServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/TaskServiceImpl.java
@@ -31,6 +31,7 @@
 import org.dinky.data.dto.TaskDTO;
 import org.dinky.data.dto.TaskRollbackVersionDTO;
 import org.dinky.data.dto.TaskSubmitDto;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.enums.JobLifeCycle;
 import org.dinky.data.enums.JobStatus;
 import org.dinky.data.enums.ProcessStepType;
@@ -61,7 +62,6 @@
 import org.dinky.function.compiler.CustomStringJavaCompiler;
 import org.dinky.function.pool.UdfCodePool;
 import org.dinky.function.util.UDFUtil;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.gateway.enums.SavePointStrategy;
 import org.dinky.gateway.enums.SavePointType;
 import org.dinky.gateway.model.FlinkClusterConfig;

diff --git a/dinky-admin/src/main/java/org/dinky/service/task/FlinkSqlEnvTask.java b/dinky-admin/src/main/java/org/dinky/service/task/FlinkSqlEnvTask.java
@@ -23,10 +23,10 @@
 import org.dinky.config.Dialect;
 import org.dinky.data.annotations.SupportDialect;
 import org.dinky.data.dto.TaskDTO;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.exception.NotSupportExecuteExcepition;
 import org.dinky.data.exception.NotSupportExplainExcepition;
 import org.dinky.data.result.SqlExplainResult;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.job.JobManager;
 import org.dinky.job.JobResult;
 import org.dinky.service.TaskService;

diff --git a/dinky-admin/src/main/java/org/dinky/service/task/FlinkSqlTask.java b/dinky-admin/src/main/java/org/dinky/service/task/FlinkSqlTask.java
@@ -23,8 +23,8 @@
 import org.dinky.config.Dialect;
 import org.dinky.data.annotations.SupportDialect;
 import org.dinky.data.dto.TaskDTO;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.result.SqlExplainResult;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.job.JobManager;
 import org.dinky.job.JobResult;
 import org.dinky.service.TaskService;

diff --git a/dinky-admin/src/main/java/org/dinky/utils/FlinkConfigOptionsUtils.java b/dinky-admin/src/main/java/org/dinky/utils/FlinkConfigOptionsUtils.java
@@ -78,7 +78,6 @@ public static List<FlinkConfigOption> loadOptionsByClassName(String name) {
                 configList.add(config);
             }
         } catch (ClassNotFoundException ignored) {
-            logger.warning("Could not get config option, class not found: " + name);
         }
         return configList;
     }

diff --git a/dinky-admin/src/main/resources/dinky-loader/FlinkConfClass b/dinky-admin/src/main/resources/dinky-loader/FlinkConfClass
@@ -27,6 +27,8 @@ org.apache.flink.configuration.NettyShuffleEnvironmentOptions
 org.apache.flink.configuration.RestartStrategyOptions
 org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions
 org.apache.flink.yarn.configuration.YarnConfigOptions
+org.apache.flink.configuration.YarnConfigOptions
 org.apache.flink.kubernetes.configuration.KubernetesConfigOptions
+org.apache.flink.configuration.KubernetesConfigOptions
 org.apache.flink.python.PythonOptions
-org.dinky.constant.CustomerConfigureOptions
+org.dinky.constant.CustomerConfigureOptions
diff --git a/dinky-app/dinky-app-base/src/main/java/org/dinky/app/flinksql/Submitter.java b/dinky-app/dinky-app-base/src/main/java/org/dinky/app/flinksql/Submitter.java
@@ -29,6 +29,7 @@
 import org.dinky.constant.FlinkSQLConstant;
 import org.dinky.data.app.AppParamConfig;
 import org.dinky.data.app.AppTask;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.model.SystemConfiguration;
 import org.dinky.executor.Executor;
 import org.dinky.executor.ExecutorConfig;
@@ -172,9 +173,9 @@ private static void loadDep(String type, Integer taskId, ExecutorConfig executor
             return;
         }
 
-        if ("kubernetes-application".equals(type)) {
+        if (GatewayType.get(type).isKubernetesApplicationMode()) {
             try {
-                String httpJar = "http://" + dinkyAddr + "/download/downloadDepJar/" + taskId;
+                String httpJar = dinkyAddr + "/download/downloadDepJar/" + taskId;
                 log.info("下载依赖 http-url为：{}", httpJar);
                 String flinkHome = System.getenv("FLINK_HOME");
                 String usrlib = flinkHome + "/usrlib";
@@ -214,7 +215,6 @@ private static void loadDep(String type, Integer taskId, ExecutorConfig executor
                     }
                 }
             } catch (IOException e) {
-                log.error("");
                 throw new RuntimeException(e);
             }
         }

diff --git a/dinky-app/dinky-app-base/src/main/java/org/dinky/app/util/FlinkAppUtil.java b/dinky-app/dinky-app-base/src/main/java/org/dinky/app/util/FlinkAppUtil.java
@@ -107,8 +107,8 @@ public static void monitorFlinkTask(JobClient jobClient, int taskId) {
     private static void sendHook(int taskId, String jobId, int reTryCount) {
         String dinkyAddr = SystemConfiguration.getInstances().getDinkyAddr().getValue();
         try {
-            String url = StrFormatter.format(
-                    "http://{}/api/jobInstance/hookJobDone?taskId={}&jobId={}", dinkyAddr, taskId, jobId);
+            String url =
+                    StrFormatter.format("{}/api/jobInstance/hookJobDone?taskId={}&jobId={}", dinkyAddr, taskId, jobId);
             String resultStr = HttpUtil.get(url);
             // TODO 这里应该使用Result实体类，但是Result.class不在comm里，迁移改动太大，暂时不搞
             String code = JsonUtils.parseObject(resultStr).get("code").toString();

diff --git a/.../org/dinky/gateway/enums/GatewayType.java → ...ava/org/dinky/data/enums/GatewayType.java b/.../org/dinky/gateway/enums/GatewayType.java → ...ava/org/dinky/data/enums/GatewayType.java
@@ -17,7 +17,7 @@
  *
  */
 
-package org.dinky.gateway.enums;
+package org.dinky.data.enums;
 
 import org.dinky.assertion.Asserts;
 
@@ -122,4 +122,14 @@ public boolean isApplicationMode() {
                 return false;
         }
     }
+
+    public boolean isKubernetesApplicationMode() {
+        switch (this) {
+            case KUBERNETES_APPLICATION:
+            case KUBERNETES_APPLICATION_OPERATOR:
+                return true;
+            default:
+                return false;
+        }
+    }
 }
diff --git a/dinky-common/src/main/resources/i18n/messages_en_US.properties b/dinky-common/src/main/resources/i18n/messages_en_US.properties
@@ -191,7 +191,7 @@ sys.maven.settings.repositoryPassword.note=Maven private server authentication p
 sys.env.settings.pythonHome=Python Env variables
 sys.env.settings.pythonHome.note=Python environment variables, used to submit Python tasks and build Python Udf
 sys.env.settings.dinkyAddr=Dinky Address
-sys.env.settings.dinkyAddr.note=The address must be the same as the address configured in the Dinky Application background url
+sys.env.settings.dinkyAddr.note=This address is an accessible Dinky address, such as http://127.0.0.1:8888
 sys.env.settings.jobResendDiffSecond=Alert anti-resend interval
 sys.env.settings.jobResendDiffSecond.note=During this interval, when the Alert information sent reaches the configured value of [Maximum number of alarm resend prevention], after reaching the threshold, the Alert information will no longer be sent; unit: seconds
 sys.env.settings.diffMinuteMaxSendCount=Maximum number of alarms to prevent resending

diff --git a/dinky-common/src/main/resources/i18n/messages_zh_CN.properties b/dinky-common/src/main/resources/i18n/messages_zh_CN.properties
@@ -191,7 +191,7 @@ sys.maven.settings.repositoryPassword.note=Maven 私服认证密码,如果需要
 sys.env.settings.pythonHome=Python 环境变量
 sys.env.settings.pythonHome.note=Python 环境变量,用于提交 Python 任务以及构建 Python Udf
 sys.env.settings.dinkyAddr=Dinky 地址
-sys.env.settings.dinkyAddr.note=该地址必须与Dinky Application后台url中配置的地址相同
+sys.env.settings.dinkyAddr.note=该地址为可访问的 Dinky 地址，如 http://127.0.0.1:8888
 sys.env.settings.jobResendDiffSecond=告警防重发间隔
 sys.env.settings.jobResendDiffSecond.note=在此间隔内,发送告警信息达到 [告警防重发最大条数] 配置的值时,达到阈值后,不再发送告警信息; 单位：秒
 sys.env.settings.diffMinuteMaxSendCount=告警防重发最大条数
@@ -281,7 +281,7 @@ process.submit.checkSql=检查作业
 process.submit.execute=执行作业
 process.submit.buildConfig=构建配置信息
 process.submit.execute.commSql=执行普通sql
-process.submit.execute.flinkSql=执行flinkSql
+process.submit.execute.flinkSql=执行FlinkSql
 process.register.exits=当前任务正在执行，请勿重复提交，如有问题请前往配置中心查看
 
 # resource

diff --git a/dinky-core/src/main/java/org/dinky/api/FlinkAPI.java b/dinky-core/src/main/java/org/dinky/api/FlinkAPI.java
@@ -22,8 +22,8 @@
 import org.dinky.assertion.Asserts;
 import org.dinky.data.constant.FlinkRestAPIConstant;
 import org.dinky.data.constant.NetConstant;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.exception.BusException;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.gateway.enums.SavePointType;
 import org.dinky.gateway.model.JobInfo;
 import org.dinky.gateway.result.SavePointResult;

diff --git a/dinky-core/src/main/java/org/dinky/executor/ExecutorConfig.java b/dinky-core/src/main/java/org/dinky/executor/ExecutorConfig.java
@@ -20,7 +20,7 @@
 package org.dinky.executor;
 
 import org.dinky.assertion.Asserts;
-import org.dinky.gateway.enums.GatewayType;
+import org.dinky.data.enums.GatewayType;
 
 import java.util.ArrayList;
 import java.util.HashMap;

diff --git a/dinky-core/src/main/java/org/dinky/explainer/Explainer.java b/dinky-core/src/main/java/org/dinky/explainer/Explainer.java
@@ -20,6 +20,7 @@
 package org.dinky.explainer;
 
 import org.dinky.assertion.Asserts;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.model.LineageRel;
 import org.dinky.data.result.ExplainResult;
 import org.dinky.data.result.SqlExplainResult;
@@ -28,7 +29,6 @@
 import org.dinky.explainer.print_table.PrintStatementExplainer;
 import org.dinky.function.data.model.UDF;
 import org.dinky.function.util.UDFUtil;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.interceptor.FlinkInterceptor;
 import org.dinky.job.JobConfig;
 import org.dinky.job.JobManager;

diff --git a/dinky-core/src/main/java/org/dinky/job/Job.java b/dinky-core/src/main/java/org/dinky/job/Job.java
@@ -19,10 +19,10 @@
 
 package org.dinky.job;
 
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.result.IResult;
 import org.dinky.executor.Executor;
 import org.dinky.executor.ExecutorConfig;
-import org.dinky.gateway.enums.GatewayType;
 
 import java.time.LocalDateTime;
 import java.util.List;

diff --git a/dinky-core/src/main/java/org/dinky/job/JobBuilder.java b/dinky-core/src/main/java/org/dinky/job/JobBuilder.java
@@ -19,8 +19,8 @@
 
 package org.dinky.job;
 
+import org.dinky.data.enums.GatewayType;
 import org.dinky.executor.Executor;
-import org.dinky.gateway.enums.GatewayType;
 
 public abstract class JobBuilder {
 

diff --git a/dinky-core/src/main/java/org/dinky/job/JobConfig.java b/dinky-core/src/main/java/org/dinky/job/JobConfig.java
@@ -21,10 +21,10 @@
 
 import org.dinky.assertion.Asserts;
 import org.dinky.data.constant.NetConstant;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.executor.ExecutorConfig;
 import org.dinky.gateway.config.FlinkConfig;
 import org.dinky.gateway.config.GatewayConfig;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.gateway.enums.SavePointStrategy;
 import org.dinky.gateway.model.FlinkClusterConfig;
 

diff --git a/dinky-core/src/main/java/org/dinky/job/JobManager.java b/dinky-core/src/main/java/org/dinky/job/JobManager.java
@@ -26,6 +26,7 @@
 import org.dinky.context.FlinkUdfPathContextHolder;
 import org.dinky.context.RowLevelPermissionsContext;
 import org.dinky.data.annotations.ProcessStep;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.enums.ProcessStepType;
 import org.dinky.data.exception.BusException;
 import org.dinky.data.model.SystemConfiguration;
@@ -45,7 +46,6 @@
 import org.dinky.gateway.config.FlinkConfig;
 import org.dinky.gateway.config.GatewayConfig;
 import org.dinky.gateway.enums.ActionType;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.gateway.enums.SavePointType;
 import org.dinky.gateway.result.GatewayResult;
 import org.dinky.gateway.result.SavePointResult;

diff --git a/dinky-core/src/main/java/org/dinky/job/builder/JobTransBuilder.java b/dinky-core/src/main/java/org/dinky/job/builder/JobTransBuilder.java
@@ -21,12 +21,12 @@
 
 import org.dinky.assertion.Asserts;
 import org.dinky.constant.FlinkSQLConstant;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.result.IResult;
 import org.dinky.data.result.InsertResult;
 import org.dinky.data.result.ResultBuilder;
 import org.dinky.executor.Executor;
 import org.dinky.gateway.Gateway;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.gateway.result.GatewayResult;
 import org.dinky.interceptor.FlinkInterceptor;
 import org.dinky.interceptor.FlinkInterceptorResult;

diff --git a/dinky-core/src/test/java/org/dinky/core/JobManagerTest.java b/dinky-core/src/test/java/org/dinky/core/JobManagerTest.java
@@ -19,9 +19,9 @@
 
 package org.dinky.core;
 
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.result.ResultPool;
 import org.dinky.data.result.SelectResult;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.job.JobConfig;
 import org.dinky.job.JobManager;
 import org.dinky.job.JobResult;

diff --git a/dinky-core/src/test/java/org/dinky/job/JobConfigTest.java b/dinky-core/src/test/java/org/dinky/job/JobConfigTest.java
@@ -21,7 +21,7 @@
 
 import static org.junit.jupiter.api.Assertions.*;
 
-import org.dinky.gateway.enums.GatewayType;
+import org.dinky.data.enums.GatewayType;
 
 import org.apache.flink.configuration.RestOptions;
 

diff --git a/dinky-function/src/main/java/org/dinky/function/util/UDFUtil.java b/dinky-function/src/main/java/org/dinky/function/util/UDFUtil.java
@@ -23,6 +23,7 @@
 import org.dinky.classloader.DinkyClassLoader;
 import org.dinky.config.Dialect;
 import org.dinky.context.FlinkUdfPathContextHolder;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.exception.DinkyException;
 import org.dinky.data.model.FlinkUdfManifest;
 import org.dinky.data.model.SystemConfiguration;
@@ -33,7 +34,6 @@
 import org.dinky.function.constant.PathConstant;
 import org.dinky.function.data.model.UDF;
 import org.dinky.function.pool.UdfCodePool;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.pool.ClassEntity;
 import org.dinky.pool.ClassPool;
 

diff --git a/dinky-gateway/src/main/java/org/dinky/gateway/AbstractGateway.java b/dinky-gateway/src/main/java/org/dinky/gateway/AbstractGateway.java
@@ -21,10 +21,10 @@
 
 import org.dinky.assertion.Asserts;
 import org.dinky.context.FlinkUdfPathContextHolder;
+import org.dinky.data.enums.GatewayType;
 import org.dinky.data.enums.JobStatus;
 import org.dinky.gateway.config.GatewayConfig;
 import org.dinky.gateway.enums.ActionType;
-import org.dinky.gateway.enums.GatewayType;
 import org.dinky.gateway.exception.GatewayException;
 import org.dinky.gateway.exception.NotSupportGetStatusException;
 import org.dinky.gateway.model.JobInfo;