From 7495642167786776e7eceaa3f348106389603166 Mon Sep 17 00:00:00 2001 From: xiangzihao <460888207@qq.com> Date: Thu, 2 Jan 2025 19:14:08 +0800 Subject: [PATCH 01/15] [Fix-16940] Fix incorrect docs link in priority parameter (#16941) --- docs/configs/docsdev.js | 8 ++++++++ docs/docs/en/guide/installation/pseudo-cluster.md | 2 +- docs/docs/zh/guide/installation/pseudo-cluster.md | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/configs/docsdev.js b/docs/configs/docsdev.js index ef599cd1183a..6942d87fb887 100644 --- a/docs/configs/docsdev.js +++ b/docs/configs/docsdev.js @@ -258,6 +258,10 @@ export default { title: 'File Parameter', link: '/en-us/docs/dev/user_doc/guide/parameter/file-parameter.html', }, + { + title: 'StartUp Parameter', + link: '/en-us/docs/dev/user_doc/guide/parameter/startup-parameter.html', + }, ], }, { @@ -969,6 +973,10 @@ export default { title: '文件参数传递', link: '/zh-cn/docs/dev/user_doc/guide/parameter/file-parameter.html', }, + { + title: '启动参数', + link: '/zh-cn/docs/dev/user_doc/guide/parameter/startup-parameter.html', + }, ], }, { diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md b/docs/docs/en/guide/installation/pseudo-cluster.md index 57550482968a..68cf3bf24444 100644 --- a/docs/docs/en/guide/installation/pseudo-cluster.md +++ b/docs/docs/en/guide/installation/pseudo-cluster.md @@ -2,7 +2,7 @@ The purpose of the pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, API server, are all on the same machine. -If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md). +If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow pseudo-cluster deployment. If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md). ## Preparation diff --git a/docs/docs/zh/guide/installation/pseudo-cluster.md b/docs/docs/zh/guide/installation/pseudo-cluster.md index eb1a1502af98..d19400ab11ec 100644 --- a/docs/docs/zh/guide/installation/pseudo-cluster.md +++ b/docs/docs/zh/guide/installation/pseudo-cluster.md @@ -2,7 +2,7 @@ 伪集群部署目的是在单台机器部署 DolphinScheduler 服务，该模式下 master、worker、api server 都在同一台机器上 -如果你是新手，想要体验 DolphinScheduler 的功能，推荐使用[Standalone](standalone.md)方式体检。如果你想体验更完整的功能，或者更大的任务量，推荐使用[伪集群部署](pseudo-cluster.md)。如果你是在生产中使用，推荐使用[集群部署](cluster.md)或者[kubernetes](kubernetes.md) +如果你是新手，想要体验 DolphinScheduler 的功能，推荐使用[Standalone](standalone.md)方式体检。如果你想体验更完整的功能，或者更大的任务量，推荐使用伪集群部署。如果你是在生产中使用，推荐使用[集群部署](cluster.md)或者[kubernetes](kubernetes.md) ## 前置准备工作 From 890376a9eebef74a930f41e390a8402c1f92493c Mon Sep 17 00:00:00 2001 From: Bruce Wong Date: Wed, 8 Jan 2025 13:45:13 +0800 Subject: [PATCH 02/15] [Fix-16918][Task] Make task working directory to 775 (#16923) --- docs/docs/en/guide/security/security.md | 4 +- docs/docs/zh/guide/security/security.md | 4 +- .../common/utils/FileUtils.java | 52 ++++++++++--------- .../common/utils/FileUtilsTest.java | 6 +-- .../storage/abs/AbsStorageOperator.java | 2 +- .../storage/cos/CosStorageOperator.java | 2 +- .../storage/gcs/GcsStorageOperator.java | 2 +- .../storage/obs/ObsStorageOperator.java | 2 +- .../storage/oss/OssStorageOperator.java | 2 +- .../plugin/storage/s3/S3StorageOperator.java | 2 +- .../utils/TaskExecutionContextUtils.java | 3 +- .../utils/TaskExecutionContextUtilsTest.java | 2 +- 12 files changed, 45 insertions(+), 38 deletions(-) diff --git a/docs/docs/en/guide/security/security.md b/docs/docs/en/guide/security/security.md index 646243a9641c..fe11755d2faa 100644 --- a/docs/docs/en/guide/security/security.md +++ b/docs/docs/en/guide/security/security.md @@ -19,7 +19,9 @@ Administrator login, default username/password: admin/dolphinscheduler123 - Tenant Code: **The tenant code is the user on Linux, unique and cannot be repeated** - The administrator enters the `Security Center->Tenant Management` page, and clicks the `Create Tenant` button to create a tenant. -> Note: Currently, only admin users can modify tenant. +> Note: +> 1. Currently, only admin users can modify tenant. +> 2. If you create a tenant manually in the Linux, you need to add the manually created tenant to the dolphinscheduler bootstrap user's group, so that the tenant will have enough working directory permissions. ![create-tenant](../../../../img/new_ui/dev/security/create-tenant.png) diff --git a/docs/docs/zh/guide/security/security.md b/docs/docs/zh/guide/security/security.md index a48954cfec2f..43e90fba1a94 100644 --- a/docs/docs/zh/guide/security/security.md +++ b/docs/docs/zh/guide/security/security.md @@ -18,7 +18,9 @@ - 租户编码：**租户编码是 Linux上 的用户，唯一，不能重复** - 管理员进入安全中心->租户管理页面，点击“创建租户”按钮，创建租户。 -> 注意：目前仅有 admin 用户可以修改租户。 +> 注意： +> 1. 目前仅有 admin 用户可以修改租户； +> 2. 如果您在 Linux 中手动创建一个租户，则需要将手动创建的租户添加到 dolphinscheduler 启动用户组，以便该租户拥有足够的工作目录权限。 ![create-tenant](../../../../img/new_ui/dev/security/create-tenant.png) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index 685dc63e4e29..c826d713ac8b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -62,7 +62,9 @@ public class FileUtils { public static final String KUBE_CONFIG_FILE = "config"; - private static final Set PERMISSION_755 = PosixFilePermissions.fromString("rwxr-xr-x"); + public static final Set PERMISSION_755 = PosixFilePermissions.fromString("rwxr-xr-x"); + + public static final Set PERMISSION_775 = PosixFilePermissions.fromString("rwxrwxr-x"); /** * get download file absolute path and name @@ -239,7 +241,7 @@ public static String getFileChecksum(String pathName) throws IOException { public static void createFileWith755(@NonNull Path path) throws IOException { final Path parent = path.getParent(); if (!parent.toFile().exists()) { - createDirectoryWith755(parent); + createDirectoryWithPermission(parent, PERMISSION_755); } if (SystemUtils.IS_OS_WINDOWS) { Files.createFile(path); @@ -249,29 +251,6 @@ public static void createFileWith755(@NonNull Path path) throws IOException { } } - public static void createDirectoryWith755(@NonNull Path path) throws IOException { - if (path.toFile().exists()) { - return; - } - if (OSUtils.isWindows()) { - Files.createDirectories(path); - } else { - Path parent = path.getParent(); - if (parent != null && !parent.toFile().exists()) { - createDirectoryWith755(parent); - } - - try { - Files.createDirectory(path); - Files.setPosixFilePermissions(path, PERMISSION_755); - } catch (FileAlreadyExistsException fileAlreadyExistsException) { - // Catch the FileAlreadyExistsException here to avoid create the same parent directory in parallel - log.debug("The directory: {} already exists", path); - } - - } - } - public static void setFileTo755(File file) throws IOException { if (OSUtils.isWindows()) { return; @@ -289,6 +268,29 @@ public static void setFileTo755(File file) throws IOException { } } + public static void createDirectoryWithPermission(@NonNull Path path, + @NonNull Set permissions) throws IOException { + if (path.toFile().exists()) { + return; + } + + if (OSUtils.isWindows()) { + Files.createDirectories(path); + } else { + Path parent = path.getParent(); + if (parent != null && !parent.toFile().exists()) { + createDirectoryWithPermission(parent, permissions); + } + + try { + Files.createDirectory(path); + Files.setPosixFilePermissions(path, permissions); + } catch (FileAlreadyExistsException fileAlreadyExistsException) { + log.error("The directory: {} already exists", path); + } + } + } + public static String concatFilePath(String... paths) { if (paths.length == 0) { throw new IllegalArgumentException("At least one path should be provided"); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index 581ee8e822ff..2396d52c205f 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -60,10 +60,10 @@ public void testGetProcessExecDir() { } @Test - public void createDirectoryWith755() throws IOException { + public void testCreateDirectoryWithPermission() throws IOException { Path path = Paths.get("/tmp/createWorkDirAndUserIfAbsent"); try { - FileUtils.createDirectoryWith755(path); + FileUtils.createDirectoryWithPermission(path, FileUtils.PERMISSION_755); File file = path.toFile(); Assertions.assertTrue(file.exists()); Assertions.assertTrue(file.isDirectory()); @@ -71,7 +71,7 @@ public void createDirectoryWith755() throws IOException { Assertions.assertTrue(file.canRead()); Assertions.assertTrue(file.canWrite()); - FileUtils.createDirectoryWith755(Paths.get("/")); + FileUtils.createDirectoryWithPermission(Paths.get("/"), FileUtils.PERMISSION_755); } catch (Exception e) { e.printStackTrace(); Assertions.fail(e.getMessage()); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java index 0b6030082e0a..edd2b86c5968 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java @@ -99,7 +99,7 @@ public void download(String srcFilePath, String dstFilePath, boolean overwrite) if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); } else { - FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath()); + FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755); } BlobClient blobClient = blobContainerClient.getBlobClient(srcFilePath); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java index e2f605ec917e..c4d1468260cd 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java @@ -148,7 +148,7 @@ public void download(String srcFilePath, String dstFilePath, boolean overwrite) if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); } else { - FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath()); + FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755); } GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, cosKey); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java index fe85a402b813..9036fb1cd1ce 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java @@ -112,7 +112,7 @@ public void download(String srcFilePath, String dstFilePath, boolean overwrite) if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); } else { - FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath()); + FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755); } Blob blob = gcsStorage.get(BlobId.of(bucketName, srcFilePath)); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java index 129b9a88281f..fd96ff16a0d7 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java @@ -111,7 +111,7 @@ public void download(String srcFilePath, String dstFilePath, boolean overwrite) if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); } else { - FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath()); + FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755); } ObsObject obsObject = obsClient.getObject(bucketName, srcFilePath); try ( diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java index 0b78ce49b40a..c243c8ac5ae2 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java @@ -169,7 +169,7 @@ public void download(String srcFilePath, if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); } else { - FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath()); + FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755); } OSSObject ossObject = ossClient.getObject(bucketName, srcFilePath); try ( diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java index d1836d7c9455..2f4954fe1c40 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java @@ -113,7 +113,7 @@ public void download(String srcFilePath, if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); } else { - FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath()); + FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755); } S3Object o = s3Client.getObject(bucketName, srcFilePath); try ( diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java index cbe2a6f8aaaa..79536bb78b3f 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java @@ -50,7 +50,8 @@ public static void createTaskInstanceWorkingDirectory(TaskExecutionContext taskE log.warn("The TaskInstance WorkingDirectory: {} is exist, will recreate again", taskInstanceWorkingDirectory); } - FileUtils.createDirectoryWith755(Paths.get(taskInstanceWorkingDirectory)); + + FileUtils.createDirectoryWithPermission(Paths.get(taskInstanceWorkingDirectory), FileUtils.PERMISSION_775); taskExecutionContext.setExecutePath(taskInstanceWorkingDirectory); taskExecutionContext.setAppInfoPath(FileUtils.getAppInfoPath(taskInstanceWorkingDirectory)); diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java index 328006f83b26..05d419696bba 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java @@ -44,7 +44,7 @@ void createTaskInstanceWorkingDirectory() throws IOException { try { // Test if the working directory is exist // will delete it and recreate - FileUtils.createDirectoryWith755(Paths.get(taskWorkingDirectory)); + FileUtils.createDirectoryWithPermission(Paths.get(taskWorkingDirectory), FileUtils.PERMISSION_775); Files.createFile(Paths.get(taskWorkingDirectory, "text.txt")); Assertions.assertTrue(Files.exists(Paths.get(taskWorkingDirectory, "text.txt"))); From b6b04617a21afa2166e7076c6fb89458b7b1fc8f Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Fri, 10 Jan 2025 16:58:04 +0800 Subject: [PATCH 03/15] [Fix-16942] Fix gloval master failover might cause master dead (#16953) Co-authored-by: xiangzihao <460888207@qq.com> --- .../master/cluster/BaseServerMetadata.java | 2 + .../master/cluster/MasterServerMetadata.java | 1 + .../master/cluster/WorkerServerMetadata.java | 1 + .../system/event/MasterFailoverEvent.java | 1 + .../master/failover/FailoverCoordinator.java | 90 ++++++++++--------- .../master/registry/MasterHeartBeatTask.java | 2 +- .../registry/api/enums/RegistryNodeType.java | 1 + .../registry/api/utils/RegistryUtils.java | 26 ++++-- .../worker/task/WorkerHeartBeatTask.java | 2 +- 9 files changed, 78 insertions(+), 48 deletions(-) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/BaseServerMetadata.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/BaseServerMetadata.java index c9b89eee7062..04c595412641 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/BaseServerMetadata.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/BaseServerMetadata.java @@ -28,6 +28,8 @@ @SuperBuilder public abstract class BaseServerMetadata implements IClusters.IServerMetadata { + private final int processId; + // The server startup time in milliseconds. private final long serverStartupTime; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterServerMetadata.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterServerMetadata.java index c49833d5dadd..f68e13d7e9f6 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterServerMetadata.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterServerMetadata.java @@ -33,6 +33,7 @@ public class MasterServerMetadata extends BaseServerMetadata implements Comparab public static MasterServerMetadata parseFromHeartBeat(final MasterHeartBeat masterHeartBeat) { return MasterServerMetadata.builder() + .processId(masterHeartBeat.getProcessId()) .serverStartupTime(masterHeartBeat.getStartupTime()) .address(masterHeartBeat.getHost() + Constants.COLON + masterHeartBeat.getPort()) .cpuUsage(masterHeartBeat.getCpuUsage()) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/WorkerServerMetadata.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/WorkerServerMetadata.java index d853c7d061ce..c4d196718b38 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/WorkerServerMetadata.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/WorkerServerMetadata.java @@ -41,6 +41,7 @@ public class WorkerServerMetadata extends BaseServerMetadata { public static WorkerServerMetadata parseFromHeartBeat(final WorkerHeartBeat workerHeartBeat) { return WorkerServerMetadata.builder() + .processId(workerHeartBeat.getProcessId()) .serverStartupTime(workerHeartBeat.getStartupTime()) .address(workerHeartBeat.getHost() + Constants.COLON + workerHeartBeat.getPort()) .workerGroup(workerHeartBeat.getWorkerGroup()) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/MasterFailoverEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/MasterFailoverEvent.java index f2086a65693f..8e18477675fd 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/MasterFailoverEvent.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/MasterFailoverEvent.java @@ -29,6 +29,7 @@ public class MasterFailoverEvent extends AbstractSystemEvent { private final MasterServerMetadata masterServerMetadata; + // The time when the event occurred. This might be different at different nodes. private final Date eventTime; private MasterFailoverEvent(final MasterServerMetadata masterServerMetadata, diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/failover/FailoverCoordinator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/failover/FailoverCoordinator.java index 7bb24907ca36..30a9ae773ba0 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/failover/FailoverCoordinator.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/failover/FailoverCoordinator.java @@ -44,7 +44,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.springframework.transaction.PlatformTransactionManager; @Slf4j @Component @@ -65,9 +64,6 @@ public class FailoverCoordinator implements IFailoverCoordinator { @Autowired private WorkflowInstanceDao workflowInstanceDao; - @Autowired - private PlatformTransactionManager platformTransactionManager; - @Autowired private WorkflowFailover workflowFailover; @@ -81,13 +77,21 @@ public void globalMasterFailover(final GlobalMasterFailoverEvent globalMasterFai final Optional aliveMasterOptional = clusterManager.getMasterClusters().getServer(masterAddress); if (aliveMasterOptional.isPresent()) { + // If the master is alive, then we use the alive master's startup time as the failover deadline. final MasterServerMetadata aliveMasterServerMetadata = aliveMasterOptional.get(); log.info("The master[{}] is alive, do global master failover on it", aliveMasterServerMetadata); - doMasterFailover(aliveMasterServerMetadata.getAddress(), - aliveMasterServerMetadata.getServerStartupTime()); + doMasterFailover( + masterAddress, + aliveMasterServerMetadata.getServerStartupTime(), + RegistryUtils.getFailoveredNodePathWhichStartupTimeIsUnknown( + masterAddress)); } else { + // If the master is not alive, then we use the event time as the failover deadline. log.info("The master[{}] is not alive, do global master failover on it", masterAddress); - doMasterFailover(masterAddress, globalMasterFailoverEvent.getEventTime().getTime()); + doMasterFailover( + masterAddress, + globalMasterFailoverEvent.getEventTime().getTime(), + RegistryUtils.getFailoveredNodePathWhichStartupTimeIsUnknown(masterAddress)); } } @@ -99,53 +103,55 @@ public void globalMasterFailover(final GlobalMasterFailoverEvent globalMasterFai public void failoverMaster(final MasterFailoverEvent masterFailoverEvent) { final MasterServerMetadata masterServerMetadata = masterFailoverEvent.getMasterServerMetadata(); log.info("Master[{}] failover starting", masterServerMetadata); + final String masterAddress = masterServerMetadata.getAddress(); final Optional aliveMasterOptional = - clusterManager.getMasterClusters().getServer(masterServerMetadata.getAddress()); + clusterManager.getMasterClusters().getServer(masterAddress); if (aliveMasterOptional.isPresent()) { final MasterServerMetadata aliveMasterServerMetadata = aliveMasterOptional.get(); if (aliveMasterServerMetadata.getServerStartupTime() == masterServerMetadata.getServerStartupTime()) { log.info("The master[{}] is alive, maybe it reconnect to registry skip failover", masterServerMetadata); - } else { - log.info("The master[{}] is alive, but the startup time is different, will failover on {}", - masterServerMetadata, - aliveMasterServerMetadata); - doMasterFailover(aliveMasterServerMetadata.getAddress(), - aliveMasterServerMetadata.getServerStartupTime()); + return; } - } else { - log.info("The master[{}] is not alive, will failover", masterServerMetadata); - doMasterFailover(masterServerMetadata.getAddress(), masterServerMetadata.getServerStartupTime()); } + doMasterFailover( + masterServerMetadata.getAddress(), + masterFailoverEvent.getEventTime().getTime(), + RegistryUtils.getFailoveredNodePath( + masterServerMetadata.getAddress(), + masterServerMetadata.getServerStartupTime(), + masterServerMetadata.getProcessId())); } /** * Do master failover. *

Will failover the workflow which is scheduled by the master and the workflow's fire time is before the maxWorkflowFireTime. */ - private void doMasterFailover(final String masterAddress, final long masterStartupTime) { + private void doMasterFailover(final String masterAddress, + final long workflowFailoverDeadline, + final String masterFailoverNodePath) { // We use lock to avoid multiple master failover at the same time. // Once the workflow has been failovered, then it's state will be changed to FAILOVER // Once the FAILOVER workflow has been refired, then it's host will be changed to the new master and have a new // start time. // So if a master has been failovered multiple times, there is no problem. final StopWatch failoverTimeCost = StopWatch.createStarted(); - registryClient.getLock(RegistryNodeType.MASTER_FAILOVER_LOCK.getRegistryPath()); + registryClient.getLock(RegistryUtils.getMasterFailoverLockPath(masterAddress)); try { - final String failoverFinishedNodePath = - RegistryUtils.getFailoverFinishedNodePath(masterAddress, masterStartupTime); - if (registryClient.exists(failoverFinishedNodePath)) { - log.error("The master[{}-{}] is exist at: {}, means it has already been failovered, skip failover", + // If the master has already been failovered, then we skip the failover. + if (registryClient.exists(masterFailoverNodePath) + && String.valueOf(workflowFailoverDeadline).equals(registryClient.get(masterFailoverNodePath))) { + log.error("The master[{}/{}] is exist at: {}, means it has already been failovered, skip failover", masterAddress, - masterStartupTime, - failoverFinishedNodePath); + workflowFailoverDeadline, + masterFailoverNodePath); return; } final List needFailoverWorkflows = - getFailoverWorkflowsForMaster(masterAddress, new Date(masterStartupTime)); + getFailoverWorkflowsForMaster(masterAddress, new Date(workflowFailoverDeadline)); needFailoverWorkflows.forEach(workflowFailover::failoverWorkflow); + registryClient.persist(masterFailoverNodePath, String.valueOf(workflowFailoverDeadline)); failoverTimeCost.stop(); - registryClient.persist(failoverFinishedNodePath, String.valueOf(System.currentTimeMillis())); log.info("Master[{}] failover {} workflows finished, cost: {}/ms", masterAddress, needFailoverWorkflows.size(), @@ -190,28 +196,30 @@ public void failoverWorker(final WorkerFailoverEvent workerFailoverEvent) { final WorkerServerMetadata aliveWorkerServerMetadata = aliveWorkerOptional.get(); if (aliveWorkerServerMetadata.getServerStartupTime() == workerServerMetadata.getServerStartupTime()) { log.info("The worker[{}] is alive, maybe it reconnect to registry skip failover", workerServerMetadata); - } else { - log.info("The worker[{}] is alive, but the startup time is different, will failover on {}", - workerServerMetadata, - aliveWorkerServerMetadata); - doWorkerFailover(aliveWorkerServerMetadata.getAddress(), - aliveWorkerServerMetadata.getServerStartupTime()); + return; } - } else { - log.info("The worker[{}] is not alive, will failover", workerServerMetadata); - doWorkerFailover(workerServerMetadata.getAddress(), workerServerMetadata.getServerStartupTime()); } + doWorkerFailover( + workerServerMetadata.getAddress(), + System.currentTimeMillis(), + RegistryUtils.getFailoveredNodePath( + workerServerMetadata.getAddress(), + workerServerMetadata.getServerStartupTime(), + workerServerMetadata.getProcessId())); } - private void doWorkerFailover(final String workerAddress, final long workerCrashTime) { + private void doWorkerFailover(final String workerAddress, + final long taskFailoverDeadline, + final String workerFailoverNodePath) { final StopWatch failoverTimeCost = StopWatch.createStarted(); + // we don't check the workerFailoverNodePath exist, since the worker may be failovered multiple master final List needFailoverTasks = - getFailoverTaskForWorker(workerAddress, new Date(workerCrashTime)); + getFailoverTaskForWorker(workerAddress, new Date(taskFailoverDeadline)); needFailoverTasks.forEach(taskFailover::failoverTask); registryClient.persist( - RegistryUtils.getFailoverFinishedNodePath(workerAddress, workerCrashTime), + workerFailoverNodePath, String.valueOf(System.currentTimeMillis())); failoverTimeCost.stop(); log.info("Worker[{}] failover {} tasks finished, cost: {}/ms", @@ -221,7 +229,7 @@ private void doWorkerFailover(final String workerAddress, final long workerCrash } private List getFailoverTaskForWorker(final String workerAddress, - final Date workerCrashTime) { + final Date taskFailoverDeadline) { return workflowRepository.getAll() .stream() .map(IWorkflowExecutionRunnable::getWorkflowExecutionGraph) @@ -237,7 +245,7 @@ private List getFailoverTaskForWorker(final String worke // The submitTime should not be null. // This is a bad case unless someone manually set the submitTime to null. final Date submitTime = taskExecutionRunnable.getTaskInstance().getSubmitTime(); - return submitTime != null && submitTime.before(workerCrashTime); + return submitTime != null && submitTime.before(taskFailoverDeadline); }) .collect(Collectors.toList()); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterHeartBeatTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterHeartBeatTask.java index cf33c4824a3b..59174bac303e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterHeartBeatTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterHeartBeatTask.java @@ -86,7 +86,7 @@ public MasterHeartBeat getHeartBeat() { @Override public void writeHeartBeat(final MasterHeartBeat masterHeartBeat) { - final String failoverNodePath = RegistryUtils.getFailoverFinishedNodePath(masterHeartBeat); + final String failoverNodePath = RegistryUtils.getFailoveredNodePath(masterHeartBeat); if (registryClient.exists(failoverNodePath)) { log.warn("The master: {} is under {}, means it has been failover will close myself", masterHeartBeat, diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java index 75deb8a02d2a..c0262315629a 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java @@ -26,6 +26,7 @@ public enum RegistryNodeType { FAILOVER_FINISH_NODES("FailoverFinishNodes", "/nodes/failover-finish-nodes"), + GLOBAL_MASTER_FAILOVER_LOCK("GlobalMasterFailoverLock", "/lock/global-master-failover"), MASTER("Master", "/nodes/master"), MASTER_FAILOVER_LOCK("MasterFailoverLock", "/lock/master-failover"), MASTER_COORDINATOR("MasterCoordinator", "/nodes/master-coordinator"), diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/utils/RegistryUtils.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/utils/RegistryUtils.java index 25ef976ed516..7edcce3c8440 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/utils/RegistryUtils.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/utils/RegistryUtils.java @@ -20,14 +20,30 @@ import org.apache.dolphinscheduler.common.model.BaseHeartBeat; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; +import com.google.common.base.Preconditions; + public class RegistryUtils { - public static String getFailoverFinishedNodePath(final BaseHeartBeat baseHeartBeat) { - return getFailoverFinishedNodePath(baseHeartBeat.getHost() + ":" + baseHeartBeat.getPort(), - baseHeartBeat.getStartupTime()); + public static String getMasterFailoverLockPath(final String masterAddress) { + Preconditions.checkNotNull(masterAddress, "master address cannot be null"); + return RegistryNodeType.MASTER_FAILOVER_LOCK.getRegistryPath() + "/" + masterAddress; + } + + public static String getFailoveredNodePathWhichStartupTimeIsUnknown(final String serverAddress) { + return RegistryNodeType.FAILOVER_FINISH_NODES.getRegistryPath() + "/" + serverAddress + "-" + "unknown" + "-" + + "unknown"; + } + + public static String getFailoveredNodePath(final BaseHeartBeat baseHeartBeat) { + return getFailoveredNodePath( + baseHeartBeat.getHost() + ":" + baseHeartBeat.getPort(), + baseHeartBeat.getStartupTime(), + baseHeartBeat.getProcessId()); } - public static String getFailoverFinishedNodePath(final String masterAddress, final long masterStartupTime) { - return RegistryNodeType.FAILOVER_FINISH_NODES.getRegistryPath() + "/" + masterAddress + "-" + masterStartupTime; + public static String getFailoveredNodePath(final String serverAddress, final long serverStartupTime, + final int processId) { + return RegistryNodeType.FAILOVER_FINISH_NODES.getRegistryPath() + "/" + serverAddress + "-" + serverStartupTime + + "-" + processId; } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java index 739cf95e2250..01ed2f541ef0 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java @@ -85,7 +85,7 @@ public WorkerHeartBeat getHeartBeat() { @Override public void writeHeartBeat(final WorkerHeartBeat workerHeartBeat) { - final String failoverNodePath = RegistryUtils.getFailoverFinishedNodePath(workerHeartBeat); + final String failoverNodePath = RegistryUtils.getFailoveredNodePath(workerHeartBeat); if (registryClient.exists(failoverNodePath)) { log.warn("The worker: {} is under {}, means it has been failover will close myself", workerHeartBeat, From 37b620a27d93942f230b1e9e73fe4368f59f023e Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 13 Jan 2025 18:18:58 +0800 Subject: [PATCH 04/15] [Chore] Remove unused alert.rpc.port in common.properties (#16957) --- deploy/kubernetes/dolphinscheduler/README.md | 1 - deploy/kubernetes/dolphinscheduler/values.yaml | 3 --- docs/docs/en/architecture/configuration.md | 1 - docs/docs/zh/architecture/configuration.md | 1 - .../test/resources/docker/file-manage/common.properties | 8 -------- .../src/main/resources/common.properties | 8 -------- .../src/test/resources/common.properties | 8 -------- .../test/resources/docker/file-manage/common.properties | 8 -------- .../src/test/resources/common.properties | 8 -------- 9 files changed, 46 deletions(-) diff --git a/deploy/kubernetes/dolphinscheduler/README.md b/deploy/kubernetes/dolphinscheduler/README.md index d64053ffdd27..bcdf1eb625ce 100644 --- a/deploy/kubernetes/dolphinscheduler/README.md +++ b/deploy/kubernetes/dolphinscheduler/README.md @@ -122,7 +122,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst | common.sharedStoragePersistence.storage | string | `"20Gi"` | `PersistentVolumeClaim` size | | common.sharedStoragePersistence.storageClassName | string | `"-"` | Shared Storage persistent volume storage class, must support the access mode: ReadWriteMany | | conf.auto | bool | `false` | auto restart, if true, all components will be restarted automatically after the common configuration is updated. if false, you need to restart the components manually. default is false | -| conf.common."alert.rpc.port" | int | `50052` | rpc port | | conf.common."appId.collect" | string | `"log"` | way to collect applicationId: log, aop | | conf.common."aws.credentials.provider.type" | string | `"AWSStaticCredentialsProvider"` | | | conf.common."aws.s3.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required | diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 5658a29c1189..2c881c774cc7 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -345,9 +345,6 @@ conf: # -- development state development.state: false - # -- rpc port - alert.rpc.port: 50052 - # -- set path of conda.sh conda.path: /opt/anaconda3/etc/profile.d/conda.sh diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index 579415296623..742e6b8be00a 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -226,7 +226,6 @@ The default configuration is as follows: | datasource.encryption.salt | !@#$%^&* | the salt of the datasource encryption | | support.hive.oneSession | false | specify whether hive SQL is executed in the same session | | sudo.enable | true | whether to enable sudo | -| alert.rpc.port | 50052 | the RPC port of Alert Server | | zeppelin.rest.url | http://localhost:8080 | the RESTful API url of zeppelin | | appId.collect | log | way to collect applicationId, if use aop, alter the configuration from log to aop, annotation of applicationId auto collection related configuration in `bin/env/dolphinscheduler_env.sh` should be removed. Note: Aop way doesn't support submitting yarn job on remote host by client mode like Beeline, and will failure if override applicationId collection-related environment configuration in dolphinscheduler_env.sh, and . | diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index d167bba45e05..83c076c7c52b 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -226,7 +226,6 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId | datasource.encryption.salt | !@#$%^&* | datasource加密使用的salt | | support.hive.oneSession | false | 设置hive SQL是否在同一个session中执行 | | sudo.enable | true | 是否开启sudo | -| alert.rpc.port | 50052 | Alert Server的RPC端口 | | zeppelin.rest.url | http://localhost:8080 | zeppelin RESTful API 接口地址 | | appId.collect | log | 收集applicationId方式， 如果用aop方法，将配置log替换为aop，并将`bin/env/dolphinscheduler_env.sh`自动收集applicationId相关环境变量配置的注释取消掉，注意：aop不支持远程主机提交yarn作业的方式比如Beeline客户端提交，且如果用户环境覆盖了dolphinscheduler_env.sh收集applicationId相关环境变量配置，aop方法会失效 | diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties index d2e08600afaa..4070c1705f97 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties @@ -84,8 +84,6 @@ datasource.encryption.enable=false # datasource encryption salt datasource.encryption.salt=!@#$%^&* -# Network IP gets priority, default inner outer - # Whether hive SQL is executed in the same session support.hive.oneSession=false @@ -98,15 +96,9 @@ sudo.enable=true # network IP gets priority, default: inner outer #dolphin.scheduler.network.priority.strategy=default -# system env path -#dolphinscheduler.env.path=dolphinscheduler_env.sh - # development state development.state=false -# rpc port -alert.rpc.port=50052 - # set path of conda.sh conda.path=/opt/anaconda3/etc/profile.d/conda.sh diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 44b9acfdfe52..00eff646ae6f 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -66,8 +66,6 @@ datasource.encryption.enable=false # datasource encryption salt datasource.encryption.salt=!@#$%^&* -# Network IP gets priority, default inner outer - # Whether hive SQL is executed in the same session support.hive.oneSession=false @@ -83,15 +81,9 @@ dolphin.scheduler.network.interface.restrict=docker0 # network IP gets priority, default: inner outer #dolphin.scheduler.network.priority.strategy=default -# system env path -#dolphinscheduler.env.path=dolphinscheduler_env.sh - # development state development.state=false -# rpc port -alert.rpc.port=50052 - # set path of conda.sh conda.path=/opt/anaconda3/etc/profile.d/conda.sh diff --git a/dolphinscheduler-common/src/test/resources/common.properties b/dolphinscheduler-common/src/test/resources/common.properties index 92609741b009..9b9eaa2e47c8 100644 --- a/dolphinscheduler-common/src/test/resources/common.properties +++ b/dolphinscheduler-common/src/test/resources/common.properties @@ -130,8 +130,6 @@ datasource.encryption.enable=false # datasource encryption salt datasource.encryption.salt=!@#$%^&* -# Network IP gets priority, default inner outer - # Whether hive SQL is executed in the same session support.hive.oneSession=false @@ -147,15 +145,9 @@ dolphin.scheduler.network.interface.restrict=docker0 # network IP gets priority, default: inner outer #dolphin.scheduler.network.priority.strategy=default -# system env path -#dolphinscheduler.env.path=dolphinscheduler_env.sh - # development state development.state=false -# rpc port -alert.rpc.port=50052 - # set path of conda.sh conda.path=/opt/anaconda3/etc/profile.d/conda.sh diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties index ff146269d9cf..6f5d24d082a1 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties @@ -84,8 +84,6 @@ datasource.encryption.enable=false # datasource encryption salt datasource.encryption.salt=!@#$%^&* -# Network IP gets priority, default inner outer - # Whether hive SQL is executed in the same session support.hive.oneSession=false @@ -98,15 +96,9 @@ sudo.enable=true # network IP gets priority, default: inner outer #dolphin.scheduler.network.priority.strategy=default -# system env path -#dolphinscheduler.env.path=dolphinscheduler_env.sh - # development state development.state=false -# rpc port -alert.rpc.port=50052 - # set path of conda.sh conda.path=/opt/anaconda3/etc/profile.d/conda.sh diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties index 05dc379c56b1..f0d9698b8bbe 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties @@ -73,8 +73,6 @@ datasource.encryption.enable=false # datasource encryption salt datasource.encryption.salt=!@#$%^&* -# Network IP gets priority, default inner outer - # Whether hive SQL is executed in the same session support.hive.oneSession=false @@ -87,15 +85,9 @@ sudo.enable=true # network IP gets priority, default: inner outer #dolphin.scheduler.network.priority.strategy=default -# system env path -#dolphinscheduler.env.path=dolphinscheduler_env.sh - # development state development.state=false -# rpc port -alert.rpc.port=50052 - # set path of conda.sh conda.path=/opt/anaconda3/etc/profile.d/conda.sh From 2ae4402fde1a16cc05057f0cda64502e4d7a9e44 Mon Sep 17 00:00:00 2001 From: LiWenXin Date: Fri, 17 Jan 2025 14:44:07 +0800 Subject: [PATCH 05/15] [Fix-16934][api] When creating workflows containing switch nodes in different orders, the copied workflows may lose associations (#16939) --- .../impl/WorkflowDefinitionServiceImpl.java | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index f9432da84f6d..1b743e2c0cea 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -2118,37 +2118,38 @@ protected void doBatchOperateWorkflowDefinition(User loginUser, List taskDefinitionLogs = taskDefinitionLogDao.queryTaskDefineLogList(workflowTaskRelations); Map taskCodeMap = new HashMap<>(); - for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { + taskDefinitionLogs.forEach(taskDefinitionLog -> { try { - long taskCode = CodeGenerateUtils.genCode(); - taskCodeMap.put(taskDefinitionLog.getCode(), taskCode); - taskDefinitionLog.setCode(taskCode); - if (TaskTypeUtils.isSwitchTask(taskDefinitionLog.getTaskType())) { - final String taskParams = taskDefinitionLog.getTaskParams(); - final SwitchParameters switchParameters = - JSONUtils.parseObject(taskParams, SwitchParameters.class); - if (switchParameters == null) { - throw new IllegalArgumentException( - "Switch task params: " + taskParams + " is invalid."); - } - SwitchParameters.SwitchResult switchResult = switchParameters.getSwitchResult(); - switchResult.getDependTaskList().forEach(switchResultVo -> { - switchResultVo.setNextNode(taskCodeMap.get(switchResultVo.getNextNode())); - }); - if (switchResult.getNextNode() != null) { - switchResult.setNextNode( - taskCodeMap.get(switchResult.getNextNode())); - } - taskDefinitionLog.setTaskParams(JSONUtils.toJsonString(switchParameters)); - } + taskCodeMap.put(taskDefinitionLog.getCode(), CodeGenerateUtils.genCode()); } catch (CodeGenerateException e) { log.error("Generate task definition code error, projectCode:{}.", targetProjectCode, e); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS); throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS); } + }); + for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { + taskDefinitionLog.setCode(taskCodeMap.get(taskDefinitionLog.getCode())); taskDefinitionLog.setProjectCode(targetProjectCode); taskDefinitionLog.setVersion(0); taskDefinitionLog.setName(taskDefinitionLog.getName()); + if (TaskTypeUtils.isSwitchTask(taskDefinitionLog.getTaskType())) { + final String taskParams = taskDefinitionLog.getTaskParams(); + final SwitchParameters switchParameters = + JSONUtils.parseObject(taskParams, SwitchParameters.class); + if (switchParameters == null) { + throw new IllegalArgumentException( + "Switch task params: " + taskParams + " is invalid."); + } + SwitchParameters.SwitchResult switchResult = switchParameters.getSwitchResult(); + switchResult.getDependTaskList().forEach(switchResultVo -> { + switchResultVo.setNextNode(taskCodeMap.get(switchResultVo.getNextNode())); + }); + if (switchResult.getNextNode() != null) { + switchResult.setNextNode( + taskCodeMap.get(switchResult.getNextNode())); + } + taskDefinitionLog.setTaskParams(JSONUtils.toJsonString(switchParameters)); + } } for (WorkflowTaskRelationLog workflowTaskRelationLog : taskRelationList) { if (workflowTaskRelationLog.getPreTaskCode() > 0) { From 352b47bd8576a47f83285ecfffec589de462fac0 Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Wed, 22 Jan 2025 19:17:22 +0800 Subject: [PATCH 06/15] [Chore] python gateway unable to execute workflow (#16969) --- .../dolphinscheduler/api/python/PythonGateway.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java index 0166e34381e7..0149e48a8836 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java @@ -33,9 +33,11 @@ import org.apache.dolphinscheduler.api.service.UsersService; import org.apache.dolphinscheduler.api.service.WorkflowDefinitionService; import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ComplementDependentMode; import org.apache.dolphinscheduler.common.enums.ExecutionOrder; import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.RunMode; @@ -370,11 +372,9 @@ private void createOrUpdateSchedule(User user, public void execWorkflowInstance(String userName, String projectName, String workflowName, - String cronTime, String workerGroup, String warningType, - Integer warningGroupId, - Integer timeout) { + Integer warningGroupId) { User user = usersService.queryUser(userName); Project project = projectMapper.queryByName(projectName); WorkflowDefinition workflowDefinition = @@ -389,6 +389,10 @@ public void execWorkflowInstance(String userName, .workerGroup(workerGroup) .warningType(WarningType.of(warningType)) .warningGroupId(warningGroupId) + .execType(CommandType.START_PROCESS) + .taskDependType(TaskDependType.TASK_POST) + .dryRun(Flag.NO) + .testFlag(Flag.NO) .build(); executorService.triggerWorkflowDefinition(workflowTriggerRequest); } From 899fb378137582ef9a20663318c79f9f95cf855e Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Fri, 24 Jan 2025 08:56:55 +0800 Subject: [PATCH 07/15] [Fix-16903] Fix monitor page cannot display well (#16968) --- .../alert/registry/AlertHeartbeatTask.java | 1 + .../api/controller/MonitorController.java | 2 +- .../meter/metrics/DefaultMetricsProvider.java | 6 ++ .../registry/api/RegistryConstants.java | 25 ++++++ .../plugin/registry/jdbc/JdbcRegistry.java | 41 +++++----- .../plugin/registry/jdbc/KeyUtils.java | 76 +++++++++++++++++++ .../jdbc/server/JdbcRegistryDataManager.java | 6 +- .../plugin/registry/jdbc/KeyUtilsTest.java | 41 ++++++++++ .../src/locales/en_US/monitor.ts | 6 +- .../src/locales/zh_CN/monitor.ts | 6 +- .../src/service/modules/monitor/types.ts | 4 +- .../servers/alert_server/index.module.scss | 5 -- .../monitor/servers/alert_server/index.tsx | 35 +++------ .../monitor/servers/master/index.module.scss | 5 -- .../views/monitor/servers/master/index.tsx | 35 +++------ .../monitor/servers/worker/index.module.scss | 5 -- .../views/monitor/servers/worker/index.tsx | 64 +++++----------- 17 files changed, 222 insertions(+), 141 deletions(-) create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConstants.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtils.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtilsTest.java diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java index 8b9775e6d808..95d26b155819 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java @@ -75,6 +75,7 @@ public AlertServerHeartBeat getHeartBeat() { .cpuUsage(systemMetrics.getSystemCpuUsagePercentage()) .memoryUsage(systemMetrics.getSystemMemoryUsedPercentage()) .jvmMemoryUsage(systemMetrics.getJvmMemoryUsedPercentage()) + .diskUsage(systemMetrics.getDiskUsedPercentage()) .serverStatus(ServerStatus.NORMAL) .isActive(alertHAServer.isActive()) .host(NetUtils.getHost()) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java index 4bf952d00630..be714032ccc8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java @@ -65,7 +65,7 @@ public class MonitorController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(LIST_MASTERS_ERROR) public Result> listServer(@PathVariable("nodeType") RegistryNodeType nodeType) { - List servers = monitorService.listServer(nodeType); + final List servers = monitorService.listServer(nodeType); return Result.success(servers); } diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java index d2276794463a..0db50ba95e88 100644 --- a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java +++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/metrics/DefaultMetricsProvider.java @@ -65,6 +65,9 @@ public SystemMetrics getSystemMetrics() { long totalSystemMemory = OSUtils.getTotalSystemMemory(); long systemMemoryAvailable = OSUtils.getSystemAvailableMemoryUsed(); + double diskToTalBytes = meterRegistry.get("disk.total").gauge().value(); + double diskFreeBytes = meterRegistry.get("disk.free").gauge().value(); + systemMetrics = SystemMetrics.builder() .systemCpuUsagePercentage(systemCpuUsage) .jvmCpuUsagePercentage(processCpuUsage) @@ -74,6 +77,9 @@ public SystemMetrics getSystemMetrics() { .systemMemoryUsed(totalSystemMemory - systemMemoryAvailable) .systemMemoryMax(totalSystemMemory) .systemMemoryUsedPercentage((double) (totalSystemMemory - systemMemoryAvailable) / totalSystemMemory) + .diskUsed(diskToTalBytes - diskFreeBytes) + .diskTotal(diskToTalBytes) + .diskUsedPercentage((diskToTalBytes - diskFreeBytes) / diskToTalBytes) .build(); lastRefreshTime = System.currentTimeMillis(); return systemMetrics; diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConstants.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConstants.java new file mode 100644 index 000000000000..93d28576bd0c --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryConstants.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.registry.api; + +public class RegistryConstants { + + public static final String PATH_SEPARATOR = "/"; + public static final char PATH_SEPARATOR_CHAR = '/'; + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java index bd468f58b792..37a301bcadf1 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java @@ -97,19 +97,19 @@ public void connectUntilTimeout(@NonNull Duration timeout) throws RegistryExcept } @Override - public void subscribe(String path, SubscribeListener listener) { - checkNotNull(path); + public void subscribe(String subscribePath, SubscribeListener listener) { + checkNotNull(subscribePath); checkNotNull(listener); jdbcRegistryClient.subscribeJdbcRegistryDataChange(new JdbcRegistryDataChangeListener() { @Override - public void onJdbcRegistryDataChanged(String key, String value) { - if (!key.startsWith(path)) { + public void onJdbcRegistryDataChanged(String eventPath, String value) { + if (!isPathMatch(subscribePath, eventPath)) { return; } - Event event = Event.builder() - .key(key) - .path(path) + final Event event = Event.builder() + .key(subscribePath) + .path(eventPath) .data(value) .type(Event.Type.UPDATE) .build(); @@ -117,31 +117,35 @@ public void onJdbcRegistryDataChanged(String key, String value) { } @Override - public void onJdbcRegistryDataDeleted(String key) { - if (!key.startsWith(path)) { + public void onJdbcRegistryDataDeleted(String eventPath) { + if (!isPathMatch(subscribePath, eventPath)) { return; } - Event event = Event.builder() - .key(key) - .path(key) + final Event event = Event.builder() + .key(subscribePath) + .path(eventPath) .type(Event.Type.REMOVE) .build(); listener.notify(event); } @Override - public void onJdbcRegistryDataAdded(String key, String value) { - if (!key.startsWith(path)) { + public void onJdbcRegistryDataAdded(String eventPath, String value) { + if (!isPathMatch(subscribePath, eventPath)) { return; } - Event event = Event.builder() - .key(key) - .path(key) + final Event event = Event.builder() + .key(subscribePath) + .path(eventPath) .data(value) .type(Event.Type.ADD) .build(); listener.notify(event); } + + private boolean isPathMatch(String subscribePath, String eventPath) { + return KeyUtils.isParent(subscribePath, eventPath) || KeyUtils.isSamePath(subscribePath, eventPath); + } }); } @@ -206,11 +210,10 @@ public void delete(String key) { @Override public Collection children(String key) { try { - List children = jdbcRegistryClient.listJdbcRegistryDataChildren(key); + final List children = jdbcRegistryClient.listJdbcRegistryDataChildren(key); return children .stream() .map(JdbcRegistryDataDTO::getDataKey) - .filter(fullPath -> fullPath.length() > key.length()) .map(fullPath -> StringUtils.substringBefore(fullPath.substring(key.length() + 1), "/")) .distinct() .collect(Collectors.toList()); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtils.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtils.java new file mode 100644 index 000000000000..84576243ca90 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtils.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry.jdbc; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.dolphinscheduler.registry.api.RegistryConstants; + +import org.apache.commons.lang3.StringUtils; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class KeyUtils { + + /** + * Whether the path is the parent path of the child + *

Only the parentPath is the parent path of the childPath, return true + *

If the parentPath is equal to the childPath, return false + */ + public static boolean isParent(final String parentPath, final String childPath) { + if (StringUtils.isEmpty(parentPath)) { + throw new IllegalArgumentException("Invalid parent path " + parentPath); + } + if (StringUtils.isEmpty(childPath)) { + throw new IllegalArgumentException("Invalid child path " + childPath); + } + final String[] parentSplit = parentPath.split(RegistryConstants.PATH_SEPARATOR); + final String[] childSplit = childPath.split(RegistryConstants.PATH_SEPARATOR); + if (parentSplit.length >= childSplit.length) { + return false; + } + for (int i = 0; i < parentSplit.length; i++) { + if (!parentSplit[i].equals(childSplit[i])) { + return false; + } + } + return true; + + } + + public static boolean isSamePath(final String path1, final String path2) { + return removeLastSlash(path1).equals(path2); + } + + private static String removeLastSlash(final String path) { + checkNotNull(path, "path is null"); + if (!path.startsWith(RegistryConstants.PATH_SEPARATOR)) { + throw new IllegalArgumentException("Invalid path " + path); + } + int length = path.length() - 1; + while (length >= 0 && path.charAt(length) == RegistryConstants.PATH_SEPARATOR_CHAR) { + length--; + } + if (length == -1) { + return RegistryConstants.PATH_SEPARATOR; + } + return path.substring(0, length + 1); + } + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/JdbcRegistryDataManager.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/JdbcRegistryDataManager.java index e0f091bde7d9..b31780cf7fff 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/JdbcRegistryDataManager.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/JdbcRegistryDataManager.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryProperties; import org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryThreadFactory; +import org.apache.dolphinscheduler.plugin.registry.jdbc.KeyUtils; import org.apache.dolphinscheduler.plugin.registry.jdbc.model.DTO.DataType; import org.apache.dolphinscheduler.plugin.registry.jdbc.model.DTO.JdbcRegistryDataChanceEventDTO; import org.apache.dolphinscheduler.plugin.registry.jdbc.model.DTO.JdbcRegistryDataDTO; @@ -147,12 +148,11 @@ public Optional getRegistryDataByKey(String key) { } @Override - public List listJdbcRegistryDataChildren(String key) { + public List listJdbcRegistryDataChildren(final String key) { checkNotNull(key); return jdbcRegistryDataRepository.selectAll() .stream() - .filter(jdbcRegistryDataDTO -> jdbcRegistryDataDTO.getDataKey().startsWith(key) - && !jdbcRegistryDataDTO.getDataKey().equals(key)) + .filter(jdbcRegistryDataDTO -> KeyUtils.isParent(key, jdbcRegistryDataDTO.getDataKey())) .collect(Collectors.toList()); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtilsTest.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtilsTest.java new file mode 100644 index 000000000000..0a8c591dc2a9 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtilsTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry.jdbc; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class KeyUtilsTest { + + @Test + void isParent() { + assertFalse(KeyUtils.isParent("/a", "/b")); + assertFalse(KeyUtils.isParent("/a", "/a")); + assertFalse(KeyUtils.isParent("/b/c", "/b")); + assertFalse(KeyUtils.isParent("/b/c", "/b/")); + + assertTrue(KeyUtils.isParent("/", "/b")); + assertTrue(KeyUtils.isParent("/b/c", "/b/c/d")); + assertTrue(KeyUtils.isParent("/b", "/b/c/d")); + assertTrue(KeyUtils.isParent("/b/", "/b/c/d")); + + } + +} diff --git a/dolphinscheduler-ui/src/locales/en_US/monitor.ts b/dolphinscheduler-ui/src/locales/en_US/monitor.ts index 561b40d64bb3..54431534322f 100644 --- a/dolphinscheduler-ui/src/locales/en_US/monitor.ts +++ b/dolphinscheduler-ui/src/locales/en_US/monitor.ts @@ -19,8 +19,7 @@ export default { master: { cpu_usage: 'CPU Usage', memory_usage: 'Memory Usage', - disk_available: 'Disk Available', - load_average: 'Load Average', + disk_usage: 'Disk Usage', create_time: 'Create Time', last_heartbeat_time: 'Last Heartbeat Time', directory_detail: 'Directory Detail', @@ -33,8 +32,7 @@ export default { worker: { cpu_usage: 'CPU Usage', memory_usage: 'Memory Usage', - disk_available: 'Disk Available', - load_average: 'Load Average', + disk_usage: 'Disk Usage', thread_pool_usage: 'Thread Pool Usage', create_time: 'Create Time', last_heartbeat_time: 'Last Heartbeat Time', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts b/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts index 2ef6a9c4609c..ef78068a61d9 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/monitor.ts @@ -19,8 +19,7 @@ export default { master: { cpu_usage: '处理器使用量', memory_usage: '内存使用量', - disk_available: '磁盘可用容量', - load_average: '平均负载量', + disk_usage: '磁盘使用量', create_time: '创建时间', last_heartbeat_time: '最后心跳时间', directory_detail: '目录详情', @@ -33,8 +32,7 @@ export default { worker: { cpu_usage: '处理器使用量', memory_usage: '内存使用量', - disk_available: '磁盘可用容量', - load_average: '平均负载量', + disk_usage: '磁盘使用量', thread_pool_usage: '线程池使用量', create_time: '创建时间', last_heartbeat_time: '最后心跳时间', diff --git a/dolphinscheduler-ui/src/service/modules/monitor/types.ts b/dolphinscheduler-ui/src/service/modules/monitor/types.ts index 0a689019ce03..11a707789aab 100644 --- a/dolphinscheduler-ui/src/service/modules/monitor/types.ts +++ b/dolphinscheduler-ui/src/service/modules/monitor/types.ts @@ -31,8 +31,8 @@ interface ServerNode { id: number host: string port: number - zkDirectory: string - resInfo: string + serverDirectory: string + heartBeatInfo: string createTime: string lastHeartbeatTime: string } diff --git a/dolphinscheduler-ui/src/views/monitor/servers/alert_server/index.module.scss b/dolphinscheduler-ui/src/views/monitor/servers/alert_server/index.module.scss index 7bde891f5b01..deedab61c054 100644 --- a/dolphinscheduler-ui/src/views/monitor/servers/alert_server/index.module.scss +++ b/dolphinscheduler-ui/src/views/monitor/servers/alert_server/index.module.scss @@ -27,11 +27,6 @@ @include base; } -.load-average { - @include base; - color: var(--n-color-target); -} - .link-btn { color: var(--n-color-target); cursor: pointer; diff --git a/dolphinscheduler-ui/src/views/monitor/servers/alert_server/index.tsx b/dolphinscheduler-ui/src/views/monitor/servers/alert_server/index.tsx index a223fcb8a809..d9f876b8c678 100644 --- a/dolphinscheduler-ui/src/views/monitor/servers/alert_server/index.tsx +++ b/dolphinscheduler-ui/src/views/monitor/servers/alert_server/index.tsx @@ -16,7 +16,7 @@ */ import { defineComponent, onMounted, ref, toRefs } from 'vue' -import { NGrid, NGi, NCard, NNumberAnimation, NSpace, NTag } from 'naive-ui' +import { NGrid, NGi, NCard, NSpace, NTag } from 'naive-ui' import { useI18n } from 'vue-i18n' import { useServerNode } from './use-server-node' import styles from './index.module.scss' @@ -64,7 +64,7 @@ const alertServer = defineComponent({ this const renderNodeServerStatusTag = (item: AlertNode) => { - const serverStatus = JSON.parse(item.resInfo)?.serverStatus + const serverStatus = JSON.parse(item.heartBeatInfo)?.serverStatus if (!serverStatus) return '' @@ -103,7 +103,7 @@ const alertServer = defineComponent({ }`} clickDetails(item.zkDirectory)} + onClick={() => clickDetails(item.serverDirectory)} > {t('monitor.master.directory_detail')} @@ -125,7 +125,7 @@ const alertServer = defineComponent({ {item && ( )} @@ -138,7 +138,7 @@ const alertServer = defineComponent({ {item && ( )} @@ -146,26 +146,13 @@ const alertServer = defineComponent({ - -

+ +
{item && ( - - )} -
- - - - -
- {item && ( - )}
diff --git a/dolphinscheduler-ui/src/views/monitor/servers/master/index.module.scss b/dolphinscheduler-ui/src/views/monitor/servers/master/index.module.scss index 7bde891f5b01..deedab61c054 100644 --- a/dolphinscheduler-ui/src/views/monitor/servers/master/index.module.scss +++ b/dolphinscheduler-ui/src/views/monitor/servers/master/index.module.scss @@ -27,11 +27,6 @@ @include base; } -.load-average { - @include base; - color: var(--n-color-target); -} - .link-btn { color: var(--n-color-target); cursor: pointer; diff --git a/dolphinscheduler-ui/src/views/monitor/servers/master/index.tsx b/dolphinscheduler-ui/src/views/monitor/servers/master/index.tsx index cf29109c270b..0766681ddf3b 100644 --- a/dolphinscheduler-ui/src/views/monitor/servers/master/index.tsx +++ b/dolphinscheduler-ui/src/views/monitor/servers/master/index.tsx @@ -16,7 +16,7 @@ */ import { defineComponent, onMounted, ref, toRefs } from 'vue' -import { NGrid, NGi, NCard, NNumberAnimation, NSpace, NTag } from 'naive-ui' +import { NGrid, NGi, NCard, NSpace, NTag } from 'naive-ui' import { useI18n } from 'vue-i18n' import { useMaster } from './use-master' import styles from './index.module.scss' @@ -64,7 +64,7 @@ const master = defineComponent({ this const renderNodeServerStatusTag = (item: MasterNode) => { - const serverStatus = JSON.parse(item.resInfo)?.serverStatus + const serverStatus = JSON.parse(item.heartBeatInfo)?.serverStatus if (!serverStatus) return '' @@ -103,7 +103,7 @@ const master = defineComponent({ }`} clickDetails(item.zkDirectory)} + onClick={() => clickDetails(item.serverDirectory)} > {t('monitor.master.directory_detail')} @@ -125,7 +125,7 @@ const master = defineComponent({ {item && ( )} @@ -138,7 +138,7 @@ const master = defineComponent({ {item && ( )} @@ -146,26 +146,13 @@ const master = defineComponent({ - -
+ +
{item && ( - - )} -
- - - - -
- {item && ( - )}
diff --git a/dolphinscheduler-ui/src/views/monitor/servers/worker/index.module.scss b/dolphinscheduler-ui/src/views/monitor/servers/worker/index.module.scss index f0a52382eeae..5f6a7c62d434 100644 --- a/dolphinscheduler-ui/src/views/monitor/servers/worker/index.module.scss +++ b/dolphinscheduler-ui/src/views/monitor/servers/worker/index.module.scss @@ -27,11 +27,6 @@ @include base; } -.load-average { - @include base; - color: var(--n-color-target); -} - .link-btn { color: var(--n-color-target); cursor: pointer; diff --git a/dolphinscheduler-ui/src/views/monitor/servers/worker/index.tsx b/dolphinscheduler-ui/src/views/monitor/servers/worker/index.tsx index 6971e78f1b5a..34ff4786acb5 100644 --- a/dolphinscheduler-ui/src/views/monitor/servers/worker/index.tsx +++ b/dolphinscheduler-ui/src/views/monitor/servers/worker/index.tsx @@ -16,7 +16,7 @@ */ import { defineComponent, onMounted, ref, toRefs } from 'vue' -import { NGrid, NGi, NCard, NNumberAnimation, NSpace, NTag } from 'naive-ui' +import { NGrid, NGi, NCard, NSpace, NTag } from 'naive-ui' import { useI18n } from 'vue-i18n' import { useWorker } from './use-worker' import styles from './index.module.scss' @@ -64,7 +64,7 @@ const worker = defineComponent({ this const renderNodeServerStatusTag = (item: WorkerNode) => { - const serverStatus = JSON.parse(item.resInfo)?.serverStatus + const serverStatus = JSON.parse(item.heartBeatInfo)?.serverStatus if (!serverStatus) return '' @@ -103,7 +103,7 @@ const worker = defineComponent({ }`} clickDetails(item.zkDirectory)} + onClick={() => clickDetails(item.serverDirectory)} > {t('monitor.worker.directory_detail')} @@ -118,14 +118,14 @@ const worker = defineComponent({ - +
{item && ( )} @@ -138,7 +138,7 @@ const worker = defineComponent({ {item && ( )} @@ -146,54 +146,28 @@ const worker = defineComponent({ - -
+ +
{item && ( - - )} -
- - - - -
- {item && ( - )}
- -
+
{item && ( - <> - - / - - + )}
From 25108c84e639e086eba9e97c8432974138f78875 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Fri, 24 Jan 2025 14:15:52 +0800 Subject: [PATCH 08/15] [Improvement-16982][Master] When master startup, initialize the cluster from registry (#16983) --- .../server/master/cluster/ClusterManager.java | 45 ++++++++++++++- .../cluster/IMasterSlotChangeListener.java | 25 +++++++++ .../master/cluster/MasterServerMetadata.java | 3 + .../MasterSlotChangeListenerAdaptor.java | 56 +++++++++++++++++++ .../master/cluster/MasterSlotManager.java | 22 +------- .../master/cluster/MasterSlotManagerTest.java | 19 ++++--- 6 files changed, 139 insertions(+), 31 deletions(-) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/IMasterSlotChangeListener.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotChangeListenerAdaptor.java diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/ClusterManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/ClusterManager.java index 3f875f7047e2..3c6f9f1ba0a5 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/ClusterManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/ClusterManager.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.server.master.cluster; +import org.apache.dolphinscheduler.common.model.MasterHeartBeat; +import org.apache.dolphinscheduler.common.model.WorkerHeartBeat; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; @@ -36,6 +39,9 @@ public class ClusterManager { @Getter private WorkerClusters workerClusters; + @Autowired + private MasterSlotManager masterSlotManager; + @Autowired private WorkerGroupChangeNotifier workerGroupChangeNotifier; @@ -48,11 +54,48 @@ public ClusterManager() { } public void start() { + initializeMasterClusters(); + initializeWorkerClusters(); + log.info("ClusterManager started..."); + } + + /** + * Initialize the master clusters. + *

1. Register master slot listener once master clusters changed. + *

2. Fetch master nodes from registry. + *

3. Subscribe the master change event. + */ + private void initializeMasterClusters() { + this.masterClusters.registerListener(new MasterSlotChangeListenerAdaptor(masterSlotManager, masterClusters)); + + registryClient.getServerList(RegistryNodeType.MASTER).forEach(server -> { + final MasterHeartBeat masterHeartBeat = + JSONUtils.parseObject(server.getHeartBeatInfo(), MasterHeartBeat.class); + masterClusters.onServerAdded(MasterServerMetadata.parseFromHeartBeat(masterHeartBeat)); + }); + log.info("Initialized MasterClusters: {}", JSONUtils.toPrettyJsonString(masterClusters.getServers())); + this.registryClient.subscribe(RegistryNodeType.MASTER.getRegistryPath(), masterClusters); + } + + /** + * Initialize the worker clusters. + *

1. Fetch worker nodes from registry. + *

2. Register worker group change notifier once worker clusters changed. + *

3. Subscribe the worker change event. + */ + private void initializeWorkerClusters() { + registryClient.getServerList(RegistryNodeType.WORKER).forEach(server -> { + final WorkerHeartBeat workerHeartBeat = + JSONUtils.parseObject(server.getHeartBeatInfo(), WorkerHeartBeat.class); + workerClusters.onServerAdded(WorkerServerMetadata.parseFromHeartBeat(workerHeartBeat)); + }); + log.info("Initialized WorkerClusters: {}", JSONUtils.toPrettyJsonString(workerClusters.getServers())); + this.registryClient.subscribe(RegistryNodeType.WORKER.getRegistryPath(), workerClusters); + this.workerGroupChangeNotifier.subscribeWorkerGroupsChange(workerClusters); this.workerGroupChangeNotifier.start(); - log.info("ClusterManager started..."); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/IMasterSlotChangeListener.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/IMasterSlotChangeListener.java new file mode 100644 index 000000000000..8918fa794d45 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/IMasterSlotChangeListener.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.cluster; + +import java.util.List; + +public interface IMasterSlotChangeListener { + + void onMasterSlotChanged(final List normalMasterServers); +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterServerMetadata.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterServerMetadata.java index f68e13d7e9f6..3ec7c35efd82 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterServerMetadata.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterServerMetadata.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.server.master.cluster; +import static com.google.common.base.Preconditions.checkNotNull; + import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.model.MasterHeartBeat; @@ -32,6 +34,7 @@ public class MasterServerMetadata extends BaseServerMetadata implements Comparable { public static MasterServerMetadata parseFromHeartBeat(final MasterHeartBeat masterHeartBeat) { + checkNotNull(masterHeartBeat); return MasterServerMetadata.builder() .processId(masterHeartBeat.getProcessId()) .serverStartupTime(masterHeartBeat.getStartupTime()) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotChangeListenerAdaptor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotChangeListenerAdaptor.java new file mode 100644 index 000000000000..be280f1ed52c --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotChangeListenerAdaptor.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.cluster; + +import java.util.List; + +public class MasterSlotChangeListenerAdaptor + implements + IMasterSlotChangeListener, + IClusters.IClustersChangeListener { + + private final MasterSlotManager masterSlotManager; + + private final MasterClusters masterClusters; + + public MasterSlotChangeListenerAdaptor(final MasterSlotManager masterSlotManager, + final MasterClusters masterClusters) { + this.masterSlotManager = masterSlotManager; + this.masterClusters = masterClusters; + } + + @Override + public void onMasterSlotChanged(final List normalMasterServers) { + masterSlotManager.doReBalance(normalMasterServers); + } + + @Override + public void onServerAdded(MasterServerMetadata server) { + onMasterSlotChanged(masterClusters.getNormalServers()); + } + + @Override + public void onServerRemove(MasterServerMetadata server) { + onMasterSlotChanged(masterClusters.getNormalServers()); + } + + @Override + public void onServerUpdate(MasterServerMetadata server) { + onMasterSlotChanged(masterClusters.getNormalServers()); + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotManager.java index 4e619b2fcc81..67660bd038a9 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotManager.java @@ -29,34 +29,14 @@ @Component public class MasterSlotManager implements IMasterSlotReBalancer { - private final MasterClusters masterClusters; - private final MasterConfig masterConfig; private volatile int currentSlot = -1; private volatile int totalSlots = 0; - public MasterSlotManager(ClusterManager clusterManager, MasterConfig masterConfig) { + public MasterSlotManager(final MasterConfig masterConfig) { this.masterConfig = masterConfig; - this.masterClusters = clusterManager.getMasterClusters(); - this.masterClusters.registerListener(new IClusters.IClustersChangeListener() { - - @Override - public void onServerAdded(MasterServerMetadata server) { - doReBalance(masterClusters.getNormalServers()); - } - - @Override - public void onServerRemove(MasterServerMetadata server) { - doReBalance(masterClusters.getNormalServers()); - } - - @Override - public void onServerUpdate(MasterServerMetadata server) { - doReBalance(masterClusters.getNormalServers()); - } - }); } /** diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotManagerTest.java index 56d119d776e8..0b4f659ebc0d 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotManagerTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotManagerTest.java @@ -29,16 +29,16 @@ class MasterSlotManagerTest { private MasterSlotManager masterSlotManager; - private ClusterManager clusterManager; + private MasterClusters masterClusters; private MasterConfig masterConfig; @BeforeEach public void setUp() { - clusterManager = new ClusterManager(); + masterClusters = new MasterClusters(); masterConfig = new MasterConfig(); masterConfig.setMasterAddress("127.0.0.1:5678"); - masterSlotManager = new MasterSlotManager(clusterManager, masterConfig); + masterSlotManager = new MasterSlotManager(masterConfig); MasterServerMetadata master1 = MasterServerMetadata.builder() .cpuUsage(0.2) .memoryUsage(0.4) @@ -63,10 +63,11 @@ public void setUp() { .serverStatus(ServerStatus.BUSY) .address("127.0.0.4:5679") .build(); - clusterManager.getMasterClusters().onServerAdded(master1); - clusterManager.getMasterClusters().onServerAdded(master2); - clusterManager.getMasterClusters().onServerAdded(master3); - clusterManager.getMasterClusters().onServerAdded(master4); + this.masterClusters.registerListener(new MasterSlotChangeListenerAdaptor(masterSlotManager, masterClusters)); + masterClusters.onServerAdded(master1); + masterClusters.onServerAdded(master2); + masterClusters.onServerAdded(master3); + masterClusters.onServerAdded(master4); } @Test @@ -98,8 +99,8 @@ void doReBalance() { .serverStatus(ServerStatus.BUSY) .address("127.0.0.4:5679") .build(); - clusterManager.getMasterClusters().onServerRemove(master2); - clusterManager.getMasterClusters().onServerRemove(master3); + masterClusters.onServerRemove(master2); + masterClusters.onServerRemove(master3); // After doReBalance, the total master slots should be 2 assertThat(masterSlotManager.getTotalMasterSlots()).isEqualTo(2); } From 44165480b2b97ca569741b995d68940f29ae12a6 Mon Sep 17 00:00:00 2001 From: lile <38578667+reele@users.noreply.github.com> Date: Wed, 29 Jan 2025 10:13:06 +0800 Subject: [PATCH 09/15] [Fix-16978][Master] Fix AbstractDelayEvent compare method is incorrect (#16980) --- .../eventbus/AbstractDelayEvent.java | 7 +- .../engine/graph/IWorkflowExecutionGraph.java | 5 ++ .../engine/graph/WorkflowExecutionGraph.java | 12 +++- .../statemachine/TaskFailureStateAction.java | 31 ++++++--- .../cases/WorkflowInstancePauseTestCase.java | 50 +++++++++++++++ .../cases/WorkflowInstanceStopTestCase.java | 50 +++++++++++++++ ...rkflow_with_fake_task_failed_retrying.yaml | 64 +++++++++++++++++++ ...rkflow_with_fake_task_failed_retrying.yaml | 64 +++++++++++++++++++ 8 files changed, 271 insertions(+), 12 deletions(-) create mode 100644 dolphinscheduler-master/src/test/resources/it/pause/workflow_with_fake_task_failed_retrying.yaml create mode 100644 dolphinscheduler-master/src/test/resources/it/stop/workflow_with_fake_task_failed_retrying.yaml diff --git a/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEvent.java b/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEvent.java index c4c61b905520..c013171377ae 100644 --- a/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEvent.java +++ b/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEvent.java @@ -39,6 +39,10 @@ public abstract class AbstractDelayEvent implements IEvent, Delayed { @Builder.Default protected long createTimeInNano = System.nanoTime(); + // set create time as default if the inheritor didn't call super() + @Builder.Default + protected long expiredTimeInNano = System.nanoTime(); + public AbstractDelayEvent() { this(DEFAULT_DELAY_TIME); } @@ -50,6 +54,7 @@ public AbstractDelayEvent(final long delayTime) { public AbstractDelayEvent(final long delayTime, final long createTimeInNano) { this.delayTime = delayTime; this.createTimeInNano = createTimeInNano; + this.expiredTimeInNano = this.delayTime * 1_000_000 + this.createTimeInNano; } @Override @@ -60,7 +65,7 @@ public long getDelay(TimeUnit unit) { @Override public int compareTo(Delayed other) { - return Long.compare(this.createTimeInNano, ((AbstractDelayEvent) other).createTimeInNano); + return Long.compare(this.expiredTimeInNano, ((AbstractDelayEvent) other).expiredTimeInNano); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/IWorkflowExecutionGraph.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/IWorkflowExecutionGraph.java index 76b04951b712..3f168cce6e30 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/IWorkflowExecutionGraph.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/IWorkflowExecutionGraph.java @@ -191,6 +191,11 @@ public interface IWorkflowExecutionGraph { */ boolean isTaskExecutionRunnableForbidden(final ITaskExecutionRunnable taskExecutionRunnable); + /** + * Whether the given task's execution is failure and waiting for retry. + */ + boolean isTaskExecutionRunnableRetrying(final ITaskExecutionRunnable taskExecutionRunnable); + /** * Whether all predecessors task is skipped. *

Once all predecessors are marked as skipped, then the task will be marked as skipped, and will trigger its successors. diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/WorkflowExecutionGraph.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/WorkflowExecutionGraph.java index 20a65d45a2e9..dd89debe6c64 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/WorkflowExecutionGraph.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/WorkflowExecutionGraph.java @@ -140,7 +140,7 @@ public ITaskExecutionRunnable getTaskExecutionRunnableByTaskCode(final Long task @Override public boolean isTaskExecutionRunnableActive(final ITaskExecutionRunnable taskExecutionRunnable) { - return activeTaskExecutionRunnable.add(taskExecutionRunnable.getName()); + return activeTaskExecutionRunnable.contains(taskExecutionRunnable.getName()); } @Override @@ -256,6 +256,16 @@ public boolean isTaskExecutionRunnableForbidden(final ITaskExecutionRunnable tas return (taskExecutionRunnable.getTaskDefinition().getFlag() == Flag.NO); } + @Override + public boolean isTaskExecutionRunnableRetrying(final ITaskExecutionRunnable taskExecutionRunnable) { + if (!taskExecutionRunnable.isTaskInstanceInitialized()) { + return false; + } + final TaskInstance taskInstance = taskExecutionRunnable.getTaskInstance(); + return taskInstance.getState() == TaskExecutionStatus.FAILURE && taskExecutionRunnable.isTaskInstanceCanRetry() + && isTaskExecutionRunnableActive(taskExecutionRunnable); + } + /** * Whether all predecessors are skipped. *

Only when all predecessors are skipped, will return true. If the given task doesn't have any predecessors, will return false. diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskFailureStateAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskFailureStateAction.java index 005a0003caab..b0b9139b8b8c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskFailureStateAction.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskFailureStateAction.java @@ -104,6 +104,12 @@ public void pauseEventAction(final IWorkflowExecutionRunnable workflowExecutionR final ITaskExecutionRunnable taskExecutionRunnable, final TaskPauseLifecycleEvent taskPauseEvent) { throwExceptionIfStateIsNotMatch(taskExecutionRunnable); + // When the failed task is awaiting retry, we can mark it as 'paused' to ignore the retry event. + if (isTaskRetrying(taskExecutionRunnable)) { + super.pausedEventAction(workflowExecutionRunnable, taskExecutionRunnable, + TaskPausedLifecycleEvent.of(taskExecutionRunnable)); + return; + } logWarningIfCannotDoAction(taskExecutionRunnable, taskPauseEvent); } @@ -112,14 +118,11 @@ public void pausedEventAction(final IWorkflowExecutionRunnable workflowExecution final ITaskExecutionRunnable taskExecutionRunnable, final TaskPausedLifecycleEvent taskPausedEvent) { throwExceptionIfStateIsNotMatch(taskExecutionRunnable); - final IWorkflowExecutionGraph workflowExecutionGraph = taskExecutionRunnable.getWorkflowExecutionGraph(); // This case happen when the task is failure but the task is in delay retry queue. // We don't remove the event in GlobalWorkflowDelayEventCoordinator the event should be dropped when the task is // killed. - if (taskExecutionRunnable.isTaskInstanceCanRetry() - && workflowExecutionGraph.isTaskExecutionRunnableActive(taskExecutionRunnable)) { - workflowExecutionGraph.markTaskExecutionRunnableChainPause(taskExecutionRunnable); - publishWorkflowInstanceTopologyLogicalTransitionEvent(taskExecutionRunnable); + if (isTaskRetrying(taskExecutionRunnable)) { + super.pausedEventAction(workflowExecutionRunnable, taskExecutionRunnable, taskPausedEvent); return; } logWarningIfCannotDoAction(taskExecutionRunnable, taskPausedEvent); @@ -130,6 +133,12 @@ public void killEventAction(final IWorkflowExecutionRunnable workflowExecutionRu final ITaskExecutionRunnable taskExecutionRunnable, final TaskKillLifecycleEvent taskKillEvent) { throwExceptionIfStateIsNotMatch(taskExecutionRunnable); + // When the failed task is awaiting retry, we can mark it as 'killed' to ignore the retry event. + if (isTaskRetrying(taskExecutionRunnable)) { + super.killedEventAction(workflowExecutionRunnable, taskExecutionRunnable, + TaskKilledLifecycleEvent.of(taskExecutionRunnable)); + return; + } logWarningIfCannotDoAction(taskExecutionRunnable, taskKillEvent); } @@ -138,14 +147,11 @@ public void killedEventAction(final IWorkflowExecutionRunnable workflowExecution final ITaskExecutionRunnable taskExecutionRunnable, final TaskKilledLifecycleEvent taskKilledEvent) { throwExceptionIfStateIsNotMatch(taskExecutionRunnable); - final IWorkflowExecutionGraph workflowExecutionGraph = taskExecutionRunnable.getWorkflowExecutionGraph(); // This case happen when the task is failure but the task is in delay retry queue. // We don't remove the event in GlobalWorkflowDelayEventCoordinator the event should be dropped when the task is // killed. - if (taskExecutionRunnable.isTaskInstanceCanRetry() - && workflowExecutionGraph.isTaskExecutionRunnableActive(taskExecutionRunnable)) { - workflowExecutionGraph.markTaskExecutionRunnableChainKill(taskExecutionRunnable); - publishWorkflowInstanceTopologyLogicalTransitionEvent(taskExecutionRunnable); + if (isTaskRetrying(taskExecutionRunnable)) { + super.killedEventAction(workflowExecutionRunnable, taskExecutionRunnable, taskKilledEvent); return; } logWarningIfCannotDoAction(taskExecutionRunnable, taskKilledEvent); @@ -179,4 +185,9 @@ public void failoverEventAction(final IWorkflowExecutionRunnable workflowExecuti public TaskExecutionStatus matchState() { return TaskExecutionStatus.FAILURE; } + + private boolean isTaskRetrying(final ITaskExecutionRunnable taskExecutionRunnable) { + final IWorkflowExecutionGraph workflowExecutionGraph = taskExecutionRunnable.getWorkflowExecutionGraph(); + return workflowExecutionGraph.isTaskExecutionRunnableRetrying(taskExecutionRunnable); + } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstancePauseTestCase.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstancePauseTestCase.java index fd9eee60f496..6594e170debe 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstancePauseTestCase.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstancePauseTestCase.java @@ -294,4 +294,54 @@ public void testPauseWorkflow_with_subWorkflowTask_success() { masterContainer.assertAllResourceReleased(); } + @Test + @DisplayName("Test pause a workflow with failed retrying task") + public void testPauseWorkflow_with_failedRetryingTask() { + final String yaml = "/it/pause/workflow_with_fake_task_failed_retrying.yaml"; + final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml); + final WorkflowDefinition workflow = context.getOneWorkflow(); + + final WorkflowOperator.WorkflowTriggerDTO workflowTriggerDTO = WorkflowOperator.WorkflowTriggerDTO.builder() + .workflowDefinition(workflow) + .runWorkflowCommandParam(new RunWorkflowCommandParam()) + .build(); + final Integer workflowInstanceId = workflowOperator.manualTriggerWorkflow(workflowTriggerDTO); + + await() + .pollInterval(Duration.ofMillis(100)) + .atMost(Duration.ofMinutes(1)) + .untilAsserted(() -> { + assertThat(repository.queryWorkflowInstance(workflowInstanceId).getState()) + .isEqualTo(WorkflowExecutionStatus.RUNNING_EXECUTION); + + assertThat(repository.queryTaskInstance(workflowInstanceId)) + .satisfiesExactly( + taskInstance -> { + assertThat(taskInstance.getName()).isEqualTo("FAILED-RETRY"); + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.FAILURE); + }); + }); + + assertThat(workflowOperator.pauseWorkflowInstance(workflowInstanceId).isSuccess()); + + await() + .pollInterval(Duration.ofMillis(100)) + .atMost(Duration.ofMinutes(1)) + .untilAsserted(() -> { + assertThat(repository.queryWorkflowInstance(workflowInstanceId)) + .satisfies( + workflowInstance -> { + assertThat(workflowInstance.getState()) + .isEqualTo(WorkflowExecutionStatus.PAUSE); + }); + + assertThat(repository.queryTaskInstance(workflowInstanceId)) + .satisfiesExactly( + taskInstance -> { + assertThat(taskInstance.getName()).isEqualTo("FAILED-RETRY"); + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.PAUSE); + }); + }); + masterContainer.assertAllResourceReleased(); + } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceStopTestCase.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceStopTestCase.java index 0b80a4b40a08..5174a2dc59fe 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceStopTestCase.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceStopTestCase.java @@ -246,4 +246,54 @@ public void testStopWorkflow_with_subWorkflowTask_success() { masterContainer.assertAllResourceReleased(); } + + @Test + @DisplayName("Test stop a workflow with failed retrying task") + public void testStopWorkflow_with_failedRetryingTask() { + final String yaml = "/it/stop/workflow_with_fake_task_failed_retrying.yaml"; + final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml); + final WorkflowDefinition workflow = context.getOneWorkflow(); + + final WorkflowOperator.WorkflowTriggerDTO workflowTriggerDTO = WorkflowOperator.WorkflowTriggerDTO.builder() + .workflowDefinition(workflow) + .runWorkflowCommandParam(new RunWorkflowCommandParam()) + .build(); + final Integer workflowInstanceId = workflowOperator.manualTriggerWorkflow(workflowTriggerDTO); + + await() + .pollInterval(Duration.ofMillis(100)) + .atMost(Duration.ofMinutes(1)) + .untilAsserted(() -> { + assertThat(repository.queryWorkflowInstance(workflowInstanceId).getState()) + .isEqualTo(WorkflowExecutionStatus.RUNNING_EXECUTION); + + assertThat(repository.queryTaskInstance(workflowInstanceId)) + .satisfiesExactly( + taskInstance -> { + assertThat(taskInstance.getName()).isEqualTo("FAILED-RETRY"); + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.FAILURE); + }); + }); + + assertThat(workflowOperator.stopWorkflowInstance(workflowInstanceId).isSuccess()); + + await() + .pollInterval(Duration.ofMillis(100)) + .atMost(Duration.ofMinutes(1)) + .untilAsserted(() -> { + assertThat(repository.queryWorkflowInstance(workflowInstanceId)) + .satisfies( + workflowInstance -> { + assertThat(workflowInstance.getState()).isEqualTo(WorkflowExecutionStatus.STOP); + }); + + assertThat(repository.queryTaskInstance(workflowInstanceId)) + .satisfiesExactly( + taskInstance -> { + assertThat(taskInstance.getName()).isEqualTo("FAILED-RETRY"); + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.KILL); + }); + }); + masterContainer.assertAllResourceReleased(); + } } diff --git a/dolphinscheduler-master/src/test/resources/it/pause/workflow_with_fake_task_failed_retrying.yaml b/dolphinscheduler-master/src/test/resources/it/pause/workflow_with_fake_task_failed_retrying.yaml new file mode 100644 index 000000000000..a86397370e53 --- /dev/null +++ b/dolphinscheduler-master/src/test/resources/it/pause/workflow_with_fake_task_failed_retrying.yaml @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +project: + name: MasterIntegrationTest + code: 1 + description: This is a fake project + userId: 1 + userName: admin + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + +workflows: + - name: workflow_with_one_fake_task_failed + code: 1 + version: 1 + projectCode: 1 + description: This is a fake workflow with single task + releaseState: ONLINE + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + userId: 1 + executionType: PARALLEL + +tasks: + - name: FAILED-RETRY + code: 1 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"ls /-"}' + workerGroup: default + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + failRetryTimes: 10 + failRetryInterval: 10 + +taskRelations: + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 0 + preTaskVersion: 0 + postTaskCode: 1 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + diff --git a/dolphinscheduler-master/src/test/resources/it/stop/workflow_with_fake_task_failed_retrying.yaml b/dolphinscheduler-master/src/test/resources/it/stop/workflow_with_fake_task_failed_retrying.yaml new file mode 100644 index 000000000000..a86397370e53 --- /dev/null +++ b/dolphinscheduler-master/src/test/resources/it/stop/workflow_with_fake_task_failed_retrying.yaml @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +project: + name: MasterIntegrationTest + code: 1 + description: This is a fake project + userId: 1 + userName: admin + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + +workflows: + - name: workflow_with_one_fake_task_failed + code: 1 + version: 1 + projectCode: 1 + description: This is a fake workflow with single task + releaseState: ONLINE + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + userId: 1 + executionType: PARALLEL + +tasks: + - name: FAILED-RETRY + code: 1 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"ls /-"}' + workerGroup: default + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + failRetryTimes: 10 + failRetryInterval: 10 + +taskRelations: + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 0 + preTaskVersion: 0 + postTaskCode: 1 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + From bf5aeb107d22f1ea0575b30669a10cbf6d3f0529 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 4 Feb 2025 17:12:02 +0800 Subject: [PATCH 10/15] [Fix-16627] [dolphinscheduler-api] LoginHandlerInterceptor.preHandle check session without expire time check (#16989) --- .../interceptor/LoginHandlerInterceptor.java | 6 ++++-- .../security/impl/AbstractAuthenticator.java | 6 +++++- .../impl/pwd/PasswordAuthenticator.java | 4 +++- .../api/service/impl/SessionServiceImpl.java | 2 +- .../api/controller/LoginControllerTest.java | 21 +++++++++++++++++++ 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java index cc1928404b12..49eacaa59c34 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java @@ -96,8 +96,10 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons } @Override - public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, - ModelAndView modelAndView) throws Exception { + public void postHandle(HttpServletRequest request, + HttpServletResponse response, + Object handler, + ModelAndView modelAndView) { ThreadLocalContext.getTimezoneThreadLocal().remove(); int code = response.getStatus(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java index 7b9c3f5a43bb..b840a0279691 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java @@ -119,10 +119,14 @@ public User getAuthUser(HttpServletRequest request) { sessionId = cookie.getValue(); } } - Session session = sessionService.getSession(sessionId); + final Session session = sessionService.getSession(sessionId); if (session == null) { return null; } + if (sessionService.isSessionExpire(session)) { + sessionService.expireSession(session.getUserId()); + return null; + } // get user object from session return userService.queryUser(session.getUserId()); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/pwd/PasswordAuthenticator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/pwd/PasswordAuthenticator.java index 86c419a60524..9770041606df 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/pwd/PasswordAuthenticator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/pwd/PasswordAuthenticator.java @@ -20,10 +20,12 @@ import org.apache.dolphinscheduler.api.security.impl.AbstractAuthenticator; import org.apache.dolphinscheduler.dao.entity.User; +import lombok.NonNull; + public class PasswordAuthenticator extends AbstractAuthenticator { @Override - public User login(String userName, String password) { + public User login(@NonNull String userName, String password) { return userService.queryUser(userName, password); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java index 79816c58d900..a3571eb19e2d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java @@ -94,7 +94,7 @@ public void expireSession(Integer userId) { @Override public boolean isSessionExpire(Session session) { - return System.currentTimeMillis() - session.getLastLoginTime().getTime() <= Constants.SESSION_TIME_OUT * 1000; + return System.currentTimeMillis() - session.getLastLoginTime().getTime() >= Constants.SESSION_TIME_OUT * 1000; } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java index bad2825145d3..c954f224989d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java @@ -27,7 +27,12 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.Session; +import org.apache.dolphinscheduler.dao.repository.SessionDao; +import org.apache.http.HttpStatus; + +import java.util.Date; import java.util.Map; import javax.servlet.http.Cookie; @@ -36,6 +41,7 @@ import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.test.web.servlet.MvcResult; @@ -49,6 +55,9 @@ public class LoginControllerTest extends AbstractControllerTest { private static final Logger logger = LoggerFactory.getLogger(LoginControllerTest.class); + @Autowired + private SessionDao sessionDao; + @Test public void testLogin() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); @@ -85,6 +94,18 @@ public void testSignOut() throws Exception { logger.info(mvcResult.getResponse().getContentAsString()); } + @Test + void testSignOutWithExpireSession() throws Exception { + final Session session = sessionDao.queryById(sessionId); + session.setLastLoginTime(new Date(System.currentTimeMillis() - Constants.SESSION_TIME_OUT * 1000 - 1)); + sessionDao.updateById(session); + + mockMvc.perform(post("/signOut") + .header("sessionId", sessionId)) + .andExpect(status().is(HttpStatus.SC_UNAUTHORIZED)) + .andReturn(); + } + @Test void testClearCookie() throws Exception { MvcResult mvcResult = mockMvc.perform(delete("/cookies") From 9d1414435e60fba35c8f4ac84eb4285a7e8e1194 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 4 Feb 2025 17:53:34 +0800 Subject: [PATCH 11/15] [Improvement-16985][Registry] SubscribeListener support set scope (#16986) --- .../AbstractClusterSubscribeListener.java | 9 +- .../dolphinscheduler/registry/api/Event.java | 111 ++---------------- .../registry/api/SubscribeListener.java | 20 +++- .../registry/api/ha/AbstractHAServer.java | 24 ++-- .../plugin/registry/etcd/EtcdRegistry.java | 69 +++++++---- .../plugin/registry/RegistryTestCase.java | 26 ++-- .../plugin/registry/jdbc/JdbcRegistry.java | 55 +-------- ...JdbcRegistryDataChangeListenerAdapter.java | 91 ++++++++++++++ .../plugin/registry/jdbc/KeyUtils.java | 6 +- .../registry/zookeeper/ZookeeperRegistry.java | 34 +----- .../ZookeeperTreeCacheListenerAdapter.java | 87 ++++++++++++++ 11 files changed, 305 insertions(+), 227 deletions(-) create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryDataChangeListenerAdapter.java create mode 100644 dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperTreeCacheListenerAdapter.java diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/AbstractClusterSubscribeListener.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/AbstractClusterSubscribeListener.java index 677687eaf13b..b730f9042680 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/AbstractClusterSubscribeListener.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/AbstractClusterSubscribeListener.java @@ -30,8 +30,8 @@ public void notify(Event event) { try { // make sure the event is processed in order synchronized (this) { - Event.Type type = event.type(); - T server = parseServerFromHeartbeat(event.data()); + Event.Type type = event.getType(); + T server = parseServerFromHeartbeat(event.getEventData()); if (server == null) { log.error("Unknown cluster change event: {}", event); return; @@ -58,6 +58,11 @@ public void notify(Event event) { } } + @Override + public SubscribeScope getSubscribeScope() { + return SubscribeScope.CHILDREN_ONLY; + } + abstract T parseServerFromHeartbeat(String serverHeartBeatJson); public abstract void onServerAdded(T serverHeartBeat); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java index d99445498285..b260a591e799 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java @@ -17,115 +17,30 @@ package org.apache.dolphinscheduler.registry.api; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; + +@Getter +@ToString +@Builder +@AllArgsConstructor public class Event { - // The prefix which is watched - private String key; + // The path which is watched + private final String watchedPath; // The full path where the event was generated - private String path; + private final String eventPath; // The value corresponding to the path - private String data; + private final String eventData; // The event type {ADD, REMOVE, UPDATE} private Type type; - public Event(String key, String path, String data, Type type) { - this.key = key; - this.path = path; - this.data = data; - this.type = type; - } - - public Event() { - } - - public static EventBuilder builder() { - return new EventBuilder(); - } - - public String key() { - return this.key; - } - - public String path() { - return this.path; - } - - public String data() { - return this.data; - } - - public Type type() { - return this.type; - } - - public Event key(String key) { - this.key = key; - return this; - } - - public Event path(String path) { - this.path = path; - return this; - } - - public Event data(String data) { - this.data = data; - return this; - } - - public Event type(Type type) { - this.type = type; - return this; - } - - public String toString() { - return "Event(key=" + this.key() + ", path=" + this.path() + ", data=" + this.data() + ", type=" + this.type() - + ")"; - } - public enum Type { ADD, REMOVE, UPDATE } - public static class EventBuilder { - - private String key; - private String path; - private String data; - private Type type; - - EventBuilder() { - } - - public EventBuilder key(String key) { - this.key = key; - return this; - } - - public EventBuilder path(String path) { - this.path = path; - return this; - } - - public EventBuilder data(String data) { - this.data = data; - return this; - } - - public EventBuilder type(Type type) { - this.type = type; - return this; - } - - public Event build() { - return new Event(key, path, data, type); - } - - public String toString() { - return "Event.EventBuilder(key=" + this.key + ", path=" + this.path + ", data=" + this.data + ", type=" - + this.type + ")"; - } - } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java index e7b434e5f6df..e63d50a2af85 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java @@ -19,5 +19,23 @@ public interface SubscribeListener { - void notify(Event event); + void notify(final Event event); + + SubscribeScope getSubscribeScope(); + + enum SubscribeScope { + /** + * Only watch the path itself + */ + PATH_ONLY, + /** + * Only watch the children of the path + */ + CHILDREN_ONLY, + /** + * Watch the path and all its children and the parent path + */ + ALL + + } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractHAServer.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractHAServer.java index c87f341a30a9..a79b14d99d5c 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractHAServer.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ha/AbstractHAServer.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.registry.api.Event; import org.apache.dolphinscheduler.registry.api.Registry; +import org.apache.dolphinscheduler.registry.api.SubscribeListener; import java.util.List; @@ -56,16 +57,25 @@ public AbstractHAServer(final Registry registry, final String selectorPath, fina @Override public void start() { - registry.subscribe(selectorPath, event -> { - if (Event.Type.REMOVE.equals(event.type())) { - if (serverIdentify.equals(event.data())) { - statusChange(ServerStatus.STAND_BY); - } else { - if (participateElection()) { - statusChange(ServerStatus.ACTIVE); + registry.subscribe(selectorPath, new SubscribeListener() { + + @Override + public void notify(Event event) { + if (Event.Type.REMOVE.equals(event.getType())) { + if (serverIdentify.equals(event.getEventData())) { + statusChange(ServerStatus.STAND_BY); + } else { + if (participateElection()) { + statusChange(ServerStatus.ACTIVE); + } } } } + + @Override + public SubscribeScope getSubscribeScope() { + return SubscribeScope.PATH_ONLY; + } }); if (participateElection()) { diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java index 54a270f047f9..9f1d031131ce 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -158,7 +159,24 @@ public void subscribe(String path, SubscribeListener listener) { watcherMap.computeIfAbsent(path, $ -> client.getWatchClient().watch(watchKey, watchOption, watchResponse -> { for (WatchEvent event : watchResponse.getEvents()) { - listener.notify(new EventAdaptor(event, path)); + final String eventPath = event.getKeyValue().getKey().toString(StandardCharsets.UTF_8); + switch (listener.getSubscribeScope()) { + case PATH_ONLY: + if (eventPath.equals(path)) { + listener.notify(toEvent(event, path)); + } + break; + case CHILDREN_ONLY: + if (!eventPath.equals(path)) { + listener.notify(toEvent(event, path)); + } + break; + case ALL: + listener.notify(toEvent(event, path)); + break; + default: + throw new RegistryException("Unknown event scope: " + listener.getSubscribeScope()); + } } })); } catch (Exception e) { @@ -373,30 +391,31 @@ private static ByteSequence byteSequence(String val) { return ByteSequence.from(val, StandardCharsets.UTF_8); } - static final class EventAdaptor extends Event { - - public EventAdaptor(WatchEvent event, String key) { - key(key); - - switch (event.getEventType()) { - case PUT: - if (event.getPrevKV().getKey().isEmpty()) { - type(Type.ADD); - } else { - type(Type.UPDATE); - } - break; - case DELETE: - type(Type.REMOVE); - break; - default: - break; - } - KeyValue keyValue = event.getKeyValue(); - if (keyValue != null) { - path(keyValue.getKey().toString(StandardCharsets.UTF_8)); - data(keyValue.getValue().toString(StandardCharsets.UTF_8)); - } + private Event toEvent(final WatchEvent watchEvent, final String watchedPath) { + Event.Type eventType = null; + switch (watchEvent.getEventType()) { + case PUT: + if (watchEvent.getPrevKV().getKey().isEmpty()) { + eventType = Event.Type.ADD; + } else { + eventType = Event.Type.UPDATE; + } + break; + case DELETE: + eventType = Event.Type.REMOVE; + break; + default: + break; } + final KeyValue keyValue = watchEvent.getKeyValue(); + return Event.builder() + .type(eventType) + .watchedPath(watchedPath) + .eventPath(Optional.ofNullable(keyValue).map(kv -> kv.getKey().toString(StandardCharsets.UTF_8)) + .orElse(null)) + .eventData(Optional.ofNullable(keyValue).map(kv -> kv.getValue().toString(StandardCharsets.UTF_8)) + .orElse(null)) + .build(); } + } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-it/src/test/java/org/apache/dolphinscheduler/plugin/registry/RegistryTestCase.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-it/src/test/java/org/apache/dolphinscheduler/plugin/registry/RegistryTestCase.java index a9a23dfa0fc3..5db9798d505d 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-it/src/test/java/org/apache/dolphinscheduler/plugin/registry/RegistryTestCase.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-it/src/test/java/org/apache/dolphinscheduler/plugin/registry/RegistryTestCase.java @@ -81,16 +81,24 @@ public void testSubscribe() { final AtomicBoolean subscribeRemoved = new AtomicBoolean(false); final AtomicBoolean subscribeUpdated = new AtomicBoolean(false); - SubscribeListener subscribeListener = event -> { - System.out.println("Receive event: " + event); - if (event.type() == Event.Type.ADD) { - subscribeAdded.compareAndSet(false, true); + final SubscribeListener subscribeListener = new SubscribeListener() { + + @Override + public void notify(Event event) { + if (event.getType() == Event.Type.ADD) { + subscribeAdded.compareAndSet(false, true); + } + if (event.getType() == Event.Type.REMOVE) { + subscribeRemoved.compareAndSet(false, true); + } + if (event.getType() == Event.Type.UPDATE) { + subscribeUpdated.compareAndSet(false, true); + } } - if (event.type() == Event.Type.REMOVE) { - subscribeRemoved.compareAndSet(false, true); - } - if (event.type() == Event.Type.UPDATE) { - subscribeUpdated.compareAndSet(false, true); + + @Override + public SubscribeScope getSubscribeScope() { + return SubscribeScope.PATH_ONLY; } }; String key = "/nodes/master" + System.nanoTime(); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java index 37a301bcadf1..f9614d3c388d 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java @@ -24,10 +24,8 @@ import org.apache.dolphinscheduler.plugin.registry.jdbc.model.DTO.JdbcRegistryDataDTO; import org.apache.dolphinscheduler.plugin.registry.jdbc.server.ConnectionStateListener; import org.apache.dolphinscheduler.plugin.registry.jdbc.server.IJdbcRegistryServer; -import org.apache.dolphinscheduler.plugin.registry.jdbc.server.JdbcRegistryDataChangeListener; import org.apache.dolphinscheduler.registry.api.ConnectionListener; import org.apache.dolphinscheduler.registry.api.ConnectionState; -import org.apache.dolphinscheduler.registry.api.Event; import org.apache.dolphinscheduler.registry.api.Registry; import org.apache.dolphinscheduler.registry.api.RegistryException; import org.apache.dolphinscheduler.registry.api.SubscribeListener; @@ -97,56 +95,11 @@ public void connectUntilTimeout(@NonNull Duration timeout) throws RegistryExcept } @Override - public void subscribe(String subscribePath, SubscribeListener listener) { - checkNotNull(subscribePath); + public void subscribe(String watchedPath, SubscribeListener listener) { + checkNotNull(watchedPath); checkNotNull(listener); - jdbcRegistryClient.subscribeJdbcRegistryDataChange(new JdbcRegistryDataChangeListener() { - - @Override - public void onJdbcRegistryDataChanged(String eventPath, String value) { - if (!isPathMatch(subscribePath, eventPath)) { - return; - } - final Event event = Event.builder() - .key(subscribePath) - .path(eventPath) - .data(value) - .type(Event.Type.UPDATE) - .build(); - listener.notify(event); - } - - @Override - public void onJdbcRegistryDataDeleted(String eventPath) { - if (!isPathMatch(subscribePath, eventPath)) { - return; - } - final Event event = Event.builder() - .key(subscribePath) - .path(eventPath) - .type(Event.Type.REMOVE) - .build(); - listener.notify(event); - } - - @Override - public void onJdbcRegistryDataAdded(String eventPath, String value) { - if (!isPathMatch(subscribePath, eventPath)) { - return; - } - final Event event = Event.builder() - .key(subscribePath) - .path(eventPath) - .data(value) - .type(Event.Type.ADD) - .build(); - listener.notify(event); - } - - private boolean isPathMatch(String subscribePath, String eventPath) { - return KeyUtils.isParent(subscribePath, eventPath) || KeyUtils.isSamePath(subscribePath, eventPath); - } - }); + jdbcRegistryClient + .subscribeJdbcRegistryDataChange(new JdbcRegistryDataChangeListenerAdapter(watchedPath, listener)); } @Override diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryDataChangeListenerAdapter.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryDataChangeListenerAdapter.java new file mode 100644 index 000000000000..ccfb95f96391 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryDataChangeListenerAdapter.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry.jdbc; + +import org.apache.dolphinscheduler.plugin.registry.jdbc.server.JdbcRegistryDataChangeListener; +import org.apache.dolphinscheduler.registry.api.Event; +import org.apache.dolphinscheduler.registry.api.SubscribeListener; + +public class JdbcRegistryDataChangeListenerAdapter implements JdbcRegistryDataChangeListener { + + private final String watchedPath; + private final SubscribeListener listener; + + public JdbcRegistryDataChangeListenerAdapter(final String watchedPath, final SubscribeListener listener) { + this.watchedPath = watchedPath; + this.listener = listener; + } + + @Override + public void onJdbcRegistryDataChanged(String eventPath, String value) { + if (!isPathMatch(watchedPath, eventPath, listener.getSubscribeScope())) { + return; + } + final Event event = Event.builder() + .watchedPath(watchedPath) + .eventPath(eventPath) + .eventData(value) + .type(Event.Type.UPDATE) + .build(); + listener.notify(event); + } + + @Override + public void onJdbcRegistryDataDeleted(String eventPath) { + if (!isPathMatch(watchedPath, eventPath, listener.getSubscribeScope())) { + return; + } + final Event event = Event.builder() + .watchedPath(watchedPath) + .eventPath(eventPath) + .type(Event.Type.REMOVE) + .build(); + listener.notify(event); + } + + @Override + public void onJdbcRegistryDataAdded(String eventPath, String value) { + if (!isPathMatch(watchedPath, eventPath, listener.getSubscribeScope())) { + return; + } + final Event event = Event.builder() + .watchedPath(watchedPath) + .eventPath(eventPath) + .eventData(value) + .type(Event.Type.ADD) + .build(); + listener.notify(event); + } + + private boolean isPathMatch(final String subscribePath, + final String eventPath, + final SubscribeListener.SubscribeScope subscribeScope) { + switch (subscribeScope) { + case PATH_ONLY: + return KeyUtils.isSamePath(subscribePath, eventPath); + case CHILDREN_ONLY: + return KeyUtils.isParent(subscribePath, eventPath); + case ALL: + return KeyUtils.isParent(subscribePath, eventPath) + || KeyUtils.isSamePath(subscribePath, eventPath); + default: + throw new IllegalArgumentException("Invalid subscribe scope " + subscribeScope); + } + } + +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtils.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtils.java index 84576243ca90..2512eb0a3fe4 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtils.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/KeyUtils.java @@ -40,8 +40,10 @@ public static boolean isParent(final String parentPath, final String childPath) if (StringUtils.isEmpty(childPath)) { throw new IllegalArgumentException("Invalid child path " + childPath); } - final String[] parentSplit = parentPath.split(RegistryConstants.PATH_SEPARATOR); - final String[] childSplit = childPath.split(RegistryConstants.PATH_SEPARATOR); + final String[] parentSplit = removeLastSlash(parentPath).split(RegistryConstants.PATH_SEPARATOR); + final String[] childSplit = removeLastSlash(childPath).split(RegistryConstants.PATH_SEPARATOR); + // If the parent path is longer than or equals the child path, it is impossible to be the parent path of the + // child path if (parentSplit.length >= childSplit.length) { return false; } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java index 70754f6a6ed7..be059ad423b5 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java @@ -20,7 +20,6 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import org.apache.dolphinscheduler.registry.api.ConnectionListener; -import org.apache.dolphinscheduler.registry.api.Event; import org.apache.dolphinscheduler.registry.api.Registry; import org.apache.dolphinscheduler.registry.api.RegistryException; import org.apache.dolphinscheduler.registry.api.SubscribeListener; @@ -30,9 +29,7 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.ACLProvider; -import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.TreeCache; -import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.curator.utils.CloseableUtils; @@ -142,9 +139,9 @@ public void connectUntilTimeout(@NonNull Duration timeout) throws RegistryExcept } @Override - public void subscribe(String path, SubscribeListener listener) { + public void subscribe(final String path, final SubscribeListener listener) { final TreeCache treeCache = treeCacheMap.computeIfAbsent(path, $ -> new TreeCache(client, path)); - treeCache.getListenable().addListener(($, event) -> listener.notify(new EventAdaptor(event, path))); + treeCache.getListenable().addListener(new ZookeeperTreeCacheListenerAdapter(path, listener)); try { treeCache.start(); } catch (Exception e) { @@ -305,31 +302,4 @@ public void close() { treeCacheMap.values().forEach(CloseableUtils::closeQuietly); CloseableUtils.closeQuietly(client); } - - static final class EventAdaptor extends Event { - - public EventAdaptor(TreeCacheEvent event, String key) { - key(key); - - switch (event.getType()) { - case NODE_ADDED: - type(Type.ADD); - break; - case NODE_UPDATED: - type(Type.UPDATE); - break; - case NODE_REMOVED: - type(Type.REMOVE); - break; - default: - break; - } - - final ChildData data = event.getData(); - if (data != null) { - path(data.getPath()); - data(new String(data.getData())); - } - } - } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperTreeCacheListenerAdapter.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperTreeCacheListenerAdapter.java new file mode 100644 index 000000000000..da35f21bbf0f --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperTreeCacheListenerAdapter.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry.zookeeper; + +import org.apache.dolphinscheduler.registry.api.Event; +import org.apache.dolphinscheduler.registry.api.RegistryException; +import org.apache.dolphinscheduler.registry.api.SubscribeListener; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.TreeCacheEvent; +import org.apache.curator.framework.recipes.cache.TreeCacheListener; + +public class ZookeeperTreeCacheListenerAdapter implements TreeCacheListener { + + private final String watchedPath; + + private final SubscribeListener listener; + + public ZookeeperTreeCacheListenerAdapter(final String watchedPath, final SubscribeListener listener) { + this.listener = listener; + this.watchedPath = watchedPath; + } + + @Override + public void childEvent(final CuratorFramework curatorFramework, final TreeCacheEvent event) { + final String eventPath = event.getData().getPath(); + switch (listener.getSubscribeScope()) { + case PATH_ONLY: + if (eventPath.equals(watchedPath)) { + listener.notify(convertToEvent(event, watchedPath)); + } + break; + case CHILDREN_ONLY: + if (!eventPath.equals(watchedPath)) { + listener.notify(convertToEvent(event, watchedPath)); + } + break; + case ALL: + listener.notify(convertToEvent(event, watchedPath)); + break; + default: + throw new RegistryException("Unknown event scope: " + listener.getSubscribeScope()); + } + } + + private Event convertToEvent(TreeCacheEvent event, String watchedPath) { + + Event.Type type; + switch (event.getType()) { + case NODE_ADDED: + type = Event.Type.ADD; + break; + case NODE_UPDATED: + type = Event.Type.UPDATE; + break; + case NODE_REMOVED: + type = Event.Type.REMOVE; + break; + default: + throw new IllegalArgumentException("Unsupported event type: " + event.getType()); + } + + final ChildData data = event.getData(); + return Event.builder() + .type(type) + .watchedPath(watchedPath) + .eventPath(data.getPath()) + .eventData(new String(data.getData())) + .build(); + } +} From 879e9665764567391facb4c491654a490a3108a3 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Fri, 7 Feb 2025 10:42:03 +0800 Subject: [PATCH 12/15] [Fix-16990] Fix WorkflowExecutionGraph#isTaskFinish is not working correctly (#16995) --- .../engine/graph/IWorkflowExecutionGraph.java | 17 +++ .../engine/graph/WorkflowExecutionGraph.java | 48 ++++--- .../AbstractWorkflowStateAction.java | 4 +- .../WorkflowReadyPauseStateAction.java | 2 +- .../WorkflowReadyStopStateAction.java | 2 +- .../WorkflowRunningStateAction.java | 2 +- .../cases/WorkflowStartTestCase.java | 84 +++++++++++ ...ask_with_multiple_predecessors_failed.yaml | 132 ++++++++++++++++++ ...sk_with_multiple_predecessors_success.yaml | 132 ++++++++++++++++++ 9 files changed, 396 insertions(+), 27 deletions(-) create mode 100644 dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_with_multiple_predecessors_failed.yaml create mode 100644 dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_with_multiple_predecessors_success.yaml diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/IWorkflowExecutionGraph.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/IWorkflowExecutionGraph.java index 3f168cce6e30..1c3af50915dc 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/IWorkflowExecutionGraph.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/IWorkflowExecutionGraph.java @@ -80,11 +80,27 @@ public interface IWorkflowExecutionGraph { */ boolean isTaskExecutionRunnableActive(final ITaskExecutionRunnable taskExecutionRunnable); + /** + * Whether the given task is inactive. + *

A task is inactive means the task has been `executed`. + */ + boolean isTaskExecutionRunnableInActive(final ITaskExecutionRunnable taskExecutionRunnable); + /** * Whether the given task is killed. */ boolean isTaskExecutionRunnableKilled(final ITaskExecutionRunnable taskExecutionRunnable); + /** + * Whether the given task is failure. + */ + boolean isTaskExecutionRunnableFailed(final ITaskExecutionRunnable taskExecutionRunnable); + + /** + * Whether the given task is paused. + */ + boolean isTaskExecutionRunnablePaused(final ITaskExecutionRunnable taskExecutionRunnable); + /** * Get the active TaskExecutionRunnable list. *

The active TaskExecutionRunnable means the task is handling in the workflow execution graph. @@ -176,6 +192,7 @@ public interface IWorkflowExecutionGraph { * Check whether the given task is the end of the task chain. *

If the given task has no successor, then it is the end of the task chain. *

If the given task is killed or paused, then it is the end of the task chain. + *

If the given task is failure, and all its successors are condition task then it is not end of a task chain. */ boolean isEndOfTaskChain(final ITaskExecutionRunnable taskExecutionRunnable); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/WorkflowExecutionGraph.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/WorkflowExecutionGraph.java index dd89debe6c64..2439d9f840ae 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/WorkflowExecutionGraph.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/WorkflowExecutionGraph.java @@ -51,6 +51,8 @@ public class WorkflowExecutionGraph implements IWorkflowExecutionGraph { private final Set activeTaskExecutionRunnable; + private final Set inActiveTaskExecutionRunnable; + public WorkflowExecutionGraph() { this.failureTaskChains = new HashSet<>(); this.pausedTaskChains = new HashSet<>(); @@ -60,6 +62,7 @@ public WorkflowExecutionGraph() { this.successors = new HashMap<>(); this.totalTaskExecuteRunnableMap = new HashMap<>(); this.activeTaskExecutionRunnable = new HashSet<>(); + this.inActiveTaskExecutionRunnable = new HashSet<>(); } @Override @@ -143,11 +146,26 @@ public boolean isTaskExecutionRunnableActive(final ITaskExecutionRunnable taskEx return activeTaskExecutionRunnable.contains(taskExecutionRunnable.getName()); } + @Override + public boolean isTaskExecutionRunnableInActive(ITaskExecutionRunnable taskExecutionRunnable) { + return inActiveTaskExecutionRunnable.contains(taskExecutionRunnable.getName()); + } + @Override public boolean isTaskExecutionRunnableKilled(final ITaskExecutionRunnable taskExecutionRunnable) { return killedTaskChains.contains(taskExecutionRunnable.getName()); } + @Override + public boolean isTaskExecutionRunnableFailed(ITaskExecutionRunnable taskExecutionRunnable) { + return failureTaskChains.contains(taskExecutionRunnable.getName()); + } + + @Override + public boolean isTaskExecutionRunnablePaused(ITaskExecutionRunnable taskExecutionRunnable) { + return pausedTaskChains.contains(taskExecutionRunnable.getName()); + } + @Override public List getActiveTaskExecutionRunnable() { return activeTaskExecutionRunnable @@ -165,10 +183,10 @@ public List getAllTaskExecutionRunnable() { public boolean isTriggerConditionMet(final ITaskExecutionRunnable taskExecutionRunnable) { return getPredecessors(taskExecutionRunnable.getName()) .stream() - .allMatch(predecessor -> isTaskFinish(predecessor) - && !isTaskFailure(predecessor) - && !isTaskPaused(predecessor) - && !isTaskKilled(predecessor)); + .allMatch(predecessor -> isTaskExecutionRunnableInActive(predecessor) + && !isTaskExecutionRunnableFailed(predecessor) + && !isTaskExecutionRunnablePaused(predecessor) + && !isTaskExecutionRunnableKilled(predecessor)); } @Override @@ -209,6 +227,7 @@ public void markTaskExecutionRunnableActive(final ITaskExecutionRunnable taskExe @Override public void markTaskExecutionRunnableInActive(final ITaskExecutionRunnable taskExecutionRunnable) { activeTaskExecutionRunnable.remove(taskExecutionRunnable.getName()); + inActiveTaskExecutionRunnable.add(taskExecutionRunnable.getName()); } @Override @@ -242,8 +261,9 @@ public void markTaskSkipped(final String taskName) { @Override public boolean isEndOfTaskChain(final ITaskExecutionRunnable taskExecutionRunnable) { return successors.get(taskExecutionRunnable.getName()).isEmpty() - || killedTaskChains.contains(taskExecutionRunnable.getName()) - || pausedTaskChains.contains(taskExecutionRunnable.getName()); + || isTaskExecutionRunnableKilled(taskExecutionRunnable) + || isTaskExecutionRunnablePaused(taskExecutionRunnable) + || isTaskExecutionRunnableFailed(taskExecutionRunnable); } @Override @@ -291,22 +311,6 @@ public boolean isAllSuccessorsAreConditionTask(final ITaskExecutionRunnable task || TaskTypeUtils.isConditionTask(taskExecutionRunnable.getTaskInstance().getTaskType())); } - private boolean isTaskFinish(final ITaskExecutionRunnable taskExecutionRunnable) { - return !activeTaskExecutionRunnable.contains(taskExecutionRunnable.getName()); - } - - private boolean isTaskFailure(final ITaskExecutionRunnable taskExecutionRunnable) { - return failureTaskChains.contains(taskExecutionRunnable.getName()); - } - - private boolean isTaskPaused(final ITaskExecutionRunnable taskExecutionRunnable) { - return pausedTaskChains.contains(taskExecutionRunnable.getName()); - } - - private boolean isTaskKilled(final ITaskExecutionRunnable taskExecutionRunnable) { - return killedTaskChains.contains(taskExecutionRunnable.getName()); - } - private void assertTaskExecutionRunnableState(final ITaskExecutionRunnable taskExecutionRunnable, final TaskExecutionStatus taskExecutionStatus) { final TaskInstance taskInstance = taskExecutionRunnable.getTaskInstance(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java index 1cd6ad21b72e..4e05255d6426 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java @@ -115,8 +115,8 @@ protected void pauseActiveTask(final IWorkflowExecutionRunnable workflowExecutio } } - protected void onTaskFinish(final IWorkflowExecutionRunnable workflowExecutionRunnable, - final ITaskExecutionRunnable taskExecutionRunnable) { + protected void tryToTriggerSuccessorsAfterTaskFinish(final IWorkflowExecutionRunnable workflowExecutionRunnable, + final ITaskExecutionRunnable taskExecutionRunnable) { final IWorkflowExecutionGraph workflowExecutionGraph = workflowExecutionRunnable.getWorkflowExecutionGraph(); if (workflowExecutionGraph.isEndOfTaskChain(taskExecutionRunnable)) { emitWorkflowFinishedEventIfApplicable(workflowExecutionRunnable); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowReadyPauseStateAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowReadyPauseStateAction.java index 1d1925f272f5..cb48b33acde2 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowReadyPauseStateAction.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowReadyPauseStateAction.java @@ -52,7 +52,7 @@ public void startEventAction(final IWorkflowExecutionRunnable workflowExecutionR public void topologyLogicalTransitionEventAction(final IWorkflowExecutionRunnable workflowExecutionRunnable, final WorkflowTopologyLogicalTransitionWithTaskFinishLifecycleEvent workflowTopologyLogicalTransitionWithTaskFinishEvent) { throwExceptionIfStateIsNotMatch(workflowExecutionRunnable); - super.onTaskFinish(workflowExecutionRunnable, + super.tryToTriggerSuccessorsAfterTaskFinish(workflowExecutionRunnable, workflowTopologyLogicalTransitionWithTaskFinishEvent.getTaskExecutionRunnable()); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowReadyStopStateAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowReadyStopStateAction.java index 8b1f393ffe63..c5c94c18c588 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowReadyStopStateAction.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowReadyStopStateAction.java @@ -52,7 +52,7 @@ public void startEventAction(final IWorkflowExecutionRunnable workflowExecutionR public void topologyLogicalTransitionEventAction(final IWorkflowExecutionRunnable workflowExecutionRunnable, final WorkflowTopologyLogicalTransitionWithTaskFinishLifecycleEvent workflowTopologyLogicalTransitionWithTaskFinishEvent) { throwExceptionIfStateIsNotMatch(workflowExecutionRunnable); - super.onTaskFinish(workflowExecutionRunnable, + super.tryToTriggerSuccessorsAfterTaskFinish(workflowExecutionRunnable, workflowTopologyLogicalTransitionWithTaskFinishEvent.getTaskExecutionRunnable()); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowRunningStateAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowRunningStateAction.java index 2ef810aba452..15e360cdb614 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowRunningStateAction.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/WorkflowRunningStateAction.java @@ -53,7 +53,7 @@ public void topologyLogicalTransitionEventAction( final IWorkflowExecutionRunnable workflowExecutionRunnable, final WorkflowTopologyLogicalTransitionWithTaskFinishLifecycleEvent workflowTopologyLogicalTransitionWithTaskFinishEvent) { throwExceptionIfStateIsNotMatch(workflowExecutionRunnable); - super.onTaskFinish(workflowExecutionRunnable, + super.tryToTriggerSuccessorsAfterTaskFinish(workflowExecutionRunnable, workflowTopologyLogicalTransitionWithTaskFinishEvent.getTaskExecutionRunnable()); } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowStartTestCase.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowStartTestCase.java index 55558eece228..1041a34a521e 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowStartTestCase.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowStartTestCase.java @@ -252,6 +252,90 @@ public void testStartWorkflow_with_subWorkflowTask_dryRunSuccess() { masterContainer.assertAllResourceReleased(); } + @Test + @DisplayName("Test start a workflow with one fake task(A) with multiple predecessors run success") + void testStartWorkflow_with_oneTaskWithMultiplePredecessors_runSuccess() { + final String yaml = "/it/start/workflow_with_one_fake_task_with_multiple_predecessors_success.yaml"; + final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml); + final WorkflowDefinition parentWorkflow = context.getOneWorkflow(); + + final WorkflowOperator.WorkflowTriggerDTO workflowTriggerDTO = WorkflowOperator.WorkflowTriggerDTO.builder() + .workflowDefinition(parentWorkflow) + .runWorkflowCommandParam(new RunWorkflowCommandParam()) + .build(); + final Integer workflowInstanceId = workflowOperator.manualTriggerWorkflow(workflowTriggerDTO); + + await() + .atMost(Duration.ofMinutes(1)) + .untilAsserted(() -> { + Assertions + .assertThat(repository.queryWorkflowInstance(workflowInstanceId)) + .matches( + workflowInstance -> workflowInstance.getState() == WorkflowExecutionStatus.SUCCESS); + + Assertions + .assertThat(repository.queryTaskInstance(workflowInstanceId)) + .hasSize(4) + .anySatisfy(taskInstance -> { + assertThat(taskInstance.getName()).isEqualTo("A"); + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.SUCCESS); + }) + .anySatisfy(taskInstance -> { + assertThat(taskInstance.getName()).isEqualTo("B"); + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.SUCCESS); + }) + .anySatisfy(taskInstance -> { + assertThat(taskInstance.getName()).isEqualTo("C"); + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.SUCCESS); + }) + .anySatisfy(taskInstance -> { + assertThat(taskInstance.getName()).isEqualTo("D"); + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.SUCCESS); + }); + }); + masterContainer.assertAllResourceReleased(); + } + + @Test + @DisplayName("Test start a workflow with one fake task(A) with multiple predecessors run failed") + void testStartWorkflow_with_oneTaskWithMultiplePredecessors_runFailed() { + final String yaml = "/it/start/workflow_with_one_fake_task_with_multiple_predecessors_failed.yaml"; + final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml); + final WorkflowDefinition parentWorkflow = context.getOneWorkflow(); + + final WorkflowOperator.WorkflowTriggerDTO workflowTriggerDTO = WorkflowOperator.WorkflowTriggerDTO.builder() + .workflowDefinition(parentWorkflow) + .runWorkflowCommandParam(new RunWorkflowCommandParam()) + .build(); + final Integer workflowInstanceId = workflowOperator.manualTriggerWorkflow(workflowTriggerDTO); + + await() + .atMost(Duration.ofMinutes(1)) + .untilAsserted(() -> { + Assertions + .assertThat(repository.queryWorkflowInstance(workflowInstanceId)) + .matches( + workflowInstance -> workflowInstance.getState() == WorkflowExecutionStatus.FAILURE); + + Assertions + .assertThat(repository.queryTaskInstance(workflowInstanceId)) + .hasSize(3) + .anySatisfy(taskInstance -> { + assertThat(taskInstance.getName()).isEqualTo("A"); + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.SUCCESS); + }) + .anySatisfy(taskInstance -> { + assertThat(taskInstance.getName()).isEqualTo("B"); + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.FAILURE); + }) + .anySatisfy(taskInstance -> { + assertThat(taskInstance.getName()).isEqualTo("C"); + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.SUCCESS); + }); + }); + masterContainer.assertAllResourceReleased(); + } + @Test @DisplayName("Test start a workflow with one sub workflow task(A) failed") public void testStartWorkflow_with_subWorkflowTask_failed() { diff --git a/dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_with_multiple_predecessors_failed.yaml b/dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_with_multiple_predecessors_failed.yaml new file mode 100644 index 000000000000..075a45da337c --- /dev/null +++ b/dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_with_multiple_predecessors_failed.yaml @@ -0,0 +1,132 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# A(success) -> B(failed) -> D(success) +# C(success) -> D(success) +project: + name: MasterIntegrationTest + code: 1 + description: This is a fake project + userId: 1 + userName: admin + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + +workflows: + - name: workflow_with_one_fake_task_with_multiple_predecessors_failed + code: 1 + version: 1 + projectCode: 1 + description: This is a fake workflow with one task which has multiple predecessors + releaseState: ONLINE + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + userId: 1 + executionType: PARALLEL + +tasks: + - name: A + code: 1 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"sleep 2"}' + workerGroup: default + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + - name: B + code: 2 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"xx"}' + workerGroup: default + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + - name: C + code: 3 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"echo success"}' + workerGroup: default + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + - name: D + code: 4 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"echo success"}' + workerGroup: default + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + +taskRelations: + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 0 + preTaskVersion: 0 + postTaskCode: 1 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 1 + preTaskVersion: 1 + postTaskCode: 2 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 2 + preTaskVersion: 1 + postTaskCode: 4 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 0 + preTaskVersion: 0 + postTaskCode: 3 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 3 + preTaskVersion: 1 + postTaskCode: 4 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 diff --git a/dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_with_multiple_predecessors_success.yaml b/dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_with_multiple_predecessors_success.yaml new file mode 100644 index 000000000000..53cc326986ac --- /dev/null +++ b/dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_with_multiple_predecessors_success.yaml @@ -0,0 +1,132 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# A(success) -> B(success) -> D(success) +# C(success) -> D(success) +project: + name: MasterIntegrationTest + code: 1 + description: This is a fake project + userId: 1 + userName: admin + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + +workflows: + - name: workflow_with_one_fake_task_with_multiple_predecessors_success + code: 1 + version: 1 + projectCode: 1 + description: This is a fake workflow with one task which has multiple predecessors + releaseState: ONLINE + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + userId: 1 + executionType: PARALLEL + +tasks: + - name: A + code: 1 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"sleep 2"}' + workerGroup: default + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + - name: B + code: 2 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"echo success"}' + workerGroup: default + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + - name: C + code: 3 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"echo success"}' + workerGroup: default + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + - name: D + code: 4 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"echo success"}' + workerGroup: default + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + +taskRelations: + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 0 + preTaskVersion: 0 + postTaskCode: 1 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 1 + preTaskVersion: 1 + postTaskCode: 2 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 2 + preTaskVersion: 1 + postTaskCode: 4 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 0 + preTaskVersion: 0 + postTaskCode: 3 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 3 + preTaskVersion: 1 + postTaskCode: 4 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 From 806f05155f1df531609ef4e1b11590799e65a97f Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Fri, 7 Feb 2025 12:09:17 +0800 Subject: [PATCH 13/15] [Fix-16991] Missing environmentConfig when retry/failover/recover task instance (#16998) --- .../dao/repository/IEnvironmentDao.java | 27 ++++ .../repository/impl/EnvironmentDaoImpl.java | 42 ++++++ .../dao/utils/TaskInstanceUtils.java | 80 ------------ .../dao/utils/TaskInstanceUtilsTest.java | 43 ------ .../executor/plugin/fake/LogicFakeTask.java | 8 +- .../runnable/AbstractTaskInstanceFactory.java | 12 -- .../runnable/FirstRunTaskInstanceFactory.java | 1 - .../runnable/TaskExecutionContextBuilder.java | 44 +++---- .../runner/TaskExecutionContextFactory.java | 110 ++++++++++------ .../integration/WorkflowTestCaseContext.java | 3 + .../WorkflowTestCaseContextFactory.java | 14 ++ .../WorkflowInstanceFailoverTestCase.java | 36 +++++ .../cases/WorkflowStartTestCase.java | 32 ++++- ...e_running_fake_task_using_environment.yaml | 123 ++++++++++++++++++ ...e_fake_task_using_environment_success.yaml | 72 ++++++++++ .../plugin/task/api/TaskExecutionContext.java | 4 - 16 files changed, 440 insertions(+), 211 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IEnvironmentDao.java create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/EnvironmentDaoImpl.java delete mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java delete mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java create mode 100644 dolphinscheduler-master/src/test/resources/it/failover/running_workflowInstance_with_one_running_fake_task_using_environment.yaml create mode 100644 dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_using_environment_success.yaml diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IEnvironmentDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IEnvironmentDao.java new file mode 100644 index 000000000000..4bcd5988044f --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/IEnvironmentDao.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository; + +import org.apache.dolphinscheduler.dao.entity.Environment; + +import java.util.Optional; + +public interface IEnvironmentDao extends IDao { + + Optional queryByEnvironmentCode(Long environmentCode); +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/EnvironmentDaoImpl.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/EnvironmentDaoImpl.java new file mode 100644 index 000000000000..00f639e55cd3 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/EnvironmentDaoImpl.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository.impl; + +import org.apache.dolphinscheduler.dao.entity.Environment; +import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper; +import org.apache.dolphinscheduler.dao.repository.BaseDao; +import org.apache.dolphinscheduler.dao.repository.IEnvironmentDao; + +import java.util.Optional; + +import lombok.NonNull; + +import org.springframework.stereotype.Repository; + +@Repository +public class EnvironmentDaoImpl extends BaseDao implements IEnvironmentDao { + + public EnvironmentDaoImpl(@NonNull EnvironmentMapper environmentMapper) { + super(environmentMapper); + } + + @Override + public Optional queryByEnvironmentCode(Long environmentCode) { + return Optional.ofNullable(mybatisMapper.queryByEnvironmentCode(environmentCode)); + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java deleted file mode 100644 index 8c7039c6999a..000000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.utils; - -import org.apache.dolphinscheduler.dao.entity.TaskInstance; - -public class TaskInstanceUtils { - - /** - * Copy the property of given source {@link TaskInstance} to target. - * - * @param source Given task instance, copy from. - * @param target Given task instance, copy to - * @return a soft copy of given task instance. - */ - public static void copyTaskInstance(TaskInstance source, TaskInstance target) { - target.setId(source.getId()); - target.setName(source.getName()); - target.setTaskType(source.getTaskType()); - target.setWorkflowInstanceId(source.getWorkflowInstanceId()); - target.setWorkflowInstanceName(source.getWorkflowInstanceName()); - target.setProjectCode(source.getProjectCode()); - target.setTaskCode(source.getTaskCode()); - target.setTaskDefinitionVersion(source.getTaskDefinitionVersion()); - target.setWorkflowInstanceName(source.getWorkflowInstanceName()); - target.setTaskGroupPriority(source.getTaskGroupPriority()); - target.setState(source.getState()); - target.setFirstSubmitTime(source.getFirstSubmitTime()); - target.setSubmitTime(source.getSubmitTime()); - target.setStartTime(source.getStartTime()); - target.setEndTime(source.getEndTime()); - target.setHost(source.getHost()); - target.setExecutePath(source.getExecutePath()); - target.setLogPath(source.getLogPath()); - target.setRetryTimes(source.getRetryTimes()); - target.setAlertFlag(source.getAlertFlag()); - target.setWorkflowInstance(source.getWorkflowInstance()); - target.setWorkflowDefinition(source.getWorkflowDefinition()); - target.setTaskDefine(source.getTaskDefine()); - target.setPid(source.getPid()); - target.setAppLink(source.getAppLink()); - target.setFlag(source.getFlag()); - // todo: we need to cpoy the task params and then copy switchDependency, since the setSwitchDependency rely on - // task params, this is really a very bad practice. - target.setTaskParams(source.getTaskParams()); - target.setDuration(source.getDuration()); - target.setMaxRetryTimes(source.getMaxRetryTimes()); - target.setRetryInterval(source.getRetryInterval()); - target.setTaskInstancePriority(source.getTaskInstancePriority()); - target.setWorkerGroup(source.getWorkerGroup()); - target.setEnvironmentCode(source.getEnvironmentCode()); - target.setEnvironmentConfig(source.getEnvironmentConfig()); - target.setExecutorId(source.getExecutorId()); - target.setVarPool(source.getVarPool()); - target.setExecutorName(source.getExecutorName()); - target.setDelayTime(source.getDelayTime()); - target.setDryRun(source.getDryRun()); - target.setTaskGroupId(source.getTaskGroupId()); - target.setCpuQuota(source.getCpuQuota()); - target.setMemoryMax(source.getMemoryMax()); - target.setTaskExecuteType(source.getTaskExecuteType()); - target.setTestFlag(source.getTestFlag()); - } - -} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java deleted file mode 100644 index cb74aeb44e03..000000000000 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.utils; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; - -import java.util.Date; -import java.util.HashMap; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class TaskInstanceUtilsTest { - - @Test - void copyTaskInstance() { - TaskInstance source = new TaskInstance(); - source.setId(1); - source.setName("source"); - source.setSubmitTime(new Date()); - source.setTaskParams(JSONUtils.toJsonString(new HashMap<>())); - TaskInstance target = new TaskInstance(); - TaskInstanceUtils.copyTaskInstance(source, target); - Assertions.assertEquals(target.getId(), source.getId()); - Assertions.assertEquals(target.getName(), source.getName()); - } -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/fake/LogicFakeTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/fake/LogicFakeTask.java index b05dc8cff498..409d52976134 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/fake/LogicFakeTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/fake/LogicFakeTask.java @@ -26,6 +26,8 @@ import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ITaskParameterDeserializer; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; +import org.apache.commons.lang3.StringUtils; + import lombok.extern.slf4j.Slf4j; import com.fasterxml.jackson.core.type.TypeReference; @@ -51,9 +53,13 @@ public void start() throws MasterTaskExecuteException { try { log.info("Begin to execute LogicFakeTask: {}", taskExecutionContext.getTaskName()); - final String shellScript = ParameterUtils.convertParameterPlaceholders( + String shellScript = ParameterUtils.convertParameterPlaceholders( taskParameters.getShellScript(), ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap())); + + if (StringUtils.isNotEmpty(taskExecutionContext.getEnvironmentConfig())) { + shellScript = taskExecutionContext.getEnvironmentConfig() + "\n" + shellScript; + } final String[] cmd = {"/bin/sh", "-c", shellScript}; process = Runtime.getRuntime().exec(cmd); int exitCode = process.waitFor(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/AbstractTaskInstanceFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/AbstractTaskInstanceFactory.java index 0db29c77a0bc..37993f9405b1 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/AbstractTaskInstanceFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/AbstractTaskInstanceFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.engine.task.runnable; -import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; @@ -62,7 +61,6 @@ protected TaskInstance cloneTaskInstance(TaskInstance originTaskInstance) { result.setTaskInstancePriority(originTaskInstance.getTaskInstancePriority()); result.setWorkerGroup(originTaskInstance.getWorkerGroup()); result.setEnvironmentCode(originTaskInstance.getEnvironmentCode()); - result.setEnvironmentConfig(originTaskInstance.getEnvironmentConfig()); result.setExecutorId(originTaskInstance.getExecutorId()); result.setVarPool(originTaskInstance.getVarPool()); result.setExecutorName(originTaskInstance.getExecutorName()); @@ -116,14 +114,4 @@ protected void injectMetadataFromWorkflowInstance(TaskInstance taskInstance, Wor taskInstance.setTestFlag(workflowInstance.getTestFlag()); } - protected void injectEnvironmentConfigFromDB(TaskInstance taskInstance) { - if (EnvironmentUtils.isEnvironmentCodeEmpty(taskInstance.getEnvironmentCode())) { - return; - } - Environment environment = environmentMapper.queryByEnvironmentCode(taskInstance.getEnvironmentCode()); - if (environment == null) { - throw new IllegalArgumentException("Cannot find the environment: " + taskInstance.getEnvironmentCode()); - } - taskInstance.setEnvironmentConfig(environment.getConfig()); - } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/FirstRunTaskInstanceFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/FirstRunTaskInstanceFactory.java index 1d624295be19..bdd0bd86b72c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/FirstRunTaskInstanceFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/FirstRunTaskInstanceFactory.java @@ -51,7 +51,6 @@ public TaskInstance createTaskInstance(FirstRunTaskInstanceBuilder builder) { TaskInstance taskInstance = new TaskInstance(); injectMetadataFromTaskDefinition(taskInstance, taskDefinition); injectMetadataFromWorkflowInstance(taskInstance, workflowInstance); - injectEnvironmentConfigFromDB(taskInstance); taskInstance.setState(TaskExecutionStatus.SUBMITTED_SUCCESS); taskInstance.setFirstSubmitTime(new Date()); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionContextBuilder.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionContextBuilder.java index b5df29ec86f0..edbfb9af4d11 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionContextBuilder.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionContextBuilder.java @@ -23,11 +23,9 @@ import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition; import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; @@ -48,7 +46,11 @@ public static TaskExecutionContextBuilder get() { return new TaskExecutionContextBuilder(); } - private TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); + private final TaskExecutionContext taskExecutionContext; + + public TaskExecutionContextBuilder() { + this.taskExecutionContext = new TaskExecutionContext(); + } /** * build taskInstance related info @@ -56,7 +58,7 @@ public static TaskExecutionContextBuilder get() { * @param taskInstance taskInstance * @return TaskExecutionContextBuilder */ - public TaskExecutionContextBuilder buildTaskInstanceRelatedInfo(TaskInstance taskInstance) { + public TaskExecutionContextBuilder buildTaskInstanceRelatedInfo(final TaskInstance taskInstance) { taskExecutionContext.setTaskInstanceId(taskInstance.getId()); taskExecutionContext.setTaskName(taskInstance.getName()); taskExecutionContext.setFirstSubmitTime(DateUtils.dateToTimeStamp(taskInstance.getFirstSubmitTime())); @@ -64,19 +66,17 @@ public TaskExecutionContextBuilder buildTaskInstanceRelatedInfo(TaskInstance tas taskExecutionContext.setTaskType(taskInstance.getTaskType()); taskExecutionContext.setLogPath(taskInstance.getLogPath()); taskExecutionContext.setWorkerGroup(taskInstance.getWorkerGroup()); - taskExecutionContext.setEnvironmentConfig(taskInstance.getEnvironmentConfig()); taskExecutionContext.setHost(taskInstance.getHost()); taskExecutionContext.setVarPool(taskInstance.getVarPool()); taskExecutionContext.setDryRun(taskInstance.getDryRun()); taskExecutionContext.setTestFlag(taskInstance.getTestFlag()); - taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUBMITTED_SUCCESS); taskExecutionContext.setCpuQuota(taskInstance.getCpuQuota()); taskExecutionContext.setMemoryMax(taskInstance.getMemoryMax()); taskExecutionContext.setAppIds(taskInstance.getAppLink()); return this; } - public TaskExecutionContextBuilder buildTaskDefinitionRelatedInfo(TaskDefinition taskDefinition) { + public TaskExecutionContextBuilder buildTaskDefinitionRelatedInfo(final TaskDefinition taskDefinition) { // todo: remove the timeout setting here the timeout strategy should be used at master taskExecutionContext.setTaskTimeout(Integer.MAX_VALUE); if (taskDefinition.getTimeoutFlag() == TimeoutFlag.OPEN) { @@ -97,7 +97,7 @@ public TaskExecutionContextBuilder buildTaskDefinitionRelatedInfo(TaskDefinition * @param workflowInstance processInstance * @return TaskExecutionContextBuilder */ - public TaskExecutionContextBuilder buildProcessInstanceRelatedInfo(WorkflowInstance workflowInstance) { + public TaskExecutionContextBuilder buildProcessInstanceRelatedInfo(final WorkflowInstance workflowInstance) { taskExecutionContext.setWorkflowInstanceId(workflowInstance.getId()); taskExecutionContext.setScheduleTime(DateUtils.dateToTimeStamp(workflowInstance.getScheduleTime())); taskExecutionContext.setGlobalParams(workflowInstance.getGlobalParams()); @@ -110,20 +110,7 @@ public TaskExecutionContextBuilder buildProcessInstanceRelatedInfo(WorkflowInsta return this; } - /** - * build processDefinition related info - * - * @param workflowDefinition processDefinition - * @return TaskExecutionContextBuilder - */ - public TaskExecutionContextBuilder buildProcessDefinitionRelatedInfo(WorkflowDefinition workflowDefinition) { - taskExecutionContext.setWorkflowDefinitionCode(workflowDefinition.getCode()); - taskExecutionContext.setWorkflowDefinitionVersion(workflowDefinition.getVersion()); - taskExecutionContext.setProjectCode(workflowDefinition.getProjectCode()); - return this; - } - - public TaskExecutionContextBuilder buildResourceParametersInfo(ResourceParametersHelper parametersHelper) { + public TaskExecutionContextBuilder buildResourceParameters(final ResourceParametersHelper parametersHelper) { taskExecutionContext.setResourceParametersHelper(parametersHelper); return this; } @@ -135,7 +122,7 @@ public TaskExecutionContextBuilder buildResourceParametersInfo(ResourceParameter * @return TaskExecutionContextBuilder */ - public TaskExecutionContextBuilder buildK8sTaskRelatedInfo(K8sTaskExecutionContext k8sTaskExecutionContext) { + public TaskExecutionContextBuilder buildK8sTaskRelatedInfo(final K8sTaskExecutionContext k8sTaskExecutionContext) { taskExecutionContext.setK8sTaskExecutionContext(k8sTaskExecutionContext); return this; } @@ -146,7 +133,7 @@ public TaskExecutionContextBuilder buildK8sTaskRelatedInfo(K8sTaskExecutionConte * @param propertyMap * @return */ - public TaskExecutionContextBuilder buildParamInfo(Map propertyMap) { + public TaskExecutionContextBuilder buildPrepareParams(final Map propertyMap) { taskExecutionContext.setPrepareParamsMap(propertyMap); return this; } @@ -157,16 +144,21 @@ public TaskExecutionContextBuilder buildParamInfo(Map property * @param businessParamsMap * @return */ - public TaskExecutionContextBuilder buildBusinessParamsMap(Map businessParamsMap) { + public TaskExecutionContextBuilder buildBusinessParams(final Map businessParamsMap) { taskExecutionContext.setParamsMap(businessParamsMap); return this; } - public TaskExecutionContextBuilder buildWorkflowInstanceHost(String masterHost) { + public TaskExecutionContextBuilder buildWorkflowInstanceHost(final String masterHost) { taskExecutionContext.setWorkflowInstanceHost(masterHost); return this; } + public TaskExecutionContextBuilder buildEnvironmentConfig(final String environmentConfig) { + taskExecutionContext.setEnvironmentConfig(environmentConfig); + return this; + } + /** * create * diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java index 2eda4689d718..327cc6de9937 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java @@ -22,10 +22,13 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition; import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; +import org.apache.dolphinscheduler.dao.repository.IEnvironmentDao; +import org.apache.dolphinscheduler.dao.utils.EnvironmentUtils; import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; @@ -46,6 +49,7 @@ import java.util.Map; import java.util.Objects; +import java.util.Optional; import lombok.extern.slf4j.Slf4j; @@ -65,60 +69,48 @@ public class TaskExecutionContextFactory { @Autowired private MasterConfig masterConfig; - public TaskExecutionContext createTaskExecutionContext(TaskExecutionContextCreateRequest request) { - TaskInstance taskInstance = request.getTaskInstance(); - WorkflowInstance workflowInstance = request.getWorkflowInstance(); - WorkflowDefinition workflowDefinition = request.getWorkflowDefinition(); - Project project = request.getProject(); - - ResourceParametersHelper resources = TaskPluginManager.getTaskChannel(taskInstance.getTaskType()) - .parseParameters(taskInstance.getTaskParams()) - .getResources(); - setTaskResourceInfo(resources); - - Map businessParamsMap = curingParamsService.preBuildBusinessParams(workflowInstance); + @Autowired + private IEnvironmentDao environmentDao; - AbstractParameters baseParam = - TaskPluginManager.parseTaskParameters(taskInstance.getTaskType(), taskInstance.getTaskParams()); + public TaskExecutionContext createTaskExecutionContext(TaskExecutionContextCreateRequest request) { + final TaskInstance taskInstance = request.getTaskInstance(); + final WorkflowInstance workflowInstance = request.getWorkflowInstance(); + final WorkflowDefinition workflowDefinition = request.getWorkflowDefinition(); + final Project project = request.getProject(); - Map propertyMap = - curingParamsService.paramParsingPreparation(taskInstance, baseParam, workflowInstance, - project.getName(), workflowDefinition.getName()); - TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get() + return TaskExecutionContextBuilder.get() .buildWorkflowInstanceHost(masterConfig.getMasterAddress()) .buildTaskInstanceRelatedInfo(taskInstance) + .buildEnvironmentConfig(getEnvironmentConfigFromDB(taskInstance).orElse(null)) .buildTaskDefinitionRelatedInfo(request.getTaskDefinition()) .buildProcessInstanceRelatedInfo(request.getWorkflowInstance()) - .buildResourceParametersInfo(resources) - .buildBusinessParamsMap(businessParamsMap) - .buildParamInfo(propertyMap) + .buildResourceParameters(getResourceParameters(taskInstance)) + .buildBusinessParams(getBusinessParams(workflowInstance)) + .buildPrepareParams(getPrepareParams(taskInstance, workflowInstance, workflowDefinition, project)) + .buildK8sTaskRelatedInfo(getK8sTaskExecutionContext(taskInstance)) .create(); - - setK8sTaskRelatedInfo(taskExecutionContext, taskInstance); - return taskExecutionContext; - } - - public void setK8sTaskRelatedInfo(TaskExecutionContext taskExecutionContext, TaskInstance taskInstance) { - K8sTaskExecutionContext k8sTaskExecutionContext = setK8sTaskRelation(taskInstance); - taskExecutionContext.setK8sTaskExecutionContext(k8sTaskExecutionContext); } - private void setTaskResourceInfo(ResourceParametersHelper resourceParametersHelper) { - if (Objects.isNull(resourceParametersHelper)) { - return; + private ResourceParametersHelper getResourceParameters(final TaskInstance taskInstance) { + final ResourceParametersHelper resourceParameters = TaskPluginManager.getTaskChannel(taskInstance.getTaskType()) + .parseParameters(taskInstance.getTaskParams()) + .getResources(); + if (resourceParameters != null) { + // todo: add DataSourceParametersAssembler to assemble DataSourceParameters + resourceParameters.getResourceMap().forEach((type, map) -> { + switch (type) { + case DATASOURCE: + assembleDataSourceParameters(map); + break; + default: + break; + } + }); } - resourceParametersHelper.getResourceMap().forEach((type, map) -> { - switch (type) { - case DATASOURCE: - setTaskDataSourceResourceInfo(map); - break; - default: - break; - } - }); + return resourceParameters; } - private void setTaskDataSourceResourceInfo(Map map) { + private void assembleDataSourceParameters(Map map) { if (MapUtils.isEmpty(map)) { return; } @@ -135,7 +127,7 @@ private void setTaskDataSourceResourceInfo(Map getBusinessParams(final WorkflowInstance workflowInstance) { + return curingParamsService.preBuildBusinessParams(workflowInstance); + } + + private Map getPrepareParams(final TaskInstance taskInstance, + final WorkflowInstance workflowInstance, + final WorkflowDefinition workflowDefinition, + final Project project) { + final AbstractParameters baseParam = TaskPluginManager.parseTaskParameters( + taskInstance.getTaskType(), + taskInstance.getTaskParams()); + + return curingParamsService.paramParsingPreparation( + taskInstance, + baseParam, + workflowInstance, + project.getName(), + workflowDefinition.getName()); + } + + private Optional getEnvironmentConfigFromDB(final TaskInstance taskInstance) { + if (EnvironmentUtils.isEnvironmentCodeEmpty(taskInstance.getEnvironmentCode())) { + return Optional.empty(); + } + final Optional environmentOptional = + environmentDao.queryByEnvironmentCode(taskInstance.getEnvironmentCode()); + if (!environmentOptional.isPresent()) { + throw new IllegalArgumentException("Cannot find the environment: " + taskInstance.getEnvironmentCode()); + } + return Optional.ofNullable(environmentOptional.get().getConfig()); + } + } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContext.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContext.java index 7d152ea284e0..a0c5c83e2eb3 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContext.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContext.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.integration; +import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskGroup; @@ -52,6 +53,8 @@ public class WorkflowTestCaseContext { private List taskGroups; + private List environments; + public WorkflowDefinition getOneWorkflow() { if (CollectionUtils.isEmpty(workflows)) { throw new IllegalStateException("workflows is empty"); diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContextFactory.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContextFactory.java index 1a5804fd99e0..c5de1a4d5d0f 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContextFactory.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContextFactory.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.integration; +import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; @@ -28,6 +29,7 @@ import org.apache.dolphinscheduler.dao.entity.WorkflowTaskRelation; import org.apache.dolphinscheduler.dao.entity.WorkflowTaskRelationLog; import org.apache.dolphinscheduler.dao.mapper.WorkflowTaskRelationMapper; +import org.apache.dolphinscheduler.dao.repository.IEnvironmentDao; import org.apache.dolphinscheduler.dao.repository.ProjectDao; import org.apache.dolphinscheduler.dao.repository.TaskDefinitionDao; import org.apache.dolphinscheduler.dao.repository.TaskDefinitionLogDao; @@ -79,6 +81,9 @@ public class WorkflowTestCaseContextFactory { @Autowired private TaskGroupDao taskGroupDao; + @Autowired + private IEnvironmentDao environmentDao; + public WorkflowTestCaseContext initializeContextFromYaml(final String yamlPath) { final WorkflowTestCaseContext workflowTestCaseContext = YamlFactory.load(yamlPath); initializeProjectToDB(workflowTestCaseContext.getProject()); @@ -94,6 +99,9 @@ public WorkflowTestCaseContext initializeContextFromYaml(final String yamlPath) if (CollectionUtils.isNotEmpty(workflowTestCaseContext.getTaskGroups())) { initializeTaskGroupsToDB(workflowTestCaseContext.getTaskGroups()); } + if (CollectionUtils.isNotEmpty(workflowTestCaseContext.getEnvironments())) { + initializeEnvironmentToDB(workflowTestCaseContext.getEnvironments()); + } return workflowTestCaseContext; } @@ -148,4 +156,10 @@ private void initializeTaskGroupsToDB(final List taskGroups) { } } + private void initializeEnvironmentToDB(final List environments) { + for (final Environment environment : environments) { + environmentDao.insert(environment); + } + } + } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceFailoverTestCase.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceFailoverTestCase.java index 43571d2b7367..4ef678fe5117 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceFailoverTestCase.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceFailoverTestCase.java @@ -146,6 +146,42 @@ public void testGlobalFailover_runningWorkflow_withRunningTasks() { } + @Test + public void testGlobalFailover_runningWorkflow_withRunningTasksUsingEnvironment() { + final String yaml = "/it/failover/running_workflowInstance_with_one_running_fake_task_using_environment.yaml"; + final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml); + final WorkflowDefinition workflow = context.getOneWorkflow(); + + systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date())); + + await() + .atMost(Duration.ofMinutes(1)) + .untilAsserted(() -> { + assertThat(repository.queryWorkflowInstance(workflow)) + .hasSize(1) + .anySatisfy(workflowInstance -> { + assertThat(workflowInstance.getState()) + .isEqualTo(WorkflowExecutionStatus.SUCCESS); + assertThat(workflowInstance.getName()) + .isEqualTo( + "running_workflowInstance_with_one_running_fake_task_using_environment-20240816071251690"); + }); + final List taskInstances = repository.queryTaskInstance(workflow); + assertThat(taskInstances) + .hasSize(2); + assertThat(taskInstances.get(0)) + .matches(t -> t.getState() == TaskExecutionStatus.NEED_FAULT_TOLERANCE) + .matches(t -> t.getFlag() == Flag.NO); + + assertThat(taskInstances.get(1)) + .matches(t -> t.getState() == TaskExecutionStatus.SUCCESS) + .matches(t -> t.getFlag() == Flag.YES) + .matches(t -> StringUtils.isNotEmpty(t.getLogPath())); + }); + masterContainer.assertAllResourceReleased(); + + } + @Test public void testGlobalFailover_runningWorkflow_withSuccessTasks() { final String yaml = "/it/failover/running_workflowInstance_with_one_success_fake_task.yaml"; diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowStartTestCase.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowStartTestCase.java index 1041a34a521e..19650f8db70d 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowStartTestCase.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowStartTestCase.java @@ -131,7 +131,6 @@ public void testStartWorkflow_with_oneSuccessTaskUsingTaskGroup() { final WorkflowOperator.WorkflowTriggerDTO workflowTriggerDTO = WorkflowOperator.WorkflowTriggerDTO.builder() .workflowDefinition(workflow) .runWorkflowCommandParam(new RunWorkflowCommandParam()) - .dryRun(Flag.YES) .build(); workflowOperator.manualTriggerWorkflow(workflowTriggerDTO); @@ -153,6 +152,37 @@ public void testStartWorkflow_with_oneSuccessTaskUsingTaskGroup() { masterContainer.assertAllResourceReleased(); } + @Test + @DisplayName("Test start a workflow with one fake task(A) using environment config") + public void testStartWorkflow_with_oneSuccessTaskUsingEnvironmentConfig() { + final String yaml = "/it/start/workflow_with_one_fake_task_using_environment_success.yaml"; + final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml); + final WorkflowDefinition workflow = context.getOneWorkflow(); + + final WorkflowOperator.WorkflowTriggerDTO workflowTriggerDTO = WorkflowOperator.WorkflowTriggerDTO.builder() + .workflowDefinition(workflow) + .runWorkflowCommandParam(new RunWorkflowCommandParam()) + .build(); + + final Integer workflowInstanceId = workflowOperator.manualTriggerWorkflow(workflowTriggerDTO); + + await() + .atMost(Duration.ofMinutes(1)) + .untilAsserted(() -> { + Assertions + .assertThat(repository.queryWorkflowInstance(workflowInstanceId)) + .matches( + workflowInstance -> workflowInstance.getState() == WorkflowExecutionStatus.SUCCESS); + Assertions + .assertThat(repository.queryTaskInstance(workflow)) + .satisfiesExactly(taskInstance -> { + assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.SUCCESS); + }); + }); + + masterContainer.assertAllResourceReleased(); + } + @Test @DisplayName("Test start a workflow with one sub workflow task(A) success") public void testStartWorkflow_with_subWorkflowTask_success() { diff --git a/dolphinscheduler-master/src/test/resources/it/failover/running_workflowInstance_with_one_running_fake_task_using_environment.yaml b/dolphinscheduler-master/src/test/resources/it/failover/running_workflowInstance_with_one_running_fake_task_using_environment.yaml new file mode 100644 index 000000000000..c34532994e08 --- /dev/null +++ b/dolphinscheduler-master/src/test/resources/it/failover/running_workflowInstance_with_one_running_fake_task_using_environment.yaml @@ -0,0 +1,123 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +project: + name: MasterIntegrationTest + code: 1 + description: This is a fake project + userId: 1 + userName: admin + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + +workflows: + - name: running_workflowInstance_with_one_running_fake_task_using_environment + code: 1 + version: 1 + projectCode: 1 + description: This is a fake workflow with single task + releaseState: ONLINE + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + userId: 1 + executionType: PARALLEL + +workflowInstance: + id: 1 + name: running_workflowInstance_with_one_running_fake_task_using_environment-20240816071251690 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + projectCode: 1 + state: RUNNING_EXECUTION + recovery: NO + startTime: 2024-08-16 07:12:52 + endTime: null + runTimes: 1 + host: 127.0.0.1:5678 + commandType: START_PROCESS + commandParam: '{"commandType":"START_PROCESS","startNodes":[],"commandParams":[],"timeZone":"UTC"}' + taskDependType: TASK_POST + commandStartTime: 2024-08-16 07:12:52 + isSubWorkflow: NO + executorId: 1 + historyCmd: START_PROCESS + workerGroup: default + globalParams: '[]' + varPool: '[]' + dryRun: 0 + +taskInstances: + - id: 1 + name: A + taskType: LogicFakeTask + workflowInstanceId: 1 + workflowInstanceName: running_workflowInstance_with_one_running_fake_task_using_environment-20240816071251690 + projectCode: 1 + taskCode: 1 + taskDefinitionVersion: 1 + state: RUNNING_EXECUTION + firstSubmitTime: 2024-08-16 07:12:52 + submitTime: 2024-08-16 07:12:57 + startTime: 2024-08-16 07:12:57 + endTime: 2024-08-16 07:12:57 + retryTimes: 0 + host: 127.0.0.1:1234 + maxRetryTimes: 0 + taskParams: '{"localParams":null,"varPool":[],"shellScript":"if [ \"${NAME}\" = \"Wenjun\" ]; then\n exit 0 \nelse\n exit 1\nfi"}' + flag: YES + retryInterval: 0 + delayTime: 0 + workerGroup: default + environmentCode: 1 + executorId: 1 + varPool: '[]' + taskExecuteType: BATCH + +tasks: + - name: A + code: 1 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"if [ \"${NAME}\" = \"Wenjun\" ]; then\n exit 0 \nelse\n exit 1\nfi"}' + workerGroup: default + environmentCode: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + +taskRelations: + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 0 + preTaskVersion: 0 + postTaskCode: 1 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + +environments: + - id: 1 + code: 1 + name: MockEnv + config: 'export NAME=Wenjun' + description: 'test' + operator: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 diff --git a/dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_using_environment_success.yaml b/dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_using_environment_success.yaml new file mode 100644 index 000000000000..0302d376b142 --- /dev/null +++ b/dolphinscheduler-master/src/test/resources/it/start/workflow_with_one_fake_task_using_environment_success.yaml @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +project: + name: MasterIntegrationTest + code: 1 + description: This is a fake project + userId: 1 + userName: admin + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + +workflows: + - name: workflow_with_one_fake_task_using_environment_success + code: 1 + version: 1 + projectCode: 1 + description: This is a fake workflow with single task + releaseState: ONLINE + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + userId: 1 + executionType: PARALLEL + +tasks: + - name: A + code: 1 + version: 1 + projectCode: 1 + userId: 1 + taskType: LogicFakeTask + taskParams: '{"localParams":null,"varPool":[],"shellScript":"if [ \"${NAME}\" = \"Wenjun\" ]; then\n exit 0 \nelse\n exit 1\nfi"}' + workerGroup: default + environmentCode: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 + taskExecuteType: BATCH + +taskRelations: + - projectCode: 1 + workflowDefinitionCode: 1 + workflowDefinitionVersion: 1 + preTaskCode: 0 + preTaskVersion: 0 + postTaskCode: 1 + postTaskVersion: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2024-08-12 00:00:00 + +environments: + - id: 1 + code: 1 + name: MockEnv + config: 'export NAME=Wenjun' + description: 'test' + operator: 1 + createTime: 2024-08-12 00:00:00 + updateTime: 2021-08-12 00:00:00 diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java index 9024be12d510..c7ac6e7ef9a9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.api; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; @@ -113,9 +112,6 @@ public class TaskExecutionContext implements Serializable { private String workerGroup; - @Deprecated - private TaskExecutionStatus currentExecutionStatus; - private ResourceParametersHelper resourceParametersHelper; private long endTime; From 934da2e6eec9b1856cbaec58921337ac0ed3400a Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Sat, 8 Feb 2025 15:20:29 +0800 Subject: [PATCH 14/15] [Improvement-16999] Fix SensitiveDataConverter cannot convert empty data (#17000) --- docs/docs/en/architecture/design.md | 2 +- docs/docs/zh/architecture/design.md | 2 +- .../common/log/SensitiveDataConverter.java | 77 ------------------- .../api/datasource/DataSourceProcessor.java | 3 +- .../datasource/api/utils/DataSourceUtils.java | 10 +-- .../src/main/resources/logback-spring.xml | 2 +- .../src/test/resources/logback.xml | 2 +- .../src/main/resources/logback-spring.xml | 2 +- .../plugin/task/api/TaskConstants.java | 1 + .../task/api/log/SensitiveDataConverter.java | 26 ++++--- .../api}/log/SensitiveDataConverterTest.java | 46 +++++------ .../plugin/task/sqoop/SqoopTaskTest.java | 2 +- .../src/main/resources/logback-spring.xml | 2 +- .../src/test/resources/logback.xml | 2 +- 14 files changed, 49 insertions(+), 130 deletions(-) delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java rename {dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common => dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api}/log/SensitiveDataConverterTest.java (74%) diff --git a/docs/docs/en/architecture/design.md b/docs/docs/en/architecture/design.md index dfd443e9631d..4bc8b06eeda6 100644 --- a/docs/docs/en/architecture/design.md +++ b/docs/docs/en/architecture/design.md @@ -197,7 +197,7 @@ In the early schedule design, if there is no priority design and use the fair sc - For details, please refer to the logback configuration of Master and Worker, as shown in the following example: ```xml - + diff --git a/docs/docs/zh/architecture/design.md b/docs/docs/zh/architecture/design.md index 246bd6b1acaf..5ceffc4e0aff 100644 --- a/docs/docs/zh/architecture/design.md +++ b/docs/docs/zh/architecture/design.md @@ -195,7 +195,7 @@ - 详情可参考Master和Worker的logback配置，如下示例： ```xml - + diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java deleted file mode 100644 index 57b2bcde59a8..000000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.log; - -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import ch.qos.logback.classic.pattern.MessageConverter; -import ch.qos.logback.classic.spi.ILoggingEvent; - -import com.google.common.base.Strings; - -/** - * sensitive data log converter - */ -public class SensitiveDataConverter extends MessageConverter { - - private static Pattern multilinePattern; - private static HashSet maskPatterns = - new HashSet<>(Arrays.asList(DataSourceConstants.DATASOURCE_PASSWORD_REGEX)); - - @Override - public String convert(ILoggingEvent event) { - - // get original log - String requestLogMsg = event.getFormattedMessage(); - - // desensitization log - return maskSensitiveData(requestLogMsg); - } - - public static void addMaskPattern(String maskPattern) { - maskPatterns.add(maskPattern); - } - - public static String maskSensitiveData(final String logMsg) { - if (StringUtils.isEmpty(logMsg)) { - return logMsg; - } - multilinePattern = Pattern.compile(String.join("|", maskPatterns), Pattern.MULTILINE); - - StringBuffer sb = new StringBuffer(logMsg.length()); - Matcher matcher = multilinePattern.matcher(logMsg); - - while (matcher.find()) { - String password = matcher.group(); - String maskPassword = Strings.repeat(Constants.STAR, password.length()); - matcher.appendReplacement(sb, maskPassword); - } - matcher.appendTail(sb); - - return sb.toString(); - } - -} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java index 7c8e313039cc..5a2338a3a596 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java @@ -36,7 +36,8 @@ public interface DataSourceProcessor { BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson); /** - * check datasource param is valid + * check datasource param is valid. + * @throws IllegalArgumentException if invalid */ void checkDatasourceParam(BaseDataSourceParamDTO datasourceParam); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java index e3e43e52f021..20e80d945f8a 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java @@ -46,16 +46,8 @@ public static void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourcePar getDatasourceProcessor(baseDataSourceParamDTO.getType()).checkDatasourceParam(baseDataSourceParamDTO); } - /** - * build connection url - * - * @param baseDataSourceParamDTO datasourceParam - */ public static ConnectionParam buildConnectionParams(BaseDataSourceParamDTO baseDataSourceParamDTO) { - ConnectionParam connectionParams = getDatasourceProcessor(baseDataSourceParamDTO.getType()) - .createConnectionParams(baseDataSourceParamDTO); - log.info("Parameters map:{}", connectionParams); - return connectionParams; + return getDatasourceProcessor(baseDataSourceParamDTO.getType()).createConnectionParams(baseDataSourceParamDTO); } public static ConnectionParam buildConnectionParams(DbType dbType, String connectionJson) { diff --git a/dolphinscheduler-master/src/main/resources/logback-spring.xml b/dolphinscheduler-master/src/main/resources/logback-spring.xml index 5ce1423d0d5e..e366eec71dcb 100644 --- a/dolphinscheduler-master/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-master/src/main/resources/logback-spring.xml @@ -30,7 +30,7 @@ + converterClass="org.apache.dolphinscheduler.plugin.task.api.log.SensitiveDataConverter"/> diff --git a/dolphinscheduler-master/src/test/resources/logback.xml b/dolphinscheduler-master/src/test/resources/logback.xml index 2af91b3daa58..18d1350c24ee 100644 --- a/dolphinscheduler-master/src/test/resources/logback.xml +++ b/dolphinscheduler-master/src/test/resources/logback.xml @@ -30,7 +30,7 @@ + converterClass="org.apache.dolphinscheduler.plugin.task.api.log.SensitiveDataConverter"/> diff --git a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml index 2d74923c30a6..6734f7c6fa52 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml @@ -51,7 +51,7 @@ + converterClass="org.apache.dolphinscheduler.plugin.task.api.log.SensitiveDataConverter"/> diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java index bec200bc309e..48376e0179c5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java @@ -387,4 +387,5 @@ private TaskConstants() { public static final String TASK_INSTANCE_ID_MDC_KEY = "taskInstanceId"; public static final String STAR = "*"; + public static final String SENSITIVE_DATA_MASK = "******"; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverter.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverter.java index 1c90b03139ef..ea70aaef7678 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverter.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverter.java @@ -21,24 +21,25 @@ import org.apache.commons.lang3.StringUtils; -import java.util.Collections; import java.util.HashSet; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import ch.qos.logback.classic.pattern.MessageConverter; import ch.qos.logback.classic.spi.ILoggingEvent; -import com.google.common.base.Strings; - /** * sensitive data log converter */ public class SensitiveDataConverter extends MessageConverter { private static Pattern multilinePattern; - private static HashSet maskPatterns = - new HashSet<>(Collections.singletonList(TaskConstants.DATASOURCE_PASSWORD_REGEX)); + private static final Set maskPatterns = new HashSet<>(); + + static { + addMaskPattern(TaskConstants.DATASOURCE_PASSWORD_REGEX); + } @Override public String convert(ILoggingEvent event) { @@ -50,23 +51,24 @@ public String convert(ILoggingEvent event) { return maskSensitiveData(requestLogMsg); } - public static void addMaskPattern(String maskPattern) { + public static synchronized void addMaskPattern(final String maskPattern) { + if (maskPatterns.contains(maskPattern)) { + return; + } maskPatterns.add(maskPattern); + multilinePattern = Pattern.compile(String.join("|", maskPatterns), Pattern.MULTILINE); } public static String maskSensitiveData(final String logMsg) { if (StringUtils.isEmpty(logMsg)) { return logMsg; } - multilinePattern = Pattern.compile(String.join("|", maskPatterns), Pattern.MULTILINE); - StringBuffer sb = new StringBuffer(logMsg.length()); - Matcher matcher = multilinePattern.matcher(logMsg); + final StringBuffer sb = new StringBuffer(logMsg.length()); + final Matcher matcher = multilinePattern.matcher(logMsg); while (matcher.find()) { - String password = matcher.group(); - String maskPassword = Strings.repeat(TaskConstants.STAR, password.length()); - matcher.appendReplacement(sb, maskPassword); + matcher.appendReplacement(sb, TaskConstants.SENSITIVE_DATA_MASK); } matcher.appendTail(sb); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverterTest.java similarity index 74% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverterTest.java index c641c296b8fb..b1d242c1105a 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverterTest.java @@ -15,26 +15,22 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.log; +package org.apache.dolphinscheduler.plugin.task.api.log; import static org.apache.dolphinscheduler.common.constants.Constants.K8S_CONFIG_REGEX; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.HashMap; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class SensitiveDataConverterTest { - - private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class); +class SensitiveDataConverterTest { /** * mask sensitive logMsg - sql task datasource password */ @Test - public void testPwdLogMsgConverter() { + void testPwdLogMsgConverter() { HashMap tcs = new HashMap<>(); tcs.put("{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + "\"database\":\"carbond\"," @@ -46,7 +42,7 @@ public void testPwdLogMsgConverter() { + "\"database\":\"carbond\"," + "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + "\"user\":\"view\"," - + "\"password\":\"*****\"}"); + + "\"password\":\"******\"}"); tcs.put("End initialize task {\n" + " \"resourceParametersHelper\" : {\n" + @@ -70,7 +66,7 @@ public void testPwdLogMsgConverter() { " \"1\" : {\n" + " \"resourceType\" : \"DATASOURCE\",\n" + " \"type\" : \"ORACLE\",\n" + - " \"connectionParams\" : \"{\\\"user\\\":\\\"user\\\",\\\"password\\\":\\\"*****\\\"}\",\n" + " \"connectionParams\" : \"{\\\"user\\\":\\\"user\\\",\\\"password\\\":\\\"******\\\"}\",\n" + " \"DATASOURCE\" : null\n" + " }\n" + @@ -81,24 +77,32 @@ public void testPwdLogMsgConverter() { for (String logMsg : tcs.keySet()) { String maskedLog = SensitiveDataConverter.maskSensitiveData(logMsg); - logger.info("original parameter : {}", logMsg); - logger.info("masked parameter : {}", maskedLog); - Assertions.assertEquals(tcs.get(logMsg), maskedLog); + assertEquals(tcs.get(logMsg), maskedLog); } } @Test - public void testPostJdbcInfoLogMsgConverter() { + void testPostJdbcInfoLogMsgConverter() { String POST_JDBC_INFO_REGEX = "(?<=(post jdbc info:)).*(?=)"; SensitiveDataConverter.addMaskPattern(POST_JDBC_INFO_REGEX); String postJdbcInfoLogMsg = "post jdbc info:clickhouse,jdbc:clickhouse://127.0.0.1:8123/td_cdp,admin,123%@@56"; final String maskedLog = SensitiveDataConverter.maskSensitiveData(postJdbcInfoLogMsg); - String expectedMsg = "post jdbc info:*****************************************************************"; - Assertions.assertEquals(expectedMsg, maskedLog); + String expectedMsg = "post jdbc info:******"; + assertEquals(expectedMsg, maskedLog); } @Test - public void testK8SLogMsgConverter() { + void testMaskSensitiveDataWithEmptyPassword() { + String postJdbcInfoLogMsg = + "MySQLConnectionParam{user='admin', password='', address='jdbc:mysql://localhost:3306', database='aa', jdbcUrl='jdbc:mysql://localhost:3306/aa', driverLocation='null', driverClassName='com.mysql.cj.jdbc.Driver', validationQuery='select 1', other='null'}"; + final String maskedLog = SensitiveDataConverter.maskSensitiveData(postJdbcInfoLogMsg); + final String expectedMsg = + "MySQLConnectionParam{user='admin', password='******', address='jdbc:mysql://localhost:3306', database='aa', jdbcUrl='jdbc:mysql://localhost:3306/aa', driverLocation='null', driverClassName='com.mysql.cj.jdbc.Driver', validationQuery='select 1', other='null'}"; + assertEquals(expectedMsg, maskedLog); + } + + @Test + void testK8SLogMsgConverter() { String msg = "End initialize task {\n" + " \"taskName\" : \"echo\",\n" + " \"k8sTaskExecutionContext\" : {\n" + @@ -110,7 +114,7 @@ public void testK8SLogMsgConverter() { String maskMsg = "End initialize task {\n" + " \"taskName\" : \"echo\",\n" + " \"k8sTaskExecutionContext\" : {\n" + - " \"configYaml\" : \"**************************************\",\n" + + " \"configYaml\" : \"******\",\n" + " \"namespace\" : \"abc\"\n" + " },\n" + " \"logBufferEnable\" : false\n" + @@ -118,10 +122,6 @@ public void testK8SLogMsgConverter() { SensitiveDataConverter.addMaskPattern(K8S_CONFIG_REGEX); final String maskedLog = SensitiveDataConverter.maskSensitiveData(msg); - logger.info("original parameter : {}", msg); - logger.info("masked parameter : {}", maskedLog); - - Assertions.assertEquals(maskMsg, maskedLog); - + assertEquals(maskMsg, maskedLog); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java index cb0b887a75ec..28c4da571455 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java @@ -30,7 +30,7 @@ public void testSqoopPasswordMask() { "sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"mypassword\" --table student --target-dir /sqoop_test --as-textfile"; final String maskScript = - "sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"**********\" --table student --target-dir /sqoop_test --as-textfile"; + "sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"******\" --table student --target-dir /sqoop_test --as-textfile"; SensitiveDataConverter.addMaskPattern(SqoopConstants.SQOOP_PASSWORD_REGEX); Assertions.assertEquals(maskScript, SensitiveDataConverter.maskSensitiveData(originalScript)); diff --git a/dolphinscheduler-worker/src/main/resources/logback-spring.xml b/dolphinscheduler-worker/src/main/resources/logback-spring.xml index 1bfbdd506de5..01b35c230006 100644 --- a/dolphinscheduler-worker/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-worker/src/main/resources/logback-spring.xml @@ -30,7 +30,7 @@ + converterClass="org.apache.dolphinscheduler.plugin.task.api.log.SensitiveDataConverter"/> diff --git a/dolphinscheduler-worker/src/test/resources/logback.xml b/dolphinscheduler-worker/src/test/resources/logback.xml index 10fc24d97434..c4cbf154681e 100644 --- a/dolphinscheduler-worker/src/test/resources/logback.xml +++ b/dolphinscheduler-worker/src/test/resources/logback.xml @@ -30,7 +30,7 @@ + converterClass="org.apache.dolphinscheduler.plugin.task.api.log.SensitiveDataConverter"/> From c4e2c66ebda764d560c6da17da82035cced1a1d0 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Sun, 9 Feb 2025 21:41:55 +0800 Subject: [PATCH 15/15] [Chore] Set update time when online/offline schedule (#17002) --- .../dolphinscheduler/api/service/impl/SchedulerServiceImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java index ffafa331ccd6..a87d36c7af60 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java @@ -706,6 +706,7 @@ private void doOnlineScheduler(Schedule schedule) { } schedule.setReleaseState(ReleaseState.ONLINE); + schedule.setUpdateTime(new Date()); scheduleMapper.updateById(schedule); Project project = projectMapper.queryByCode(workflowDefinition.getProjectCode()); @@ -735,6 +736,7 @@ private void doOfflineScheduler(Schedule schedule) { log.debug("The schedule is already offline, scheduleId:{}.", schedule.getId()); return; } + schedule.setUpdateTime(new Date()); schedule.setReleaseState(ReleaseState.OFFLINE); scheduleMapper.updateById(schedule); WorkflowDefinition workflowDefinition =