paths = new ArrayList<>();
+ service.supplyFlowCategoryPath(paths, project, flowId);
+ System.out.println(StringUtils.join(paths, File.separator));
+
+ AliyunEmrExportRequest request = new AliyunEmrExportRequest();
+ File dir = new File(Objects.requireNonNull(AliyunEmrServiceTest.class.getClassLoader().getResource(".")).getFile());
+ request.setFolder(new File(Objects.requireNonNull(AliyunEmrServiceTest.class.getClassLoader().getResource(".")).getFile()));
+ request.setProjects(Collections.singletonList("Default"));
+ service.dump(request);
+ Assert.assertTrue(dir.listFiles() != null && Objects.requireNonNull(dir.listFiles()).length > 0);
+ }
+}
diff --git a/client/migrationx-domain/migrationx-domain-aliyunemr/src/test/java/com/aliyun/dataworks/migrationx/domain/dataworks/aliyunemr/CronUtilTest.java b/client/migrationx-domain/migrationx-domain-aliyunemr/src/test/java/com/aliyun/dataworks/migrationx/domain/dataworks/aliyunemr/CronUtilTest.java
new file mode 100644
index 0000000..cee7227
--- /dev/null
+++ b/client/migrationx-domain/migrationx-domain-aliyunemr/src/test/java/com/aliyun/dataworks/migrationx/domain/dataworks/aliyunemr/CronUtilTest.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2024, Alibaba Cloud;
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.aliyun.dataworks.migrationx.domain.dataworks.aliyunemr;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * @author 聿剑
+ * @date 2024/5/11
+ */
+public class CronUtilTest {
+ @Test
+ public void test() {
+ Assert.assertEquals("0 0 9 ? * 1", CronUtil.cronToDwCron("0 0 9 ? * 2"));
+ Assert.assertEquals("0 0 9 ? * 7,1,2,3,4,5,6", CronUtil.cronToDwCron("0 0 9 ? * 1,2,3,4,5,6,7"));
+ Assert.assertEquals("0 0 9 ? * 6-1", CronUtil.cronToDwCron("0 0 9 ? * 7-2"));
+ Assert.assertEquals("0 0 9 ? * ?", CronUtil.cronToDwCron("0 0 9 ? * ?"));
+
+ // failed case
+ Assert.assertEquals(" ", CronUtil.cronToDwCron(" "));
+ Assert.assertEquals("0 0 9 ? * 2 * ", CronUtil.cronToDwCron("0 0 9 ? * 2 * "));
+ }
+}
diff --git a/client/migrationx-domain/migrationx-domain-aliyunemr/src/test/java/com/aliyun/dataworks/migrationx/domain/dataworks/aliyunemr/ParamUtilTest.java b/client/migrationx-domain/migrationx-domain-aliyunemr/src/test/java/com/aliyun/dataworks/migrationx/domain/dataworks/aliyunemr/ParamUtilTest.java
new file mode 100644
index 0000000..e167157
--- /dev/null
+++ b/client/migrationx-domain/migrationx-domain-aliyunemr/src/test/java/com/aliyun/dataworks/migrationx/domain/dataworks/aliyunemr/ParamUtilTest.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2024, Alibaba Cloud;
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.aliyun.dataworks.migrationx.domain.dataworks.aliyunemr;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * TODO 概要描述
+ * TODO 详细描述
+ *
+ * @author 聿剑
+ * @date 2024/5/12
+ */
+public class ParamUtilTest {
+ @Test
+ public void test() {
+ Assert.assertEquals("$[yyyymmddhh24-7/24]", ParamUtil.convertParameterExpression("${yyyyMMddHH-7h}"));
+ Assert.assertEquals("${yyyymm-7}", ParamUtil.convertParameterExpression("${yyyyMM-7m}"));
+ Assert.assertEquals("${yyyy-7}", ParamUtil.convertParameterExpression("${yyyy-7y}"));
+ Assert.assertEquals("$[yyyymmdd-26]", ParamUtil.convertParameterExpression("${yyyyMMdd-26d}"));
+ Assert.assertEquals("$[ddmmyyyy-26]", ParamUtil.convertParameterExpression("${ddMMyyyy-26d}"));
+ Assert.assertEquals("$[ddmmyyyy-26]", ParamUtil.convertParameterExpression("${ddMMyyyy - 26d}"));
+ Assert.assertEquals("$[yyyymmdd hh24:mi:ss]",
+ ParamUtil.convertParameterExpression("${yyyyMMdd HH:mm:ss}"));
+ Assert.assertEquals("$[yyyymmdd-4]", ParamUtil.convertParameterExpression("${yyyyMMdd-4d}"));
+ }
+}
diff --git a/client/migrationx-domain/migrationx-domain-azkaban/pom.xml b/client/migrationx-domain/migrationx-domain-azkaban/pom.xml
index f0a7a61..8a14567 100644
--- a/client/migrationx-domain/migrationx-domain-azkaban/pom.xml
+++ b/client/migrationx-domain/migrationx-domain-azkaban/pom.xml
@@ -20,7 +20,7 @@
migrationx-domain
com.aliyun.dataworks
- 1.1.4
+ 1.1.5
4.0.0
diff --git a/client/migrationx-domain/migrationx-domain-caiyunjian/pom.xml b/client/migrationx-domain/migrationx-domain-caiyunjian/pom.xml
index a6c228d..c760154 100644
--- a/client/migrationx-domain/migrationx-domain-caiyunjian/pom.xml
+++ b/client/migrationx-domain/migrationx-domain-caiyunjian/pom.xml
@@ -20,7 +20,7 @@
migrationx-domain
com.aliyun.dataworks
- 1.1.4
+ 1.1.5
4.0.0
@@ -46,7 +46,7 @@
com.aliyun.dataworks
migrationx-domain-dataworks
- 1.1.4
+ 1.1.5
diff --git a/client/migrationx-domain/migrationx-domain-core/pom.xml b/client/migrationx-domain/migrationx-domain-core/pom.xml
index bbb27e2..83e9beb 100644
--- a/client/migrationx-domain/migrationx-domain-core/pom.xml
+++ b/client/migrationx-domain/migrationx-domain-core/pom.xml
@@ -21,7 +21,7 @@
com.aliyun.dataworks
migrationx-domain
- 1.1.4
+ 1.1.5
migrationx-domain-core
diff --git a/client/migrationx-domain/migrationx-domain-datago/pom.xml b/client/migrationx-domain/migrationx-domain-datago/pom.xml
index 21a51ae..df32979 100644
--- a/client/migrationx-domain/migrationx-domain-datago/pom.xml
+++ b/client/migrationx-domain/migrationx-domain-datago/pom.xml
@@ -20,7 +20,7 @@
migrationx-domain
com.aliyun.dataworks
- 1.1.4
+ 1.1.5
4.0.0
@@ -38,17 +38,17 @@
com.aliyun.dataworks
migrationx-domain-caiyunjian
- 1.1.4
+ 1.1.5
com.aliyun.dataworks
migrationx-common
- 1.1.4
+ 1.1.5
com.aliyun.dataworks
migrationx-domain-dataworks
- 1.1.4
+ 1.1.5
org.apache.velocity
diff --git a/client/migrationx-domain/migrationx-domain-dataworks/pom.xml b/client/migrationx-domain/migrationx-domain-dataworks/pom.xml
index a1e450f..444101c 100644
--- a/client/migrationx-domain/migrationx-domain-dataworks/pom.xml
+++ b/client/migrationx-domain/migrationx-domain-dataworks/pom.xml
@@ -20,7 +20,7 @@
migrationx-domain
com.aliyun.dataworks
- 1.1.4
+ 1.1.5
4.0.0
@@ -39,7 +39,7 @@
com.aliyun.dataworks
migrationx-domain-core
- 1.1.4
+ 1.1.5
com.alibaba
diff --git a/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/constants/ChangeType.java b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/constants/ChangeType.java
new file mode 100644
index 0000000..0a64717
--- /dev/null
+++ b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/constants/ChangeType.java
@@ -0,0 +1,46 @@
+package com.aliyun.dataworks.migrationx.domain.dataworks.constants;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+/**
+ * 文件的变更类型
+ *
+ * @author 戒迷
+ * @date 2024/04/17
+ */
+@RequiredArgsConstructor
+@Getter
+public enum ChangeType {
+
+ /**
+ * NEW
+ */
+ NEW(0),
+ /**
+ * UPDATE
+ */
+ UPDATE(1),
+ /**
+ * DELETE
+ */
+ DELETE(2),
+ ;
+
+ private final Integer code;
+
+ /**
+ * 通过code来获取枚举对象
+ *
+ * @param code
+ * @return
+ */
+ public static ChangeType getByCode(Integer code) {
+ for (ChangeType type : ChangeType.values()) {
+ if (code.equals(type.getCode())) {
+ return type;
+ }
+ }
+ return null;
+ }
+}
diff --git a/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/File.java b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/File.java
index 36ac10a..10166d7 100644
--- a/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/File.java
+++ b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/File.java
@@ -52,6 +52,7 @@ public class File {
private String fileName;
private Integer filePublish;
private Integer fileType;
+ private String fileTypeStr;
private String galaxyResultTableSql;
private String galaxySourceTableSql;
private String galaxyTaskConfig;
diff --git a/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/FileNodeCfg.java b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/FileNodeCfg.java
index 552a328..69e7067 100644
--- a/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/FileNodeCfg.java
+++ b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/FileNodeCfg.java
@@ -15,12 +15,17 @@
package com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.client;
+import java.util.Date;
+import java.util.List;
+
+import com.alibaba.fastjson.JSON;
+
+import com.aliyun.migrationx.common.utils.GsonUtils;
import lombok.Data;
import lombok.ToString;
import lombok.experimental.Accessors;
-
-import java.util.Date;
-import java.util.List;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
/**
* @author sam.liux
@@ -66,4 +71,16 @@ public class FileNodeCfg {
Integer taskRerunInterval;
Integer taskRerunTime;
String extConfig;
+
+ public void setOutputByOutputList() {
+ if (StringUtils.isBlank(output) && !CollectionUtils.isEmpty(outputList)) {
+ this.output = GsonUtils.toJsonString(outputList);
+ }
+ }
+
+ public void setInputByInputList() {
+ if (StringUtils.isBlank(input) && !CollectionUtils.isEmpty(inputList)) {
+ this.input = GsonUtils.toJsonString(inputList);
+ }
+ }
}
diff --git a/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/FileVersion.java b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/FileVersion.java
new file mode 100644
index 0000000..c03f138
--- /dev/null
+++ b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/FileVersion.java
@@ -0,0 +1,24 @@
+package com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.client;
+
+import lombok.Data;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+
+/**
+ * @author 戒迷
+ * @date 2024/4/16
+ */
+@Data
+@Accessors(chain = true)
+@ToString(exclude = {"fileContent"})
+public class FileVersion {
+ private Long fileId;
+ private String fileContent;
+ private String commitUser;
+ private Integer fileType;
+ private String changeType;
+ private String fileName;
+ private Long cloudUuid;
+ private String comment;
+ private String useType;
+}
diff --git a/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/converter/DataWorksSpecNodeConverter.java b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/converter/DataWorksSpecNodeConverter.java
new file mode 100644
index 0000000..70e8fad
--- /dev/null
+++ b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/converter/DataWorksSpecNodeConverter.java
@@ -0,0 +1,375 @@
+package com.aliyun.dataworks.migrationx.domain.dataworks.service.converter;
+
+import java.nio.file.Paths;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import com.aliyun.dataworks.common.spec.domain.DataWorksWorkflowSpec;
+import com.aliyun.dataworks.common.spec.domain.SpecRefEntity;
+import com.aliyun.dataworks.common.spec.domain.Specification;
+import com.aliyun.dataworks.common.spec.domain.dw.nodemodel.DataWorksNodeAdapter;
+import com.aliyun.dataworks.common.spec.domain.dw.nodemodel.DwNodeDependentTypeInfo;
+import com.aliyun.dataworks.common.spec.domain.dw.nodemodel.OutputContext;
+import com.aliyun.dataworks.common.spec.domain.enums.ArtifactType;
+import com.aliyun.dataworks.common.spec.domain.enums.NodeInstanceModeType;
+import com.aliyun.dataworks.common.spec.domain.enums.NodeRerunModeType;
+import com.aliyun.dataworks.common.spec.domain.enums.SpecKind;
+import com.aliyun.dataworks.common.spec.domain.ref.SpecArtifact;
+import com.aliyun.dataworks.common.spec.domain.ref.SpecDatasource;
+import com.aliyun.dataworks.common.spec.domain.ref.SpecFile;
+import com.aliyun.dataworks.common.spec.domain.ref.SpecNode;
+import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput;
+import com.aliyun.dataworks.common.spec.domain.ref.SpecRuntimeResource;
+import com.aliyun.dataworks.common.spec.domain.ref.SpecScript;
+import com.aliyun.dataworks.common.spec.domain.ref.SpecTrigger;
+import com.aliyun.dataworks.common.spec.domain.ref.file.SpecLocalFile;
+import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime;
+import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.client.File;
+import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.client.FileDetail;
+import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.client.FileNodeCfg;
+import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.client.FileNodeInputOutput;
+import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.client.FileNodeInputOutputContext;
+import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.IoParseType;
+import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.NodeUseType;
+import com.aliyun.dataworks.migrationx.domain.dataworks.utils.CronExpressUtil;
+import com.aliyun.migrationx.common.utils.DateUtils;
+import com.google.common.base.Joiner;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.ListUtils;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * DataWorks SpecNode 转化为 DataWorks对象
+ *
+ * @author 戒迷
+ * @date 2024/4/16
+ */
+@Slf4j
+public class DataWorksSpecNodeConverter {
+ private DataWorksSpecNodeConverter() {
+ throw new IllegalStateException("Utility class");
+ }
+
+ public static FileDetail functionSpecToFileDetail(Specification spec, String resourceId) {
+ FileDetail fileDetail = new FileDetail();
+ File file = functionSpecToFile(spec, resourceId);
+ fileDetail.setFile(file);
+ fileDetail.setNodeCfg(initFileNodeCfgByFile(file));
+ return fileDetail;
+ }
+
+ public static FileDetail resourceSpecToFileDetail(Specification spec, String resourceId) {
+ FileDetail fileDetail = new FileDetail();
+ File file = resourceSpecToFile(spec, resourceId);
+ fileDetail.setFile(file);
+ fileDetail.setNodeCfg(initFileNodeCfgByFile(file));
+ return fileDetail;
+ }
+
+ private static FileNodeCfg initFileNodeCfgByFile(File file) {
+ FileNodeCfg fileNodeCfg = new FileNodeCfg();
+ fileNodeCfg.setNodeName(file.getFileName());
+ fileNodeCfg.setNodeId(file.getFileId());
+ return fileNodeCfg;
+ }
+
+ public static FileDetail resourceSpecToFileDetail(Specification spec) {
+ return resourceSpecToFileDetail(spec, null);
+ }
+
+ public static FileDetail functionSpecToFileDetail(Specification spec) {
+ return functionSpecToFileDetail(spec, null);
+ }
+
+ private static File functionSpecToFile(Specification spec, String functionId) {
+ DataWorksWorkflowSpec dataWorksWorkflowSpec = spec.getSpec();
+ if (spec.getSpec() == null) {
+ log.warn("dataworks resource spec is null");
+ return null;
+ }
+
+ return ListUtils.emptyIfNull(dataWorksWorkflowSpec.getFunctions()).stream()
+ .filter(x -> StringUtils.isBlank(functionId) || StringUtils.equals(x.getId(), functionId))
+ .findFirst()
+ .map(specFunc -> {
+ File dwFunc = new File();
+ dwFunc.setFileName(specFunc.getName());
+ dwFunc.setOwner(Optional.ofNullable(specFunc.getMetadata()).map(m -> (String)m.get("owner")).orElse(null));
+ dwFunc.setFileTypeStr(Optional.ofNullable(specFunc.getScript()).map(SpecScript::getRuntime).map(SpecScriptRuntime::getCommand)
+ .orElse(null));
+ dwFunc.setConnName(Optional.ofNullable(specFunc.getDatasource()).map(SpecDatasource::getName).orElse(null));
+ return dwFunc;
+ }).orElse(null);
+ }
+
+ private static File resourceSpecToFile(Specification spec, String resourceId) {
+ DataWorksWorkflowSpec dataWorksWorkflowSpec = spec.getSpec();
+ if (spec.getSpec() == null) {
+ log.warn("dataworks resource spec is null");
+ return null;
+ }
+
+ return ListUtils.emptyIfNull(dataWorksWorkflowSpec.getFileResources()).stream()
+ .filter(x -> StringUtils.isBlank(resourceId) || StringUtils.equals(x.getId(), resourceId))
+ .findFirst()
+ .map(specRes -> {
+ File dwRes = new File();
+ dwRes.setFileName(specRes.getName());
+ dwRes.setOwner(Optional.ofNullable(specRes.getMetadata()).map(m -> (String)m.get("owner")).orElse(null));
+ String fileName = Optional.ofNullable(specRes.getFile()).filter(SpecLocalFile.class::isInstance).map(f -> (SpecLocalFile)f)
+ .map(f -> Paths.get(f.getPath()).toFile().getName()).orElse(specRes.getName());
+ dwRes.setFileTypeStr(Optional.ofNullable(specRes.getScript()).map(SpecScript::getRuntime).map(SpecScriptRuntime::getCommand)
+ .orElse(null));
+ dwRes.setOriginResourceName(fileName);
+ dwRes.setConnName(Optional.ofNullable(specRes.getDatasource()).map(SpecDatasource::getName).orElse(null));
+ return dwRes;
+ }).orElse(null);
+ }
+
+ public static FileDetail nodeSpecToFileDetail(Specification spec, String nodeId) {
+ FileDetail fileDetail = new FileDetail();
+ fileDetail.setFile(nodeSpecToFile(spec, nodeId));
+ fileDetail.setNodeCfg(nodeSpecToNodeCfg(spec, nodeId));
+ return fileDetail;
+ }
+
+ public static FileDetail nodeSpecToFileDetail(Specification spec) {
+ FileDetail fileDetail = new FileDetail();
+ fileDetail.setFile(nodeSpecToFile(spec, null));
+ fileDetail.setNodeCfg(nodeSpecToNodeCfg(spec, null));
+ return fileDetail;
+ }
+
+ public static File nodeSpecToFile(Specification spec, String nodeId) {
+ DataWorksWorkflowSpec dataWorksWorkflowSpec = spec.getSpec();
+ if (spec.getSpec() == null) {
+ log.warn("dataworks workflow spec is null");
+ return null;
+ }
+
+ return Optional.ofNullable(getMatchSpecNode(dataWorksWorkflowSpec, nodeId)).map(specNode -> {
+ File file = new File();
+ file.setAppId(null);
+ file.setBizId(null);
+ file.setCloudUuid(null);
+ file.setCommitStatus(null);
+ file.setConnName(Optional.ofNullable(specNode.getDatasource()).map(SpecDatasource::getName).orElse(null));
+ file.setContent(Optional.ofNullable(specNode.getScript()).map(SpecScript::getContent).orElse(null));
+ file.setCreateTime(null);
+ file.setCreateUser(null);
+ file.setCurrentVersion(null);
+ file.setExtend(null);
+ file.setExtraContent(null);
+ file.setFileDagUrl(null);
+ file.setFileDelete(null);
+ file.setFileDesc(specNode.getDescription());
+ file.setFileFolderId(null);
+ file.setFileFolderPath(Optional.ofNullable(specNode.getScript()).map(SpecFile::getPath).orElse(null));
+ file.setFileId(Long.valueOf(specNode.getId()));
+ file.setFileLockStatus(null);
+ file.setFileLockUser(null);
+ file.setFileLockUserName(null);
+ file.setFileName(specNode.getName());
+ file.setFilePublish(null);
+ file.setFileTypeStr(Optional.ofNullable(specNode.getScript())
+ .map(SpecScript::getRuntime).map(SpecScriptRuntime::getCommand).orElse(null));
+ file.setGalaxyResultTableSql(null);
+ file.setGalaxySourceTableSql(null);
+ file.setGalaxyTaskConfig(null);
+ file.setInstanceInfo(null);
+ file.setIsAutoParse(null);
+ file.setIsLarge(null);
+ file.setIsOdps(null);
+ file.setIsProtected(null);
+ file.setLabelId(null);
+ file.setLastEditTime(null);
+ file.setLastEditUser(null);
+ file.setLastEditUserName(null);
+ file.setLimit(null);
+ file.setLocked(null);
+ file.setLockedBy(null);
+ file.setLockedByName(null);
+ file.setNodeId(null);
+ file.setOriginResourceName(null);
+ file.setOwner(specNode.getOwner());
+ file.setOwnerName(null);
+ file.setParentId(null);
+ file.setParentType(null);
+ file.setPosition(null);
+ file.setReference(null);
+ file.setRegion(null);
+ file.setSourceApp(null);
+ file.setStart(null);
+ file.setTenantId(null);
+ file.setTtContent(null);
+ if (SpecKind.CYCLE_WORKFLOW.getLabel().equals(spec.getKind())) {
+ file.setUseType(NodeUseType.SCHEDULED.getValue());
+ } else if (SpecKind.MANUAL_WORKFLOW.getLabel().equals(spec.getKind())) {
+ file.setUseType(NodeUseType.MANUAL_WORKFLOW.getValue());
+ } else if (SpecKind.TEMPORARY_WORKFLOW.getLabel().equals(spec.getKind())) {
+ file.setUseType(NodeUseType.MANUAL.getValue());
+ }
+ file.setWorkspaceUrl(null);
+ file.setIgnoreLock(null);
+
+ return file;
+ }).orElse(null);
+ }
+
+ public static SpecNode getMatchSpecNode(DataWorksWorkflowSpec dataWorksWorkflowSpec, String nodeId) {
+ for (SpecNode node : dataWorksWorkflowSpec.getNodes()) {
+ if (StringUtils.isBlank(nodeId) || StringUtils.equalsIgnoreCase(node.getId(), nodeId)) {
+ return node;
+ }
+ for (SpecNode innerNode : node.getInnerNodes()) {
+ if (StringUtils.isBlank(nodeId) || StringUtils.equalsIgnoreCase(innerNode.getId(), nodeId)) {
+ return innerNode;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * 处理Node类型的Spec
+ *
+ * @param spec
+ * @param nodeId
+ * @return
+ */
+ public static FileNodeCfg nodeSpecToNodeCfg(Specification spec, String nodeId) {
+ DataWorksWorkflowSpec dataWorksWorkflowSpec = spec.getSpec();
+ if (spec.getSpec() == null) {
+ log.warn("dataworks workflow spec is null");
+ return null;
+ }
+
+ return Optional.ofNullable(getMatchSpecNode(dataWorksWorkflowSpec, nodeId)).map(specNode -> {
+ FileNodeCfg nodeCfg = new FileNodeCfg();
+ nodeCfg.setAppId(null);
+ nodeCfg.setBaselineId(null);
+ nodeCfg.setCreateTime(null);
+ nodeCfg.setCreateUser(null);
+ nodeCfg.setCronExpress(Optional.ofNullable(specNode.getTrigger()).map(SpecTrigger::getCron).orElse(null));
+ nodeCfg.setCycleType(CronExpressUtil.parseCronToCycleType(nodeCfg.getCronExpress()));
+ nodeCfg.setDataxFileId(null);
+ nodeCfg.setDataxFileVersion(null);
+
+ nodeCfg.setDependentType(0);
+ nodeCfg.setDescription(specNode.getDescription());
+ nodeCfg.setEndEffectDate(Optional.ofNullable(specNode.getTrigger()).map(SpecTrigger::getEndTime)
+ .map(DateUtils::convertStringToDate).orElse(null));
+ nodeCfg.setFileId(Optional.ofNullable(specNode.getId()).map(Long::valueOf).orElse(null));
+
+ nodeCfg.setIsAutoParse(null);
+ nodeCfg.setIsStop(null);
+ nodeCfg.setLastModifyTime(null);
+ nodeCfg.setLastModifyUser(null);
+ nodeCfg.setMultiinstCheckType(null);
+ nodeCfg.setNodeId(Long.valueOf(specNode.getId()));
+ nodeCfg.setNodeName(specNode.getName());
+ nodeCfg.setOwner(specNode.getOwner());
+ nodeCfg.setPriority(specNode.getPriority());
+ nodeCfg.setResgroupId(Optional.ofNullable(specNode.getRuntimeResource()).map(SpecRuntimeResource::getResourceGroupId)
+ .map(Long::valueOf).orElse(null));
+ nodeCfg.setStartEffectDate(Optional.ofNullable(specNode.getTrigger()).map(SpecTrigger::getStartTime)
+ .map(DateUtils::convertStringToDate).orElse(null));
+ nodeCfg.setStartRightNow(Optional.ofNullable(specNode.getInstanceMode())
+ .map(instanceMode -> instanceMode == NodeInstanceModeType.IMMEDIATELY)
+ .orElse(false));
+ nodeCfg.setTaskRerunInterval(specNode.getRerunInterval());
+ nodeCfg.setTaskRerunTime(specNode.getRerunTimes());
+
+ setRerunMode(specNode, nodeCfg);
+ setInputOutputList(specNode, nodeCfg);
+ setByAdaptor(spec, specNode, nodeCfg);
+ return nodeCfg;
+ }).orElse(null);
+ }
+
+ private static void setRerunMode(SpecNode specNode, FileNodeCfg nodeCfg) {
+ if (null == specNode.getRerunMode() || NodeRerunModeType.ALL_ALLOWED == specNode.getRerunMode()) {
+ nodeCfg.setReRunAble(1);
+ } else if (NodeRerunModeType.ALL_DENIED == specNode.getRerunMode()) {
+ nodeCfg.setReRunAble(2);
+ } else if (NodeRerunModeType.FAILURE_ALLOWED == specNode.getRerunMode()) {
+ nodeCfg.setReRunAble(0);
+ }
+ }
+
+ private static void setInputOutputList(SpecNode specNode, FileNodeCfg nodeCfg) {
+ nodeCfg.setInputList(ListUtils.emptyIfNull(specNode.getInputs()).stream()
+ .filter(SpecArtifact.class::isInstance)
+ .map(io -> (SpecArtifact)io)
+ .filter(io -> Objects.equals(io.getArtifactType(), ArtifactType.NODE_OUTPUT))
+ .map(io -> (SpecNodeOutput)io)
+ .map(io -> {
+ FileNodeInputOutput in = new FileNodeInputOutput();
+ in.setStr(io.getData());
+ in.setParseType(IoParseType.MANUAL.getCode());
+ in.setRefTableName(io.getRefTableName());
+ return in;
+ }).collect(Collectors.toList()));
+ nodeCfg.setInputByInputList();
+
+ nodeCfg.setOutputList(ListUtils.emptyIfNull(specNode.getOutputs()).stream()
+ .filter(SpecArtifact.class::isInstance)
+ .map(io -> (SpecArtifact)io)
+ .filter(io -> Objects.equals(io.getArtifactType(), ArtifactType.NODE_OUTPUT))
+ .map(io -> (SpecNodeOutput)io)
+ .map(io -> {
+ FileNodeInputOutput out = new FileNodeInputOutput();
+ out.setStr(io.getData());
+ out.setParseType(IoParseType.MANUAL.getCode());
+ out.setRefTableName(io.getRefTableName());
+ return out;
+ }).collect(Collectors.toList()));
+ nodeCfg.setOutputByOutputList();
+ }
+
+ private static void setByAdaptor(Specification spec, SpecNode specNode, FileNodeCfg nodeCfg) {
+ DataWorksNodeAdapter adapter = new DataWorksNodeAdapter(spec, specNode);
+
+ ListUtils.emptyIfNull(spec.getSpec().getFlow()).stream()
+ .filter(f -> StringUtils.equals(specNode.getId(), Optional.ofNullable(f.getNodeId()).map(SpecRefEntity::getId).orElse(null)))
+ .findFirst().ifPresent(flow -> {
+ DwNodeDependentTypeInfo depInfo = adapter.getDependentType(list -> null);
+ nodeCfg.setDependentType(depInfo.getDependentType());
+ if (CollectionUtils.isNotEmpty(depInfo.getDependentNodeOutputList())) {
+ nodeCfg.setDependentDataNode(Joiner.on(",").join(depInfo.getDependentNodeOutputList()));
+ }
+ });
+
+ nodeCfg.setInputContextList(ListUtils.emptyIfNull(adapter.getInputContexts()).stream().map(ctx -> {
+ FileNodeInputOutputContext nc = new FileNodeInputOutputContext();
+ nc.setType(0); // input ctx
+ nc.setParamName(ctx.getKey());
+ nc.setParseType(IoParseType.MANUAL.getCode());
+ nc.setParamValue(ctx.getRefKey());
+ return nc;
+ }).collect(Collectors.toList()));
+ nodeCfg.setOutputContextList(ListUtils.emptyIfNull(adapter.getOutputContexts()).stream().map(ctx -> {
+ FileNodeInputOutputContext nc = new FileNodeInputOutputContext();
+ nc.setType(1); // output ctx
+ nc.setParamName(ctx.getKey());
+ nc.setParseType(IoParseType.MANUAL.getCode());
+ nc.setParamValue(ctx.getValueExpr());
+
+ if (StringUtils.equalsIgnoreCase(OutputContext.CTX_TYPE_CONST, ctx.getCtxType())) {
+ nc.setParamType(1);
+ } else if (StringUtils.equalsIgnoreCase(OutputContext.CTX_TYPE_CONST_SYSTEM_VARIABLE, ctx.getCtxType())
+ || StringUtils.equalsIgnoreCase(OutputContext.CTX_TYPE_SCRIPT_OUTPUTS, ctx.getCtxType())) {
+ nc.setParamType(2);
+ nc.setParseType(IoParseType.SYSTEM.getCode());
+ } else if (StringUtils.equalsIgnoreCase(OutputContext.CTX_TYPE_PARAMETER_NODE_OUTPUTS, ctx.getCtxType())) {
+ nc.setParamType(3);
+ }
+ return nc;
+ }).collect(Collectors.toList()));
+ nodeCfg.setParaValue(adapter.getParaValue());
+ nodeCfg.setExtConfig(adapter.getExtConfig());
+ }
+
+}
diff --git a/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/utils/CronExpressUtil.java b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/utils/CronExpressUtil.java
index 1a40979..4ec9e0b 100644
--- a/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/utils/CronExpressUtil.java
+++ b/client/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/utils/CronExpressUtil.java
@@ -15,6 +15,7 @@
package com.aliyun.dataworks.migrationx.domain.dataworks.utils;
+import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.CycleType;
import com.aliyun.dataworks.migrationx.domain.dataworks.utils.quartz.CronExpression;
import com.aliyun.dataworks.migrationx.domain.dataworks.utils.quartz.ExtendedQuartzCronExpression;
import com.google.common.base.Joiner;
@@ -25,6 +26,8 @@
import java.text.ParseException;
import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -344,4 +347,32 @@ private static String normalizePart(String part) {
}
return part;
}
+
+
+ public static Integer parseCronToCycleType(String cronExpression) {
+ if (StringUtils.isBlank(cronExpression) || "day".equalsIgnoreCase(cronExpression)) {
+ return CycleType.DAY.getCode();
+ }
+
+ try {
+ String[] cronExp = cronExpression.split("\\s+");
+ String pattern = "[-/,*]";
+ Pattern regex = Pattern.compile(pattern);
+
+ String minute = cronExp[1];
+ String hour = cronExp[2];
+
+ Matcher matchM = regex.matcher(minute);
+ Matcher matchH = regex.matcher(hour);
+
+ if (matchM.find() || matchH.find()) {
+ //计算得到的小时调度和分钟调度才重新算cycleType
+ return CycleType.NOT_DAY.getCode();
+ } else {
+ return CycleType.DAY.getCode();
+ }
+ } catch (Exception e) {
+ return CycleType.DAY.getCode();
+ }
+ }
}
diff --git a/client/migrationx-domain/migrationx-domain-dataworks/src/test/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/converter/DataWorksSpecNodeConverterTest.java b/client/migrationx-domain/migrationx-domain-dataworks/src/test/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/converter/DataWorksSpecNodeConverterTest.java
new file mode 100644
index 0000000..29c4f30
--- /dev/null
+++ b/client/migrationx-domain/migrationx-domain-dataworks/src/test/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/converter/DataWorksSpecNodeConverterTest.java
@@ -0,0 +1,198 @@
+package com.aliyun.dataworks.migrationx.domain.dataworks.service.converter;
+
+import com.aliyun.dataworks.common.spec.SpecUtil;
+import com.aliyun.dataworks.common.spec.domain.DataWorksWorkflowSpec;
+import com.aliyun.dataworks.common.spec.domain.Specification;
+import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.client.FileDetail;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * @author 戒迷
+ * @date 2024/4/16
+ */
+public class DataWorksSpecNodeConverterTest {
+
+ @Test
+ public void testHandleNodeSpec() throws Exception {
+ String specStr = "{\n"
+ + "\t\"version\":\"1.1.0\",\n"
+ + "\t\"kind\":\"CycleWorkflow\",\n"
+ + "\t\"spec\":{\n"
+ + "\t\t\"nodes\":[\n"
+ + "\t\t\t{\n"
+ + "\t\t\t\t\"recurrence\":\"Normal\",\n"
+ + "\t\t\t\t\"id\":\"26248077\",\n"
+ + "\t\t\t\t\"timeout\":0,\n"
+ + "\t\t\t\t\"instanceMode\":\"T+1\",\n"
+ + "\t\t\t\t\"rerunMode\":\"Allowed\",\n"
+ + "\t\t\t\t\"rerunTimes\":0,\n"
+ + "\t\t\t\t\"rerunInterval\":120000,\n"
+ + "\t\t\t\t\"datasource\":{\n"
+ + "\t\t\t\t\t\"name\":\"odps_first\",\n"
+ + "\t\t\t\t\t\"type\":\"odps\"\n"
+ + "\t\t\t\t},\n"
+ + "\t\t\t\t\"script\":{\n"
+ + "\t\t\t\t\t\"path\":\"业务流程/建模引擎/MaxCompute/数据开发/config_driver数据同步/model_table\",\n"
+ + "\t\t\t\t\t\"runtime\":{\n"
+ + "\t\t\t\t\t\t\"command\":\"ODPS_SQL\"\n"
+ + "\t\t\t\t\t},\n"
+ + "\t\t\t\t\t\"parameters\":[\n"
+ + "\t\t\t\t\t\t{\n"
+ + "\t\t\t\t\t\t\t\"name\":\"bizdate\",\n"
+ + "\t\t\t\t\t\t\t\"artifactType\":\"Variable\",\n"
+ + "\t\t\t\t\t\t\t\"scope\":\"NodeParameter\",\n"
+ + "\t\t\t\t\t\t\t\"type\":\"System\",\n"
+ + "\t\t\t\t\t\t\t\"value\":\"$[yyyymmdd-1]\"\n"
+ + "\t\t\t\t\t\t}\n"
+ + "\t\t\t\t\t]\n"
+ + "\t\t\t\t},\n"
+ + "\t\t\t\t\"trigger\":{\n"
+ + "\t\t\t\t\t\"type\":\"Scheduler\",\n"
+ + "\t\t\t\t\t\"cron\":\"00 29 00 * * ?\",\n"
+ + "\t\t\t\t\t\"startTime\":\"1970-01-01 00:00:00\",\n"
+ + "\t\t\t\t\t\"endTime\":\"9999-01-01 15:12:51\",\n"
+ + "\t\t\t\t\t\"timezone\":\"Asia/Shanghai\"\n"
+ + "\t\t\t\t},\n"
+ + "\t\t\t\t\"runtimeResource\":{\n"
+ + "\t\t\t\t\t\"resourceGroup\":\"group_20051853\",\n"
+ + "\t\t\t\t\t\"resourceGroupId\":\"20051853\"\n"
+ + "\t\t\t\t},\n"
+ + "\t\t\t\t\"name\":\"model_table\",\n"
+ + "\t\t\t\t\"owner\":\"370260\",\n"
+ + "\t\t\t\t\"inputs\":{\n"
+ + "\t\t\t\t\t\"nodeOutputs\":[\n"
+ + "\t\t\t\t\t\t{\n"
+ + "\t\t\t\t\t\t\t\"data\":\"dataworks_meta.dwd_base_config_driver_data_jsondata_df\",\n"
+ + "\t\t\t\t\t\t\t\"artifactType\":\"NodeOutput\"\n"
+ + "\t\t\t\t\t\t}\n"
+ + "\t\t\t\t\t]\n"
+ + "\t\t\t\t},\n"
+ + "\t\t\t\t\"outputs\":{\n"
+ + "\t\t\t\t\t\"nodeOutputs\":[\n"
+ + "\t\t\t\t\t\t{\n"
+ + "\t\t\t\t\t\t\t\"data\":\"dataworks_analyze.26248077_out\",\n"
+ + "\t\t\t\t\t\t\t\"artifactType\":\"NodeOutput\"\n"
+ + "\t\t\t\t\t\t},\n"
+ + "\t\t\t\t\t\t{\n"
+ + "\t\t\t\t\t\t\t\"data\":\"dataworks_analyze.model_table_config_driver\",\n"
+ + "\t\t\t\t\t\t\t\"artifactType\":\"NodeOutput\"\n"
+ + "\t\t\t\t\t\t}\n"
+ + "\t\t\t\t\t]\n"
+ + "\t\t\t\t}\n"
+ + "\t\t\t}\n"
+ + "\t\t],\n"
+ + "\t\t\"flow\":[\n"
+ + "\t\t\t{\n"
+ + "\t\t\t\t\"nodeId\":\"26248077\",\n"
+ + "\t\t\t\t\"depends\":[\n"
+ + "\t\t\t\t\t{\n"
+ + "\t\t\t\t\t\t\"type\":\"Normal\",\n"
+ + "\t\t\t\t\t\t\"output\":\"dataworks_meta.dwd_base_config_driver_data_jsondata_df\"\n"
+ + "\t\t\t\t\t}\n"
+ + "\t\t\t\t]\n"
+ + "\t\t\t}\n"
+ + "\t\t]\n"
+ + "\t}\n"
+ + "}";
+ Specification spec = SpecUtil.parseToDomain(specStr);
+ FileDetail result = DataWorksSpecNodeConverter.nodeSpecToFileDetail(spec);
+ Assert.assertNotNull(result);
+ Assert.assertNotNull(result.getFile());
+ Assert.assertNotNull(result.getNodeCfg());
+ }
+
+ @Test
+ public void testHandleResourceSpec() throws Exception {
+ String specStr = "{\n"
+ + " \"version\":\"1.1.0\",\n"
+ + " \"kind\":\"CycleWorkflow\",\n"
+ + " \"spec\":{\n"
+ + " \"fileResources\":[\n"
+ + " {\n"
+ + " \"name\":\"mc.py\",\n"
+ + " \"id\":\"6300484767235409791\",\n"
+ + " \"script\":{\n"
+ + " \"path\":\"戒迷/资源/mc.py\",\n"
+ + " \"runtime\":{\n"
+ + " \"command\":\"ODPS_PYTHON\"\n"
+ + " }\n"
+ + " },\n"
+ + " \"runtimeResource\":{\n"
+ + " \"id\":\"5623679673296125496\",\n"
+ + " \"resourceGroup\":\"group_2\",\n"
+ + " \"resourceGroupId\":\"2\"\n"
+ + " },\n"
+ + " \"type\":\"python\",\n"
+ + " \"file\":{\n"
+ + " \"storage\":{\n"
+ + " \"type\": \"oss\"\n"
+ + " }\n"
+ + " },\n"
+ + " \"datasource\":{\n"
+ + " \"name\":\"odps_first\",\n"
+ + " \"type\":\"odps\"\n"
+ + " },\n"
+ + " \"metadata\":{\n"
+ + " \"owner\":\"370260\"\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + "}";
+ Specification spec = SpecUtil.parseToDomain(specStr);
+ FileDetail result = DataWorksSpecNodeConverter.resourceSpecToFileDetail(spec);
+ Assert.assertNotNull(result);
+ Assert.assertNotNull(result.getFile());
+ Assert.assertNotNull(result.getNodeCfg());
+ }
+
+ @Test
+ public void testHandleFunction() throws Exception {
+ String specStr = "{\n"
+ + " \"version\":\"1.1.0\",\n"
+ + " \"kind\":\"CycleWorkflow\",\n"
+ + " \"spec\":{\n"
+ + " \"functions\":[\n"
+ + " {\n"
+ + " \"name\":\"odps_function\",\n"
+ + " \"id\":\"6615080895197716196\",\n"
+ + " \"script\":{\n"
+ + " \"path\":\"戒迷/函数/odps_function\",\n"
+ + " \"runtime\":{\n"
+ + " \"command\":\"ODPS_FUNCTION\"\n"
+ + " }\n"
+ + " },\n"
+ + " \"type\":\"other\",\n"
+ + " \"className\":\"main\",\n"
+ + " \"datasource\":{\n"
+ + " \"name\":\"odps_first\",\n"
+ + " \"type\":\"odps\"\n"
+ + " },\n"
+ + " \"runtimeResource\":{\n"
+ + " \"resourceGroup\":\"group_2\",\n"
+ + " \"id\":\"5623679673296125496\",\n"
+ + " \"resourceGroupId\":\"2\"\n"
+ + " },\n"
+ + " \"resourceType\":\"file\",\n"
+ + " \"metadata\":{\n"
+ + " \"owner\":\"370260\"\n"
+ + " },\n"
+ + " \"fileResources\":[\n"
+ + " {\n"
+ + " \"name\":\"mc.py\"\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + "}";
+ Specification spec = SpecUtil.parseToDomain(specStr);
+ FileDetail result = DataWorksSpecNodeConverter.functionSpecToFileDetail(spec);
+ Assert.assertNotNull(result);
+ Assert.assertNotNull(result.getFile());
+ Assert.assertNotNull(result.getNodeCfg());
+ }
+}
+
+// Generated with love by TestMe :) Please report issues and submit feature requests at: http://weirddev.com/forum#!/testme
\ No newline at end of file
diff --git a/client/migrationx-domain/migrationx-domain-dolphinscheduler/pom.xml b/client/migrationx-domain/migrationx-domain-dolphinscheduler/pom.xml
index e409dba..95e3180 100644
--- a/client/migrationx-domain/migrationx-domain-dolphinscheduler/pom.xml
+++ b/client/migrationx-domain/migrationx-domain-dolphinscheduler/pom.xml
@@ -20,7 +20,7 @@
migrationx-domain
com.aliyun.dataworks
- 1.1.4
+ 1.1.5
4.0.0
@@ -35,7 +35,7 @@
com.aliyun.dataworks
migrationx-domain-core
- 1.1.4
+ 1.1.5
com.fasterxml.jackson.core
diff --git a/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/Project.java b/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/Project.java
index fe55eb8..76d5d9e 100644
--- a/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/Project.java
+++ b/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/Project.java
@@ -32,5 +32,6 @@ public class Project {
private Integer id;
private String name;
private Integer userId;
+ private String code;
private String description;
}
diff --git a/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerApiService.java b/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerApiService.java
index e7c2999..6a4e7f6 100644
--- a/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerApiService.java
+++ b/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerApiService.java
@@ -15,6 +15,18 @@
package com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.text.MessageFormat;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
import com.aliyun.migrationx.common.http.HttpClientUtil;
import com.aliyun.migrationx.common.utils.GsonUtils;
import com.google.common.base.Joiner;
@@ -31,18 +43,6 @@
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.text.MessageFormat;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
/**
* @author 聿剑
* @date 2022/10/20
@@ -155,8 +155,7 @@ public Response> queryAllProjectList(DolphinSchedulerRequest re
HttpClientUtil client = new HttpClientUtil();
HttpGet httpGet = new HttpGet();
httpGet.setHeader("token", token);
- String url = MessageFormat.format("{0}/dolphinscheduler/projects/query-project-list",
- endpoint, request.getPageNo(), request.getPageSize());
+ String url = MessageFormat.format("{0}/dolphinscheduler/projects/query-project-list", endpoint);
httpGet.setURI(new URI(url));
String responseStr = client.executeAndGet(httpGet);
return GsonUtils.fromJsonString(responseStr, new TypeToken>>() {}.getType());
diff --git a/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerProjectRequest.java b/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerProjectRequest.java
index b829685..ed5843f 100644
--- a/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerProjectRequest.java
+++ b/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerProjectRequest.java
@@ -15,13 +15,13 @@
package com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1;
+import javax.validation.constraints.NotBlank;
+
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import lombok.experimental.Accessors;
-import javax.validation.constraints.NotBlank;
-
/**
* @author 聿剑
* @date 2022/10/20
@@ -33,4 +33,5 @@
public class DolphinSchedulerProjectRequest extends DolphinSchedulerRequest {
@NotBlank(message = "dolphinscheduler project name needed")
private String projectName;
+ private String projectCode;
}
diff --git a/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DownloadResourceRequest.java b/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DownloadResourceRequest.java
index 18b206e..7b8349e 100644
--- a/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DownloadResourceRequest.java
+++ b/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DownloadResourceRequest.java
@@ -15,13 +15,13 @@
package com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1;
+import javax.validation.constraints.NotNull;
+
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import lombok.experimental.Accessors;
-import javax.validation.constraints.NotNull;
-
/**
* @author 聿剑
* @date 2022/10/20
@@ -33,4 +33,5 @@
public class DownloadResourceRequest extends DolphinSchedulerRequest {
@NotNull(message = "resource id needed")
private Integer id;
+ private String fullName;
}
diff --git a/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v3/DolphinschedulerApiV3Service.java b/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v3/DolphinschedulerApiV3Service.java
new file mode 100644
index 0000000..6896ba5
--- /dev/null
+++ b/client/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v3/DolphinschedulerApiV3Service.java
@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) 2024, Alibaba Cloud;
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.text.MessageFormat;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.BatchExportProcessDefinitionByIdsRequest;
+import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerApi;
+import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerRequest;
+import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DownloadResourceRequest;
+import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.PaginateResponse;
+import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.QueryDataSourceListByPaginateRequest;
+import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.QueryProcessDefinitionByPaginateRequest;
+import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.QueryResourceListRequest;
+import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.QueryUdfFuncListByPaginateRequest;
+import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.Response;
+import com.aliyun.migrationx.common.http.HttpClientUtil;
+import com.aliyun.migrationx.common.utils.GsonUtils;
+import com.google.common.base.Joiner;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.JsonObject;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.ListUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RegExUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.Header;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.util.EntityUtils;
+
+/**
+ * V3 Dolphinscheduler Api Implementations
+ *
+ * @author 聿剑
+ * @date 2024/4/20
+ */
+@Slf4j
+public class DolphinschedulerApiV3Service implements DolphinSchedulerApi {
+ private static final String HEADER_TOKEN = "token";
+ private final String endpoint;
+ private final String token;
+
+ public DolphinschedulerApiV3Service(String endpoint, String token) {
+ this.endpoint = endpoint;
+ this.token = token;
+ }
+
+ private HttpGet newHttpGet(String url) throws URISyntaxException {
+ HttpGet httpGet = new HttpGet();
+ httpGet.setHeader(HEADER_TOKEN, token);
+ String finalUrl = MessageFormat.format("{0}/dolphinscheduler/{1}", endpoint, url);
+ httpGet.setURI(new URI(finalUrl));
+ return httpGet;
+ }
+
+ private HttpPost newHttpPost(String url) throws URISyntaxException {
+ HttpPost httpPost = new HttpPost();
+ httpPost.setHeader(HEADER_TOKEN, token);
+ String finalUrl = MessageFormat.format("{0}/dolphinscheduler/{1}", endpoint, url);
+ httpPost.setURI(new URI(finalUrl));
+ return httpPost;
+ }
+
+ @Override
+ public PaginateResponse queryProcessDefinitionByPaging(QueryProcessDefinitionByPaginateRequest request) throws Exception {
+ HttpClientUtil client = new HttpClientUtil();
+ String url = MessageFormat.format("projects/{0}/process-definition/list&pageNo={1}&pageSize={2}",
+ request.getProjectCode(), request.getPageNo(), request.getPageSize());
+ String responseStr = client.executeAndGet(newHttpGet(url));
+ return GsonUtils.fromJsonString(responseStr, new TypeToken>() {}.getType());
+ }
+
+ @Override
+ public String batchExportProcessDefinitionByIds(BatchExportProcessDefinitionByIdsRequest request) throws Exception {
+ HttpClientUtil client = new HttpClientUtil();
+ String url = MessageFormat.format("projects/{0}/process-definition/batch-export?codes={1}",
+ request.getProjectCode(),
+ Joiner.on(",").join(ListUtils.emptyIfNull(request.getIds()).stream().distinct().collect(Collectors.toList())));
+ return client.executeAndGet(newHttpPost(url));
+ }
+
+ @Override
+ public Response> queryResourceList(QueryResourceListRequest request) throws Exception {
+ HttpClientUtil client = new HttpClientUtil();
+ String url = MessageFormat.format("resources/query-by-type?type={0}", request.getType());
+ HttpGet httpGet = newHttpGet(url);
+ String responseStr = client.executeAndGet(httpGet);
+ return GsonUtils.fromJsonString(responseStr, new TypeToken>>() {}.getType());
+ }
+
+ private String getSuggestedFileName(Header contentDispositionHeader) {
+ String value = contentDispositionHeader.getValue();
+ return Arrays.stream(StringUtils.split(value, ";"))
+ .map(StringUtils::trim)
+ .filter(token -> StringUtils.startsWithIgnoreCase(token, "filename="))
+ .findFirst()
+ .map(fileNamePart -> StringUtils.replace(fileNamePart, "filename=", ""))
+ .map(fileName -> RegExUtils.replaceAll(fileName, "^\"", ""))
+ .map(fileName -> RegExUtils.replaceAll(fileName, "\"$", ""))
+ .orElse(null);
+ }
+
+ @Override
+ public File downloadResource(DownloadResourceRequest request) throws Exception {
+ HttpClientUtil client = new HttpClientUtil();
+ String url = MessageFormat.format("resources/download?fullName={0}", request.getFullName());
+ HttpGet httpGet = newHttpGet(url);
+ HttpResponse resp = client.executeAndGetHttpResponse(httpGet);
+ InputStream inputStream = resp.getEntity().getContent();
+ String fileName = Stream.of(resp.getAllHeaders())
+ .filter(header -> StringUtils.equalsIgnoreCase(header.getName(), "Content-Disposition"))
+ .findFirst()
+ .map(this::getSuggestedFileName)
+ .orElse(null);
+
+ if (StringUtils.isBlank(fileName)) {
+ String content = EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8);
+ Response