diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..c02456b --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,59 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Deploy Java jar with maven + +on: + release: + types: [published] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout project + uses: actions/checkout@v3 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Build with Maven + run: mvn -B package -DskipTests -Prelease --file pom.xml + + - name: Set up Apache Maven Central + uses: actions/setup-java@v3 + with: # running setup-java again overwrites the settings.xml + java-version: '11' + distribution: 'temurin' + server-id: ossrh # Value of the distributionManagement/repository/id field of the pom.xml + server-username: MAVEN_USERNAME # env variable for username in deploy + server-password: MAVEN_PASSWORD # env variable for token in deploy + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} # Value of the GPG private key to import + gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase + + - name: Publish to the Maven Central Repository + run: + mvn clean --no-transfer-progress --batch-mode -Prelease -DskipTests deploy --file pom.xml -pl highflip-proto,highflip-core,highflip-clients/highflip-sdk -am + env: + MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY_PASSWORD }} + + # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive + # - name: Update dependency graph + # uses: advanced-security/maven-dependency-submission-action@v2.0.1 diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 1d25004..2431618 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -6,13 +6,12 @@ # separate terms of service, privacy policy, and support # documentation. -name: Java CI with Maven +name: Build Java Project on: push: - branches: [ "master" ] pull_request: - branches: [ "master" ] + workflow_dispatch: jobs: build: @@ -20,16 +19,14 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - java-version: '11' - distribution: 'temurin' - cache: maven - - name: Build with Maven - run: mvn -B package --file pom.xml - - # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive - # - name: Update dependency graph - # uses: advanced-security/maven-dependency-submission-action@v2.0.1 + - name: Checkout project + uses: actions/checkout@v3 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + + - name: Build with Maven + run: mvn -B package -DskipTests -Prelease --file pom.xml diff --git a/highflip-build/pom.xml b/highflip-build/pom.xml index 6d5cf4e..5ae3101 100644 --- a/highflip-build/pom.xml +++ b/highflip-build/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-build jar 1.0.0-SNAPSHOT diff --git a/highflip-clients/highflip-console/pom.xml b/highflip-clients/highflip-console/pom.xml index 3047fff..1815e6d 100644 --- a/highflip-clients/highflip-console/pom.xml +++ b/highflip-clients/highflip-console/pom.xml @@ -11,16 +11,16 @@ --> - com.baidu + com.baidu.highflip highflip-console 1.0.0-SNAPSHOT highflip-console - com.baidu - highflip + com.baidu.highflip + highflip-clients 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml @@ -44,7 +44,7 @@ spring-boot-starter-json - com.baidu + com.baidu.highflip highflip-sdk 1.0.0-SNAPSHOT diff --git a/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java b/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java index 724295b..5c969f0 100644 --- a/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java +++ b/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java @@ -17,12 +17,13 @@ public class TaskCommand { @Autowired HighFlipClient client; - @ShellMethod(key = "task list", value = "List all task ids") + @ShellMethod(key = "task list", value = "List job all task ids") public Iterable list( + @ShellOption String jobId, @ShellOption(defaultValue = "0") Integer offset, @ShellOption(defaultValue = "0") Integer limit) { - return client.listTasks(offset, limit); + return client.listTasks(jobId, offset, limit); } @ShellMethod(key = "task get", value = "Get a task information") diff --git a/highflip-clients/highflip-sdk/pom.xml b/highflip-clients/highflip-sdk/pom.xml index d982e95..49e9514 100644 --- a/highflip-clients/highflip-sdk/pom.xml +++ b/highflip-clients/highflip-sdk/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-sdk jar 1.0.0-SNAPSHOT @@ -10,17 +10,17 @@ https://maven.apache.org - com.baidu - highflip + com.baidu.highflip + highflip-clients 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml - com.baidu + com.baidu.highflip highflip-proto - 1.0.0-SNAPSHOT + ${project.version} org.projectlombok diff --git a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java index 9a57d8c..3cd80e0 100644 --- a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java +++ b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java @@ -42,6 +42,12 @@ public HighFlipClient() { this.blockingStub = null; } + public static HighFlipClient newHighFlipClient(String target) { + HighFlipClient highFlipClient = new HighFlipClient(); + highFlipClient.connect(target); + return highFlipClient; + } + public void connect(String target) { close(); @@ -283,13 +289,17 @@ public Iterable getJobLog(String jobId){ /** * + * @param jobId job id in highflip * @param offset * @param limit * @return */ - public Iterable listTasks(int offset, int limit){ + public Iterable listTasks(String jobId, int offset, int limit) { Highflip.TaskListRequest request = Highflip.TaskListRequest .newBuilder() + .setJobId(jobId) + .setOffset(offset) + .setLimit(limit) .build(); Iterator response = getBlockingStub() diff --git a/highflip-clients/pom.xml b/highflip-clients/pom.xml index 3de5034..1457d0f 100644 --- a/highflip-clients/pom.xml +++ b/highflip-clients/pom.xml @@ -4,13 +4,13 @@ highflip - com.baidu + com.baidu.highflip 1.0.0-SNAPSHOT ../pom.xml - com.baidu + com.baidu.highflip highflip-clients 1.0.0-SNAPSHOT pom diff --git a/highflip-core/pom.xml b/highflip-core/pom.xml index ff39144..22b019e 100644 --- a/highflip-core/pom.xml +++ b/highflip-core/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-core jar 1.0.0-SNAPSHOT @@ -11,7 +11,7 @@ http://maven.apache.org - com.baidu + com.baidu.highflip highflip 1.0.0-SNAPSHOT ../pom.xml @@ -19,7 +19,7 @@ - com.baidu + com.baidu.highflip highflip-proto 1.0.0-SNAPSHOT diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java b/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java new file mode 100644 index 0000000..104cfc1 --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java @@ -0,0 +1,10 @@ +package com.baidu.highflip.core.adaptor; + +public interface ServiceAdaptor { + + String getUrl(); + + String getPartyId(); + + String getRole(); +} diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java b/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java index 70522ff..f156486 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java @@ -25,4 +25,16 @@ public class AdaptorPropsList { public static final String PROPS_HIGHFLIP_ADAPTOR_PLATFORM_VERSION_DEFAULT = "0.0.0"; + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL = "highflip.adaptor.service.url"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL_DEFAULT = "http://127.0.0.1:9380"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID = "highflip.adaptor.service.party.id"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID_DEFAULT = "9999"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE = "highflip.adaptor.service.role"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE_DEFAULT = "guest"; + } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java b/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java index ef166ec..1a34c7f 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java @@ -25,4 +25,7 @@ public class InstanceNameList { public static final String HIGHFLIP_ADAPTOR_PARTNER = "highflip.adaptor.partner"; public static final String HIGHFLIP_ADAPTOR_USER = "highflip.adaptor.user"; + + public static final String HIGHFLIP_ADAPTOR_SERVICE = "highflip.adaptor.service"; + } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java index 26f069a..21a56a6 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java @@ -6,6 +6,7 @@ import com.baidu.highflip.core.entity.runtime.Partner; import com.baidu.highflip.core.entity.runtime.Task; import com.baidu.highflip.core.entity.runtime.User; +import com.baidu.highflip.core.entity.runtime.basic.Status; public interface HighFlipRuntime { @@ -26,4 +27,10 @@ public interface HighFlipRuntime { User getUser(String userId); Operator getOperator(String operatorId); + + Data registerData(Data data); + + Iterable listTask(String jobId); + + void updateTask(Task task); } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java index 2224ee4..73eacfe 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java @@ -3,6 +3,8 @@ import com.baidu.highflip.core.entity.dag.codec.AttributeMap; import com.baidu.highflip.core.entity.dag.common.NamedAttributeObject; import com.baidu.highflip.core.utils.ProtoUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; + import highflip.HighflipMeta; import lombok.Data; import lombok.NoArgsConstructor; @@ -96,7 +98,7 @@ public Iterable listParties() { return getParties().keySet(); } - + @JsonIgnore protected void setNodeCategory() { calcMiddleNodes(); calcOutputNodes(); diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java index 2089128..78d73c0 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java @@ -6,16 +6,19 @@ import com.baidu.highflip.core.entity.dag.common.NodeInputRef; import com.baidu.highflip.core.entity.dag.common.NodeOutputRef; import com.baidu.highflip.core.utils.ProtoUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; + import highflip.HighflipMeta; import lombok.Data; -import javax.persistence.Transient; import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import org.springframework.data.annotation.Transient; + @Data public class Node extends NamedAttributeObject implements Serializable { @@ -25,7 +28,7 @@ public class Node extends NamedAttributeObject implements Serializable { String description; - @Transient + @JsonIgnore Graph graph; Category category; @@ -108,6 +111,7 @@ public Node getInputNode(String name) { getInputs().get(name).getFromNode()); } + @JsonIgnore public List getInputNodes() { return getInputs() .values() diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java index c96e552..5d096f0 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java @@ -3,6 +3,7 @@ import com.baidu.highflip.core.entity.dag.codec.AttributeMap; import com.baidu.highflip.core.entity.dag.common.NamedAttributeObject; import com.baidu.highflip.core.utils.ProtoUtils; + import highflip.HighflipMeta; import lombok.Data; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java index b5227f4..1a43401 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java @@ -1,5 +1,12 @@ package com.baidu.highflip.core.entity.dag.codec; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.google.protobuf.ByteString; + import highflip.HighflipMeta; public class TypeValue { @@ -17,11 +24,76 @@ public static Object fromProto(HighflipMeta.TypedValueProto value) { return value.getValue().getFloat(); } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.STRING.getNumber()) { return value.getValue().getString(); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.BYTES.getNumber()) { + return value.getValue().getBytes(); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.LIST.getNumber()) { + return listProtoConvert2List(value.getValue().getList()); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.MAP.getNumber()) { + return mapProtoConvert2Map(value.getValue().getMap()); } else { throw new IllegalArgumentException(); } } + private static List listProtoConvert2List( + HighflipMeta.ListProto listProto) { + if (listProto == null) { + return new ArrayList<>(); + } + List result = new ArrayList<>(); + List listProtoList = listProto.getListList(); + for (HighflipMeta.ValueProto valueProto : listProtoList) { + result.add(convertValueProto(valueProto)); + } + return result; + } + + private static Map mapProtoConvert2Map(HighflipMeta.MapProto mapProto) { + if (mapProto == null) { + return new HashMap<>(); + } + Map map = mapProto.getMapMap(); + Map result = new HashMap<>(); + for (Map.Entry entry : + map.entrySet()) { + String key = entry.getKey(); + HighflipMeta.ValueProto value = entry.getValue(); + result.put(key, convertValueProto(value)); + } + return result; + } + + private static Object convertValueProto(HighflipMeta.ValueProto value) { + switch (value.getValueCase()) { + case BOOL: + return value.getBool(); + case INT: + return value.getInt(); + case LONG: + return value.getLong(); + case FLOAT: + return value.getFloat(); + case DOUBLE: + return value.getDouble(); + case STRING: + return value.getString(); + case BYTES: + return value.getBytes(); + case LIST: + List list = + value.getList().getListList(); + List resultList = new ArrayList<>(); + for(HighflipMeta.ValueProto listValue: list) { + resultList.add(convertValueProto(listValue)); + } + return resultList; + case MAP: + return mapProtoConvert2Map(value.getMap()); + default: + throw new RuntimeException("NOT_SUPPORTED_TYPE"); + } + } + public static HighflipMeta.TypedValueProto toProto(Object object) { HighflipMeta.TypedValueProto.Builder builder = HighflipMeta.TypedValueProto .newBuilder(); @@ -56,14 +128,31 @@ public static HighflipMeta.TypedValueProto toProto(Object object) { .newBuilder() .setDouble((Double) object) .build()); - } else { + } else if (object instanceof String) { builder.setType(HighflipMeta.TypedValueProto.TypeProto.STRING); builder.setValue(HighflipMeta.ValueProto .newBuilder() .setString(object.toString()) .build()); + } else if (object instanceof ByteString) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.BYTES); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setBytes((ByteString) object) + .build()); + } else if (object instanceof HighflipMeta.ListProto) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.LIST); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setList((HighflipMeta.ListProto) object) + .build()); + } else if (object instanceof HighflipMeta.MapProto) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.MAP); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setMap((HighflipMeta.MapProto) object) + .build()); } - return builder.build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java index e0f1377..0b9f14a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java @@ -2,15 +2,18 @@ import lombok.Data; -import javax.persistence.Transient; import java.util.List; import java.util.Map; import java.util.TreeMap; +import org.springframework.data.annotation.Transient; + +import com.fasterxml.jackson.annotation.JsonIgnore; + @Data public class NamedAttributeObject implements Comparable { - @Transient + @JsonIgnore List parents = List.of(); String name; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java index a6d9a63..d6b2eef 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java @@ -19,12 +19,15 @@ public class NodeInputRef { String fromOutput; + String value; + public static HighflipMeta.NodeInputProto toProto(com.baidu.highflip.core.entity.dag.common.NodeInputRef ref) { HighflipMeta.NodeInputProto.Builder builder = HighflipMeta.NodeInputProto .newBuilder() .setName(ref.getName()) .setFromNode(ref.getFromNode()) - .setFromOutput(ref.getFromOutput()); + .setFromOutput(ref.getFromOutput()) + .setValue(ref.getValue()); ProtoUtils.setOptional(builder, "Description", ProtoUtils.ofString(ref.getDescription())); return builder.build(); @@ -37,6 +40,7 @@ public static com.baidu.highflip.core.entity.dag.common.NodeInputRef fromProto(H .setDescription(proto.getDescription()) .setFromNode(proto.getFromNode()) .setFromOutput(proto.getFromOutput()) + .setValue(proto.getValue()) .build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java index e9fd012..270288a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java @@ -15,10 +15,13 @@ public class NodeOutputRef { String description; + String value; + public static HighflipMeta.NodeOutputProto toProto(NodeOutputRef ref) { HighflipMeta.NodeOutputProto.Builder builder = HighflipMeta.NodeOutputProto .newBuilder() - .setName(ref.getName()); + .setName(ref.getName()) + .setValue(ref.getValue()); ProtoUtils.setOptional(builder, "Description", ProtoUtils.ofString(ref.getDescription())); return builder.build(); @@ -29,6 +32,7 @@ public static NodeOutputRef fromProto(HighflipMeta.NodeOutputProto proto) { .builder() .setName(proto.getName()) .setDescription(proto.getDescription()) + .setValue(proto.getValue()) .build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java index 8086c0a..bc4239b 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java @@ -33,7 +33,7 @@ public class Data { @Id @Column(name = "data_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String dataId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java index 9d506fc..aaf9d3f 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java @@ -37,7 +37,7 @@ public class Job { @Id @Column(name = "job_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String jobId; @@ -59,7 +59,7 @@ public class Job { DateTime finishTime; @Type(type = "json") - @Column(name = "graph") + @Column(name = "graph", length = 10240) Graph graph; @Column(name = "status") diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java index d8e9054..1af34ec 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java @@ -28,7 +28,7 @@ public class Operator { @Id @Column(name = "operator_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String operatorId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java index 41f25f2..6ab0f5a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java @@ -27,7 +27,7 @@ public class Partner { @Id @Column(name = "partner_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String partnerId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java index db5eaf6..8d7936a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java @@ -33,7 +33,7 @@ public class Platform { @Id @Column(name = "platform_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String platformId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java index 07c3e35..f4ed6ad 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java @@ -31,10 +31,14 @@ public class Task { @Id @Column(name = "task_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String taskid; + /** + * job id is not the job id in highflip but the job id in corresponding + * federated learning service + */ @Column(name = "job_id") String jobid; @@ -81,4 +85,9 @@ public class Task { @Type(type = "json") @Column(name = "binding") Map binding; + + @Type(type = "json") + @Column(name = "output_data") + List outputData; + } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java b/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java new file mode 100644 index 0000000..4e9ea66 --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java @@ -0,0 +1,24 @@ +package com.baidu.highflip.core.utils; + +import java.io.Serializable; + +import org.hibernate.HibernateException; +import org.hibernate.engine.spi.SharedSessionContractImplementor; +import org.hibernate.id.UUIDGenerator; + +public class CustomUuidGenerator extends UUIDGenerator { + + @Override + public Serializable generate(SharedSessionContractImplementor s, Object obj) + throws HibernateException { + Serializable id = s.getEntityPersister(null, obj).getClassMetadata() + .getIdentifier(obj, s); + + if (id != null && String.valueOf(id).length() > 0) { + return id; + } else { + return super.generate(s, obj); + } + } + +} diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java b/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java new file mode 100644 index 0000000..d252f4a --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java @@ -0,0 +1,64 @@ +package com.baidu.highflip.core.utils; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.springframework.lang.Nullable; + +public class SerializerUtils { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private SerializerUtils() { + } + + public static Map mapObjectDeserialize(String object, + Class tClass) + throws JsonProcessingException { + if (object == null || object.isBlank()) { + return null; + } + Map ret = new HashMap<>(); + Map map = objectMapper.readValue(object, + new TypeReference>() { + }); + for (String tempKey : map.keySet()) { + Object tempValue = map.get(tempKey); + if (tempValue != null) { + String decodeValue = objectMapper.writeValueAsString(tempValue); + if (tClass.isAssignableFrom(String.class)) { + //noinspection unchecked + ret.put(tempKey, (T) decodeValue); + } else { + T result = objectMapper.readValue(decodeValue, tClass); + ret.put(tempKey, result); + } + } + } + return ret; + } + + public static String toJsonString(Object object) + throws JsonProcessingException { + return objectMapper.writeValueAsString(object); + } + + public static T deserialize(String json, Class tClass) + throws JsonProcessingException { + return objectMapper.readValue(json, tClass); + } + + @Nullable + public static T deserializeType(String json, + TypeReference typeReference) + throws JsonProcessingException { + if (json == null || json.isBlank()) { + return null; + } + return objectMapper.readValue(json, typeReference); + } +} diff --git a/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java b/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java index 7584d1a..b7f2135 100644 --- a/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java +++ b/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java @@ -6,12 +6,18 @@ import com.baidu.highflip.core.entity.dag.PartyNode; import com.baidu.highflip.core.entity.dag.codec.TypeValue; import com.baidu.highflip.core.entity.dag.common.MappedNode; +import com.baidu.highflip.core.utils.SerializerUtils; +import com.fasterxml.jackson.core.JsonProcessingException; + import highflip.HighflipMeta; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; @Slf4j @@ -19,10 +25,7 @@ public class TestDag { @Test public void testGraph() { - HighflipMeta.GraphProto define = HighflipMeta.GraphProto - .newBuilder() - .setName("test_graph") - .build(); + HighflipMeta.GraphProto define = constructHighflipGraph(); var graph = Graph.fromProto(define); log.info("graph = {}", graph); @@ -229,4 +232,353 @@ public void testDag() { var proto = Graph.toProto(graph); log.info("proto = {}", proto); } + + private HighflipMeta.GraphProto constructHighflipGraph() { + // reader + List readerNodeOutputProtos = + new ArrayList<>(); + + HighflipMeta.NodeOutputProto outputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + readerNodeOutputProtos.add(outputProto); + + HighflipMeta.NodeProto reader = + HighflipMeta.NodeProto.newBuilder().setName("reader_0") + .setType("Reader") + .addAllOutputs(readerNodeOutputProtos) + .build(); + + // DataTransform + List dataTransformNodeOutputProtos = + new ArrayList<>(); + + HighflipMeta.NodeOutputProto dataTransformOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + HighflipMeta.NodeOutputProto dataTransformOutputProto2 = + HighflipMeta.NodeOutputProto.newBuilder().setName("model").setValue(getJsonStringOfModelOutput()).build(); + dataTransformNodeOutputProtos.add(dataTransformOutputProto); + dataTransformNodeOutputProtos.add(dataTransformOutputProto2); + + + + List dataTransformNodeInputProtos = + new ArrayList<>(); + String json = getDataTransformInputJsonString(); + HighflipMeta.NodeInputProto dataTransformInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(json) + .build(); + + + dataTransformNodeInputProtos.add(dataTransformInputProto); + + HighflipMeta.NodeProto dataTransform = + HighflipMeta.NodeProto.newBuilder().setName("data_transform_0") + .setType("DataTransform") + .addAllInputs( + dataTransformNodeInputProtos) + .addAllOutputs( + dataTransformNodeOutputProtos) + .build(); + + // intersect + List intersectNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto intersectOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + intersectNodeOutputProtos.add(intersectOutputProto); + + List intersectNodeInputProtos = + new ArrayList<>(); + + String dataTransformInputJsonString = getIntersectionInputJsonString(); + HighflipMeta.NodeInputProto intersectInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(dataTransformInputJsonString) + .build(); + + intersectNodeInputProtos.add(intersectInputProto); + + HighflipMeta.NodeProto intersect = + HighflipMeta.NodeProto.newBuilder().setName("intersect_0") + .setType("Intersection") + .addAllInputs(intersectNodeInputProtos) + .addAllOutputs(intersectNodeOutputProtos) + .build(); + + + // Map tableMap = new HashMap<>(); + // tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + // tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + // HighflipMeta.MapProto mapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + // HighflipMeta.ValueProto valueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + // highflip.HighflipMeta.TypedValueProto typedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + HighflipMeta.PartyProto.PartyNode guestPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getGuestPartyReaderNode()) + .build(); + // + HighflipMeta.TypedValueProto withLabel= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + + HighflipMeta.TypedValueProto outputFormat= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("dense") + .build()).build(); + + HighflipMeta.PartyProto.PartyNode guestPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + // common conf + /* + + "intersect_0": { + "intersect_method": "rsa", + "sync_intersect_ids": false, + "only_output_key": true, + "rsa_params": { + "hash_method": "sha256", + "final_hash_method": "sha256", + "split_calculation": false, + "key_length": 2048 + } + } + */ + HighflipMeta.TypedValueProto intersectMethod= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("rsa") + .build()).build(); + HighflipMeta.TypedValueProto syncIntersectIds= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + HighflipMeta.TypedValueProto onlyOutputKey= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(true) + .build()).build(); + + // Map commonTableMap = + // new HashMap<>(); + + // tableMap.put("hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // tableMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // tableMap.put("split_calculation", HighflipMeta.ValueProto.newBuilder().setBool(false).build()); + // tableMap.put("key_length", HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + + + + + // Map commonRsaParamConfMap = + // new HashMap<>(); + // commonRsaParamConfMap.put("hash_method", + // HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // commonRsaParamConfMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // commonRsaParamConfMap.put("split_calculation", + // HighflipMeta.ValueProto.newBuilder().setBool( + // false).build()); + // commonRsaParamConfMap.put("key_length", + // HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + // HighflipMeta.MapProto commonValueMapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(commonRsaParamConfMap).build(); + // HighflipMeta.ValueProto commonValueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(commonValueMapProto).build(); + // highflip.HighflipMeta.TypedValueProto commonTypedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(commonValueProto).build(); + + HighflipMeta.PartyProto.PartyNode commonPartyNode3 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(intersect.getName()) + .putAttributes("intersect_method",intersectMethod) + .putAttributes("sync_intersect_ids",syncIntersectIds) + .putAttributes("only_output_key",onlyOutputKey) + .putAttributes("rsa_params", + getCommonConfRsaParamTypedValueProto()) + .build(); + + List guestPartyNodes = + new ArrayList<>(); + guestPartyNodes.add(guestPartyNode1); + guestPartyNodes.add(guestPartyNode2); + guestPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto guest = + HighflipMeta.PartyProto.newBuilder().setRole( + HighflipMeta.PartyRole.GUEST).addAllNodes(guestPartyNodes).setName("9999").build(); + // guest 已经组装好 + + // 开始组装host + + // Map hostTableMap = + // new HashMap<>(); + // hostTableMap.put("name",HighflipMeta.ValueProto.newBuilder().setString("csv2").build()); + // hostTableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + // HighflipMeta.MapProto hostMapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(hostTableMap).build(); + // HighflipMeta.ValueProto hostValueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(hostMapProto).build(); + // highflip.HighflipMeta.TypedValueProto hostTypedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(hostValueProto).build(); + + HighflipMeta.PartyProto.PartyNode hostPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getHostReaderTypedValueProto()) + .build(); + + // + HighflipMeta.PartyProto.PartyNode hostPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + List hostPartyNodes = + new ArrayList<>(); + hostPartyNodes.add(hostPartyNode1); + hostPartyNodes.add(hostPartyNode2); + hostPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto host = + HighflipMeta.PartyProto.newBuilder().setRole( + HighflipMeta.PartyRole.HOST).addAllNodes(hostPartyNodes).setName("10000").build(); + + List node = new ArrayList<>(); + node.add(reader); + node.add(dataTransform); + node.add(intersect); + + List partyProtoList = new ArrayList<>(); + partyProtoList.add(guest); + partyProtoList.add(host); + + HighflipMeta.GraphProto graphProto = + HighflipMeta.GraphProto.newBuilder().setName("jobId-202301") + .addAllNodes(node) + .addAllParties(partyProtoList).build(); + + return graphProto; + } + + private String getIntersectionInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("data_transform_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getDataTransformInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("reader_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfDataOutput() { + List data = new ArrayList<>(); + data.add("data"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfModelOutput() { + List data = new ArrayList<>(); + data.add("model"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private HighflipMeta.TypedValueProto getGuestPartyReaderNode() { + Map tableMap = new HashMap<>(); + tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto mapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + HighflipMeta.ValueProto valueProto = + HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + highflip.HighflipMeta.TypedValueProto typedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + return typedValueProto; + } + + private HighflipMeta.TypedValueProto getCommonConfRsaParamTypedValueProto(){ + Map commonRsaParamConfMap = + new HashMap<>(); + commonRsaParamConfMap.put("hash_method", + HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("split_calculation", + HighflipMeta.ValueProto.newBuilder().setBool( + false).build()); + commonRsaParamConfMap.put("key_length", + HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + HighflipMeta.MapProto commonValueMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(commonRsaParamConfMap).build(); + HighflipMeta.ValueProto commonValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(commonValueMapProto).build(); + highflip.HighflipMeta.TypedValueProto commonTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(commonValueProto).build(); + return commonTypedValueProto; + } + + private HighflipMeta.TypedValueProto getHostReaderTypedValueProto() { + Map hostTableMap = + new HashMap<>(); + hostTableMap.put("name",HighflipMeta.ValueProto.newBuilder().setString("csv2").build()); + hostTableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto hostMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(hostTableMap).build(); + HighflipMeta.ValueProto hostValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(hostMapProto).build(); + highflip.HighflipMeta.TypedValueProto hostTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(hostValueProto).build(); + return hostTypedValueProto; + } + } diff --git a/highflip-doc/pom.xml b/highflip-doc/pom.xml index 0d8e57a..3ace508 100644 --- a/highflip-doc/pom.xml +++ b/highflip-doc/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-doc pom 1.0.0-SNAPSHOT diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml new file mode 100644 index 0000000..57aa44c --- /dev/null +++ b/highflip-editor/pom.xml @@ -0,0 +1,97 @@ + + + + highflip + com.baidu.highflip + 1.0.0-SNAPSHOT + + 4.0.0 + + highflip-editor + + + + com.baidu.highflip + highflip-sdk + ${project.version} + + + com.baidu.highflip + highflip-proto + ${project.version} + + + + org.springframework.boot + spring-boot-starter-web + ${spring.boot.version} + + + org.springframework.boot + spring-boot-starter-logging + + + + + org.springframework.boot + spring-boot-starter-aop + ${spring.boot.version} + + + ch.qos.logback + logback-classic + + + + + org.springframework.boot + spring-boot-autoconfigure + ${spring.boot.version} + + + org.springframework.boot + spring-boot-starter-test + ${spring.boot.version} + test + + + + org.springframework.security + spring-security-config + ${spring.security.version} + + + + + org.springframework.boot + spring-boot-starter-log4j + 1.3.8.RELEASE + + + + org.projectlombok + lombok + ${lombok.version} + + + org.testng + testng + ${testng.version} + test + + + org.jmockit + jmockit + ${jmockit.version} + test + + + com.google.guava + guava + ${guava.version} + + + + \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java new file mode 100644 index 0000000..007dc29 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java @@ -0,0 +1,34 @@ +package com.baidu.highflip.editor.model; + +public class HighFlipEditorException extends RuntimeException { + + private int code; + private String message; + + public HighFlipEditorException(int code, String message) { + super(message); + this.code = code; + this.message = message; + } + + public HighFlipEditorException(String message) { + super(message); + this.code = 400; + this.message = message; + } + + public HighFlipEditorException(int code, String message, Throwable cause) { + super(message, cause); + this.code = code; + this.message = message; + } + + public int getCode() { + return code; + } + + @Override + public String getMessage() { + return message; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java new file mode 100644 index 0000000..dba016a --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java @@ -0,0 +1,66 @@ +package com.baidu.highflip.editor.model; + +/** + * customized response of controller + */ +public class Result { + /** + * similar to http status code + */ + private int code; + private String message; + private T data; + + public Result() { + } + + public Result(int code, String message, T data) { + this.code = code; + this.message = message; + this.data = data; + } + + public static Result success() { + Result result = new Result(); + result.code = 200; + return result; + } + + public static Result success(T data) { + Result result = new Result(); + result.code = 200; + result.data = data; + return result; + } + + public static Result failure(int errorCode, String message) { + Result result = new Result(); + result.code = errorCode; + result.message = message; + return result; + } + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + public T getData() { + return data; + } + + public void setCode(int code) { + this.code = code; + } + + public void setMessage(String message) { + this.message = message; + } + + public void setData(T data) { + this.data = data; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java new file mode 100644 index 0000000..4e0fbc5 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java @@ -0,0 +1,50 @@ +package com.baidu.highflip.editor.utils; + + +import java.io.IOException; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * utils for serialization and deserialization + * + * @author songtingyu + */ +public class JacksonUtils { + + private static final ObjectMapper mapper = new ObjectMapper(); + + static { + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + mapper.configure(MapperFeature.ALLOW_FINAL_FIELDS_AS_MUTATORS, false); + } + + private JacksonUtils() { + + } + + public static String toJson(T obj) { + try { + return mapper.writeValueAsString(obj); + } catch (JsonProcessingException e) { + throw new HighFlipEditorException(400, "failed to serialize obj.", e); + } + } + + public static T fromJson(String json, Class valueType) { + if (json == null || json.isEmpty()) { + throw new HighFlipEditorException(400, "json is empty!"); + } + try { + return mapper.readValue(json, valueType); + } catch (IOException e) { + throw new HighFlipEditorException(400, e.getMessage(), e); + } + } +} + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java new file mode 100644 index 0000000..e1ea1b1 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java @@ -0,0 +1,21 @@ +package com.baidu.highflip.editor.utils; + +public class StringUtils { + + public static boolean isNotBlank(final CharSequence cs) { + return !isBlank(cs); + } + + public static boolean isBlank(final CharSequence cs) { + int strLen; + if (cs == null || (strLen = cs.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (Character.isWhitespace(cs.charAt(i)) == false) { + return false; + } + } + return true; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java new file mode 100644 index 0000000..40a49a9 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java @@ -0,0 +1,17 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListDataRequest { + + String partnerId; + + String highFlipServerIp; + + int highFlipServerPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java new file mode 100644 index 0000000..e813154 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java @@ -0,0 +1,16 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListOperatorsRequest { + + String highFlipServerIp; + + int highFlipServerPort; + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java new file mode 100644 index 0000000..a240873 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java @@ -0,0 +1,15 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListPartnersRequest { + + String highFlipServerIp; + + int highFlipServerPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java new file mode 100644 index 0000000..c0cf92b --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java @@ -0,0 +1,19 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class LoginRequest { + + private String username; + + private String password; + + private String serverIp; + + private int serverPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java new file mode 100644 index 0000000..e25ec31 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java @@ -0,0 +1,13 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class SaveJobRequest { + + String json; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java new file mode 100644 index 0000000..f53568e --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java @@ -0,0 +1,39 @@ +package com.baidu.highflip.editor.web; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.baidu.highflip.editor.model.Result; + +/** + * global exception handler + */ +@ControllerAdvice +public class RestResponseEntityExceptionHandler + extends ResponseEntityExceptionHandler { + + @ExceptionHandler(value = {RuntimeException.class}) + protected ResponseEntity handleRuntimeException( + RuntimeException e, WebRequest request) { + Result result = Result.failure(400, e.getMessage()); + return handleExceptionInternal(e, result, new HttpHeaders(), + HttpStatus.INTERNAL_SERVER_ERROR, + request); + } + + @ExceptionHandler(value = {HighFlipEditorException.class}) + protected ResponseEntity handleDsException( + HighFlipEditorException e, WebRequest request) { + Result result = Result.failure(e.getCode(), e.getMessage()); + return handleExceptionInternal(e, result, new HttpHeaders(), + HttpStatus.OK, request); + } +} + + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java new file mode 100644 index 0000000..6d903d2 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java @@ -0,0 +1,11 @@ +package com.baidu.highflip.editor.web; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class WebApplication { + public static void main(String[] args) { + SpringApplication.run(WebApplication.class, args); + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java new file mode 100644 index 0000000..cc8f409 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java @@ -0,0 +1,5 @@ +package com.baidu.highflip.editor.web; + +public class WebSecurityConfig { + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java new file mode 100644 index 0000000..77c9a77 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java @@ -0,0 +1,17 @@ +package com.baidu.highflip.editor.web.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * ignore request body when print access log. + * use in {@link com.baidu.highflip.editor.web.aop.AccessLogAop} + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface IgnoreRequestBody { +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java new file mode 100644 index 0000000..8179e89 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java @@ -0,0 +1,91 @@ +package com.baidu.highflip.editor.web.aop; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.Set; + +import javax.servlet.http.HttpServletRequest; + +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; +import org.aspectj.lang.reflect.MethodSignature; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +import com.baidu.highflip.editor.utils.JacksonUtils; +import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; + +@Aspect +@Component +@Order(0) +public class AccessLogAop extends RequestAspect { + + private static final String LOG_CONTENT = + "RequestMethod = {}, RequestUri = {}, RemoteAddress = {}, " + + "RequestParams = `{}`, RequestBody = {}"; + // when request method is post or put, log request body + private static final Set REQUEST_BODY_METHOD = + ImmutableSet.of(RequestMethod.POST.name(), + RequestMethod.PUT.name()); + + private Map loggerHolder = Maps.newConcurrentMap(); + + @Before(CONTROLLER_METHOD_NAME) + public void before(JoinPoint joinPoint) { + Class targetClass = joinPoint.getTarget().getClass(); + + MethodSignature signature = (MethodSignature) joinPoint.getSignature(); + Method method = signature.getMethod(); + + Object[] args = joinPoint.getArgs(); + + log(targetClass, method, args); + } + + private void log(Class targetClass, Method method, Object[] args) { + // 1. get logger + Logger logger = loggerHolder.putIfAbsent( + targetClass, LoggerFactory.getLogger(targetClass)); + logger = logger == null ? loggerHolder.get(targetClass) : logger; + + // 2. get request + ServletRequestAttributes requestAttributes = + (ServletRequestAttributes) RequestContextHolder + .currentRequestAttributes(); + HttpServletRequest request = requestAttributes.getRequest(); + + // 3. get log content + // request body is a method arg, that has @RequestBody annotation. + String remoteAddress = request.getRemoteAddr(); + String url = request.getRequestURI(); + String requestParams = request.getQueryString(); + Object body = null; + if (REQUEST_BODY_METHOD.stream() + .anyMatch(m -> m.equals(request.getMethod()))) { + if (method.getAnnotation(IgnoreRequestBody.class) == null) { + Annotation[][] annotations = method.getParameterAnnotations(); + for (int i = 0; i < annotations.length; i++) { + for (int j = 0; j < annotations[i].length; j++) { + if (annotations[i][j] instanceof RequestBody) { + body = args[i]; + } + } + } + } + } + + logger.info(LOG_CONTENT, + targetClass.getSimpleName() + "." + method.getName(), + url, remoteAddress, requestParams, JacksonUtils.toJson(body)); + } +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java new file mode 100644 index 0000000..85ab08f --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java @@ -0,0 +1,13 @@ +package com.baidu.highflip.editor.web.aop; + +import org.aspectj.lang.annotation.Pointcut; + +public class RequestAspect { + + protected static final String CONTROLLER_METHOD_NAME = "controllerMethod()"; + + @Pointcut("@annotation(org.springframework.web.bind.annotation.RequestMapping)") + protected void controllerMethod() { + + } +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java new file mode 100644 index 0000000..3300b95 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java @@ -0,0 +1,105 @@ +package com.baidu.highflip.editor.web.controller; + +import java.util.List; + +import javax.servlet.http.HttpServletRequest; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.baidu.highflip.editor.model.Result; +import com.baidu.highflip.editor.utils.StringUtils; +import com.baidu.highflip.editor.vo.ListDataRequest; +import com.baidu.highflip.editor.vo.ListOperatorsRequest; +import com.baidu.highflip.editor.vo.ListPartnersRequest; +import com.baidu.highflip.editor.vo.SaveJobRequest; +import com.baidu.highflip.editor.web.services.HighFlipEditorService; +import com.baidu.highflip.editor.vo.LoginRequest; +import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; + +import highflip.v1.Highflip; + +@RestController +@RequestMapping(value = "highflip") +public class HighFlipController { + + private static String TOKEN = "Token"; + + @Autowired + HighFlipEditorService highFlipEditorService; + + @IgnoreRequestBody + @RequestMapping(value = "login", method = RequestMethod.POST) + public Result login(@RequestBody LoginRequest request) { + highFlipEditorService.login(request.getUsername(), + request.getPassword(), + request.getServerIp(), + request.getServerPort()); + return Result.success("login successfully"); + } + + @RequestMapping(value = "operators", method = RequestMethod.GET) + public Result listOperators(ListOperatorsRequest request) { + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List operators = + highFlipEditorService.getOperators( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password); + + return Result.success(operators); + } + + @RequestMapping(value = "partners", method = RequestMethod.GET) + public Result listPartners(ListPartnersRequest request){ + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List partners = + highFlipEditorService.getPartners( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password); + return Result.success(partners); + } + + @RequestMapping(value = "data", method = RequestMethod.GET) + public Result listData(ListDataRequest request) { + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List dataGetResponseList = + highFlipEditorService.getDataList( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password, request.getPartnerId()); + return Result.success(dataGetResponseList); + } + + @RequestMapping(value = "job", method = RequestMethod.POST) + public Result saveJob(HttpServletRequest request, + @RequestBody SaveJobRequest saveJobRequest) { + String token = request.getHeader(TOKEN); + if (StringUtils.isBlank(token)) { + return Result.failure(400, "The required header TOKEN has not found"); + } + highFlipEditorService.saveJob(token, saveJobRequest.getJson()); + return Result.success(); + } + + @RequestMapping(value = "job", method = RequestMethod.GET) + public Result getJob(HttpServletRequest request) { + String token = request.getHeader(TOKEN); + if (StringUtils.isBlank(token)) { + return Result.failure(400, "The required header TOKEN has not found"); + } + String jobDag = highFlipEditorService.getJob(token); + return Result.success(jobDag); + } + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java new file mode 100644 index 0000000..57e0e7f --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java @@ -0,0 +1,47 @@ +package com.baidu.highflip.editor.web.filter; + +import java.io.IOException; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.baidu.highflip.editor.model.Result; +import com.fasterxml.jackson.databind.ObjectMapper; + +@Component +public class GlobalExceptionFilter extends OncePerRequestFilter { + + @Autowired + protected ObjectMapper objectMapper; + + @Override + protected void doFilterInternal(HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain) + throws ServletException, IOException { + try { + filterChain.doFilter(request, response); + } catch (Exception e) { + response.setContentType("application/json;charset=UTF-8"); + // handle TicketException + if (e instanceof HighFlipEditorException) { + HighFlipEditorException ticketException = (HighFlipEditorException) e; + objectMapper.writeValue(response.getWriter(), + Result.failure( + ticketException.getCode(), + ticketException.getMessage())); + } else { + objectMapper.writeValue(response.getWriter(), + Result.failure(400, e.getMessage())); + } + } + } +} + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java new file mode 100644 index 0000000..9e274dc --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java @@ -0,0 +1,113 @@ +package com.baidu.highflip.editor.web.services; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.baidu.highflip.client.HighFlipClient; +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.baidu.highflip.editor.utils.StringUtils; + +import highflip.v1.Highflip; + +@Service +public class HighFlipEditorService { + + private static final Logger LOG = + LoggerFactory.getLogger(HighFlipEditorService.class); + + private ConcurrentHashMap jobs; + + public HighFlipEditorService() { + jobs = new ConcurrentHashMap<>(); + } + + public void login(String username, String password, + String serverIp, int serverPort) { + // TODO create grpc client to login + } + + public List getOperators(String ip, + int port, + String username, + String password) { + // TODO: construct targetUri + String targetUriDemo ="grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + // rpc call + Iterable operatorIds = highFlipClient.listOperators(0, 1000); + List operators = new ArrayList<>(); + for(String operatorId : operatorIds) { + Highflip.OperatorGetResponse operator = + highFlipClient.getOperator(operatorId); + operators.add(operator); + } + return operators; + } + + public List getPartners(String ip, + int port, + String username, + String password) { + // TODO: construct targetUri + String targetUriDemo = "grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + + Iterable partnerIds = highFlipClient.listPartners(0, 1000); + List partners = new ArrayList<>(); + for (String partnerId : partnerIds) { + Highflip.PartnerGetResponse partner = + highFlipClient.getPartner(partnerId); + partners.add(partner); + } + return partners; + } + + public List getDataList(String ip, + int port, + String username, + String password, + String targetPartnerId) { + // TODO: construct targetUri + String targetUriDemo = "grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + + Iterable dataIds = highFlipClient.listData(0, 1000); + List dataList = new ArrayList<>(); + for (String dataId : dataIds) { + Highflip.DataGetResponse data = highFlipClient.getData(dataId); + if (StringUtils.isBlank(targetPartnerId) || + (StringUtils.isNotBlank(data.getPartyId()) && + data.getPartyId().equals(targetPartnerId))) { + dataList.add(data); + } + } + return dataList; + } + + // save job + public void saveJob(String key, String jsonOfJob) { + if (jobs == null) { + jobs = new ConcurrentHashMap<>(); + } + jobs.put(key, jsonOfJob); + } + + public String getJob(String key){ + if(StringUtils.isBlank(key)){ + throw new HighFlipEditorException("Key is blank"); + } + return jobs.get(key); + } + +} diff --git a/highflip-editor/src/main/resources/application.properties b/highflip-editor/src/main/resources/application.properties new file mode 100644 index 0000000..b9cb39f --- /dev/null +++ b/highflip-editor/src/main/resources/application.properties @@ -0,0 +1,8 @@ +server.port=8090 + +server.servlet.session.timeout=30m +server.tomcat.connection-timeout=600000 +# httpOnly +server.servlet.session.cookie.http-only=true + +spring.web.resources.static-locations=classpath:/ui/ \ No newline at end of file diff --git a/highflip-editor/src/main/resources/log4j.properties b/highflip-editor/src/main/resources/log4j.properties new file mode 100644 index 0000000..ebec8d1 --- /dev/null +++ b/highflip-editor/src/main/resources/log4j.properties @@ -0,0 +1,14 @@ +# LOG4J configuration +log4j.rootLogger=info, console, file + +# Console Logger +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d [%p] %l %x - %m%n + +# File Logger +log4j.appender.file=org.apache.log4j.DailyRollingFileAppender +log4j.appender.file.File=./logs/highflip-editor.log +log4j.appender.file.DatePattern='-'yyyy-MM-dd'.log' +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d [%p] %l %x - %m%n \ No newline at end of file diff --git a/highflip-proto/pom.xml b/highflip-proto/pom.xml index 1584c1c..868c091 100644 --- a/highflip-proto/pom.xml +++ b/highflip-proto/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-proto jar 1.0.0-SNAPSHOT @@ -16,6 +16,19 @@ 1.49.0 + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + + io.grpc diff --git a/highflip-proto/src/main/proto/highflip-meta.proto b/highflip-proto/src/main/proto/highflip-meta.proto index a093630..a11552c 100644 --- a/highflip-proto/src/main/proto/highflip-meta.proto +++ b/highflip-proto/src/main/proto/highflip-meta.proto @@ -65,6 +65,8 @@ message NodeInputProto{ string from_node = 4; string from_output = 5; + + string value = 6; } message NodeOutputProto{ @@ -74,6 +76,8 @@ message NodeOutputProto{ string name = 2; optional string description = 3; + + string value = 4; } message NodeProto{ diff --git a/highflip-proto/src/main/proto/highflip.proto b/highflip-proto/src/main/proto/highflip.proto index 880a5b6..9549533 100644 --- a/highflip-proto/src/main/proto/highflip.proto +++ b/highflip-proto/src/main/proto/highflip.proto @@ -230,6 +230,8 @@ message TaskGetResponse{ repeated string input_tasks = 9; repeated string output_tasks = 10; + + repeated string output_data_ids = 11; } @@ -559,7 +561,6 @@ message ConfigListResponse{ string key = 1; } - //////////////////////////////////////////////////////////////////////////////// // SERVICE //////////////////////////////////////////////////////////////////////////////// diff --git a/highflip-server/pom.xml b/highflip-server/pom.xml index 5055c3e..104b332 100644 --- a/highflip-server/pom.xml +++ b/highflip-server/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-server jar 1.0.0-SNAPSHOT @@ -11,7 +11,7 @@ http://maven.apache.org - com.baidu + com.baidu.highflip highflip 1.0.0-SNAPSHOT ../pom.xml @@ -70,7 +70,7 @@ - com.baidu + com.baidu.highflip highflip-core 1.0.0-SNAPSHOT diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java index a9fdf38..e183cd5 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java @@ -2,10 +2,12 @@ import com.baidu.highflip.core.common.InstanceNameList; import com.baidu.highflip.core.engine.HighFlipAdaptor; +import com.baidu.highflip.core.engine.HighFlipRuntime; import com.baidu.highflip.core.engine.InstanceRegister; import com.baidu.highflip.server.adaptor.impl.ConfigurableOperatorAdaptor; import com.baidu.highflip.server.adaptor.impl.ConfigurablePartnerAdaptor; import com.baidu.highflip.server.adaptor.impl.ConfigurablePlatformAdaptor; +import com.baidu.highflip.server.adaptor.impl.ConfigurableServiceAdaptor; import com.baidu.highflip.server.adaptor.impl.DumbJobAdaptor; import com.baidu.highflip.server.adaptor.impl.DumbTaskAdaptor; import com.baidu.highflip.server.adaptor.impl.FixedSingleDataAdaptor; @@ -22,6 +24,17 @@ public class DefaultAdaptor implements HighFlipAdaptor { public static final String PLATFORM_PROPERTIES = "/adaptor/highflip.platform.properties"; + /** + * 用于配置本侧服务的配置信息 + */ + public static final String SERVICE_PROPERTIES = "/adaptor/highflip.service.properties"; + + private HighFlipRuntime highFlipRuntime; + + public DefaultAdaptor(HighFlipRuntime highFlipRuntime) { + this.highFlipRuntime = highFlipRuntime; + } + @Override public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_JOB, new DumbJobAdaptor()); @@ -37,6 +50,20 @@ public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_PLATFORM, new ConfigurablePlatformAdaptor(HighFlipUtils.getProperty(PLATFORM_PROPERTIES))); + + final ConfigurableServiceAdaptor configurableServiceAdaptor = + new ConfigurableServiceAdaptor( + HighFlipUtils.getProperty(SERVICE_PROPERTIES)); + register.register(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE, configurableServiceAdaptor); + + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.url", + configurableServiceAdaptor.getUrl()); + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.party.id", + configurableServiceAdaptor.getPartyId()); + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.role", + configurableServiceAdaptor.getRole()); + + register.register(InstanceNameList.HIGHFLIP_RUNTIME, highFlipRuntime); } @Override diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java new file mode 100644 index 0000000..800bbc8 --- /dev/null +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java @@ -0,0 +1,42 @@ +package com.baidu.highflip.server.adaptor.impl; + +import java.util.Properties; + +import com.baidu.highflip.core.adaptor.ServiceAdaptor; +import com.baidu.highflip.core.common.AdaptorPropsList; + +public class ConfigurableServiceAdaptor implements ServiceAdaptor { + + String url; + + String partyId; + + String role; + + public ConfigurableServiceAdaptor(Properties props) { + + this.url = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL_DEFAULT); + + this.partyId = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID_DEFAULT); + + this.role = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE_DEFAULT); + } + + @Override + public String getUrl() { + return this.url; + } + + @Override + public String getPartyId() { + return this.partyId; + } + + @Override + public String getRole() { + return this.role; + } +} diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java index 0c99b7e..2a7827d 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java @@ -3,6 +3,8 @@ import com.baidu.highflip.core.engine.InstanceRegister; import com.baidu.highflip.server.adaptor.DefaultAdaptor; import com.baidu.highflip.server.adaptor.loader.AdaptorLoader; +import com.baidu.highflip.server.engine.component.HighFlipRuntime; + import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -23,6 +25,9 @@ public class AdaptorConfig { @Autowired InstanceRegister register; + @Autowired + HighFlipRuntime highFlipRuntime; + AdaptorLoader loader = null; @PostConstruct @@ -71,7 +76,7 @@ void initialAdaptor() { } void initialDefaultAdaptor() { - DefaultAdaptor adaptor = new DefaultAdaptor(); + DefaultAdaptor adaptor = new DefaultAdaptor(highFlipRuntime); adaptor.setup(register); } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java index 525db92..05443c3 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; @@ -64,7 +65,7 @@ public class HighFlipEngine { @Autowired PlatformTransactionManager transactionManager; - ConcurrentMap activeJobs; + ConcurrentMap activeJobs = new ConcurrentHashMap<>(); @Autowired AsyncTaskExecutor executor; @@ -290,7 +291,6 @@ public Job createJob(String name, String description, Graph graph) { job = getContext().getJobAdaptor() .createJob(job); - job = getContext().getJobRepository() .save(job); @@ -314,12 +314,12 @@ public Job createJob(String name, String description, Graph graph) { getContext().getTaskRepository() .saveAll(news); } + activeJobs.putIfAbsent(job.getJobId(), job); return job; } - // @Scheduled - protected void updateJob() { + public void updateJob() { JobAdaptor adaptor = getContext().getJobAdaptor(); @@ -328,6 +328,10 @@ protected void updateJob() { if (status != job.getStatus()) { job.setStatus(status); getContext().getJobRepository().save(job); + getContext().getJobAdaptor().updateJob(job); + } + if (status == Status.SUCCEEDED || status == Status.FAILED) { + activeJobs.remove(job.getJobId()); } }); } @@ -413,14 +417,15 @@ public void initializeTasks() { } // @Scheduled - private void updateTask() { - + public void updateTask(Task task) { + getContext().getTaskRepository().save(task); } public Iterable listTask(String jobid) { - - return getContext().getTaskRepository() - .findAllByJobid(jobid); + final List tasks = getContext().getTaskRepository() + .findAllByJobid(jobid); + log.info("tasks size: {}", tasks.size()); + return tasks; } @Cacheable(value = "tasks") diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java index d204511..3d2a07b 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java @@ -5,6 +5,7 @@ import com.baidu.highflip.core.adaptor.OperatorAdaptor; import com.baidu.highflip.core.adaptor.PartnerAdaptor; import com.baidu.highflip.core.adaptor.PlatformAdaptor; +import com.baidu.highflip.core.adaptor.ServiceAdaptor; import com.baidu.highflip.core.adaptor.TaskAdaptor; import com.baidu.highflip.core.adaptor.UserAdaptor; import com.baidu.highflip.core.common.InstanceNameList; @@ -125,4 +126,9 @@ public UserAdaptor getUserAdaptor(){ public UserRepository getUserRepository() { return userReps; } + + // SERVICE + public ServiceAdaptor getServiceAdaptor() { + return getInstance(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE); + } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java index 55c0412..791cce8 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java @@ -7,6 +7,8 @@ import com.baidu.highflip.core.entity.runtime.Partner; import com.baidu.highflip.core.entity.runtime.Task; import com.baidu.highflip.core.entity.runtime.User; +import com.baidu.highflip.core.entity.runtime.basic.Status; + import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -18,13 +20,16 @@ public class HighFlipRuntime implements com.baidu.highflip.core.engine.HighFlipR @Autowired HighFlipContext context; + @Autowired + HighFlipConfiguration configuration; + HighFlipContext getContext() { return context; } @Override public Configuration getConfiguration() { - return null; + return configuration; } @Override @@ -69,4 +74,21 @@ public User getUser(String userId) { public Operator getOperator(String operatorId) { return null; } + + @Override + public Data registerData(Data data) { + Data dataSaved = getContext().getDataRepository().save(data); + return dataSaved; + } + + @Override + public Iterable listTask(String jobId) { + return getContext().getTaskRepository() + .findAllByJobid(jobId); + } + + @Override + public void updateTask(Task task) { + getContext().getTaskRepository().save(task); + } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java index aaa80e2..050051a 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java @@ -1,5 +1,6 @@ package com.baidu.highflip.server.rpc.v1; +import com.baidu.highflip.core.adaptor.ServiceAdaptor; import com.baidu.highflip.core.entity.dag.Graph; import com.baidu.highflip.core.entity.runtime.Config; import com.baidu.highflip.core.entity.runtime.Data; @@ -84,6 +85,7 @@ public void listConfig(Highflip.ConfigListRequest request, * @param request * @param responseObserver */ + @Override public void getConfig(Highflip.ConfigId request, StreamObserver responseObserver) { @@ -105,6 +107,7 @@ public void getConfig(Highflip.ConfigId request, * @param request * @param responseObserver */ + @Override public void setConfig(Highflip.ConfigSetRequest request, StreamObserver responseObserver) { @@ -113,6 +116,7 @@ public void setConfig(Highflip.ConfigSetRequest request, returnVoid(responseObserver); } + @Override public void deleteConfig(Highflip.ConfigId request, StreamObserver responseObserver) { @@ -129,6 +133,7 @@ public void deleteConfig(Highflip.ConfigId request, * @param request * @param responseObserver */ + @Override public void getPlatform(Highflip.Void request, StreamObserver responseObserver) { @@ -147,6 +152,7 @@ public void getPlatform(Highflip.Void request, * @param request * @param responseObserver */ + @Override public void matchPlatform(Highflip.PlatformMatchRequest request, StreamObserver responseObserver) { @@ -189,6 +195,7 @@ public void createJob(Highflip.JobCreateRequest request, * @param request * @param responseObserver */ + @Override public void getJob(Highflip.JobId request, StreamObserver responseObserver) { Job job = getEngine().getJob(request.getJobId()); @@ -210,16 +217,24 @@ public void getJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void checkJob(Highflip.JobId request, StreamObserver responseObserver) { Job job = getEngine().getJob(request.getJobId()); + com.baidu.highflip.core.entity.runtime.basic.Status jobStatus = + getEngine().getContext() + .getJobAdaptor() + .getJobStatus(job); + // check and update job + getEngine().updateJob(); Highflip.JobCheckResponse response = Highflip.JobCheckResponse .newBuilder() .setJobId(job.getJobId()) + .setStatus(Highflip.JobCheckResponse.JobStatus.valueOf( + jobStatus.toString().toUpperCase())) .build(); - returnOne(responseObserver, response); } @@ -227,6 +242,7 @@ public void checkJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void deleteJob(Highflip.JobId request, StreamObserver responseObserver) { @@ -239,6 +255,7 @@ public void deleteJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void listJob(Highflip.JobListRequest request, StreamObserver responseObserver) { @@ -259,6 +276,7 @@ public void listJob(Highflip.JobListRequest request, * @param request * @param responseObserver */ + @Override public void controlJob(Highflip.JobControlRequest request, StreamObserver responseObserver) { @@ -270,6 +288,7 @@ public void controlJob(Highflip.JobControlRequest request, returnVoid(responseObserver); } + @Override public void getJobLog(Highflip.JobLogRequest request, StreamObserver responseObserver) { @@ -282,9 +301,12 @@ public void getJobLog(Highflip.JobLogRequest request, * @param request * @param responseObserver */ + @Override public void listTask(Highflip.TaskListRequest request, StreamObserver responseObserver) { - + final Job job = getEngine().getJob(request.getJobId()); + log.info("job id: {}, job binging id: {}", job.getJobId(), + job.getBingingId()); Iterator response = Streams .stream(getEngine().listTask(request.getJobId())) .map(t -> Highflip.TaskListResponse @@ -300,6 +322,7 @@ public void listTask(Highflip.TaskListRequest request, * @param request * @param responseObserver */ + @Override public void getTask(Highflip.TaskId request, StreamObserver responseObserver) { @@ -309,8 +332,10 @@ public void getTask(Highflip.TaskId request, .newBuilder() .setTaskId(task.getTaskid()) .setJobId(task.getJobid()) + .setNodeName(task.getNodeName()) .setCreateTime(task.getCreateTime().toString()) .setUpdateTime(task.getUpdateTime().toString()) + .addAllOutputDataIds(task.getOutputData()) .build(); returnOne(responseObserver, response); @@ -320,6 +345,7 @@ public void getTask(Highflip.TaskId request, * @param request * @param responseObserver */ + @Override public void checkTask(Highflip.TaskId request, StreamObserver responseObserver) { @@ -340,6 +366,7 @@ public void checkTask(Highflip.TaskId request, * @param request * @param responseObserver */ + @Override public void controlTask(Highflip.TaskControlRequest request, StreamObserver responseObserver) { @@ -355,6 +382,7 @@ public void controlTask(Highflip.TaskControlRequest request, * @param request * @param responseObserver */ + @Override public void getTaskLog(Highflip.TaskLogRequest request, StreamObserver responseObserver) { @@ -368,6 +396,7 @@ public void getTaskLog(Highflip.TaskLogRequest request, * @param request * @param responseObserver */ + @Override public void listData(Highflip.DataListRequest request, StreamObserver responseObserver) { @@ -383,6 +412,7 @@ public void listData(Highflip.DataListRequest request, * @param request * @param responseObserver */ + @Override public void getData(Highflip.DataId request, StreamObserver responseObserver) { @@ -413,6 +443,7 @@ public void getData(Highflip.DataId request, * @param responseObserver * @return */ + @Override public StreamObserver pushData( StreamObserver responseObserver) { @@ -496,6 +527,7 @@ public void onCompleted() { * @param request * @param responseObserver */ + @Override public void pullData(Highflip.DataPullRequest request, StreamObserver responseObserver) { @@ -537,6 +569,7 @@ public void pullData(Highflip.DataPullRequest request, * @param request * @param responseObserver */ + @Override public void deleteData(Highflip.DataId request, StreamObserver responseObserver) { @@ -552,6 +585,7 @@ public void deleteData(Highflip.DataId request, * @param request * @param responseObserver */ + @Override public void listOperator(Highflip.OperatorListRequest request, StreamObserver responseObserver) { Iterator response = Streams @@ -569,6 +603,7 @@ public void listOperator(Highflip.OperatorListRequest request, * @param request * @param responseObserver */ + @Override public void getOperator(Highflip.OperatorId request, StreamObserver responseObserver) { @@ -594,6 +629,7 @@ public void getOperator(Highflip.OperatorId request, * @param request * @param responseObserver */ + @Override public void createPartner(Highflip.PartnerCreateRequest request, StreamObserver responseObserver) { @@ -614,6 +650,7 @@ public void createPartner(Highflip.PartnerCreateRequest request, * @param request * @param responseObserver */ + @Override public void getPartner(Highflip.PartnerId request, StreamObserver responseObserver) { @@ -635,6 +672,7 @@ public void getPartner(Highflip.PartnerId request, * @param request * @param responseObserver */ + @Override public void listPartner(Highflip.PartnerListRequest request, StreamObserver responseObserver) { @@ -653,9 +691,9 @@ public void listPartner(Highflip.PartnerListRequest request, * @param request * @param responseObserver */ + @Override public void controlPartner(Highflip.PartnerControlRequest request, StreamObserver responseObserver) { } - } diff --git a/highflip-server/src/main/resources/adaptor/highflip.service.properties b/highflip-server/src/main/resources/adaptor/highflip.service.properties new file mode 100644 index 0000000..b36b940 --- /dev/null +++ b/highflip-server/src/main/resources/adaptor/highflip.service.properties @@ -0,0 +1,3 @@ +highflip.adaptor.service.url=http://10.27.130.41:8380 +highflip.adaptor.service.party.id=9999 +highflip.adaptor.service.role=guest \ No newline at end of file diff --git a/highflip-server/src/main/resources/application.properties b/highflip-server/src/main/resources/application.properties index e7c6ac5..deced10 100644 --- a/highflip-server/src/main/resources/application.properties +++ b/highflip-server/src/main/resources/application.properties @@ -30,7 +30,9 @@ spring.jpa.properties.jadira.usertype.autoRegisterUserTypes=true # h2 spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect spring.datasource.driver-class-name=org.h2.Driver -spring.datasource.url=jdbc:h2:file:~/.highflip/db/highflip.db +spring.datasource.url=jdbc:h2:file:~/.highflip/db/highflip +spring.datasource.username=highflip +spring.datasource.password=changeMe ######################################################### # logging ######################################################### diff --git a/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java b/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java new file mode 100644 index 0000000..d4651d8 --- /dev/null +++ b/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java @@ -0,0 +1,5 @@ +package com.baidu.highflip.server.engine; + +public class HighFlipEngineTest { + +} diff --git a/highflip-vendors/highflip-adaptor-demo/pom.xml b/highflip-vendors/highflip-adaptor-demo/pom.xml index a022666..871c819 100644 --- a/highflip-vendors/highflip-adaptor-demo/pom.xml +++ b/highflip-vendors/highflip-adaptor-demo/pom.xml @@ -3,7 +3,7 @@ 4.0.0 - com.baidu + com.baidu.highflip highflip-adaptor-demo jar 1.0.0-SNAPSHOT @@ -18,7 +18,7 @@ - com.baidu + com.baidu.highflip highflip-core 1.0.0-SNAPSHOT compile diff --git a/highflip-vendors/highflip-adaptor-fate/pom.xml b/highflip-vendors/highflip-adaptor-fate/pom.xml index 481c757..c6622a8 100644 --- a/highflip-vendors/highflip-adaptor-fate/pom.xml +++ b/highflip-vendors/highflip-adaptor-fate/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-adaptor-fate jar 1.0.0-SNAPSHOT @@ -17,6 +17,16 @@ + + com.baidu.highflip + highflip-proto + 1.0.0-SNAPSHOT + + + com.baidu.highflip + highflip-core + 1.0.0-SNAPSHOT + org.junit.jupiter junit-jupiter @@ -30,21 +40,42 @@ test - com.baidu - highflip-core - 1.0.0-SNAPSHOT - compile + com.google.protobuf + protobuf-java + 3.21.1 io.github.openfeign feign-core 11.10 + + org.slf4j + slf4j-api + 1.7.36 + + + + org.slf4j + slf4j-simple + 1.7.36 + io.github.openfeign feign-jackson 11.10 + + io.github.openfeign + feign-httpclient + 11.10 + + + + org.apache.commons + commons-compress + 1.21 + @@ -54,6 +85,25 @@ maven-surefire-plugin 2.22.2 + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java index 52b367f..45952eb 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java @@ -1,10 +1,167 @@ package com.webank.ai.fate.adaptor; +import static com.webank.ai.fate.common.FateConstants.DATA_ID_SEPARATOR; + +import com.baidu.highflip.core.entity.runtime.Data; +import com.baidu.highflip.core.entity.runtime.basic.DataCategory; +import com.baidu.highflip.core.entity.runtime.basic.KeyPair; +import com.baidu.highflip.core.utils.Foreach; +import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.common.DataMultipartFile; +import com.webank.ai.fate.common.DecompressUtils; import com.webank.ai.fate.context.FateContext; +import feign.Response; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +@Slf4j +@Getter +public class DataAdaptor implements com.baidu.highflip.core.adaptor.DataAdaptor { -public class DataAdaptor { + private final FateContext context; + + private final String DEFAULT_NAMESPACES = "HIGH-FLIP"; + + private final String DEFAULT_DELIMITER = ","; public DataAdaptor(FateContext context) { + this.context = context; + } + + @Override + public Data updateData(Data data) { + return null; + } + + @Override + public int getDataCount() { + return 0; + } + + @Override + public Data getDataByIndex(int index, Data data) { + return null; + } + + @Override + public void deleteData(Data data) { + getContext().getClient().deleteData(data.getBingingId(), DEFAULT_NAMESPACES); + } + + @Override + public InputStream readDataRaw(Data data) { + return null; + } + + @Override + public Iterator> readDataDense(Data data) { + if(data == null) { + throw new RuntimeException("data is null."); + } + log.info("data:{}", data); + + if (DataCategory.RESULT_DATA.equals(data.getCategory())) { + String jobId = (String) data.getBinding().get("jobId"); + String componentName = (String) data.getBinding().get("componentName"); + String role = (String) data.getBinding().get("role"); + String partyId = (String) data.getBinding().get("partyId"); + log.info("jobId: {}, componentName: {}, role: {}, partyId: {}", + jobId, componentName, role, partyId); + + try (Response response = getContext().getClient() + .downloadComponentResultData(jobId, componentName, + role, partyId)) { + String content = DecompressUtils.decompressTarGzToStringMap( + response.body().asInputStream(), + s -> s.contains("csv")).get("data.csv"); + return Arrays.stream(content.split("\n")) + .map(s -> Arrays.stream(s.split(DEFAULT_DELIMITER)) + .map(d -> (Object) d) + .collect(Collectors.toList())) + .collect(Collectors.toList()).iterator(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + try (Response response = getContext().getClient() + .downloadData(data.getName(), + DEFAULT_NAMESPACES)) { + String content = DecompressUtils.decompressTarGzToStringMap( + response.body().asInputStream(), + s -> s.contains("csv")).get("table.csv"); + return Arrays.stream(content.split("\n")) + .map(s -> Arrays.stream(s.split(DEFAULT_DELIMITER)) + .map(d -> (Object) d) + .collect(Collectors.toList())) + .collect(Collectors.toList()).iterator(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public Iterator> readDataSparse(Data data) { + return null; + } + + @Override + public Data createData(Data data) { + return null; + } + + @Override + public void writeDataRaw(Data data, InputStream body) { + + } + + @Override + public void writeDataDense(Data data, Iterator> body) { + StringBuilder stringBuilder = new StringBuilder(); + try { + String headers = + data.getColumns().stream().map(c -> c.getName()) + .collect(Collectors.joining(DEFAULT_DELIMITER)); + stringBuilder.append(headers).append("\n"); + final Foreach> foreach = Foreach.from(body); + for (List dataList : foreach) { + String column = dataList.stream().map(Object::toString).collect( + Collectors.joining(DEFAULT_DELIMITER)); + stringBuilder.append(column).append("\n"); + } + } catch (NoSuchElementException e) { + log.warn("body has no next element"); + } catch (Exception e) { + throw e; + } + String tableName = data.getName(); + log.info("push table:{} data:{}", tableName, data); + + MultipartFile multipartFile = new DataMultipartFile(tableName, + stringBuilder.toString().getBytes(StandardCharsets.UTF_8)); + // drop default 1, which means overwrite the dataset + ResultForm dataResultForm = + getContext().getClient() + .pushData(multipartFile, DEFAULT_DELIMITER, "1", + "4", tableName, DEFAULT_NAMESPACES, + null, null, + null, "1"); + data.setBingingId(dataResultForm.getData().getJob_id()); + } + + @Override + public void writeDataSparse(Data data, Iterator> body) { } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java index 6fb2c7b..a8b992d 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java @@ -1,20 +1,29 @@ package com.webank.ai.fate.adaptor; import com.baidu.highflip.core.engine.HighFlipRuntime; -import com.baidu.highflip.core.entity.runtime.Job; +import com.baidu.highflip.core.entity.runtime.Data; import com.baidu.highflip.core.entity.runtime.Task; import com.baidu.highflip.core.entity.runtime.basic.Action; +import com.baidu.highflip.core.entity.runtime.basic.DataCategory; +import com.baidu.highflip.core.entity.runtime.basic.DataMode; import com.baidu.highflip.core.entity.runtime.basic.Status; -import com.webank.ai.fate.client.form.JobQueryResponseForm; import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.client.form.job.QueryJob; +import com.webank.ai.fate.client.form.task.FFateTask; +import com.webank.ai.fate.client.form.task.FateTask; import com.webank.ai.fate.context.FateContext; import com.webank.ai.fate.translator.DSLTranslator; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.util.CollectionUtils; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; @Slf4j @Getter @@ -28,11 +37,11 @@ public JobAdaptor(FateContext context) { @Override public List getFeatures() { - return null; + throw new UnsupportedOperationException(); } @Override - public Job createJob(Job job) { + public com.baidu.highflip.core.entity.runtime.Job createJob(com.baidu.highflip.core.entity.runtime.Job job) { DSLTranslator.FateDAG dag = getContext() .getTranslator() @@ -40,83 +49,153 @@ public Job createJob(Job job) { String bindId = getContext() .getClient() - .jobSumbit(dag.getDsl(), dag.getDsl()); + .jobSubmit(dag.getDsl(), dag.getConf()).getData().getJob_id(); job.setBingingId(bindId); return job; } @Override - public Job updateJob(Job job) { - return null; + public com.baidu.highflip.core.entity.runtime.Job updateJob(com.baidu.highflip.core.entity.runtime.Job job) { + + String bindId = job.getJobId(); + final Iterable tasks = + getContext().getHighFlipRuntime().listTask(bindId); + for (Task task : tasks) { + ResultForm> result = + getContext().getClient().taskQuery(task.getBingingId()); + if (CollectionUtils.isEmpty(result.getData())) { + throw new RuntimeException("task not found, task: " + task); + } + final FFateTask fFateTask = result.getData().get(0); + final String f_status = fFateTask.getF_status(); + final Status taskStatus = convertToHighFlipStatus(f_status); + + if (task.getStatus().equals(taskStatus)) { + // task status has not changed + continue; + } + + if (taskStatus == Status.SUCCEEDED) { + // register data and bind to task + Data data = new Data(); + data.setBinding(Map.of("jobId", job.getBingingId(), + "taskId", task.getTaskid(), + "taskBingingId", task.getBingingId(), + "componentName", task.getName(), + "role",fFateTask.getF_role(), + "partyId",fFateTask.getF_party_id())); + data.setFormat(DataMode.DENSE); + data.setCategory(DataCategory.RESULT_DATA); + getContext().getHighFlipRuntime().registerData(data); + log.info("data: {}", data); + task.setOutputData(List.of(data.getDataId())); + } + task.setStatus(taskStatus.name()); + getContext().getHighFlipRuntime().updateTask(task); + } + return job; + } + + private Status convertToHighFlipStatus(String fateStatus) { + switch (fateStatus) { + case "success": + return Status.SUCCEEDED; + case "timeout": + return Status.FAILED; + case "failed": + return Status.FAILED; + default: + return Status.UNKNOWN; + } } @Override - public boolean hasJob(Job job) { - return false; + public boolean hasJob(com.baidu.highflip.core.entity.runtime.Job job) { + throw new UnsupportedOperationException(); } @Override - public Status getJobStatus(Job job) { + public Status getJobStatus(com.baidu.highflip.core.entity.runtime.Job job) { String bindId = job.getBingingId(); - ResultForm> result = getContext() + ResultForm> result = getContext() .getClient() .jobQuery(bindId); String status = result.getData() .get(0) - .getStatus(); + .getF_status(); - return Status.valueOf(status); + if ("SUCCESS".equals(status.toUpperCase())) { + return Status.SUCCEEDED; + } else { + return Status.valueOf(status.toUpperCase()); + } } @Override - public void deleteJob(Job job) { - + public void deleteJob(com.baidu.highflip.core.entity.runtime.Job job) { + throw new UnsupportedOperationException(); } @Override - public Job controlJob(Job job, Action action) { + public com.baidu.highflip.core.entity.runtime.Job controlJob(com.baidu.highflip.core.entity.runtime.Job job, Action action) { switch (action) { case STOP: getContext().getClient().jobStop(job.getBingingId()); break; + default: + throw new UnsupportedOperationException(); } return null; } @Override public int getJobCount() { - return 0; + log.info("client get job count"); + return getContext().getClient().listJob(1, Integer.MAX_VALUE).getData().getCount(); } @Override - public Job getJobByIndex(int index, Job job) { - return null; + public com.baidu.highflip.core.entity.runtime.Job getJobByIndex(int index, com.baidu.highflip.core.entity.runtime.Job job) { + log.info("client get job by index"); + FateJob response = getContext().getClient().listJob(index + 1, 1).getData().getJobs().get(0); + return FateJob.convertToEntity(response); } @Override - public Optional moreJob(Job job, HighFlipRuntime runtime) { + public Optional moreJob(com.baidu.highflip.core.entity.runtime.Job job, HighFlipRuntime runtime) { return Optional.empty(); } @Override - public int getTaskCount(Job job) { - return 0; + public int getTaskCount(com.baidu.highflip.core.entity.runtime.Job job) { + return getContext().getClient().listTask(job.getBingingId(), 1, Integer.MAX_VALUE).getData().getCount(); } @Override - public List getTaskList(Job job, List task) { - return null; + public List getTaskList(com.baidu.highflip.core.entity.runtime.Job job, List tasks) { + List queryResult = getContext().getClient().listTask(job.getBingingId(), 1, Integer.MAX_VALUE).getData().getTasks().stream() + .map(FateTask::convertToEntity) + .collect(Collectors.toList()); + for (int i = 0; i < tasks.size(); i++) { + Task queryTask = queryResult.get(i); + Task ret = tasks.get(i); + // task bind job id in highflip + queryTask.setJobid(job.getJobId()); + queryTask.setTaskid(ret.getTaskid()); + BeanUtils.copyProperties(queryTask, ret); + } + return tasks; } @Override - public int getJobLogCount(Job job) { + public int getJobLogCount(com.baidu.highflip.core.entity.runtime.Job job) { return 0; } @Override - public Iterator getJobLog(Job job, int offset, int limit) { + public Iterator getJobLog(com.baidu.highflip.core.entity.runtime.Job job, int offset, int limit) { return null; } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java index 17c3625..bb66014 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java @@ -1,4 +1,30 @@ package com.webank.ai.fate.adaptor; -public class OperatorAdaptor { +import com.baidu.highflip.core.entity.runtime.Operator; + +public class OperatorAdaptor implements com.baidu.highflip.core.adaptor.OperatorAdaptor { + @Override + public int getOperatorCount() { + return 0; + } + + @Override + public Operator getOperatorByIndex(int index, Operator operator) { + return null; + } + + @Override + public Operator updateOperator(Operator operator) { + return null; + } + + @Override + public boolean hasOperator(Operator operator) { + return false; + } + + @Override + public void deleteOperator(Operator operator) { + + } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java index 0fb81dc..b8a1e5d 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java @@ -1,16 +1,98 @@ package com.webank.ai.fate.adaptor; +import com.baidu.highflip.core.entity.runtime.Task; +import com.baidu.highflip.core.entity.runtime.basic.Action; +import com.baidu.highflip.core.entity.runtime.basic.Status; +import com.baidu.highflip.core.exception.HighFlipException; +import com.webank.ai.fate.client.form.task.FFateTask; +import com.webank.ai.fate.client.form.task.FateTask; import com.webank.ai.fate.context.FateContext; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.springframework.util.CollectionUtils; + @Slf4j @Getter -public class TaskAdaptor { +public class TaskAdaptor implements com.baidu.highflip.core.adaptor.TaskAdaptor { FateContext context; public TaskAdaptor(FateContext context) { this.context = context; } + + @Override + public Task updateTask(Task task) { + return null; + } + + @Override + public void deleteTask(Task task) { + + } + + @Override + public boolean hasTask(Task task) { + return false; + } + + @Override + public Status getTaskStatus(Task task) { + final List tasks = + getContext().getClient().taskQuery(task.getBingingId()) + .getData().stream() + .map(FFateTask::convertToEntity) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(tasks)) { + throw new RuntimeException("task not found: " + task); + } + final String status = tasks.get(0).getStatus(); + switch (status) { + case "success": + return Status.SUCCEEDED; + case "timeout": + return Status.FAILED; + case "failed": + return Status.FAILED; + default: + return Status.UNKNOWN; + } + } + + @Override + public Task controlTask(Task task, Action action, Map config) { + return null; + } + + @Override + public int getTaskCount() { + return getContext().getClient().listTask(null, 1, Integer.MAX_VALUE).getData().getCount(); + } + + @Override + public Task getTaskByIndex(int index, Task task) { + log.info("get task by index:{}", index); + return FateTask.convertToEntity(getContext().getClient().listTask(null, index+1, 1).getData().getTasks().get(0)); + } + + @Override + public void invokeTask(Task task) { + + } + + @Override + public int getTaskLogCount(Task task) { + return 0; + } + + @Override + public Iterator getTaskLog(Task task, int offset, int limit) { + return null; + } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java index 116a10b..3ea0b0c 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java @@ -1,85 +1,93 @@ package com.webank.ai.fate.client; -import com.webank.ai.fate.client.form.JobQueryResponseForm; -import com.webank.ai.fate.client.form.JsonResultForm; import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.data.Data; +import com.webank.ai.fate.client.form.data.DataId; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.client.form.job.JobData; +import com.webank.ai.fate.client.form.job.QueryJob; +import com.webank.ai.fate.client.form.task.FFateTask; +import com.webank.ai.fate.client.form.task.TaskData; +import com.webank.ai.fate.common.deserializer.FeignSpringFormEncoder; import feign.Feign; import feign.Headers; +import feign.Logger; import feign.Param; import feign.Request; import feign.RequestLine; +import feign.Response; +import feign.httpclient.ApacheHttpClient; import feign.jackson.JacksonDecoder; -import feign.jackson.JacksonEncoder; +import org.springframework.lang.Nullable; +import org.springframework.web.multipart.MultipartFile; -import java.io.File; import java.util.List; import java.util.concurrent.TimeUnit; -@Headers("Content-Type: application/json") public interface FateClient { static FateClient connect(String url) { - return Feign.builder() - .decoder(new JacksonDecoder()) - .encoder(new JacksonEncoder()) - .options(new Request.Options( - 10, TimeUnit.SECONDS, - 60, TimeUnit.SECONDS, - true)) + return Feign.builder().client(new ApacheHttpClient()).decoder(new JacksonDecoder()) + .encoder(new FeignSpringFormEncoder()).logLevel(Logger.Level.NONE) + .options(new Request.Options(10, TimeUnit.SECONDS, 60, TimeUnit.SECONDS, true)) .target(FateClient.class, url); } - @RequestLine("POST /v1/version/get") - JsonResultForm version(); - - @RequestLine("POST /v1/version/get") - JsonResultForm versionGet(@Param("module") String module); - @RequestLine("POST /v1/job/submit") - String jobSumbit(@Param("job_dsl") String dsl, - @Param("job_runtime_conf") String conf); - - - @RequestLine("POST /v1/job/list/job") - JsonResultForm jobList(@Param("limit") Integer limit); + @Headers("Content-Type: application/json") + ResultForm jobSubmit(@Param("job_dsl") Dsl dsl, @Param("job_runtime_conf") DslConf conf); @RequestLine("POST /v1/job/stop") + @Headers("Content-Type: application/json") void jobStop(@Param("job_id") String jobId); @RequestLine("POST /v1/job/query") - ResultForm> jobQuery(@Param("job_id") String jobId); - - - /** - * - * @param head - * @param id_delimiter - * @param partition - * @param table_name - * @param namespace - * @param storage_engine - * @param destory - * @param extend_sid - * @return - */ + @Headers("Content-Type: application/json") + ResultForm> jobQuery(@Param("job_id") String jobId); + + @RequestLine("POST /v1/job/list/job") + @Headers("Content-Type: application/json") + ResultForm listJob(@Param("page") int page, @Param("limit") int limit); + + @RequestLine("POST /v1/job/task/query") + @Headers("Content-Type: application/json") + ResultForm> taskQuery(@Param("task_id") String taskId); + + @RequestLine("POST /v1/job/list/task") + @Headers("Content-Type: application/json") + ResultForm listTask(@Param("job_id") @Nullable String jobId, @Param("page") int page, + @Param("limit") int limit); + + @RequestLine("POST /v1/table/delete") + @Headers("Content-Type: application/json") + ResultForm deleteData(@Param(value = "table_name") String tableName, + @Param(value = "namespace") String nameSpace); + + @RequestLine("GET /v1/table/download") + @Headers("Content-Type: application/json") + Response downloadData(@Param(value = "name") String tableName, @Param(value = "namespace") String nameSpace); + + @RequestLine("POST /v1/data/upload") @Headers("Content-Type: application/octet-stream") - @RequestLine("POST /data/upload") - JsonResultForm dataUpload( - @Param("file") File file, - @Param("head") Integer head, - @Param("id_delimiter") String idDelimiter, - @Param("partition") Integer partition, - @Param("table_name") String tableName, - @Param("namespace") String namespace, - @Param("storage_engine") String storageEngine, - @Param("destory") Integer destory, - @Param("extend_sid") Integer extendSid); - - - @RequestLine("POST /data/download") - JsonResultForm dataDownload( - @Param("table_name") String tableName, - @Param("namespace") String namespace, - @Param("output_path") String outputPath, - @Param("delimiter") String delimiter); + ResultForm pushData(@Param(value = "file") MultipartFile file, + @Param(value = "id_delimiter") String id_delimiter, @Param(value = "head") String head, + @Param(value = "partition") String partition, @Param(value = "table_name") String table_name, + @Param(value = "namespace") String namespace, @Param(value = "storage_engine") String storage_engine, + @Param(value = "destroy") String destroy, @Param(value = "extend_sid") String extend_sid, + @Param(value = "drop") String drop); + + @RequestLine("POST /v1/data/upload/history") + @Headers("Content-Type: application/json") + ResultForm getDataUploadHistory(@Param(value = "limit") int limit, + @Param(value = "job_id") String job_id); + + @RequestLine("GET /v1/tracking/component/output/data/download") + @Headers("Content-Type: application/json") + Response downloadComponentResultData(@Param(value = "job_id") String tableName, + @Param(value = "component_name") String component_name, + @Param(value = "role") String role, + @Param(value = "party_id") String party_id); + } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/FateStatus.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/FateStatus.java new file mode 100644 index 0000000..2bacde1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/FateStatus.java @@ -0,0 +1,45 @@ +package com.webank.ai.fate.client.form; + +import com.baidu.highflip.core.entity.runtime.basic.Status; + +public enum FateStatus { + + WAITING("waiting"), + RUNNING("running"), + SUCCESS("success"), + FAILED("failed"), + ; + FateStatus(String value) { + this.value = value; + } + + private final String value; + + public String getValue() { + return value; + } + + public static FateStatus fromValue(String value) { + for (FateStatus fateStatus : FateStatus.values()) { + if (fateStatus.getValue().equalsIgnoreCase(value)) { + return fateStatus; + } + } + throw new IllegalArgumentException(); + } + + public static Status convertToEntity(FateStatus fateStatus) { + switch (fateStatus) { + case WAITING: + return Status.APPENDING; + case RUNNING: + return Status.RUNNING; + case FAILED: + return Status.FAILED; + case SUCCESS: + return Status.SUCCEEDED; + } + throw new IllegalArgumentException(); + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/Data.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/Data.java new file mode 100644 index 0000000..51c7b64 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/Data.java @@ -0,0 +1,21 @@ +package com.webank.ai.fate.client.form.data; + +import java.util.Map; + +@lombok.Data +public class Data { + + private String board_url; + private int code; + private String dsl_path; + private String job_id; + private String logs_directory; + private String message; + private Map model_info; + private String namespace; + private String pipeline_dsl_path; + private String runtime_conf_on_party_path; + private String runtime_conf_path; + private String table_name; + private String train_runtime_conf_path; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/DataId.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/DataId.java new file mode 100644 index 0000000..a44fcf5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/DataId.java @@ -0,0 +1,12 @@ +package com.webank.ai.fate.client.form.data; + +import lombok.Data; + +@Data +public class DataId { + + private String table_name; + + private String namespaces; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Component.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Component.java new file mode 100644 index 0000000..30b6179 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Component.java @@ -0,0 +1,14 @@ +package com.webank.ai.fate.client.form.dsl; + +import lombok.Data; + +@Data +public class Component { + + private String module; + + private Input input; + + private Output output; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/ComponentParameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/ComponentParameters.java new file mode 100644 index 0000000..5811b18 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/ComponentParameters.java @@ -0,0 +1,17 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapMapObjectDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class ComponentParameters { + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> common; + + private RoleConf role; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Components.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Components.java new file mode 100644 index 0000000..39f40fa --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Components.java @@ -0,0 +1,12 @@ +package com.webank.ai.fate.client.form.dsl; + +import lombok.Data; + +import java.util.Map; + +@Data +public class Components { + + Map components; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Dsl.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Dsl.java new file mode 100644 index 0000000..d90467c --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Dsl.java @@ -0,0 +1,15 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapComponentsDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class Dsl { + + @JsonDeserialize(using = JsonMapComponentsDeserializer.class) + Map components; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/DslConf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/DslConf.java new file mode 100644 index 0000000..57498ef --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/DslConf.java @@ -0,0 +1,22 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapListStringDeserializer; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +public class DslConf { + + private String dsl_version; + + private Site initiator; + + @JsonDeserialize(using = JsonMapListStringDeserializer.class) + private Map> role; + + private ComponentParameters component_parameters; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Input.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Input.java new file mode 100644 index 0000000..051fa04 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Input.java @@ -0,0 +1,23 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; +import com.webank.ai.fate.common.deserializer.JsonMapListStringDeserializer; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +public class Input { + + @JsonDeserialize(using = JsonMapListStringDeserializer.class) + private Map> data; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List model; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List cache; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Output.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Output.java new file mode 100644 index 0000000..c42906e --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Output.java @@ -0,0 +1,21 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; +import lombok.Data; + +import java.util.List; + +@Data +public class Output { + + @JsonDeserialize(using = JsonListDeserializer.class) + private List data; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List model; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List cache; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/RoleConf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/RoleConf.java new file mode 100644 index 0000000..ecc2fbe --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/RoleConf.java @@ -0,0 +1,21 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapMapObjectDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class RoleConf { + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> host; + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> guest; + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> arbiter; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Site.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Site.java new file mode 100644 index 0000000..6c048fd --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Site.java @@ -0,0 +1,12 @@ +package com.webank.ai.fate.client.form.dsl; + +import lombok.Data; + +@Data +public class Site { + + private String role; + + private String party_id; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Adaptation_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Adaptation_parameters.java new file mode 100644 index 0000000..8fe798b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Adaptation_parameters.java @@ -0,0 +1,22 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Adaptation_parameters { + + private boolean if_initiator_baseline; + + private int request_task_cores; + + private int task_cores_per_node; + + private int task_memory_per_node; + + private int task_nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Common.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Common.java new file mode 100644 index 0000000..89cbcac --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Common.java @@ -0,0 +1,14 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Common { + + private Hetero_feature_binning hetero_feature_binning_0; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Components.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Components.java new file mode 100644 index 0000000..74fe323 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Components.java @@ -0,0 +1,49 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Components { + + private Dataio dataio_0; + + private Hetero_feature_binning hetero_feature_binning_0; + + private Intersection intersection_0; + + private Reader reader_0; + + public void setDataio_0(Dataio dataio_0) { + this.dataio_0 = dataio_0; + } + + public Dataio getDataio_0() { + return dataio_0; + } + + public void setHetero_feature_binning_0(Hetero_feature_binning hetero_feature_binning_0) { + this.hetero_feature_binning_0 = hetero_feature_binning_0; + } + + public Hetero_feature_binning getHetero_feature_binning_0() { + return hetero_feature_binning_0; + } + + public void setIntersection_0(Intersection intersection_0) { + this.intersection_0 = intersection_0; + } + + public Intersection getIntersection_0() { + return intersection_0; + } + + public void setReader_0(Reader reader_0) { + this.reader_0 = reader_0; + } + + public Reader getReader_0() { + return reader_0; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Computing.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Computing.java new file mode 100644 index 0000000..9456eae --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Computing.java @@ -0,0 +1,16 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Computing { + + private int cores_per_node; + + private int nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Dataio.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Dataio.java new file mode 100644 index 0000000..89c7093 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Dataio.java @@ -0,0 +1,19 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Dataio { + + private boolean with_label; + + public void setWith_label(boolean with_label) { + this.with_label = with_label; + } + + public boolean getWith_label() { + return with_label; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Eggroll_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Eggroll_run.java new file mode 100644 index 0000000..bdd2bd9 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Eggroll_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Eggroll_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Engines_address.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Engines_address.java new file mode 100644 index 0000000..97f5d33 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Engines_address.java @@ -0,0 +1,18 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Engines_address { + + private Computing computing; + + private Federation federation; + + private Storage storage; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/FateJob.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/FateJob.java new file mode 100644 index 0000000..6b640fd --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/FateJob.java @@ -0,0 +1,133 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import com.baidu.highflip.core.entity.runtime.basic.Status; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.FateStatus; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import com.webank.ai.fate.common.deserializer.JsonStringDeserializer; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.joda.time.DateTime; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +@Slf4j +@Data +public class FateJob { + + private long apply_resource_time; + + private boolean cancel_signal; + + private String cancel_time; + + private int cores; + + private String create_date; + + private Date create_time; + + private String description; + + private Dsl dsl; + + private long elapsed; + + private String end_date; + + private int end_scheduling_updates; + + private Date end_time; + + private String engine_name; + + private String engine_type; + + private String initiator_party_id; + + private String initiator_role; + + private boolean is_initiator; + + private String job_id; + + private int memory; + + private String name; + + private List partners; + + private int party_id; + + private int progress; + + private boolean ready_signal; + + private String ready_time; + + private int remaining_cores; + + private int remaining_memory; + + private boolean rerun_signal; + + private boolean resource_in_use; + + private long return_resource_time; + + @JsonDeserialize(using = JsonStringDeserializer.class) + private String role; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map roles; + + private Runtime_conf runtime_conf; + + private Runtime_conf_on_party runtime_conf_on_party; + + private String start_date; + + private Date start_time; + + private String status; + + private String status_code; + + private String tag; + + private Train_runtime_conf train_runtime_conf; + + private String update_date; + + private Date update_time; + + private User user; + + private String user_id; + + public static com.baidu.highflip.core.entity.runtime.Job convertToEntity(FateJob response) { + com.baidu.highflip.core.entity.runtime.Job job = new com.baidu.highflip.core.entity.runtime.Job(); +// job.setJobId(); + job.setJobName(response.getName()); + job.setDescription(response.getDescription()); + job.setCreateTime(new DateTime(response.getCreate_time())); + job.setUpdateTime(new DateTime(response.getUpdate_time())); + job.setFinishTime(new DateTime(response.getEnd_time())); +// job.setGraph(); + job.setStatus(FateStatus.convertToEntity(FateStatus.fromValue(response.getStatus()))); +// job.setMessage(); +// job.setIsDeleted(); +// job.setInputTasks(); +// job.setOutputTasks(); + job.setBingingId(response.getJob_id()); +// job.setBinding(); + return job; + } +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Federation.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Federation.java new file mode 100644 index 0000000..9fa754e --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Federation.java @@ -0,0 +1,16 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Federation { + + private int cores_per_node; + + private int nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Hetero_feature_binning.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Hetero_feature_binning.java new file mode 100644 index 0000000..00920c5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Hetero_feature_binning.java @@ -0,0 +1,19 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Hetero_feature_binning { + + private Transform_param transform_param; + + public void setTransform_param(Transform_param transform_param) { + this.transform_param = transform_param; + } + + public Transform_param getTransform_param() { + return transform_param; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Input.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Input.java new file mode 100644 index 0000000..5db61a9 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Input.java @@ -0,0 +1,21 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.List; +@Data +public class Input { + + private InputData data; + + @Data + public static class InputData { + + private List data; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Intersection.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Intersection.java new file mode 100644 index 0000000..f0d298d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Intersection.java @@ -0,0 +1,39 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Intersection { + + private Input input; + + private String module; + + private Output output; + + public void setInput(Input input) { + this.input = input; + } + + public Input getInput() { + return input; + } + + public void setModule(String module) { + this.module = module; + } + + public String getModule() { + return module; + } + + public void setOutput(Output output) { + this.output = output; + } + + public Output getOutput() { + return output; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/JobData.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/JobData.java new file mode 100644 index 0000000..b9937ec --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/JobData.java @@ -0,0 +1,18 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.List; + +@Data +public class JobData { + + private int count; + + private List jobs; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Job_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Job_parameters.java new file mode 100644 index 0000000..8631a12 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Job_parameters.java @@ -0,0 +1,50 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.Map; + +@Data +public class Job_parameters { + + private Adaptation_parameters adaptation_parameters; + + private int auto_retries; + + private int auto_retry_delay; + + private String computing_engine; + + private int computing_partitions; + + private Eggroll_run eggroll_run; + + private Engines_address engines_address; + + private String federated_mode; + + private String federated_status_collect_type; + + private String federation_engine; + + private String job_type; + + private String model_id; + + private String model_version; + + private Map pulsar_run; + + private Map rabbitmq_run; + + private Map spark_run; + + private String storage_engine; + + private int task_parallelism; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Output.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Output.java new file mode 100644 index 0000000..7844095 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Output.java @@ -0,0 +1,21 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import java.util.List; + +public class Output { + + private List data; + + public void setData(List data) { + this.data = data; + } + + public List getData() { + return data; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Pulsar_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Pulsar_run.java new file mode 100644 index 0000000..2837c57 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Pulsar_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Pulsar_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/QueryJob.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/QueryJob.java new file mode 100644 index 0000000..8e758c4 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/QueryJob.java @@ -0,0 +1,106 @@ +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import com.webank.ai.fate.common.deserializer.JsonStringDeserializer; +import lombok.Data; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +@Data +public class QueryJob { + + private long f_apply_resource_time; + + private boolean f_cancel_signal; + + private String f_cancel_time; + + private int f_cores; + + private String f_create_date; + + private Date f_create_time; + + private String f_description; + + private Dsl f_dsl; + + private long f_elapsed; + + private String f_end_date; + + private int f_end_scheduling_updates; + + private Date f_end_time; + + private String f_engine_name; + + private String f_engine_type; + + private String f_initiator_party_id; + + private String f_initiator_role; + + private boolean f_is_initiator; + + private String f_job_id; + + private int f_memory; + + private String f_name; + + private List f_partners; + + private int f_party_id; + + private int f_progress; + + private boolean f_ready_signal; + + private String f_ready_time; + + private int f_remaining_cores; + + private int f_remaining_memory; + + private boolean f_rerun_signal; + + private boolean f_resource_in_use; + + private long f_return_resource_time; + + @JsonDeserialize(using = JsonStringDeserializer.class) + private String f_role; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map f_roles; + + private Runtime_conf f_runtime_conf; + + private Runtime_conf_on_party f_runtime_conf_on_party; + + private String f_start_date; + + private Date f_start_time; + + private String f_status; + + private String f_status_code; + + private String f_tag; + + private Train_runtime_conf f_train_runtime_conf; + + private String f_update_date; + + private Date f_update_time; + + private User f_user; + + private String f_user_id; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Rabbitmq_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Rabbitmq_run.java new file mode 100644 index 0000000..69e7e6b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Rabbitmq_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Rabbitmq_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Reader.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Reader.java new file mode 100644 index 0000000..5ca5261 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Reader.java @@ -0,0 +1,19 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Reader { + + private Table table; + + public void setTable(Table table) { + this.table = table; + } + + public Table getTable() { + return table; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Roles.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Roles.java new file mode 100644 index 0000000..5b41ac9 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Roles.java @@ -0,0 +1,18 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.List; + +@Data +public class Roles { + + private List guest; + + private List host; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf.java new file mode 100644 index 0000000..302937b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf.java @@ -0,0 +1,34 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.dsl.ComponentParameters; +import com.webank.ai.fate.client.form.dsl.Site; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class Runtime_conf { + + private ComponentParameters component_parameters; + + private String conf_path; + + private String dsl_path; + + private int dsl_version; + + private Site initiator; + + private Job_parameters job_parameters; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map role; + + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf_on_party.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf_on_party.java new file mode 100644 index 0000000..3a4c14c --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf_on_party.java @@ -0,0 +1,33 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.dsl.ComponentParameters; +import com.webank.ai.fate.client.form.dsl.Site; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class Runtime_conf_on_party { + + private ComponentParameters component_parameters; + + private String conf_path; + + private String dsl_path; + + private int dsl_version; + + private Site initiator; + + private Job_parameters job_parameters; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Spark_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Spark_run.java new file mode 100644 index 0000000..09d7be4 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Spark_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Spark_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Storage.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Storage.java new file mode 100644 index 0000000..70df472 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Storage.java @@ -0,0 +1,16 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Storage { + + private int cores_per_node; + + private int nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Table.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Table.java new file mode 100644 index 0000000..93866f6 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Table.java @@ -0,0 +1,17 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + + +import lombok.Data; + +@Data +public class Table { + + private String name; + + private String namespace; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Train_runtime_conf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Train_runtime_conf.java new file mode 100644 index 0000000..ae6c312 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Train_runtime_conf.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Train_runtime_conf { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Transform_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Transform_param.java new file mode 100644 index 0000000..166cc37 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Transform_param.java @@ -0,0 +1,14 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Transform_param { + + private String transform_type; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/User.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/User.java new file mode 100644 index 0000000..04c9b02 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/User.java @@ -0,0 +1,18 @@ +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class User { + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map guest; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map host; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Adaptation_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Adaptation_parameters.java new file mode 100644 index 0000000..011bd6f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Adaptation_parameters.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; + +import lombok.Data; + +@Data +public class Adaptation_parameters { + + private boolean if_initiator_baseline; + private int request_task_cores; + private int task_cores_per_node; + private int task_memory_per_node; + private int task_nodes; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Class_path.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Class_path.java new file mode 100644 index 0000000..b4228cf --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Class_path.java @@ -0,0 +1,16 @@ + +package com.webank.ai.fate.client.form.task; + +import lombok.Data; + + +@Data +public class Class_path { + + private String feature_instance; + private String feature_vector; + private String homo_model_convert; + private String Interface; + private String model; + private String model_migrate; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Common.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Common.java new file mode 100644 index 0000000..7afbc31 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Common.java @@ -0,0 +1,28 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Common { + + private Adaptation_parameters adaptation_parameters; + private int auto_retries; + private int auto_retry_delay; + private String computing_engine; + private int computing_partitions; + private Eggroll_run eggroll_run; + private Engines_address engines_address; + private String federated_mode; + private String federated_status_collect_type; + private String federation_engine; + private String job_type; + private String model_id; + private String model_version; + private Pulsar_run pulsar_run; + private Rabbitmq_run rabbitmq_run; + private Spark_run spark_run; + private String storage_engine; + private int task_parallelism; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/ComponentParam.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/ComponentParam.java new file mode 100644 index 0000000..b6de203 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/ComponentParam.java @@ -0,0 +1,31 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.List; + + +@Data +public class ComponentParam { + + private List _feeded_deprecated_params; + private boolean _is_raw_conf; + private String _name; + private List _user_feeded_params; + private double adjustment_factor; + private int bin_indexes; + private String bin_names; + private int bin_num; + private String category_indexes; + private String category_names; + private int compress_thres; + private Encrypt_param encrypt_param; + private double error; + private int head_size; + private boolean local_only; + private String method; + private boolean need_run; + private Optimal_binning_param optimal_binning_param; + private boolean skip_static; + private Transform_param transform_param; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Component_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Component_parameters.java new file mode 100644 index 0000000..51cde37 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Component_parameters.java @@ -0,0 +1,21 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Component_parameters { + + private String CodePath; + private ComponentParam ComponentParam; + private String conf_path; + private String dsl_path; + private int dsl_version; + private Initiator initiator; + private Job_parameters job_parameters; + private Local local; + private String module; + private Role role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Eggroll_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Eggroll_run.java new file mode 100644 index 0000000..bfe6f19 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Eggroll_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Eggroll_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Encrypt_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Encrypt_param.java new file mode 100644 index 0000000..d64a45e --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Encrypt_param.java @@ -0,0 +1,13 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Encrypt_param { + + private int key_length; + private String method; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engine_conf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engine_conf.java new file mode 100644 index 0000000..c710198 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engine_conf.java @@ -0,0 +1,12 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Engine_conf { + + private String computing_engine; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engines_address.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engines_address.java new file mode 100644 index 0000000..bba10cf --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engines_address.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Engines_address { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Env.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Env.java new file mode 100644 index 0000000..76b0cee --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Env.java @@ -0,0 +1,11 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Env { + + private String PYTHONPATH; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java new file mode 100644 index 0000000..a639960 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java @@ -0,0 +1,63 @@ +package com.webank.ai.fate.client.form.task; + +import java.util.Date; +import java.util.List; + +import org.joda.time.DateTime; + +import com.baidu.highflip.core.entity.runtime.Task; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; + +import lombok.Data; + +@Data +public class FFateTask { + private int f_auto_retries; + private int f_auto_retry_delay; + @JsonDeserialize(using = JsonListDeserializer.class) + private List f_cmd; + private String f_component_module; + private String f_component_name; + private Component_parameters f_component_parameters; + private Date f_create_time; + private int f_elapsed; + private Date f_end_time; + private Engine_conf f_engine_conf; + private String f_federated_mode; + private String f_federated_status_collect_type; + private String f_initiator_party_id; + private String f_initiator_role; + // job_id is the job id in FATE + private String f_job_id; + private String f_party_id; + private String f_party_status; + private Provider_info f_provider_info; + private String f_role; + private String f_run_ip; + private boolean f_run_on_this_party; + private int f_run_pid; + private Date f_start_time; + /** + * f_status is one of waiting,running,canceled,timeout,failed,pass,success + */ + private String f_status; + private String f_status_code; + private String f_task_id; + private int f_task_version; + private Date f_update_time; + private String f_worker_id; + + public static Task convertToEntity(FFateTask data) { + Task task = new Task(); + task.setName(data.getF_component_name()); + task.setCreateTime(new DateTime(data.getF_create_time())); + task.setUpdateTime(new DateTime(data.getF_update_time())); + task.setFinishTime(new DateTime(data.getF_end_time())); + task.setNodeName(data.getF_component_name()); + task.setStatus(data.getF_status()); + task.setBingingId(data.getF_task_id()); + return task; + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java new file mode 100644 index 0000000..e0a3b8b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java @@ -0,0 +1,61 @@ + +package com.webank.ai.fate.client.form.task; + +import com.baidu.highflip.core.entity.runtime.Task; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; +import lombok.Data; +import org.joda.time.DateTime; + +import java.util.Date; +import java.util.List; + + +@Data +public class FateTask { + + private int auto_retries; + private int auto_retry_delay; + @JsonDeserialize(using = JsonListDeserializer.class) + private List cmd; + private String component_module; + private String component_name; + private Component_parameters component_parameters; + private Date create_time; + private int elapsed; + private Date end_time; + private Engine_conf engine_conf; + private String federated_mode; + private String federated_status_collect_type; + private String initiator_party_id; + private String initiator_role; + // job_id is the job id in FATE + private String job_id; + private String party_id; + private String party_status; + private Provider_info provider_info; + private String role; + private String run_ip; + private boolean run_on_this_party; + private int run_pid; + private Date start_time; + private String status; + private String status_code; + private String task_id; + private int task_version; + private Date update_time; + private String worker_id; + + public static Task convertToEntity(FateTask data) { + Task task = new Task(); + task.setName(data.getComponent_name()); + task.setCreateTime(new DateTime(data.getCreate_time())); + task.setUpdateTime(new DateTime(data.getUpdate_time())); + task.setFinishTime(new DateTime(data.getEnd_time())); + task.setNodeName(data.getComponent_name()); + task.setStatus(data.getStatus()); + task.setBingingId(data.getTask_id()); + return task; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Initiator.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Initiator.java new file mode 100644 index 0000000..d4746fc --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Initiator.java @@ -0,0 +1,13 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Initiator { + + private int party_id; + private String role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Job_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Job_parameters.java new file mode 100644 index 0000000..35e1cb7 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Job_parameters.java @@ -0,0 +1,12 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Job_parameters { + + private Common common; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Local.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Local.java new file mode 100644 index 0000000..2e66e13 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Local.java @@ -0,0 +1,13 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Local { + + private int party_id; + private String role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Optimal_binning_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Optimal_binning_param.java new file mode 100644 index 0000000..9cdaf7d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Optimal_binning_param.java @@ -0,0 +1,18 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Optimal_binning_param { + + private String adjustment_factor; + private int init_bin_nums; + private String init_bucket_method; + private String max_bin; + private int max_bin_pct; + private String metric_method; + private double min_bin_pct; + private boolean mixture; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Provider_info.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Provider_info.java new file mode 100644 index 0000000..211fab1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Provider_info.java @@ -0,0 +1,15 @@ + +package com.webank.ai.fate.client.form.task; + +import lombok.Data; + + +@Data +public class Provider_info { + + private Class_path class_path; + private Env env; + private String name; + private String path; + private String version; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Pulsar_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Pulsar_run.java new file mode 100644 index 0000000..a5b6e13 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Pulsar_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Pulsar_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Rabbitmq_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Rabbitmq_run.java new file mode 100644 index 0000000..ef0032b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Rabbitmq_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Rabbitmq_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Role.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Role.java new file mode 100644 index 0000000..51c34d1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Role.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.List; + + +@Data +public class Role { + + private List guest; + private List host; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Spark_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Spark_run.java new file mode 100644 index 0000000..6412c4f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Spark_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Spark_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/TaskData.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/TaskData.java new file mode 100644 index 0000000..9c3a8c5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/TaskData.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.List; + + +@Data +public class TaskData { + + private int count; + private List tasks; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Transform_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Transform_param.java new file mode 100644 index 0000000..5fefc0f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Transform_param.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.Date; + + +@Data +public class Transform_param { + + private Date transform_cols; + private String transform_names; + private String transform_type; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/ConfigConst.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/ConfigConst.java new file mode 100644 index 0000000..ab137c1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/ConfigConst.java @@ -0,0 +1,7 @@ +package com.webank.ai.fate.common; + +public interface ConfigConst { + + String FLOW_URL_CONFIG="highflip.adaptor.fate.fate-flow.url"; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DataMultipartFile.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DataMultipartFile.java new file mode 100644 index 0000000..ed844dd --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DataMultipartFile.java @@ -0,0 +1,76 @@ +package com.webank.ai.fate.common; + +import org.springframework.lang.NonNull; +import org.springframework.util.Base64Utils; +import org.springframework.util.FileCopyUtils; +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +public class DataMultipartFile implements MultipartFile { + + private String originalFilename = "file"; + + private final String contentType = "application/octet-stream"; + + private final byte[] content; + + public DataMultipartFile(String filename, byte[] bytes) { + if (filename != null) { + this.originalFilename = filename; + } + this.content = bytes; + } + + public DataMultipartFile(String filename, String base64str) { + if (filename != null) { + this.originalFilename = filename; + } + this.content = Base64Utils.decodeFromString(base64str); + } + + @Override + @NonNull + public String getName() { + return this.originalFilename; + } + + @Override + public String getOriginalFilename() { + return this.originalFilename; + } + + @Override + public String getContentType() { + return this.contentType; + } + + @Override + public boolean isEmpty() { + return (this.content.length == 0); + } + + @Override + public long getSize() { + return this.content.length; + } + + @Override + public byte[] getBytes() throws IOException { + return this.content; + } + + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(this.content); + } + + @Override + public void transferTo(File dest) throws IOException, IllegalStateException { + FileCopyUtils.copy(this.content, dest); + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DecompressUtils.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DecompressUtils.java new file mode 100644 index 0000000..aa2d2b1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DecompressUtils.java @@ -0,0 +1,39 @@ +package com.webank.ai.fate.common; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.springframework.lang.Nullable; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; + +public class DecompressUtils { + + public static Map decompressTarGzToStringMap(InputStream tarGzInputStream, @Nullable Function fileNameFilter) { + if (fileNameFilter == null) { + fileNameFilter = s -> true; + } + Map result = new HashMap<>(); + try (GZIPInputStream gis = new GZIPInputStream(tarGzInputStream);TarArchiveInputStream tais = new TarArchiveInputStream(gis)) { + int i = 0; + TarArchiveEntry entry; + while ((entry = tais.getNextTarEntry()) != null) { + if (fileNameFilter.apply(entry.getName())) { + String text = new BufferedReader(new InputStreamReader(tais, StandardCharsets.UTF_8)).lines().collect(Collectors.joining("\n")); + result.put(entry.getName(), text); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return result; + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/FateConstants.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/FateConstants.java new file mode 100644 index 0000000..7931da6 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/FateConstants.java @@ -0,0 +1,7 @@ +package com.webank.ai.fate.common; + +public class FateConstants { + public static final String DATA_ID_SEPARATOR = "\\+"; + + public static final String DATA_ID_CONNECTOR = "+"; +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/GZIPUtils.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/GZIPUtils.java new file mode 100644 index 0000000..bbc891f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/GZIPUtils.java @@ -0,0 +1,84 @@ +package com.webank.ai.fate.common; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +public class GZIPUtils { + public static final String GZIP_ENCODE_UTF_8 = "UTF-8"; + public static final String GZIP_ENCODE_ISO_8859_1 = "ISO-8859-1"; + + + public static byte[] compress(String str, String encoding) { + if (str == null || str.length() == 0) { + return null; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gzip; + try { + gzip = new GZIPOutputStream(out); + gzip.write(str.getBytes(encoding)); + gzip.close(); + } catch ( Exception e) { + e.printStackTrace(); + } + return out.toByteArray(); + } + + public static byte[] compress(String str) throws IOException { + return compress(str, GZIP_ENCODE_UTF_8); + } + + public static byte[] uncompress(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayInputStream in = new ByteArrayInputStream(bytes); + try { + GZIPInputStream ungzip = new GZIPInputStream(in); + byte[] buffer = new byte[256]; + int n; + while ((n = ungzip.read(buffer)) >= 0) { + out.write(buffer, 0, n); + } + } catch (Exception e) { + e.printStackTrace(); + } + return out.toByteArray(); + } + + public static String uncompressToString(byte[] bytes, String encoding) { + if (bytes == null || bytes.length == 0) { + return null; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayInputStream in = new ByteArrayInputStream(bytes); + try { + GZIPInputStream ungzip = new GZIPInputStream(in); + byte[] buffer = new byte[256]; + int n; + while ((n = ungzip.read(buffer)) >= 0) { + out.write(buffer, 0, n); + } + return out.toString(encoding); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public static String uncompressToString(byte[] bytes) { + return uncompressToString(bytes, GZIP_ENCODE_UTF_8); + } + + public static void main(String[] args) throws IOException { + String s = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + System.out.println("字符串长度："+s.length()); + System.out.println("压缩后：："+compress(s).length); + System.out.println("解压后："+uncompress(compress(s)).length); + System.out.println("解压字符串后：："+uncompressToString(compress(s)).length()); + } +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/FeignSpringFormEncoder.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/FeignSpringFormEncoder.java new file mode 100644 index 0000000..5da179c --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/FeignSpringFormEncoder.java @@ -0,0 +1,255 @@ +package com.webank.ai.fate.common.deserializer; + +import feign.RequestTemplate; +import feign.codec.EncodeException; +import feign.codec.Encoder; +import feign.jackson.JacksonEncoder; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpOutputMessage; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Type; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class FeignSpringFormEncoder extends JacksonEncoder implements Encoder { + + private final List> converters = new RestTemplate().getMessageConverters(); + + public static final Charset UTF_8 = Charset.forName("UTF-8"); + + public FeignSpringFormEncoder() {} + + /** + * 实现一个 HttpOutputMessage + */ + private class HttpOutputMessageImpl implements HttpOutputMessage { + /** + * 输出流，请求体 + */ + private final OutputStream body; + /** + * 请求头 + */ + private final HttpHeaders headers; + + public HttpOutputMessageImpl(OutputStream body, HttpHeaders headers) { + this.body = body; + this.headers = headers; + } + + @Override + public OutputStream getBody() throws IOException { + return body; + } + + @Override + public HttpHeaders getHeaders() { + return headers; + } + } + + /** + * 内部静态类,保存 MultipartFile 数据 + */ + static class MultipartFileResource extends InputStreamResource { + /** + * 文件名 + */ + private final String filename; + /** + * 文件大小 + */ + private final long size; + + /** + * 构造方法 + */ + public MultipartFileResource(InputStream inputStream, String filename, long size) { + super(inputStream); + this.filename = filename; + this.size = size; + } + + @Override + public String getFilename() { + return this.filename; + } + + @Override + public InputStream getInputStream() throws IOException, IllegalStateException { + return super.getInputStream(); + } + + @Override + public long contentLength() throws IOException { + return size; + } + } + + /** + * 重写编码器 + */ + @Override + public void encode(Object object, Type bodyType, RequestTemplate template) throws EncodeException { + if (isJsonRequest(template)){ + super.encode(object, bodyType, template); + } else { + final HttpHeaders multipartHeaders = new HttpHeaders(); + multipartHeaders.setContentType(MediaType.MULTIPART_FORM_DATA); + encodeMultipartFormRequest((Map) object, multipartHeaders, template); + } + } + + private boolean isJsonRequest(RequestTemplate requestTemplate) { + for (Map.Entry> header : requestTemplate.headers().entrySet()) { + if (header.getKey().equalsIgnoreCase("Content-Type")) { + for (String type : header.getValue()) { + if (type.contains("json")) { + return true; + } + } + } + } + return false; + } + + /** + * 对有文件、表单的进行编码 + */ + private void encodeMultipartFormRequest(Map formMap, HttpHeaders multipartHeaders, RequestTemplate template){ + if (formMap == null){ + throw new EncodeException("无法对格式为null的请求进行编码。"); + } + + LinkedMultiValueMap map = new LinkedMultiValueMap<>(); + //对每个参数进行检查校验 + for (Map.Entry entry : formMap.entrySet()){ + Object value = entry.getValue(); + //不同的数据类型进行不同的编码逻辑处理 + if (isMultipartFile(value)){ + //单个文件 + map.add(entry.getKey(), encodeMultipartFile((MultipartFile)value)); + + } else if (isMultipartFileArray(value)){ + //多个文件 + encodeMultipartFiles(map, (String) entry.getKey(), Arrays.asList((MultipartFile[]) value)); + + } else { + //普通请求数据 + map.add(entry.getKey(), encodeJsonObject(value)); + } + } + + encodeRequest(map, multipartHeaders, template); + } + + /** + * 对请求进行编码 + */ + private void encodeRequest(Object value, HttpHeaders requestHeaders, RequestTemplate template){ + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + HttpOutputMessage dummyRequest = new HttpOutputMessageImpl(outputStream, requestHeaders); + + try { + Class requestType = value.getClass(); + MediaType requestContentType = requestHeaders.getContentType(); + for (HttpMessageConverter messageConverter : converters){ + if (messageConverter.canWrite(requestType, requestContentType)){ + ((HttpMessageConverter) messageConverter).write(value, requestContentType, dummyRequest); + break; + } + } + } catch (IOException e) { + throw new EncodeException("无法对请求进行编码:", e); + } + + HttpHeaders headers = dummyRequest.getHeaders(); + if (headers != null){ + for (Map.Entry> entry : headers.entrySet()){ + template.header(entry.getKey(), entry.getValue()); + } + } + + /* + 请使用模板输出流。。。如果文件太大，这将导致问题，因为整个请求都将在内存中。 + */ + template.body(outputStream.toByteArray(), UTF_8); + } + + /** + * 编码为json对象 + */ + private HttpEntity encodeJsonObject(Object obj){ + HttpHeaders jsonPartHeaders = new HttpHeaders(); + jsonPartHeaders.setContentType(MediaType.APPLICATION_JSON); + return new HttpEntity<>(obj, jsonPartHeaders); + } + + /** + * 编码MultipartFile文件，将其转换为HttpEntity，同时设置 Content-type 为 application/octet-stream + * @param map 当前请求 map. + * @param name 数组字段的名称 + * @param fileList 要处理的文件 + */ + private void encodeMultipartFiles(LinkedMultiValueMap map, String name, List fileList){ + HttpHeaders filePartHeaders = new HttpHeaders(); + //设置 Content-type + filePartHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM); + try { + for (MultipartFile file : fileList){ + Resource multipartFileResource = new MultipartFileResource(file.getInputStream(), file.getOriginalFilename(), file.getSize()); + map.add(name, new HttpEntity<>(multipartFileResource, filePartHeaders)); + } + } catch (IOException e) { + throw new EncodeException("无法对请求进行编码:", e); + } + } + + /** + * 编码MultipartFile文件，将其转换为HttpEntity，同时设置 Content-type 为 application/octet-stream + * @param file 要编码的文件 + */ + private HttpEntity encodeMultipartFile(MultipartFile file){ + HttpHeaders filePartHeaders = new HttpHeaders(); + //设置 Content-type + filePartHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM); + try { + Resource multipartFileResource = new MultipartFileResource(file.getInputStream(), file.getOriginalFilename(), file.getSize()); + return new HttpEntity<>(multipartFileResource, filePartHeaders); + } catch (IOException e) { + throw new EncodeException("无法对请求进行编码:", e); + } + } + + /** + * 判断是否多个 MultipartFile + */ + private boolean isMultipartFileArray(Object object){ + return object != null && object.getClass().isArray() && MultipartFile.class.isAssignableFrom(object.getClass().getComponentType()); + } + + /** + * 判断是否MultipartFile文件 + * @param object 要判断的对象 + */ + private boolean isMultipartFile(Object object){ + return object instanceof MultipartFile; + } + + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonListDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonListDeserializer.java new file mode 100644 index 0000000..33f2b0d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonListDeserializer.java @@ -0,0 +1,34 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class JsonListDeserializer extends StdDeserializer> { + + public JsonListDeserializer() { + super(List.class); + } + + @Override + public List deserialize(JsonParser parser, DeserializationContext context) + throws IOException { + JsonNode node = parser.getCodec().readTree(parser); + List result = new ArrayList<>(); + if (node.isArray()) { + for (JsonNode element : node) { + result.add(element.asText()); + } + } else if (node.isObject()) { + result.add(node.asText()); + } else { + //maybe nothing? + } + return result; + } +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapComponentsDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapComponentsDeserializer.java new file mode 100644 index 0000000..580d056 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapComponentsDeserializer.java @@ -0,0 +1,36 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.webank.ai.fate.client.form.dsl.Component; + +import java.io.IOException; +import java.util.Map; + +public class JsonMapComponentsDeserializer extends StdDeserializer> { + + protected JsonMapComponentsDeserializer() { + super(Map.class); + } + + @Override + public Map deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { +// Map ret = new HashMap<>(); +// Map temp = jsonParser.getCodec().readValue(jsonParser, new TypeReference>() { +// }); +// for (String key : temp.keySet()) { +// Object tempValue = temp.get(key); +// if (tempValue != null) { +// ObjectMapper objectMapper = new ObjectMapper(); +// String decode = objectMapper.writeValueAsString(tempValue); +// Component temp1 = objectMapper.readValue(decode, new TypeReference() { +// }); +// ret.put(key, temp1); +// } +// } +// return ret; + return SerializerUtils.mapObjectDeserialize(jsonParser.getCodec().readTree(jsonParser).toString(), Component.class); + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapListStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapListStringDeserializer.java new file mode 100644 index 0000000..cce54a5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapListStringDeserializer.java @@ -0,0 +1,38 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class JsonMapListStringDeserializer extends StdDeserializer>> { + + protected JsonMapListStringDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { + Map> ret = new HashMap<>(); + Map temp = jsonParser.getCodec().readValue(jsonParser, new TypeReference>() { + }); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + List temp1 = objectMapper.readValue(decode, new TypeReference>() { + }); + ret.put(key, temp1); + } + } + return ret; + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapDeserializer.java new file mode 100644 index 0000000..0f37715 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapDeserializer.java @@ -0,0 +1,44 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapMapDeserializer extends StdDeserializer>> { + + public JsonMapMapDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map> ret = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + Map temp1 = objectMapper.readValue(decode, typeReference); + Map map = new HashMap<>(); + for (String key1 : temp.keySet()) { + if (temp1.get(key1) != null) { + map.put(key, temp1.get(key1).toString()); + } + } + ret.put(key, map); + } + } + return ret; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapObjectDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapObjectDeserializer.java new file mode 100644 index 0000000..c83131d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapObjectDeserializer.java @@ -0,0 +1,38 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapMapObjectDeserializer extends StdDeserializer>> { + + public JsonMapMapObjectDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map> ret = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + Map temp1 = objectMapper.readValue(decode, typeReference); + ret.put(key, temp1); + } + } + return ret; + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapStringDeserializer.java new file mode 100644 index 0000000..3e3b543 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapStringDeserializer.java @@ -0,0 +1,44 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapMapStringDeserializer extends StdDeserializer>> { + + public JsonMapMapStringDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map> ret = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + Map temp1 = objectMapper.readValue(decode, typeReference); + Map map = new HashMap<>(); + for (String key1 : temp1.keySet()) { + if (temp1.get(key1) != null) { + map.put(key, temp1.get(key1).toString()); + } + } + ret.put(key, map); + } + } + return ret; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapStringDeserializer.java new file mode 100644 index 0000000..4c555ac --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapStringDeserializer.java @@ -0,0 +1,33 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapStringDeserializer extends StdDeserializer> { + + public JsonMapStringDeserializer(){ + super(Map.class); + } + + @Override + public Map deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map map = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + if (temp.get(key) != null) { + map.put(key, temp.get(key).toString()); + } + } + return map; +// return SerializerUtils.mapObjectDeserialize(jsonParser.getCodec().readTree(jsonParser).toString(), String.class); + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonStringDeserializer.java new file mode 100644 index 0000000..ec89c61 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonStringDeserializer.java @@ -0,0 +1,23 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; + +public class JsonStringDeserializer extends StdDeserializer { + + public JsonStringDeserializer() { + super(String.class); + } + + @Override + public String deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + return node.toString().replace("\"", ""); + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/SerializerUtils.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/SerializerUtils.java new file mode 100644 index 0000000..356a417 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/SerializerUtils.java @@ -0,0 +1,416 @@ +package com.webank.ai.fate.common.deserializer; + +import com.baidu.highflip.core.entity.dag.Graph; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.translator.DSLTranslator; +import org.springframework.lang.Nullable; + +import java.util.HashMap; +import java.util.Map; + +public class SerializerUtils { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private SerializerUtils() { + } + + public static Map mapObjectDeserialize(String object, Class tClass) + throws JsonProcessingException { + if (object == null || object.isBlank()) { + return null; + } + Map ret = new HashMap<>(); + Map map = objectMapper.readValue(object, new TypeReference>() { + }); + for (String tempKey : map.keySet()) { + Object tempValue = map.get(tempKey); + if (tempValue != null) { + String decodeValue = objectMapper.writeValueAsString(tempValue); + if (tClass.isAssignableFrom(String.class)) { + //noinspection unchecked + ret.put(tempKey, (T) decodeValue); + } else { + T result = objectMapper.readValue(decodeValue, tClass); + ret.put(tempKey, result); + } + } + } + return ret; + } + + public static String toJsonString(Object object) throws JsonProcessingException { + return objectMapper.writeValueAsString(object); + } + + public static T deserialize(String json, Class tClass) throws JsonProcessingException { + return objectMapper.readValue(json, tClass); + } + + @Nullable + public static T deserializeType(String json, TypeReference typeReference) throws JsonProcessingException { + if (json == null || json.isBlank()) { + return null; + } + return objectMapper.readValue(json, typeReference); + } + + public static void main(String[] args) throws JsonProcessingException { + Dsl dsl = SerializerUtils.deserialize("{\n" + " \"components\": {\n" + " \"reader_0\": {\n" + + " \"module\": \"Reader\",\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"reader_1\": {\n" + + " \"module\": \"Reader\",\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"data_transform_0\": {\n" + + " \"module\": \"DataTransform\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"reader_0.data\"\n" + " ]\n" + " }\n" + + " },\n" + " \"output\": {\n" + " \"data\": [\n" + + " \"data\"\n" + " ],\n" + " \"model\": [\n" + + " \"model\"\n" + " ]\n" + " }\n" + " },\n" + + " \"data_transform_1\": {\n" + " \"module\": \"DataTransform\",\n" + + " \"input\": {\n" + " \"data\": {\n" + " \"data\": [\n" + + " \"reader_1.data\"\n" + " ]\n" + " }\n" + + " },\n" + " \"output\": {\n" + " \"data\": [\n" + + " \"data\"\n" + " ],\n" + " \"model\": [\n" + + " \"model\"\n" + " ]\n" + " }\n" + " },\n" + + " \"intersection_0\": {\n" + " \"module\": \"Intersection\",\n" + + " \"input\": {\n" + " \"data\": {\n" + " \"data\": [\n" + + " \"data_transform_0.data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"cache\": [\n" + " \"cache\"\n" + " ]\n" + + " }\n" + " },\n" + " \"intersection_1\": {\n" + + " \"module\": \"Intersection\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"data_transform_1.data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"cache\": [\n" + " \"cache\"\n" + " ]\n" + + " }\n" + " },\n" + " \"hetero_nn_0\": {\n" + + " \"module\": \"HeteroNN\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"train_data\": [\n" + + " \"intersection_0.data\"\n" + " ],\n" + + " \"validate_data\": [\n" + " \"intersection_1.data\"\n" + + " ]\n" + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"model\": [\n" + " \"model\"\n" + " ]\n" + + " }\n" + " },\n" + " \"hetero_nn_1\": {\n" + + " \"module\": \"HeteroNN\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"test_data\": [\n" + + " \"intersection_1.data\"\n" + " ]\n" + + " },\n" + " \"model\": [\n" + + " \"hetero_nn_0.model\"\n" + " ]\n" + " },\n" + + " \"output\": {\n" + " \"data\": [\n" + " \"data\"\n" + + " ],\n" + " \"model\": [\n" + " \"model\"\n" + + " ]\n" + " }\n" + " },\n" + " \"evaluation_0\": {\n" + + " \"module\": \"Evaluation\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"hetero_nn_0.data\",\n" + " \"hetero_nn_1.data\"\n" + + " ]\n" + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " }\n" + " }\n" + "}", Dsl.class); + DslConf dslConf = SerializerUtils.deserialize("{\n" + " \"dsl_version\": 2,\n" + " \"role\": {\n" + + " \"host\": [\n" + " 10000\n" + " ],\n" + " \"guest\": [\n" + + " 9999\n" + " ]\n" + " },\n" + " \"component_parameters\": {\n" + + " \"role\": {\n" + " \"guest\": {\n" + " \"0\": {\n" + + " \"data_transform_0\": {\n" + " \"with_label\": true\n" + + " },\n" + " \"hetero_nn_0\": {\n" + + " \"bottom_nn_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 10\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 10\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 3,\n" + + " \"activation\": \"relu\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " },\n" + + " \"top_nn_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_2\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_2_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_2\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 1,\n" + + " \"activation\": \"sigmoid\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " },\n" + + " \"interactive_layer_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_1\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_1_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_1\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 2,\n" + + " \"activation\": \"linear\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " }\n" + + " },\n" + " \"reader_1\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_hetero_guest\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " },\n" + " \"data_transform_1\": {\n" + + " \"with_label\": true\n" + " },\n" + + " \"reader_0\": {\n" + " \"table\": {\n" + + " \"name\": \"breast_hetero_guest\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " }\n" + " }\n" + " },\n" + " \"host\": {\n" + + " \"0\": {\n" + " \"data_transform_0\": {\n" + + " \"with_label\": false\n" + " },\n" + + " \"hetero_nn_0\": {\n" + " \"bottom_nn_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_3\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 20\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_3_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_3\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 20\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 3,\n" + + " \"activation\": \"relu\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " },\n" + + " \"interactive_layer_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_4\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_4_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_4\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 2,\n" + + " \"activation\": \"linear\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " }\n" + + " },\n" + " \"reader_1\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_hetero_host\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " },\n" + " \"data_transform_1\": {\n" + + " \"with_label\": false\n" + " },\n" + + " \"reader_0\": {\n" + " \"table\": {\n" + + " \"name\": \"breast_hetero_host\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " }\n" + " }\n" + " }\n" + " },\n" + + " \"common\": {\n" + " \"hetero_nn_0\": {\n" + " \"epochs\": 3,\n" + + " \"interactive_layer_lr\": 0.15,\n" + " \"batch_size\": -1,\n" + + " \"early_stop\": \"diff\",\n" + " \"callback_param\": {\n" + + " \"callbacks\": [\n" + " \"ModelCheckpoint\"\n" + + " ],\n" + " \"save_freq\": \"epoch\",\n" + + " \"validation_freqs\": 1,\n" + + " \"use_first_metric_only\": true,\n" + + " \"early_stopping_rounds\": 15,\n" + " \"metrics\": [\n" + + " \"AUC\"\n" + " ]\n" + " },\n" + + " \"optimizer\": {\n" + " \"learning_rate\": 0.15,\n" + + " \"decay\": 0.0,\n" + " \"momentum\": 0.0,\n" + + " \"nesterov\": false,\n" + " \"optimizer\": \"SGD\"\n" + + " },\n" + " \"loss\": \"binary_crossentropy\",\n" + + " \"bottom_nn_define\": null,\n" + " \"top_nn_define\": null,\n" + + " \"interactive_layer_define\": null,\n" + + " \"config_type\": \"keras\"\n" + " },\n" + + " \"hetero_nn_1\": {\n" + " \"optimizer\": null,\n" + + " \"bottom_nn_define\": null,\n" + " \"top_nn_define\": null,\n" + + " \"interactive_layer_define\": null,\n" + + " \"config_type\": \"keras\"\n" + " }\n" + " }\n" + " }\n" + "}", DslConf.class); + DSLTranslator.FateDAG fateDAG = new DSLTranslator.FateDAG(); + fateDAG.setConf(dslConf); + fateDAG.setDsl(dsl); + DSLTranslator dslTranslator = new DSLTranslator(); + Graph graph = dslTranslator.translate(fateDAG); + DSLTranslator.FateDAG fateDAG1 = dslTranslator.translate(graph); + System.out.println(toJsonString(fateDAG1)); + System.out.println(toJsonString(fateDAG)); + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java index d0d7353..9219ba2 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java @@ -1,7 +1,9 @@ package com.webank.ai.fate.context; +import com.baidu.highflip.core.adaptor.ServiceAdaptor; import com.baidu.highflip.core.common.InstanceNameList; import com.baidu.highflip.core.engine.Configuration; +import com.baidu.highflip.core.engine.HighFlipRuntime; import com.baidu.highflip.core.engine.InstanceRegister; import com.webank.ai.fate.client.FateClient; import com.webank.ai.fate.translator.DSLTranslator; @@ -18,6 +20,8 @@ public class FateContext { Configuration config; + HighFlipRuntime highFlipRuntime; + FateClient client; DSLTranslator translator = new DSLTranslator(); @@ -26,8 +30,16 @@ public static FateContext from(InstanceRegister register) { FateContext context = new FateContext(); Configuration config = (Configuration) register.getInstance(InstanceNameList.HIGHFLIP_CONFIGURATION); + HighFlipRuntime highFlipRuntime = + (HighFlipRuntime) register.getInstance(InstanceNameList.HIGHFLIP_RUNTIME); context.setConfig(config); - + context.setHighFlipRuntime(highFlipRuntime); + ServiceAdaptor serviceAdaptor = + (ServiceAdaptor) register.getInstance(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE); + log.info("connecting to fate flow http service:{}", serviceAdaptor.getUrl()); + context.setClient(FateClient.connect(serviceAdaptor.getUrl())); + context.getTranslator().setPartyId(serviceAdaptor.getPartyId()); + context.getTranslator().setRole(serviceAdaptor.getRole()); return context; } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java index 117f719..3b8af09 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java @@ -1,17 +1,360 @@ package com.webank.ai.fate.translator; import com.baidu.highflip.core.entity.dag.Graph; +import com.baidu.highflip.core.entity.dag.Node; +import com.baidu.highflip.core.entity.dag.Party; +import com.baidu.highflip.core.entity.dag.PartyNode; +import com.baidu.highflip.core.entity.dag.codec.TypeValue; +import com.baidu.highflip.core.entity.dag.common.NodeInputRef; +import com.baidu.highflip.core.entity.dag.common.NodeOutputRef; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.webank.ai.fate.client.form.dsl.Component; +import com.webank.ai.fate.client.form.dsl.ComponentParameters; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.client.form.dsl.Input; +import com.webank.ai.fate.client.form.dsl.Output; +import com.webank.ai.fate.client.form.dsl.RoleConf; +import com.webank.ai.fate.client.form.dsl.Site; +import com.webank.ai.fate.common.deserializer.SerializerUtils; + +import highflip.HighflipMeta; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.util.CollectionUtils; + @Slf4j -@NoArgsConstructor +@Data public class DSLTranslator { + String partyId; + + String role; + public FateDAG translate(Graph dag) { - return new FateDAG(); + Map componentMap = new HashMap<>(); + Map> componentConfMap = new HashMap<>(); + Map> role = new HashMap<>(); + Map> hostConf = new HashMap<>(); + Map> guestConf = new HashMap<>(); + Map> arbiterConf = new HashMap<>(); + for (Node node : dag.getNodes().values()) { + Component component = new Component(); + component.setInput(transInput(node.getInputs())); + component.setOutput(transOutput(node.getOutputs())); + component.setModule(node.getType()); + Map conf = new HashMap<>(); + for (String key : node.getAttributes().keySet()) { + conf.put(key, TypeValue.fromProto( + (HighflipMeta.TypedValueProto) node.getAttribute(key, null))); + } + componentConfMap.put(node.getName(), conf); + componentMap.put(node.getName(), component); + } + int guestIndex = 0; + int hostIndex = 0; + int arbiterIndex = 0; + Map> roleMap = new HashMap<>(); + List guest = new ArrayList<>(); + List host = new ArrayList<>(); + List arbiter = new ArrayList<>(); + for (Party party : dag.getParties().values()) { + role.putIfAbsent(party.getRole(), new ArrayList<>()); + role.get(party.getRole()).add(party.getName()); + Map componentAttr = new HashMap<>(); + for (PartyNode node : party.getNodes().values()) { + Map attr = node.getAttributes(); + componentAttr.put(node.getName(), attr); + } + if (party.getRole().equalsIgnoreCase("host")) { + hostConf.put(hostIndex + "", componentAttr); + host.add(party.getName()); + hostIndex++; + } + if (party.getRole().equalsIgnoreCase("guest")) { + guestConf.put(guestIndex + "", componentAttr); + guest.add(party.getName()); + guestIndex++; + } + if (party.getRole().equalsIgnoreCase("arbiter")) { + arbiterConf.put(arbiterIndex + "", componentAttr); + arbiter.add(party.getName()); + arbiterIndex++; + } + } + if (!guest.isEmpty()) { + roleMap.put("guest", guest); + } + if (!host.isEmpty()) { + roleMap.put("host", host); + } + if (!arbiter.isEmpty()) { + roleMap.put("arbiter", arbiter); + } + Dsl dsl = new Dsl(); + dsl.setComponents(componentMap); + DslConf dslConf = new DslConf(); + dslConf.setDsl_version("2"); + dslConf.setRole(roleMap); + ComponentParameters componentParameters = new ComponentParameters(); + componentParameters.setCommon(componentConfMap); + RoleConf roleConf = new RoleConf(); + if (!hostConf.isEmpty()) { + roleConf.setHost(hostConf); + } + if (!guestConf.isEmpty()) { + roleConf.setGuest(guestConf); + } + if (!arbiterConf.isEmpty()) { + roleConf.setArbiter(arbiterConf); + } + componentParameters.setRole(roleConf); + dslConf.setComponent_parameters(componentParameters); + FateDAG fateDAG = new FateDAG(); + fateDAG.setDsl(dsl); + fateDAG.setConf(dslConf); + // set initiator to fateDAG + Site initiator = new Site(); + initiator.setRole(this.role); + initiator.setParty_id(this.partyId); + fateDAG.getConf().setInitiator(initiator); + return fateDAG; + } + + public Graph translate(FateDAG dag) { + Graph graph = new Graph(); + Dsl dsl = dag.getDsl(); + Node pre = null; + Map nodeMap = new HashMap<>(); + Map partyMap = new HashMap<>(); + Map> rolePartyMap = new HashMap<>(); + graph.setNodes(nodeMap); + graph.setParties(partyMap); + for (Map.Entry entity : dsl.getComponents().entrySet()) { + String key = entity.getKey(); + Component component = entity.getValue(); + Node node = new Node(); + node.setName(key); + node.setType(component.getModule()); + if (component.getInput() != null) { + node.setInputs(transInput(component.getInput())); + } + if (component.getOutput() != null) { + node.setOutputs(transOutput(component.getOutput())); + } + if (pre != null) { + node.setParent(pre); + } + pre = node; + nodeMap.put(key, node); + } + DslConf dslConf = dag.getConf(); + for (Map.Entry> roleEntity : dslConf.getRole().entrySet()) { + String role = roleEntity.getKey(); + rolePartyMap.putIfAbsent(role.toLowerCase(), new ArrayList<>()); + for (String partyId : roleEntity.getValue()) { + Party party = new Party(); + party.setName(partyId); + party.setRole(role); + party.setNodes(new HashMap<>()); + partyMap.put(role + "-" + partyId, party); + rolePartyMap.get(role.toLowerCase()).add(party); + } + } + ComponentParameters componentParameters = dslConf.getComponent_parameters(); + for (Map.Entry> attrMap : componentParameters.getCommon().entrySet()) { + String componentName = attrMap.getKey(); + Map attr = attrMap.getValue(); + Node node = nodeMap.get(componentName); + if (node == null) { + throw new RuntimeException("dsl conf component not match"); + } + attr.forEach(node::setAttribute); + } + if (componentParameters.getRole().getGuest() != null) { + for (Map.Entry> attr : componentParameters.getRole().getGuest().entrySet()) { + String index = attr.getKey(); + Map componentsAttrs = attr.getValue(); + Party party = rolePartyMap.get("guest").get(Integer.parseInt(index)); + addPartyNodes(componentsAttrs, party); + } + } + if (componentParameters.getRole().getHost() != null) { + for (Map.Entry> attr : componentParameters.getRole().getHost().entrySet()) { + String index = attr.getKey(); + Map componentsAttrs = attr.getValue(); + Party party = rolePartyMap.get("host").get(Integer.parseInt(index)); + addPartyNodes(componentsAttrs, party); + } + } + if (componentParameters.getRole().getArbiter() != null) { + for (Map.Entry> attr : componentParameters.getRole().getArbiter().entrySet()) { + String index = attr.getKey(); + Map componentsAttrs = attr.getValue(); + Party party = rolePartyMap.get("arbiter").get(Integer.parseInt(index)); + addPartyNodes(componentsAttrs, party); + } + } + return graph; + } + + private void addPartyNodes(Map componentsAttrs, Party party) { + if (party == null) { + throw new RuntimeException("dsl conf role not match"); + } + for (Map.Entry componentEntry : componentsAttrs.entrySet()) { + String componentName = componentEntry.getKey(); + Object componentAttr = componentEntry.getValue(); + PartyNode partyNode = new PartyNode(); + partyNode.setParent(party); + partyNode.setName(componentName); + Map attrMap; + try { + attrMap = SerializerUtils.deserializeType(SerializerUtils.toJsonString(componentAttr), + new TypeReference>() { + }); + if (attrMap != null) { + partyNode.setAttributes(attrMap); + } + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + party.getNodes().put(componentName, partyNode); + } + } + + private Map transInput(Input input) { + try { + Map map = new HashMap<>(); + if (input.getData() != null) { + HighflipMeta.NodeInputProto nodeInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(SerializerUtils.toJsonString(input.getData())) + .build(); + map.put("data", NodeInputRef.fromProto(nodeInputProto)); + } + if (input.getModel() != null) { + HighflipMeta.NodeInputProto nodeInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("model") + .setValue(SerializerUtils.toJsonString(input.getModel())) + .build(); + map.put("model", NodeInputRef.fromProto(nodeInputProto)); + } + if (input.getCache() != null) { + HighflipMeta.NodeInputProto nodeInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("cache") + .setValue(SerializerUtils.toJsonString(input.getCache())) + .build(); + map.put("cache", NodeInputRef.fromProto(nodeInputProto)); + } + return map; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Input transInput(Map map) { + try { + if (map == null || CollectionUtils.isEmpty(map)) { + return null; + } + Input input = new Input(); + TypeReference>> typeReference = new TypeReference<>() { + }; + if (map.containsKey("data")) { + String content = map.get("data").getValue(); + Map> data = + SerializerUtils.deserializeType(content, typeReference); + input.setData(data); + } + TypeReference> typeReference2 = new TypeReference<>() { + }; + if (map.containsKey("model")) { + List model = SerializerUtils.deserializeType(map.get( + "model").getValue(), typeReference2); + input.setModel(model); + } + + if (map.containsKey("cache")) { + List cache = SerializerUtils.deserializeType(map.get( + "cache").getValue(), typeReference2); + input.setCache(cache); + } + return input; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Map transOutput(Output output) { + try { + Map map = new HashMap<>(); + HighflipMeta.NodeOutputProto dataNodeOutputProto = + HighflipMeta.NodeOutputProto.newBuilder() + .setName("data") + .setValue(SerializerUtils.toJsonString(output.getData())) + .build(); + map.put("data", NodeOutputRef.fromProto(dataNodeOutputProto)); + + HighflipMeta.NodeOutputProto modelNodeOutputProto = + HighflipMeta.NodeOutputProto.newBuilder() + .setName("model") + .setValue(SerializerUtils.toJsonString(output.getModel())) + .build(); + map.put("model", NodeOutputRef.fromProto(modelNodeOutputProto)); + + HighflipMeta.NodeOutputProto cacheNodeOutputProto = + HighflipMeta.NodeOutputProto.newBuilder() + .setName("cache") + .setValue(SerializerUtils.toJsonString(output.getCache())) + .build(); + map.put("cache", NodeOutputRef.fromProto(cacheNodeOutputProto)); + return map; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Output transOutput(Map map) { + try { + if (map == null || CollectionUtils.isEmpty(map)) { + return null; + } + TypeReference> typeReference2 = new TypeReference<>() { + }; + Output output = new Output(); + if (map.containsKey("data")) { + List data = SerializerUtils.deserializeType( + map.get("data").getValue(), typeReference2); + output.setData(data); + } + if (map.containsKey("model")) { + List model = SerializerUtils.deserializeType( + map.get("model").getValue(), typeReference2); + output.setModel(model); + + } + if (map.containsKey("cache")) { + List cache = SerializerUtils.deserializeType( + map.get("cache").getValue(), typeReference2); + output.setCache(cache); + } + return output; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } } @Data @@ -19,8 +362,9 @@ public FateDAG translate(Graph dag) { @NoArgsConstructor public static class FateDAG { - String dsl; + Dsl dsl; + + DslConf conf; - String conf; } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/adaptor/DataAdaptorTest.java b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/adaptor/DataAdaptorTest.java new file mode 100644 index 0000000..c47cf48 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/adaptor/DataAdaptorTest.java @@ -0,0 +1,44 @@ +package com.webank.ai.fate.adaptor; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; + +import com.baidu.highflip.core.utils.Foreach; + +public class DataAdaptorTest { + private final String DEFAULT_DELIMITER = ","; + + @Test + public void writeDataDenseTest() { + List> body = new ArrayList<>(); + List rowHeader = List.of("ID","name"); + List rowOne = List.of("1","bob"); + List rowTwo = List.of("2","joe"); + List rowThree = List.of("3","alice"); + body.add(rowHeader); + body.add(rowOne); + body.add(rowTwo); + body.add(rowThree); + Iterator> iterator = body.iterator(); + + StringBuilder stringBuilder = new StringBuilder(); + for (List dataList : Foreach.from(iterator)) { + String column = dataList.stream().map(Object::toString).collect( + Collectors.joining(DEFAULT_DELIMITER)); + stringBuilder.append(column).append("\n"); + } + } + + @Test + public void testSplit(){ + String dataId = "8a53e2c4-7e04-482b-9eef-4dd6c8376b1e+intersect_0" + + "+guest+9999"; + final String[] split = dataId.split("\\+"); + System.out.println(List.of(split)); + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java index d445c3a..aad5c89 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java +++ b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java @@ -1,38 +1,158 @@ package com.webank.ai.fate.client; -import com.webank.ai.fate.client.form.JsonResultForm; +import com.fasterxml.jackson.core.JsonProcessingException; import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.client.form.job.JobData; +import com.webank.ai.fate.client.form.task.TaskData; +import com.webank.ai.fate.common.DataMultipartFile; +import com.webank.ai.fate.common.DecompressUtils; +import com.webank.ai.fate.common.deserializer.SerializerUtils; +import feign.Response; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.springframework.web.multipart.MultipartFile; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; @Slf4j @Data @Disabled public class FateClientTest { - public static final String TEST_FLOW_URL = "http://127.0.0.1:9380"; + public static final String TEST_FLOW_URL = "http://10.27.130.41:8380"; FateClient client = FateClient.connect(TEST_FLOW_URL); + // @Test + // public void testVersionGet() { + // JsonResultForm result = getClient().version(); + // log.info("result = {}", result.getData().toPrettyString()); + // } + @Test - public void testVersionGet() { - JsonResultForm result = getClient().version(); - log.info("result = {}", result.getData().toPrettyString()); + public void testJobList() { + ResultForm result = getClient().listJob(1, 10); + log.info("result = {}", result); } @Test - public void testJobList() { - JsonResultForm result = getClient().jobList(10); - log.info("result = {}", result.getData().toPrettyString()); + public void testTaskList() { + ResultForm result = getClient().listTask(null, 18, 1); + log.info("result = {}", result); } @Test public void testJobQuery() { - ResultForm result = getClient().jobQuery("202210260752357956950"); + ResultForm result = getClient().jobQuery("202212011128410103660"); log.info("result = {}", result.getData().toString()); } + + @Test + public void testPushData() { + String data = "id,y,x0,x1,x2,x3,x4,x5\n" + "274,0,0.963102,1.467675,0.829202,0.772457,-0.038076,-0.468613\n" + + "199,0,0.426758,0.723479,0.316885,0.287273,1.000835,0.962702\n" + + "133,1,0.254879,-1.046633,0.209656,0.074214,-0.441366,-0.377645\n" + + "273,1,-1.142928,-0.781198,-1.166747,-0.923578,0.62823,-1.021418\n" + + "551,1,-0.879933,0.420589,-0.877527,-0.780484,-1.037534,-0.48388\n" + + "175,1,-1.451067,-1.406518,-1.456564,-1.092337,-0.708765,-1.168557\n"; + MultipartFile multipartFile = new DataMultipartFile("unittest_data3.csv", data.getBytes(StandardCharsets.UTF_8)); + ResultForm resultForm = getClient().pushData(multipartFile, ",", "1", "4", "highflip_test1", "experiment", null, null, null,"1"); + log.info("result = {}", resultForm); + } + + @Test + public void testJobSubmit() throws JsonProcessingException { + Dsl dsl = SerializerUtils.deserialize("{\n" + " \"components\": {\n" + " \"reader_0\": {\n" + + " \"module\": \"Reader\",\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"data_transform_0\": {\n" + + " \"module\": \"DataTransform\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"reader_0.data\"\n" + " ]\n" + " }\n" + + " },\n" + " \"output\": {\n" + " \"data\": [\n" + + " \"data\"\n" + " ],\n" + " \"model\": [\n" + + " \"model\"\n" + " ]\n" + " }\n" + " },\n" + + " \"homo_nn_0\": {\n" + " \"module\": \"HomoNN\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"train_data\": [\n" + + " \"data_transform_0.data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"model\": [\n" + " \"model\"\n" + " ]\n" + + " }\n" + " }\n" + " }\n" + "}", Dsl.class); + DslConf dslConf = SerializerUtils.deserialize("{\n" + " \"dsl_version\": 2,\n" + " \"initiator\": {\n" + + " \"role\": \"guest\",\n" + " \"party_id\": 10000\n" + " },\n" + " \"role\": {\n" + + " \"arbiter\": [\n" + " 10000\n" + " ],\n" + " \"host\": [\n" + + " 10000\n" + " ],\n" + " \"guest\": [\n" + " 10000\n" + + " ]\n" + " },\n" + " \"component_parameters\": {\n" + " \"common\": {\n" + + " \"data_transform_0\": {\n" + " \"with_label\": true,\n" + + " \"label_name\": \"y\",\n" + " \"label_type\": \"int\",\n" + + " \"output_format\": \"dense\"\n" + " },\n" + + " \"homo_nn_0\": {\n" + " \"config_type\": \"pytorch\",\n" + + " \"nn_define\": [\n" + " {\n" + + " \"layer\": \"Linear\",\n" + " \"name\": \"line\",\n" + + " \"type\": \"normal\",\n" + " \"config\": [\n" + + " 30,\n" + " 1\n" + + " ]\n" + " },\n" + " {\n" + + " \"layer\": \"Sigmoid\",\n" + + " \"type\": \"activate\",\n" + + " \"name\": \"sigmoid\"\n" + " }\n" + " ],\n" + + " \"batch_size\": -1,\n" + " \"optimizer\": {\n" + + " \"optimizer\": \"Adam\",\n" + " \"lr\": 0.05\n" + + " },\n" + " \"early_stop\": {\n" + + " \"early_stop\": \"diff\",\n" + " \"eps\": 0.0001\n" + + " },\n" + " \"loss\": \"BCELoss\",\n" + + " \"metrics\": [\n" + " \"accuracy\"\n" + " ],\n" + + " \"max_iter\": 2\n" + " }\n" + " },\n" + " \"role\": {\n" + + " \"host\": {\n" + " \"0\": {\n" + " \"reader_0\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_homo_host\",\n" + + " \"namespace\": \"HIGH-FLIP\"\n" + " }\n" + + " }\n" + " }\n" + " },\n" + " \"guest\": {\n" + + " \"0\": {\n" + " \"reader_0\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_homo_guest\",\n" + + " \"namespace\": \"HIGH-FLIP\"\n" + " }\n" + + " }\n" + " }\n" + " }\n" + " }\n" + " }\n" + + "}", DslConf.class); + log.info("dsl:{}", SerializerUtils.toJsonString(dsl)); + log.info("dsl conf:{}", SerializerUtils.toJsonString(dslConf)); + ResultForm job = getClient().jobSubmit(dsl, dslConf); + log.info("result = {}", job); + } + + @Test + public void testDownloadData() { + try(Response response = getClient().downloadData("unittest_data3", "experiment")) { + InputStream inputStream = response.body().asInputStream(); + Map result = DecompressUtils.decompressTarGzToStringMap(inputStream, s -> s.contains("csv")); + log.info("result = {}, head = {}", result, response.headers()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testDownloadComponentResultData(){ + String jobId = "202305231618271012050"; + String componentName = "intersect_0"; + String role = "guest"; + String partyId = "9999"; + try (Response response = getClient().downloadComponentResultData(jobId, componentName, role, partyId)) { + InputStream inputStream = response.body().asInputStream(); + + Map result = DecompressUtils.decompressTarGzToStringMap(inputStream, s -> s.contains("csv")); + log.info("result = {}, head = {}", result, response.headers()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/translator/DSLTranslatorTest.java b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/translator/DSLTranslatorTest.java new file mode 100644 index 0000000..7f4db49 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/translator/DSLTranslatorTest.java @@ -0,0 +1,434 @@ +package com.webank.ai.fate.translator; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.baidu.highflip.core.entity.dag.Graph; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.ai.fate.client.FateClient; +import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.common.deserializer.SerializerUtils; + +import highflip.HighflipMeta; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DSLTranslatorTest { + + @Test + public void translateTest() { + // step 1: construct a highflip graph + HighflipMeta.GraphProto graphProto = constructHighflipGraph(); + + Graph graph = Graph.fromProto(graphProto); + // step 2: translate highflip graph to fate dag (dsl and conf) + DSLTranslator dslTranslator = new DSLTranslator(); + + DSLTranslator.FateDAG translate = dslTranslator.translate(graph); + + // fate flow http service + String TEST_FLOW_URL = "http://10.27.130.41:8380"; + + FateClient client = FateClient.connect(TEST_FLOW_URL); + + try { + log.info("conf = {}", + SerializerUtils.toJsonString(translate.getDsl())); + log.info("dsl = {}", + SerializerUtils.toJsonString(translate.getConf())); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + ResultForm job = client.jobSubmit(translate.getDsl(), + translate.getConf()); + log.info("result = {}", job); + } + + private HighflipMeta.GraphProto constructHighflipGraph() { + // reader + List readerNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto outputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data") + .setValue(getJsonStringOfDataOutput()) + .build(); + readerNodeOutputProtos.add(outputProto); + HighflipMeta.NodeProto reader = + HighflipMeta.NodeProto.newBuilder().setName("reader_0") + .setType("Reader") + .addAllOutputs(readerNodeOutputProtos) + .build(); + + // DataTransform + List dataTransformNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto dataTransformOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data") + .setValue(getJsonStringOfDataOutput()) + .build(); + HighflipMeta.NodeOutputProto dataTransformOutputProto2 = + HighflipMeta.NodeOutputProto.newBuilder().setName("model") + .setValue(getJsonStringOfModelOutput()) + .build(); + dataTransformNodeOutputProtos.add(dataTransformOutputProto); + dataTransformNodeOutputProtos.add(dataTransformOutputProto2); + + List dataTransformNodeInputProtos = + new ArrayList<>(); + String json = getDataTransformInputJsonString(); + HighflipMeta.NodeInputProto dataTransformInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(json) + .build(); + dataTransformNodeInputProtos.add(dataTransformInputProto); + HighflipMeta.NodeProto dataTransform = + HighflipMeta.NodeProto.newBuilder().setName("data_transform_0") + .setType("DataTransform") + .addAllInputs(dataTransformNodeInputProtos) + .addAllOutputs(dataTransformNodeOutputProtos) + .build(); + + // intersect + List intersectNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto intersectOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data") + .setValue(getJsonStringOfDataOutput()) + .build(); + intersectNodeOutputProtos.add(intersectOutputProto); + List intersectNodeInputProtos = + new ArrayList<>(); + String dataTransformInputJsonString = getIntersectionInputJsonString(); + HighflipMeta.NodeInputProto intersectInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(dataTransformInputJsonString) + .build(); + intersectNodeInputProtos.add(intersectInputProto); + HighflipMeta.NodeProto intersect = + HighflipMeta.NodeProto.newBuilder().setName("intersect_0") + .setType("Intersection") + .addAllInputs(intersectNodeInputProtos) + .addAllOutputs(intersectNodeOutputProtos) + .build(); + + // construct guest + HighflipMeta.PartyProto.PartyNode guestPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getGuestPartyReaderNode()) + .build(); + HighflipMeta.TypedValueProto withLabel= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + HighflipMeta.TypedValueProto outputFormat= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("dense") + .build()).build(); + + HighflipMeta.PartyProto.PartyNode guestPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + // common conf + /* + "intersect_0": { + "intersect_method": "rsa", + "sync_intersect_ids": false, + "only_output_key": true, + "rsa_params": { + "hash_method": "sha256", + "final_hash_method": "sha256", + "split_calculation": false, + "key_length": 2048 + } + } + */ + HighflipMeta.TypedValueProto intersectMethod= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("rsa") + .build()).build(); + HighflipMeta.TypedValueProto syncIntersectIds= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + HighflipMeta.TypedValueProto onlyOutputKey= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(true) + .build()).build(); + + HighflipMeta.PartyProto.PartyNode commonPartyNode3 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(intersect.getName()) + .putAttributes("intersect_method",intersectMethod) + .putAttributes("sync_intersect_ids",syncIntersectIds) + .putAttributes("only_output_key",onlyOutputKey) + .putAttributes("rsa_params", + getCommonConfRsaParamTypedValueProto()) + .build(); + + List guestPartyNodes = + new ArrayList<>(); + guestPartyNodes.add(guestPartyNode1); + guestPartyNodes.add(guestPartyNode2); + guestPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto guest = + HighflipMeta.PartyProto.newBuilder() + .setRole(HighflipMeta.PartyRole.GUEST) + .addAllNodes(guestPartyNodes) + .setName("9999") + .build(); + + // construct host + HighflipMeta.PartyProto.PartyNode hostPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getHostReaderTypedValueProto()) + .build(); + + HighflipMeta.PartyProto.PartyNode hostPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + List hostPartyNodes = + new ArrayList<>(); + hostPartyNodes.add(hostPartyNode1); + hostPartyNodes.add(hostPartyNode2); + hostPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto host = + HighflipMeta.PartyProto.newBuilder() + .setRole(HighflipMeta.PartyRole.HOST) + .addAllNodes(hostPartyNodes) + .setName("10000") + .build(); + + // components + List node = new ArrayList<>(); + node.add(reader); + node.add(dataTransform); + node.add(intersect); + + List partyProtoList = new ArrayList<>(); + partyProtoList.add(guest); + partyProtoList.add(host); + + HighflipMeta.GraphProto graphProto = + HighflipMeta.GraphProto.newBuilder().setName("jobId-202301") + .addAllNodes(node) + .addAllParties(partyProtoList).build(); + + return graphProto; + } + + private String getIntersectionInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("data_transform_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getDataTransformInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("reader_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfDataOutput() { + List data = new ArrayList<>(); + data.add("data"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfModelOutput() { + List data = new ArrayList<>(); + data.add("model"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private HighflipMeta.TypedValueProto getGuestPartyReaderNode() { + Map tableMap = new HashMap<>(); + tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv2").build()); + tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto mapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + HighflipMeta.ValueProto valueProto = + HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + highflip.HighflipMeta.TypedValueProto typedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + return typedValueProto; + } + + private HighflipMeta.TypedValueProto getCommonConfRsaParamTypedValueProto(){ + Map commonRsaParamConfMap = + new HashMap<>(); + commonRsaParamConfMap.put("hash_method", + HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("split_calculation", + HighflipMeta.ValueProto.newBuilder().setBool( + false).build()); + commonRsaParamConfMap.put("key_length", + HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + HighflipMeta.MapProto commonValueMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(commonRsaParamConfMap).build(); + HighflipMeta.ValueProto commonValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(commonValueMapProto).build(); + highflip.HighflipMeta.TypedValueProto commonTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(commonValueProto).build(); + return commonTypedValueProto; + } + + private HighflipMeta.TypedValueProto getHostReaderTypedValueProto() { + Map hostTableMap = + new HashMap<>(); + hostTableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + hostTableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto hostMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(hostTableMap).build(); + HighflipMeta.ValueProto hostValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(hostMapProto).build(); + highflip.HighflipMeta.TypedValueProto hostTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(hostValueProto).build(); + return hostTypedValueProto; + } + + @Test + public void testValueProto() { + Map tableMap2 = + new HashMap<>(); + tableMap2.put("name", HighflipMeta.ValueProto.newBuilder().setString( + "csv1").build()); + tableMap2.put("namespace", + HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + + Map tableMap = new HashMap<>(); + tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + tableMap.put("bo", + HighflipMeta.ValueProto.newBuilder().setBool(true).build()); + + HighflipMeta.MapProto mapProto2 = HighflipMeta.MapProto.newBuilder().putAllMap(tableMap2).build(); + tableMap.put("mp", + HighflipMeta.ValueProto.newBuilder().setMap(mapProto2).build()); + + List list = new ArrayList<>(); + list.add(HighflipMeta.ValueProto.newBuilder().setMap(mapProto2).build()); + + HighflipMeta.ListProto listProto = + HighflipMeta.ListProto.newBuilder().addAllList(list).build(); + + tableMap.put("list", + HighflipMeta.ValueProto.newBuilder().setList(listProto).build()); + + HighflipMeta.MapProto mapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + HighflipMeta.ValueProto valueProto = + HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + + highflip.HighflipMeta.TypedValueProto typedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + HighflipMeta.MapProto map = typedValueProto.getValue().getMap(); + Map map1 = mapProtoConvert2Map(map); + try{ + String s = SerializerUtils.toJsonString(map1); + System.out.println(s); + }catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Map mapProtoConvert2Map(HighflipMeta.MapProto mapProto){ + Map result = new HashMap<>(); + Map map = mapProto.getMapMap(); + for(Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + HighflipMeta.ValueProto value = entry.getValue(); + result.put(key, convertValueProto(value)); + } + return result; + } + + private Object convertValueProto(HighflipMeta.ValueProto value){ + switch (value.getValueCase()) { + case BOOL: + return value.getBool(); + case INT: + return value.getInt(); + case LONG: + return value.getLong(); + case FLOAT: + return value.getFloat(); + case DOUBLE: + return value.getDouble(); + case STRING: + return value.getString(); + case BYTES: + return value.getBytes(); + case LIST: + List list = + value.getList().getListList(); + List resultList = new ArrayList<>(); + for(HighflipMeta.ValueProto listValue: list) { + resultList.add(convertValueProto(listValue)); + } + return resultList; + case MAP: + // Map + return mapProtoConvert2Map(value.getMap()); + default: + throw new RuntimeException("NOT_SUPPORTED_TYPE"); + } + } + +} diff --git a/highflip-vendors/pom.xml b/highflip-vendors/pom.xml index 35c88ca..38a435b 100644 --- a/highflip-vendors/pom.xml +++ b/highflip-vendors/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-vendors pom 1.0.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index 2d5fe7b..5358d7a 100644 --- a/pom.xml +++ b/pom.xml @@ -2,13 +2,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip pom 1.0.0-SNAPSHOT highflip - - http://maven.apache.org + High layer Federated Learning Intercommunication Protocol + https://github.com/baidu/highflip @@ -29,6 +29,17 @@ 11 11 1.49.0 + 2.7.3 + 1.18.22 + 6.11 + 1.43 + 31.1-jre + 6.0.2 + + + 3.2.0 + 1.6 + 1.6.7 @@ -38,8 +49,26 @@ highflip-server highflip-doc highflip-clients + highflip-editor + + + Baidu dianshi + dianshi-help@baidu.com + https://dianshi.baidu.com/ + dianshi + https://dianshi.baidu.com/ + + + + + scm:git:${project.scm.url} + scm:git:${project.scm.url} + https://github.com/baidu/highflip.git + HEAD + + @@ -119,6 +148,55 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadoc + + jar + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-source + + jar + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + prepare-agent + + + + + report + test + + report + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven.jar.plugin.version} + @@ -129,23 +207,114 @@ org.jacoco jacoco-maven-plugin - 0.8.2 - - - - prepare-agent - - - - - report - test - - report - - - + + + org.apache.maven.plugins + maven-jar-plugin + + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + + + + + local + + true + + + + + release + + + + maven-source-plugin + + + create-source-jar + + jar + test-jar + + + + + + maven-jar-plugin + + + + test-jar + + + + true + + + + + + maven-javadoc-plugin + + + create-javadoc-jar + + javadoc + jar + + package + + + + ${java.version} + + + + org.apache.maven.plugins + maven-gpg-plugin + ${maven.gpg.plugin.version} + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + ${nexus.staging.maven.plugin.version} + true + + ossrh + https://oss.sonatype.org + true + + + + + + \ No newline at end of file