2.0 to master

.github/workflows/deploy.yml

@@ -0,0 +1,59 @@
+# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Deploy Java jar with maven
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v3
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          server-id: ossrh
+          server-username: MAVEN_USERNAME
+          server-password: MAVEN_PASSWORD
+          gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }}
+          gpg-passphrase: MAVEN_GPG_PASSPHRASE
+
+      - name: Build with Maven
+        run: mvn -B package -DskipTests -Prelease --file pom.xml
+
+      - name: Set up Apache Maven Central
+        uses: actions/setup-java@v3
+        with: # running setup-java again overwrites the settings.xml
+          java-version: '11'
+          distribution: 'temurin'
+          server-id: ossrh # Value of the distributionManagement/repository/id field of the pom.xml
+          server-username: MAVEN_USERNAME # env variable for username in deploy
+          server-password: MAVEN_PASSWORD # env variable for token in deploy
+          gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} # Value of the GPG private key to import
+          gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase
+
+      - name: Publish to the Maven Central Repository
+        run:
+          mvn clean --no-transfer-progress --batch-mode -Prelease -DskipTests deploy --file pom.xml -pl highflip-proto,highflip-core,highflip-clients/highflip-sdk -am
+        env:
+          MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }}
+          MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }}
+          MAVEN_GPG_PASSPHRASE: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY_PASSWORD }}
+
+    # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive
+    # - name: Update dependency graph
+    #  uses: advanced-security/maven-dependency-submission-action@v2.0.1

.github/workflows/maven.yml

@@ -6,30 +6,27 @@
 # separate terms of service, privacy policy, and support
 # documentation.
 
-name: Java CI with Maven
+name: Build Java Project
 
 on:
   push:
-    branches: [ "master" ]
   pull_request:
-    branches: [ "master" ]
+  workflow_dispatch:
 
 jobs:
   build:
 
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up JDK 11
-      uses: actions/setup-java@v3
-      with:
-        java-version: '11'
-        distribution: 'temurin'
-        cache: maven
-    - name: Build with Maven
-      run: mvn -B package --file pom.xml
-
-    # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive
-    # - name: Update dependency graph
-    #  uses: advanced-security/maven-dependency-submission-action@v2.0.1
+      - name: Checkout project
+        uses: actions/checkout@v3
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+
+      - name: Build with Maven
+        run: mvn -B package -DskipTests -Prelease --file pom.xml

highflip-build/pom.xml

@@ -2,7 +2,7 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 
     <modelVersion>4.0.0</modelVersion>
-    <groupId>com.baidu</groupId>
+    <groupId>com.baidu.highflip</groupId>
     <artifactId>highflip-build</artifactId>
     <packaging>jar</packaging>
     <version>1.0.0-SNAPSHOT</version>

highflip-clients/highflip-console/pom.xml

@@ -11,16 +11,16 @@
         <relativePath/>
     </parent>
     -->
-    <groupId>com.baidu</groupId>
+    <groupId>com.baidu.highflip</groupId>
     <artifactId>highflip-console</artifactId>
     <version>1.0.0-SNAPSHOT</version>
     <name>highflip-console</name>
 
     <parent>
-        <groupId>com.baidu</groupId>
-        <artifactId>highflip</artifactId>
+        <groupId>com.baidu.highflip</groupId>
+        <artifactId>highflip-clients</artifactId>
         <version>1.0.0-SNAPSHOT</version>
-        <relativePath>../../pom.xml</relativePath>
+        <relativePath>../pom.xml</relativePath>
     </parent>
 
     <properties>
@@ -44,7 +44,7 @@
             <artifactId>spring-boot-starter-json</artifactId>
         </dependency>
         <dependency>
-            <groupId>com.baidu</groupId>
+            <groupId>com.baidu.highflip</groupId>
             <artifactId>highflip-sdk</artifactId>
             <version>1.0.0-SNAPSHOT</version>
         </dependency>

highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java

@@ -17,12 +17,13 @@ public class TaskCommand {
     @Autowired
     HighFlipClient client;
 
-    @ShellMethod(key = "task list", value = "List all task ids")
+    @ShellMethod(key = "task list", value = "List job all task ids")
     public Iterable<String> list(
+            @ShellOption String jobId,
             @ShellOption(defaultValue = "0") Integer offset,
             @ShellOption(defaultValue = "0") Integer limit) {
 
-        return client.listTasks(offset, limit);
+        return client.listTasks(jobId, offset, limit);
     }
 
     @ShellMethod(key = "task get", value = "Get a task information")

highflip-clients/highflip-sdk/pom.xml

@@ -2,25 +2,25 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 
     <modelVersion>4.0.0</modelVersion>
-    <groupId>com.baidu</groupId>
+    <groupId>com.baidu.highflip</groupId>
     <artifactId>highflip-sdk</artifactId>
     <packaging>jar</packaging>
     <version>1.0.0-SNAPSHOT</version>
     <name>highflip-sdk</name>
     <url>https://maven.apache.org</url>
 
     <parent>
-        <groupId>com.baidu</groupId>
-        <artifactId>highflip</artifactId>
+        <groupId>com.baidu.highflip</groupId>
+        <artifactId>highflip-clients</artifactId>
         <version>1.0.0-SNAPSHOT</version>
-        <relativePath>../../pom.xml</relativePath>
+        <relativePath>../pom.xml</relativePath>
     </parent>
 
     <dependencies>
         <dependency>
-            <groupId>com.baidu</groupId>
+            <groupId>com.baidu.highflip</groupId>
             <artifactId>highflip-proto</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
+            <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>org.projectlombok</groupId>

highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java

@@ -42,6 +42,12 @@ public HighFlipClient() {
         this.blockingStub = null;
     }
 
+    public static HighFlipClient newHighFlipClient(String target) {
+        HighFlipClient highFlipClient = new HighFlipClient();
+        highFlipClient.connect(target);
+        return highFlipClient;
+    }
+
     public void connect(String target) {
         close();
 
@@ -283,13 +289,17 @@ public Iterable<String> getJobLog(String jobId){
 
     /**
      *
+     * @param jobId  job id in highflip
      * @param offset
      * @param limit
      * @return
      */
-    public Iterable<String> listTasks(int offset, int limit){
+    public Iterable<String> listTasks(String jobId, int offset, int limit) {
         Highflip.TaskListRequest request = Highflip.TaskListRequest
                 .newBuilder()
+                .setJobId(jobId)
+                .setOffset(offset)
+                .setLimit(limit)
                 .build();
 
         Iterator<Highflip.TaskListResponse> response = getBlockingStub()

highflip-clients/pom.xml

@@ -4,13 +4,13 @@
 
     <parent>
         <artifactId>highflip</artifactId>
-        <groupId>com.baidu</groupId>
+        <groupId>com.baidu.highflip</groupId>
         <version>1.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
 
-    <groupId>com.baidu</groupId>
+    <groupId>com.baidu.highflip</groupId>
     <artifactId>highflip-clients</artifactId>
     <version>1.0.0-SNAPSHOT</version>
     <packaging>pom</packaging>

highflip-core/pom.xml

@@ -2,7 +2,7 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 
     <modelVersion>4.0.0</modelVersion>
-    <groupId>com.baidu</groupId>
+    <groupId>com.baidu.highflip</groupId>
     <artifactId>highflip-core</artifactId>
     <packaging>jar</packaging>
     <version>1.0.0-SNAPSHOT</version>
@@ -11,15 +11,15 @@
     <url>http://maven.apache.org</url>
 
     <parent>
-        <groupId>com.baidu</groupId>
+        <groupId>com.baidu.highflip</groupId>
         <artifactId>highflip</artifactId>
         <version>1.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
     <dependencies>
         <dependency>
-            <groupId>com.baidu</groupId>
+            <groupId>com.baidu.highflip</groupId>
             <artifactId>highflip-proto</artifactId>
             <version>1.0.0-SNAPSHOT</version>
         </dependency>

highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java

@@ -0,0 +1,10 @@
+package com.baidu.highflip.core.adaptor;
+
+public interface ServiceAdaptor {
+
+    String getUrl();
+
+    String getPartyId();
+
+    String getRole();
+}

highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java

@@ -25,4 +25,16 @@ public class AdaptorPropsList {
 
     public static final String PROPS_HIGHFLIP_ADAPTOR_PLATFORM_VERSION_DEFAULT = "0.0.0";
 
+    public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL = "highflip.adaptor.service.url";
+
+    public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL_DEFAULT = "http://127.0.0.1:9380";
+
+    public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID = "highflip.adaptor.service.party.id";
+
+    public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID_DEFAULT = "9999";
+
+    public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE = "highflip.adaptor.service.role";
+
+    public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE_DEFAULT = "guest";
+
 }

highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java

@@ -25,4 +25,7 @@ public class InstanceNameList {
     public static final String HIGHFLIP_ADAPTOR_PARTNER = "highflip.adaptor.partner";
 
     public static final String HIGHFLIP_ADAPTOR_USER = "highflip.adaptor.user";
+
+    public static final String HIGHFLIP_ADAPTOR_SERVICE = "highflip.adaptor.service";
+
 }

highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java

@@ -6,6 +6,7 @@
 import com.baidu.highflip.core.entity.runtime.Partner;
 import com.baidu.highflip.core.entity.runtime.Task;
 import com.baidu.highflip.core.entity.runtime.User;
+import com.baidu.highflip.core.entity.runtime.basic.Status;
 
 public interface HighFlipRuntime {
 
@@ -26,4 +27,10 @@ public interface HighFlipRuntime {
     User getUser(String userId);
 
     Operator getOperator(String operatorId);
+
+    Data registerData(Data data);
+
+    Iterable<Task> listTask(String jobId);
+
+    void updateTask(Task task);
 }

highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java

@@ -3,6 +3,8 @@
 import com.baidu.highflip.core.entity.dag.codec.AttributeMap;
 import com.baidu.highflip.core.entity.dag.common.NamedAttributeObject;
 import com.baidu.highflip.core.utils.ProtoUtils;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
 import highflip.HighflipMeta;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -96,7 +98,7 @@ public Iterable<String> listParties() {
         return getParties().keySet();
     }
 
-
+    @JsonIgnore
     protected void setNodeCategory() {
         calcMiddleNodes();
         calcOutputNodes();

highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java

@@ -6,16 +6,19 @@
 import com.baidu.highflip.core.entity.dag.common.NodeInputRef;
 import com.baidu.highflip.core.entity.dag.common.NodeOutputRef;
 import com.baidu.highflip.core.utils.ProtoUtils;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
 import highflip.HighflipMeta;
 import lombok.Data;
 
-import javax.persistence.Transient;
 import java.io.Serializable;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+import org.springframework.data.annotation.Transient;
+
 @Data
 public class Node extends NamedAttributeObject implements Serializable {
 
@@ -25,7 +28,7 @@ public class Node extends NamedAttributeObject implements Serializable {
 
     String description;
 
-    @Transient
+    @JsonIgnore
     Graph graph;
 
     Category category;
@@ -108,6 +111,7 @@ public Node getInputNode(String name) {
                 getInputs().get(name).getFromNode());
     }
 
+    @JsonIgnore
     public List<Node> getInputNodes() {
         return getInputs()
                 .values()

highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java

@@ -3,6 +3,7 @@
 import com.baidu.highflip.core.entity.dag.codec.AttributeMap;
 import com.baidu.highflip.core.entity.dag.common.NamedAttributeObject;
 import com.baidu.highflip.core.utils.ProtoUtils;
+
 import highflip.HighflipMeta;
 import lombok.Data;