Add pytest-timeout to requirements-dev.txt and fixes a deprecated war…

…nings when running tests (alibaba#2145)

* Add pytest-timeout to requirements-dev.txt
* Filter out the deprecated warnings in compat.py

Signed-off-by: Tao He <sighingnow@gmail.com>

Rename pegasus.hosts to network.servers for executor (alibaba#2147)

refactor GraphScopeAnnotationProcessor

able to generate

add file as entrance

rename

dev

codegen ok, shell script not finished

analytical_engine/java/grape-jdk/pom.xml

@@ -44,6 +44,10 @@
       <groupId>com.alibaba.fastffi</groupId>
       <artifactId>ffi</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.alibaba.graphscope</groupId>
+      <artifactId>grape-annotation</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.alibaba.fastffi</groupId>
       <artifactId>annotation-processor</artifactId>

analytical_engine/java/grape-runtime/pom.xml

@@ -64,14 +64,18 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
-    <dependency>
+      <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-api</artifactId>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-core</artifactId>
+      </dependency>
+      <dependency>
       <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-api</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-core</artifactId>
-    </dependency>
+        <artifactId>log4j-slf4j-impl</artifactId>
+      </dependency>
   </dependencies>
   <build>
     <plugins>

analytical_engine/java/grape-runtime/src/main/java/com/alibaba/graphscope/annotation/ArrowProjectedGenerator.java

@@ -1,5 +1,6 @@
 package com.alibaba.graphscope.annotation;
 
+import static com.alibaba.graphscope.annotation.Utils.addCXXTemplate;
 import static com.alibaba.graphscope.annotation.Utils.addColumn;
 import static com.alibaba.graphscope.annotation.Utils.addSharedPtr;
 import static com.alibaba.graphscope.annotation.Utils.getMessageTypes;
@@ -10,8 +11,13 @@
 import com.alibaba.fastffi.FFIFunGen;
 import com.alibaba.fastffi.FFIGen;
 import com.alibaba.graphscope.context.ffi.FFIVertexDataContext;
+import com.alibaba.graphscope.ds.GSVertexArray;
 import com.alibaba.graphscope.ds.ProjectedAdjList;
 import com.alibaba.graphscope.ds.ProjectedNbr;
+import com.alibaba.graphscope.ds.PropertyNbrUnit;
+import com.alibaba.graphscope.ds.TypedArray;
+import com.alibaba.graphscope.ds.VertexArray;
+import com.alibaba.graphscope.ds.VertexRange;
 import com.alibaba.graphscope.fragment.ArrowProjectedFragment;
 import com.alibaba.graphscope.parallel.MessageInBuffer;
 import com.alibaba.graphscope.parallel.ParallelMessageManager;
@@ -27,15 +33,34 @@ public class ArrowProjectedGenerator {
 
     private static Logger logger =
         LoggerFactory.getLogger(ArrowProjectedGenerator.class.getName());
+    private static String LONG_MSG_FULL_NAME = "com.alibaba.graphscope.parallel.message.LongMsg";
+    private static String DOUBLE_MSG_FULL_NAME = "com.alibaba.graphscope.parallel.message.DoubleMsg";
+    private static String EMPTY_TYPE = "com.alibaba.graphscope.ds.EmptyType";
+    private static String UNUSED_CLASS_NAME = "com.alibaba.graphscope.runtime.UnusedImpl";
     private AnnotationSpec.Builder ffiGenBatchBuilder;
     private String cppOid, cppVid, cppVdata, cppEdata;
     private String javaOid, javaVid, javaVdata, javaEdata;
+    private String simpleJavaOid, simpleJavaVid, simpleJavaVdata, simpleJavaEdata;
     private String cppFragName, javaFragName;
     private String[][] messageTypePairs;
 
+
     public ArrowProjectedGenerator(AnnotationSpec.Builder ffiGenBatchBuilder,
         String cppOid, String cppVid, String cppVdata, String cppEdata,
         String javaOid, String javaVid, String javaVdata, String javaEdata) {
+        this.ffiGenBatchBuilder = ffiGenBatchBuilder;
+        this.cppOid = cppOid;
+        this.cppVid = cppVid;
+        this.cppVdata = cppVdata;
+        this.cppEdata = cppEdata;
+        this.javaOid = javaOid;
+        this.javaVid = javaVid;
+        this.javaVdata = javaVdata;
+        this.javaEdata = javaEdata;
+        this.simpleJavaOid = typeNameToSimple(javaOid);
+        this.simpleJavaVid = typeNameToSimple(javaVid);
+        this.simpleJavaVdata = typeNameToSimple(javaVdata);
+        this.simpleJavaEdata = typeNameToSimple(javaEdata);
         cppFragName = String.format("%s<%s,%s,%s,%s>", ARROW_PROJECTED_FRAGMENT, cppOid, cppVid,
             cppVdata, cppEdata);
         javaFragName = String.format("%s<%s,%s,%s,%s>", ArrowProjectedFragment.class.getName(),
@@ -60,6 +85,13 @@ public ArrowProjectedGenerator(AnnotationSpec.Builder ffiGenBatchBuilder,
     }
 
     public void generate() {
+        addTemplate(TypedArray.class.getName(), new String[]{cppEdata}, new String[]{javaEdata});
+        addTemplate(TypedArray.class.getName(), new String[]{cppVdata}, new String[]{javaVdata});
+        addTemplate(PropertyNbrUnit.class.getName(), new String[]{cppVid}, new String[]{javaVid});
+        addTemplate(ProjectedNbr.class.getName(), cppVid, cppEdata, javaVid, javaEdata);
+        addTemplate(ProjectedAdjList.class.getName(), cppVid, cppEdata, javaVid, javaEdata);
+        addTemplate(VertexRange.class.getName(), new String[]{cppVid}, new String[]{javaVid});
+        addVertexArray();
         //only add if edata is not on of primitives ones.
         addProjectedNbr();
         //only add if edata is not on of primitives ones.
@@ -87,6 +119,17 @@ public void generate() {
             javaFragName);
     }
 
+    private void addVertexArray() {
+        AnnotationSpec.Builder ffiGen1 = AnnotationSpec.builder(FFIGen.class);
+        ffiGen1.addMember("type", "$S", VertexArray.class.getName());
+        addCXXTemplate(ffiGen1, "uint64_t", "double", "Long", "Double");
+        addCXXTemplate(ffiGen1, "uint64_t", "int64_t", "Long", "Long");
+        addCXXTemplate(ffiGen1, "uint64_t", "int32_t", "Long", "Integer");
+        ffiGenBatchBuilder.addMember("value", "$L", ffiGen1.build());
+
+        addTemplate(GSVertexArray.class.getName(), new String[]{"int32_t", "int64_t", "double"}, new String[]{"Integer", "Long", "Double"});
+    }
+
     private void addVertexDataContext(
         AnnotationSpec.Builder ffiGenBatchBuilder,
         String foreignFragName,
@@ -106,7 +149,7 @@ private void addProjectedAdjList() {
     }
 
     private void addArrowProjectedFragment() {
-        addTemplate(ArrowProjectedFragment.class.getName(), cppOid, cppVid, cppVid, cppEdata,
+        addTemplate(ArrowProjectedFragment.class.getName(), cppOid, cppVid, cppVdata, cppEdata,
             javaOid, javaVid, javaVdata, javaEdata);
     }
 
@@ -125,7 +168,8 @@ private void addMessageInBuffer() {
             AnnotationSpec.Builder methodBuilder =
                 AnnotationSpec.builder(FFIFunGen.class)
                     .addMember("name", "$S", "getPureMessage")
-                    .addMember("returnType", "$S", "boolean");
+                    .addMember("returnType", "$S", "boolean")
+                    .addMember("parameterTypes", "$S", "MSG_T");
             AnnotationSpec.Builder templateBuilder =
                 AnnotationSpec.builder(CXXTemplate.class)
                     .addMember("cxx", "$S", "std::vector<char>")
@@ -161,6 +205,28 @@ private void addTemplate(String typeName, String... templates) {
         ffiGenBatchBuilder.addMember("value", "$L", curBuilder.build());
     }
 
+    /** add template with multiple cxx template*/
+    private void addTemplate(String typeName, String[] cxxs, String javas[]) {
+        if (cxxs.length != javas.length) {
+            throw new IllegalStateException(
+                "Java cxx template length not equal");
+        }
+        int num = cxxs.length;
+        AnnotationSpec.Builder curBuilder = AnnotationSpec.builder(FFIGen.class);
+        curBuilder.addMember("type", "$S", typeName);
+        //build cxx template.
+        for (int i = 0; i < num; ++i) {
+            AnnotationSpec.Builder cxxTemplateBuilder = AnnotationSpec.builder(CXXTemplate.class);
+            cxxTemplateBuilder.addMember("cxx", "$S", cxxs[i]);
+            cxxTemplateBuilder.addMember("java", "$S", javas[i]);
+            curBuilder.addMember(
+                "templates",
+                "$L", cxxTemplateBuilder.build());
+        }
+
+        ffiGenBatchBuilder.addMember("value", "$L", curBuilder.build());
+    }
+
     private void parallelMessageManagerAddMessages(
         AnnotationSpec.Builder parallelMessageBuilder,
         String cppFragName,
@@ -243,10 +309,31 @@ public void addFuncGenMethodNoMsg(AnnotationSpec.Builder parallelMessageBuilder,
     }
 
     public String getUnusedTypeName(String msgType) {
-        return javaVdata + javaEdata + msgType;
+        return UNUSED_CLASS_NAME + "." + simpleJavaVdata + simpleJavaEdata + typeNameToSimple(msgType);
     }
 
     public String getUnusedTypeName() {
-        return javaVdata + javaEdata;
+        return UNUSED_CLASS_NAME + "." + simpleJavaVdata + simpleJavaEdata;
+    }
+
+    private static String typeNameToSimple(String fullName){
+        if (fullName.equals("java.lang.Long") || fullName.equals(LONG_MSG_FULL_NAME)){
+            return "Long";
+        }
+        else if (fullName.equals("java.lang.Double") || fullName.equals(DOUBLE_MSG_FULL_NAME)){
+            return "Double";
+        }
+        else if (fullName.equals("java.lang.Integer")){
+            return "Integer";
+        }
+        else if (fullName.equals("java.lang.String")){
+            return "String";
+        }
+        else if (fullName.equals("Empty") || fullName.equals(EMPTY_TYPE)){
+            return "Empty";
+        }
+        else {
+            throw new IllegalStateException("Unrecognized full name " + fullName);
+        }
     }
 }

analytical_engine/java/grape-runtime/src/main/java/com/alibaba/graphscope/annotation/GraphScopeAppScanner.java

@@ -67,6 +67,7 @@
 public class GraphScopeAppScanner {
 
     private static Logger logger = LoggerFactory.getLogger(GraphScopeAppScanner.class.getName());
+    private boolean javaFragName;
     private String classpath;
     private String configPath;
     private String outputDirectory;
@@ -78,7 +79,7 @@ public class GraphScopeAppScanner {
     private Map<String, String> ffiMirrors = new HashMap<>();
 
     private GraphScopeAppScanner(
-            String classpath, String configPath, String outputDirectory, String graphTemplateStr) {
+            String classpath, String configPath, String outputDirectory, String graphTemplateStr, boolean javaFragName) {
         this.classpath = classpath;
         this.configPath = configPath;
         this.outputDirectory = outputDirectory;
@@ -98,6 +99,11 @@ private GraphScopeAppScanner(
                                 .toArray(URL[]::new),
                         ClassLoader.getSystemClassLoader());
         this.graphTemplateStr = graphTemplateStr;
+        if (this.graphTemplateStr.startsWith("\"") && this.graphTemplateStr.endsWith("\"")){
+            this.graphTemplateStr = this.graphTemplateStr.substring(1, this.graphTemplateStr.length() - 1);
+        }
+        logger.info("Graph template string {}", graphTemplateStr);
+        this.javaFragName = javaFragName;
     }
 
     /**
@@ -111,8 +117,8 @@ private GraphScopeAppScanner(
      * @return absolute output path
      */
     public static String scanAppAndGenerate(
-            String classpath, String outputDirectory, String graphTemplateString) {
-        return new GraphScopeAppScanner(classpath, "empty", outputDirectory, graphTemplateString)
+            String classpath, String outputDirectory, String graphTemplateString, boolean javaFragName) {
+        return new GraphScopeAppScanner(classpath, "empty", outputDirectory, graphTemplateString, javaFragName)
                 .scanAppAndGenerateImpl();
     }
 
@@ -140,8 +146,8 @@ private String scanAppAndGenerateImpl() {
         }
         // add support for doubleMsg and LongMsg
         //
-        // sb.append("gs::DoubleMsg=com.alibaba.graphscopescope.parallel.message.DoubleMsg,");
-        //        sb.append("gs::LongMsg=com.alibaba.graphscopescope.parallel.message.LongMsg,");
+        sb.append("gs::DoubleMsg=com.alibaba.graphscope.parallel.message.DoubleMsg,");
+        sb.append("gs::LongMsg=com.alibaba.graphscope.parallel.message.LongMsg,");
         String temp = sb.toString();
         String messageTypes = "";
         if (!temp.isEmpty()) {
@@ -156,18 +162,35 @@ private String scanAppAndGenerateImpl() {
             return this.outputDirectory;
         }
         if (Objects.nonNull(parsed) && parsed.length == 5) {
-            graphConfig =
-                    new GraphConfig(
-                            cpp2Java(parsed[1]),
-                            cpp2Java(parsed[2]),
-                            cpp2Java(parsed[3]),
-                            cpp2Java(parsed[4]),
-                            messageTypes,
-                            parsed[0],
-                            parsed[1],
-                            parsed[2],
-                            parsed[3],
-                            parsed[4]);
+            if (javaFragName){
+                graphConfig =
+                new GraphConfig(
+                        parsed[1],
+                        parsed[2],
+                        parsed[3],
+                        parsed[4],
+                        messageTypes,
+                        parsed[0],
+                        java2Cpp(parsed[1], true),
+                        java2Cpp(parsed[2], false),
+                        java2Cpp(parsed[3], true),
+                        java2Cpp(parsed[4], true));
+            }
+            else {
+                graphConfig =
+                new GraphConfig(
+                        cpp2Java(parsed[1]),
+                        cpp2Java(parsed[2]),
+                        cpp2Java(parsed[3]),
+                        cpp2Java(parsed[4]),
+                        messageTypes,
+                        parsed[0],
+                        parsed[1],
+                        parsed[2],
+                        parsed[3],
+                        parsed[4]);
+            }
+
         } else {
             graphConfig =
                     new GraphConfig(
@@ -197,20 +220,38 @@ private String cpp2Java(String cppType) {
         return null;
     }
 
+    private String java2Cpp(String javaType, boolean signed) {
+        if (javaType.equals("java.lang.Long") || javaType.equals("Long")) {
+            if (signed){
+                return "int64_t";
+            }
+            else return "uint64_t";
+        } else if (javaType.equals("java.lang.Integer") || javaType.equals("Integer")) {
+            if (signed){
+                return "int32_t";
+            }
+            else return "uint32_t";
+        } else if (javaType.equals("java.lang.Double") || javaType.equals("Double")) {
+            return "double";
+        }
+        throw new IllegalStateException("Unrecognized type " + javaType + " sign: " + signed);
+    }
+
     // gs::ArrowProjectedFragment
+    //can also be ArrowProjectedFragment<java.lang.Long,....>
     private String[] parseGraphTemplateStr(String graphTemplateStr) {
         String[] first = graphTemplateStr.split("<");
         if (first.length != 2) {
             logger.error("parse fragment type error" + graphTemplateStr);
             return first;
-        } else if (first[0].equals("gs::ArrowProjectedFragment")) {
+        } else if (first[0].equals("gs::ArrowProjectedFragment") || first[0].equals("ArrowProjectedFragment")) {
             String typeParams = first[1].substring(0, first[1].length() - 1);
             String[] second = typeParams.split(",");
             if (second.length != 4) {
                 logger.error("Inproper num of type params" + typeParams);
             }
             return new String[] {
-                first[0].trim(),
+                "gs::ArrowProjectedFragment",
                 second[0].trim(),
                 second[1].trim(),
                 second[2].trim(),

analytical_engine/java/grape-runtime/src/main/java/com/alibaba/graphscope/annotation/Main.java

@@ -29,12 +29,17 @@ public class Main {
     private static Logger logger = LoggerFactory.getLogger(Main.class.getName());
 
     public static void main(String[] args) {
-        if (args.length != 3) {
-            logger.error("Expected 3 params.");
+        if (args.length < 3 || args.length > 4) {
+            logger.error("Expected 3 or 4 params.");
             return;
         }
-        logger.info(
-                "Files are generated in "
-                        + GraphScopeAppScanner.scanAppAndGenerate(args[0], args[1], args[2]));
+        String res;
+        if (args.length == 3){
+            res = GraphScopeAppScanner.scanAppAndGenerate(args[0], args[1], args[2], false);
+        }
+        else {
+            res = GraphScopeAppScanner.scanAppAndGenerate(args[0], args[1], args[2], Boolean.valueOf(args[3]));
+        }
+        logger.info( "Files are generated in " + res);
     }
 }