refactor(interactive): Parse Graph Schema from Yaml Configuration

flex/tests/hqps/hqps_cypher_test.sh

@@ -36,7 +36,7 @@ GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yam
 GRAPH_BULK_LOAD_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml
 COMPILER_GRAPH_SCHEMA=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json
 GRAPH_CSR_DATA_DIR=${HOME}/csr-data-dir/
-HQPS_IR_CONF=/tmp/hqps.ir.conf
+HQPS_IR_CONF=/tmp/hqps.ir.properties
 # check if GRAPH_SCHEMA_YAML exists
 if [ ! -f ${GRAPH_SCHEMA_YAML} ]; then
   echo "GRAPH_SCHEMA_YAML: ${GRAPH_SCHEMA_YAML} not found"

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/Configs.java

@@ -16,6 +16,8 @@
 
 package com.alibaba.graphscope.common.config;
 
+import com.alibaba.graphscope.common.utils.FileUtils;
+
 import org.apache.commons.lang3.NotImplementedException;
 import org.apache.commons.lang3.StringUtils;
 
@@ -81,10 +83,15 @@ public String toString() {
 
     public static class Factory {
         public static Configs create(String file) throws Exception {
-            if (file.endsWith(".yaml")) {
-                return new YamlConfigs(file);
-            } else {
-                return new Configs(file);
+            switch (FileUtils.getFormatType(file)) {
+                case YAML:
+                    return new YamlConfigs(file);
+                case PROPERTIES:
+                    return new Configs(file);
+                case JSON:
+                default:
+                    throw new UnsupportedOperationException(
+                            "can not initiate Configs from the file " + file);
             }
         }
     }

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/GraphStoredProcedures.java

@@ -32,6 +32,7 @@ public GraphStoredProcedures(MetaDataReader reader) throws Exception {
         for (InputStream inputStream : reader.getStoredProcedures()) {
             StoredProcedureMeta createdMeta = StoredProcedureMeta.Deserializer.perform(inputStream);
             this.storedProcedureMetaMap.put(createdMeta.getName(), createdMeta);
+            inputStream.close();
         }
     }
 

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/FileFormatType.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.graphscope.common.ir.meta.reader;
+
+public enum FileFormatType {
+    YAML,
+    JSON,
+    PROPERTIES
+}

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java

@@ -19,19 +19,20 @@
 import com.alibaba.graphscope.common.config.Configs;
 import com.alibaba.graphscope.common.config.GraphConfig;
 import com.alibaba.graphscope.common.config.Utils;
+import com.alibaba.graphscope.common.utils.FileUtils;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
-import java.nio.file.Paths;
+import java.io.*;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 // a local file system implementation of MetaDataReader
@@ -44,7 +45,7 @@ public LocalMetaDataReader(Configs configs) {
     }
 
     @Override
-    public List<InputStream> getStoredProcedures() throws FileNotFoundException {
+    public List<InputStream> getStoredProcedures() throws IOException {
         String procedurePath = GraphConfig.GRAPH_STORED_PROCEDURES.get(configs);
         File procedureDir = new File(procedurePath);
         if (!procedureDir.exists() || !procedureDir.isDirectory()) {
@@ -60,27 +61,49 @@ public List<InputStream> getStoredProcedures() throws FileNotFoundException {
                 procedureInputs.add(new FileInputStream(file));
             }
         } else {
+            Map<String, InputStream> procedureInputMap =
+                    getProcedureNameWithInputStream(procedureDir);
             for (String enableProcedure : enableProcedureList) {
-                File procedureFile =
-                        new File(Paths.get(procedurePath, enableProcedure + ".yaml").toString());
-                if (!procedureFile.exists()) {
-                    logger.warn(
-                            "procedure {} not exist in directory {}",
-                            procedureFile.getName(),
-                            procedurePath);
-                } else {
-                    procedureInputs.add(new FileInputStream(procedureFile));
-                }
+                InputStream enableInput = procedureInputMap.get(enableProcedure);
+                Preconditions.checkArgument(
+                        enableInput != null,
+                        "can not find procedure with name=%s under directory=%s, candidates are %s",
+                        enableProcedure,
+                        procedureDir,
+                        procedureInputMap.keySet());
+                procedureInputs.add(enableInput);
             }
         }
         return Collections.unmodifiableList(procedureInputs);
     }
 
+    private Map<String, InputStream> getProcedureNameWithInputStream(File procedureDir)
+            throws IOException {
+        Map<String, InputStream> procedureInputMap = Maps.newHashMap();
+        for (File file : procedureDir.listFiles()) {
+            String procedureName = getProcedureName(file);
+            procedureInputMap.put(procedureName, new FileInputStream(file));
+        }
+        return procedureInputMap;
+    }
+
+    private String getProcedureName(File file) throws IOException {
+        try (InputStream inputStream = new FileInputStream(file)) {
+            Yaml yaml = new Yaml();
+            Map<String, Object> map = yaml.load(inputStream);
+            Object procedureName = map.get("name");
+            Preconditions.checkArgument(
+                    procedureName != null, "procedure name not exist in %s", file.getName());
+            return procedureName.toString();
+        }
+    }
+
     @Override
-    public InputStream getGraphSchema() throws FileNotFoundException {
+    public SchemaInputStream getGraphSchema() throws IOException {
         String schemaPath =
                 Objects.requireNonNull(
                         GraphConfig.GRAPH_SCHEMA.get(configs), "schema path not exist");
-        return new FileInputStream(schemaPath);
+        return new SchemaInputStream(
+                new FileInputStream(schemaPath), FileUtils.getFormatType(schemaPath));
     }
 }

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/MetaDataReader.java

@@ -26,5 +26,5 @@ public interface MetaDataReader {
     // if enableProcedures is null, return all stored procedures
     List<InputStream> getStoredProcedures() throws Exception;
 
-    InputStream getGraphSchema() throws Exception;
+    SchemaInputStream getGraphSchema() throws Exception;
 }

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/StatisticSchema.java → interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/SchemaInputStream.java

@@ -14,24 +14,24 @@
  * limitations under the License.
  */
 
-package com.alibaba.graphscope.common.ir.meta.schema;
+package com.alibaba.graphscope.common.ir.meta.reader;
 
-import com.alibaba.graphscope.groot.common.schema.api.GraphSchema;
+import java.io.InputStream;
 
-import org.apache.calcite.schema.Statistic;
+public class SchemaInputStream {
+    private final InputStream inputStream;
+    private final FileFormatType formatType;
 
-import java.util.List;
+    public SchemaInputStream(InputStream inputStream, FileFormatType formatType) {
+        this.inputStream = inputStream;
+        this.formatType = formatType;
+    }
 
-/**
- * Extends {@link GraphSchema} to add {@link Statistic}
- */
-public interface StatisticSchema extends GraphSchema {
-    // get meta for CBO
-    Statistic getStatistic(List<String> tableName);
-
-    // if the property name need to be converted to id
-    boolean isColumnId();
+    public InputStream getInputStream() {
+        return inputStream;
+    }
 
-    // schema json for ir core
-    String schemaJson();
+    public FileFormatType getFormatType() {
+        return formatType;
+    }
 }

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptSchema.java

@@ -24,7 +24,6 @@
 import org.apache.calcite.plan.RelOptSchema;
 import org.apache.calcite.plan.RelOptTable;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.Statistic;
 import org.apache.calcite.util.Static;
 import org.apache.commons.lang3.ObjectUtils;
 import org.checkerframework.checker.nullness.qual.Nullable;
@@ -37,9 +36,9 @@
  */
 public class GraphOptSchema implements RelOptSchema {
     private RelOptCluster optCluster;
-    private StatisticSchema rootSchema;
+    private IrGraphSchema rootSchema;
 
-    public GraphOptSchema(@Nullable RelOptCluster optCluster, StatisticSchema rootSchema) {
+    public GraphOptSchema(@Nullable RelOptCluster optCluster, IrGraphSchema rootSchema) {
         this.optCluster = optCluster;
         this.rootSchema = Objects.requireNonNull(rootSchema);
     }
@@ -57,15 +56,14 @@ public RelOptTable getTableForMember(List<String> tableName) {
         String labelName = tableName.get(0);
         try {
             GraphElement element = rootSchema.getElement(labelName);
-            return createRelOptTable(tableName, element, rootSchema.getStatistic(tableName));
+            return createRelOptTable(tableName, element);
         } catch (GraphElementNotFoundException e) {
             throw Static.RESOURCE.tableNotFound(labelName).ex();
         }
     }
 
-    private RelOptTable createRelOptTable(
-            List<String> tableName, GraphElement element, Statistic statistic) {
-        return new GraphOptTable(this, tableName, element, statistic);
+    private RelOptTable createRelOptTable(List<String> tableName, GraphElement element) {
+        return new GraphOptTable(this, tableName, element);
     }
 
     /**
@@ -78,7 +76,7 @@ public RelDataTypeFactory getTypeFactory() {
         return this.optCluster.getTypeFactory();
     }
 
-    public StatisticSchema getRootSchema() {
+    public IrGraphSchema getRootSchema() {
         return this.rootSchema;
     }
 

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptTable.java

@@ -52,16 +52,10 @@ public class GraphOptTable implements RelOptTable {
     private List<String> tableName;
     private RelOptSchema schema;
     private RelDataType dataType;
-    private Statistic statistic;
 
-    protected GraphOptTable(
-            RelOptSchema schema,
-            List<String> tableName,
-            GraphElement element,
-            Statistic statistic) {
+    protected GraphOptTable(RelOptSchema schema, List<String> tableName, GraphElement element) {
         this.schema = schema;
         this.tableName = tableName;
-        this.statistic = statistic;
         this.dataType = deriveType(element);
     }
 
@@ -155,12 +149,12 @@ public RelDataType getRowType() {
 
     @Override
     public boolean isKey(ImmutableBitSet immutableBitSet) {
-        return this.statistic.isKey(immutableBitSet);
+        throw new UnsupportedOperationException("is key is unsupported yet in statistics");
     }
 
     @Override
     public @Nullable List<ImmutableBitSet> getKeys() {
-        return this.statistic.getKeys();
+        throw new UnsupportedOperationException("get keys is unsupported yet in statistics");
     }
 
     @Override