Skip to content

Commit

Permalink
fix(interactive): Introduce API to Get Meta Data From Remote Service (#…
Browse files Browse the repository at this point in the history
…3879)

<!--
Thanks for your contribution! please review
https://github.com/alibaba/GraphScope/blob/main/CONTRIBUTING.md before
opening an issue.
-->

## What do these changes do?
as titled.

<!-- Please give a short brief about these changes. -->

## Related issue number

<!-- Are there any issues opened that will be resolved by merging this
change? -->

Fixes

---------

Co-authored-by: xiaolei.zl <xiaolei.zl@alibaba-inc.com>
Co-authored-by: Longbin Lai <longbin.lailb@alibaba-inc.com>
  • Loading branch information
3 people committed Jun 6, 2024
1 parent fb308b7 commit 5b53359
Show file tree
Hide file tree
Showing 64 changed files with 918 additions and 698 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,14 @@
import com.alibaba.graphscope.common.config.Configs;
import com.alibaba.graphscope.common.config.FrontendConfig;
import com.alibaba.graphscope.common.config.GraphConfig;
import com.alibaba.graphscope.common.ir.meta.reader.LocalMetaDataReader;
import com.alibaba.graphscope.common.ir.meta.IrMeta;
import com.alibaba.graphscope.common.ir.meta.IrMetaTracker;
import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher;
import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher;
import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader;
import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer;
import com.alibaba.graphscope.common.ir.tools.*;
import com.alibaba.graphscope.common.manager.IrMetaQueryCallback;
import com.alibaba.graphscope.common.store.ExperimentalMetaFetcher;
import com.alibaba.graphscope.common.store.IrMeta;
import com.alibaba.graphscope.cypher.antlr4.parser.CypherAntlr4Parser;
import com.alibaba.graphscope.cypher.antlr4.visitor.LogicalPlanVisitor;
import com.alibaba.graphscope.cypher.service.CypherBootstrapper;
Expand Down Expand Up @@ -57,6 +60,7 @@ public class GraphServer {
private final ChannelFetcher channelFetcher;
private final IrMetaQueryCallback metaQueryCallback;
private final GraphProperties testGraph;
private final GraphRelOptimizer optimizer;

private IrGremlinServer gremlinServer;
private CypherBootstrapper cypherBootstrapper;
Expand All @@ -65,11 +69,13 @@ public GraphServer(
Configs configs,
ChannelFetcher channelFetcher,
IrMetaQueryCallback metaQueryCallback,
GraphProperties testGraph) {
GraphProperties testGraph,
GraphRelOptimizer optimizer) {
this.configs = configs;
this.channelFetcher = channelFetcher;
this.metaQueryCallback = metaQueryCallback;
this.testGraph = testGraph;
this.optimizer = optimizer;
}

public void start() throws Exception {
Expand All @@ -83,7 +89,8 @@ public void start() throws Exception {
new LogicalPlan(
new GraphBuilderVisitor(builder)
.visit(new GremlinAntlr4Parser().parse(query))
.build()));
.build()),
optimizer);
QueryCache queryCache = new QueryCache(configs, graphPlanner);
this.gremlinServer =
new IrGremlinServer(
Expand All @@ -102,7 +109,8 @@ public void start() throws Exception {
configs,
(GraphBuilder builder, IrMeta irMeta, String query) ->
new LogicalPlanVisitor(builder, irMeta)
.visit(new CypherAntlr4Parser().parse(query)));
.visit(new CypherAntlr4Parser().parse(query)),
optimizer);
QueryCache queryCache = new QueryCache(configs, graphPlanner);
this.cypherBootstrapper =
new CypherBootstrapper(
Expand Down Expand Up @@ -164,15 +172,24 @@ public static void main(String[] args) throws Exception {
throw new IllegalArgumentException("usage: GraphServer '<path_to_config_file>'");
}
Configs configs = Configs.Factory.create(args[0]);
GraphRelOptimizer optimizer = new GraphRelOptimizer(configs);
IrMetaQueryCallback queryCallback =
new IrMetaQueryCallback(
new ExperimentalMetaFetcher(new LocalMetaDataReader(configs)));
new IrMetaQueryCallback(createIrMetaFetcher(configs, optimizer.getGlogueHolder()));
GraphServer server =
new GraphServer(
configs, getChannelFetcher(configs), queryCallback, getTestGraph(configs));
configs,
getChannelFetcher(configs),
queryCallback,
getTestGraph(configs),
optimizer);
server.start();
}

private static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker tracker)
throws IOException {
return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker);
}

private static GraphProperties getTestGraph(Configs configs) {
GraphProperties testGraph;
switch (GraphConfig.GRAPH_STORE.get(configs)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@
import com.alibaba.graphscope.common.intermediate.process.SinkArg;
import com.alibaba.graphscope.common.intermediate.process.SinkByColumns;
import com.alibaba.graphscope.common.intermediate.process.SinkGraph;
import com.alibaba.graphscope.common.ir.meta.IrMeta;
import com.alibaba.graphscope.common.jna.IrCoreLibrary;
import com.alibaba.graphscope.common.jna.type.*;
import com.alibaba.graphscope.common.store.IrMeta;
import com.alibaba.graphscope.common.utils.ClassUtils;
import com.alibaba.graphscope.gaia.proto.Common;
import com.alibaba.graphscope.gaia.proto.GraphAlgebra;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,4 @@ public class GraphConfig {
public static final Config<String> GRAPH_STATISTICS =
Config.stringConfig("graph.statistics", "");
public static final Config<String> GRAPH_STORE = Config.stringConfig("graph.store", "exp");

@Deprecated
public static final Config<String> GRAPH_STORED_PROCEDURES =
Config.stringConfig("graph.stored.procedures", "");

@Deprecated
public static final Config<String> GRAPH_STORED_PROCEDURES_ENABLE_LISTS =
Config.stringConfig("graph.stored.procedures.enable.lists", "");

// denote stored procedures in yaml format, refer to test resource file
// 'config/modern/graph.yaml' for more info about
// the format
public static final Config<String> GRAPH_STORED_PROCEDURES_YAML =
Config.stringConfig("graph.stored.procedures.yaml", "");
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.apache.commons.lang3.StringUtils;
import org.yaml.snakeyaml.Yaml;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
Expand Down Expand Up @@ -75,26 +74,6 @@ public class YamlConfigs extends Configs {
return "cpp-mcsr";
}
})
.put(
"graph.stored.procedures.yaml",
(Configs configs) -> {
File schemaFile = new File(GraphConfig.GRAPH_SCHEMA.get(configs));
if (!schemaFile.exists() || !schemaFile.getName().endsWith(".yaml")) {
return null;
}
try {
Yaml yaml = new Yaml();
Map<String, Object> yamlAsMap =
yaml.load(new FileInputStream(schemaFile));
Object value;
if ((value = yamlAsMap.get("stored_procedures")) == null) {
return null;
}
return yaml.dump(value);
} catch (IOException e) {
throw new RuntimeException(e);
}
})
.put(
"pegasus.worker.num",
(Configs configs) -> {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
*
* * Copyright 2020 Alibaba Group Holding Limited.
* *
* * Licensed under the Apache License, Version 2.0 (the "License");
* * you may not use this file except in compliance with the License.
* * You may obtain a copy of the License at
* *
* * http://www.apache.org/licenses/LICENSE-2.0
* *
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS,
* * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* * See the License for the specific language governing permissions and
* * limitations under the License.
*
*/

package com.alibaba.graphscope.common.ir.meta;

import org.checkerframework.checker.nullness.qual.Nullable;

/**
* This class is used to uniquely identify a graph instance in scenarios that support multiple graphs.
* @param <T>
*/
public class GraphId<T> {
public static final GraphId DEFAULT = new GraphId(null);

private final @Nullable T id;

public GraphId(T id) {
this.id = id;
}

public @Nullable T getId() {
return id;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
*
* * Copyright 2020 Alibaba Group Holding Limited.
* *
* * Licensed under the Apache License, Version 2.0 (the "License");
* * you may not use this file except in compliance with the License.
* * You may obtain a copy of the License at
* *
* * http://www.apache.org/licenses/LICENSE-2.0
* *
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS,
* * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* * See the License for the specific language governing permissions and
* * limitations under the License.
*
*/

package com.alibaba.graphscope.common.ir.meta;

import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures;
import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema;

import org.checkerframework.checker.nullness.qual.Nullable;

import java.util.Objects;

/**
* This class describes basic metadata information required by IR, including Schema, Procedures.
* They are versioned using the same snapshot id.
*/
public class IrMeta {
protected final GraphId graphId;
protected final SnapshotId snapshotId;
protected final IrGraphSchema schema;
protected final @Nullable GraphStoredProcedures storedProcedures;

public IrMeta(IrGraphSchema schema) {
this(SnapshotId.createEmpty(), schema);
}

public IrMeta(IrGraphSchema schema, GraphStoredProcedures storedProcedures) {
this(GraphId.DEFAULT, SnapshotId.createEmpty(), schema, storedProcedures);
}

public IrMeta(SnapshotId snapshotId, IrGraphSchema schema) {
this(GraphId.DEFAULT, snapshotId, schema, null);
}

public IrMeta(
GraphId graphId,
SnapshotId snapshotId,
IrGraphSchema schema,
GraphStoredProcedures storedProcedures) {
this.graphId = graphId;
this.snapshotId = Objects.requireNonNull(snapshotId);
this.schema = Objects.requireNonNull(schema);
this.storedProcedures = storedProcedures;
}

public IrGraphSchema getSchema() {
return schema;
}

public SnapshotId getSnapshotId() {
return snapshotId;
}

public @Nullable GraphStoredProcedures getStoredProcedures() {
return storedProcedures;
}

public GraphId getGraphId() {
return graphId;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
*
* * Copyright 2020 Alibaba Group Holding Limited.
* *
* * Licensed under the Apache License, Version 2.0 (the "License");
* * you may not use this file except in compliance with the License.
* * You may obtain a copy of the License at
* *
* * http://www.apache.org/licenses/LICENSE-2.0
* *
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS,
* * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* * See the License for the specific language governing permissions and
* * limitations under the License.
*
*/

package com.alibaba.graphscope.common.ir.meta;

import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures;
import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema;
import com.alibaba.graphscope.groot.common.schema.api.GraphStatistics;

import org.checkerframework.checker.nullness.qual.Nullable;

/**
* This class describes all metadata information required by IR, including Schema, Procedures and Statistics.
*/
public class IrMetaStats extends IrMeta {
private final @Nullable GraphStatistics statistics;

public IrMetaStats(
SnapshotId snapshotId,
IrGraphSchema schema,
GraphStoredProcedures storedProcedures,
GraphStatistics statistics) {
this(GraphId.DEFAULT, snapshotId, schema, storedProcedures, statistics);
}

public IrMetaStats(
GraphId graphId,
SnapshotId snapshotId,
IrGraphSchema schema,
GraphStoredProcedures storedProcedures,
GraphStatistics statistics) {
super(graphId, snapshotId, schema, storedProcedures);
this.statistics = statistics;
}

public @Nullable GraphStatistics getStatistics() {
return this.statistics;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
*
* * Copyright 2020 Alibaba Group Holding Limited.
* *
* * Licensed under the Apache License, Version 2.0 (the "License");
* * you may not use this file except in compliance with the License.
* * You may obtain a copy of the License at
* *
* * http://www.apache.org/licenses/LICENSE-2.0
* *
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS,
* * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* * See the License for the specific language governing permissions and
* * limitations under the License.
*
*/

package com.alibaba.graphscope.common.ir.meta;

// This interface defines the callback to be invoked when IrMeta changes.
public interface IrMetaTracker {
void onChanged(IrMeta meta);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
*
* * Copyright 2020 Alibaba Group Holding Limited.
* *
* * Licensed under the Apache License, Version 2.0 (the "License");
* * you may not use this file except in compliance with the License.
* * You may obtain a copy of the License at
* *
* * http://www.apache.org/licenses/LICENSE-2.0
* *
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS,
* * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* * See the License for the specific language governing permissions and
* * limitations under the License.
*
*/

package com.alibaba.graphscope.common.ir.meta;

public class SnapshotId {
public static final SnapshotId createEmpty() {
return new SnapshotId(false, -1);
}

private final boolean acquired;
private final long id;

public SnapshotId(boolean acquired, long id) {
this.acquired = acquired;
this.id = id;
}

public boolean isAcquired() {
return acquired;
}

public long getId() {
return id;
}
}
Loading

0 comments on commit 5b53359

Please sign in to comment.