From aa9791f5a8dbfda61db2e09c057c8b7fbaf86cd6 Mon Sep 17 00:00:00 2001 From: Boxuan Li Date: Wed, 12 May 2021 18:28:06 +0800 Subject: [PATCH] Super Node: Add proxy node type This aims to fix the super node problem by introducing proxy nodes (which works like the partition node before), except that we introduce proxy nodes in application layer only when needed. This commit focuses on how to make users a seamless use experience as if the proxy node does not exist. What is not clear right now is, if A is a super node, and A connects to a number of vertices with different labels (and possibly different properties like timestamp). Let's say we have a proxy node for A, Vpa, then how many edges should we create between Va and Vpa? 1) Create 0 edge between Va and proxies. We store id(Vpa) as a vertex property in Va. Every time we need to traverse from Va, we always fetch Vpa, and do the traversal from there. Let's say Va -.-.-.-> Vpa -------> Vb, then when we traverse from Vb, we will find Vpa first, and then we retrieve Va because Vpa is just a proxy for Va. This is very similar if not exactly the same as vertex-cut partition in JanusGraph. 2) Create 1 edge between Va and each Vpa. No specific benefit; drawback is, this edge will be queried, which means we need to remove this edge. 3) Create 1 edge per label between Va and and each Vpa. Benefit is that for queries with label constraint that does not apply here, we don't even need to traverse Vpa because it won't be found with this label. Drawback is also very obvious, if Vpa ---label1---> Vb1, and Vpa ---label2---> Vb2, then if we traverse from Vb1 to Va, we will see two edges, unless we somehow see that we use label1 to go from Vb1 to Vpa thus we shall only use label1 to go from Vpa to Va. This is very cumbersome so we shall not use it. This commit actually uses 3) which makes it very difficult to pass all tests. 4) Create one proxy node for a particular edge type (label + props). Suppose edges are all very similar except for a few properties. Then for each unique edge type, we create a proxy node. Benefit is we can fully utilize VCI because these edges actually represent all physical edges. Drawback is this is very application-specific. For example, if we only need rundate in traversal, then one or more proxy nodes will represent a particular rundate. If original query is g.V(Va).outE().has("rundate", "20210512").inV(), then we can utilize VCI for rundate on Va. Note that there is only 1 edge between Va and each proxy node. Another big challenge of 2, 3, and 4 is how to control edge traversal. If Vpa connects to Vb1 and Vb2, we must avoid wrong traversal from Vb1 to Vb2 via Vpa (when the user aims to traverse from Vb1 to Va), which is difficult in practice. This commit uses option (1): Create proxy nodes but don't draw connection between canonical node and proxy node Signed-off-by: Boxuan Li --- .../janusgraph/graphdb/JanusGraphTest.java | 82 +++++++++++++++ .../java/org/janusgraph/core/VertexLabel.java | 2 + .../core/schema/VertexLabelMaker.java | 2 + .../graphdb/database/StandardJanusGraph.java | 4 +- .../database/idassigner/VertexIDAssigner.java | 8 +- .../graphdb/idmanagement/IDManager.java | 25 ++++- .../condition/RelationTypeCondition.java | 4 + .../BasicVertexCentricQueryBuilder.java | 86 +++++++++++++++- .../JanusGraphProxyTraversalStrategy.java | 99 +++++++++++++++++++ .../types/StandardVertexLabelMaker.java | 11 +++ .../graphdb/types/TypeDefinitionCategory.java | 3 +- .../graphdb/types/VertexLabelVertex.java | 5 + .../graphdb/types/system/BaseVertexLabel.java | 5 + 13 files changed, 328 insertions(+), 8 deletions(-) create mode 100644 janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphProxyTraversalStrategy.java diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java index 19cd0c3e8e..4f5139f0c5 100644 --- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java @@ -16,6 +16,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; @@ -8846,6 +8848,86 @@ public void performReindexAndVerifyEdgeCount(String indexName, String edgeLabel, } } + @Test + public void testNeighborCountWithProxyNode() { + mgmt.makePropertyKey("proxies").dataType(Long.class).cardinality(Cardinality.LIST).make(); + mgmt.makeVertexLabel("proxy").setProxy().make(); + mgmt.commit(); + + newTx(); + + Vertex v0 = graph.addVertex("vertexId", "v0"); + Vertex v1 = graph.addVertex("vertexId", "v1"); + Vertex v2 = graph.addVertex("vertexId", "v2"); + Vertex v3a = graph.addVertex("vertexId", "v3a"); + Vertex v3b = graph.addVertex("vertexId", "v3b"); + Vertex v4 = graph.addVertex("vertexId", "v4"); + + v0.addEdge("normal-edge", v1); + v2.addEdge("normal-edge", v3a); + v1.addEdge("labelX", v4); + + // assume now v1 becomes a super node and we decide to introduce proxy node(s). The previous edges are retained. + // assume the application logic adds an edge from v1 to v2 with labelX, another edge from v1 to v3a with labelY, + // and another edge from v1 to v3b with labelY. + // what happens under the hood is v1 connects to v2/v3a/v3b via vProxy. + Vertex vProxy = graph.addVertex(T.label, "proxy", "canonicalId", v1.id()); + vProxy.addEdge("labelX", v2, "runDate", "01"); + vProxy.addEdge("labelY", v3a, "runDate", "02"); + vProxy.addEdge("labelY", v3b, "runDate", "02"); + // be careful: we need to make sure we assign id right after vertex is created + v1.property("proxies", vProxy.id()); + graph.tx().commit(); + + // ensure proxy node is not involved when we retrieve canonical node's properties + assertEquals(ImmutableList.of("v1"), graph.traversal().V(v1).values("vertexId").toList()); + + // queries without label constraints + assertEquals(4, graph.traversal().V(v1).out().count().next()); + assertEquals(4, graph.traversal().V(v1).out().toList().size()); + assertEquals(ImmutableSet.of("v2", "v3a", "v3b", "v4"), new HashSet<>(graph.traversal().V(v1).out().values("vertexId").toList())); + assertEquals(ImmutableSet.of("v2", "v3a", "v3b", "v4"), new HashSet<>(graph.traversal().V(v1).outE().inV().values("vertexId").toList())); + assertEquals(ImmutableSet.of("v0", "v2", "v3a", "v3b", "v4"), new HashSet<>(graph.traversal().V(v1).both().values("vertexId").toList())); + assertTrue(graph.traversal().V(v1).outE().where(__.otherV().has("vertexId", "v2")).hasNext()); + + assertEquals(2, graph.traversal().V(v3a).in().count().next()); + assertEquals(2, graph.traversal().V(v3a).both().count().next()); + assertEquals(0, graph.traversal().V(v3a).out().count().next()); + assertEquals(2, graph.traversal().V(v3a).inE().count().next()); + assertEquals(2, graph.traversal().V(v3a).bothE().count().next()); + assertEquals(0, graph.traversal().V(v3a).outE().count().next()); + assertEquals(ImmutableSet.of("v1", "v2"), new HashSet<>(graph.traversal().V(v3a).in().values("vertexId").toList())); + assertEquals(ImmutableSet.of("v1", "v2"), new HashSet<>(graph.traversal().V(v3a).both().values("vertexId").toList())); + + assertEquals(ImmutableSet.of("v1", "v3a"), new HashSet<>(graph.traversal().V(v2).both().values("vertexId").toList())); + assertEquals(ImmutableSet.of("v1"), new HashSet<>(graph.traversal().V(v2).in().values("vertexId").toList())); + assertEquals(ImmutableSet.of("v3a"), new HashSet<>(graph.traversal().V(v2).out().values("vertexId").toList())); + assertEquals(ImmutableSet.of("v3a"), new HashSet<>(graph.traversal().V(v2).outE().inV().values("vertexId").toList())); + assertEquals(ImmutableSet.of("v3a"), new HashSet<>(graph.traversal().V(v2).outE().otherV().values("vertexId").toList())); + + // queries with label constraints + assertEquals(ImmutableSet.of("v2", "v4"), new HashSet<>(graph.traversal().V(v1).out("labelX").values("vertexId").toList())); + assertEquals(ImmutableSet.of("v3a", "v3b"), new HashSet<>(graph.traversal().V(v1).out("labelY").values("vertexId").toList())); + assertEquals(ImmutableSet.of("v2", "v4"), new HashSet<>(graph.traversal().V(v1).both("labelX").values("vertexId").toList())); + assertEquals(ImmutableSet.of("v3a", "v3b"), new HashSet<>(graph.traversal().V(v1).both("labelY").values("vertexId").toList())); + assertEquals(2, graph.traversal().V(v1).out("labelX").count().next()); + assertEquals(2, graph.traversal().V(v1).out("labelY").count().next()); + + assertEquals(ImmutableSet.of("v1"), new HashSet<>(graph.traversal().V(v3a).both("labelY").values("vertexId").toList())); + assertEquals(ImmutableSet.of("v2"), new HashSet<>(graph.traversal().V(v3a).both("normal-edge").values("vertexId").toList())); + assertEquals(ImmutableSet.of("v1"), new HashSet<>(graph.traversal().V(v3a).in("labelY").values("vertexId").toList())); + assertEquals(ImmutableSet.of("v1"), new HashSet<>(graph.traversal().V(v3a).inE("labelY").outV().values("vertexId").toList())); + assertEquals(ImmutableSet.of("v2"), new HashSet<>(graph.traversal().V(v3a).in("normal-edge").values("vertexId").toList())); + assertEquals(4, graph.traversal().V(v3a).in("labelY").out().count().next()); + assertEquals(4, graph.traversal().V(v3a).in("labelY").out().toList().size()); + assertEquals(4, graph.traversal().V(v3a).in("labelY").out().values("vertexId").toList().size()); + assertEquals(ImmutableSet.of("v2", "v3a", "v3b", "v4"), new HashSet<>(graph.traversal().V(v3a).in("labelY").out().values("vertexId").toList())); + + assertEquals("02", graph.traversal().V(v3a).inE("labelY").where(__.otherV().has("vertexId", "v1")).next().property("runDate").value()); + + + } + @Test public void testMultipleOrClauses() { clopen(); diff --git a/janusgraph-core/src/main/java/org/janusgraph/core/VertexLabel.java b/janusgraph-core/src/main/java/org/janusgraph/core/VertexLabel.java index 5e2ab36943..e49ee5f674 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/core/VertexLabel.java +++ b/janusgraph-core/src/main/java/org/janusgraph/core/VertexLabel.java @@ -43,6 +43,8 @@ public interface VertexLabel extends JanusGraphVertex, JanusGraphSchemaType { */ boolean isStatic(); + boolean isProxy(); + //TTL /** diff --git a/janusgraph-core/src/main/java/org/janusgraph/core/schema/VertexLabelMaker.java b/janusgraph-core/src/main/java/org/janusgraph/core/schema/VertexLabelMaker.java index 85ef66455c..7eed52f120 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/core/schema/VertexLabelMaker.java +++ b/janusgraph-core/src/main/java/org/janusgraph/core/schema/VertexLabelMaker.java @@ -52,6 +52,8 @@ public interface VertexLabelMaker { */ VertexLabelMaker setStatic(); + VertexLabelMaker setProxy(); + /** * Creates a {@link VertexLabel} according to the specifications of this builder. * diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/StandardJanusGraph.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/StandardJanusGraph.java index d4f8bc4292..a349dfe77a 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/StandardJanusGraph.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/StandardJanusGraph.java @@ -98,6 +98,7 @@ import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMixedIndexAggStrategy; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMixedIndexCountStrategy; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMultiQueryStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphProxyTraversalStrategy; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphStepStrategy; import org.janusgraph.graphdb.transaction.StandardJanusGraphTx; import org.janusgraph.graphdb.transaction.StandardTransactionBuilder; @@ -160,7 +161,8 @@ public class StandardJanusGraph extends JanusGraphBlueprintsGraph { JanusGraphMixedIndexAggStrategy.instance(), JanusGraphMixedIndexCountStrategy.instance(), JanusGraphStepStrategy.instance(), - JanusGraphIoRegistrationStrategy.instance()); + JanusGraphIoRegistrationStrategy.instance(), + JanusGraphProxyTraversalStrategy.instance()); //Register with cache TraversalStrategies.GlobalCache.registerStrategies(StandardJanusGraph.class, graphStrategies); diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/idassigner/VertexIDAssigner.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/idassigner/VertexIDAssigner.java index a3c228e39d..ee5dc49721 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/idassigner/VertexIDAssigner.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/idassigner/VertexIDAssigner.java @@ -381,6 +381,8 @@ private static IDManager.VertexIDType getVertexIDType(VertexLabel vertexLabel) { return IDManager.VertexIDType.PartitionedVertex; } else if (vertexLabel.isStatic()) { return IDManager.VertexIDType.UnmodifiableVertex; + } else if (vertexLabel.isProxy()) { + return IDManager.VertexIDType.ProxyVertex; } else { return IDManager.VertexIDType.NormalVertex; } @@ -403,6 +405,7 @@ private class SimpleVertexIDBlockSizer implements IDBlockSizer { public long getBlockSize(int idNamespace) { switch (PoolType.getPoolType(idNamespace)) { case NORMAL_VERTEX: + case PROXY_VERTEX: return baseBlockSize; case UNMODIFIABLE_VERTEX: return Math.max(10,baseBlockSize/10); @@ -426,7 +429,7 @@ public long getIdUpperBound(int idNamespace) { private enum PoolType { - NORMAL_VERTEX, UNMODIFIABLE_VERTEX, PARTITIONED_VERTEX, RELATION, SCHEMA; + NORMAL_VERTEX, PROXY_VERTEX, UNMODIFIABLE_VERTEX, PARTITIONED_VERTEX, RELATION, SCHEMA; public int getIDNamespace() { return ordinal(); @@ -437,6 +440,7 @@ public long getCountBound(IDManager idManager) { case NORMAL_VERTEX: case UNMODIFIABLE_VERTEX: case PARTITIONED_VERTEX: + case PROXY_VERTEX: return idManager.getVertexCountBound(); case RELATION: return idManager.getRelationCountBound(); case SCHEMA: return IDManager.getSchemaCountBound(); @@ -449,6 +453,7 @@ public boolean hasOnePerPartition() { case NORMAL_VERTEX: case UNMODIFIABLE_VERTEX: case RELATION: + case PROXY_VERTEX: return true; default: return false; } @@ -458,6 +463,7 @@ public static PoolType getPoolTypeFor(IDManager.VertexIDType idType) { if (idType==IDManager.VertexIDType.NormalVertex) return NORMAL_VERTEX; else if (idType== IDManager.VertexIDType.UnmodifiableVertex) return UNMODIFIABLE_VERTEX; else if (idType== IDManager.VertexIDType.PartitionedVertex) return PARTITIONED_VERTEX; + else if (idType== IDManager.VertexIDType.ProxyVertex) return PROXY_VERTEX; else if (IDManager.VertexIDType.Schema.isSubType(idType)) return SCHEMA; else throw new IllegalArgumentException("Invalid id type: " + idType); } diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/idmanagement/IDManager.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/idmanagement/IDManager.java index 07eac44a76..716d28dcfc 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/idmanagement/IDManager.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/idmanagement/IDManager.java @@ -40,7 +40,7 @@ public class IDManager { * 000 - * Normal vertices * 010 - * Partitioned vertices * 100 - * Unmodifiable (e.g. TTL'ed) vertices - * 110 - + Reserved for additional vertex type + * 110 - * Proxy vertices * 1 - + Invisible * 11 - * Invisible (user created/triggered) Vertex [for later] * 01 - + Schema related vertices @@ -125,6 +125,22 @@ final boolean isProper() { return true; } }, + ProxyVertex { + @Override + final long offset() { + return 3L; + } + + @Override + final long suffix() { + return 6L; + } + + @Override + final boolean isProper() { + return true; + } + }, Invisible { @Override final long offset() { @@ -463,6 +479,7 @@ private static VertexIDType getUserVertexIDType(Object vertexId) { if (VertexIDType.NormalVertex.is(vertexId)) type=VertexIDType.NormalVertex; else if (VertexIDType.PartitionedVertex.is(vertexId)) type=VertexIDType.PartitionedVertex; else if (VertexIDType.UnmodifiableVertex.is(vertexId)) type=VertexIDType.UnmodifiableVertex; + else if (VertexIDType.ProxyVertex.is(vertexId)) type=VertexIDType.ProxyVertex; if (null == type) { throw new InvalidIDException("Vertex ID " + vertexId + " has unrecognized type"); } @@ -471,7 +488,7 @@ private static VertexIDType getUserVertexIDType(Object vertexId) { public final boolean isUserVertexId(Object vertexId) { if (vertexId instanceof Number) { - return (VertexIDType.NormalVertex.is(vertexId) || VertexIDType.PartitionedVertex.is(vertexId) || VertexIDType.UnmodifiableVertex.is(vertexId)) + return (VertexIDType.NormalVertex.is(vertexId) || VertexIDType.PartitionedVertex.is(vertexId) || VertexIDType.UnmodifiableVertex.is(vertexId) || VertexIDType.ProxyVertex.is(vertexId)) && ((((Number) vertexId).longValue() >>> (partitionBits+USERVERTEX_PADDING_BITWIDTH))>0); } else { return true; @@ -639,6 +656,10 @@ public boolean isPartitionedVertex(Object id) { return isUserVertexId(id) && VertexIDType.PartitionedVertex.is(id); } + public boolean isProxyVertex(Object id) { + return isUserVertexId(id) && VertexIDType.ProxyVertex.is(id); + } + public long getRelationCountBound() { return relationCountBound; } diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/condition/RelationTypeCondition.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/condition/RelationTypeCondition.java index 756bcc837b..69a186634b 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/condition/RelationTypeCondition.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/condition/RelationTypeCondition.java @@ -55,4 +55,8 @@ public boolean equals(Object other) { public String toString() { return "type["+ relationType.toString()+"]"; } + + public RelationType getRelationType() { + return this.relationType; + } } diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/BasicVertexCentricQueryBuilder.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/BasicVertexCentricQueryBuilder.java index 593449a836..8859d7709b 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/BasicVertexCentricQueryBuilder.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/BasicVertexCentricQueryBuilder.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.janusgraph.core.BaseVertexQuery; import org.janusgraph.core.JanusGraphEdge; import org.janusgraph.core.JanusGraphRelation; @@ -53,8 +54,10 @@ import org.janusgraph.graphdb.relations.RelationComparator; import org.janusgraph.graphdb.relations.StandardVertexProperty; import org.janusgraph.graphdb.transaction.StandardJanusGraphTx; +import org.janusgraph.graphdb.types.system.BaseLabel; import org.janusgraph.graphdb.types.system.ImplicitKey; import org.janusgraph.graphdb.types.system.SystemRelationType; +import org.janusgraph.graphdb.util.CloseableAbstractIterator; import org.janusgraph.graphdb.vertices.CacheVertex; import org.janusgraph.util.datastructures.Interval; import org.janusgraph.util.datastructures.PointInterval; @@ -69,6 +72,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -340,7 +344,8 @@ protected Iterable executeRelations(InternalVertex vertex, B return ResultSetIterator.wrap(merge,baseQuery.getLimit()); } else vertex = tx.getCanonicalVertex(vertex); } - return executeIndividualRelations(vertex,baseQuery); + Iterator iter = new EdgeProxyProcessor(vertex, baseQuery); + return () -> iter; } /** @@ -374,7 +379,7 @@ protected Iterable loadRelationsFromCache(InternalVertex ver merge = ResultMergeSortIterator.mergeSort(merge, iterable, relationComparator, false); } } - + return ResultSetIterator.wrap(merge, baseQuery.getLimit()); } } else { @@ -413,7 +418,8 @@ public Iterable executeVertices(InternalVertex vertex, BaseVer return ResultSetIterator.wrap(merge,baseQuery.getLimit()); } else vertex = tx.getCanonicalVertex(vertex); } - return executeIndividualVertices(vertex,baseQuery); + Iterator iter = new VertexProxyProcessor(vertex, baseQuery); + return () -> iter; } private Iterable executeIndividualVertices(InternalVertex vertex, @@ -859,5 +865,79 @@ private int computeLimit(int remainingConditions, int baseLimit) { return baseLimit; } + // TODO: we should not use CloseableAbstractIterator here since it is unmodifiable + public class VertexProxyProcessor extends CloseableAbstractIterator { + + private List proxyIds = new ArrayList<>(); + private int offset; + private Iterator iter; + private BaseVertexCentricQuery baseQuery; + + public VertexProxyProcessor(InternalVertex vertex, BaseVertexCentricQuery baseQuery) { + Condition condition = baseQuery.getCondition(); + if (condition instanceof And && ((RelationCategory) ((And) condition).get(0)).name().equals(RelationCategory.EDGE.name()) + || condition instanceof RelationTypeCondition && ((RelationTypeCondition) condition).getRelationType().isEdgeLabel() + && !(((RelationTypeCondition) condition).getRelationType() instanceof BaseLabel)) { + Iterator> proxyIter = vertex.properties("proxies"); + while (proxyIter.hasNext()) { + proxyIds.add(proxyIter.next().value()); + } + } + iter = executeIndividualVertices(vertex, baseQuery).iterator(); + this.baseQuery = baseQuery; + } + + @Override + protected JanusGraphVertex computeNext() { + if (iter.hasNext()) { + JanusGraphVertex v = iter.next(); + if (tx.getIdInspector().isProxyVertex(v.id())) { + long canonicalId = (long) v.property("canonicalId").value(); + JanusGraphVertex canonicalV = tx.getVertex(canonicalId); + return canonicalV; + } + return v; + } else if (offset < proxyIds.size()) { + InternalVertex proxyV = (InternalVertex) tx.getVertex(proxyIds.get(offset++)); + iter = executeIndividualVertices(proxyV, baseQuery).iterator(); + return computeNext(); + } + return endOfData(); + } + } + + public class EdgeProxyProcessor extends CloseableAbstractIterator { + + private List proxyIds = new ArrayList<>(); + private int offset; + private Iterator iter; + private BaseVertexCentricQuery baseQuery; + + public EdgeProxyProcessor(InternalVertex vertex, BaseVertexCentricQuery baseQuery) { + Condition condition = baseQuery.getCondition(); + if (condition instanceof And && ((RelationCategory) ((And) condition).get(0)).name() == RelationCategory.EDGE.name() + || condition instanceof RelationTypeCondition && ((RelationTypeCondition) condition).getRelationType().isEdgeLabel() + && !(((RelationTypeCondition) condition).getRelationType() instanceof BaseLabel)) { + Iterator> proxyIter = vertex.properties("proxies"); + while (proxyIter.hasNext()) { + proxyIds.add(proxyIter.next().value()); + } + } + iter = executeIndividualRelations(vertex, baseQuery).iterator(); + this.baseQuery = baseQuery; + } + + @Override + protected JanusGraphRelation computeNext() { + if (iter.hasNext()) { + return iter.next(); + } else if (offset < proxyIds.size()) { + InternalVertex proxyV = (InternalVertex) tx.getVertex(proxyIds.get(offset++)); + iter = executeIndividualRelations(proxyV, baseQuery).iterator(); + return computeNext(); + } + return endOfData(); + } + } } diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphProxyTraversalStrategy.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphProxyTraversalStrategy.java new file mode 100644 index 0000000000..8f83c9eb11 --- /dev/null +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphProxyTraversalStrategy.java @@ -0,0 +1,99 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.graphdb.tinkerpop.optimize.strategy; + +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.Traverser; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeOtherVertexStep; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.util.ElementHelper; +import org.janusgraph.graphdb.database.StandardJanusGraph; +import org.janusgraph.graphdb.transaction.StandardJanusGraphTx; +import org.janusgraph.graphdb.vertices.AbstractVertex; + +import java.util.List; + +/** + * This strategy is to ensure traversals work properly even if it encounters proxy nodes + */ +public class JanusGraphProxyTraversalStrategy extends AbstractTraversalStrategy implements TraversalStrategy.ProviderOptimizationStrategy { + + private static final JanusGraphProxyTraversalStrategy INSTANCE = new JanusGraphProxyTraversalStrategy(); + + private JanusGraphProxyTraversalStrategy() { + } + + @Override + public void apply(final Traversal.Admin traversal) { + TraversalHelper.getStepsOfClass(EdgeOtherVertexStep.class, traversal).forEach(originalStep -> { + final JanusGraphEdgeOtherVertexStep step = new JanusGraphEdgeOtherVertexStep(traversal); + TraversalHelper.replaceStep(originalStep, step, traversal); + }); + } + + public static JanusGraphProxyTraversalStrategy instance() { + return INSTANCE; + } + + public static class JanusGraphEdgeOtherVertexStep extends EdgeOtherVertexStep { + + private StandardJanusGraph janusGraph; + + public JanusGraphEdgeOtherVertexStep(final Traversal.Admin traversal) { + super(traversal); + final Graph graph = traversal.getGraph().get(); + janusGraph = graph instanceof StandardJanusGraphTx ? ((StandardJanusGraphTx) graph).getGraph() : (StandardJanusGraph) graph; + } + + @Override + protected Vertex map(final Traverser.Admin traverser) { + final List objects = traverser.path().objects(); + for (int i = objects.size() - 2; i >= 0; i--) { + if (objects.get(i) instanceof Vertex) { + final Edge edge = traverser.get(); + final Vertex outVertex = edge.outVertex(); + final Vertex inVertex = edge.inVertex(); + final Vertex v = (Vertex) objects.get(i); + AbstractVertex otherV; + if (ElementHelper.areEqual(v, edge.outVertex())) { + otherV = (AbstractVertex) inVertex; + } else if (ElementHelper.areEqual(v, edge.inVertex())) { + otherV = (AbstractVertex) outVertex; + } else { + // at least one endpoint of this edge is a proxy node + if (janusGraph.getIDManager().isProxyVertex((long) outVertex.id()) && outVertex.property("canonicalId").value().equals(v.id())) { + otherV = (AbstractVertex) inVertex; + } else { + otherV = (AbstractVertex) outVertex; + } + } + if (janusGraph.getIDManager().isProxyVertex((long) otherV.id())) { + long canonicalId = (long) otherV.property("canonicalId").value(); + return otherV.tx().getVertex(canonicalId); + } else { + return otherV; + } + } + } + throw new IllegalStateException("The path history of the traverser does not contain a previous vertex: " + traverser.path()); + } + + } +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/StandardVertexLabelMaker.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/StandardVertexLabelMaker.java index 2b1c4f41ef..6dca4d5047 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/StandardVertexLabelMaker.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/StandardVertexLabelMaker.java @@ -22,6 +22,7 @@ import org.janusgraph.graphdb.types.system.SystemTypeManager; import static org.janusgraph.graphdb.types.TypeDefinitionCategory.PARTITIONED; +import static org.janusgraph.graphdb.types.TypeDefinitionCategory.PROXY; import static org.janusgraph.graphdb.types.TypeDefinitionCategory.STATIC; /** @@ -34,6 +35,7 @@ public class StandardVertexLabelMaker implements VertexLabelMaker { private String name; private boolean partitioned; private boolean isStatic; + private boolean isProxy; public StandardVertexLabelMaker(StandardJanusGraphTx tx) { this.tx = tx; @@ -63,12 +65,21 @@ public StandardVertexLabelMaker setStatic() { return this; } + @Override + public StandardVertexLabelMaker setProxy() { + isProxy = true; + return this; + } + @Override public VertexLabel make() { Preconditions.checkArgument(!partitioned || !isStatic,"A vertex label cannot be partitioned and static at the same time"); + Preconditions.checkArgument(!partitioned || !isProxy, "A vertex label cannot be partitioned and proxy at the same time"); + Preconditions.checkArgument(!isStatic || !isProxy, "A vertex label cannot be proxy and static at the same time"); TypeDefinitionMap def = new TypeDefinitionMap(); def.setValue(PARTITIONED, partitioned); def.setValue(STATIC, isStatic); + def.setValue(PROXY, isProxy); return (VertexLabelVertex)tx.makeSchemaVertex(JanusGraphSchemaCategory.VERTEXLABEL,name,def); } diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/TypeDefinitionCategory.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/TypeDefinitionCategory.java index cb1c97ea1a..66fac987d4 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/TypeDefinitionCategory.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/TypeDefinitionCategory.java @@ -67,6 +67,7 @@ public enum TypeDefinitionCategory { //Vertex Label PARTITIONED(Boolean.class), STATIC(Boolean.class), + PROXY(Boolean.class), //Schema Edges RELATIONTYPE_INDEX(), @@ -80,7 +81,7 @@ public enum TypeDefinitionCategory { public static final Set PROPERTYKEY_DEFINITION_CATEGORIES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(STATUS, INVISIBLE, SORT_KEY, SORT_ORDER, SIGNATURE, MULTIPLICITY, DATATYPE))); public static final Set EDGELABEL_DEFINITION_CATEGORIES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(STATUS, INVISIBLE, SORT_KEY, SORT_ORDER, SIGNATURE, MULTIPLICITY, UNIDIRECTIONAL))); public static final Set INDEX_DEFINITION_CATEGORIES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(STATUS, ELEMENT_CATEGORY,INDEX_CARDINALITY,INTERNAL_INDEX, BACKING_INDEX,INDEXSTORE_NAME))); - public static final Set VERTEXLABEL_DEFINITION_CATEGORIES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(PARTITIONED, STATIC))); + public static final Set VERTEXLABEL_DEFINITION_CATEGORIES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(PARTITIONED, STATIC, PROXY))); public static final Set TYPE_MODIFIER_DEFINITION_CATEGORIES = Collections.unmodifiableSet(Stream.of(ModifierType.values()).map(ModifierType::getCategory).collect(Collectors.toSet())); private final RelationCategory relationCategory; diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/VertexLabelVertex.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/VertexLabelVertex.java index b8209b9948..6fa2d337ca 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/VertexLabelVertex.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/VertexLabelVertex.java @@ -46,6 +46,11 @@ public boolean isStatic() { return getDefinition().getValue(TypeDefinitionCategory.STATIC, Boolean.class); } + @Override + public boolean isProxy() { + return getDefinition().getValue(TypeDefinitionCategory.PROXY, Boolean.class); + } + @Override public Collection mappedProperties() { return CollectionsUtil.toArrayList( diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/system/BaseVertexLabel.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/system/BaseVertexLabel.java index 293266bf6d..e544409803 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/system/BaseVertexLabel.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/types/system/BaseVertexLabel.java @@ -45,6 +45,11 @@ public boolean isStatic() { return false; } + @Override + public boolean isProxy() { + return false; + } + @Override public String name() { return name;