From 32e8ff6da32969d23863ab64ebcf7fe2b30ca0ed Mon Sep 17 00:00:00 2001 From: ebice Date: Fri, 19 Jun 2015 08:53:59 -0400 Subject: [PATCH 01/13] Starting to create a GraphSONReader capable of reading TP2 GraphSON and usable via hadoop-gremlin --- .../io/graphson/GraphSONLegacyReader.java | 275 ++++++++++++++++++ .../gremlin/structure/io/IoTest.java | 5 +- .../graphson/GraphSONLegacyInputFormat.java | 32 ++ .../graphson/GraphSONLegacyRecordReader.java | 88 ++++++ .../GraphSONLegacyRecordReaderTest.java | 10 + 5 files changed, 407 insertions(+), 3 deletions(-) create mode 100644 gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java create mode 100644 hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java create mode 100644 hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReader.java create mode 100644 hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java new file mode 100644 index 00000000000..0c53ad14272 --- /dev/null +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.structure.io.graphson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Property; +import org.apache.tinkerpop.gremlin.structure.T; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.apache.tinkerpop.gremlin.structure.io.GraphReader; +import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; +import org.apache.tinkerpop.gremlin.structure.io.Io; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter; +import org.apache.tinkerpop.gremlin.structure.util.Attachable; +import org.apache.tinkerpop.gremlin.structure.util.Host; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedProperty; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty; +import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph; +import org.apache.tinkerpop.gremlin.structure.util.star.StarGraphGraphSONSerializer; +import org.apache.tinkerpop.gremlin.util.function.FunctionUtils; +import org.javatuples.Pair; + +import java.io.*; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; + +/** + * + * + * @author Edi Bice + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public final class GraphSONLegacyReader implements GraphReader { + private final ObjectMapper mapper; + private final long batchSize; + + final TypeReference> mapTypeReference = new TypeReference>() { + }; + + private GraphSONLegacyReader(final GraphSONMapper mapper, final long batchSize) { + this.mapper = mapper.createMapper(); + this.batchSize = batchSize; + } + + /** + * Read data into a {@link Graph} from output generated by any of the {@link GraphSONWriter} {@code writeVertex} or + * {@code writeVertices} methods or by {@link GryoWriter#writeGraph(OutputStream, Graph)}. + * + * @param inputStream a stream containing an entire graph of vertices and edges as defined by the accompanying + * {@link GraphSONWriter#writeGraph(OutputStream, Graph)}. + * @param graphToWriteTo the graph to write to when reading from the stream. + */ + @Override + public void readGraph(final InputStream inputStream, final Graph graphToWriteTo) throws IOException { + // dual pass - create all vertices and store to cache the ids. then create edges. as long as we don't + // have vertex labels in the output we can't do this single pass + final Map cache = new HashMap<>(); + final AtomicLong counter = new AtomicLong(0); + + final boolean supportsTx = graphToWriteTo.features().graph().supportsTransactions(); + final Graph.Features.EdgeFeatures edgeFeatures = graphToWriteTo.features().edge(); + + final BufferedReader br = new BufferedReader(new InputStreamReader(inputStream)); + br.lines().map(FunctionUtils.wrapFunction(line -> readVertex(new ByteArrayInputStream(line.getBytes()), null, null, Direction.OUT))).forEach(vertex -> { + final Attachable attachable = (Attachable) vertex; + cache.put((StarGraph.StarVertex) attachable.get(), attachable.attach(Attachable.Method.create(graphToWriteTo))); + if (supportsTx && counter.incrementAndGet() % batchSize == 0) + graphToWriteTo.tx().commit(); + }); + cache.entrySet().forEach(kv -> kv.getKey().edges(Direction.OUT).forEachRemaining(e -> { + // can't use a standard Attachable attach method here because we have to use the cache for those + // graphs that don't support userSuppliedIds on edges. note that outVertex/inVertex methods return + // StarAdjacentVertex whose equality should match StarVertex. + final Vertex cachedOutV = cache.get(e.outVertex()); + final Vertex cachedInV = cache.get(e.inVertex()); + final Edge newEdge = edgeFeatures.willAllowId(e.id()) ? cachedOutV.addEdge(e.label(), cachedInV, T.id, e.id()) : cachedOutV.addEdge(e.label(), cachedInV); + e.properties().forEachRemaining(p -> newEdge.property(p.key(), p.value())); + if (supportsTx && counter.incrementAndGet() % batchSize == 0) + graphToWriteTo.tx().commit(); + })); + + if (supportsTx) graphToWriteTo.tx().commit(); + } + + /** + * Read {@link Vertex} objects from output generated by any of the {@link GraphSONWriter} {@code writeVertex} or + * {@code writeVertices} methods or by {@link GraphSONWriter#writeGraph(OutputStream, Graph)}. + * + * @param inputStream a stream containing at least one {@link Vertex} as defined by the accompanying + * {@link GraphWriter#writeVertices(OutputStream, Iterator, Direction)} or + * {@link GraphWriter#writeVertices(OutputStream, Iterator)} methods. + * @param vertexAttachMethod a function that creates re-attaches a {@link Vertex} to a {@link Host} object. + * @param edgeAttachMethod a function that creates re-attaches a {@link Edge} to a {@link Host} object. + * @param attachEdgesOfThisDirection only edges of this direction are passed to the {@code edgeMaker}. + */ + @Override + public Iterator readVertices(final InputStream inputStream, + final Function, Vertex> vertexAttachMethod, + final Function, Edge> edgeAttachMethod, + final Direction attachEdgesOfThisDirection) throws IOException { + final BufferedReader br = new BufferedReader(new InputStreamReader(inputStream)); + return br.lines().map(FunctionUtils.wrapFunction(line -> readVertex(new ByteArrayInputStream(line.getBytes()), vertexAttachMethod, edgeAttachMethod, attachEdgesOfThisDirection))).iterator(); + } + + /** + * Read a {@link Vertex} from output generated by any of the {@link GraphSONWriter} {@code writeVertex} or + * {@code writeVertices} methods or by {@link GraphSONWriter#writeGraph(OutputStream, Graph)}. + * + * @param inputStream a stream containing at least a single vertex as defined by the accompanying + * {@link GraphWriter#writeVertex(OutputStream, Vertex)}. + * @param vertexAttachMethod a function that creates re-attaches a {@link Vertex} to a {@link Host} object. + */ + @Override + public Vertex readVertex(final InputStream inputStream, final Function, Vertex> vertexAttachMethod) throws IOException { + return readVertex(inputStream, vertexAttachMethod, null, null); + } + + /** + * Read a {@link Vertex} from output generated by any of the {@link GraphSONWriter} {@code writeVertex} or + * {@code writeVertices} methods or by {@link GraphSONWriter#writeGraph(OutputStream, Graph)}. + * + * @param inputStream a stream containing at least one {@link Vertex} as defined by the accompanying + * {@link GraphWriter#writeVertices(OutputStream, Iterator, Direction)} method. + * @param vertexAttachMethod a function that creates re-attaches a {@link Vertex} to a {@link Host} object. + * @param edgeAttachMethod a function that creates re-attaches a {@link Edge} to a {@link Host} object. + * @param attachEdgesOfThisDirection only edges of this direction are passed to the {@code edgeMaker}. + */ + @Override + public Vertex readVertex(final InputStream inputStream, + final Function, Vertex> vertexAttachMethod, + final Function, Edge> edgeAttachMethod, + final Direction attachEdgesOfThisDirection) throws IOException { + final Map vertexData = mapper.readValue(inputStream, mapTypeReference); + final StarGraph starGraph = StarGraphGraphSONSerializer.readStarGraphVertex(vertexData); + if (vertexAttachMethod != null) vertexAttachMethod.apply(starGraph.getStarVertex()); + + if (vertexData.containsKey(GraphSONTokens.OUT_E) && (attachEdgesOfThisDirection == Direction.BOTH || attachEdgesOfThisDirection == Direction.OUT)) + StarGraphGraphSONSerializer.readStarGraphEdges(edgeAttachMethod, starGraph, vertexData, GraphSONTokens.OUT_E); + + if (vertexData.containsKey(GraphSONTokens.IN_E) && (attachEdgesOfThisDirection == Direction.BOTH || attachEdgesOfThisDirection == Direction.IN)) + StarGraphGraphSONSerializer.readStarGraphEdges(edgeAttachMethod, starGraph, vertexData, GraphSONTokens.IN_E); + + return starGraph.getStarVertex(); + } + + /** + * Read an {@link Edge} from output generated by {@link GraphSONWriter#writeEdge(OutputStream, Edge)} or via + * an {@link Edge} passed to {@link GraphSONWriter#writeObject(OutputStream, Object)}. + * + * @param inputStream a stream containing at least one {@link Edge} as defined by the accompanying + * {@link GraphWriter#writeEdge(OutputStream, Edge)} method. + * @param edgeAttachMethod a function that creates re-attaches a {@link Edge} to a {@link Host} object. + */ + @Override + public Edge readEdge(final InputStream inputStream, final Function, Edge> edgeAttachMethod) throws IOException { + final Map edgeData = mapper.readValue(inputStream, mapTypeReference); + + final Map edgeProperties = edgeData.containsKey(GraphSONTokens.PROPERTIES) ? + (Map) edgeData.get(GraphSONTokens.PROPERTIES) : Collections.EMPTY_MAP; + final DetachedEdge edge = new DetachedEdge(edgeData.get(GraphSONTokens.ID), + edgeData.get(GraphSONTokens.LABEL).toString(), + edgeProperties, + Pair.with(edgeData.get(GraphSONTokens.OUT), edgeData.get(GraphSONTokens.OUT_LABEL).toString()), + Pair.with(edgeData.get(GraphSONTokens.IN), edgeData.get(GraphSONTokens.IN_LABEL).toString())); + + return edgeAttachMethod.apply(edge); + } + + /** + * Read a {@link VertexProperty} from output generated by + * {@link GraphSONWriter#writeVertexProperty(OutputStream, VertexProperty)} or via an {@link VertexProperty} passed + * to {@link GraphSONWriter#writeObject(OutputStream, Object)}. + * + * @param inputStream a stream containing at least one {@link VertexProperty} as written by the accompanying + * {@link GraphWriter#writeVertexProperty(OutputStream, VertexProperty)} method. + * @param vertexPropertyAttachMethod a function that creates re-attaches a {@link VertexProperty} to a + * {@link Host} object. + */ + @Override + public VertexProperty readVertexProperty(final InputStream inputStream, + final Function, VertexProperty> vertexPropertyAttachMethod) throws IOException { + final Map vpData = mapper.readValue(inputStream, mapTypeReference); + final Map metaProperties = (Map) vpData.get(GraphSONTokens.PROPERTIES); + final DetachedVertexProperty vp = new DetachedVertexProperty(vpData.get(GraphSONTokens.ID), + vpData.get(GraphSONTokens.LABEL).toString(), + vpData.get(GraphSONTokens.VALUE), metaProperties); + return vertexPropertyAttachMethod.apply(vp); + } + + /** + * Read a {@link Property} from output generated by {@link GraphSONWriter#writeProperty(OutputStream, Property)} or + * via an {@link Property} passed to {@link GraphSONWriter#writeObject(OutputStream, Object)}. + * + * @param inputStream a stream containing at least one {@link Property} as written by the accompanying + * {@link GraphWriter#writeProperty(OutputStream, Property)} method. + * @param propertyAttachMethod a function that creates re-attaches a {@link Property} to a {@link Host} object. + */ + @Override + public Property readProperty(final InputStream inputStream, + final Function, Property> propertyAttachMethod) throws IOException { + final Map propertyData = mapper.readValue(inputStream, mapTypeReference); + final DetachedProperty p = new DetachedProperty(propertyData.get(GraphSONTokens.KEY).toString(), propertyData.get(GraphSONTokens.VALUE)); + return propertyAttachMethod.apply(p); + } + + /** + * {@inheritDoc} + */ + @Override + public C readObject(final InputStream inputStream, final Class clazz) throws IOException { + return mapper.readValue(inputStream, clazz); + } + + public static Builder build() { + return new Builder(); + } + + public final static class Builder implements ReaderBuilder { + private long batchSize = 10000; + + private GraphSONMapper mapper = GraphSONMapper.build().create(); + + private Builder() {} + + /** + * Number of mutations to perform before a commit is executed when using + * {@link GraphSONReader#readGraph(InputStream, Graph)}. + */ + public Builder batchSize(final long batchSize) { + this.batchSize = batchSize; + return this; + } + + /** + * Override all of the {@link GraphSONMapper} builder + * options with this mapper. If this value is set to something other than null then that value will be + * used to construct the writer. + */ + public Builder mapper(final GraphSONMapper mapper) { + this.mapper = mapper; + return this; + } + + public GraphSONLegacyReader create() { + return new GraphSONLegacyReader(mapper, batchSize); + } + } +} diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoTest.java index 3ed7e544d45..4b8236e00f8 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoTest.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoTest.java @@ -30,7 +30,6 @@ import org.apache.commons.configuration.Configuration; import org.apache.tinkerpop.gremlin.AbstractGremlinTest; import org.apache.tinkerpop.gremlin.FeatureRequirement; -import org.apache.tinkerpop.gremlin.GraphManager; import org.apache.tinkerpop.gremlin.LoadGraphWith; import org.apache.tinkerpop.gremlin.TestHelper; import org.apache.tinkerpop.gremlin.structure.Direction; @@ -54,7 +53,7 @@ import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; -import org.apache.tinkerpop.gremlin.structure.io.graphson.LegacyGraphSONReader; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONLegacyReader; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; @@ -1737,7 +1736,7 @@ public void shouldReadWriteVertexWithBOTHEdgesToGraphSONWithTypes() throws Excep @FeatureRequirement(featureClass = VertexPropertyFeatures.class, feature = FEATURE_INTEGER_VALUES) @FeatureRequirement(featureClass = EdgePropertyFeatures.class, feature = EdgePropertyFeatures.FEATURE_FLOAT_VALUES) public void shouldReadLegacyGraphSON() throws IOException { - final GraphReader reader = LegacyGraphSONReader.build().create(); + final GraphReader reader = GraphSONLegacyReader.build().create(); try (final InputStream stream = IoTest.class.getResourceAsStream(TestHelper.convertPackageToResourcePath(GraphSONResourceAccess.class) + "tinkerpop-classic-legacy.json")) { reader.readGraph(stream, graph); } diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java new file mode 100644 index 00000000000..ed38f468fba --- /dev/null +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java @@ -0,0 +1,32 @@ +package org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.compress.CompressionCodecFactory; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolsConfigurable; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; + +import java.io.IOException; + +/** + * Created by Edi Bice on 6/18/2015. + */ +public final class GraphSONLegacyInputFormat extends FileInputFormat implements HadoopPoolsConfigurable { + + @Override + public RecordReader createRecordReader(final InputSplit split, final TaskAttemptContext context) throws IOException, InterruptedException { + RecordReader reader = new GraphSONRecordReader(); + reader.initialize(split, context); + return reader; + } + + @Override + protected boolean isSplitable(final JobContext context, final Path file) { + return null == new CompressionCodecFactory(context.getConfiguration()).getCodec(file); + } +} diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReader.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReader.java new file mode 100644 index 00000000000..9960397e656 --- /dev/null +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReader.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson; + +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; +import org.apache.tinkerpop.gremlin.hadoop.Constants; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONLegacyReader; +import org.apache.tinkerpop.gremlin.structure.util.Attachable; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * @author Edi Bice + */ +public final class GraphSONLegacyRecordReader extends RecordReader { + + private final GraphSONLegacyReader graphsonLegacyReader = GraphSONLegacyReader.build().create(); + private final VertexWritable vertexWritable = new VertexWritable(); + private final LineRecordReader lineRecordReader; + private boolean hasEdges; + + public GraphSONLegacyRecordReader() { + this.lineRecordReader = new LineRecordReader(); + } + + @Override + public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException { + this.lineRecordReader.initialize(genericSplit, context); + this.hasEdges = context.getConfiguration().getBoolean(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT_HAS_EDGES, true); + } + + @Override + public boolean nextKeyValue() throws IOException { + if (!this.lineRecordReader.nextKeyValue()) + return false; + + try (InputStream in = new ByteArrayInputStream(this.lineRecordReader.getCurrentValue().getBytes())) { + this.vertexWritable.set(this.hasEdges ? + this.graphsonLegacyReader.readVertex(in, Attachable::get, Attachable::get, Direction.BOTH) : + this.graphsonLegacyReader.readVertex(in, Attachable::get)); + return true; + } + } + + @Override + public NullWritable getCurrentKey() { + return NullWritable.get(); + } + + @Override + public VertexWritable getCurrentValue() { + return this.vertexWritable; + } + + @Override + public float getProgress() throws IOException { + return this.lineRecordReader.getProgress(); + } + + @Override + public synchronized void close() throws IOException { + this.lineRecordReader.close(); + } +} diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java new file mode 100644 index 00000000000..75608def801 --- /dev/null +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java @@ -0,0 +1,10 @@ +package org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson; + +import junit.framework.TestCase; + +/** + * Created by ebice on 6/18/2015. + */ +public class GraphSONLegacyRecordReaderTest extends TestCase { + +} \ No newline at end of file From 536946e36afd84dbc5c921bb74ef8010c5822778 Mon Sep 17 00:00:00 2001 From: ebice Date: Fri, 19 Jun 2015 16:35:38 -0400 Subject: [PATCH 02/13] Merge branch 'master' of https://github.com/apache/incubator-tinkerpop Conflicts: gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoTest.java Continued fleshing out GraphSONLegacyReader for use with hadoop-gremlin --- .../io/graphson/GraphSONLegacyReader.java | 70 +++++-- .../io/graphson/GraphSONTokensTP2.java | 33 +++- .../io/graphson/LegacyGraphSONReader.java | 172 ------------------ .../io/graphson/LegacyGraphSONUtility.java | 160 +++++++++++++++- .../structure/util/star/StarGraph.java | 4 +- .../graphson/GraphSONLegacyInputFormat.java | 2 +- 6 files changed, 244 insertions(+), 197 deletions(-) diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java index 0c53ad14272..bdf3e43281e 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java @@ -82,10 +82,12 @@ public void readGraph(final InputStream inputStream, final Graph graphToWriteTo) // dual pass - create all vertices and store to cache the ids. then create edges. as long as we don't // have vertex labels in the output we can't do this single pass final Map cache = new HashMap<>(); + //final Map cache = new HashMap<>(); final AtomicLong counter = new AtomicLong(0); final boolean supportsTx = graphToWriteTo.features().graph().supportsTransactions(); final Graph.Features.EdgeFeatures edgeFeatures = graphToWriteTo.features().edge(); + final Graph.Features.VertexFeatures vertexFeatures = graphToWriteTo.features().vertex(); final BufferedReader br = new BufferedReader(new InputStreamReader(inputStream)); br.lines().map(FunctionUtils.wrapFunction(line -> readVertex(new ByteArrayInputStream(line.getBytes()), null, null, Direction.OUT))).forEach(vertex -> { @@ -157,19 +159,57 @@ public Vertex readVertex(final InputStream inputStream, final Function, Vertex> vertexAttachMethod, final Function, Edge> edgeAttachMethod, final Direction attachEdgesOfThisDirection) throws IOException { - final Map vertexData = mapper.readValue(inputStream, mapTypeReference); - final StarGraph starGraph = StarGraphGraphSONSerializer.readStarGraphVertex(vertexData); - if (vertexAttachMethod != null) vertexAttachMethod.apply(starGraph.getStarVertex()); - if (vertexData.containsKey(GraphSONTokens.OUT_E) && (attachEdgesOfThisDirection == Direction.BOTH || attachEdgesOfThisDirection == Direction.OUT)) - StarGraphGraphSONSerializer.readStarGraphEdges(edgeAttachMethod, starGraph, vertexData, GraphSONTokens.OUT_E); + final StarGraph starGraph = StarGraph.open(); - if (vertexData.containsKey(GraphSONTokens.IN_E) && (attachEdgesOfThisDirection == Direction.BOTH || attachEdgesOfThisDirection == Direction.IN)) - StarGraphGraphSONSerializer.readStarGraphEdges(edgeAttachMethod, starGraph, vertexData, GraphSONTokens.IN_E); + final JsonFactory factory = mapper.getFactory(); + try (JsonParser parser = factory.createParser(inputStream)) { + + final JsonNode node = parser.readValueAsTree(); + final Map vertexData = LegacyGraphSONUtility.readProperties(node); + starGraph.addVertex(T.id, vertexData.get(GraphSONTokensTP2._ID), T.label, vertexData.get(GraphSONTokensTP2._LABEL)); + for (Map.Entry p : vertexData.entrySet()) { + if (LegacyGraphSONUtility.isReservedKey(p.getKey())) + continue; + starGraph.getStarVertex().property(p.getKey(), p.getValue()); //cardinality? .getCardinality(p.getKey()) + } + if (vertexAttachMethod != null) vertexAttachMethod.apply(starGraph.getStarVertex()); + + if (vertexData.containsKey(GraphSONTokensTP2._OUT_E) && (attachEdgesOfThisDirection == Direction.BOTH || attachEdgesOfThisDirection == Direction.OUT)) + readEdges(edgeAttachMethod, starGraph, vertexData, GraphSONTokens.OUT_E); + + if (vertexData.containsKey(GraphSONTokensTP2._IN_E) && (attachEdgesOfThisDirection == Direction.BOTH || attachEdgesOfThisDirection == Direction.IN)) + readEdges(edgeAttachMethod, starGraph, vertexData, GraphSONTokens.IN_E); + + } catch (Exception ex) { + throw new IOException(ex); + } return starGraph.getStarVertex(); } + public static void readEdges(final Function, Edge> edgeMaker, + final StarGraph starGraph, + final Map vertexData, + final String direction) throws IOException { + final Map>> edgeDatas = (Map>>) vertexData.get(direction); + for (Map.Entry>> edgeData : edgeDatas.entrySet()) { + for (Map inner : edgeData.getValue()) { + final StarGraph.StarEdge starEdge; + String edgeLabel = inner.get(GraphSONTokensTP2._LABEL).toString(); + if (direction.equals(GraphSONTokens.OUT_E)) + starEdge = (StarGraph.StarEdge) starGraph.getStarVertex().addOutEdge(edgeLabel, starGraph.addVertex(T.id, inner.get(GraphSONTokensTP2._IN_V)), T.id, inner.get(GraphSONTokensTP2._ID)); + else + starEdge = (StarGraph.StarEdge) starGraph.getStarVertex().addInEdge(edgeLabel, starGraph.addVertex(T.id, inner.get(GraphSONTokensTP2._OUT_V)), T.id, inner.get(GraphSONTokensTP2._ID)); + for (Map.Entry epd : inner.entrySet()) { + if (!LegacyGraphSONUtility.isReservedKey(epd.getKey())) + starEdge.property(epd.getKey(), epd.getValue()); + } + if (edgeMaker != null) edgeMaker.apply(starEdge); + } + } + } + /** * Read an {@link Edge} from output generated by {@link GraphSONWriter#writeEdge(OutputStream, Edge)} or via * an {@link Edge} passed to {@link GraphSONWriter#writeObject(OutputStream, Object)}. @@ -184,11 +224,11 @@ public Edge readEdge(final InputStream inputStream, final Function edgeProperties = edgeData.containsKey(GraphSONTokens.PROPERTIES) ? (Map) edgeData.get(GraphSONTokens.PROPERTIES) : Collections.EMPTY_MAP; - final DetachedEdge edge = new DetachedEdge(edgeData.get(GraphSONTokens.ID), - edgeData.get(GraphSONTokens.LABEL).toString(), + final DetachedEdge edge = new DetachedEdge(edgeData.get(GraphSONTokensTP2._ID), + edgeData.get(GraphSONTokensTP2._LABEL).toString(), edgeProperties, - Pair.with(edgeData.get(GraphSONTokens.OUT), edgeData.get(GraphSONTokens.OUT_LABEL).toString()), - Pair.with(edgeData.get(GraphSONTokens.IN), edgeData.get(GraphSONTokens.IN_LABEL).toString())); + Pair.with(edgeData.get(GraphSONTokensTP2._OUT_V), null), + Pair.with(edgeData.get(GraphSONTokensTP2._IN_V), null)); return edgeAttachMethod.apply(edge); } @@ -206,12 +246,14 @@ public Edge readEdge(final InputStream inputStream, final Function, VertexProperty> vertexPropertyAttachMethod) throws IOException { - final Map vpData = mapper.readValue(inputStream, mapTypeReference); + /*final Map vpData = mapper.readValue(inputStream, mapTypeReference); final Map metaProperties = (Map) vpData.get(GraphSONTokens.PROPERTIES); final DetachedVertexProperty vp = new DetachedVertexProperty(vpData.get(GraphSONTokens.ID), vpData.get(GraphSONTokens.LABEL).toString(), vpData.get(GraphSONTokens.VALUE), metaProperties); return vertexPropertyAttachMethod.apply(vp); + */ + return null; } /** @@ -225,9 +267,11 @@ public VertexProperty readVertexProperty(final InputStream inputStream, @Override public Property readProperty(final InputStream inputStream, final Function, Property> propertyAttachMethod) throws IOException { - final Map propertyData = mapper.readValue(inputStream, mapTypeReference); + /*final Map propertyData = mapper.readValue(inputStream, mapTypeReference); final DetachedProperty p = new DetachedProperty(propertyData.get(GraphSONTokens.KEY).toString(), propertyData.get(GraphSONTokens.VALUE)); return propertyAttachMethod.apply(p); + */ + return null; } /** diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java index f7f1814b837..c7575a151a8 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java @@ -1,7 +1,32 @@ package org.apache.tinkerpop.gremlin.structure.io.graphson; -/** - * Created by ebice on 6/19/2015. - */ -public class GraphSONTokensTP2 { +public final class GraphSONTokensTP2 { + + private GraphSONTokensTP2() {} + + public static final String _ID = "_id"; + public static final String _LABEL = "_label"; + public static final String _TYPE = "_type"; + public static final String _OUT_V = "_outV"; + public static final String _IN_V = "_inV"; + public static final String _OUT_E = "_outE"; + public static final String _IN_E = "_inE"; + public static final String VALUE = "value"; + public static final String TYPE = "type"; + public static final String TYPE_LIST = "list"; + public static final String TYPE_STRING = "string"; + public static final String TYPE_DOUBLE = "double"; + public static final String TYPE_INTEGER = "integer"; + public static final String TYPE_FLOAT = "float"; + public static final String TYPE_MAP = "map"; + public static final String TYPE_BOOLEAN = "boolean"; + public static final String TYPE_LONG = "long"; + public static final String TYPE_SHORT = "short"; + public static final String TYPE_BYTE = "byte"; + public static final String TYPE_UNKNOWN = "unknown"; + + public static final String VERTICES = "vertices"; + public static final String EDGES = "edges"; + public static final String MODE = "mode"; + } diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONReader.java index ba8255abb52..d7270685225 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONReader.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONReader.java @@ -244,176 +244,4 @@ public LegacyGraphSONReader create() { } } - static class LegacyGraphSONUtility { - private static final String EMPTY_STRING = ""; - private final Graph g; - private final Graph.Features.VertexFeatures vertexFeatures; - private final Graph.Features.EdgeFeatures edgeFeatures; - private final Map cache; - - public LegacyGraphSONUtility(final Graph g, final Graph.Features.VertexFeatures vertexFeatures, - final Graph.Features.EdgeFeatures edgeFeatures, - final Map cache) { - this.g = g; - this.vertexFeatures = vertexFeatures; - this.edgeFeatures = edgeFeatures; - this.cache = cache; - } - - public Vertex vertexFromJson(final JsonNode json) throws IOException { - final Map props = readProperties(json); - - final Object vertexId = getTypedValueFromJsonNode(json.get(GraphSONTokensTP2._ID)); - final Vertex v = vertexFeatures.willAllowId(vertexId) ? g.addVertex(T.id, vertexId) : g.addVertex(); - cache.put(vertexId, v); - - for (Map.Entry entry : props.entrySet()) { - v.property(g.features().vertex().getCardinality(entry.getKey()), entry.getKey(), entry.getValue()); - } - - return v; - } - - public Edge edgeFromJson(final JsonNode json, final Vertex out, final Vertex in) throws IOException { - final Map props = LegacyGraphSONUtility.readProperties(json); - - final Object edgeId = getTypedValueFromJsonNode(json.get(GraphSONTokensTP2._ID)); - final JsonNode nodeLabel = json.get(GraphSONTokensTP2._LABEL); - final String label = nodeLabel == null ? EMPTY_STRING : nodeLabel.textValue(); - - final Edge e = edgeFeatures.willAllowId(edgeId) ? out.addEdge(label, in, T.id, edgeId) : out.addEdge(label, in) ; - for (Map.Entry entry : props.entrySet()) { - e.property(entry.getKey(), entry.getValue()); - } - - return e; - } - - static Map readProperties(final JsonNode node) { - final Map map = new HashMap<>(); - - final Iterator> iterator = node.fields(); - while (iterator.hasNext()) { - final Map.Entry entry = iterator.next(); - - if (!isReservedKey(entry.getKey())) { - // it generally shouldn't be as such but graphson containing null values can't be shoved into - // element property keys or it will result in error - final Object o = readProperty(entry.getValue()); - if (o != null) { - map.put(entry.getKey(), o); - } - } - } - - return map; - } - - private static boolean isReservedKey(final String key) { - return key.equals(GraphSONTokensTP2._ID) || key.equals(GraphSONTokensTP2._TYPE) || key.equals(GraphSONTokensTP2._LABEL) - || key.equals(GraphSONTokensTP2._OUT_V) || key.equals(GraphSONTokensTP2._IN_V); - } - - private static Object readProperty(final JsonNode node) { - final Object propertyValue; - - if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_UNKNOWN)) { - propertyValue = null; - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BOOLEAN)) { - propertyValue = node.get(GraphSONTokensTP2.VALUE).booleanValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_FLOAT)) { - propertyValue = Float.parseFloat(node.get(GraphSONTokensTP2.VALUE).asText()); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BYTE)) { - propertyValue = Byte.parseByte(node.get(GraphSONTokensTP2.VALUE).asText()); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_SHORT)) { - propertyValue = Short.parseShort(node.get(GraphSONTokensTP2.VALUE).asText()); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_DOUBLE)) { - propertyValue = node.get(GraphSONTokensTP2.VALUE).doubleValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_INTEGER)) { - propertyValue = node.get(GraphSONTokensTP2.VALUE).intValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LONG)) { - propertyValue = node.get(GraphSONTokensTP2.VALUE).longValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_STRING)) { - propertyValue = node.get(GraphSONTokensTP2.VALUE).textValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LIST)) { - propertyValue = readProperties(node.get(GraphSONTokensTP2.VALUE).elements()); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_MAP)) { - propertyValue = readProperties(node.get(GraphSONTokensTP2.VALUE)); - } else { - propertyValue = node.textValue(); - } - - return propertyValue; - } - - private static List readProperties(final Iterator listOfNodes) { - final List array = new ArrayList<>(); - - while (listOfNodes.hasNext()) { - array.add(readProperty(listOfNodes.next())); - } - - return array; - } - - static Object getTypedValueFromJsonNode(final JsonNode node) { - Object theValue = null; - - if (node != null && !node.isNull()) { - if (node.isBoolean()) { - theValue = node.booleanValue(); - } else if (node.isDouble()) { - theValue = node.doubleValue(); - } else if (node.isFloatingPointNumber()) { - theValue = node.floatValue(); - } else if (node.isInt()) { - theValue = node.intValue(); - } else if (node.isLong()) { - theValue = node.longValue(); - } else if (node.isTextual()) { - theValue = node.textValue(); - } else if (node.isArray()) { - // this is an array so just send it back so that it can be - // reprocessed to its primitive components - theValue = node; - } else if (node.isObject()) { - // this is an object so just send it back so that it can be - // reprocessed to its primitive components - theValue = node; - } else { - theValue = node.textValue(); - } - } - - return theValue; - } - } - - public final static class GraphSONTokensTP2 { - - private GraphSONTokensTP2() {} - - public static final String _ID = "_id"; - public static final String _LABEL = "_label"; - public static final String _TYPE = "_type"; - public static final String _OUT_V = "_outV"; - public static final String _IN_V = "_inV"; - public static final String VALUE = "value"; - public static final String TYPE = "type"; - public static final String TYPE_LIST = "list"; - public static final String TYPE_STRING = "string"; - public static final String TYPE_DOUBLE = "double"; - public static final String TYPE_INTEGER = "integer"; - public static final String TYPE_FLOAT = "float"; - public static final String TYPE_MAP = "map"; - public static final String TYPE_BOOLEAN = "boolean"; - public static final String TYPE_LONG = "long"; - public static final String TYPE_SHORT = "short"; - public static final String TYPE_BYTE = "byte"; - public static final String TYPE_UNKNOWN = "unknown"; - - public static final String VERTICES = "vertices"; - public static final String EDGES = "edges"; - public static final String MODE = "mode"; - } } diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java index 932212b8080..5d306b2c6e7 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java @@ -1,7 +1,157 @@ package org.apache.tinkerpop.gremlin.structure.io.graphson; -/** - * Created by ebice on 6/19/2015. - */ -public class LegacyGraphSONUtility { -} +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.T; +import org.apache.tinkerpop.gremlin.structure.Vertex; + +import java.io.IOException; +import java.util.*; + +public final class LegacyGraphSONUtility { + + private static final String EMPTY_STRING = ""; + private final Graph g; + private final Graph.Features.VertexFeatures vertexFeatures; + private final Graph.Features.EdgeFeatures edgeFeatures; + private final Map cache; + + public LegacyGraphSONUtility(final Graph g, final Graph.Features.VertexFeatures vertexFeatures, + final Graph.Features.EdgeFeatures edgeFeatures, + final Map cache) { + this.g = g; + this.vertexFeatures = vertexFeatures; + this.edgeFeatures = edgeFeatures; + this.cache = cache; + } + + public Vertex vertexFromJson(final JsonNode json) throws IOException { + final Map props = readProperties(json); + + final Object vertexId = getTypedValueFromJsonNode(json.get(GraphSONTokensTP2._ID)); + final Vertex v = vertexFeatures.willAllowId(vertexId) ? g.addVertex(T.id, vertexId) : g.addVertex(); + cache.put(vertexId, v); + + for (Map.Entry entry : props.entrySet()) { + v.property(g.features().vertex().getCardinality(entry.getKey()), entry.getKey(), entry.getValue()); + } + + return v; + } + + public Edge edgeFromJson(final JsonNode json, final Vertex out, final Vertex in) throws IOException { + final Map props = LegacyGraphSONUtility.readProperties(json); + + final Object edgeId = getTypedValueFromJsonNode(json.get(GraphSONTokensTP2._ID)); + final JsonNode nodeLabel = json.get(GraphSONTokensTP2._LABEL); + final String label = nodeLabel == null ? EMPTY_STRING : nodeLabel.textValue(); + + final Edge e = edgeFeatures.willAllowId(edgeId) ? out.addEdge(label, in, T.id, edgeId) : out.addEdge(label, in) ; + for (Map.Entry entry : props.entrySet()) { + e.property(entry.getKey(), entry.getValue()); + } + + return e; + } + + public static Map readProperties(final JsonNode node) { + final Map map = new HashMap<>(); + + final Iterator> iterator = node.fields(); + while (iterator.hasNext()) { + final Map.Entry entry = iterator.next(); + + if (!isReservedKey(entry.getKey())) { + // it generally shouldn't be as such but graphson containing null values can't be shoved into + // element property keys or it will result in error + final Object o = readProperty(entry.getValue()); + if (o != null) { + map.put(entry.getKey(), o); + } + } + } + + return map; + } + + public static boolean isReservedKey(final String key) { + return key.equals(GraphSONTokensTP2._ID) || key.equals(GraphSONTokensTP2._TYPE) || key.equals(GraphSONTokensTP2._LABEL) + || key.equals(GraphSONTokensTP2._OUT_V) || key.equals(GraphSONTokensTP2._IN_V) + || key.equals(GraphSONTokensTP2._IN_E) || key.equals(GraphSONTokensTP2._OUT_E); + } + + private static Object readProperty(final JsonNode node) { + final Object propertyValue; + + if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_UNKNOWN)) { + propertyValue = null; + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BOOLEAN)) { + propertyValue = node.get(GraphSONTokensTP2.VALUE).booleanValue(); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_FLOAT)) { + propertyValue = Float.parseFloat(node.get(GraphSONTokensTP2.VALUE).asText()); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BYTE)) { + propertyValue = Byte.parseByte(node.get(GraphSONTokensTP2.VALUE).asText()); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_SHORT)) { + propertyValue = Short.parseShort(node.get(GraphSONTokensTP2.VALUE).asText()); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_DOUBLE)) { + propertyValue = node.get(GraphSONTokensTP2.VALUE).doubleValue(); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_INTEGER)) { + propertyValue = node.get(GraphSONTokensTP2.VALUE).intValue(); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LONG)) { + propertyValue = node.get(GraphSONTokensTP2.VALUE).longValue(); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_STRING)) { + propertyValue = node.get(GraphSONTokensTP2.VALUE).textValue(); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LIST)) { + propertyValue = readProperties(node.get(GraphSONTokensTP2.VALUE).elements()); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_MAP)) { + propertyValue = readProperties(node.get(GraphSONTokensTP2.VALUE)); + } else { + propertyValue = node.textValue(); + } + + return propertyValue; + } + + private static List readProperties(final Iterator listOfNodes) { + final List array = new ArrayList<>(); + + while (listOfNodes.hasNext()) { + array.add(readProperty(listOfNodes.next())); + } + + return array; + } + + static Object getTypedValueFromJsonNode(final JsonNode node) { + Object theValue = null; + + if (node != null && !node.isNull()) { + if (node.isBoolean()) { + theValue = node.booleanValue(); + } else if (node.isDouble()) { + theValue = node.doubleValue(); + } else if (node.isFloatingPointNumber()) { + theValue = node.floatValue(); + } else if (node.isInt()) { + theValue = node.intValue(); + } else if (node.isLong()) { + theValue = node.longValue(); + } else if (node.isTextual()) { + theValue = node.textValue(); + } else if (node.isArray()) { + // this is an array so just send it back so that it can be + // reprocessed to its primitive components + theValue = node; + } else if (node.isObject()) { + // this is an object so just send it back so that it can be + // reprocessed to its primitive components + theValue = node; + } else { + theValue = node.textValue(); + } + } + + return theValue; + } +} \ No newline at end of file diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraph.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraph.java index 9541cdf1ef4..bd5953fd086 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraph.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraph.java @@ -286,7 +286,7 @@ public VertexProperty property(final String key, final V value, final Obj return this.property(VertexProperty.Cardinality.single, key, value, keyValues); } - Edge addOutEdge(final String label, final Vertex inVertex, final Object... keyValues) { + public Edge addOutEdge(final String label, final Vertex inVertex, final Object... keyValues) { ElementHelper.validateLabel(label); ElementHelper.legalPropertyKeyValueArray(keyValues); if (null == this.outEdges) @@ -302,7 +302,7 @@ Edge addOutEdge(final String label, final Vertex inVertex, final Object... keyVa return outEdge; } - Edge addInEdge(final String label, final Vertex outVertex, final Object... keyValues) { + public Edge addInEdge(final String label, final Vertex outVertex, final Object... keyValues) { ElementHelper.validateLabel(label); ElementHelper.legalPropertyKeyValueArray(keyValues); if (null == this.inEdges) diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java index ed38f468fba..af66d111aaf 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java @@ -20,7 +20,7 @@ public final class GraphSONLegacyInputFormat extends FileInputFormat createRecordReader(final InputSplit split, final TaskAttemptContext context) throws IOException, InterruptedException { - RecordReader reader = new GraphSONRecordReader(); + RecordReader reader = new GraphSONLegacyRecordReader(); reader.initialize(split, context); return reader; } From 04f3b993c24cd5dbd38f16c1a582f788ecc1cefe Mon Sep 17 00:00:00 2001 From: ebice Date: Mon, 22 Jun 2015 10:52:16 -0400 Subject: [PATCH 03/13] Added missing Apache license --- .../io/graphson/GraphSONTokensTP2.java | 18 ++++++++++++++++++ .../io/graphson/LegacyGraphSONUtility.java | 18 ++++++++++++++++++ .../GraphSONLegacyRecordReaderTest.java | 19 +++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java index c7575a151a8..479b473635a 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.tinkerpop.gremlin.structure.io.graphson; public final class GraphSONTokensTP2 { diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java index 5d306b2c6e7..a8fedf32217 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.tinkerpop.gremlin.structure.io.graphson; import com.fasterxml.jackson.databind.JsonNode; diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java index 75608def801..c3f413f2292 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson; import junit.framework.TestCase; From 1d2f30cb7c7f1f1b5f5c3a4e09e1c5a6fec1ffae Mon Sep 17 00:00:00 2001 From: ebice Date: Mon, 22 Jun 2015 10:54:08 -0400 Subject: [PATCH 04/13] Added missing Apache license --- .../io/graphson/GraphSONLegacyInputFormat.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java index af66d111aaf..b0471ba163e 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyInputFormat.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson; import org.apache.hadoop.fs.Path; From 207416a33c179eb845706e012cd9f49ba272d06a Mon Sep 17 00:00:00 2001 From: ebice Date: Tue, 23 Jun 2015 15:12:04 -0400 Subject: [PATCH 05/13] I needed a test resource in the legacy GraphSON adjacency list format and went with the following http://s3.thinkaurelius.com/docs/titan/0.5.4/hadoop-getting-started.html#_graph_of_the_gods_with_hadoop Not sure if it's a good idea to (re)introduce this data set. May be better to convert one of the others, say Grateful Dead, into the legacy GraphSON adjacency list format. --- data/graph-of-gods.json | 12 ++++++++++ .../tinkerpop/gremlin/LoadGraphWith.java | 17 ++++++++++++- .../gremlin/hadoop/HadoopGraphProvider.java | 21 +++++----------- .../GraphSONLegacyRecordReaderTest.java | 24 ++++++++++++++++--- 4 files changed, 55 insertions(+), 19 deletions(-) create mode 100644 data/graph-of-gods.json diff --git a/data/graph-of-gods.json b/data/graph-of-gods.json new file mode 100644 index 00000000000..65b5d212bf1 --- /dev/null +++ b/data/graph-of-gods.json @@ -0,0 +1,12 @@ +{"name":"saturn","type":"titan","_id":0,"_inE":[{"_label":"father","_id":12,"_outV":1}]} +{"name":"jupiter","type":"god","_id":1,"_outE":[{"_label":"lives","_id":13,"_inV":4},{"_label":"brother","_id":16,"_inV":3},{"_label":"brother","_id":14,"_inV":2},{"_label":"father","_id":12,"_inV":0}],"_inE":[{"_label":"brother","_id":17,"_outV":3},{"_label":"brother","_id":15,"_outV":2},{"_label":"father","_id":24,"_outV":7}]} +{"name":"neptune","type":"god","_id":2,"_outE":[{"_label":"lives","_id":20,"_inV":5},{"_label":"brother","_id":19,"_inV":3},{"_label":"brother","_id":15,"_inV":1}],"_inE":[{"_label":"brother","_id":18,"_outV":3},{"_label":"brother","_id":14,"_outV":1}]} +{"name":"pluto","type":"god","_id":3,"_outE":[{"_label":"pet","_id":23,"_inV":11},{"_label":"lives","_id":21,"_inV":6},{"_label":"brother","_id":17,"_inV":1},{"_label":"brother","_id":18,"_inV":2}],"_inE":[{"_label":"brother","_id":19,"_outV":2},{"_label":"brother","_id":16,"_outV":1}]} +{"name":"sky","type":"location","_id":4,"_inE":[{"_label":"lives","_id":13,"_outV":1}]} +{"name":"sea","type":"location","_id":5,"_inE":[{"_label":"lives","_id":20,"_outV":2}]} +{"name":"tartarus","type":"location","_id":6,"_inE":[{"_label":"lives","_id":21,"_outV":3},{"_label":"lives","_id":22,"_outV":11}]} +{"name":"hercules","type":"demigod","_id":7,"_outE":[{"_label":"mother","_id":25,"_inV":8},{"time":1,"_label":"battled","_id":26,"_inV":9},{"time":2,"_label":"battled","_id":27,"_inV":10},{"time":12,"_label":"battled","_id":28,"_inV":11},{"_label":"father","_id":24,"_inV":1}]} +{"name":"alcmene","type":"human","_id":8,"_inE":[{"_label":"mother","_id":25,"_outV":7}]} +{"name":"nemean","type":"monster","_id":9,"_inE":[{"time":1,"_label":"battled","_id":26,"_outV":7}]} +{"name":"hydra","type":"monster","_id":10,"_inE":[{"time":2,"_label":"battled","_id":27,"_outV":7}]} +{"name":"cerberus","type":"monster","_id":11,"_outE":[{"_label":"lives","_id":22,"_inV":6}],"_inE":[{"_label":"pet","_id":23,"_outV":3},{"time":12,"_label":"battled","_id":28,"_outV":7}]} \ No newline at end of file diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java index 17893e5ee7a..944a972bc0a 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java @@ -77,7 +77,12 @@ public enum GraphData { * Loads the "grateful dead" graph which is a "large" graph which provides for the construction of more * complex traversals. */ - GRATEFUL; + GRATEFUL, + + /** + * Loads the "Graph of the Gods" graph which is a "toy" graph with 12 vertices and 17 edges. + */ + GODS; private static final List featuresRequiredByClassic = new ArrayList() {{ add(FeatureRequirement.Factory.create(FEATURE_STRING_VALUES, VertexPropertyFeatures.class)); @@ -104,6 +109,14 @@ public enum GraphData { add(FeatureRequirement.Factory.create(FEATURE_INTEGER_VALUES, VertexPropertyFeatures.class)); }}; + private static final List featuresRequiredByGods = new ArrayList() {{ + add(FeatureRequirement.Factory.create(FEATURE_STRING_VALUES, VertexPropertyFeatures.class)); + add(FeatureRequirement.Factory.create(FEATURE_INTEGER_VALUES, VertexPropertyFeatures.class)); + add(FeatureRequirement.Factory.create(FEATURE_BOOLEAN_VALUES, VertexPropertyFeatures.class)); + add(FeatureRequirement.Factory.create(FEATURE_META_PROPERTIES, Graph.Features.VertexFeatures.class)); + add(FeatureRequirement.Factory.create(FEATURE_MULTI_PROPERTIES, Graph.Features.VertexFeatures.class)); + }}; + public String location() { switch (this) { case CLASSIC: @@ -129,6 +142,8 @@ public List featuresRequired() { return featuresRequiredByModern; case GRATEFUL: return featuresRequiredByGrateful; + case GODS: + return featuresRequiredByGods; } throw new RuntimeException("No features for this GraphData type"); diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java index 27b84f0e4d1..65bb87b7ca2 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java @@ -20,16 +20,10 @@ import org.apache.commons.configuration.Configuration; import org.apache.giraph.conf.GiraphConstants; -import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; import org.apache.tinkerpop.gremlin.AbstractGraphProvider; import org.apache.tinkerpop.gremlin.LoadGraphWith; import org.apache.tinkerpop.gremlin.TestHelper; -import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge; -import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement; -import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph; -import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty; -import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex; -import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty; +import org.apache.tinkerpop.gremlin.hadoop.structure.*; import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat; import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat; @@ -38,13 +32,7 @@ import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess; import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; +import java.util.*; import java.util.concurrent.TimeUnit; /** @@ -82,7 +70,8 @@ public class HadoopGraphProvider extends AbstractGraphProvider { "tinkerpop-modern.json", "grateful-dead.json", "tinkerpop-classic.json", - "tinkerpop-crew.json"); + "tinkerpop-crew.json", + "graph-of-gods.json"); for (final String fileName : graphsonResources) { PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath()); } @@ -159,6 +148,8 @@ public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.Grap ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type)); } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type)); + } else if (graphData.equals(LoadGraphWith.GraphData.GODS) && type.equals("json")) { + ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("graph-of-gods." + type)); } else { throw new RuntimeException("Could not load graph with " + graphData); } diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java index c3f413f2292..47fde1290fa 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java @@ -19,11 +19,29 @@ package org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson; -import junit.framework.TestCase; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.OutputFormat; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.RecordReaderWriterTest; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; /** - * Created by ebice on 6/18/2015. + * Created by Edi Bice on 6/18/2015. */ -public class GraphSONLegacyRecordReaderTest extends TestCase { +public class GraphSONLegacyRecordReaderTest extends RecordReaderWriterTest { + @Override + protected String getInputFilename() { + return "graph-of-gods.json"; + } + + @Override + protected Class> getInputFormat() { + return GraphSONLegacyInputFormat.class; + } + + @Override + protected Class> getOutputFormat() { + return GraphSONOutputFormat.class; + } } \ No newline at end of file From 2e1a195a4b7e2a435106abc49b7427371b424ce3 Mon Sep 17 00:00:00 2001 From: ebice Date: Tue, 30 Jun 2015 15:26:48 -0400 Subject: [PATCH 06/13] Stupid change to commit before pull & merge --- .../gremlin/structure/io/graphson/TP2GraphSONUtility.java | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java new file mode 100644 index 00000000000..faa77d83c5c --- /dev/null +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java @@ -0,0 +1,8 @@ +package org.apache.tinkerpop.gremlin.structure.io.graphson; + +/** + * Created by Edi Bice on 6/24/2015. + */ +public class TP2GraphSONUtility { + +} From 12cadd2a8d9a506d17613774dc1a298d653f12de Mon Sep 17 00:00:00 2001 From: ebice Date: Wed, 1 Jul 2015 16:50:16 -0400 Subject: [PATCH 07/13] Added GraphSONLegacyWriter and TP2GraphSONUtility ported over from TP2 GraphSONWriter and GraphSONUtility. --- gremlin-core/pom.xml | 5 + .../io/graphson/ElementPropertyConfig.java | 112 +++ .../io/graphson/GraphSONLegacyWriter.java | 242 +++++ .../io/graphson/TP2GraphSONUtility.java | 847 +++++++++++++++++- 4 files changed, 1204 insertions(+), 2 deletions(-) create mode 100644 gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/ElementPropertyConfig.java create mode 100644 gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java diff --git a/gremlin-core/pom.xml b/gremlin-core/pom.xml index ec15ba873ca..af6c342af99 100644 --- a/gremlin-core/pom.xml +++ b/gremlin-core/pom.xml @@ -103,6 +103,11 @@ limitations under the License. 3.2.1 test + + org.codehaus.jettison + jettison + 1.3.5 + ${basedir}/target diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/ElementPropertyConfig.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/ElementPropertyConfig.java new file mode 100644 index 00000000000..3c2756058e3 --- /dev/null +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/ElementPropertyConfig.java @@ -0,0 +1,112 @@ +package org.apache.tinkerpop.gremlin.structure.io.graphson; + + import java.util.ArrayList; + import java.util.Collections; + import java.util.List; + import java.util.Set; + +/** + * Configure how the GraphSON utility treats edge and vertex properties. + * + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public class ElementPropertyConfig { + + public static enum ElementPropertiesRule { + INCLUDE, EXCLUDE + } + + private final List vertexPropertyKeys; + private final List edgePropertyKeys; + private final ElementPropertiesRule vertexPropertiesRule; + private final ElementPropertiesRule edgePropertiesRule; + private final boolean normalized; + + /** + * A configuration that includes all properties of vertices and edges. + */ + public static final ElementPropertyConfig AllProperties = new ElementPropertyConfig(null, null, + ElementPropertiesRule.INCLUDE, ElementPropertiesRule.INCLUDE, false); + + + public ElementPropertyConfig(final Set vertexPropertyKeys, final Set edgePropertyKeys, + final ElementPropertiesRule vertexPropertiesRule, + final ElementPropertiesRule edgePropertiesRule) { + this(vertexPropertyKeys, edgePropertyKeys, vertexPropertiesRule, edgePropertiesRule, false); + } + + public ElementPropertyConfig(final Set vertexPropertyKeys, final Set edgePropertyKeys, + final ElementPropertiesRule vertexPropertiesRule, + final ElementPropertiesRule edgePropertiesRule, final boolean normalized) { + this.vertexPropertiesRule = vertexPropertiesRule; + this.vertexPropertyKeys = sortKeys(vertexPropertyKeys, normalized); + this.edgePropertiesRule = edgePropertiesRule; + this.edgePropertyKeys = sortKeys(edgePropertyKeys, normalized); + this.normalized = normalized; + } + + /** + * Construct a configuration that includes the specified properties from both vertices and edges. + */ + public static ElementPropertyConfig includeProperties(final Set vertexPropertyKeys, + final Set edgePropertyKeys) { + return new ElementPropertyConfig(vertexPropertyKeys, edgePropertyKeys, ElementPropertiesRule.INCLUDE, + ElementPropertiesRule.INCLUDE); + } + + public static ElementPropertyConfig includeProperties(final Set vertexPropertyKeys, + final Set edgePropertyKeys, final boolean normalized) { + return new ElementPropertyConfig(vertexPropertyKeys, edgePropertyKeys, ElementPropertiesRule.INCLUDE, + ElementPropertiesRule.INCLUDE, normalized); + } + + /** + * Construct a configuration that excludes the specified properties from both vertices and edges. + */ + public static ElementPropertyConfig excludeProperties(final Set vertexPropertyKeys, + final Set edgePropertyKeys) { + return new ElementPropertyConfig(vertexPropertyKeys, edgePropertyKeys, ElementPropertiesRule.EXCLUDE, + ElementPropertiesRule.EXCLUDE); + } + + public static ElementPropertyConfig excludeProperties(final Set vertexPropertyKeys, + final Set edgePropertyKeys, final boolean normalized) { + return new ElementPropertyConfig(vertexPropertyKeys, edgePropertyKeys, ElementPropertiesRule.EXCLUDE, + ElementPropertiesRule.EXCLUDE, normalized); + } + + public List getVertexPropertyKeys() { + return vertexPropertyKeys; + } + + public List getEdgePropertyKeys() { + return edgePropertyKeys; + } + + public ElementPropertiesRule getVertexPropertiesRule() { + return vertexPropertiesRule; + } + + public ElementPropertiesRule getEdgePropertiesRule() { + return edgePropertiesRule; + } + + public boolean isNormalized() { + return normalized; + } + + private static List sortKeys(final Set keys, final boolean normalized) { + final List propertyKeyList; + if (keys != null) { + if (normalized) { + final List sorted = new ArrayList(keys); + Collections.sort(sorted); + propertyKeyList = sorted; + } else + propertyKeyList = new ArrayList(keys); + } else + propertyKeyList = null; + + return propertyKeyList; + } +} \ No newline at end of file diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java new file mode 100644 index 00000000000..5f5e137fb37 --- /dev/null +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java @@ -0,0 +1,242 @@ +package org.apache.tinkerpop.gremlin.structure.io.graphson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.MappingJsonFactory; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Element; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.io.graphson.TP2GraphSONUtility.GraphSONMode; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.*; + +/** + * GraphSONWriter writes a Graph to a TinkerPop JSON OutputStream. + * + * @author Edi Bice + * @author Stephen Mallette + */ +public class GraphSONLegacyWriter { + + /** + * Elements are sorted in lexicographical order of IDs. + */ + public class LexicographicalElementComparator implements Comparator { + + @Override + public int compare(final Element a, final Element b) { + return a.id().toString().compareTo(b.id().toString()); + } + } + + private static final JsonFactory jsonFactory = new MappingJsonFactory(); + private final Graph graph; + + /** + * @param graph the Graph to pull the data from + */ + public GraphSONLegacyWriter(final Graph graph) { + this.graph = graph; + } + + /** + * Write the data in a Graph to a JSON OutputStream. + * + * @param filename the JSON file to write the Graph data to + * @param vertexPropertyKeys the keys of the vertex elements to write to JSON + * @param edgePropertyKeys the keys of the edge elements to write to JSON + * @param mode determines the format of the GraphSON + * @throws IOException thrown if there is an error generating the JSON data + */ + public void outputGraph(final String filename, final Set vertexPropertyKeys, + final Set edgePropertyKeys, final GraphSONMode mode) throws IOException { + final FileOutputStream fos = new FileOutputStream(filename); + outputGraph(fos, vertexPropertyKeys, edgePropertyKeys, mode); + fos.close(); + } + + /** + * Write the data in a Graph to a JSON OutputStream. + * + * @param jsonOutputStream the JSON OutputStream to write the Graph data to + * @param vertexPropertyKeys the keys of the vertex elements to write to JSON + * @param edgePropertyKeys the keys of the edge elements to write to JSON + * @param mode determines the format of the GraphSON + * @throws IOException thrown if there is an error generating the JSON data + */ + public void outputGraph(final OutputStream jsonOutputStream, final Set vertexPropertyKeys, + final Set edgePropertyKeys, final GraphSONMode mode) throws IOException { + outputGraph(jsonOutputStream, vertexPropertyKeys, edgePropertyKeys, mode, false); + } + + + public void outputGraph(final OutputStream jsonOutputStream, final Set vertexPropertyKeys, + final Set edgePropertyKeys, final GraphSONMode mode, final boolean normalize) throws IOException { + final JsonGenerator jg = jsonFactory.createGenerator(jsonOutputStream); + + // don't let the JsonGenerator close the underlying stream...leave that to the client passing in the stream + jg.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); + + final TP2GraphSONUtility graphson = new TP2GraphSONUtility(mode, null, + ElementPropertyConfig.includeProperties(vertexPropertyKeys, edgePropertyKeys, normalize)); + + jg.writeStartObject(); + + jg.writeStringField(GraphSONTokensTP2.MODE, mode.toString()); + + jg.writeArrayFieldStart(GraphSONTokensTP2.VERTICES); + + final Iterable vertices = vertices(normalize); + for (Vertex v : vertices) { + jg.writeTree(graphson.objectNodeFromElement(v)); + } + + jg.writeEndArray(); + + jg.writeArrayFieldStart(GraphSONTokensTP2.EDGES); + + final Iterable edges = edges(normalize); + for (Edge e : edges) { + jg.writeTree(graphson.objectNodeFromElement(e)); + } + jg.writeEndArray(); + + jg.writeEndObject(); + + jg.flush(); + jg.close(); + } + + private Iterable vertices(boolean normalize) { + Iterable vertices; + if (normalize) { + vertices = new ArrayList(); + Iterator viter = graph.vertices(null); + while (viter.hasNext()) { + ((Collection) vertices).add(viter.next()); + } + Collections.sort((List) vertices, new LexicographicalElementComparator()); + } else { + vertices = new Iterable() { + @Override + public Iterator iterator() { + return graph.vertices(null); + } + }; + } + return vertices; + } + + private Iterable edges(boolean normalize) { + Iterable edges; + if (normalize) { + edges = new ArrayList(); + Iterator eiter = graph.edges(null); + while (eiter.hasNext()) { + ((Collection) edges).add(eiter.next()); + } + Collections.sort((List) edges, new LexicographicalElementComparator()); + } else { + edges = new Iterable() { + @Override + public Iterator iterator() { + return graph.edges(null); + } + }; + } + return edges; + } + + /** + * Write the data in a Graph to a JSON OutputStream. All keys are written to JSON. Utilizing + * GraphSONMode.NORMAL. + * + * @param graph the graph to serialize to JSON + * @param jsonOutputStream the JSON OutputStream to write the Graph data to + * @throws IOException thrown if there is an error generating the JSON data + */ + public static void outputGraph(final Graph graph, final OutputStream jsonOutputStream) throws IOException { + final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); + writer.outputGraph(jsonOutputStream, null, null, GraphSONMode.NORMAL); + } + + /** + * Write the data in a Graph to a JSON OutputStream. All keys are written to JSON. Utilizing + * GraphSONMode.NORMAL. + * + * @param graph the graph to serialize to JSON + * @param filename the JSON file to write the Graph data to + * @throws IOException thrown if there is an error generating the JSON data + */ + public static void outputGraph(final Graph graph, final String filename) throws IOException { + final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); + writer.outputGraph(filename, null, null, GraphSONMode.NORMAL); + } + + /** + * Write the data in a Graph to a JSON OutputStream. All keys are written to JSON. + * + * @param graph the graph to serialize to JSON + * @param jsonOutputStream the JSON OutputStream to write the Graph data to + * @param mode determines the format of the GraphSON + * @throws IOException thrown if there is an error generating the JSON data + */ + public static void outputGraph(final Graph graph, final OutputStream jsonOutputStream, + final GraphSONMode mode) throws IOException { + final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); + writer.outputGraph(jsonOutputStream, null, null, mode); + } + + /** + * Write the data in a Graph to a JSON OutputStream. All keys are written to JSON. + * + * @param graph the graph to serialize to JSON + * @param filename the JSON file to write the Graph data to + * @param mode determines the format of the GraphSON + * @throws IOException thrown if there is an error generating the JSON data + */ + public static void outputGraph(final Graph graph, final String filename, + final GraphSONMode mode) throws IOException { + final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); + writer.outputGraph(filename, null, null, mode); + } + + /** + * Write the data in a Graph to a JSON OutputStream. + * + * @param graph the graph to serialize to JSON + * @param jsonOutputStream the JSON OutputStream to write the Graph data to + * @param vertexPropertyKeys the keys of the vertex elements to write to JSON + * @param edgePropertyKeys the keys of the edge elements to write to JSON + * @param mode determines the format of the GraphSON + * @throws IOException thrown if there is an error generating the JSON data + */ + public static void outputGraph(final Graph graph, final OutputStream jsonOutputStream, + final Set vertexPropertyKeys, final Set edgePropertyKeys, + final GraphSONMode mode) throws IOException { + final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); + writer.outputGraph(jsonOutputStream, vertexPropertyKeys, edgePropertyKeys, mode); + } + + /** + * Write the data in a Graph to a JSON OutputStream. + * + * @param graph the graph to serialize to JSON + * @param filename the JSON file to write the Graph data to + * @param vertexPropertyKeys the keys of the vertex elements to write to JSON + * @param edgePropertyKeys the keys of the edge elements to write to JSON + * @param mode determines the format of the GraphSON + * @throws IOException thrown if there is an error generating the JSON data + */ + public static void outputGraph(final Graph graph, final String filename, + final Set vertexPropertyKeys, final Set edgePropertyKeys, + final GraphSONMode mode) throws IOException { + final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); + writer.outputGraph(filename, vertexPropertyKeys, edgePropertyKeys, mode); + } + +} \ No newline at end of file diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java index faa77d83c5c..73e895fab6b 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java @@ -1,8 +1,851 @@ package org.apache.tinkerpop.gremlin.structure.io.graphson; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MappingJsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Element; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.io.graphson.ElementPropertyConfig.ElementPropertiesRule; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.codehaus.jettison.json.JSONTokener; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Array; +import java.util.*; + +//import com.tinkerpop.blueprints.Direction; +//import com.tinkerpop.blueprints.Element; + /** - * Created by Edi Bice on 6/24/2015. + * Helps write individual graph elements to TinkerPop JSON format known as GraphSON. + * + * @author Stephen Mallette (http://stephen.genoprime.com) */ public class TP2GraphSONUtility { -} + /** + * Modes of operation of the GraphSONUtility. + * + * @author Stephen Mallette + */ + public enum GraphSONMode { + /** + * COMPACT constructs GraphSON on the assumption that all property keys + * are fair game for exclusion including _type, _inV, _outV, _label and _id. + * It is possible to write GraphSON that cannot be read back into Graph, + * if some or all of these keys are excluded. + */ + COMPACT, + + /** + * NORMAL includes the _type field and JSON data typing. + */ + NORMAL, + + /** + * EXTENDED includes the _type field and explicit data typing. + */ + EXTENDED + } + + /** + * A factory responsible for creating graph elements. Abstracts the way that graph elements are created. In + * most cases a Graph is responsible for element creation, but there are cases where more control over + * how vertices and edges are constructed. + * + * @author Stephen Mallette (http://stephen.genoprime.com) + */ + public interface ElementFactory { + /** + * Creates a new Edge instance. + */ + E createEdge(final Object id, final V out, final V in, final String label); + + /** + * Creates a new Vertex instance. + */ + V createVertex(final Object id); + } + + private static final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.instance; + private static final JsonFactory jsonFactory = new MappingJsonFactory(); + private static final String EMPTY_STRING = ""; + + private static final ObjectMapper mapper = new ObjectMapper(); + + private final GraphSONMode mode; + private final List vertexPropertyKeys; + private final List edgePropertyKeys; + private final ElementFactory factory; + private final boolean hasEmbeddedTypes; + private final ElementPropertiesRule vertexPropertiesRule; + private final ElementPropertiesRule edgePropertiesRule; + private final boolean normalized; + + private final boolean includeReservedVertexId; + private final boolean includeReservedEdgeId; + private final boolean includeReservedVertexType; + private final boolean includeReservedEdgeType; + private final boolean includeReservedEdgeLabel; + private final boolean includeReservedEdgeOutV; + private final boolean includeReservedEdgeInV; + + /** + * A GraphSONUtiltiy that includes all properties of vertices and edges. + */ + public TP2GraphSONUtility(final GraphSONMode mode, final ElementFactory factory) { + this(mode, factory, ElementPropertyConfig.AllProperties); + } + + /** + * A GraphSONUtility that includes the specified properties. + */ + public TP2GraphSONUtility(final GraphSONMode mode, final ElementFactory factory, + final Set vertexPropertyKeys, final Set edgePropertyKeys) { + this(mode, factory, ElementPropertyConfig.includeProperties(vertexPropertyKeys, edgePropertyKeys)); + } + + public TP2GraphSONUtility(final GraphSONMode mode, final ElementFactory factory, + final ElementPropertyConfig config) { + this.vertexPropertyKeys = config.getVertexPropertyKeys(); + this.edgePropertyKeys = config.getEdgePropertyKeys(); + this.vertexPropertiesRule = config.getVertexPropertiesRule(); + this.edgePropertiesRule = config.getEdgePropertiesRule(); + this.normalized = config.isNormalized(); + + this.mode = mode; + this.factory = factory; + this.hasEmbeddedTypes = mode == GraphSONMode.EXTENDED; + + this.includeReservedVertexId = includeReservedKey(mode, GraphSONTokensTP2._ID, vertexPropertyKeys, this.vertexPropertiesRule); + this.includeReservedEdgeId = includeReservedKey(mode, GraphSONTokensTP2._ID, edgePropertyKeys, this.edgePropertiesRule); + this.includeReservedVertexType = includeReservedKey(mode, GraphSONTokensTP2._TYPE, vertexPropertyKeys, this.vertexPropertiesRule); + this.includeReservedEdgeType = includeReservedKey(mode, GraphSONTokensTP2._TYPE, edgePropertyKeys, this.edgePropertiesRule); + this.includeReservedEdgeLabel = includeReservedKey(mode, GraphSONTokensTP2._LABEL, edgePropertyKeys, this.edgePropertiesRule); + this.includeReservedEdgeOutV = includeReservedKey(mode, GraphSONTokensTP2._OUT_V, edgePropertyKeys, this.edgePropertiesRule); + this.includeReservedEdgeInV = includeReservedKey(mode, GraphSONTokensTP2._IN_V, edgePropertyKeys, this.edgePropertiesRule); + } + + /** + * Creates a vertex from GraphSON using settings supplied in the constructor. + */ + public Vertex vertexFromJson(final JSONObject json) throws IOException { + return this.vertexFromJson(json.toString()); + } + + /** + * Creates a vertex from GraphSON using settings supplied in the constructor. + */ + public Vertex vertexFromJson(final String json) throws IOException { + final JsonParser jp = jsonFactory.createParser(json); + final JsonNode node = jp.readValueAsTree(); + return this.vertexFromJson(node); + } + + /** + * Creates a vertex from GraphSON using settings supplied in the constructor. + */ + public Vertex vertexFromJson(final InputStream json) throws IOException { + final JsonParser jp = jsonFactory.createParser(json); + final JsonNode node = jp.readValueAsTree(); + return this.vertexFromJson(node); + } + + /** + * Creates a vertex from GraphSON using settings supplied in the constructor. + */ + public Vertex vertexFromJson(final JsonNode json) throws IOException { + final Map props = readProperties(json, true, this.hasEmbeddedTypes); + + final Object vertexId = getTypedValueFromJsonNode(json.get(GraphSONTokensTP2._ID)); + final Vertex v = factory.createVertex(vertexId); + + for (Map.Entry entry : props.entrySet()) { + //if (this.vertexPropertyKeys == null || vertexPropertyKeys.contains(entry.getKey())) { + if (includeKey(entry.getKey(), vertexPropertyKeys, this.vertexPropertiesRule)) { + v.property(entry.getKey(), entry.getValue()); + } + } + + return v; + } + + /** + * Creates an edge from GraphSON using settings supplied in the constructor. + */ + public Edge edgeFromJson(final JSONObject json, final Vertex out, final Vertex in) throws IOException { + return this.edgeFromJson(json.toString(), out, in); + } + + /** + * Creates an edge from GraphSON using settings supplied in the constructor. + */ + public Edge edgeFromJson(final String json, final Vertex out, final Vertex in) throws IOException { + final JsonParser jp = jsonFactory.createParser(json); + final JsonNode node = jp.readValueAsTree(); + return this.edgeFromJson(node, out, in); + } + + /** + * Creates an edge from GraphSON using settings supplied in the constructor. + */ + public Edge edgeFromJson(final InputStream json, final Vertex out, final Vertex in) throws IOException { + final JsonParser jp = jsonFactory.createParser(json); + final JsonNode node = jp.readValueAsTree(); + return this.edgeFromJson(node, out, in); + } + + /** + * Creates an edge from GraphSON using settings supplied in the constructor. + */ + public Edge edgeFromJson(final JsonNode json, final Vertex out, final Vertex in) throws IOException { + final Map props = TP2GraphSONUtility.readProperties(json, true, this.hasEmbeddedTypes); + + final Object edgeId = getTypedValueFromJsonNode(json.get(GraphSONTokensTP2._ID)); + final JsonNode nodeLabel = json.get(GraphSONTokensTP2._LABEL); + + // assigned an empty string edge label in cases where one does not exist. this gets around the requirement + // that blueprints graphs have a non-null label while ensuring that GraphSON can stay flexible in parsing + // partial bits from the JSON. Not sure if there is any gotchas developing out of this. + final String label = nodeLabel == null ? EMPTY_STRING : nodeLabel.textValue(); + + final Edge e = factory.createEdge(edgeId, out, in, label); + + for (Map.Entry entry : props.entrySet()) { + // if (this.edgePropertyKeys == null || this.edgePropertyKeys.contains(entry.getKey())) { + if (includeKey(entry.getKey(), edgePropertyKeys, this.edgePropertiesRule)) { + e.property(entry.getKey(), entry.getValue()); + } + } + + return e; + } + + /** + * Creates GraphSON for a single graph element. + */ + public JSONObject jsonFromElement(final Element element) throws JSONException { + final ObjectNode objectNode = this.objectNodeFromElement(element); + + try { + return new JSONObject(new JSONTokener(mapper.writeValueAsString(objectNode))); + } catch (IOException ioe) { + // repackage this as a JSONException...seems sensible as the caller will only know about + // the jettison object not being created + throw new JSONException(ioe); + } + } + + /** + * Creates GraphSON for a single graph element. + */ + public ObjectNode objectNodeFromElement(final Element element) { + final boolean isEdge = element instanceof Edge; + final boolean showTypes = mode == GraphSONMode.EXTENDED; + final List propertyKeys = isEdge ? this.edgePropertyKeys : this.vertexPropertyKeys; + final ElementPropertiesRule elementPropertyConfig = isEdge ? this.edgePropertiesRule : this.vertexPropertiesRule; + + final ObjectNode jsonElement = createJSONMap(createPropertyMap(element, propertyKeys, elementPropertyConfig, normalized), propertyKeys, showTypes); + + if ((isEdge && this.includeReservedEdgeId) || (!isEdge && this.includeReservedVertexId)) { + putObject(jsonElement, GraphSONTokensTP2._ID, element.id()); + } + + // it's important to keep the order of these straight. check Edge first and then Vertex because there + // are graph implementations that have Edge extend from Vertex + if (element instanceof Edge) { + final Edge edge = (Edge) element; + + if (this.includeReservedEdgeId) { + putObject(jsonElement, GraphSONTokensTP2._ID, element.id()); + } + + if (this.includeReservedEdgeType) { + jsonElement.put(GraphSONTokensTP2._TYPE, GraphSONTokens.EDGE); + } + + if (this.includeReservedEdgeOutV) { + putObject(jsonElement, GraphSONTokensTP2._OUT_V, edge.outVertex().id()); + } + + if (this.includeReservedEdgeInV) { + putObject(jsonElement, GraphSONTokensTP2._IN_V, edge.inVertex().id()); + } + + if (this.includeReservedEdgeLabel) { + jsonElement.put(GraphSONTokensTP2._LABEL, edge.label()); + } + } else if (element instanceof Vertex) { + if (this.includeReservedVertexId) { + putObject(jsonElement, GraphSONTokensTP2._ID, element.id()); + } + + if (this.includeReservedVertexType) { + jsonElement.put(GraphSONTokensTP2._TYPE, GraphSONTokens.VERTEX); + } + } + + return jsonElement; + } + + /** + * Reads an individual Vertex from JSON. The vertex must match the accepted GraphSON format. + * + * @param json a single vertex in GraphSON format as Jettison JSONObject + * @param factory the factory responsible for constructing graph elements + * @param mode the mode of the GraphSON + * @param propertyKeys a list of keys to include on reading of element properties + */ + public static Vertex vertexFromJson(final JSONObject json, final ElementFactory factory, final GraphSONMode mode, + final Set propertyKeys) throws IOException { + final TP2GraphSONUtility graphson = new TP2GraphSONUtility(mode, factory, propertyKeys, null); + return graphson.vertexFromJson(json); + } + + /** + * Reads an individual Vertex from JSON. The vertex must match the accepted GraphSON format. + * + * @param json a single vertex in GraphSON format as a String. + * @param factory the factory responsible for constructing graph elements + * @param mode the mode of the GraphSON + * @param propertyKeys a list of keys to include on reading of element properties + */ + public static Vertex vertexFromJson(final String json, final ElementFactory factory, final GraphSONMode mode, + final Set propertyKeys) throws IOException { + final TP2GraphSONUtility graphson = new TP2GraphSONUtility(mode, factory, propertyKeys, null); + return graphson.vertexFromJson(json); + } + + /** + * Reads an individual Vertex from JSON. The vertex must match the accepted GraphSON format. + * + * @param json a single vertex in GraphSON format as an InputStream. + * @param factory the factory responsible for constructing graph elements + * @param mode the mode of the GraphSON + * @param propertyKeys a list of keys to include on reading of element properties + */ + public static Vertex vertexFromJson(final InputStream json, final ElementFactory factory, final GraphSONMode mode, + final Set propertyKeys) throws IOException { + final TP2GraphSONUtility graphson = new TP2GraphSONUtility(mode, factory, propertyKeys, null); + return graphson.vertexFromJson(json); + } + + /** + * Reads an individual Vertex from JSON. The vertex must match the accepted GraphSON format. + * + * @param json a single vertex in GraphSON format as Jackson JsonNode + * @param factory the factory responsible for constructing graph elements + * @param mode the mode of the GraphSON + * @param propertyKeys a list of keys to include on reading of element properties + */ + public static Vertex vertexFromJson(final JsonNode json, final ElementFactory factory, final GraphSONMode mode, + final Set propertyKeys) throws IOException { + final TP2GraphSONUtility graphson = new TP2GraphSONUtility(mode, factory, propertyKeys, null); + return graphson.vertexFromJson(json); + } + + /** + * Reads an individual Edge from JSON. The edge must match the accepted GraphSON format. + * + * @param json a single edge in GraphSON format as a Jettison JSONObject + * @param factory the factory responsible for constructing graph elements + * @param mode the mode of the GraphSON + * @param propertyKeys a list of keys to include when reading of element properties + */ + public static Edge edgeFromJson(final JSONObject json, final Vertex out, final Vertex in, + final ElementFactory factory, final GraphSONMode mode, + final Set propertyKeys) throws IOException { + final TP2GraphSONUtility graphson = new TP2GraphSONUtility(mode, factory, null, propertyKeys); + return graphson.edgeFromJson(json, out, in); + } + + /** + * Reads an individual Edge from JSON. The edge must match the accepted GraphSON format. + * + * @param json a single edge in GraphSON format as a String + * @param factory the factory responsible for constructing graph elements + * @param mode the mode of the GraphSON + * @param propertyKeys a list of keys to include when reading of element properties + */ + public static Edge edgeFromJson(final String json, final Vertex out, final Vertex in, + final ElementFactory factory, final GraphSONMode mode, + final Set propertyKeys) throws IOException { + final TP2GraphSONUtility graphson = new TP2GraphSONUtility(mode, factory, null, propertyKeys); + return graphson.edgeFromJson(json, out, in); + } + + /** + * Reads an individual Edge from JSON. The edge must match the accepted GraphSON format. + * + * @param json a single edge in GraphSON format as an InputStream + * @param factory the factory responsible for constructing graph elements + * @param mode the mode of the GraphSON + * @param propertyKeys a list of keys to include when reading of element properties + */ + public static Edge edgeFromJson(final InputStream json, final Vertex out, final Vertex in, + final ElementFactory factory, final GraphSONMode mode, + final Set propertyKeys) throws IOException { + final TP2GraphSONUtility graphson = new TP2GraphSONUtility(mode, factory, null, propertyKeys); + return graphson.edgeFromJson(json, out, in); + } + + /** + * Reads an individual Edge from JSON. The edge must match the accepted GraphSON format. + * + * @param json a single edge in GraphSON format as a Jackson JsonNode + * @param factory the factory responsible for constructing graph elements + * @param mode the mode of the GraphSON + * @param propertyKeys a list of keys to include when reading of element properties + */ + public static Edge edgeFromJson(final JsonNode json, final Vertex out, final Vertex in, + final ElementFactory factory, final GraphSONMode mode, + final Set propertyKeys) throws IOException { + final TP2GraphSONUtility graphson = new TP2GraphSONUtility(mode, factory, null, propertyKeys); + return graphson.edgeFromJson(json, out, in); + } + + /** + * Creates a Jettison JSONObject from a graph element. + * + * @param element the graph element to convert to JSON. + * @param propertyKeys The property keys at the root of the element to serialize. If null, then all keys are serialized. + * @param mode the type of GraphSON to be generated. + */ + public static JSONObject jsonFromElement(final Element element, final Set propertyKeys, + final GraphSONMode mode) throws JSONException { + final TP2GraphSONUtility graphson = element instanceof Edge ? new TP2GraphSONUtility(mode, null, null, propertyKeys) + : new TP2GraphSONUtility(mode, null, propertyKeys, null); + return graphson.jsonFromElement(element); + } + + /** + * Creates a Jackson ObjectNode from a graph element. + * + * @param element the graph element to convert to JSON. + * @param propertyKeys The property keys at the root of the element to serialize. If null, then all keys are serialized. + * @param mode The type of GraphSON to generate. + */ + public static ObjectNode objectNodeFromElement(final Element element, final Set propertyKeys, final GraphSONMode mode) { + final TP2GraphSONUtility graphson = element instanceof Edge ? new TP2GraphSONUtility(mode, null, null, propertyKeys) + : new TP2GraphSONUtility(mode, null, propertyKeys, null); + return graphson.objectNodeFromElement(element); + } + + private static ObjectNode objectNodeFromElement(final Element element, final List propertyKeys, final GraphSONMode mode) { + final TP2GraphSONUtility graphson = element instanceof Edge ? new TP2GraphSONUtility(mode, null, null, new HashSet(propertyKeys)) + : new TP2GraphSONUtility(mode, null, new HashSet(propertyKeys), null); + return graphson.objectNodeFromElement(element); + } + + static Map readProperties(final JsonNode node, final boolean ignoreReservedKeys, final boolean hasEmbeddedTypes) { + final Map map = new HashMap(); + + final Iterator> iterator = node.fields(); + while (iterator.hasNext()) { + final Map.Entry entry = iterator.next(); + + if (!ignoreReservedKeys || !isReservedKey(entry.getKey())) { + // it generally shouldn't be as such but graphson containing null values can't be shoved into + // element property keys or it will result in error + final Object o = readProperty(entry.getValue(), hasEmbeddedTypes); + if (o != null) { + map.put(entry.getKey(), o); + } + } + } + + return map; + } + + private static boolean includeReservedKey(final GraphSONMode mode, final String key, + final List propertyKeys, + final ElementPropertiesRule rule) { + // the key is always included in modes other than compact. if it is compact, then validate that the + // key is in the property key list + return mode != GraphSONMode.COMPACT || includeKey(key, propertyKeys, rule); + } + + private static boolean includeKey(final String key, final List propertyKeys, + final ElementPropertiesRule rule) { + if (propertyKeys == null) { + // when null always include the key and shortcut this piece + return true; + } + + // default the key situation. if it's included then it should be explicitly defined in the + // property keys list to be included or the reverse otherwise + boolean keySituation = rule == ElementPropertiesRule.INCLUDE; + + switch (rule) { + case INCLUDE: + keySituation = propertyKeys.contains(key); + break; + case EXCLUDE: + keySituation = !propertyKeys.contains(key); + break; + } + + return keySituation; + } + + private static boolean isReservedKey(final String key) { + return key.equals(GraphSONTokensTP2._ID) || key.equals(GraphSONTokensTP2._TYPE) || key.equals(GraphSONTokensTP2._LABEL) + || key.equals(GraphSONTokensTP2._OUT_V) || key.equals(GraphSONTokensTP2._IN_V); + } + + private static Object readProperty(final JsonNode node, final boolean hasEmbeddedTypes) { + final Object propertyValue; + + if (hasEmbeddedTypes) { + if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_UNKNOWN)) { + propertyValue = null; + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BOOLEAN)) { + propertyValue = node.get(GraphSONTokensTP2.VALUE).booleanValue(); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_FLOAT)) { + propertyValue = Float.parseFloat(node.get(GraphSONTokensTP2.VALUE).asText()); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BYTE)) { + propertyValue = Byte.parseByte(node.get(GraphSONTokensTP2.VALUE).asText()); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_SHORT)) { + propertyValue = Short.parseShort(node.get(GraphSONTokensTP2.VALUE).asText()); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_DOUBLE)) { + propertyValue = node.get(GraphSONTokensTP2.VALUE).doubleValue(); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_INTEGER)) { + propertyValue = node.get(GraphSONTokensTP2.VALUE).intValue(); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LONG)) { + propertyValue = node.get(GraphSONTokensTP2.VALUE).longValue(); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_STRING)) { + propertyValue = node.get(GraphSONTokensTP2.VALUE).textValue(); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LIST)) { + propertyValue = readProperties(node.get(GraphSONTokensTP2.VALUE).elements(), hasEmbeddedTypes); + } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_MAP)) { + propertyValue = readProperties(node.get(GraphSONTokensTP2.VALUE), false, hasEmbeddedTypes); + } else { + propertyValue = node.textValue(); + } + } else { + if (node.isNull()) { + propertyValue = null; + } else if (node.isBoolean()) { + propertyValue = node.booleanValue(); + } else if (node.isDouble()) { + propertyValue = node.doubleValue(); + } else if (node.isInt()) { + propertyValue = node.intValue(); + } else if (node.isLong()) { + propertyValue = node.longValue(); + } else if (node.isTextual()) { + propertyValue = node.textValue(); + } else if (node.isArray()) { + propertyValue = readProperties(node.elements(), hasEmbeddedTypes); + } else if (node.isObject()) { + propertyValue = readProperties(node, false, hasEmbeddedTypes); + } else { + propertyValue = node.textValue(); + } + } + + return propertyValue; + } + + private static List readProperties(final Iterator listOfNodes, final boolean hasEmbeddedTypes) { + final List array = new ArrayList(); + + while (listOfNodes.hasNext()) { + array.add(readProperty(listOfNodes.next(), hasEmbeddedTypes)); + } + + return array; + } + + private static ArrayNode createJSONList(final List list, final List propertyKeys, final boolean showTypes) { + final ArrayNode jsonList = jsonNodeFactory.arrayNode(); + for (Object item : list) { + if (item instanceof Element) { + jsonList.add(objectNodeFromElement((Element) item, propertyKeys, + showTypes ? GraphSONMode.EXTENDED : GraphSONMode.NORMAL)); + } else if (item instanceof List) { + jsonList.add(createJSONList((List) item, propertyKeys, showTypes)); + } else if (item instanceof Map) { + jsonList.add(createJSONMap((Map) item, propertyKeys, showTypes)); + } else if (item != null && item.getClass().isArray()) { + jsonList.add(createJSONList(convertArrayToList(item), propertyKeys, showTypes)); + } else { + addObject(jsonList, item); + } + } + return jsonList; + } + + private static ObjectNode createJSONMap(final Map map, final List propertyKeys, final boolean showTypes) { + final ObjectNode jsonMap = jsonNodeFactory.objectNode(); + for (Object key : map.keySet()) { + Object value = map.get(key); + if (value != null) { + if (value instanceof List) { + value = createJSONList((List) value, propertyKeys, showTypes); + } else if (value instanceof Iterable){ + value = createJSONList(getList((Iterable) value), propertyKeys, showTypes); + } else if (value instanceof Iterator){ + value = createJSONList(getList((Iterator) value), propertyKeys, showTypes); + } else if (value instanceof Map) { + value = createJSONMap((Map) value, propertyKeys, showTypes); + } else if (value instanceof Element) { + value = objectNodeFromElement((Element) value, propertyKeys, + showTypes ? GraphSONMode.EXTENDED : GraphSONMode.NORMAL); + } else if (value.getClass().isArray()) { + value = createJSONList(convertArrayToList(value), propertyKeys, showTypes); + } + } + + putObject(jsonMap, key.toString(), getValue(value, showTypes)); + } + return jsonMap; + + } + + private static List getList(final Iterable value) { + return getList(value.iterator()); + } + + private static List getList(final Iterator value) { + final List result = new ArrayList(); + while (value.hasNext()) { + result.add(value.next()); + } + return result; + } + + private static void addObject(final ArrayNode jsonList, final Object value) { + if (value == null) { + jsonList.add((JsonNode) null); + } else if (value.getClass() == Boolean.class) { + jsonList.add((Boolean) value); + } else if (value.getClass() == Long.class) { + jsonList.add((Long) value); + } else if (value.getClass() == Integer.class) { + jsonList.add((Integer) value); + } else if (value.getClass() == Float.class) { + jsonList.add((Float) value); + } else if (value.getClass() == Double.class) { + jsonList.add((Double) value); + } else if (value.getClass() == Byte.class) { + jsonList.add((Byte) value); + } else if (value.getClass() == Short.class) { + jsonList.add((Short) value); + } else if (value.getClass() == String.class) { + jsonList.add((String) value); + } else if (value instanceof ObjectNode) { + jsonList.add((ObjectNode) value); + } else if (value instanceof ArrayNode) { + jsonList.add((ArrayNode) value); + } else { + jsonList.add(value.toString()); + } + } + + private static void putObject(final ObjectNode jsonMap, final String key, final Object value) { + if (value == null) { + jsonMap.put(key, (JsonNode) null); + } else if (value.getClass() == Boolean.class) { + jsonMap.put(key, (Boolean) value); + } else if (value.getClass() == Long.class) { + jsonMap.put(key, (Long) value); + } else if (value.getClass() == Integer.class) { + jsonMap.put(key, (Integer) value); + } else if (value.getClass() == Float.class) { + jsonMap.put(key, (Float) value); + } else if (value.getClass() == Double.class) { + jsonMap.put(key, (Double) value); + } else if (value.getClass() == Short.class) { + jsonMap.put(key, (Short) value); + } else if (value.getClass() == Byte.class) { + jsonMap.put(key, (Byte) value); + } else if (value.getClass() == String.class) { + jsonMap.put(key, (String) value); + } else if (value instanceof ObjectNode) { + jsonMap.put(key, (ObjectNode) value); + } else if (value instanceof ArrayNode) { + jsonMap.put(key, (ArrayNode) value); + } else { + jsonMap.put(key, value.toString()); + } + } + + private static Map createPropertyMap(final Element element, final List propertyKeys, + final ElementPropertiesRule rule, final boolean normalized) { + final Map map = new HashMap(); + final List propertyKeyList; + if (normalized) { + final List sorted = new ArrayList(element.keys()); + Collections.sort(sorted); + propertyKeyList = sorted; + } else + propertyKeyList = new ArrayList(element.keys()); + + if (propertyKeys == null) { + for (String key : propertyKeyList) { + final Object valToPutInMap = element.property(key); + if (valToPutInMap != null) { + map.put(key, valToPutInMap); + } + } + } else { + if (rule == ElementPropertiesRule.INCLUDE) { + for (String key : propertyKeys) { + final Object valToPutInMap = element.property(key); + if (valToPutInMap != null) { + map.put(key, valToPutInMap); + } + } + } else { + for (String key : propertyKeyList) { + if (!propertyKeys.contains(key)) { + final Object valToPutInMap = element.property(key); + if (valToPutInMap != null) { + map.put(key, valToPutInMap); + } + } + } + } + } + + return map; + } + + private static Object getValue(Object value, final boolean includeType) { + + Object returnValue = value; + + // if the includeType is set to true then show the data types of the properties + if (includeType) { + + // type will be one of: map, list, string, long, int, double, float. + // in the event of a complex object it will call a toString and store as a + // string + String type = determineType(value); + + ObjectNode valueAndType = jsonNodeFactory.objectNode(); + valueAndType.put(GraphSONTokens.TYPE, type); + + if (type.equals(GraphSONTokensTP2.TYPE_LIST)) { + + // values of lists must be accumulated as ObjectNode objects under the value key. + // will return as a ArrayNode. called recursively to traverse the entire + // object graph of each item in the array. + ArrayNode list = (ArrayNode) value; + + // there is a set of values that must be accumulated as an array under a key + ArrayNode valueArray = valueAndType.putArray(GraphSONTokens.VALUE); + for (int ix = 0; ix < list.size(); ix++) { + // the value of each item in the array is a node object from an ArrayNode...must + // get the value of it. + addObject(valueArray, getValue(getTypedValueFromJsonNode(list.get(ix)), includeType)); + } + + } else if (type.equals(GraphSONTokensTP2.TYPE_MAP)) { + + // maps are converted to a ObjectNode. called recursively to traverse + // the entire object graph within the map. + ObjectNode convertedMap = jsonNodeFactory.objectNode(); + ObjectNode jsonObject = (ObjectNode) value; + Iterator keyIterator = jsonObject.fieldNames(); + while (keyIterator.hasNext()) { + Object key = keyIterator.next(); + + // no need to getValue() here as this is already a ObjectNode and should have type info + convertedMap.put(key.toString(), jsonObject.get(key.toString())); + } + + valueAndType.put(GraphSONTokens.VALUE, convertedMap); + } else { + + // this must be a primitive value or a complex object. if a complex + // object it will be handled by a call to toString and stored as a + // string value + putObject(valueAndType, GraphSONTokens.VALUE, value); + } + + // this goes back as a JSONObject with data type and value + returnValue = valueAndType; + } + + return returnValue; + } + + static Object getTypedValueFromJsonNode(JsonNode node) { + Object theValue = null; + + if (node != null && !node.isNull()) { + if (node.isBoolean()) { + theValue = node.booleanValue(); + } else if (node.isDouble()) { + theValue = node.doubleValue(); + } else if (node.isFloatingPointNumber()) { + theValue = node.floatValue(); + } else if (node.isInt()) { + theValue = node.intValue(); + } else if (node.isLong()) { + theValue = node.longValue(); + } else if (node.isTextual()) { + theValue = node.textValue(); + } else if (node.isArray()) { + // this is an array so just send it back so that it can be + // reprocessed to its primitive components + theValue = node; + } else if (node.isObject()) { + // this is an object so just send it back so that it can be + // reprocessed to its primitive components + theValue = node; + } else { + theValue = node.textValue(); + } + } + + return theValue; + } + + private static List convertArrayToList(final Object value) { + final ArrayList list = new ArrayList(); + int arrlength = Array.getLength(value); + for (int i = 0; i < arrlength; i++) { + Object object = Array.get(value, i); + list.add(object); + } + return list; + } + + private static String determineType(final Object value) { + String type = GraphSONTokensTP2.TYPE_STRING; + if (value == null) { + type = GraphSONTokensTP2.TYPE_UNKNOWN; + } else if (value.getClass() == Double.class) { + type = GraphSONTokensTP2.TYPE_DOUBLE; + } else if (value.getClass() == Float.class) { + type = GraphSONTokensTP2.TYPE_FLOAT; + } else if (value.getClass() == Byte.class) { + type = GraphSONTokensTP2.TYPE_BYTE; + } else if (value.getClass() == Short.class) { + type = GraphSONTokensTP2.TYPE_SHORT; + } else if (value.getClass() == Integer.class) { + type = GraphSONTokensTP2.TYPE_INTEGER; + } else if (value.getClass() == Long.class) { + type = GraphSONTokensTP2.TYPE_LONG; + } else if (value.getClass() == Boolean.class) { + type = GraphSONTokensTP2.TYPE_BOOLEAN; + } else if (value instanceof ArrayNode) { + type = GraphSONTokensTP2.TYPE_LIST; + } else if (value instanceof ObjectNode) { + type = GraphSONTokensTP2.TYPE_MAP; + } + + return type; + } +} \ No newline at end of file From 081e403c892c9e7b08d7d934fae7ffad3c7a04e6 Mon Sep 17 00:00:00 2001 From: ebice Date: Thu, 2 Jul 2015 15:09:52 -0400 Subject: [PATCH 08/13] Added shouldWriteXYZGraphASTP2GraphSON tests which utilize GraphSONLegacyWriter and TP2GraphSONUtility. CrewGraph is the only one failing due to "multiple properties for same key" location --- .../io/graphson/GraphSONLegacyWriter.java | 8 ++-- .../io/graphson/TP2GraphSONUtility.java | 11 ++--- .../structure/IoDataGenerationTest.java | 41 ++++++++++++++++--- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java index 5f5e137fb37..8d517a22efe 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java @@ -115,7 +115,7 @@ private Iterable vertices(boolean normalize) { Iterable vertices; if (normalize) { vertices = new ArrayList(); - Iterator viter = graph.vertices(null); + Iterator viter = graph.vertices(); while (viter.hasNext()) { ((Collection) vertices).add(viter.next()); } @@ -124,7 +124,7 @@ private Iterable vertices(boolean normalize) { vertices = new Iterable() { @Override public Iterator iterator() { - return graph.vertices(null); + return graph.vertices(); } }; } @@ -135,7 +135,7 @@ private Iterable edges(boolean normalize) { Iterable edges; if (normalize) { edges = new ArrayList(); - Iterator eiter = graph.edges(null); + Iterator eiter = graph.edges(); while (eiter.hasNext()) { ((Collection) edges).add(eiter.next()); } @@ -144,7 +144,7 @@ private Iterable edges(boolean normalize) { edges = new Iterable() { @Override public Iterator iterator() { - return graph.edges(null); + return graph.edges(); } }; } diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java index 73e895fab6b..4806a0d8c7f 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java @@ -440,8 +440,9 @@ public static ObjectNode objectNodeFromElement(final Element element, final Set< } private static ObjectNode objectNodeFromElement(final Element element, final List propertyKeys, final GraphSONMode mode) { - final TP2GraphSONUtility graphson = element instanceof Edge ? new TP2GraphSONUtility(mode, null, null, new HashSet(propertyKeys)) - : new TP2GraphSONUtility(mode, null, new HashSet(propertyKeys), null); + Set propKeySet = (propertyKeys!=null)? new HashSet(propertyKeys) : null; + final TP2GraphSONUtility graphson = element instanceof Edge ? new TP2GraphSONUtility(mode, null, null, propKeySet) + : new TP2GraphSONUtility(mode, null, propKeySet, null); return graphson.objectNodeFromElement(element); } @@ -692,7 +693,7 @@ private static Map createPropertyMap(final Element element, final List p if (propertyKeys == null) { for (String key : propertyKeyList) { - final Object valToPutInMap = element.property(key); + final Object valToPutInMap = element.property(key).value(); if (valToPutInMap != null) { map.put(key, valToPutInMap); } @@ -700,7 +701,7 @@ private static Map createPropertyMap(final Element element, final List p } else { if (rule == ElementPropertiesRule.INCLUDE) { for (String key : propertyKeys) { - final Object valToPutInMap = element.property(key); + final Object valToPutInMap = element.property(key).value(); if (valToPutInMap != null) { map.put(key, valToPutInMap); } @@ -708,7 +709,7 @@ private static Map createPropertyMap(final Element element, final List p } else { for (String key : propertyKeyList) { if (!propertyKeys.contains(key)) { - final Object valToPutInMap = element.property(key); + final Object valToPutInMap = element.property(key).value(); if (valToPutInMap != null) { map.put(key, valToPutInMap); } diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java index 86fc733b1ba..7882778f563 100644 --- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java +++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java @@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.io.GraphReader; import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONLegacyWriter; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; @@ -36,11 +37,7 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.util.stream.IntStream; /** @@ -122,6 +119,16 @@ public void shouldWriteClassicGraphAsGraphSONNoTypes() throws IOException { os.close(); } + /** + * No assertions. Just write out the graph for convenience. + */ + @Test + public void shouldWriteClassicGraphAsTP2GraphSONNoTypes() throws IOException { + final OutputStream os = new FileOutputStream(tempPath + "tinkerpop2-classic.json"); + GraphSONLegacyWriter.outputGraph(TinkerFactory.createClassic(), os); + os.close(); + } + /** * No assertions. Just write out the graph for convenience. */ @@ -132,6 +139,16 @@ public void shouldWriteModernGraphAsGraphSONNoTypes() throws IOException { os.close(); } + /** + * No assertions. Just write out the graph for convenience. + */ + @Test + public void shouldWriteModernGraphAsTP2GraphSONNoTypes() throws IOException { + final OutputStream os = new FileOutputStream(tempPath + "tinkerpop2-modern.json"); + GraphSONLegacyWriter.outputGraph(TinkerFactory.createModern(), os); + os.close(); + } + /** * No assertions. Just write out the graph for convenience. */ @@ -142,6 +159,16 @@ public void shouldWriteCrewGraphAsGraphSONNoTypes() throws IOException { os.close(); } + /** + * No assertions. Just write out the graph for convenience. + */ + @Test + public void shouldWriteCrewGraphAsTP2GraphSONNoTypes() throws IOException { + final OutputStream os = new FileOutputStream(tempPath + "tinkerpop2-crew.json"); + GraphSONLegacyWriter.outputGraph(TinkerFactory.createTheCrew(), os); + os.close(); + } + /** * No assertions. Just write out the graph for convenience. */ @@ -274,5 +301,9 @@ else if (e.label().equals("writtenBy")) final OutputStream os4 = new FileOutputStream(tempPath + "grateful-dead-typed.json"); GraphSONWriter.build().mapper(GraphSONMapper.build().embedTypes(true).create()).create().writeGraph(os4, g); os4.close(); + + final OutputStream os5 = new FileOutputStream(tempPath + "grateful-dead-tp2.json"); + GraphSONLegacyWriter.outputGraph(g, os5); + os5.close(); } } From f11b4a4d991326a3bf802469e23782dff2bd1e75 Mon Sep 17 00:00:00 2001 From: ebice Date: Mon, 6 Jul 2015 09:53:58 -0400 Subject: [PATCH 09/13] Added Apache License headers --- .../io/graphson/GraphSONLegacyWriter.java | 18 ++++++++++++++++++ .../io/graphson/TP2GraphSONUtility.java | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java index 8d517a22efe..c47c8069713 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.tinkerpop.gremlin.structure.io.graphson; import com.fasterxml.jackson.core.JsonFactory; diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java index 4806a0d8c7f..e331f1c967e 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.tinkerpop.gremlin.structure.io.graphson; import com.fasterxml.jackson.core.JsonFactory; From 8119bc81b5cac42f29d7eab0d0bafc6b05230894 Mon Sep 17 00:00:00 2001 From: ebice Date: Mon, 6 Jul 2015 13:28:15 -0400 Subject: [PATCH 10/13] Fixed conflicting jettison version issue. Now generating test resources for all graphs in TP2 legacy GraphSON format. Still need to do adjacency flavor of legacy GraphSON. --- gremlin-core/pom.xml | 2 +- .../io/graphson/ElementPropertyConfig.java | 26 +++++++++++-- .../tinkerpop/gremlin/LoadGraphWith.java | 37 +++++++++++++------ .../gremlin/hadoop/HadoopGraphProvider.java | 15 ++++++-- .../GraphSONLegacyRecordReaderTest.java | 2 +- .../structure/IoDataGenerationTest.java | 3 ++ 6 files changed, 64 insertions(+), 21 deletions(-) diff --git a/gremlin-core/pom.xml b/gremlin-core/pom.xml index af6c342af99..d67ddd81f1d 100644 --- a/gremlin-core/pom.xml +++ b/gremlin-core/pom.xml @@ -106,7 +106,7 @@ limitations under the License. org.codehaus.jettison jettison - 1.3.5 + 1.1 diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/ElementPropertyConfig.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/ElementPropertyConfig.java index 3c2756058e3..57bb241820f 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/ElementPropertyConfig.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/ElementPropertyConfig.java @@ -1,9 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.tinkerpop.gremlin.structure.io.graphson; - import java.util.ArrayList; - import java.util.Collections; - import java.util.List; - import java.util.Set; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; /** * Configure how the GraphSON utility treats edge and vertex properties. diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java index 944a972bc0a..b11c9993a31 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java @@ -61,6 +61,11 @@ public enum GraphData { */ CLASSIC, + /** + * Classic in TinkerPop2 legacy format. + */ + CLASSIC_TP2, + /** * Loads the "modern" TinkerPop toy graph which is like "classic", but with the "weight" value on edges stored * as double and labels added for vertices. This should be the most commonly used graph instance for testing @@ -68,11 +73,21 @@ public enum GraphData { */ MODERN, + /** + * Modern in TinkerPop2 legacy format. + */ + MODERN_TP2, + /** * Load "The Crew" TinkerPop3 toy graph which includes {@link org.apache.tinkerpop.gremlin.structure.VertexProperty} data. */ CREW, + /** + * Crew in TinkerPop2 legacy format. + */ + CREW_TP2, + /** * Loads the "grateful dead" graph which is a "large" graph which provides for the construction of more * complex traversals. @@ -80,9 +95,9 @@ public enum GraphData { GRATEFUL, /** - * Loads the "Graph of the Gods" graph which is a "toy" graph with 12 vertices and 17 edges. + * Grateful in TinkerPop2 legacy format. */ - GODS; + GRATEFUL_TP2; private static final List featuresRequiredByClassic = new ArrayList() {{ add(FeatureRequirement.Factory.create(FEATURE_STRING_VALUES, VertexPropertyFeatures.class)); @@ -109,14 +124,6 @@ public enum GraphData { add(FeatureRequirement.Factory.create(FEATURE_INTEGER_VALUES, VertexPropertyFeatures.class)); }}; - private static final List featuresRequiredByGods = new ArrayList() {{ - add(FeatureRequirement.Factory.create(FEATURE_STRING_VALUES, VertexPropertyFeatures.class)); - add(FeatureRequirement.Factory.create(FEATURE_INTEGER_VALUES, VertexPropertyFeatures.class)); - add(FeatureRequirement.Factory.create(FEATURE_BOOLEAN_VALUES, VertexPropertyFeatures.class)); - add(FeatureRequirement.Factory.create(FEATURE_META_PROPERTIES, Graph.Features.VertexFeatures.class)); - add(FeatureRequirement.Factory.create(FEATURE_MULTI_PROPERTIES, Graph.Features.VertexFeatures.class)); - }}; - public String location() { switch (this) { case CLASSIC: @@ -136,14 +143,20 @@ public List featuresRequired() { switch (this) { case CLASSIC: return featuresRequiredByClassic; + case CLASSIC_TP2: + return featuresRequiredByClassic; case CREW: return featuresRequiredByCrew; + case CREW_TP2: + return featuresRequiredByCrew; case MODERN: return featuresRequiredByModern; + case MODERN_TP2: + return featuresRequiredByModern; case GRATEFUL: return featuresRequiredByGrateful; - case GODS: - return featuresRequiredByGods; + case GRATEFUL_TP2: + return featuresRequiredByGrateful; } throw new RuntimeException("No features for this GraphData type"); diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java index 65bb87b7ca2..68f62cd4ab4 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java @@ -68,10 +68,13 @@ public class HadoopGraphProvider extends AbstractGraphProvider { final List graphsonResources = Arrays.asList( "tinkerpop-modern.json", + "tinkerpop2-modern.json", "grateful-dead.json", + "grateful-dead-tp2.json", "tinkerpop-classic.json", + "tinkerpop2-classic.json", "tinkerpop-crew.json", - "graph-of-gods.json"); + "tinkerpop2-crew.json"); for (final String fileName : graphsonResources) { PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath()); } @@ -142,14 +145,20 @@ public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.Grap if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type)); + } else if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL_TP2) && type.equals("json")) { + ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead-tp2." + type)); } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type)); + } else if (graphData.equals(LoadGraphWith.GraphData.MODERN_TP2) && type.equals("json")) { + ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop2-modern." + type)); } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type)); + } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC_TP2) && type.equals("json")) { + ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop2-classic." + type)); } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type)); - } else if (graphData.equals(LoadGraphWith.GraphData.GODS) && type.equals("json")) { - ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("graph-of-gods." + type)); + } else if (graphData.equals(LoadGraphWith.GraphData.CREW_TP2) && type.equals("json")) { + ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop2-crew." + type)); } else { throw new RuntimeException("Could not load graph with " + graphData); } diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java index 47fde1290fa..618c4140951 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java @@ -32,7 +32,7 @@ public class GraphSONLegacyRecordReaderTest extends RecordReaderWriterTest { @Override protected String getInputFilename() { - return "graph-of-gods.json"; + return "grateful-dead-tp2.json"; } @Override diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java index 7882778f563..9111b806128 100644 --- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java +++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java @@ -35,6 +35,7 @@ import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import java.io.*; @@ -110,6 +111,7 @@ public void shouldWriteModernGraphAsGraphML() throws IOException { } /** + * * No assertions. Just write out the graph for convenience. */ @Test @@ -162,6 +164,7 @@ public void shouldWriteCrewGraphAsGraphSONNoTypes() throws IOException { /** * No assertions. Just write out the graph for convenience. */ + @Ignore @Test public void shouldWriteCrewGraphAsTP2GraphSONNoTypes() throws IOException { final OutputStream os = new FileOutputStream(tempPath + "tinkerpop2-crew.json"); From b92cd799ea9eb2502bbd46906f1796bddf9b21cd Mon Sep 17 00:00:00 2001 From: ebice Date: Tue, 7 Jul 2015 17:17:23 -0400 Subject: [PATCH 11/13] Added FaunusGraphSONUtility - based loosely on same from Aurelius Faunus project - to be merged into TP2GraphSONUtility as the adjacency list functionality provider --- .../io/graphson/FaunusGraphSONUtility.java | 172 ++++++++++++++++++ .../io/graphson/GraphSONLegacyWriter.java | 82 +++++++-- .../structure/IoDataGenerationTest.java | 43 ++++- 3 files changed, 274 insertions(+), 23 deletions(-) create mode 100644 gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/FaunusGraphSONUtility.java diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/FaunusGraphSONUtility.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/FaunusGraphSONUtility.java new file mode 100644 index 00000000000..e40a77f833c --- /dev/null +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/FaunusGraphSONUtility.java @@ -0,0 +1,172 @@ +package org.apache.tinkerpop.gremlin.structure.io.graphson; + +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Element; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.io.graphson.TP2GraphSONUtility.ElementFactory; +import org.apache.tinkerpop.gremlin.structure.io.graphson.TP2GraphSONUtility.GraphSONMode; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.codehaus.jettison.json.JSONTokener; +import org.javatuples.Pair; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.*; + +import static org.apache.tinkerpop.gremlin.structure.Direction.IN; +import static org.apache.tinkerpop.gremlin.structure.Direction.OUT; + +/** + * @author Marko A. Rodriguez (http://markorodriguez.com) + * @author Stephen Mallette (http://stephen.genoprime.com) + * @author Edi Bice + */ +public class FaunusGraphSONUtility { + + //private static final String _OUT_E = "_outE"; + //private static final String _IN_E = "_inE"; + private static final String EMPTY_STRING = ""; + + private static final Set VERTEX_IGNORE = new HashSet(Arrays.asList(GraphSONTokensTP2._TYPE, GraphSONTokensTP2._OUT_E, GraphSONTokensTP2._IN_E)); + private static final Set EDGE_IGNORE = new HashSet(Arrays.asList(GraphSONTokensTP2._TYPE, GraphSONTokensTP2._OUT_V, GraphSONTokensTP2._IN_V)); + + private static final ElementFactory elementFactory = new MyElementFactory(); + + private static final TP2GraphSONUtility graphson = new TP2GraphSONUtility(GraphSONMode.COMPACT, elementFactory, + ElementPropertyConfig.excludeProperties(VERTEX_IGNORE, EDGE_IGNORE)); + + public static List fromJSON(final InputStream in) throws IOException { + final List vertices = new LinkedList(); + final BufferedReader bfs = new BufferedReader(new InputStreamReader(in)); + String line = ""; + while ((line = bfs.readLine()) != null) { + vertices.add(FaunusGraphSONUtility.fromJSON(line)); + } + bfs.close(); + return vertices; + + } + + public static Vertex fromJSON(String line) throws IOException { + try { + final JSONObject json = new JSONObject(new JSONTokener(line)); + line = EMPTY_STRING; // clear up some memory + + final Vertex vertex = graphson.vertexFromJson(json); + + fromJSONEdges(vertex, json.optJSONArray(GraphSONTokensTP2._OUT_E), OUT); + json.remove(GraphSONTokensTP2._OUT_E); // clear up some memory + fromJSONEdges(vertex, json.optJSONArray(GraphSONTokensTP2._IN_E), IN); + json.remove(GraphSONTokensTP2._IN_E); // clear up some memory + + return vertex; + } catch (Exception e) { + throw new IOException(e.getMessage(), e); + } + } + + private static void fromJSONEdges(final Vertex vertex, final JSONArray edges, final Direction direction) throws JSONException, IOException { + if (null != edges) { + for (int i = 0; i < edges.length(); i++) { + final JSONObject jsonEdge = edges.optJSONObject(i); + Edge edge = null; + if (direction.equals(Direction.IN)) { + DetachedVertex outVertex = new DetachedVertex(jsonEdge.optLong(GraphSONTokensTP2._OUT_V),null,new HashMap<>()); + edge = graphson.edgeFromJson(jsonEdge, outVertex, vertex); + outVertex.addEdge(jsonEdge.optString(GraphSONTokensTP2._LABEL), vertex); + } else if (direction.equals(Direction.OUT)) { + DetachedVertex inVertex = new DetachedVertex(jsonEdge.optLong(GraphSONTokensTP2._IN_V),null,new HashMap<>()); + edge = graphson.edgeFromJson(jsonEdge, vertex, inVertex); + vertex.addEdge(jsonEdge.optString(GraphSONTokensTP2._LABEL), inVertex); + } + } + } + } + + public static JSONObject toJSON(final Vertex vertex) throws IOException { + try { + final JSONObject object = TP2GraphSONUtility.jsonFromElement(vertex, getElementPropertyKeys(vertex, false), GraphSONMode.COMPACT); + + // force the ID to long. with blueprints, most implementations will send back a long, but + // some like TinkerGraph will return a string. the same is done for edges below + object.put(GraphSONTokensTP2._ID, Long.valueOf(object.remove(GraphSONTokensTP2._ID).toString())); + + List edges = (List) vertex.edges(OUT); + if (!edges.isEmpty()) { + final JSONArray outEdgesArray = new JSONArray(); + for (final Edge outEdge : edges) { + final JSONObject edgeObject = TP2GraphSONUtility.jsonFromElement(outEdge, getElementPropertyKeys(outEdge, true), GraphSONMode.COMPACT); + outEdgesArray.put(edgeObject); + } + object.put(GraphSONTokensTP2._OUT_E, outEdgesArray); + } + + edges = (List) vertex.edges(IN); + if (!edges.isEmpty()) { + final JSONArray inEdgesArray = new JSONArray(); + for (final Edge inEdge : edges) { + final JSONObject edgeObject = TP2GraphSONUtility.jsonFromElement(inEdge, getElementPropertyKeys(inEdge, false), GraphSONMode.COMPACT); + inEdgesArray.put(edgeObject); + } + object.put(GraphSONTokensTP2._IN_E, inEdgesArray); + } + + return object; + } catch (JSONException e) { + throw new IOException(e); + } + } + + private static Set getElementPropertyKeys(final Element element, final boolean edgeIn) { + final Set elementPropertyKeys = new HashSet(element.keys()); // .getPropertyKeys()); + elementPropertyKeys.add(GraphSONTokensTP2._ID); + if (element instanceof Edge) { + if (edgeIn) { + elementPropertyKeys.add(GraphSONTokensTP2._IN_V); + } else { + elementPropertyKeys.add(GraphSONTokensTP2._OUT_V); + } + + elementPropertyKeys.add(GraphSONTokensTP2._LABEL); + } + + return elementPropertyKeys; + } + + private static class MyElementFactory implements ElementFactory { + + @Override + public Edge createEdge(final Object id, final Vertex out, final Vertex in, final String label) { + Pair outPair = new Pair(out.id(), out.label()); + Pair inPair = new Pair(in.id(), in.label()); + return new DetachedEdge(id, label, Collections.EMPTY_MAP, outPair, inPair); + } + + @Override + public DetachedVertex createVertex(final Object id) { + return new DetachedVertex(convertIdentifier(id), null, new HashMap<>()); + } + + private long convertIdentifier(final Object id) { + if (id instanceof Long) + return (Long) id; + + long identifier = -1l; + if (id != null) { + try { + identifier = Long.parseLong(id.toString()); + } catch (final NumberFormatException e) { + identifier = -1l; + } + } + return identifier; + } + } +} \ No newline at end of file diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java index c47c8069713..b032a8531b5 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java @@ -68,12 +68,14 @@ public GraphSONLegacyWriter(final Graph graph) { * @param vertexPropertyKeys the keys of the vertex elements to write to JSON * @param edgePropertyKeys the keys of the edge elements to write to JSON * @param mode determines the format of the GraphSON + * @param adjacencyListFormat whether to output in adjacency list or vertices/edges format * @throws IOException thrown if there is an error generating the JSON data */ public void outputGraph(final String filename, final Set vertexPropertyKeys, - final Set edgePropertyKeys, final GraphSONMode mode) throws IOException { + final Set edgePropertyKeys, final GraphSONMode mode, + final boolean adjacencyListFormat) throws IOException { final FileOutputStream fos = new FileOutputStream(filename); - outputGraph(fos, vertexPropertyKeys, edgePropertyKeys, mode); + outputGraph(fos, vertexPropertyKeys, edgePropertyKeys, mode, adjacencyListFormat); fos.close(); } @@ -84,16 +86,17 @@ public void outputGraph(final String filename, final Set vertexPropertyK * @param vertexPropertyKeys the keys of the vertex elements to write to JSON * @param edgePropertyKeys the keys of the edge elements to write to JSON * @param mode determines the format of the GraphSON + * @param adjacencyListFormat whether to output in adjacency list or vertices/edges format * @throws IOException thrown if there is an error generating the JSON data */ public void outputGraph(final OutputStream jsonOutputStream, final Set vertexPropertyKeys, - final Set edgePropertyKeys, final GraphSONMode mode) throws IOException { - outputGraph(jsonOutputStream, vertexPropertyKeys, edgePropertyKeys, mode, false); + final Set edgePropertyKeys, final GraphSONMode mode, + final boolean adjacencyListFormat) throws IOException { + outputGraph(jsonOutputStream, vertexPropertyKeys, edgePropertyKeys, mode, false, adjacencyListFormat); } - - public void outputGraph(final OutputStream jsonOutputStream, final Set vertexPropertyKeys, - final Set edgePropertyKeys, final GraphSONMode mode, final boolean normalize) throws IOException { + private void outputVerticesEdgesGraph(final OutputStream jsonOutputStream, final Set vertexPropertyKeys, + final Set edgePropertyKeys, final GraphSONMode mode, final boolean normalize) throws IOException { final JsonGenerator jg = jsonFactory.createGenerator(jsonOutputStream); // don't let the JsonGenerator close the underlying stream...leave that to the client passing in the stream @@ -127,6 +130,41 @@ public void outputGraph(final OutputStream jsonOutputStream, final Set v jg.flush(); jg.close(); + + } + + private void outputAdjListGraph(final OutputStream jsonOutputStream, final Set vertexPropertyKeys, + final Set edgePropertyKeys, final GraphSONMode mode, final boolean normalize) throws IOException { + + final JsonGenerator jg = jsonFactory.createGenerator(jsonOutputStream); + + // don't let the JsonGenerator close the underlying stream...leave that to the client passing in the stream + jg.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); + + final TP2GraphSONUtility graphson = new TP2GraphSONUtility(mode, null, + ElementPropertyConfig.includeProperties(vertexPropertyKeys, edgePropertyKeys, normalize)); + + //jg.writeStartObject(); + + final Iterable vertices = vertices(normalize); + for (Vertex v : vertices) { + jg.writeTree(graphson.objectNodeFromElement(v)); + } + + //jg.writeEndObject(); + + jg.flush(); + jg.close(); + } + + public void outputGraph(final OutputStream jsonOutputStream, final Set vertexPropertyKeys, + final Set edgePropertyKeys, final GraphSONMode mode, final boolean normalize, + final boolean adjacencyListFormat) throws IOException { + if (adjacencyListFormat) { + outputAdjListGraph(jsonOutputStream, vertexPropertyKeys, edgePropertyKeys, mode, normalize); + } else { + outputVerticesEdgesGraph(jsonOutputStream, vertexPropertyKeys, edgePropertyKeys, mode, normalize); + } } private Iterable vertices(boolean normalize) { @@ -175,11 +213,12 @@ public Iterator iterator() { * * @param graph the graph to serialize to JSON * @param jsonOutputStream the JSON OutputStream to write the Graph data to + * @param adjacencyListFormat whether to output in adjacency list or vertices/edges format * @throws IOException thrown if there is an error generating the JSON data */ - public static void outputGraph(final Graph graph, final OutputStream jsonOutputStream) throws IOException { + public static void outputGraph(final Graph graph, final OutputStream jsonOutputStream, final boolean adjacencyListFormat) throws IOException { final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); - writer.outputGraph(jsonOutputStream, null, null, GraphSONMode.NORMAL); + writer.outputGraph(jsonOutputStream, null, null, GraphSONMode.NORMAL, adjacencyListFormat); } /** @@ -188,11 +227,12 @@ public static void outputGraph(final Graph graph, final OutputStream jsonOutputS * * @param graph the graph to serialize to JSON * @param filename the JSON file to write the Graph data to + * @param adjacencyListFormat whether to output in adjacency list or vertices/edges format * @throws IOException thrown if there is an error generating the JSON data */ - public static void outputGraph(final Graph graph, final String filename) throws IOException { + public static void outputGraph(final Graph graph, final String filename, final boolean adjacencyListFormat) throws IOException { final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); - writer.outputGraph(filename, null, null, GraphSONMode.NORMAL); + writer.outputGraph(filename, null, null, GraphSONMode.NORMAL, adjacencyListFormat); } /** @@ -201,12 +241,13 @@ public static void outputGraph(final Graph graph, final String filename) throws * @param graph the graph to serialize to JSON * @param jsonOutputStream the JSON OutputStream to write the Graph data to * @param mode determines the format of the GraphSON + * @param adjacencyListFormat whether to output in adjacency list or vertices/edges format * @throws IOException thrown if there is an error generating the JSON data */ public static void outputGraph(final Graph graph, final OutputStream jsonOutputStream, - final GraphSONMode mode) throws IOException { + final GraphSONMode mode, final boolean adjacencyListFormat) throws IOException { final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); - writer.outputGraph(jsonOutputStream, null, null, mode); + writer.outputGraph(jsonOutputStream, null, null, mode, adjacencyListFormat); } /** @@ -215,12 +256,13 @@ public static void outputGraph(final Graph graph, final OutputStream jsonOutputS * @param graph the graph to serialize to JSON * @param filename the JSON file to write the Graph data to * @param mode determines the format of the GraphSON + * @param adjacencyListFormat whether to output in adjacency list or vertices/edges format * @throws IOException thrown if there is an error generating the JSON data */ public static void outputGraph(final Graph graph, final String filename, - final GraphSONMode mode) throws IOException { + final GraphSONMode mode, final boolean adjacencyListFormat) throws IOException { final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); - writer.outputGraph(filename, null, null, mode); + writer.outputGraph(filename, null, null, mode, adjacencyListFormat); } /** @@ -231,13 +273,14 @@ public static void outputGraph(final Graph graph, final String filename, * @param vertexPropertyKeys the keys of the vertex elements to write to JSON * @param edgePropertyKeys the keys of the edge elements to write to JSON * @param mode determines the format of the GraphSON + * @param adjacencyListFormat whether to output in adjacency list or vertices/edges format * @throws IOException thrown if there is an error generating the JSON data */ public static void outputGraph(final Graph graph, final OutputStream jsonOutputStream, final Set vertexPropertyKeys, final Set edgePropertyKeys, - final GraphSONMode mode) throws IOException { + final GraphSONMode mode, final boolean adjacencyListFormat) throws IOException { final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); - writer.outputGraph(jsonOutputStream, vertexPropertyKeys, edgePropertyKeys, mode); + writer.outputGraph(jsonOutputStream, vertexPropertyKeys, edgePropertyKeys, mode, adjacencyListFormat); } /** @@ -248,13 +291,14 @@ public static void outputGraph(final Graph graph, final OutputStream jsonOutputS * @param vertexPropertyKeys the keys of the vertex elements to write to JSON * @param edgePropertyKeys the keys of the edge elements to write to JSON * @param mode determines the format of the GraphSON + * @param adjacencyListFormat whether to output in adjacency list or vertices/edges format * @throws IOException thrown if there is an error generating the JSON data */ public static void outputGraph(final Graph graph, final String filename, final Set vertexPropertyKeys, final Set edgePropertyKeys, - final GraphSONMode mode) throws IOException { + final GraphSONMode mode, final boolean adjacencyListFormat) throws IOException { final GraphSONLegacyWriter writer = new GraphSONLegacyWriter(graph); - writer.outputGraph(filename, vertexPropertyKeys, edgePropertyKeys, mode); + writer.outputGraph(filename, vertexPropertyKeys, edgePropertyKeys, mode, adjacencyListFormat); } } \ No newline at end of file diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java index 9111b806128..3ec7bf28a5a 100644 --- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java +++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/IoDataGenerationTest.java @@ -127,7 +127,17 @@ public void shouldWriteClassicGraphAsGraphSONNoTypes() throws IOException { @Test public void shouldWriteClassicGraphAsTP2GraphSONNoTypes() throws IOException { final OutputStream os = new FileOutputStream(tempPath + "tinkerpop2-classic.json"); - GraphSONLegacyWriter.outputGraph(TinkerFactory.createClassic(), os); + GraphSONLegacyWriter.outputGraph(TinkerFactory.createClassic(), os, false); + os.close(); + } + + /** + * No assertions. Just write out the graph for convenience. + */ + @Test + public void shouldWriteClassicGraphAsTP2AdjGraphSONNoTypes() throws IOException { + final OutputStream os = new FileOutputStream(tempPath + "tinkerpop2adj-classic.json"); + GraphSONLegacyWriter.outputGraph(TinkerFactory.createClassic(), os, true); os.close(); } @@ -147,7 +157,17 @@ public void shouldWriteModernGraphAsGraphSONNoTypes() throws IOException { @Test public void shouldWriteModernGraphAsTP2GraphSONNoTypes() throws IOException { final OutputStream os = new FileOutputStream(tempPath + "tinkerpop2-modern.json"); - GraphSONLegacyWriter.outputGraph(TinkerFactory.createModern(), os); + GraphSONLegacyWriter.outputGraph(TinkerFactory.createModern(), os, false); + os.close(); + } + + /** + * No assertions. Just write out the graph for convenience. + */ + @Test + public void shouldWriteModernGraphAsTP2AdjGraphSONNoTypes() throws IOException { + final OutputStream os = new FileOutputStream(tempPath + "tinkerpop2adj-modern.json"); + GraphSONLegacyWriter.outputGraph(TinkerFactory.createModern(), os, true); os.close(); } @@ -168,7 +188,18 @@ public void shouldWriteCrewGraphAsGraphSONNoTypes() throws IOException { @Test public void shouldWriteCrewGraphAsTP2GraphSONNoTypes() throws IOException { final OutputStream os = new FileOutputStream(tempPath + "tinkerpop2-crew.json"); - GraphSONLegacyWriter.outputGraph(TinkerFactory.createTheCrew(), os); + GraphSONLegacyWriter.outputGraph(TinkerFactory.createTheCrew(), os, false); + os.close(); + } + + /** + * No assertions. Just write out the graph for convenience. + */ + @Ignore + @Test + public void shouldWriteCrewGraphAsTP2AdjGraphSONNoTypes() throws IOException { + final OutputStream os = new FileOutputStream(tempPath + "tinkerpop2adj-crew.json"); + GraphSONLegacyWriter.outputGraph(TinkerFactory.createTheCrew(), os, true); os.close(); } @@ -306,7 +337,11 @@ else if (e.label().equals("writtenBy")) os4.close(); final OutputStream os5 = new FileOutputStream(tempPath + "grateful-dead-tp2.json"); - GraphSONLegacyWriter.outputGraph(g, os5); + GraphSONLegacyWriter.outputGraph(g, os5, false); os5.close(); + + final OutputStream os6 = new FileOutputStream(tempPath + "grateful-dead-tp2adj.json"); + GraphSONLegacyWriter.outputGraph(g, os6, true); + os6.close(); } } From 456f844e945254aba1e17c39c89595289ea4c331 Mon Sep 17 00:00:00 2001 From: ebice Date: Thu, 9 Jul 2015 11:24:26 -0400 Subject: [PATCH 12/13] Now successfully passing all IODataGeneration tests - including and especially those tests which write out legacy TP2 GraphSON in both vertices/edges- and adjacency list (as output by Faunus during export to GraphSON) format flavors. --- .../io/graphson/FaunusGraphSONUtility.java | 46 ++++++++++++++----- .../io/graphson/GraphSONLegacyWriter.java | 32 +++++-------- .../io/graphson/TP2GraphSONUtility.java | 23 ++++++---- .../GraphSONLegacyRecordReaderTest.java | 2 +- 4 files changed, 60 insertions(+), 43 deletions(-) diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/FaunusGraphSONUtility.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/FaunusGraphSONUtility.java index e40a77f833c..e455d5e9440 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/FaunusGraphSONUtility.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/FaunusGraphSONUtility.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.tinkerpop.gremlin.structure.io.graphson; import org.apache.tinkerpop.gremlin.structure.Direction; @@ -92,27 +110,31 @@ private static void fromJSONEdges(final Vertex vertex, final JSONArray edges, fi public static JSONObject toJSON(final Vertex vertex) throws IOException { try { - final JSONObject object = TP2GraphSONUtility.jsonFromElement(vertex, getElementPropertyKeys(vertex, false), GraphSONMode.COMPACT); + //final JSONObject object = TP2GraphSONUtility.jsonFromElement(vertex, getElementPropertyKeys(vertex, false), GraphSONMode.COMPACT); + final JSONObject object = TP2GraphSONUtility.jsonFromElement(vertex, null, GraphSONMode.COMPACT); // force the ID to long. with blueprints, most implementations will send back a long, but // some like TinkerGraph will return a string. the same is done for edges below object.put(GraphSONTokensTP2._ID, Long.valueOf(object.remove(GraphSONTokensTP2._ID).toString())); - List edges = (List) vertex.edges(OUT); + List edges = new ArrayList(); + vertex.edges(OUT).forEachRemaining(edges::add); if (!edges.isEmpty()) { final JSONArray outEdgesArray = new JSONArray(); for (final Edge outEdge : edges) { - final JSONObject edgeObject = TP2GraphSONUtility.jsonFromElement(outEdge, getElementPropertyKeys(outEdge, true), GraphSONMode.COMPACT); + //final JSONObject edgeObject = TP2GraphSONUtility.jsonFromElement(outEdge, getElementPropertyKeys(outEdge, true), GraphSONMode.COMPACT); + final JSONObject edgeObject = TP2GraphSONUtility.jsonFromElement(outEdge, null, GraphSONMode.COMPACT); outEdgesArray.put(edgeObject); } object.put(GraphSONTokensTP2._OUT_E, outEdgesArray); } - edges = (List) vertex.edges(IN); + vertex.edges(IN).forEachRemaining(edges::add); if (!edges.isEmpty()) { final JSONArray inEdgesArray = new JSONArray(); for (final Edge inEdge : edges) { - final JSONObject edgeObject = TP2GraphSONUtility.jsonFromElement(inEdge, getElementPropertyKeys(inEdge, false), GraphSONMode.COMPACT); + //final JSONObject edgeObject = TP2GraphSONUtility.jsonFromElement(inEdge, getElementPropertyKeys(inEdge, false), GraphSONMode.COMPACT); + final JSONObject edgeObject = TP2GraphSONUtility.jsonFromElement(inEdge, null, GraphSONMode.COMPACT); inEdgesArray.put(edgeObject); } object.put(GraphSONTokensTP2._IN_E, inEdgesArray); @@ -126,21 +148,21 @@ public static JSONObject toJSON(final Vertex vertex) throws IOException { private static Set getElementPropertyKeys(final Element element, final boolean edgeIn) { final Set elementPropertyKeys = new HashSet(element.keys()); // .getPropertyKeys()); - elementPropertyKeys.add(GraphSONTokensTP2._ID); + // exclude reserved keys? not really properties? + /*elementPropertyKeys.add(GraphSONTokens.ID); if (element instanceof Edge) { if (edgeIn) { - elementPropertyKeys.add(GraphSONTokensTP2._IN_V); + elementPropertyKeys.add(GraphSONTokens.IN); } else { - elementPropertyKeys.add(GraphSONTokensTP2._OUT_V); + elementPropertyKeys.add(GraphSONTokens.OUT); } - elementPropertyKeys.add(GraphSONTokensTP2._LABEL); - } - + elementPropertyKeys.add(GraphSONTokens.LABEL); + }*/ return elementPropertyKeys; } - private static class MyElementFactory implements ElementFactory { + public static class MyElementFactory implements ElementFactory { @Override public Edge createEdge(final Object id, final Vertex out, final Vertex in, final String label) { diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java index b032a8531b5..f10d040f418 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyWriter.java @@ -26,10 +26,9 @@ import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.io.graphson.TP2GraphSONUtility.GraphSONMode; +import org.codehaus.jettison.json.JSONException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; +import java.io.*; import java.util.*; /** @@ -135,26 +134,19 @@ private void outputVerticesEdgesGraph(final OutputStream jsonOutputStream, final private void outputAdjListGraph(final OutputStream jsonOutputStream, final Set vertexPropertyKeys, final Set edgePropertyKeys, final GraphSONMode mode, final boolean normalize) throws IOException { - - final JsonGenerator jg = jsonFactory.createGenerator(jsonOutputStream); - - // don't let the JsonGenerator close the underlying stream...leave that to the client passing in the stream - jg.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); - - final TP2GraphSONUtility graphson = new TP2GraphSONUtility(mode, null, - ElementPropertyConfig.includeProperties(vertexPropertyKeys, edgePropertyKeys, normalize)); - - //jg.writeStartObject(); - + final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(jsonOutputStream)); final Iterable vertices = vertices(normalize); for (Vertex v : vertices) { - jg.writeTree(graphson.objectNodeFromElement(v)); + try { + FaunusGraphSONUtility.toJSON(v).write(writer); + } catch (JSONException e) { + e.printStackTrace(); + throw new IOException(e); + } + writer.newLine(); } - - //jg.writeEndObject(); - - jg.flush(); - jg.close(); + writer.flush(); + writer.close(); } public void outputGraph(final OutputStream jsonOutputStream, final Set vertexPropertyKeys, diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java index e331f1c967e..33be1d0484a 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java @@ -273,13 +273,13 @@ public ObjectNode objectNodeFromElement(final Element element) { final ObjectNode jsonElement = createJSONMap(createPropertyMap(element, propertyKeys, elementPropertyConfig, normalized), propertyKeys, showTypes); - if ((isEdge && this.includeReservedEdgeId) || (!isEdge && this.includeReservedVertexId)) { - putObject(jsonElement, GraphSONTokensTP2._ID, element.id()); - } + //if ((isEdge && this.includeReservedEdgeId) || (!isEdge && this.includeReservedVertexId)) { + // putObject(jsonElement, GraphSONTokensTP2._ID, element.id()); + //} // it's important to keep the order of these straight. check Edge first and then Vertex because there // are graph implementations that have Edge extend from Vertex - if (element instanceof Edge) { + if (isEdge) { //element instanceof Edge) { final Edge edge = (Edge) element; if (this.includeReservedEdgeId) { @@ -439,8 +439,9 @@ public static Edge edgeFromJson(final JsonNode json, final Vertex out, final Ver */ public static JSONObject jsonFromElement(final Element element, final Set propertyKeys, final GraphSONMode mode) throws JSONException { - final TP2GraphSONUtility graphson = element instanceof Edge ? new TP2GraphSONUtility(mode, null, null, propertyKeys) - : new TP2GraphSONUtility(mode, null, propertyKeys, null); + ElementFactory factory = new FaunusGraphSONUtility.MyElementFactory(); + final TP2GraphSONUtility graphson = element instanceof Edge ? new TP2GraphSONUtility(mode, factory, null, propertyKeys) + : new TP2GraphSONUtility(mode, factory, propertyKeys, null); return graphson.jsonFromElement(element); } @@ -452,15 +453,17 @@ public static JSONObject jsonFromElement(final Element element, final Set propertyKeys, final GraphSONMode mode) { - final TP2GraphSONUtility graphson = element instanceof Edge ? new TP2GraphSONUtility(mode, null, null, propertyKeys) - : new TP2GraphSONUtility(mode, null, propertyKeys, null); + ElementFactory factory = new FaunusGraphSONUtility.MyElementFactory(); + final TP2GraphSONUtility graphson = element instanceof Edge ? new TP2GraphSONUtility(mode, factory, null, propertyKeys) + : new TP2GraphSONUtility(mode, factory, propertyKeys, null); return graphson.objectNodeFromElement(element); } private static ObjectNode objectNodeFromElement(final Element element, final List propertyKeys, final GraphSONMode mode) { + ElementFactory factory = new FaunusGraphSONUtility.MyElementFactory(); Set propKeySet = (propertyKeys!=null)? new HashSet(propertyKeys) : null; - final TP2GraphSONUtility graphson = element instanceof Edge ? new TP2GraphSONUtility(mode, null, null, propKeySet) - : new TP2GraphSONUtility(mode, null, propKeySet, null); + final TP2GraphSONUtility graphson = element instanceof Edge ? new TP2GraphSONUtility(mode, factory, null, propKeySet) + : new TP2GraphSONUtility(mode, factory, propKeySet, null); return graphson.objectNodeFromElement(element); } diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java index 618c4140951..4a1926e491f 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java @@ -32,7 +32,7 @@ public class GraphSONLegacyRecordReaderTest extends RecordReaderWriterTest { @Override protected String getInputFilename() { - return "grateful-dead-tp2.json"; + return "tinkerpop2adj-classic.json"; } @Override From 2e30517f428be29666b8afad7af3f95b3953e72d Mon Sep 17 00:00:00 2001 From: ebice Date: Fri, 10 Jul 2015 13:40:18 -0400 Subject: [PATCH 13/13] GraphSONLegacyRecordReaderTest in hadoop-gremlin now runs through to completion but fails assertion. --- .../io/graphson/GraphSONLegacyReader.java | 40 +++++++------------ .../io/graphson/GraphSONTokensTP2.java | 2 +- .../io/graphson/LegacyGraphSONUtility.java | 22 +++++----- .../io/graphson/TP2GraphSONUtility.java | 22 +++++----- .../tinkerpop/gremlin/LoadGraphWith.java | 39 +++++++++++------- .../gremlin/hadoop/HadoopGraphProvider.java | 12 ++++++ .../GraphSONLegacyRecordReaderTest.java | 3 +- 7 files changed, 77 insertions(+), 63 deletions(-) diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java index bdf3e43281e..9fc74774a8b 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONLegacyReader.java @@ -20,29 +20,17 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; -import org.apache.tinkerpop.gremlin.structure.Direction; -import org.apache.tinkerpop.gremlin.structure.Property; -import org.apache.tinkerpop.gremlin.structure.T; -import org.apache.tinkerpop.gremlin.structure.Edge; -import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.Vertex; -import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.apache.tinkerpop.gremlin.structure.*; import org.apache.tinkerpop.gremlin.structure.io.GraphReader; import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; -import org.apache.tinkerpop.gremlin.structure.io.Io; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter; import org.apache.tinkerpop.gremlin.structure.util.Attachable; import org.apache.tinkerpop.gremlin.structure.util.Host; import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge; -import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedProperty; -import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty; import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph; -import org.apache.tinkerpop.gremlin.structure.util.star.StarGraphGraphSONSerializer; import org.apache.tinkerpop.gremlin.util.function.FunctionUtils; import org.javatuples.Pair; @@ -166,8 +154,10 @@ public Vertex readVertex(final InputStream inputStream, try (JsonParser parser = factory.createParser(inputStream)) { final JsonNode node = parser.readValueAsTree(); - final Map vertexData = LegacyGraphSONUtility.readProperties(node); - starGraph.addVertex(T.id, vertexData.get(GraphSONTokensTP2._ID), T.label, vertexData.get(GraphSONTokensTP2._LABEL)); + //final Map vertexData = LegacyGraphSONUtility.readProperties(node); + final Map vertexData = TP2GraphSONUtility.readProperties(node, false, false); + //starGraph.addVertex(T.id, vertexData.get(GraphSONTokensTP2._ID), T.label, vertexData.get(GraphSONTokensTP2._LABEL)); + starGraph.addVertex(T.id, vertexData.get(GraphSONTokensTP2._ID)); // for (Map.Entry p : vertexData.entrySet()) { if (LegacyGraphSONUtility.isReservedKey(p.getKey())) continue; @@ -176,10 +166,10 @@ public Vertex readVertex(final InputStream inputStream, if (vertexAttachMethod != null) vertexAttachMethod.apply(starGraph.getStarVertex()); if (vertexData.containsKey(GraphSONTokensTP2._OUT_E) && (attachEdgesOfThisDirection == Direction.BOTH || attachEdgesOfThisDirection == Direction.OUT)) - readEdges(edgeAttachMethod, starGraph, vertexData, GraphSONTokens.OUT_E); + readEdges(edgeAttachMethod, starGraph, vertexData, GraphSONTokensTP2._OUT_E); if (vertexData.containsKey(GraphSONTokensTP2._IN_E) && (attachEdgesOfThisDirection == Direction.BOTH || attachEdgesOfThisDirection == Direction.IN)) - readEdges(edgeAttachMethod, starGraph, vertexData, GraphSONTokens.IN_E); + readEdges(edgeAttachMethod, starGraph, vertexData, GraphSONTokensTP2._IN_E); } catch (Exception ex) { throw new IOException(ex); @@ -192,22 +182,22 @@ public static void readEdges(final Function, Edge> edgeMaker, final StarGraph starGraph, final Map vertexData, final String direction) throws IOException { - final Map>> edgeDatas = (Map>>) vertexData.get(direction); - for (Map.Entry>> edgeData : edgeDatas.entrySet()) { - for (Map inner : edgeData.getValue()) { + //final Map>> edgeDatas = (Map>>) vertexData.get(direction); + final List> edgeDatas = (List>) vertexData.get(direction); + //for (Map.Entry>> edgeData : edgeDatas.entrySet()) { + for (Map edgeData : edgeDatas) { //edgeData.getValue()) { final StarGraph.StarEdge starEdge; - String edgeLabel = inner.get(GraphSONTokensTP2._LABEL).toString(); + String edgeLabel = edgeData.get(GraphSONTokensTP2._LABEL).toString(); if (direction.equals(GraphSONTokens.OUT_E)) - starEdge = (StarGraph.StarEdge) starGraph.getStarVertex().addOutEdge(edgeLabel, starGraph.addVertex(T.id, inner.get(GraphSONTokensTP2._IN_V)), T.id, inner.get(GraphSONTokensTP2._ID)); + starEdge = (StarGraph.StarEdge) starGraph.getStarVertex().addOutEdge(edgeLabel, starGraph.addVertex(T.id, edgeData.get(GraphSONTokensTP2._IN_V)), T.id, edgeData.get(GraphSONTokensTP2._ID)); else - starEdge = (StarGraph.StarEdge) starGraph.getStarVertex().addInEdge(edgeLabel, starGraph.addVertex(T.id, inner.get(GraphSONTokensTP2._OUT_V)), T.id, inner.get(GraphSONTokensTP2._ID)); - for (Map.Entry epd : inner.entrySet()) { + starEdge = (StarGraph.StarEdge) starGraph.getStarVertex().addInEdge(edgeLabel, starGraph.addVertex(T.id, edgeData.get(GraphSONTokensTP2._OUT_V)), T.id, edgeData.get(GraphSONTokensTP2._ID)); + for (Map.Entry epd : edgeData.entrySet()) { if (!LegacyGraphSONUtility.isReservedKey(epd.getKey())) starEdge.property(epd.getKey(), epd.getValue()); } if (edgeMaker != null) edgeMaker.apply(starEdge); } - } } /** diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java index 479b473635a..d5cbf2680d5 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONTokensTP2.java @@ -30,7 +30,7 @@ private GraphSONTokensTP2() {} public static final String _OUT_E = "_outE"; public static final String _IN_E = "_inE"; public static final String VALUE = "value"; - public static final String TYPE = "type"; + //public static final String TYPE = "type"; public static final String TYPE_LIST = "list"; public static final String TYPE_STRING = "string"; public static final String TYPE_DOUBLE = "double"; diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java index a8fedf32217..0f18d057e38 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONUtility.java @@ -102,27 +102,27 @@ public static boolean isReservedKey(final String key) { private static Object readProperty(final JsonNode node) { final Object propertyValue; - if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_UNKNOWN)) { + if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_UNKNOWN)) { propertyValue = null; - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BOOLEAN)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BOOLEAN)) { propertyValue = node.get(GraphSONTokensTP2.VALUE).booleanValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_FLOAT)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_FLOAT)) { propertyValue = Float.parseFloat(node.get(GraphSONTokensTP2.VALUE).asText()); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BYTE)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BYTE)) { propertyValue = Byte.parseByte(node.get(GraphSONTokensTP2.VALUE).asText()); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_SHORT)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_SHORT)) { propertyValue = Short.parseShort(node.get(GraphSONTokensTP2.VALUE).asText()); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_DOUBLE)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_DOUBLE)) { propertyValue = node.get(GraphSONTokensTP2.VALUE).doubleValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_INTEGER)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_INTEGER)) { propertyValue = node.get(GraphSONTokensTP2.VALUE).intValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LONG)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LONG)) { propertyValue = node.get(GraphSONTokensTP2.VALUE).longValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_STRING)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_STRING)) { propertyValue = node.get(GraphSONTokensTP2.VALUE).textValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LIST)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LIST)) { propertyValue = readProperties(node.get(GraphSONTokensTP2.VALUE).elements()); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_MAP)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_MAP)) { propertyValue = readProperties(node.get(GraphSONTokensTP2.VALUE)); } else { propertyValue = node.textValue(); diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java index 33be1d0484a..eed2a7ab14e 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TP2GraphSONUtility.java @@ -527,27 +527,27 @@ private static Object readProperty(final JsonNode node, final boolean hasEmbedde final Object propertyValue; if (hasEmbeddedTypes) { - if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_UNKNOWN)) { + if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_UNKNOWN)) { propertyValue = null; - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BOOLEAN)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BOOLEAN)) { propertyValue = node.get(GraphSONTokensTP2.VALUE).booleanValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_FLOAT)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_FLOAT)) { propertyValue = Float.parseFloat(node.get(GraphSONTokensTP2.VALUE).asText()); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BYTE)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_BYTE)) { propertyValue = Byte.parseByte(node.get(GraphSONTokensTP2.VALUE).asText()); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_SHORT)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_SHORT)) { propertyValue = Short.parseShort(node.get(GraphSONTokensTP2.VALUE).asText()); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_DOUBLE)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_DOUBLE)) { propertyValue = node.get(GraphSONTokensTP2.VALUE).doubleValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_INTEGER)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_INTEGER)) { propertyValue = node.get(GraphSONTokensTP2.VALUE).intValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LONG)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LONG)) { propertyValue = node.get(GraphSONTokensTP2.VALUE).longValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_STRING)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_STRING)) { propertyValue = node.get(GraphSONTokensTP2.VALUE).textValue(); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LIST)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_LIST)) { propertyValue = readProperties(node.get(GraphSONTokensTP2.VALUE).elements(), hasEmbeddedTypes); - } else if (node.get(GraphSONTokensTP2.TYPE).textValue().equals(GraphSONTokensTP2.TYPE_MAP)) { + } else if (node.get(GraphSONTokensTP2._TYPE).textValue().equals(GraphSONTokensTP2.TYPE_MAP)) { propertyValue = readProperties(node.get(GraphSONTokensTP2.VALUE), false, hasEmbeddedTypes); } else { propertyValue = node.textValue(); diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java index b11c9993a31..6c88ae5a97f 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/LoadGraphWith.java @@ -66,6 +66,11 @@ public enum GraphData { */ CLASSIC_TP2, + /** + * Classic in TinkerPop2 adjacency list format. + */ + CLASSIC_TP2_ADJ, + /** * Loads the "modern" TinkerPop toy graph which is like "classic", but with the "weight" value on edges stored * as double and labels added for vertices. This should be the most commonly used graph instance for testing @@ -78,6 +83,11 @@ public enum GraphData { */ MODERN_TP2, + /** + * Modern in TinkerPop2 adjacency list format. + */ + MODERN_TP2_ADJ, + /** * Load "The Crew" TinkerPop3 toy graph which includes {@link org.apache.tinkerpop.gremlin.structure.VertexProperty} data. */ @@ -88,6 +98,11 @@ public enum GraphData { */ CREW_TP2, + /** + * Crew in TinkerPop2 adjacency list format. + */ + CREW_TP2_ADJ, + /** * Loads the "grateful dead" graph which is a "large" graph which provides for the construction of more * complex traversals. @@ -97,7 +112,12 @@ public enum GraphData { /** * Grateful in TinkerPop2 legacy format. */ - GRATEFUL_TP2; + GRATEFUL_TP2, + + /** + * Grateful in TinkerPop2 adjacency list format. + */ + GRATEFUL_TP2_ADJ; private static final List featuresRequiredByClassic = new ArrayList() {{ add(FeatureRequirement.Factory.create(FEATURE_STRING_VALUES, VertexPropertyFeatures.class)); @@ -141,24 +161,15 @@ public String location() { public List featuresRequired() { switch (this) { - case CLASSIC: - return featuresRequiredByClassic; - case CLASSIC_TP2: + case CLASSIC: case CLASSIC_TP2: CLASSIC_TP2_ADJ: return featuresRequiredByClassic; - case CREW: - return featuresRequiredByCrew; - case CREW_TP2: + case CREW: case CREW_TP2: case CREW_TP2_ADJ: return featuresRequiredByCrew; - case MODERN: - return featuresRequiredByModern; - case MODERN_TP2: + case MODERN: case MODERN_TP2: case MODERN_TP2_ADJ: return featuresRequiredByModern; - case GRATEFUL: - return featuresRequiredByGrateful; - case GRATEFUL_TP2: + case GRATEFUL: case GRATEFUL_TP2: case GRATEFUL_TP2_ADJ: return featuresRequiredByGrateful; } - throw new RuntimeException("No features for this GraphData type"); } } diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java index 68f62cd4ab4..153db76a35a 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java @@ -69,11 +69,15 @@ public class HadoopGraphProvider extends AbstractGraphProvider { final List graphsonResources = Arrays.asList( "tinkerpop-modern.json", "tinkerpop2-modern.json", + "tinkerpop2adj-modern.json", "grateful-dead.json", "grateful-dead-tp2.json", + "grateful-dead-tp2adj.json", "tinkerpop-classic.json", "tinkerpop2-classic.json", + "tinkerpop2adj-classic.json", "tinkerpop-crew.json", + //"tinkerpop2adj-crew.json", // todo Add back when resolving issue with multivalued properties "tinkerpop2-crew.json"); for (final String fileName : graphsonResources) { PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath()); @@ -147,18 +151,26 @@ public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.Grap ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type)); } else if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL_TP2) && type.equals("json")) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead-tp2." + type)); + } else if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL_TP2_ADJ) && type.equals("json")) { + ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead-tp2adj." + type)); } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type)); } else if (graphData.equals(LoadGraphWith.GraphData.MODERN_TP2) && type.equals("json")) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop2-modern." + type)); + } else if (graphData.equals(LoadGraphWith.GraphData.MODERN_TP2_ADJ) && type.equals("json")) { + ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop2adj-modern." + type)); } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type)); } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC_TP2) && type.equals("json")) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop2-classic." + type)); + } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC_TP2_ADJ) && type.equals("json")) { + ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop2adj-classic." + type)); } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type)); } else if (graphData.equals(LoadGraphWith.GraphData.CREW_TP2) && type.equals("json")) { ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop2-crew." + type)); + } else if (graphData.equals(LoadGraphWith.GraphData.CREW_TP2_ADJ) && type.equals("json")) { + ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop2adj-crew." + type)); } else { throw new RuntimeException("Could not load graph with " + graphData); } diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java index 4a1926e491f..490b0e51bf2 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONLegacyRecordReaderTest.java @@ -32,7 +32,8 @@ public class GraphSONLegacyRecordReaderTest extends RecordReaderWriterTest { @Override protected String getInputFilename() { - return "tinkerpop2adj-classic.json"; + return "grateful-dead-tp2adj.json"; + //return "tinkerpop2adj-classic.json"; } @Override