From b294275f196564fe55e2e23e559b6a4ea7edba79 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:58:20 +0000 Subject: [PATCH 1/2] Gh-3183 Remove Gafferpop specifics from operation chains (#3184) * Make element generation happen outside of operation chain * stop view filtering out all vertexes * fix edge seeding --- .../gchq/gaffer/tinkerpop/GafferPopGraph.java | 140 ++++++++++-------- 1 file changed, 79 insertions(+), 61 deletions(-) diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java index 6ca4e591b66..f9cce2b7653 100755 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java @@ -43,8 +43,6 @@ import uk.gov.gchq.gaffer.operation.data.EntitySeed; import uk.gov.gchq.gaffer.operation.graph.SeededGraphFilters.IncludeIncomingOutgoingType; import uk.gov.gchq.gaffer.operation.impl.add.AddElements; -import uk.gov.gchq.gaffer.operation.impl.generate.GenerateElements; -import uk.gov.gchq.gaffer.operation.impl.generate.GenerateObjects; import uk.gov.gchq.gaffer.operation.impl.get.GetAdjacentIds; import uk.gov.gchq.gaffer.operation.impl.get.GetAllElements; import uk.gov.gchq.gaffer.operation.impl.get.GetElements; @@ -53,12 +51,9 @@ import uk.gov.gchq.gaffer.store.schema.Schema; import uk.gov.gchq.gaffer.tinkerpop.generator.GafferEdgeGenerator; import uk.gov.gchq.gaffer.tinkerpop.generator.GafferEntityGenerator; -import uk.gov.gchq.gaffer.tinkerpop.generator.GafferPopEdgeGenerator; import uk.gov.gchq.gaffer.tinkerpop.generator.GafferPopElementGenerator; -import uk.gov.gchq.gaffer.tinkerpop.generator.GafferPopVertexGenerator; import uk.gov.gchq.gaffer.tinkerpop.service.GafferPopNamedOperationServiceFactory; import uk.gov.gchq.gaffer.user.User; -import uk.gov.gchq.koryphe.iterable.ChainedIterable; import uk.gov.gchq.koryphe.iterable.MappedIterable; import java.nio.charset.StandardCharsets; @@ -322,12 +317,11 @@ public Vertex addVertex(final Object... keyValues) { } public void addVertex(final GafferPopVertex vertex) { + // Create the entity and add to graph execute(new OperationChain.Builder() - .first(new GenerateElements.Builder() - .input(vertex) - .generator(new GafferEntityGenerator()) - .build()) - .then(new AddElements()) + .first(new AddElements.Builder() + .input(new GafferEntityGenerator()._apply(vertex)) + .build()) .build()); // Set read only if not told otherwise @@ -337,13 +331,12 @@ public void addVertex(final GafferPopVertex vertex) { } public void addEdge(final GafferPopEdge edge) { + // Create the edge and add to graph execute(new OperationChain.Builder() - .first(new GenerateElements.Builder() - .input(edge) - .generator(new GafferEdgeGenerator()) - .build()) - .then(new AddElements()) - .build()); + .first(new AddElements.Builder() + .input(new GafferEdgeGenerator()._apply(edge)) + .build()) + .build()); } /** @@ -377,16 +370,21 @@ public Iterator vertices(final Object... vertexIds) { .build()) .build(); } - - final Iterable result = execute(new Builder() + // Run requested chain on the graph + final Iterable result = execute(new Builder() .first(getOperation) - .then(new GenerateObjects.Builder() - .generator(new GafferPopElementGenerator(this)) - .build()) .build()); - return (Iterator) result.iterator(); - } + // Translate results to Gafferpop elements + final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); + final Iterable translatedResults = () -> StreamSupport.stream(result.spliterator(), false) + .map(generator::_apply) + .filter(Vertex.class::isInstance) + .map(e -> (Vertex) e) + .iterator(); + + return translatedResults.iterator(); + } /** * This performs getRelatedEntities operation on Gaffer. @@ -524,13 +522,20 @@ public Iterator edges(final Object... elementIds) { .build(); } - return (Iterator) execute(new Builder() + // Run requested chain on the graph + final Iterable result = execute(new Builder() .first(getOperation) - .then(new GenerateObjects.Builder() - .generator(new GafferPopEdgeGenerator(this)) - .build()) - .build()) + .build()); + + // Translate results to Gafferpop elements + final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); + final Iterable translatedResults = () -> StreamSupport.stream(result.spliterator(), false) + .map(generator::_apply) + .filter(Edge.class::isInstance) + .map(e -> (Edge) e) .iterator(); + + return translatedResults.iterator(); } /** @@ -691,18 +696,21 @@ private Iterator verticesWithSeedsAndView(final List result = execute(new OperationChain.Builder() + // Run operation on graph + final Iterable result = execute(new OperationChain.Builder() .first(getOperation) - .then(new GenerateObjects.Builder() - .generator(new GafferPopVertexGenerator(this)) - .build()) .build()); - if (!idVertices.isEmpty()) { - return new ChainedIterable(result, idVertices).iterator(); - } else { - return (Iterator) result.iterator(); - } + // Translate results to Gafferpop elements + final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); + final Iterable translatedResults = () -> StreamSupport.stream(result.spliterator(), false) + .map(generator::_apply) + .filter(GafferPopVertex.class::isInstance) + .map(e -> (GafferPopVertex) e) + .iterator(); + + return translatedResults.iterator(); + } private Iterator adjVerticesWithSeedsAndView(final List seeds, final Direction direction, final View view) { @@ -711,26 +719,25 @@ private Iterator adjVerticesWithSeedsAndView(final List see } View processedView = view == null ? createAllEntitiesView() : view; - - final Iterable result = execute(new OperationChain.Builder() + final Iterable result = execute(new OperationChain.Builder() .first(new GetAdjacentIds.Builder() .input(seeds) .view(processedView) .inOutType(getInOutType(direction)) .build()) - .then(new GetElements.Builder() - .view(processedView) - .build()) - .then(new GenerateObjects.Builder() - .generator(new GafferPopElementGenerator(this)) - .build()) + // GetAdjacentIds provides list of entity seeds so run a GetElements to get the actual Entities + .then(new GetElements()) .build()); - final Iterable resultVertexes = () -> StreamSupport.stream(result.spliterator(), false) - .filter(Vertex.class::isInstance) - .map(e -> (Vertex) e) - .iterator(); - return resultVertexes.iterator(); + // Translate results to Gafferpop elements + final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); + final Iterable translatedResults = () -> StreamSupport.stream(result.spliterator(), false) + .map(generator::_apply) + .filter(Vertex.class::isInstance) + .map(e -> (Vertex) e) + .iterator(); + + return translatedResults.iterator(); } private Iterator edgesWithSeedsAndView(final List seeds, final Direction direction, final View view) { @@ -761,12 +768,20 @@ private Iterator edgesWithSeedsAndView(final List seeds, fina .build(); } - return (Iterator) execute(new OperationChain.Builder() + // Run requested chain on the graph + final Iterable result = execute(new Builder() .first(getOperation) - .then(new GenerateObjects.Builder() - .generator(new GafferPopEdgeGenerator(this, true)) - .build()) - .build()).iterator(); + .build()); + + // Translate results to Gafferpop elements + final GafferPopElementGenerator generator = new GafferPopElementGenerator(this, true); + final Iterable translatedResults = () -> StreamSupport.stream(result.spliterator(), false) + .map(generator::_apply) + .filter(Edge.class::isInstance) + .map(e -> (Edge) e) + .iterator(); + + return translatedResults.iterator(); } private View createViewWithEntities(final String... labels) { @@ -790,7 +805,7 @@ private View createView(final String... labels) { View view = null; if (null != labels && 0 < labels.length) { final View.Builder viewBuilder = new View.Builder(); - final Schema schema = ((Schema) variables().get(GafferPopGraphVariables.SCHEMA).get()); + final Schema schema = graph.getSchema(); for (final String label : labels) { if (schema.isEntity(label)) { viewBuilder.entity(label); @@ -807,8 +822,7 @@ private View createView(final String... labels) { private View createAllEntitiesView() { final View.Builder viewBuilder = new View.Builder(); - final Schema schema = ((Schema) variables().get(GafferPopGraphVariables.SCHEMA).get()); - for (final String group : schema.getEntityGroups()) { + for (final String group : graph.getSchema().getEntityGroups()) { viewBuilder.entity(group); } return viewBuilder.build(); @@ -837,11 +851,15 @@ private List getElementSeeds(final Iterable ids) { } else if (id instanceof Edge) { seeds.add(new EdgeSeed(((Edge) id).outVertex().id(), ((Edge) id).inVertex().id(), true)); // Extract source and destination from ID list - } else if (id instanceof List) { - edgeIdList = (List) id; + } else if (id instanceof Iterable) { + ((Iterable) id).forEach(edgeIdList::add); // Attempt to extract source and destination IDs from a string form of an array/list } else if ((id instanceof String) && (((String) id).matches("^\\[.*,.*\\]$"))) { - edgeIdList = Arrays.asList(((String) id).replace("[", "").replace("]", "").split(",")); + edgeIdList = Arrays.asList(((String) id) + .replaceAll("\\s", "") + .replace("[", "") + .replace("]", "") + .split(",")); // Assume entity ID as fallback } else { seeds.add(new EntitySeed(id)); From fe1d93a7e97792b201d070ffccea6204a9a8eb9b Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Mon, 8 Apr 2024 16:34:04 +0000 Subject: [PATCH 2/2] Gh-2994: Make gremlin logging configurable (#3180) * only show info logs if they are enabled * move check and fix formatting * change info to debug --- .../gchq/gaffer/tinkerpop/GafferPopGraph.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java index f9cce2b7653..2b1177da6eb 100755 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java @@ -638,21 +638,20 @@ public T execute(final OperationChain opChain) { for (final Operation operation : opChain.getOperations()) { operation.setOptions(opOptions); - if (operation instanceof Input) { + if (LOGGER.isDebugEnabled() && operation instanceof Input) { Object input = ((Input) operation).getInput(); - if (input != null) { - if (input instanceof MappedIterable) { - ((MappedIterable) input).forEach(item -> { LOGGER.info("GafferPop operation input: " + item.toString()); }); - } else { - LOGGER.info("GafferPop operation input: " + input.toString()); - } + if (input instanceof MappedIterable) { + ((MappedIterable) input).forEach(item -> { + LOGGER.debug("GafferPop operation input: {}", item); + }); + } else { + LOGGER.debug("GafferPop operation input: {}", input); } } - } try { - LOGGER.info("GafferPop operation chain called: " + opChain.toString()); + LOGGER.info("GafferPop operation chain called: {}", opChain.toOverviewString()); return graph.execute(opChain, user); } catch (final Exception e) { LOGGER.error("Operation chain failed: " + e.getMessage(), e);