Fix openCypher neighbor expansion and counts (#449)

* Add normalize function for tests

* Fix openCypher neighbor count limit

* Fix source and target node types

* Add test for unfetched count

* Fix limit and offset during expand

* Build cypher with multiple lines

* Fix limit so all edges returned

* Update changelog

Changelog.md

@@ -12,6 +12,12 @@
   (<https://github.com/aws/graph-explorer/pull/436>)
 - Add node expansion limit per connection
   (<https://github.com/aws/graph-explorer/pull/447>)
+- Fixed many bugs around neighbor expansion and counts for openCypher
+  (<https://github.com/aws/graph-explorer/pull/449>)
+  - Fixed expand limit to be type based when expanding from sidebar
+  - Fixed expand query to respect limit and offset properly so multiple
+    expansions return unique results
+  - Fixed expand query so all edges are returned between source and target nodes
 
 **Bug Fixes and Minor Changes**
 

packages/graph-explorer/src/connector/openCypher/templates/neighborsCountTemplate.test.ts

@@ -1,3 +1,4 @@
+import normalize from "../../../utils/testing/normalize";
 import neighborsCountTemplate from "./neighborsCountTemplate";
 
 describe("OpenCypher > neighborsCountTemplate", () => {
@@ -7,8 +8,15 @@ describe("OpenCypher > neighborsCountTemplate", () => {
       idType: "string",
     });
 
-    expect(template).toBe(
-      'MATCH (v) -[e]- (t) WHERE ID(v) = "12" RETURN labels(t) AS vertexLabel, count(DISTINCT t) AS count'
+    expect(normalize(template)).toBe(
+      normalize(
+        `
+          MATCH (v)-[]-(neighbor) 
+          WHERE ID(v) = "12" 
+          WITH DISTINCT neighbor 
+          RETURN labels(neighbor) AS vertexLabel, count(DISTINCT neighbor) AS count
+        `
+      )
     );
   });
 
@@ -19,8 +27,16 @@ describe("OpenCypher > neighborsCountTemplate", () => {
       limit: 20,
     });
 
-    expect(template).toBe(
-      'MATCH (v) -[e]- (t) WHERE ID(v) = "12" RETURN labels(t) AS vertexLabel, count(DISTINCT t) AS count LIMIT 20'
+    expect(normalize(template)).toBe(
+      normalize(
+        `
+        MATCH (v)-[]-(neighbor) 
+        WHERE ID(v) = "12"
+        WITH DISTINCT neighbor 
+        LIMIT 20
+        RETURN labels(neighbor) AS vertexLabel, count(DISTINCT neighbor) AS count 
+        `
+      )
     );
   });
 
@@ -31,8 +47,15 @@ describe("OpenCypher > neighborsCountTemplate", () => {
       limit: 0,
     });
 
-    expect(template).toBe(
-      'MATCH (v) -[e]- (t) WHERE ID(v) = "12" RETURN labels(t) AS vertexLabel, count(DISTINCT t) AS count'
+    expect(normalize(template)).toBe(
+      normalize(
+        `
+        MATCH (v)-[]-(neighbor)
+        WHERE ID(v) = "12"
+        WITH DISTINCT neighbor 
+        RETURN labels(neighbor) AS vertexLabel, count(DISTINCT neighbor) AS count
+        `
+      )
     );
   });
 });

packages/graph-explorer/src/connector/openCypher/templates/neighborsCountTemplate.ts

@@ -1,3 +1,4 @@
+import dedent from "dedent";
 import type { NeighborsCountRequest } from "../../useGEFetchTypes";
 
 /**
@@ -8,24 +9,21 @@ import type { NeighborsCountRequest } from "../../useGEFetchTypes";
  * ids = "44"
  * limit = 10
  *
- * MATCH (v) -[e]- (t)
+ * MATCH (v) -[]- (neighbor)
  * WHERE ID(v) = "44"
- * RETURN labels(t) AS vertexLabel, count(DISTINCT t) AS count
- * LIMIT 10
+ * WITH DISTINCT neighbor LIMIT 500
+ * RETURN labels(t) AS vertexLabel, count(neighbor) AS count
  *
  */
-const neighborsCountTemplate = ({
+export default function neighborsCountTemplate({
   vertexId,
   limit = 0,
-}: NeighborsCountRequest) => {
-  let template = "";
-  template = `MATCH (v) -[e]- (t) WHERE ID(v) = "${vertexId}" RETURN labels(t) AS vertexLabel, count(DISTINCT t) AS count`;
-
-  if (limit > 0) {
-    template += ` LIMIT ${limit}`;
-  }
-
-  return template;
-};
-
-export default neighborsCountTemplate;
+}: NeighborsCountRequest) {
+  return dedent`
+      MATCH (v)-[]-(neighbor)
+      WHERE ID(v) = "${vertexId}" 
+      WITH DISTINCT neighbor
+      ${limit > 0 ? `LIMIT ${limit}` : ``}
+      RETURN labels(neighbor) AS vertexLabel, count(DISTINCT neighbor) AS count
+    `;
+}

packages/graph-explorer/src/connector/openCypher/templates/oneHopTemplate.test.ts

@@ -1,3 +1,4 @@
+import normalize from "../../../utils/testing/normalize";
 import oneHopTemplate from "./oneHopTemplate";
 
 describe("OpenCypher > oneHopTemplate", () => {
@@ -7,8 +8,18 @@ describe("OpenCypher > oneHopTemplate", () => {
       idType: "string",
     });
 
-    expect(template).toBe(
-      'MATCH (v)-[e]-(tgt) WHERE ID(v) = "12" WITH collect(DISTINCT tgt) AS vObjects, collect({edge: e, sourceType: labels(v), targetType: labels(tgt)}) AS eObjects RETURN vObjects, eObjects'
+    expect(normalize(template)).toEqual(
+      normalize(`
+        MATCH (v)-[e]-(tgt)
+        WHERE ID(v) = "12"
+        WITH DISTINCT v, tgt 
+        ORDER BY toInteger(ID(tgt)) 
+        MATCH (v)-[e]-(tgt)
+        WITH
+          collect(DISTINCT tgt) AS vObjects,
+          collect({ edge: e, sourceType: labels(startNode(e)), targetType: labels(endNode(e)) }) AS eObjects
+        RETURN vObjects, eObjects
+      `)
     );
   });
 
@@ -20,8 +31,20 @@ describe("OpenCypher > oneHopTemplate", () => {
       limit: 5,
     });
 
-    expect(template).toBe(
-      'MATCH (v)-[e]-(tgt) WHERE ID(v) = "12" WITH collect(DISTINCT tgt)[..5] AS vObjects, collect({edge: e, sourceType: labels(v), targetType: labels(tgt)})[..5] AS eObjects RETURN vObjects, eObjects'
+    expect(normalize(template)).toBe(
+      normalize(`
+        MATCH (v)-[e]-(tgt) 
+        WHERE ID(v) = "12" 
+        WITH DISTINCT v, tgt 
+        ORDER BY toInteger(ID(tgt)) 
+        SKIP 5 
+        LIMIT 5
+        MATCH (v)-[e]-(tgt)
+        WITH 
+          collect(DISTINCT tgt) AS vObjects, 
+          collect({ edge: e, sourceType: labels(startNode(e)), targetType: labels(endNode(e)) }) AS eObjects 
+        RETURN vObjects, eObjects
+      `)
     );
   });
 
@@ -34,8 +57,68 @@ describe("OpenCypher > oneHopTemplate", () => {
       limit: 10,
     });
 
-    expect(template).toBe(
-      'MATCH (v)-[e]-(tgt:country) WHERE ID(v) = "12" WITH collect(DISTINCT tgt)[..10] AS vObjects, collect({edge: e, sourceType: labels(v), targetType: labels(tgt)})[..10] AS eObjects RETURN vObjects, eObjects'
+    expect(normalize(template)).toBe(
+      normalize(`
+        MATCH (v)-[e]-(tgt:country) 
+        WHERE ID(v) = "12" 
+        WITH DISTINCT v, tgt 
+        ORDER BY toInteger(ID(tgt)) 
+        SKIP 5 
+        LIMIT 10
+        MATCH (v)-[e]-(tgt)
+        WITH 
+          collect(DISTINCT tgt) AS vObjects, 
+          collect({ edge: e, sourceType: labels(startNode(e)), targetType: labels(endNode(e)) }) AS eObjects 
+        RETURN vObjects, eObjects
+      `)
+    );
+  });
+
+  it("Should return a template for many vertex types", () => {
+    const template = oneHopTemplate({
+      vertexId: "12",
+      idType: "string",
+      filterByVertexTypes: ["country", "continent", "airport", "person"],
+    });
+
+    expect(normalize(template)).toBe(
+      normalize(`
+        MATCH (v)-[e]-(tgt)
+        WHERE ID(v) = "12" AND (v:country OR v:continent OR v:airport OR v:person)
+        WITH DISTINCT v, tgt 
+        ORDER BY toInteger(ID(tgt)) 
+        MATCH (v)-[e]-(tgt)
+        WITH
+          collect(DISTINCT tgt) AS vObjects,
+          collect({ edge: e, sourceType: labels(startNode(e)), targetType: labels(endNode(e)) }) AS eObjects
+        RETURN vObjects, eObjects
+      `)
+    );
+  });
+
+  it("Should return a template for specific edge type", () => {
+    const template = oneHopTemplate({
+      vertexId: "12",
+      idType: "string",
+      edgeTypes: ["locatedIn"],
+      offset: 5,
+      limit: 10,
+    });
+
+    expect(normalize(template)).toBe(
+      normalize(`
+        MATCH (v)-[e:locatedIn]-(tgt) 
+        WHERE ID(v) = "12" 
+        WITH DISTINCT v, tgt 
+        ORDER BY toInteger(ID(tgt)) 
+        SKIP 5 
+        LIMIT 10
+        MATCH (v)-[e:locatedIn]-(tgt)
+        WITH 
+          collect(DISTINCT tgt) AS vObjects, 
+          collect({ edge: e, sourceType: labels(startNode(e)), targetType: labels(endNode(e)) }) AS eObjects 
+        RETURN vObjects, eObjects
+      `)
     );
   });
 
@@ -52,8 +135,20 @@ describe("OpenCypher > oneHopTemplate", () => {
       limit: 10,
     });
 
-    expect(template).toBe(
-      'MATCH (v)-[e]-(tgt:country) WHERE ID(v) = "12" AND tgt.longest >= 10000 AND tgt.country CONTAINS "ES" WITH collect(DISTINCT tgt)[..10] AS vObjects, collect({edge: e, sourceType: labels(v), targetType: labels(tgt)})[..10] AS eObjects RETURN vObjects, eObjects'
+    expect(normalize(template)).toBe(
+      normalize(`
+        MATCH (v)-[e]-(tgt:country) 
+        WHERE ID(v) = "12" AND tgt.longest >= 10000 AND tgt.country CONTAINS "ES" 
+        WITH DISTINCT v, tgt 
+        ORDER BY toInteger(ID(tgt)) 
+        SKIP 5 
+        LIMIT 10
+        MATCH (v)-[e]-(tgt)
+        WITH 
+          collect(DISTINCT tgt) AS vObjects, 
+          collect({ edge: e, sourceType: labels(startNode(e)), targetType: labels(endNode(e)) }) AS eObjects 
+        RETURN vObjects, eObjects
+      `)
     );
   });
 });

packages/graph-explorer/src/connector/openCypher/templates/oneHopTemplate.ts

@@ -1,3 +1,4 @@
+import dedent from "dedent";
 import type { Criterion, NeighborsRequest } from "../../useGEFetchTypes";
 
 const criterionNumberTemplate = ({
@@ -107,41 +108,47 @@ const oneHopTemplate = ({
   edgeTypes = [],
   filterCriteria = [],
   limit = 0,
+  offset = 0,
 }: Omit<NeighborsRequest, "vertexType">): string => {
-  let template = `MATCH (v)`;
-
-  const formattedVertexTypes = filterByVertexTypes
-    .flatMap(type => type.split("::"))
-    .map(type => `v:${type}`)
-    .join(" OR ");
+  // List of possible vertex types
+  const formattedVertexTypes =
+    filterByVertexTypes.length > 1
+      ? `(${filterByVertexTypes
+          .flatMap(type => type.split("::"))
+          .map(type => `v:${type}`)
+          .join(" OR ")})`
+      : "";
   const formattedEdgeTypes = edgeTypes.map(type => `${type}`).join("|");
 
-  if (edgeTypes.length > 0) {
-    template += `-[e:${formattedEdgeTypes}]-`;
-  } else {
-    template += `-[e]-`;
-  }
+  // Specify edge type if provided
+  const edgeMatch = edgeTypes.length > 0 ? `e:${formattedEdgeTypes}` : `e`;
 
-  if (filterByVertexTypes.length == 1) {
-    template += `(tgt:${filterByVertexTypes[0]}) WHERE ID(v) = "${vertexId}" `;
-  } else if (filterByVertexTypes.length > 1) {
-    template += `(tgt) WHERE ID(v) = "${vertexId}" AND ${formattedVertexTypes}`;
-  } else {
-    template += `(tgt) WHERE ID(v) = "${vertexId}" `;
-  }
+  // Specify node type for target if provided and only one
+  const targetMatch =
+    filterByVertexTypes.length == 1 ? `tgt:${filterByVertexTypes[0]}` : `tgt`;
 
-  const filterCriteriaTemplate = filterCriteria
-    ?.map(criterionTemplate)
+  // Combine all the WHERE conditions
+  const whereConditions = [
+    `ID(v) = "${vertexId}"`,
+    formattedVertexTypes,
+    ...(filterCriteria?.map(criterionTemplate) ?? []),
+  ]
+    .filter(Boolean)
     .join(" AND ");
-  if (filterCriteriaTemplate) {
-    template += `AND ${filterCriteriaTemplate} `;
-  }
-
-  const limitTemplate = limit > 0 ? `[..${limit}]` : "";
-
-  template += `WITH collect(DISTINCT tgt)${limitTemplate} AS vObjects, collect({edge: e, sourceType: labels(v), targetType: labels(tgt)})${limitTemplate} AS eObjects RETURN vObjects, eObjects`;
 
-  return template;
+  return dedent`
+    MATCH (v)-[${edgeMatch}]-(${targetMatch})
+    WHERE ${whereConditions}
+    WITH DISTINCT v, tgt
+    ORDER BY toInteger(ID(tgt))
+    ${limit > 0 && offset > 0 ? `SKIP ${offset}` : ``}
+    ${limit > 0 ? `LIMIT ${limit}` : ``}
+    MATCH (v)-[${edgeMatch}]-(tgt)
+    WITH
+      collect(DISTINCT tgt) AS vObjects, 
+      collect({ edge: e, sourceType: labels(startNode(e)), targetType: labels(endNode(e)) }) AS eObjects
+    RETURN vObjects, eObjects
+  `;
 };
 
 export default oneHopTemplate;

packages/graph-explorer/src/hooks/useEntities.test.tsx

@@ -2,9 +2,12 @@ import { useRecoilValue } from "recoil";
 import useEntities from "./useEntities";
 import { Vertex } from "../@types/entities";
 import {
+  createRandomEdge,
   createRandomEntities,
+  createRandomInteger,
   createRandomName,
   createRandomSchema,
+  createRandomVertex,
 } from "../utils/testing/randomData";
 import { schemaAtom } from "../core/StateProvider/schema";
 import { activeConfigurationAtom } from "../core/StateProvider/configuration";
@@ -220,6 +223,34 @@ describe("useEntities", () => {
     ).toEqual(0);
   });
 
+  it("should calculate stats after adding new nodes and edges", async () => {
+    const node1 = createRandomVertex();
+    const node2 = createRandomVertex();
+    const randomNeighborCount = createRandomInteger(500);
+    node1.data.neighborsCount = randomNeighborCount;
+    node1.data.neighborsCountByType[node2.data.type] = randomNeighborCount;
+    const edge1to2 = createRandomEdge(node1, node2);
+
+    const { result } = renderHookWithRecoilRoot(() => {
+      const [entities, setEntities] = useEntities({ disableFilters: true });
+      return { entities, setEntities };
+    });
+
+    result.current.setEntities({ nodes: [node1, node2], edges: [edge1to2] });
+
+    await waitForValueToChange(() => result.current.entities);
+
+    const actualNode1 = result.current.entities.nodes.find(
+      n => n.data.id === node1.data.id
+    )!;
+    expect(actualNode1.data.__unfetchedNeighborCount).toEqual(
+      randomNeighborCount - 1
+    );
+    expect(
+      actualNode1.data.__unfetchedNeighborCounts![node2.data.type]
+    ).toEqual(randomNeighborCount - 1);
+  });
+
   it("should return original entities before any filters were applied", async () => {
     // Define newNode and newEdge
     const newNode = {