From 81de19127af5c096cda61571d5accb952ef8fc58 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Fri, 18 Jan 2019 11:00:57 +0100 Subject: [PATCH 01/26] WIP --- .../java/org/neo4j/graphalgo/LouvainProc.java | 17 +- .../neo4j/graphalgo/impl/louvain/Louvain.java | 8 +- .../impl/louvain/ModularityOptimization.java | 31 +--- .../graphalgo/results/CommunityResult.java | 150 ++++++++++++++++++ .../graphalgo/results/HppcMapComparator.java | 56 +++++++ 5 files changed, 225 insertions(+), 37 deletions(-) create mode 100644 algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java create mode 100644 algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index c79df1a2e..9d7e7eaf4 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -30,6 +30,7 @@ import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.impl.louvain.*; +import org.neo4j.graphalgo.results.CommunityResult; import org.neo4j.graphalgo.results.LouvainResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.impl.store.PropertyType; @@ -66,7 +67,7 @@ public class LouvainProc { @Description("CALL algo.louvain(label:String, relationship:String, " + "{weightProperty:'weight', defaultValue:1.0, write: true, writeProperty:'community', concurrency:4, communityProperty:'propertyOfPredefinedCommunity'}) " + "YIELD nodes, communityCount, iterations, loadMillis, computeMillis, writeMillis") - public Stream louvain( + public Stream louvain( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { @@ -75,14 +76,14 @@ public Stream louvain( .overrideNodeLabelOrQuery(label) .overrideRelationshipTypeOrQuery(relationship); - LouvainResult.Builder builder = LouvainResult.builder(); + final CommunityResult.Builder builder = CommunityResult.builder(); final Graph graph; try (ProgressTimer timer = builder.timeLoad()) { graph = graph(label, relationship, configuration); } - builder.withNodeCount(graph.nodeCount()); + builder.withNodes(graph.nodeCount()); if(graph.nodeCount() == 0) { graph.release(); @@ -102,10 +103,12 @@ public Stream louvain( } else { louvain.compute(configuration.getIterations(10), configuration.get("innerIterations", 10)); } - builder.withIterations(louvain.getLevel()) - .withCommunityCount(louvain.getCommunityCount()) - .withModularities(louvain.getModularities()) - .withFinalModularity(louvain.getFinalModularity()); + + builder.withCommunities(louvain.getCommunityIds()); + builder.withIterations(louvain.getLevel()); // in real we have threads * innerIterations * maxLevel iterations +// builder.withModularities(louvain.getModularities()) + builder.withConvergence(true); // only subsequent modularity optimizations can converge + } if (configuration.isWriteFlag()) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/Louvain.java b/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/Louvain.java index fdbb552e1..692c677d4 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/Louvain.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/Louvain.java @@ -64,7 +64,7 @@ public class Louvain extends Algorithm { private int[][] dendrogram; private double[] nodeWeights; private Graph root; - private int communityCount = 0; + private int communityCount; public Louvain(Graph graph, ExecutorService pool, @@ -137,7 +137,7 @@ public Louvain compute(WeightMapping communityMap, int maxLevel, int maxIteratio dendrogram = new int[maxLevel][]; modularities = new double[maxLevel]; - for (level = 0; level < maxLevel; level++) { + for (level = 0; level < maxLevel && terminationFlag.running(); level++) { // start modularity optimization final ModularityOptimization modularityOptimization = new ModularityOptimization(graph, @@ -213,9 +213,7 @@ private int[] rebuildCommunityStructure(int[] communityIds) { // rebuild community array assert rootNodeCount == communities.length; final int[] ints = new int[rootNodeCount]; - Arrays.setAll(ints, i -> { - return communityIds[communities[i]]; - }); + Arrays.setAll(ints, i -> communityIds[communities[i]]); communities = ints; return communities; } diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/ModularityOptimization.java b/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/ModularityOptimization.java index e96009d02..037f8f855 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/ModularityOptimization.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/ModularityOptimization.java @@ -29,6 +29,7 @@ import org.neo4j.graphalgo.core.utils.ParallelUtil; import org.neo4j.graphalgo.core.utils.Pointer; import org.neo4j.graphalgo.core.utils.ProgressLogger; +import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.impl.Algorithm; import org.neo4j.graphdb.Direction; @@ -170,6 +171,7 @@ private void init() { * @return */ public ModularityOptimization compute(int maxIterations) { + final TerminationFlag terminationFlag = getTerminationFlag(); // init helper values & initial community structure init(); // create an array of tasks for parallel exec @@ -180,7 +182,7 @@ public ModularityOptimization compute(int maxIterations) { // (2x double + 1x int) * N * threads tracker.add(20 * nodeCount * concurrency); // as long as maxIterations is not reached - for (iterations = 0; iterations < maxIterations; iterations++) { + for (iterations = 0; iterations < maxIterations && terminationFlag.running(); iterations++) { // reset node counter (for logging) counter.set(0); // run all tasks @@ -267,6 +269,7 @@ private class Task implements Runnable { final double[] sTot, sIn; final int[] localCommunities; + private final TerminationFlag terminationFlag; double bestGain, bestWeight, q = MINIMUM_MODULARITY; int bestCommunity; boolean improvement = false; @@ -276,6 +279,7 @@ private class Task implements Runnable { * and initializes its helper arrays */ Task() { + terminationFlag = getTerminationFlag(); sTot = new double[nodeCount]; System.arraycopy(ki, 0, sTot, 0, nodeCount); // ki -> sTot localCommunities = new int[nodeCount]; @@ -307,7 +311,7 @@ public void run() { counter.getAndIncrement(), denominator, () -> String.format("round %d", iterations + 1)); - return true; + return terminationFlag.running(); }); this.q = calcModularity(); } @@ -383,29 +387,6 @@ private void removeWeightForSelfRelationships(int node, IntDoubleMap communityWe }); } - - /** - * apply consumer to each connected community one time - * - * @param node node nodeId - * @param consumer community nodeId consumer - */ - private void forEachConnectedCommunity(int node, IntConsumer consumer) { - final BitSet visited = new BitSet(nodeCount); - graph.forEachRelationship(node, D, (s, t, r) -> { - final int c = localCommunities[t]; - if (c == NONE) { - return true; - } - if (visited.get(c)) { - return true; - } - visited.set(c); - consumer.accept(c); - return true; - }); - } - /** * sum weights from node into community c * diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java new file mode 100644 index 000000000..a202e2a42 --- /dev/null +++ b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java @@ -0,0 +1,150 @@ +package org.neo4j.graphalgo.results; + +import com.carrotsearch.hppc.IntIntHashMap; +import com.carrotsearch.hppc.cursors.IntIntCursor; +import com.carrotsearch.hppc.sorting.IndirectComparator; +import com.carrotsearch.hppc.sorting.IndirectSort; +import org.HdrHistogram.Histogram; + +import java.util.Arrays; +import java.util.Iterator; + +/** + * @author mknblch + */ +public class CommunityResult { + + /* + + YIELD + + */ + + public final long loadDuration; + public final long evalDuration; + public final long writeDuration; + public final long nodes; + public final long communityCount; + public final long iterations; + public final boolean convergence; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final int[] biggestCommunities; + + CommunityResult(long loadDuration, + long evalDuration, + long writeDuration, + long nodes, + long communityCount, + long iterations, + boolean convergence, + long p99, + long p95, + long p90, + long p75, + long p50, + int[] biggestCommunities) { + this.loadDuration = loadDuration; + this.evalDuration = evalDuration; + this.writeDuration = writeDuration; + this.nodes = nodes; + this.communityCount = communityCount; + this.iterations = iterations; + this.convergence = convergence; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.biggestCommunities = biggestCommunities; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder extends AbstractResultBuilder { + + private long nodes = 1; + private long iterations = 1; + private int[] communities = new int[]{}; + private boolean convergence = false; + + public Builder withIterations(long iterations) { + this.iterations = iterations; + return this; + } + + public Builder withConvergence(boolean convergence) { + this.convergence = convergence; + return this; + } + + public Builder withNodes(long nodes) { + this.nodes = nodes; + return this; + } + + public Builder withCommunities(int[] communities) { + this.communities = communities; + return this; + } + + private static int[] mapSortTopN(IntIntHashMap map, int topN) { + return mapSortTopN_(topN, map.keys, new HppcMapComparator(map)); + } + + public static int[] mapSortTopN_dense(IntIntHashMap map, int topN) { + int size = map.size(); + int[] keys = new int[size]; + int[] values = new int[size]; + Iterator cursor = map.iterator(); + for (int index = 0; cursor.hasNext(); ++index) { + IntIntCursor entry = cursor.next(); + keys[index] = entry.key; + values[index] = entry.value; + } + + return mapSortTopN_(topN, keys, new IndirectComparator.DescendingIntComparator(values)); + } + + private static int[] mapSortTopN_(int topN, int[] keys, IndirectComparator comp) { + int[] sortedKeys = IndirectSort.mergesort(0, keys.length, comp); + topN = Math.min(topN, keys.length); + for (int i = 0; i < topN; i++) { + sortedKeys[i] = keys[sortedKeys[i]]; + } + return Arrays.copyOf(sortedKeys, topN); + } + + @Override + public CommunityResult build() { + // evaluate count and biggest communities + final IntIntHashMap map = new IntIntHashMap(); + final Histogram histogram = new Histogram(0); + + for (int i = 0; i < communities.length; i++) { + map.addTo(communities[i], 1); + histogram.recordValue(communities[i]); + } + + return new CommunityResult( + loadDuration, + evalDuration, + writeDuration, + nodes, + map.size(), + iterations, convergence, + histogram.getValueAtPercentile(.99), + histogram.getValueAtPercentile(.95), + histogram.getValueAtPercentile(.90), + histogram.getValueAtPercentile(.75), + histogram.getValueAtPercentile(.50), + mapSortTopN(map, 3)); + } + } + +} diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java b/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java new file mode 100644 index 000000000..8cb848a58 --- /dev/null +++ b/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java @@ -0,0 +1,56 @@ +package org.neo4j.graphalgo.results; + +import com.carrotsearch.hppc.IntIntHashMap; +import com.carrotsearch.hppc.sorting.IndirectComparator; + +public final class HppcMapComparator implements IndirectComparator { + + private final int[] values; + private final int[] keys; + private final int max; + private boolean hasEmptyKey; + + HppcMapComparator(IntIntHashMap map) { + values = map.values; + keys = map.keys; + max = keys.length - 1; + int lastValue = values[max]; + hasEmptyKey = map.getOrDefault(0, ~lastValue) == lastValue; + } + + @Override + public int compare(final int indexA, final int indexB) { + assert validIndex(indexA); + assert validIndex(indexB); + + // assigned slot for A + if ((keys[indexA] != 0) || (indexA == max && hasEmptyKey)) { + + // assigned slot for B + if ((keys[indexB] != 0) || (indexB == max && hasEmptyKey)) { + + // reverse order because descending + return Integer.compare(values[indexB], values[indexA]); + } + + // empty slot for B, put it at the end, A is "smaller" + return -1; + } + + // assigned slot for B + if ((keys[indexB] != 0) || (indexB == max && hasEmptyKey)) { + + // empty slot for A, put it at the end, B is "smaller" + return 1; + } + + // empty slot for A and B, both are equal + return 0; + } + + private boolean validIndex(int index) { + assert index >= 0 : "The index " + index + " must point at an existing key."; + assert index < max || (index == max && hasEmptyKey); + return true; + } + } \ No newline at end of file From 8f8d6d831d7b253521f5df1111a2f8fb587813d7 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Fri, 18 Jan 2019 14:18:17 +0100 Subject: [PATCH 02/26] impl unified community algo result, chg louvain result --- .../graphalgo/results/CommunityResult.java | 26 ++++++++++--------- .../graphalgo/results/HppcMapComparator.java | 2 +- .../LouvainClusteringIntegrationTest.java | 5 +++- .../graphalgo/impl/LouvainMultiLevelTest.java | 2 ++ .../impl/LouvainWeightedGraphTest.java | 2 ++ 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java index a202e2a42..7b0dc6d8c 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java @@ -8,6 +8,8 @@ import java.util.Arrays; import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; /** * @author mknblch @@ -20,9 +22,9 @@ public class CommunityResult { */ - public final long loadDuration; - public final long evalDuration; - public final long writeDuration; + public final long loadMillis; + public final long computeMillis; + public final long writeMillis; public final long nodes; public final long communityCount; public final long iterations; @@ -32,11 +34,11 @@ public class CommunityResult { public final long p90; public final long p75; public final long p50; - public final int[] biggestCommunities; + public final List top; - CommunityResult(long loadDuration, - long evalDuration, - long writeDuration, + CommunityResult(long loadMillis, + long computeMillis, + long writeMillis, long nodes, long communityCount, long iterations, @@ -47,9 +49,9 @@ public class CommunityResult { long p75, long p50, int[] biggestCommunities) { - this.loadDuration = loadDuration; - this.evalDuration = evalDuration; - this.writeDuration = writeDuration; + this.loadMillis = loadMillis; + this.computeMillis = computeMillis; + this.writeMillis = writeMillis; this.nodes = nodes; this.communityCount = communityCount; this.iterations = iterations; @@ -59,7 +61,7 @@ public class CommunityResult { this.p90 = p90; this.p75 = p75; this.p50 = p50; - this.biggestCommunities = biggestCommunities; + this.top = Arrays.stream(biggestCommunities).asLongStream().boxed().collect(Collectors.toList()); } public static Builder builder() { @@ -69,7 +71,7 @@ public static Builder builder() { public static class Builder extends AbstractResultBuilder { private long nodes = 1; - private long iterations = 1; + private long iterations = -1; private int[] communities = new int[]{}; private boolean convergence = false; diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java b/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java index 8cb848a58..6739cb340 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java @@ -50,7 +50,7 @@ public int compare(final int indexA, final int indexB) { private boolean validIndex(int index) { assert index >= 0 : "The index " + index + " must point at an existing key."; - assert index < max || (index == max && hasEmptyKey); + assert index <= max || (index == max && hasEmptyKey); return true; } } \ No newline at end of file diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java index 0d431c11b..b35765750 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java @@ -98,7 +98,7 @@ public void clearCommunities() { @Test public void test() { final String cypher = "CALL algo.louvain('', '', {concurrency:1}) " + - "YIELD nodes, communityCount, iterations, loadMillis, computeMillis, writeMillis"; + "YIELD nodes, communityCount, iterations, loadMillis, computeMillis, writeMillis, top, p99"; DB.execute(cypher).accept(row -> { final long nodes = row.getNumber("nodes").longValue(); @@ -110,6 +110,9 @@ public void test() { System.out.println("nodes = " + nodes); System.out.println("communityCount = " + communityCount); System.out.println("iterations = " + iterations); + System.out.println("p99 = " + row.get("p99")); + System.out.println("top = " + row.get("top")); + assertEquals("invalid node count",9, nodes); assertEquals("wrong community count", 3, communityCount); assertTrue("invalid loadTime", loadMillis >= 0); diff --git a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainMultiLevelTest.java b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainMultiLevelTest.java index 0ab2a837b..c2d69aedc 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainMultiLevelTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainMultiLevelTest.java @@ -30,6 +30,7 @@ import org.neo4j.graphalgo.core.heavyweight.HeavyGraphFactory; import org.neo4j.graphalgo.core.huge.HugeGraphFactory; import org.neo4j.graphalgo.core.utils.Pools; +import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.impl.louvain.Louvain; import org.neo4j.graphdb.Label; @@ -129,6 +130,7 @@ public void testComplex() throws Exception { setup(COMPLEX_CYPHER); final Louvain algorithm = new Louvain(graph, Pools.DEFAULT, 1, AllocationTracker.EMPTY) .withProgressLogger(TestProgressLogger.INSTANCE) + .withTerminationFlag(TerminationFlag.RUNNING_TRUE) .compute(10, 10); final int[][] dendogram = algorithm.getDendrogram(); for (int i = 1; i <= dendogram.length; i++) { diff --git a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainWeightedGraphTest.java b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainWeightedGraphTest.java index 19d387915..207023686 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainWeightedGraphTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainWeightedGraphTest.java @@ -31,6 +31,7 @@ import org.neo4j.graphalgo.core.heavyweight.HeavyGraphFactory; import org.neo4j.graphalgo.core.huge.HugeGraphFactory; import org.neo4j.graphalgo.core.utils.Pools; +import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.core.utils.paged.HugeLongArray; import org.neo4j.graphalgo.impl.louvain.*; @@ -148,6 +149,7 @@ public void testWeightedLouvain() throws Exception { final Louvain louvain = new Louvain(graph,Pools.DEFAULT, 1, AllocationTracker.EMPTY) .withProgressLogger(TestProgressLogger.INSTANCE) + .withTerminationFlag(TerminationFlag.RUNNING_TRUE) .compute(10, 10); final int[][] dendogram = louvain.getDendrogram(); From 610b6a6274164473f4f35307a9daac4576e73139 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Fri, 18 Jan 2019 14:57:51 +0100 Subject: [PATCH 03/26] adapt UnionFind result to CommunityResult --- .../org/neo4j/graphalgo/UnionFindProc.java | 3 ++- .../org/neo4j/graphalgo/UnionFindProc2.java | 3 ++- .../org/neo4j/graphalgo/UnionFindProc3.java | 3 ++- .../org/neo4j/graphalgo/UnionFindProc4.java | 3 ++- .../org/neo4j/graphalgo/impl/DSSResult.java | 16 ++++++++++++ .../graphalgo/impl/UnionFindProcExec.java | 25 +++++++++++++------ .../graphalgo/results/CommunityResult.java | 23 +++++++++++++---- .../core/utils/dss/DisjointSetStruct.java | 14 +++++------ .../utils/paged/PagedDisjointSetStruct.java | 4 +++ .../LouvainClusteringIntegrationTest.java | 4 +-- .../algo/UnionFindProcIntegrationTest.java | 12 ++++----- 11 files changed, 78 insertions(+), 32 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java index 899803288..34660c50b 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java @@ -21,6 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; +import org.neo4j.graphalgo.results.CommunityResult; import org.neo4j.graphalgo.results.UnionFindResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -52,7 +53,7 @@ public class UnionFindProc { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{weightProperty:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition'}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java index 1de2545cf..6e6283694 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java @@ -21,6 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; +import org.neo4j.graphalgo.results.CommunityResult; import org.neo4j.graphalgo.results.UnionFindResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -52,7 +53,7 @@ public class UnionFindProc2 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition',concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java index 5cd9ec59a..ebea457c8 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java @@ -21,6 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; +import org.neo4j.graphalgo.results.CommunityResult; import org.neo4j.graphalgo.results.UnionFindResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -52,7 +53,7 @@ public class UnionFindProc3 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition', concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java index 7ff7b1249..42000b691 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java @@ -21,6 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; +import org.neo4j.graphalgo.results.CommunityResult; import org.neo4j.graphalgo.results.UnionFindResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -52,7 +53,7 @@ public class UnionFindProc4 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition',concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/DSSResult.java b/algo/src/main/java/org/neo4j/graphalgo/impl/DSSResult.java index c3352e408..94e0bff8e 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/DSSResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/DSSResult.java @@ -27,9 +27,12 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.core.utils.paged.PagedDisjointSetStruct; +import java.util.Arrays; import java.util.stream.Stream; public final class DSSResult { + + public final boolean isHuge; public final DisjointSetStruct struct; public final PagedDisjointSetStruct hugeStruct; @@ -45,6 +48,19 @@ private DSSResult(DisjointSetStruct struct, PagedDisjointSetStruct hugeStruct) { assert (struct != null && hugeStruct == null) || (struct == null && hugeStruct != null); this.struct = struct; this.hugeStruct = hugeStruct; + isHuge = hugeStruct != null; + } + + public int[] getCommunities() { + + if (isHuge) { + return new int[0]; // not supported + } + + final int size = struct.capacity(); + final int[] communities = new int[size]; + Arrays.parallelSetAll(communities, struct::findNoOpt); + return communities; } public int getSetCount() { diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java index 8dd4d80c1..bdebc187e 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java @@ -19,6 +19,7 @@ package org.neo4j.graphalgo.impl; import org.neo4j.graphalgo.api.Graph; +import org.neo4j.graphalgo.api.HugeGraph; import org.neo4j.graphalgo.core.GraphLoader; import org.neo4j.graphalgo.core.ProcedureConfiguration; import org.neo4j.graphalgo.core.utils.Pools; @@ -29,6 +30,7 @@ import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.core.utils.paged.PagedDisjointSetStruct; import org.neo4j.graphalgo.core.write.Exporter; +import org.neo4j.graphalgo.results.CommunityResult; import org.neo4j.graphalgo.results.UnionFindResult; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.api.KernelTransaction; @@ -53,7 +55,7 @@ public final class UnionFindProcExec implements BiConsumer> private final UnionFindAlgo sequential; private final UnionFindAlgo parallel; - public static Stream run( + public static Stream run( Map config, String label, String relationship, @@ -64,7 +66,8 @@ public static Stream run( .overrideRelationshipTypeOrQuery(relationship); AllocationTracker tracker = AllocationTracker.create(); - UnionFindResult.Builder builder = UnionFindResult.builder(); + + final CommunityResult.Builder builder = CommunityResult.builder(); UnionFindProcExec uf = unionFind.get(); @@ -73,8 +76,8 @@ public static Stream run( if (graph.nodeCount() == 0) { graph.release(); return Stream.of(builder - .withNodeCount(graph.nodeCount()) - .withSetCount(0) + .withNodes(graph.nodeCount()) + .withoutCommunities() .build()); } @@ -89,10 +92,16 @@ public static Stream run( uf.write(builder::timeWrite, graph, dssResult, configuration); } - return Stream.of(builder - .withNodeCount(graph.nodeCount()) - .withSetCount(dssResult.getSetCount()) - .build()); + final CommunityResult.Builder result = builder + .withNodes(graph.nodeCount()); + + if (graph instanceof HugeGraph) { + result.withCommunityCount(dssResult.getSetCount()); + } else { + result.withCommunities(dssResult.getCommunities()); + } + + return Stream.of(result.build()); } public static Stream stream( diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java index 7b0dc6d8c..05a723bbb 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java @@ -18,7 +18,7 @@ public class CommunityResult { /* - YIELD + YIELD loadMillis, computeMillis, writeMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, top3 */ @@ -34,7 +34,7 @@ public class CommunityResult { public final long p90; public final long p75; public final long p50; - public final List top; + public final List top3; CommunityResult(long loadMillis, long computeMillis, @@ -61,7 +61,7 @@ public class CommunityResult { this.p90 = p90; this.p75 = p75; this.p50 = p50; - this.top = Arrays.stream(biggestCommunities).asLongStream().boxed().collect(Collectors.toList()); + this.top3 = Arrays.stream(biggestCommunities).asLongStream().boxed().collect(Collectors.toList()); } public static Builder builder() { @@ -74,6 +74,7 @@ public static class Builder extends AbstractResultBuilder { private long iterations = -1; private int[] communities = new int[]{}; private boolean convergence = false; + private long communityCount = -1; public Builder withIterations(long iterations) { this.iterations = iterations; @@ -95,6 +96,17 @@ public Builder withCommunities(int[] communities) { return this; } + // overwrite community count + public Builder withCommunityCount(long communityCount) { + this.communityCount = communityCount; + return this; + } + + public Builder withoutCommunities() { + this.communities = new int[0]; + return this; + } + private static int[] mapSortTopN(IntIntHashMap map, int topN) { return mapSortTopN_(topN, map.keys, new HppcMapComparator(map)); } @@ -138,8 +150,9 @@ public CommunityResult build() { evalDuration, writeDuration, nodes, - map.size(), - iterations, convergence, + communityCount == -1 ? map.size() : communityCount, + iterations, + convergence, histogram.getValueAtPercentile(.99), histogram.getValueAtPercentile(.95), histogram.getValueAtPercentile(.90), diff --git a/core/src/main/java/org/neo4j/graphalgo/core/utils/dss/DisjointSetStruct.java b/core/src/main/java/org/neo4j/graphalgo/core/utils/dss/DisjointSetStruct.java index 27f6affd7..8bca215b4 100644 --- a/core/src/main/java/org/neo4j/graphalgo/core/utils/dss/DisjointSetStruct.java +++ b/core/src/main/java/org/neo4j/graphalgo/core/utils/dss/DisjointSetStruct.java @@ -122,11 +122,11 @@ public Stream resultStream(IdMapping idMapping) { } /** - * element count + * element (node) count * * @return the element count */ - public int count() { + public int capacity() { return parent.length; } @@ -234,11 +234,11 @@ public IntIntMap getSetSize() { public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("\n"); - for (int i = 0; i < count(); i++) { + for (int i = 0; i < capacity(); i++) { builder.append(String.format(" %d ", i)); } builder.append("\n"); - for (int i = 0; i < count(); i++) { + for (int i = 0; i < capacity(); i++) { builder.append(String.format("[%d]", find(i))); } return builder.toString(); @@ -282,7 +282,7 @@ private static class NodeSetIterator implements Iterator { private NodeSetIterator(DisjointSetStruct struct) { this.struct = struct; - this.length = struct.count(); + this.length = struct.capacity(); } @Override @@ -311,8 +311,8 @@ private static class ConcurrentNodeSetIterator implements Iterator { private ConcurrentNodeSetIterator(DisjointSetStruct struct, int startOffset, int length) { this.struct = struct; - this.length = length + offset > struct.count() - ? struct.count() - offset + this.length = length + offset > struct.capacity() + ? struct.capacity() - offset : length; this.offset = startOffset; } diff --git a/core/src/main/java/org/neo4j/graphalgo/core/utils/paged/PagedDisjointSetStruct.java b/core/src/main/java/org/neo4j/graphalgo/core/utils/paged/PagedDisjointSetStruct.java index 3b1bcfdb1..f00b30617 100644 --- a/core/src/main/java/org/neo4j/graphalgo/core/utils/paged/PagedDisjointSetStruct.java +++ b/core/src/main/java/org/neo4j/graphalgo/core/utils/paged/PagedDisjointSetStruct.java @@ -44,6 +44,10 @@ public PagedDisjointSetStruct reset() { return this; } + public long capacity() { + return capacity; + } + public boolean connected(long p, long q) { return find(p) == find(q); } diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java index b35765750..84a18a11e 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java @@ -98,7 +98,7 @@ public void clearCommunities() { @Test public void test() { final String cypher = "CALL algo.louvain('', '', {concurrency:1}) " + - "YIELD nodes, communityCount, iterations, loadMillis, computeMillis, writeMillis, top, p99"; + "YIELD nodes, communityCount, iterations, loadMillis, computeMillis, writeMillis, top3, p99"; DB.execute(cypher).accept(row -> { final long nodes = row.getNumber("nodes").longValue(); @@ -111,7 +111,7 @@ public void test() { System.out.println("communityCount = " + communityCount); System.out.println("iterations = " + iterations); System.out.println("p99 = " + row.get("p99")); - System.out.println("top = " + row.get("top")); + System.out.println("top3 = " + row.get("top3")); assertEquals("invalid node count",9, nodes); assertEquals("wrong community count", 3, communityCount); diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java index 0a57f9716..5ba32d3b9 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java @@ -110,29 +110,29 @@ public static Collection data() { @Test public void testUnionFind() throws Exception { - db.execute("CALL algo.unionFind('', '',{graph:'"+graphImpl+"'}) YIELD setCount") + db.execute("CALL algo.unionFind('', '',{graph:'"+graphImpl+"'}) YIELD communityCount") .accept((Result.ResultVisitor) row -> { - assertEquals(3L, row.getNumber("setCount")); + assertEquals(3L, row.getNumber("communityCount")); return true; }); } @Test public void testUnionFindWithLabel() throws Exception { - db.execute("CALL algo.unionFind('Label', '',{graph:'"+graphImpl+"'}) YIELD setCount") + db.execute("CALL algo.unionFind('Label', '',{graph:'"+graphImpl+"'}) YIELD communityCount") .accept((Result.ResultVisitor) row -> { - assertEquals(1L, row.getNumber("setCount")); + assertEquals(1L, row.getNumber("communityCount")); return true; }); } @Test public void testUnionFindWriteBack() throws Exception { - db.execute("CALL algo.unionFind('', 'TYPE', {write:true,graph:'"+graphImpl+"'}) YIELD setCount, writeMillis, nodes") + db.execute("CALL algo.unionFind('', 'TYPE', {write:true,graph:'"+graphImpl+"'}) YIELD communityCount, writeMillis, nodes") .accept((Result.ResultVisitor) row -> { assertNotEquals(-1L, row.getNumber("writeMillis")); assertNotEquals(-1L, row.getNumber("nodes")); - assertEquals(3L, row.getNumber("setCount")); + assertEquals(3L, row.getNumber("communityCount")); return false; }); } From f195db16bde046a14156e2ea6d12987c2d5aba92 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Fri, 18 Jan 2019 16:18:30 +0100 Subject: [PATCH 04/26] WIP --- .../java/org/neo4j/graphalgo/LouvainProc.java | 6 +- .../graphalgo/impl/UnionFindProcExec.java | 13 +- .../graphalgo/results/CommunityResult.java | 116 +++++++----------- .../graphalgo/results/HppcMapComparator.java | 2 +- .../graphalgo/results/LouvainResult.java | 36 +++--- .../graphalgo/results/UnionFindResult.java | 79 ++++++++---- 6 files changed, 122 insertions(+), 130 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index 9d7e7eaf4..b6f33673c 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -21,9 +21,6 @@ import org.neo4j.graphalgo.api.*; import org.neo4j.graphalgo.core.GraphLoader; import org.neo4j.graphalgo.core.ProcedureConfiguration; -import org.neo4j.graphalgo.core.ProcedureConstants; -import org.neo4j.graphalgo.core.heavyweight.HeavyGraph; -import org.neo4j.graphalgo.core.heavyweight.HeavyGraphFactory; import org.neo4j.graphalgo.core.utils.Pools; import org.neo4j.graphalgo.core.utils.ProgressLogger; import org.neo4j.graphalgo.core.utils.ProgressTimer; @@ -33,7 +30,6 @@ import org.neo4j.graphalgo.results.CommunityResult; import org.neo4j.graphalgo.results.LouvainResult; import org.neo4j.kernel.api.KernelTransaction; -import org.neo4j.kernel.impl.store.PropertyType; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; import org.neo4j.procedure.*; @@ -76,7 +72,7 @@ public Stream louvain( .overrideNodeLabelOrQuery(label) .overrideRelationshipTypeOrQuery(relationship); - final CommunityResult.Builder builder = CommunityResult.builder(); + final LouvainResult.Builder builder = LouvainResult.builder(); final Graph graph; try (ProgressTimer timer = builder.timeLoad()) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java index bdebc187e..67ccf41db 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java @@ -67,7 +67,7 @@ public static Stream run( AllocationTracker tracker = AllocationTracker.create(); - final CommunityResult.Builder builder = CommunityResult.builder(); + final UnionFindResult.Builder builder = UnionFindResult.builder(); UnionFindProcExec uf = unionFind.get(); @@ -92,16 +92,7 @@ public static Stream run( uf.write(builder::timeWrite, graph, dssResult, configuration); } - final CommunityResult.Builder result = builder - .withNodes(graph.nodeCount()); - - if (graph instanceof HugeGraph) { - result.withCommunityCount(dssResult.getSetCount()); - } else { - result.withCommunities(dssResult.getCommunities()); - } - - return Stream.of(result.build()); + return Stream.of(builder.withNodes(graph.nodeCount()).withDSSResult(dssResult)); } public static Stream stream( diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java index 05a723bbb..c7db18b07 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java @@ -1,20 +1,21 @@ package org.neo4j.graphalgo.results; import com.carrotsearch.hppc.IntIntHashMap; -import com.carrotsearch.hppc.cursors.IntIntCursor; +import com.carrotsearch.hppc.LongLongHashMap; import com.carrotsearch.hppc.sorting.IndirectComparator; import com.carrotsearch.hppc.sorting.IndirectSort; import org.HdrHistogram.Histogram; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Iterator; import java.util.List; +import java.util.function.ToLongFunction; import java.util.stream.Collectors; /** * @author mknblch */ -public class CommunityResult { +public abstract class CommunityResult { /* @@ -64,102 +65,79 @@ public class CommunityResult { this.top3 = Arrays.stream(biggestCommunities).asLongStream().boxed().collect(Collectors.toList()); } - public static Builder builder() { - return new Builder(); - } - - public static class Builder extends AbstractResultBuilder { + public static abstract class AbstractCommunityBuilder extends AbstractResultBuilder { - private long nodes = 1; - private long iterations = -1; - private int[] communities = new int[]{}; - private boolean convergence = false; - private long communityCount = -1; + protected long nodes = 1; + protected long iterations = -1; + protected boolean convergence = false; + protected long communityCount = -1; + protected final Histogram histogram = new Histogram(2); + protected final IntIntHashMap communityMap = new IntIntHashMap(); - public Builder withIterations(long iterations) { + public AbstractCommunityBuilder withIterations(long iterations) { this.iterations = iterations; return this; } - public Builder withConvergence(boolean convergence) { + public AbstractCommunityBuilder withConvergence(boolean convergence) { this.convergence = convergence; return this; } - public Builder withNodes(long nodes) { + public AbstractCommunityBuilder withNodes(long nodes) { this.nodes = nodes; return this; } - public Builder withCommunities(int[] communities) { - this.communities = communities; + public AbstractCommunityBuilder withCommunities(int[] communities) { + for (int i = 0; i < communities.length; i++) { + communityMap.addTo(communities[i], 1); + histogram.recordValue(communities[i]); + } return this; } // overwrite community count - public Builder withCommunityCount(long communityCount) { + public AbstractCommunityBuilder withCommunityCount(long communityCount) { this.communityCount = communityCount; return this; } - public Builder withoutCommunities() { - this.communities = new int[0]; + public AbstractCommunityBuilder withoutCommunities() { return this; } - private static int[] mapSortTopN(IntIntHashMap map, int topN) { - return mapSortTopN_(topN, map.keys, new HppcMapComparator(map)); - } - - public static int[] mapSortTopN_dense(IntIntHashMap map, int topN) { - int size = map.size(); - int[] keys = new int[size]; - int[] values = new int[size]; - Iterator cursor = map.iterator(); - for (int index = 0; cursor.hasNext(); ++index) { - IntIntCursor entry = cursor.next(); - keys[index] = entry.key; - values[index] = entry.value; + public static List top3(long elements, ToLongFunction fun) { + + long t1 = -1L, t2 = -1L, t3 = -1L; + + for (long i = 0; i < elements; i++) { + final long r = fun.applyAsLong(i); + if (r > t1) { + t3 = t2; + t2 = t1; + t1 = r; + continue; + } + if (r > t2) { + t3 = t2; + t2 = r; + continue; + } + if (r > t3) { + t3 = r; + } } - return mapSortTopN_(topN, keys, new IndirectComparator.DescendingIntComparator(values)); + final ArrayList longs = new ArrayList<>(); + longs.add(t1); + longs.add(t2); + longs.add(t3); + return longs; } - private static int[] mapSortTopN_(int topN, int[] keys, IndirectComparator comp) { - int[] sortedKeys = IndirectSort.mergesort(0, keys.length, comp); - topN = Math.min(topN, keys.length); - for (int i = 0; i < topN; i++) { - sortedKeys[i] = keys[sortedKeys[i]]; - } - return Arrays.copyOf(sortedKeys, topN); - } - - @Override - public CommunityResult build() { - // evaluate count and biggest communities - final IntIntHashMap map = new IntIntHashMap(); - final Histogram histogram = new Histogram(0); + public abstract T build(); - for (int i = 0; i < communities.length; i++) { - map.addTo(communities[i], 1); - histogram.recordValue(communities[i]); - } - - return new CommunityResult( - loadDuration, - evalDuration, - writeDuration, - nodes, - communityCount == -1 ? map.size() : communityCount, - iterations, - convergence, - histogram.getValueAtPercentile(.99), - histogram.getValueAtPercentile(.95), - histogram.getValueAtPercentile(.90), - histogram.getValueAtPercentile(.75), - histogram.getValueAtPercentile(.50), - mapSortTopN(map, 3)); - } } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java b/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java index 6739cb340..d3b1169dc 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java @@ -50,7 +50,7 @@ public int compare(final int indexA, final int indexB) { private boolean validIndex(int index) { assert index >= 0 : "The index " + index + " must point at an existing key."; - assert index <= max || (index == max && hasEmptyKey); + assert index <= max; return true; } } \ No newline at end of file diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/LouvainResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/LouvainResult.java index 77896a3c0..0df63c548 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/LouvainResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/LouvainResult.java @@ -24,7 +24,7 @@ /** * @author mknblch */ -public class LouvainResult { +public class LouvainResult extends CommunityResult { public final long loadMillis; public final long computeMillis; @@ -35,27 +35,16 @@ public class LouvainResult { public final List modularities; public final double modularity; - private LouvainResult(long loadMillis, long computeMillis, long writeMillis, long nodes, long iterations, - long communityCount, double[] modularities, double modularity) { - this.loadMillis = loadMillis; - this.computeMillis = computeMillis; - this.writeMillis = writeMillis; - this.nodes = nodes; - this.iterations = iterations; - this.communityCount = communityCount; - this.modularities = new ArrayList<>(modularities.length); - for (double mod : modularities) this.modularities.add(mod); - - - this.modularity = modularity; + private LouvainResult(long loadMillis, long computeMillis, long writeMillis, long nodes, long communityCount, long iterations, boolean convergence, long p99, long p95, long p90, long p75, long p50, int[] biggestCommunities) { + super(loadMillis, computeMillis, writeMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, biggestCommunities); } public static Builder builder() { return new Builder(); } - public static class Builder extends AbstractResultBuilder { + public static class Builder extends CommunityResult.AbstractCommunityBuilder { private long nodes = 0; private long communityCount = 0; @@ -88,9 +77,22 @@ public Builder withFinalModularity(double modularity) { return this; } + @Override public LouvainResult build() { - return new LouvainResult(loadDuration, evalDuration, writeDuration, nodes, iterations, communityCount, - modularities, modularity); + return new LouvainResult( + loadDuration, + evalDuration, + writeDuration, + nodes, + communityCount == -1 ? communityMap.size() : communityCount, + iterations, convergence, + histogram.getValueAtPercentile(.99), + histogram.getValueAtPercentile(.95), + histogram.getValueAtPercentile(.90), + histogram.getValueAtPercentile(.75), + histogram.getValueAtPercentile(.50), + getTopN(3)); } } + } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java index d4ac91970..51ef784ee 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java @@ -1,63 +1,88 @@ /** * Copyright (c) 2017 "Neo4j, Inc." - * + *

* This file is part of Neo4j Graph Algorithms . - * + *

* Neo4j Graph Algorithms is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + *

* This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + *

* You should have received a copy of the GNU General Public License * along with this program. If not, see . */ package org.neo4j.graphalgo.results; +import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; +import org.neo4j.graphalgo.core.utils.paged.PagedDisjointSetStruct; +import org.neo4j.graphalgo.impl.DSSResult; + /** * @author mknblch */ -public class UnionFindResult { - - public final Long loadMillis; - public final Long computeMillis; - public final Long writeMillis; - public final Long nodes; - public final Long setCount; - - private UnionFindResult(Long loadMillis, Long computeMillis, Long writeMillis, Long nodes, Long setCount) { - this.loadMillis = loadMillis; - this.computeMillis = computeMillis; - this.writeMillis = writeMillis; - this.nodes = nodes; - this.setCount = setCount; +public class UnionFindResult extends CommunityResult { + + UnionFindResult(long loadMillis, long computeMillis, long writeMillis, long nodes, long communityCount, long iterations, boolean convergence, long p99, long p95, long p90, long p75, long p50, int[] biggestCommunities) { + super(loadMillis, computeMillis, writeMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, biggestCommunities); } public static Builder builder() { return new Builder(); } - public static class Builder extends AbstractResultBuilder { + public static class Builder extends CommunityResult.AbstractCommunityBuilder { - private long nodes = 0; - private long setCount = 0; - public Builder withSetCount(long setCount) { - this.setCount = setCount; + public Builder withDSSResult(DSSResult result) { + if (result.isHuge) { + withStruct(result.hugeStruct); + } else { + withStruct(result.struct); + } + return this; + } + + public Builder withStruct(DisjointSetStruct setStruct) { + for (int i = 0; i < setStruct.capacity(); i++) { + final int c = setStruct.find(i); + histogram.recordValue(c); + communityMap.addTo(i, 1); + } return this; } - public Builder withNodeCount(long nodes) { - this.nodes = nodes; + public Builder withStruct(PagedDisjointSetStruct setStruct) { + for (long i = 0; i < setStruct.capacity(); i++) { + final long c = setStruct.find(i); + histogram.recordValue(c); + communityMap.addTo(i, 1); + } return this; } - public UnionFindResult build() { - return new UnionFindResult(loadDuration, evalDuration, writeDuration, nodes, setCount); + @Override + public CommunityResult build() { + + return new UnionFindResult( + loadDuration, + evalDuration, + writeDuration, + nodes, + communityMap.size(), + iterations, convergence, + histogram.getValueAtPercentile(.99), + histogram.getValueAtPercentile(.95), + histogram.getValueAtPercentile(.90), + histogram.getValueAtPercentile(.75), + histogram.getValueAtPercentile(.50), + getTopN(3)); } } + + } From f78fee396007098dc34db8128025358230c36b96 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Mon, 21 Jan 2019 11:26:12 +0100 Subject: [PATCH 05/26] WIP --- .../java/org/neo4j/graphalgo/LouvainProc.java | 11 +-- .../graphalgo/impl/UnionFindProcExec.java | 9 +- .../graphalgo/results/CommunityResult.java | 84 ++++++++----------- .../graphalgo/results/UnionFindResult.java | 16 +--- 4 files changed, 44 insertions(+), 76 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index b6f33673c..f6120b7a5 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -79,8 +79,6 @@ public Stream louvain( graph = graph(label, relationship, configuration); } - builder.withNodes(graph.nodeCount()); - if(graph.nodeCount() == 0) { graph.release(); return Stream.of(builder.build()); @@ -100,11 +98,10 @@ public Stream louvain( louvain.compute(configuration.getIterations(10), configuration.get("innerIterations", 10)); } - builder.withCommunities(louvain.getCommunityIds()); - builder.withIterations(louvain.getLevel()); // in real we have threads * innerIterations * maxLevel iterations -// builder.withModularities(louvain.getModularities()) - builder.withConvergence(true); // only subsequent modularity optimizations can converge - + final int[] communityIds = louvain.getCommunityIds(); + builder.withCommunities(Math.toIntExact(graph.nodeCount()), n -> communityIds[n]); + builder.withIterations(louvain.getLevel()); // we have threads * innerIterations * maxLevel iterations + builder.withConvergence(true); // only modularity optimization can converge } if (configuration.isWriteFlag()) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java index 67ccf41db..bec7b9fc3 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java @@ -52,7 +52,7 @@ public final class UnionFindProcExec implements BiConsumer> private final GraphDatabaseAPI api; private final Log log; private final KernelTransaction transaction; - private final UnionFindAlgo sequential; + private final UnionFindAlgo sequential; private final UnionFindAlgo parallel; public static Stream run( @@ -75,10 +75,7 @@ public static Stream run( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder - .withNodes(graph.nodeCount()) - .withoutCommunities() - .build()); + return Stream.of(builder.withCommunities(Math.toIntExact(graph.nodeCount()), i -> -1).build()); } DSSResult dssResult = uf.evaluate( @@ -92,7 +89,7 @@ public static Stream run( uf.write(builder::timeWrite, graph, dssResult, configuration); } - return Stream.of(builder.withNodes(graph.nodeCount()).withDSSResult(dssResult)); + return Stream.of(builder.withDSSResult(dssResult)); } public static Stream stream( diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java index c7db18b07..d75ce42c3 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java @@ -9,6 +9,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.IntToLongFunction; +import java.util.function.LongFunction; import java.util.function.ToLongFunction; import java.util.stream.Collectors; @@ -35,7 +37,7 @@ public abstract class CommunityResult { public final long p90; public final long p75; public final long p50; - public final List top3; + public final long[] top3; CommunityResult(long loadMillis, long computeMillis, @@ -49,7 +51,7 @@ public abstract class CommunityResult { long p90, long p75, long p50, - int[] biggestCommunities) { + long[] biggestCommunities) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.writeMillis = writeMillis; @@ -62,7 +64,7 @@ public abstract class CommunityResult { this.p90 = p90; this.p75 = p75; this.p50 = p50; - this.top3 = Arrays.stream(biggestCommunities).asLongStream().boxed().collect(Collectors.toList()); + this.top3 = biggestCommunities; } public static abstract class AbstractCommunityBuilder extends AbstractResultBuilder { @@ -72,72 +74,52 @@ public static abstract class AbstractCommunityBuilder protected boolean convergence = false; protected long communityCount = -1; protected final Histogram histogram = new Histogram(2); - protected final IntIntHashMap communityMap = new IntIntHashMap(); + protected final LongLongHashMap communityMap = new LongLongHashMap(); + protected final long[] top3 = new long[]{-1L, -1L, -1L}; - public AbstractCommunityBuilder withIterations(long iterations) { + + public AbstractCommunityBuilder withIterations(long iterations) { this.iterations = iterations; return this; } - public AbstractCommunityBuilder withConvergence(boolean convergence) { + public AbstractCommunityBuilder withConvergence(boolean convergence) { this.convergence = convergence; return this; } - public AbstractCommunityBuilder withNodes(long nodes) { - this.nodes = nodes; - return this; - } - - public AbstractCommunityBuilder withCommunities(int[] communities) { - for (int i = 0; i < communities.length; i++) { - communityMap.addTo(communities[i], 1); - histogram.recordValue(communities[i]); - } - return this; - } + public AbstractCommunityBuilder withCommunities(long nodes, LongFunction fun) { - // overwrite community count - public AbstractCommunityBuilder withCommunityCount(long communityCount) { - this.communityCount = communityCount; - return this; - } + top3[0] = -1L; + top3[1] = -1L; + top3[2] = -1L; - public AbstractCommunityBuilder withoutCommunities() { - return this; - } - - public static List top3(long elements, ToLongFunction fun) { - - long t1 = -1L, t2 = -1L, t3 = -1L; + this.nodes = nodes; - for (long i = 0; i < elements; i++) { - final long r = fun.applyAsLong(i); - if (r > t1) { - t3 = t2; - t2 = t1; - t1 = r; - continue; - } - if (r > t2) { - t3 = t2; - t2 = r; - continue; - } - if (r > t3) { - t3 = r; + for (int i = 0; i < nodes; i++) { + // map to community id + final long r = fun.apply(i); + // aggregate + communityMap.addTo(r, 1); + // create histogram + histogram.recordValue(r); + // eval top 3 communities + if (r > top3[0]) { + top3[2] = top3[1]; + top3[1] = top3[0]; + top3[0] = r; + } else if (r > top3[1]) { + top3[2] = top3[1]; + top3[1] = r; + } else if (r > top3[2]) { + top3[2] = r; } } - final ArrayList longs = new ArrayList<>(); - longs.add(t1); - longs.add(t2); - longs.add(t3); - return longs; + return this; } public abstract T build(); } - } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java index 51ef784ee..3fbecac94 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java @@ -27,7 +27,7 @@ */ public class UnionFindResult extends CommunityResult { - UnionFindResult(long loadMillis, long computeMillis, long writeMillis, long nodes, long communityCount, long iterations, boolean convergence, long p99, long p95, long p90, long p75, long p50, int[] biggestCommunities) { + UnionFindResult(long loadMillis, long computeMillis, long writeMillis, long nodes, long communityCount, long iterations, boolean convergence, long p99, long p95, long p90, long p75, long p50, long[] biggestCommunities) { super(loadMillis, computeMillis, writeMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, biggestCommunities); } @@ -48,20 +48,12 @@ public Builder withDSSResult(DSSResult result) { } public Builder withStruct(DisjointSetStruct setStruct) { - for (int i = 0; i < setStruct.capacity(); i++) { - final int c = setStruct.find(i); - histogram.recordValue(c); - communityMap.addTo(i, 1); - } + withCommunities(setStruct.capacity(), i -> (long) setStruct.find((int) i)); return this; } public Builder withStruct(PagedDisjointSetStruct setStruct) { - for (long i = 0; i < setStruct.capacity(); i++) { - final long c = setStruct.find(i); - histogram.recordValue(c); - communityMap.addTo(i, 1); - } + withCommunities(setStruct.capacity(), setStruct::find); return this; } @@ -80,7 +72,7 @@ public CommunityResult build() { histogram.getValueAtPercentile(.90), histogram.getValueAtPercentile(.75), histogram.getValueAtPercentile(.50), - getTopN(3)); + top3); } } From f305ca69a6f3548f40cc681ac190212e418abca3 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Mon, 21 Jan 2019 14:26:56 +0100 Subject: [PATCH 06/26] WIP --- .../graphalgo/impl/UnionFindProcExec.java | 4 +- .../graphalgo/results/CommunityResult.java | 222 ++++++++++++++---- .../graphalgo/results/LouvainResult.java | 4 +- .../graphalgo/results/UnionFindResult.java | 45 +--- 4 files changed, 185 insertions(+), 90 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java index bec7b9fc3..38ad9e443 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java @@ -75,7 +75,7 @@ public static Stream run( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder.withCommunities(Math.toIntExact(graph.nodeCount()), i -> -1).build()); + return Stream.of(builder.withCommunities(graph.nodeCount(), i -> -1L).build()); } DSSResult dssResult = uf.evaluate( @@ -89,7 +89,7 @@ public static Stream run( uf.write(builder::timeWrite, graph, dssResult, configuration); } - return Stream.of(builder.withDSSResult(dssResult)); + return Stream.of(builder.build(dssResult)); } public static Stream stream( diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java index d75ce42c3..40962d805 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java @@ -1,32 +1,24 @@ package org.neo4j.graphalgo.results; -import com.carrotsearch.hppc.IntIntHashMap; import com.carrotsearch.hppc.LongLongHashMap; -import com.carrotsearch.hppc.sorting.IndirectComparator; -import com.carrotsearch.hppc.sorting.IndirectSort; import org.HdrHistogram.Histogram; +import org.neo4j.graphalgo.core.utils.ProgressTimer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.function.IntToLongFunction; import java.util.function.LongFunction; -import java.util.function.ToLongFunction; -import java.util.stream.Collectors; /** * @author mknblch */ -public abstract class CommunityResult { +@SuppressWarnings("WeakerAccess") +public class CommunityResult { /* - - YIELD loadMillis, computeMillis, writeMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, top3 - + YIELD loadMillis, computeMillis, writeMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, p25, p10, p05, p01, top3 */ public final long loadMillis; public final long computeMillis; + public final long postProcessingMillis; public final long writeMillis; public final long nodes; public final long communityCount; @@ -37,24 +29,33 @@ public abstract class CommunityResult { public final long p90; public final long p75; public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; public final long[] top3; - CommunityResult(long loadMillis, - long computeMillis, - long writeMillis, - long nodes, - long communityCount, - long iterations, - boolean convergence, - long p99, - long p95, - long p90, - long p75, - long p50, - long[] biggestCommunities) { + public CommunityResult(long loadMillis, + long computeMillis, + long writeMillis, + long postProcessingMillis, long nodes, + long communityCount, + long iterations, + boolean convergence, + long p99, + long p95, + long p90, + long p75, + long p50, + long p25, + long p10, + long p05, + long p01, + long[] biggestCommunities) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.writeMillis = writeMillis; + this.postProcessingMillis = postProcessingMillis; this.nodes = nodes; this.communityCount = communityCount; this.iterations = iterations; @@ -64,44 +65,149 @@ public abstract class CommunityResult { this.p90 = p90; this.p75 = p75; this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; this.top3 = biggestCommunities; } - public static abstract class AbstractCommunityBuilder extends AbstractResultBuilder { + /** + * Helper class for creating Builders for community algo results + * @param concrete type of the result + */ + public static abstract class AbstractCommunityResultBuilder { - protected long nodes = 1; - protected long iterations = -1; - protected boolean convergence = false; - protected long communityCount = -1; - protected final Histogram histogram = new Histogram(2); - protected final LongLongHashMap communityMap = new LongLongHashMap(); - protected final long[] top3 = new long[]{-1L, -1L, -1L}; + private long loadDuration = -1; + private long evalDuration = -1; + private long writeDuration = -1; + private long iterations = -1; + private boolean convergence = false; + public AbstractCommunityResultBuilder withLoadDuration(long loadDuration) { + this.loadDuration = loadDuration; + return this; + } - public AbstractCommunityBuilder withIterations(long iterations) { - this.iterations = iterations; + public AbstractCommunityResultBuilder withEvalDuration(long evalDuration) { + this.evalDuration = evalDuration; return this; } - public AbstractCommunityBuilder withConvergence(boolean convergence) { - this.convergence = convergence; + public AbstractCommunityResultBuilder withWriteDuration(long writeDuration) { + this.writeDuration = writeDuration; return this; } - public AbstractCommunityBuilder withCommunities(long nodes, LongFunction fun) { + /** + * returns an AutoClosable which measures the time + * until it gets closed. Saves the duration as loadMillis + * @return + */ + public ProgressTimer timeLoad() { + return ProgressTimer.start(this::withLoadDuration); + } + + /** + * returns an AutoClosable which measures the time + * until it gets closed. Saves the duration as evalMillis + * @return + */ + public ProgressTimer timeEval() { + return ProgressTimer.start(this::withEvalDuration); + } + + /** + * returns an AutoClosable which measures the time + * until it gets closed. Saves the duration as writeMillis + * @return + */ + public ProgressTimer timeWrite() { + return ProgressTimer.start(this::withWriteDuration); + } - top3[0] = -1L; - top3[1] = -1L; - top3[2] = -1L; + /** + * evaluates loadMillis + * @param runnable + */ + public void timeLoad(Runnable runnable) { + try (ProgressTimer timer = timeLoad()) { + runnable.run(); + } + } - this.nodes = nodes; + /** + * evaluates comuteMillis + * @param runnable + */ + public void timeEval(Runnable runnable) { + try (ProgressTimer timer = timeEval()) { + runnable.run(); + } + } + /** + * evaluates writeMillis + * @param runnable + */ + public void timeWrite(Runnable runnable) { + try (ProgressTimer timer = timeWrite()) { + runnable.run(); + } + } + + + /** + * number of iterations the algorithm did in total. Can be omitted. + * @param iterations + * @return + */ + public AbstractCommunityResultBuilder withIterations(long iterations) { + this.iterations = iterations; + return this; + } + + /** + * should be set to true if the algorithm did (and is able to) converge to an + * optimum. Should be set to false if the algorithm ran into an artificial limit + * (like maximum iterations without convergence). Can be omitted if the algo + * does not have such thing + * + * @param convergence + * @return + */ + public AbstractCommunityResultBuilder withConvergence(boolean convergence) { + this.convergence = convergence; + return this; + } + + // do it + protected abstract T build( + long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, + long nodeCount, long communityCount, long iterations, boolean convergence, + long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, + long[] top3Communities + ); + + /** + * build result + * @param nodes number of nodes in the graph + * @param fun nodeId to communityId mapping function + * @return result + */ + public T build(long nodes, LongFunction fun) { + + final long[] top3 = new long[]{-1L, -1L, -1L}; + final Histogram histogram = new Histogram(2); + final LongLongHashMap communityMap = new LongLongHashMap(); + + final ProgressTimer timer = ProgressTimer.start(); for (int i = 0; i < nodes; i++) { // map to community id final long r = fun.apply(i); - // aggregate + // aggregate community size communityMap.addTo(r, 1); - // create histogram + // fill histogram histogram.recordValue(r); // eval top 3 communities if (r > top3[0]) { @@ -115,11 +221,29 @@ public AbstractCommunityBuilder withCommunities(long nodes, LongFunction { + public static class Builder extends AbstractCommunityResultBuilder { private long nodes = 0; private long communityCount = 0; diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java index 3fbecac94..f4a76b4ce 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java @@ -18,8 +18,6 @@ */ package org.neo4j.graphalgo.results; -import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; -import org.neo4j.graphalgo.core.utils.paged.PagedDisjointSetStruct; import org.neo4j.graphalgo.impl.DSSResult; /** @@ -27,54 +25,27 @@ */ public class UnionFindResult extends CommunityResult { - UnionFindResult(long loadMillis, long computeMillis, long writeMillis, long nodes, long communityCount, long iterations, boolean convergence, long p99, long p95, long p90, long p75, long p50, long[] biggestCommunities) { - super(loadMillis, computeMillis, writeMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, biggestCommunities); + public UnionFindResult(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodes, long communityCount, long iterations, boolean convergence, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long[] biggestCommunities) { + super(loadMillis, computeMillis, writeMillis, postProcessingMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, p25, p10, p05, p01, biggestCommunities); } public static Builder builder() { return new Builder(); } - public static class Builder extends CommunityResult.AbstractCommunityBuilder { + public static class Builder extends CommunityResult.AbstractCommunityResultBuilder { - - public Builder withDSSResult(DSSResult result) { + public UnionFindResult build(DSSResult result) { if (result.isHuge) { - withStruct(result.hugeStruct); + return build(result.hugeStruct.capacity(), result.hugeStruct::find); } else { - withStruct(result.struct); + return build(result.struct.capacity(), l -> (long) result.struct.find((int) l)); } - return this; - } - - public Builder withStruct(DisjointSetStruct setStruct) { - withCommunities(setStruct.capacity(), i -> (long) setStruct.find((int) i)); - return this; - } - - public Builder withStruct(PagedDisjointSetStruct setStruct) { - withCommunities(setStruct.capacity(), setStruct::find); - return this; } @Override - public CommunityResult build() { - - return new UnionFindResult( - loadDuration, - evalDuration, - writeDuration, - nodes, - communityMap.size(), - iterations, convergence, - histogram.getValueAtPercentile(.99), - histogram.getValueAtPercentile(.95), - histogram.getValueAtPercentile(.90), - histogram.getValueAtPercentile(.75), - histogram.getValueAtPercentile(.50), - top3); + protected UnionFindResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, long iterations, boolean convergence, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long[] top3Communities) { + return new UnionFindResult(loadMillis, computeMillis, writeMillis, postProcessingMillis, communityCount, communityCount, iterations, convergence, p99, p95, p90, p75, p50, p25, p10, p05, p01, top3Communities); } } - - } From b2e901d053f5ce8b965841c04d8e8dafe1043a1b Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Mon, 21 Jan 2019 16:11:40 +0100 Subject: [PATCH 07/26] WIP --- .../neo4j/graphalgo/LabelPropagationProc.java | 7 +- .../java/org/neo4j/graphalgo/LouvainProc.java | 15 +-- .../org/neo4j/graphalgo/MSColoringProc.java | 10 +- .../org/neo4j/graphalgo/UnionFindProc.java | 1 - .../org/neo4j/graphalgo/UnionFindProc2.java | 1 - .../org/neo4j/graphalgo/UnionFindProc3.java | 1 - .../org/neo4j/graphalgo/UnionFindProc4.java | 1 - .../graphalgo/impl/UnionFindProcExec.java | 18 ++- .../graphalgo/results/CommunityResult.java | 123 +++++++++--------- .../results/LabelPropagationStats.java | 96 +++++++++----- .../graphalgo/results/LouvainResult.java | 98 -------------- .../graphalgo/results/UnionFindResult.java | 51 -------- pom.xml | 2 +- .../LabelPropagationProcIntegrationTest.java | 2 +- 14 files changed, 160 insertions(+), 266 deletions(-) delete mode 100644 algo/src/main/java/org/neo4j/graphalgo/results/LouvainResult.java delete mode 100644 algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java diff --git a/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java b/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java index 0faf44f11..9d7a98290 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java @@ -105,15 +105,15 @@ public Stream labelPropagation( if(graph.nodeCount() == 0) { graph.release(); - return Stream.of(stats.build()); + return Stream.of(stats.emptyResult()); } - int[] labels = compute(direction, iterations, batchSize, concurrency, graph, stats); + final int[] labels = compute(direction, iterations, batchSize, concurrency, graph, stats); if (configuration.isWriteFlag(DEFAULT_WRITE) && partitionProperty != null) { write(concurrency, partitionProperty, graph, labels, stats); } - return Stream.of(stats.build()); + return Stream.of(stats.build(graph.nodeCount(), l -> (long) labels[(int) l])); } @Procedure(value = "algo.labelPropagation.stream") @@ -205,7 +205,6 @@ private int[] compute( stats.iterations(labelPropagation.ranIterations()); stats.didConverge(labelPropagation.didConverge()); - stats.nodes(result.length); labelPropagation.release(); graph.release(); diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index f6120b7a5..d10e44967 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -28,7 +28,6 @@ import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.impl.louvain.*; import org.neo4j.graphalgo.results.CommunityResult; -import org.neo4j.graphalgo.results.LouvainResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -72,7 +71,7 @@ public Stream louvain( .overrideNodeLabelOrQuery(label) .overrideRelationshipTypeOrQuery(relationship); - final LouvainResult.Builder builder = LouvainResult.builder(); + final CommunityResult.CommunityResultBuilder builder = new CommunityResult.CommunityResultBuilder(); final Graph graph; try (ProgressTimer timer = builder.timeLoad()) { @@ -81,7 +80,7 @@ public Stream louvain( if(graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder.build()); + return Stream.of(builder.buildEmpty()); } final Louvain louvain = new Louvain(graph, Pools.DEFAULT, configuration.getConcurrency(), AllocationTracker.create()) @@ -97,18 +96,14 @@ public Stream louvain( } else { louvain.compute(configuration.getIterations(10), configuration.get("innerIterations", 10)); } - - final int[] communityIds = louvain.getCommunityIds(); - builder.withCommunities(Math.toIntExact(graph.nodeCount()), n -> communityIds[n]); - builder.withIterations(louvain.getLevel()); // we have threads * innerIterations * maxLevel iterations - builder.withConvergence(true); // only modularity optimization can converge } if (configuration.isWriteFlag()) { builder.timeWrite(() -> write(graph, louvain.getDendrogram(), louvain.getCommunityIds(), configuration)); } - return Stream.of(builder.build()); + final int[] communityIds = louvain.getCommunityIds(); + return Stream.of(builder.build(graph.nodeCount(), n -> (long) communityIds[(int) n])); } @Procedure(value = "algo.louvain.stream") @@ -179,4 +174,6 @@ private void write(Graph graph, int[][] allCommunities, int[] finalCommunities, configuration.get(INTERMEDIATE_COMMUNITIES_WRITE_PROPERTY, "communities")) .export(allCommunities, finalCommunities, includeIntermediateCommunities); } + + } diff --git a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java index 3b7bd6103..4313fd0a2 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java @@ -26,7 +26,7 @@ import org.neo4j.graphalgo.core.write.Exporter; import org.neo4j.graphalgo.core.write.Translators; import org.neo4j.graphalgo.impl.MSColoring; -import org.neo4j.graphalgo.results.UnionFindResult; +import org.neo4j.graphalgo.results.CommunityResult; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -54,7 +54,7 @@ public class MSColoringProc { @Description("CALL algo.unionFind.mscoloring(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition', concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { @@ -63,7 +63,7 @@ public Stream unionFind( .overrideNodeLabelOrQuery(label) .overrideRelationshipTypeOrQuery(relationship); - UnionFindResult.Builder builder = UnionFindResult.builder(); + CommunityResult.CommunityResultBuilder builder = new CommunityResult.CommunityResultBuilder<>(); // loading final Graph graph; @@ -73,7 +73,7 @@ public Stream unionFind( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder.build()); + return Stream.of(builder.buildEmpty()); } // evaluation @@ -88,7 +88,7 @@ public Stream unionFind( write(graph, struct, configuration)); } - return Stream.of(builder.build()); + return Stream.of(builder.build(graph.nodeCount(), n -> (long) struct.get((int) n))); } @Procedure(value = "algo.unionFind.mscoloring.stream") diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java index 34660c50b..62cc8ded4 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java @@ -22,7 +22,6 @@ import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; import org.neo4j.graphalgo.results.CommunityResult; -import org.neo4j.graphalgo.results.UnionFindResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java index 6e6283694..a2636db81 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java @@ -22,7 +22,6 @@ import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; import org.neo4j.graphalgo.results.CommunityResult; -import org.neo4j.graphalgo.results.UnionFindResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java index ebea457c8..d314e2136 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java @@ -22,7 +22,6 @@ import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; import org.neo4j.graphalgo.results.CommunityResult; -import org.neo4j.graphalgo.results.UnionFindResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java index 42000b691..92fe9986a 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java @@ -22,7 +22,6 @@ import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; import org.neo4j.graphalgo.results.CommunityResult; -import org.neo4j.graphalgo.results.UnionFindResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java index 38ad9e443..6c2574745 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java @@ -19,7 +19,6 @@ package org.neo4j.graphalgo.impl; import org.neo4j.graphalgo.api.Graph; -import org.neo4j.graphalgo.api.HugeGraph; import org.neo4j.graphalgo.core.GraphLoader; import org.neo4j.graphalgo.core.ProcedureConfiguration; import org.neo4j.graphalgo.core.utils.Pools; @@ -31,7 +30,6 @@ import org.neo4j.graphalgo.core.utils.paged.PagedDisjointSetStruct; import org.neo4j.graphalgo.core.write.Exporter; import org.neo4j.graphalgo.results.CommunityResult; -import org.neo4j.graphalgo.results.UnionFindResult; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -67,7 +65,7 @@ public static Stream run( AllocationTracker tracker = AllocationTracker.create(); - final UnionFindResult.Builder builder = UnionFindResult.builder(); + final UnionFindResultBuilder builder = new UnionFindResultBuilder(); UnionFindProcExec uf = unionFind.get(); @@ -75,7 +73,7 @@ public static Stream run( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder.withCommunities(graph.nodeCount(), i -> -1L).build()); + return Stream.of(builder.buildEmpty()); } DSSResult dssResult = uf.evaluate( @@ -239,4 +237,16 @@ private void write( PagedDisjointSetStruct.Translator.INSTANCE); } + + public static class UnionFindResultBuilder extends CommunityResult.CommunityResultBuilder { + + public CommunityResult build(DSSResult result) { + if (result.isHuge) { + return build(result.hugeStruct.capacity(), result.hugeStruct::find); + } else { + return build(result.struct.capacity(), l -> (long) result.struct.find((int) l)); + } + } + } + } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java index 40962d805..b40d7bfc7 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java @@ -1,29 +1,33 @@ package org.neo4j.graphalgo.results; +import com.carrotsearch.hppc.IntStack; import com.carrotsearch.hppc.LongLongHashMap; import org.HdrHistogram.Histogram; import org.neo4j.graphalgo.core.utils.ProgressTimer; +import java.util.Arrays; +import java.util.List; import java.util.function.LongFunction; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; /** + * unified community algo result + * + * YIELD loadMillis, computeMillis, writeMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, p25, p10, p05, p01, top3 + * * @author mknblch */ @SuppressWarnings("WeakerAccess") public class CommunityResult { - /* - YIELD loadMillis, computeMillis, writeMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, p25, p10, p05, p01, top3 - */ - public final long loadMillis; public final long computeMillis; public final long postProcessingMillis; public final long writeMillis; public final long nodes; public final long communityCount; - public final long iterations; - public final boolean convergence; public final long p99; public final long p95; public final long p90; @@ -33,15 +37,13 @@ public class CommunityResult { public final long p10; public final long p05; public final long p01; - public final long[] top3; + public final List top3; public CommunityResult(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodes, long communityCount, - long iterations, - boolean convergence, long p99, long p95, long p90, @@ -51,15 +53,13 @@ public CommunityResult(long loadMillis, long p10, long p05, long p01, - long[] biggestCommunities) { + Long[] biggestCommunities) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.writeMillis = writeMillis; this.postProcessingMillis = postProcessingMillis; this.nodes = nodes; this.communityCount = communityCount; - this.iterations = iterations; - this.convergence = convergence; this.p99 = p99; this.p95 = p95; this.p90 = p90; @@ -69,32 +69,30 @@ public CommunityResult(long loadMillis, this.p10 = p10; this.p05 = p05; this.p01 = p01; - this.top3 = biggestCommunities; + this.top3 = Arrays.asList(biggestCommunities); + } /** * Helper class for creating Builders for community algo results - * @param concrete type of the result */ - public static abstract class AbstractCommunityResultBuilder { + public static class CommunityResultBuilder { - private long loadDuration = -1; - private long evalDuration = -1; - private long writeDuration = -1; - private long iterations = -1; - private boolean convergence = false; + protected long loadDuration = -1; + protected long evalDuration = -1; + protected long writeDuration = -1; - public AbstractCommunityResultBuilder withLoadDuration(long loadDuration) { + public CommunityResultBuilder withLoadDuration(long loadDuration) { this.loadDuration = loadDuration; return this; } - public AbstractCommunityResultBuilder withEvalDuration(long evalDuration) { + public CommunityResultBuilder withEvalDuration(long evalDuration) { this.evalDuration = evalDuration; return this; } - public AbstractCommunityResultBuilder withWriteDuration(long writeDuration) { + public CommunityResultBuilder withWriteDuration(long writeDuration) { this.writeDuration = writeDuration; return this; } @@ -156,38 +154,29 @@ public void timeWrite(Runnable runnable) { } } - - /** - * number of iterations the algorithm did in total. Can be omitted. - * @param iterations - * @return - */ - public AbstractCommunityResultBuilder withIterations(long iterations) { - this.iterations = iterations; - return this; - } - - /** - * should be set to true if the algorithm did (and is able to) converge to an - * optimum. Should be set to false if the algorithm ran into an artificial limit - * (like maximum iterations without convergence). Can be omitted if the algo - * does not have such thing - * - * @param convergence - * @return - */ - public AbstractCommunityResultBuilder withConvergence(boolean convergence) { - this.convergence = convergence; - return this; - } - - // do it - protected abstract T build( + protected T build( long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, - long nodeCount, long communityCount, long iterations, boolean convergence, + long nodeCount, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, - long[] top3Communities - ); + Long[] top3Communities) { + //noinspection unchecked + return (T) new CommunityResult(loadDuration, + evalDuration, + writeDuration, + postProcessingMillis, + nodeCount, + communityCount, + p99, + p95, + p90, + p75, + p50, + p25, + p10, + p05, + p01, + top3Communities); + } /** * build result @@ -197,7 +186,7 @@ protected abstract T build( */ public T build(long nodes, LongFunction fun) { - final long[] top3 = new long[]{-1L, -1L, -1L}; + final Long[] top3 = new Long[]{-1L, -1L, -1L}; final Histogram histogram = new Histogram(2); final LongLongHashMap communityMap = new LongLongHashMap(); @@ -223,15 +212,12 @@ public T build(long nodes, LongFunction fun) { } timer.stop(); - return build( - loadDuration, + return build(loadDuration, evalDuration, writeDuration, timer.getDuration(), nodes, communityMap.size(), - iterations, - convergence, histogram.getValueAtPercentile(.99), histogram.getValueAtPercentile(.95), histogram.getValueAtPercentile(.9), @@ -241,9 +227,26 @@ public T build(long nodes, LongFunction fun) { histogram.getValueAtPercentile(.1), histogram.getValueAtPercentile(.05), histogram.getValueAtPercentile(.01), - top3 - ); + top3); } + public CommunityResult buildEmpty() { + return new CommunityResult(loadDuration, + evalDuration, + writeDuration, + -1, + 0, + 0, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + new Long[]{-1L, -1L, -1L}); + } } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java index 89aa0548a..7141c430a 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java @@ -1,64 +1,48 @@ /** * Copyright (c) 2017 "Neo4j, Inc." - * + *

* This file is part of Neo4j Graph Algorithms . - * + *

* Neo4j Graph Algorithms is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + *

* This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + *

* You should have received a copy of the GNU General Public License * along with this program. If not, see . */ package org.neo4j.graphalgo.results; -public class LabelPropagationStats { +public class LabelPropagationStats extends CommunityResult { - public final long nodes, iterations, loadMillis, computeMillis, writeMillis; + public final long iterations; + public final long nodes; public final boolean write, didConverge; public final String weightProperty, partitionProperty; - public LabelPropagationStats( - final long nodes, - final long iterations, - final long loadMillis, - final long computeMillis, - final long writeMillis, - final boolean write, - final boolean didConverge, - final String weightProperty, - final String partitionProperty) { - this.nodes = nodes; + public LabelPropagationStats(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, Long[] biggestCommunities, long iterations, boolean write, boolean didConverge, String weightProperty, String partitionProperty) { + super(loadMillis, computeMillis, writeMillis, postProcessingMillis, nodes, communityCount, p99, p95, p90, p75, p50, p25, p10, p05, p01, biggestCommunities); this.iterations = iterations; - this.loadMillis = loadMillis; - this.computeMillis = computeMillis; - this.writeMillis = writeMillis; this.write = write; this.didConverge = didConverge; this.weightProperty = weightProperty; this.partitionProperty = partitionProperty; + this.nodes = nodes; } - public static class Builder extends AbstractResultBuilder { + public static class Builder extends CommunityResultBuilder { - private long nodes = 0; private long iterations = 0; private boolean didConverge = false; private boolean write; private String weightProperty; private String partitionProperty; - public Builder nodes(final long nodes) { - this.nodes = nodes; - return this; - } - public Builder iterations(final long iterations) { this.iterations = iterations; return this; @@ -84,13 +68,67 @@ public Builder partitionProperty(final String partitionProperty) { return this; } - public LabelPropagationStats build() { + @Override + protected LabelPropagationStats build(long loadMillis, + long computeMillis, + long writeMillis, + long postProcessingMillis, + long nodeCount, + long communityCount, + long p99, + long p95, + long p90, + long p75, + long p50, + long p25, + long p10, + long p05, + long p01, + Long[] top3Communities) { return new LabelPropagationStats( - nodes, + loadMillis, + computeMillis, + writeMillis, + postProcessingMillis, + nodeCount, + communityCount, + p99, + p95, + p90, + p75, + p50, + p25, + p10, + p05, + p01, + top3Communities, iterations, + write, + didConverge, + weightProperty, + partitionProperty + ); + } + + public LabelPropagationStats emptyResult() { + return new LabelPropagationStats( loadDuration, evalDuration, writeDuration, + -1, + 0, + 0, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + new Long[]{-1L, -1L, -1L}, + iterations, write, didConverge, weightProperty, diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/LouvainResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/LouvainResult.java deleted file mode 100644 index 54d825204..000000000 --- a/algo/src/main/java/org/neo4j/graphalgo/results/LouvainResult.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright (c) 2017 "Neo4j, Inc." - * - * This file is part of Neo4j Graph Algorithms . - * - * Neo4j Graph Algorithms is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.graphalgo.results; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author mknblch - */ -public class LouvainResult extends CommunityResult { - - public final long loadMillis; - public final long computeMillis; - public final long writeMillis; - public final long nodes; - public final long iterations; - public final long communityCount; - public final List modularities; - public final double modularity; - - - private LouvainResult(long loadMillis, long computeMillis, long writeMillis, long nodes, long communityCount, long iterations, boolean convergence, long p99, long p95, long p90, long p75, long p50, int[] biggestCommunities) { - super(loadMillis, computeMillis, writeMillis, postProcessingMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, p25, p10, p05, p01, biggestCommunities); - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder extends AbstractCommunityResultBuilder { - - private long nodes = 0; - private long communityCount = 0; - private long iterations = 1; - private double[] modularities = new double[]{}; - private double modularity = -1.0; - - public Builder withIterations(long iterations) { - this.iterations = iterations; - return this; - } - - public Builder withCommunityCount(long setCount) { - this.communityCount = setCount; - return this; - } - - public Builder withNodeCount(long nodes) { - this.nodes = nodes; - return this; - } - - public Builder withModularities(double[] modularities) { - this.modularities = modularities; - return this; - } - - public Builder withFinalModularity(double modularity) { - this.modularity = modularity; - return this; - } - - @Override - public LouvainResult build() { - return new LouvainResult( - loadDuration, - evalDuration, - writeDuration, - nodes, - communityCount == -1 ? communityMap.size() : communityCount, - iterations, convergence, - histogram.getValueAtPercentile(.99), - histogram.getValueAtPercentile(.95), - histogram.getValueAtPercentile(.90), - histogram.getValueAtPercentile(.75), - histogram.getValueAtPercentile(.50), - getTopN(3)); - } - } - -} diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java deleted file mode 100644 index f4a76b4ce..000000000 --- a/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright (c) 2017 "Neo4j, Inc." - *

- * This file is part of Neo4j Graph Algorithms . - *

- * Neo4j Graph Algorithms is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - *

- * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *

- * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.graphalgo.results; - -import org.neo4j.graphalgo.impl.DSSResult; - -/** - * @author mknblch - */ -public class UnionFindResult extends CommunityResult { - - public UnionFindResult(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodes, long communityCount, long iterations, boolean convergence, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long[] biggestCommunities) { - super(loadMillis, computeMillis, writeMillis, postProcessingMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, p25, p10, p05, p01, biggestCommunities); - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder extends CommunityResult.AbstractCommunityResultBuilder { - - public UnionFindResult build(DSSResult result) { - if (result.isHuge) { - return build(result.hugeStruct.capacity(), result.hugeStruct::find); - } else { - return build(result.struct.capacity(), l -> (long) result.struct.find((int) l)); - } - } - - @Override - protected UnionFindResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, long iterations, boolean convergence, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long[] top3Communities) { - return new UnionFindResult(loadMillis, computeMillis, writeMillis, postProcessingMillis, communityCount, communityCount, iterations, convergence, p99, p95, p90, p75, p50, p25, p10, p05, p01, top3Communities); - } - } -} diff --git a/pom.xml b/pom.xml index f0531a1c0..d4fc9a390 100644 --- a/pom.xml +++ b/pom.xml @@ -104,7 +104,7 @@ net.biville.florent neo4j-sproc-compiler - 1.2 + 1.2.1 provided true diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java index 3b77c9ed3..810f623f3 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java @@ -115,7 +115,7 @@ public void shouldTakeParametersFromConfig() { @Test public void shouldRunLabelPropagation() { - String query = "CALL algo.labelPropagation(null, 'X', 'OUTGOING', {batchSize:$batchSize,concurrency:$concurrency})"; + String query = "CALL algo.labelPropagation(null, 'X', 'OUTGOING', {batchSize:$batchSize,concurrency:$concurrency}) YIELD loadMillis"; String check = "MATCH (n) WHERE n.id IN [0,1] RETURN n.partition AS partition"; runQuery(query, parParams(), row -> { From d10bad685aa66e4a6b6c56527cbdb1754629b8d5 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Tue, 22 Jan 2019 15:25:03 +0100 Subject: [PATCH 08/26] WIP --- .../java/org/neo4j/graphalgo/LouvainProc.java | 34 +++++- .../org/neo4j/graphalgo/MSColoringProc.java | 6 +- .../org/neo4j/graphalgo/UnionFindProc.java | 4 +- .../org/neo4j/graphalgo/UnionFindProc2.java | 4 +- .../org/neo4j/graphalgo/UnionFindProc3.java | 4 +- .../org/neo4j/graphalgo/UnionFindProc4.java | 4 +- .../graphalgo/impl/UnionFindProcExec.java | 9 +- ...sult.java => AbstractCommunityResult.java} | 108 +++++++++++------- .../results/LabelPropagationStats.java | 12 +- .../LabelPropagationProcIntegrationTest.java | 2 +- .../LouvainClusteringIntegrationTest.java | 2 - 11 files changed, 119 insertions(+), 70 deletions(-) rename algo/src/main/java/org/neo4j/graphalgo/results/{CommunityResult.java => AbstractCommunityResult.java} (68%) diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index d10e44967..565ffb8c3 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -27,12 +27,14 @@ import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.impl.louvain.*; -import org.neo4j.graphalgo.results.CommunityResult; +import org.neo4j.graphalgo.results.AbstractCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; import org.neo4j.procedure.*; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -62,7 +64,7 @@ public class LouvainProc { @Description("CALL algo.louvain(label:String, relationship:String, " + "{weightProperty:'weight', defaultValue:1.0, write: true, writeProperty:'community', concurrency:4, communityProperty:'propertyOfPredefinedCommunity'}) " + "YIELD nodes, communityCount, iterations, loadMillis, computeMillis, writeMillis") - public Stream louvain( + public Stream louvain( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { @@ -71,7 +73,7 @@ public Stream louvain( .overrideNodeLabelOrQuery(label) .overrideRelationshipTypeOrQuery(relationship); - final CommunityResult.CommunityResultBuilder builder = new CommunityResult.CommunityResultBuilder(); + final AbstractCommunityResult.CommunityResultBuilder builder = new AbstractCommunityResult.CommunityResultBuilder<>(); final Graph graph; try (ProgressTimer timer = builder.timeLoad()) { @@ -80,7 +82,7 @@ public Stream louvain( if(graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder.buildEmpty()); + return Stream.of(LouvainResult.EMPTY); } final Louvain louvain = new Louvain(graph, Pools.DEFAULT, configuration.getConcurrency(), AllocationTracker.create()) @@ -175,5 +177,29 @@ private void write(Graph graph, int[][] allCommunities, int[] finalCommunities, .export(allCommunities, finalCommunities, includeIntermediateCommunities); } + private static class LouvainResult extends AbstractCommunityResult { + public LouvainResult(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top) { + super(loadMillis, computeMillis, writeMillis, postProcessingMillis, nodes, communityCount, p99, p95, p90, p75, p50, p25, p10, p05, p01, top); + } + + public static final LouvainResult EMPTY = new LouvainResult( + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + Collections.emptyList() + ); + } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java index 4313fd0a2..9b9124950 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java @@ -26,7 +26,7 @@ import org.neo4j.graphalgo.core.write.Exporter; import org.neo4j.graphalgo.core.write.Translators; import org.neo4j.graphalgo.impl.MSColoring; -import org.neo4j.graphalgo.results.CommunityResult; +import org.neo4j.graphalgo.results.AbstractCommunityResult; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -54,7 +54,7 @@ public class MSColoringProc { @Description("CALL algo.unionFind.mscoloring(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition', concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { @@ -63,7 +63,7 @@ public Stream unionFind( .overrideNodeLabelOrQuery(label) .overrideRelationshipTypeOrQuery(relationship); - CommunityResult.CommunityResultBuilder builder = new CommunityResult.CommunityResultBuilder<>(); + AbstractCommunityResult.CommunityResultBuilder builder = new AbstractCommunityResult.CommunityResultBuilder<>(); // loading final Graph graph; diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java index 62cc8ded4..bdf235c5b 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.CommunityResult; +import org.neo4j.graphalgo.results.AbstractCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{weightProperty:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition'}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java index a2636db81..928a524c7 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.CommunityResult; +import org.neo4j.graphalgo.results.AbstractCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc2 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition',concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java index d314e2136..4547ab3da 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.CommunityResult; +import org.neo4j.graphalgo.results.AbstractCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc3 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition', concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java index 92fe9986a..008cf9aee 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.CommunityResult; +import org.neo4j.graphalgo.results.AbstractCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc4 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition',concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java index 6c2574745..125376c64 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java @@ -29,7 +29,7 @@ import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.core.utils.paged.PagedDisjointSetStruct; import org.neo4j.graphalgo.core.write.Exporter; -import org.neo4j.graphalgo.results.CommunityResult; +import org.neo4j.graphalgo.results.AbstractCommunityResult; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -53,7 +53,7 @@ public final class UnionFindProcExec implements BiConsumer> private final UnionFindAlgo sequential; private final UnionFindAlgo parallel; - public static Stream run( + public static Stream run( Map config, String label, String relationship, @@ -238,9 +238,9 @@ private void write( } - public static class UnionFindResultBuilder extends CommunityResult.CommunityResultBuilder { + public static class UnionFindResultBuilder extends AbstractCommunityResult.CommunityResultBuilder { - public CommunityResult build(DSSResult result) { + public AbstractCommunityResult build(DSSResult result) { if (result.isHuge) { return build(result.hugeStruct.capacity(), result.hugeStruct::find); } else { @@ -248,5 +248,4 @@ public CommunityResult build(DSSResult result) { } } } - } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResult.java similarity index 68% rename from algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java rename to algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResult.java index b40d7bfc7..4d032084a 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResult.java @@ -1,16 +1,14 @@ package org.neo4j.graphalgo.results; -import com.carrotsearch.hppc.IntStack; -import com.carrotsearch.hppc.LongLongHashMap; +import com.carrotsearch.hppc.*; +import com.carrotsearch.hppc.cursors.LongLongCursor; import org.HdrHistogram.Histogram; import org.neo4j.graphalgo.core.utils.ProgressTimer; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.function.LongFunction; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.LongStream; /** * unified community algo result @@ -20,7 +18,7 @@ * @author mknblch */ @SuppressWarnings("WeakerAccess") -public class CommunityResult { +public abstract class AbstractCommunityResult { public final long loadMillis; public final long computeMillis; @@ -39,21 +37,21 @@ public class CommunityResult { public final long p01; public final List top3; - public CommunityResult(long loadMillis, - long computeMillis, - long writeMillis, - long postProcessingMillis, long nodes, - long communityCount, - long p99, - long p95, - long p90, - long p75, - long p50, - long p25, - long p10, - long p05, - long p01, - Long[] biggestCommunities) { + public AbstractCommunityResult(long loadMillis, + long computeMillis, + long writeMillis, + long postProcessingMillis, long nodes, + long communityCount, + long p99, + long p95, + long p90, + long p75, + long p50, + long p25, + long p10, + long p05, + long p01, + List top) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.writeMillis = writeMillis; @@ -69,14 +67,14 @@ public CommunityResult(long loadMillis, this.p10 = p10; this.p05 = p05; this.p01 = p01; - this.top3 = Arrays.asList(biggestCommunities); + this.top3 = top; } /** * Helper class for creating Builders for community algo results */ - public static class CommunityResultBuilder { + public static class CommunityResultBuilder { protected long loadDuration = -1; protected long evalDuration = -1; @@ -158,9 +156,9 @@ protected T build( long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, - Long[] top3Communities) { + List top3Communities) { //noinspection unchecked - return (T) new CommunityResult(loadDuration, + return (T) new DefaultCommunityResult(loadDuration, evalDuration, writeDuration, postProcessingMillis, @@ -178,6 +176,35 @@ protected T build( top3Communities); } + private static List top3(LongLongMap assignment) { + + // index of top 3 biggest communities + final Long[] t3idx = new Long[]{-1L, -1L, -1L}; + // size of the top 3 communities + final long[] top3 = new long[]{-1L, -1L, -1L}; + + for (LongLongCursor cursor : assignment) { + if (cursor.value > top3[0]) { + top3[2] = top3[1]; + top3[1] = top3[0]; + top3[0] = cursor.value; + t3idx[2] = t3idx[1]; + t3idx[1] = t3idx[0]; + t3idx[0] = cursor.key; + } else if (cursor.value > top3[1]) { + top3[2] = top3[1]; + top3[1] = cursor.value; + t3idx[2] = t3idx[1]; + t3idx[1] = cursor.key; + } else if (cursor.value > top3[2]) { + top3[2] = cursor.value; + t3idx[2] = cursor.key; + } + } + + return Arrays.asList(t3idx); + } + /** * build result * @param nodes number of nodes in the graph @@ -186,9 +213,8 @@ protected T build( */ public T build(long nodes, LongFunction fun) { - final Long[] top3 = new Long[]{-1L, -1L, -1L}; final Histogram histogram = new Histogram(2); - final LongLongHashMap communityMap = new LongLongHashMap(); + final LongLongMap communityMap = new LongLongScatterMap(); final ProgressTimer timer = ProgressTimer.start(); for (int i = 0; i < nodes; i++) { @@ -198,17 +224,6 @@ public T build(long nodes, LongFunction fun) { communityMap.addTo(r, 1); // fill histogram histogram.recordValue(r); - // eval top 3 communities - if (r > top3[0]) { - top3[2] = top3[1]; - top3[1] = top3[0]; - top3[0] = r; - } else if (r > top3[1]) { - top3[2] = top3[1]; - top3[1] = r; - } else if (r > top3[2]) { - top3[2] = r; - } } timer.stop(); @@ -227,11 +242,12 @@ public T build(long nodes, LongFunction fun) { histogram.getValueAtPercentile(.1), histogram.getValueAtPercentile(.05), histogram.getValueAtPercentile(.01), - top3); + top3(communityMap)); } - public CommunityResult buildEmpty() { - return new CommunityResult(loadDuration, + public AbstractCommunityResult buildEmpty() { + return new DefaultCommunityResult( + loadDuration, evalDuration, writeDuration, -1, @@ -246,7 +262,15 @@ public CommunityResult buildEmpty() { -1, -1, -1, - new Long[]{-1L, -1L, -1L}); + Collections.emptyList()); + } + + } + + private static class DefaultCommunityResult extends AbstractCommunityResult { + + public DefaultCommunityResult(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List biggestCommunities) { + super(loadMillis, computeMillis, writeMillis, postProcessingMillis, nodes, communityCount, p99, p95, p90, p75, p50, p25, p10, p05, p01, biggestCommunities); } } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java index 7141c430a..bb208ad63 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java @@ -18,14 +18,17 @@ */ package org.neo4j.graphalgo.results; -public class LabelPropagationStats extends CommunityResult { +import java.util.Collections; +import java.util.List; + +public class LabelPropagationStats extends AbstractCommunityResult { public final long iterations; public final long nodes; public final boolean write, didConverge; public final String weightProperty, partitionProperty; - public LabelPropagationStats(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, Long[] biggestCommunities, long iterations, boolean write, boolean didConverge, String weightProperty, String partitionProperty) { + public LabelPropagationStats(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List biggestCommunities, long iterations, boolean write, boolean didConverge, String weightProperty, String partitionProperty) { super(loadMillis, computeMillis, writeMillis, postProcessingMillis, nodes, communityCount, p99, p95, p90, p75, p50, p25, p10, p05, p01, biggestCommunities); this.iterations = iterations; this.write = write; @@ -68,7 +71,6 @@ public Builder partitionProperty(final String partitionProperty) { return this; } - @Override protected LabelPropagationStats build(long loadMillis, long computeMillis, long writeMillis, @@ -84,7 +86,7 @@ protected LabelPropagationStats build(long loadMillis, long p10, long p05, long p01, - Long[] top3Communities) { + List top3Communities) { return new LabelPropagationStats( loadMillis, computeMillis, @@ -127,7 +129,7 @@ public LabelPropagationStats emptyResult() { -1, -1, -1, - new Long[]{-1L, -1L, -1L}, + Collections.emptyList(), iterations, write, didConverge, diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java index 810f623f3..a7b856f8b 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java @@ -115,7 +115,7 @@ public void shouldTakeParametersFromConfig() { @Test public void shouldRunLabelPropagation() { - String query = "CALL algo.labelPropagation(null, 'X', 'OUTGOING', {batchSize:$batchSize,concurrency:$concurrency}) YIELD loadMillis"; + String query = "CALL algo.labelPropagation(null, 'X', 'OUTGOING', {batchSize:$batchSize,concurrency:$concurrency}) RETURN loadMillis"; String check = "MATCH (n) WHERE n.id IN [0,1] RETURN n.partition AS partition"; runQuery(query, parParams(), row -> { diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java index 84a18a11e..0a75112bc 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java @@ -28,7 +28,6 @@ import java.util.Arrays; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import static junit.framework.TestCase.assertNull; @@ -336,5 +335,4 @@ public int[] getClusterId(String nodeName) { }); return id; } - } From 21ea3e54ebe6aa76db4cdfbb4c9f3d14d75b0ec5 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Tue, 22 Jan 2019 17:04:36 +0100 Subject: [PATCH 09/26] WIP --- .../results/CommunityResultBuilder.java | 207 ++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 algo/src/main/java/org/neo4j/graphalgo/results/CommunityResultBuilder.java diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResultBuilder.java b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResultBuilder.java new file mode 100644 index 000000000..10e762120 --- /dev/null +++ b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResultBuilder.java @@ -0,0 +1,207 @@ +package org.neo4j.graphalgo.results; + +import com.carrotsearch.hppc.LongLongMap; +import com.carrotsearch.hppc.LongLongScatterMap; +import com.carrotsearch.hppc.cursors.LongLongCursor; +import org.HdrHistogram.Histogram; +import org.neo4j.graphalgo.api.IdMapping; +import org.neo4j.graphalgo.core.utils.ProgressTimer; + +import java.util.Arrays; +import java.util.List; +import java.util.function.LongFunction; +import java.util.stream.Collectors; + +/** + * @author mknblch + */ +public abstract class CommunityResultBuilder { + + private final IdMapping idMapping; + + protected long loadDuration = -1; + protected long evalDuration = -1; + protected long writeDuration = -1; + + protected CommunityResultBuilder(IdMapping idMapping) { + this.idMapping = idMapping; + } + + public CommunityResultBuilder withLoadDuration(long loadDuration) { + + this.loadDuration = loadDuration; + return this; + } + + + public CommunityResultBuilder withEvalDuration(long evalDuration) { + + this.evalDuration = evalDuration; + return this; + } + + + public CommunityResultBuilder withWriteDuration(long writeDuration) { + + this.writeDuration = writeDuration; + return this; + } + + /** + * returns an AutoClosable which measures the time + * until it gets closed. Saves the duration as loadMillis + * + * @return + */ + public ProgressTimer timeLoad() { + + return ProgressTimer.start(this::withLoadDuration); + } + + /** + * returns an AutoClosable which measures the time + * until it gets closed. Saves the duration as evalMillis + * + * @return + */ + public ProgressTimer timeEval() { + + return ProgressTimer.start(this::withEvalDuration); + } + + /** + * returns an AutoClosable which measures the time + * until it gets closed. Saves the duration as writeMillis + * + * @return + */ + public ProgressTimer timeWrite() { + + return ProgressTimer.start(this::withWriteDuration); + } + + /** + * evaluates loadMillis + * + * @param runnable + */ + public void timeLoad(Runnable runnable) { + + try (ProgressTimer timer = timeLoad()) { + runnable.run(); + } + } + + /** + * evaluates comuteMillis + * + * @param runnable + */ + public void timeEval(Runnable runnable) { + + try (ProgressTimer timer = timeEval()) { + runnable.run(); + } + } + + /** + * evaluates writeMillis + * + * @param runnable + */ + public void timeWrite(Runnable runnable) { + + try (ProgressTimer timer = timeWrite()) { + runnable.run(); + } + } + + /** + * build result + * + * @param nodes number of nodes in the graph + * @param fun nodeId to communityId mapping function + * @return result + */ + public T build(long nodes, LongFunction fun) { + + final Histogram histogram = new Histogram(2); + final LongLongMap communityMap = new LongLongScatterMap(); + + final ProgressTimer timer = ProgressTimer.start(); + for (int i = 0; i < nodes; i++) { + // map to community id + final long r = fun.apply(i); + // aggregate community size + communityMap.addTo(r, 1); + // fill histogram + histogram.recordValue(r); + } + timer.stop(); + + return build(loadDuration, + evalDuration, + writeDuration, + timer.getDuration(), + nodes, + communityMap.size(), + histogram.getValueAtPercentile(.99), + histogram.getValueAtPercentile(.95), + histogram.getValueAtPercentile(.9), + histogram.getValueAtPercentile(.75), + histogram.getValueAtPercentile(.5), + histogram.getValueAtPercentile(.25), + histogram.getValueAtPercentile(.1), + histogram.getValueAtPercentile(.05), + histogram.getValueAtPercentile(.01), + top3(communityMap)); + } + + protected abstract T build( + long loadMillis, + long computeMillis, + long writeMillis, + long postProcessingMillis, + long nodeCount, + long communityCount, + long p99, + long p95, + long p90, + long p75, + long p50, + long p25, + long p10, + long p05, + long p01, + List top3Communities); + + private List top3(LongLongMap assignment) { + // index of top 3 biggest communities + final Long[] t3idx = new Long[]{-1L, -1L, -1L}; + // size of the top 3 communities + final long[] top3 = new long[]{-1L, -1L, -1L}; + for (LongLongCursor cursor : assignment) { + if (cursor.value > top3[0]) { + top3[2] = top3[1]; + top3[1] = top3[0]; + top3[0] = cursor.value; + t3idx[2] = t3idx[1]; + t3idx[1] = t3idx[0]; + t3idx[0] = cursor.key; + } else if (cursor.value > top3[1]) { + top3[2] = top3[1]; + top3[1] = cursor.value; + t3idx[2] = t3idx[1]; + t3idx[1] = cursor.key; + } else if (cursor.value > top3[2]) { + top3[2] = cursor.value; + t3idx[2] = cursor.key; + } + } + + Arrays.stream(t3idx) + .map(l -> idMapping.toOriginalNodeId(((Integer) l))) + .collect(Collectors.toList()) + + } +} From 88146e10719a84b2b6378faf7526cd62fa136cf3 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Wed, 23 Jan 2019 13:50:40 +0100 Subject: [PATCH 10/26] wip --- .../neo4j/graphalgo/LabelPropagationProc.java | 4 +- .../java/org/neo4j/graphalgo/LouvainProc.java | 128 +++++++- .../org/neo4j/graphalgo/MSColoringProc.java | 10 +- .../org/neo4j/graphalgo/UnionFindProc.java | 4 +- .../org/neo4j/graphalgo/UnionFindProc2.java | 4 +- .../org/neo4j/graphalgo/UnionFindProc3.java | 4 +- .../org/neo4j/graphalgo/UnionFindProc4.java | 4 +- .../graphalgo/impl/UnionFindProcExec.java | 25 +- .../results/AbstractCommunityResult.java | 276 ------------------ ...va => AbstractCommunityResultBuilder.java} | 56 +--- .../results/DefaultCommunityResult.java | 92 ++++++ .../results/LabelPropagationStats.java | 128 ++++---- .../LabelPropagationProcIntegrationTest.java | 2 +- .../LouvainClusteringIntegrationTest.java | 5 +- .../neo4j/graphalgo/impl/LouvainTest1.java | 3 + 15 files changed, 325 insertions(+), 420 deletions(-) delete mode 100644 algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResult.java rename algo/src/main/java/org/neo4j/graphalgo/results/{CommunityResultBuilder.java => AbstractCommunityResultBuilder.java} (73%) create mode 100644 algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java diff --git a/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java b/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java index 9d7a98290..df2dcdc18 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java @@ -105,7 +105,7 @@ public Stream labelPropagation( if(graph.nodeCount() == 0) { graph.release(); - return Stream.of(stats.emptyResult()); + return Stream.of(LabelPropagationStats.EMPTY); } final int[] labels = compute(direction, iterations, batchSize, concurrency, graph, stats); @@ -113,7 +113,7 @@ public Stream labelPropagation( write(concurrency, partitionProperty, graph, labels, stats); } - return Stream.of(stats.build(graph.nodeCount(), l -> (long) labels[(int) l])); + return Stream.of(stats.build(graph, l -> (long) labels[(int) l])); } @Procedure(value = "algo.labelPropagation.stream") diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index 565ffb8c3..4ba56e0b7 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -18,6 +18,8 @@ */ package org.neo4j.graphalgo; +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; import org.neo4j.graphalgo.api.*; import org.neo4j.graphalgo.core.GraphLoader; import org.neo4j.graphalgo.core.ProcedureConfiguration; @@ -27,12 +29,14 @@ import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.impl.louvain.*; -import org.neo4j.graphalgo.results.AbstractCommunityResult; +import org.neo4j.graphalgo.results.AbstractCommunityResultBuilder; +import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; import org.neo4j.procedure.*; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -73,7 +77,8 @@ public Stream louvain( .overrideNodeLabelOrQuery(label) .overrideRelationshipTypeOrQuery(relationship); - final AbstractCommunityResult.CommunityResultBuilder builder = new AbstractCommunityResult.CommunityResultBuilder<>(); + + final Builder builder = new Builder(); final Graph graph; try (ProgressTimer timer = builder.timeLoad()) { @@ -105,7 +110,7 @@ public Stream louvain( } final int[] communityIds = louvain.getCommunityIds(); - return Stream.of(builder.build(graph.nodeCount(), n -> (long) communityIds[(int) n])); + return Stream.of(builder.build(graph, n -> (long) communityIds[(int) n])); } @Procedure(value = "algo.louvain.stream") @@ -177,13 +182,15 @@ private void write(Graph graph, int[][] allCommunities, int[] finalCommunities, .export(allCommunities, finalCommunities, includeIntermediateCommunities); } - private static class LouvainResult extends AbstractCommunityResult { - - public LouvainResult(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top) { - super(loadMillis, computeMillis, writeMillis, postProcessingMillis, nodes, communityCount, p99, p95, p90, p75, p50, p25, p10, p05, p01, top); - } + public static class LouvainResult { public static final LouvainResult EMPTY = new LouvainResult( + 0, + 0, + 0, + 0, + 0, + 0, -1, -1, -1, @@ -193,13 +200,104 @@ public LouvainResult(long loadMillis, long computeMillis, long writeMillis, long -1, -1, -1, - -1, - -1, - -1, - -1, - -1, - -1, - Collections.emptyList() + Collections.emptyList(), + 0 ); + + public final long loadMillis; + public final long computeMillis; + public final long postProcessingMillis; + public final long writeMillis; + public final long nodes; + public final long communityCount; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; + public final List top3; + public final long iterations; + + public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations) { + this.loadMillis = loadMillis; + this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; + this.writeMillis = writeMillis; + this.nodes = nodes; + this.communityCount = communityCount; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; + this.top3 = top3; + this.iterations = iterations; + } + + @Override + public String toString() { + return "LouvainResult{" + + "loadMillis=" + loadMillis + + ", computeMillis=" + computeMillis + + ", postProcessingMillis=" + postProcessingMillis + + ", writeMillis=" + writeMillis + + ", nodes=" + nodes + + ", communityCount=" + communityCount + + ", p99=" + p99 + + ", p95=" + p95 + + ", p90=" + p90 + + ", p75=" + p75 + + ", p50=" + p50 + + ", p25=" + p25 + + ", p10=" + p10 + + ", p05=" + p05 + + ", p01=" + p01 + + ", top3=" + top3 + + ", iterations=" + iterations + + '}'; + } } + + public static class Builder extends AbstractCommunityResultBuilder { + + private long iterations = -1; + + public Builder withIterations(long iterations) { + this.iterations = iterations; + return this; + } + + @Override + protected LouvainResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, List top3Communities) { + return new LouvainResult( + loadMillis, + computeMillis, + postProcessingMillis, + writeMillis, + nodeCount, + communityCount, + communityHistogram.getValueAtPercentile(.99), + communityHistogram.getValueAtPercentile(.95), + communityHistogram.getValueAtPercentile(.9), + communityHistogram.getValueAtPercentile(.75), + communityHistogram.getValueAtPercentile(.5), + communityHistogram.getValueAtPercentile(.25), + communityHistogram.getValueAtPercentile(.1), + communityHistogram.getValueAtPercentile(.05), + communityHistogram.getValueAtPercentile(.01), + top3Communities, + iterations + ); + } + } + + } diff --git a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java index 9b9124950..86273f72d 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java @@ -26,7 +26,7 @@ import org.neo4j.graphalgo.core.write.Exporter; import org.neo4j.graphalgo.core.write.Translators; import org.neo4j.graphalgo.impl.MSColoring; -import org.neo4j.graphalgo.results.AbstractCommunityResult; +import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -54,7 +54,7 @@ public class MSColoringProc { @Description("CALL algo.unionFind.mscoloring(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition', concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { @@ -63,7 +63,7 @@ public Stream unionFind( .overrideNodeLabelOrQuery(label) .overrideRelationshipTypeOrQuery(relationship); - AbstractCommunityResult.CommunityResultBuilder builder = new AbstractCommunityResult.CommunityResultBuilder<>(); + final DefaultCommunityResult.DefaultCommunityResultBuilder builder = new DefaultCommunityResult.DefaultCommunityResultBuilder(); // loading final Graph graph; @@ -73,7 +73,7 @@ public Stream unionFind( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder.buildEmpty()); + return Stream.of(DefaultCommunityResult.EMPTY); } // evaluation @@ -88,7 +88,7 @@ public Stream unionFind( write(graph, struct, configuration)); } - return Stream.of(builder.build(graph.nodeCount(), n -> (long) struct.get((int) n))); + return Stream.of(builder.build(graph, n -> (long) struct.get((int) n))); } @Procedure(value = "algo.unionFind.mscoloring.stream") diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java index bdf235c5b..93919d4f5 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.AbstractCommunityResult; +import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{weightProperty:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition'}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java index 928a524c7..74191c419 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.AbstractCommunityResult; +import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc2 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition',concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java index 4547ab3da..71ed14c08 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.AbstractCommunityResult; +import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc3 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition', concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java index 008cf9aee..2ad3a3b93 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.AbstractCommunityResult; +import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc4 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition',concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java index 125376c64..ba5e3d670 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java @@ -29,7 +29,7 @@ import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.core.utils.paged.PagedDisjointSetStruct; import org.neo4j.graphalgo.core.write.Exporter; -import org.neo4j.graphalgo.results.AbstractCommunityResult; +import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -53,7 +53,7 @@ public final class UnionFindProcExec implements BiConsumer> private final UnionFindAlgo sequential; private final UnionFindAlgo parallel; - public static Stream run( + public static Stream run( Map config, String label, String relationship, @@ -65,7 +65,7 @@ public static Stream run( AllocationTracker tracker = AllocationTracker.create(); - final UnionFindResultBuilder builder = new UnionFindResultBuilder(); + final DefaultCommunityResult.DefaultCommunityResultBuilder builder = new DefaultCommunityResult.DefaultCommunityResultBuilder(); UnionFindProcExec uf = unionFind.get(); @@ -73,10 +73,10 @@ public static Stream run( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder.buildEmpty()); + return Stream.of(DefaultCommunityResult.EMPTY); } - DSSResult dssResult = uf.evaluate( + final DSSResult dssResult = uf.evaluate( builder::timeEval, graph, configuration, @@ -87,7 +87,11 @@ public static Stream run( uf.write(builder::timeWrite, graph, dssResult, configuration); } - return Stream.of(builder.build(dssResult)); + if (dssResult.isHuge) { + return Stream.of(builder.build(graph, dssResult.hugeStruct::find)); + } else { + return Stream.of(builder.build(graph, l -> (long) dssResult.struct.find((int) l))); + } } public static Stream stream( @@ -238,14 +242,5 @@ private void write( } - public static class UnionFindResultBuilder extends AbstractCommunityResult.CommunityResultBuilder { - public AbstractCommunityResult build(DSSResult result) { - if (result.isHuge) { - return build(result.hugeStruct.capacity(), result.hugeStruct::find); - } else { - return build(result.struct.capacity(), l -> (long) result.struct.find((int) l)); - } - } - } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResult.java deleted file mode 100644 index 4d032084a..000000000 --- a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResult.java +++ /dev/null @@ -1,276 +0,0 @@ -package org.neo4j.graphalgo.results; - -import com.carrotsearch.hppc.*; -import com.carrotsearch.hppc.cursors.LongLongCursor; -import org.HdrHistogram.Histogram; -import org.neo4j.graphalgo.core.utils.ProgressTimer; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.function.LongFunction; - -/** - * unified community algo result - * - * YIELD loadMillis, computeMillis, writeMillis, nodes, communityCount, iterations, convergence, p99, p95, p90, p75, p50, p25, p10, p05, p01, top3 - * - * @author mknblch - */ -@SuppressWarnings("WeakerAccess") -public abstract class AbstractCommunityResult { - - public final long loadMillis; - public final long computeMillis; - public final long postProcessingMillis; - public final long writeMillis; - public final long nodes; - public final long communityCount; - public final long p99; - public final long p95; - public final long p90; - public final long p75; - public final long p50; - public final long p25; - public final long p10; - public final long p05; - public final long p01; - public final List top3; - - public AbstractCommunityResult(long loadMillis, - long computeMillis, - long writeMillis, - long postProcessingMillis, long nodes, - long communityCount, - long p99, - long p95, - long p90, - long p75, - long p50, - long p25, - long p10, - long p05, - long p01, - List top) { - this.loadMillis = loadMillis; - this.computeMillis = computeMillis; - this.writeMillis = writeMillis; - this.postProcessingMillis = postProcessingMillis; - this.nodes = nodes; - this.communityCount = communityCount; - this.p99 = p99; - this.p95 = p95; - this.p90 = p90; - this.p75 = p75; - this.p50 = p50; - this.p25 = p25; - this.p10 = p10; - this.p05 = p05; - this.p01 = p01; - this.top3 = top; - - } - - /** - * Helper class for creating Builders for community algo results - */ - public static class CommunityResultBuilder { - - protected long loadDuration = -1; - protected long evalDuration = -1; - protected long writeDuration = -1; - - public CommunityResultBuilder withLoadDuration(long loadDuration) { - this.loadDuration = loadDuration; - return this; - } - - public CommunityResultBuilder withEvalDuration(long evalDuration) { - this.evalDuration = evalDuration; - return this; - } - - public CommunityResultBuilder withWriteDuration(long writeDuration) { - this.writeDuration = writeDuration; - return this; - } - - /** - * returns an AutoClosable which measures the time - * until it gets closed. Saves the duration as loadMillis - * @return - */ - public ProgressTimer timeLoad() { - return ProgressTimer.start(this::withLoadDuration); - } - - /** - * returns an AutoClosable which measures the time - * until it gets closed. Saves the duration as evalMillis - * @return - */ - public ProgressTimer timeEval() { - return ProgressTimer.start(this::withEvalDuration); - } - - /** - * returns an AutoClosable which measures the time - * until it gets closed. Saves the duration as writeMillis - * @return - */ - public ProgressTimer timeWrite() { - return ProgressTimer.start(this::withWriteDuration); - } - - /** - * evaluates loadMillis - * @param runnable - */ - public void timeLoad(Runnable runnable) { - try (ProgressTimer timer = timeLoad()) { - runnable.run(); - } - } - - /** - * evaluates comuteMillis - * @param runnable - */ - public void timeEval(Runnable runnable) { - try (ProgressTimer timer = timeEval()) { - runnable.run(); - } - } - - /** - * evaluates writeMillis - * @param runnable - */ - public void timeWrite(Runnable runnable) { - try (ProgressTimer timer = timeWrite()) { - runnable.run(); - } - } - - protected T build( - long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, - long nodeCount, long communityCount, - long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, - List top3Communities) { - //noinspection unchecked - return (T) new DefaultCommunityResult(loadDuration, - evalDuration, - writeDuration, - postProcessingMillis, - nodeCount, - communityCount, - p99, - p95, - p90, - p75, - p50, - p25, - p10, - p05, - p01, - top3Communities); - } - - private static List top3(LongLongMap assignment) { - - // index of top 3 biggest communities - final Long[] t3idx = new Long[]{-1L, -1L, -1L}; - // size of the top 3 communities - final long[] top3 = new long[]{-1L, -1L, -1L}; - - for (LongLongCursor cursor : assignment) { - if (cursor.value > top3[0]) { - top3[2] = top3[1]; - top3[1] = top3[0]; - top3[0] = cursor.value; - t3idx[2] = t3idx[1]; - t3idx[1] = t3idx[0]; - t3idx[0] = cursor.key; - } else if (cursor.value > top3[1]) { - top3[2] = top3[1]; - top3[1] = cursor.value; - t3idx[2] = t3idx[1]; - t3idx[1] = cursor.key; - } else if (cursor.value > top3[2]) { - top3[2] = cursor.value; - t3idx[2] = cursor.key; - } - } - - return Arrays.asList(t3idx); - } - - /** - * build result - * @param nodes number of nodes in the graph - * @param fun nodeId to communityId mapping function - * @return result - */ - public T build(long nodes, LongFunction fun) { - - final Histogram histogram = new Histogram(2); - final LongLongMap communityMap = new LongLongScatterMap(); - - final ProgressTimer timer = ProgressTimer.start(); - for (int i = 0; i < nodes; i++) { - // map to community id - final long r = fun.apply(i); - // aggregate community size - communityMap.addTo(r, 1); - // fill histogram - histogram.recordValue(r); - } - timer.stop(); - - return build(loadDuration, - evalDuration, - writeDuration, - timer.getDuration(), - nodes, - communityMap.size(), - histogram.getValueAtPercentile(.99), - histogram.getValueAtPercentile(.95), - histogram.getValueAtPercentile(.9), - histogram.getValueAtPercentile(.75), - histogram.getValueAtPercentile(.5), - histogram.getValueAtPercentile(.25), - histogram.getValueAtPercentile(.1), - histogram.getValueAtPercentile(.05), - histogram.getValueAtPercentile(.01), - top3(communityMap)); - } - - public AbstractCommunityResult buildEmpty() { - return new DefaultCommunityResult( - loadDuration, - evalDuration, - writeDuration, - -1, - 0, - 0, - -1, - -1, - -1, - -1, - -1, - -1, - -1, - -1, - -1, - Collections.emptyList()); - } - - } - - private static class DefaultCommunityResult extends AbstractCommunityResult { - - public DefaultCommunityResult(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List biggestCommunities) { - super(loadMillis, computeMillis, writeMillis, postProcessingMillis, nodes, communityCount, p99, p95, p90, p75, p50, p25, p10, p05, p01, biggestCommunities); - } - } -} diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResultBuilder.java b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java similarity index 73% rename from algo/src/main/java/org/neo4j/graphalgo/results/CommunityResultBuilder.java rename to algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java index 10e762120..f204aac18 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityResultBuilder.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java @@ -10,38 +10,31 @@ import java.util.Arrays; import java.util.List; import java.util.function.LongFunction; -import java.util.stream.Collectors; /** * @author mknblch */ -public abstract class CommunityResultBuilder { - - private final IdMapping idMapping; +public abstract class AbstractCommunityResultBuilder { protected long loadDuration = -1; protected long evalDuration = -1; protected long writeDuration = -1; - protected CommunityResultBuilder(IdMapping idMapping) { - this.idMapping = idMapping; - } - - public CommunityResultBuilder withLoadDuration(long loadDuration) { + public AbstractCommunityResultBuilder withLoadDuration(long loadDuration) { this.loadDuration = loadDuration; return this; } - public CommunityResultBuilder withEvalDuration(long evalDuration) { + public AbstractCommunityResultBuilder withEvalDuration(long evalDuration) { this.evalDuration = evalDuration; return this; } - public CommunityResultBuilder withWriteDuration(long writeDuration) { + public AbstractCommunityResultBuilder withWriteDuration(long writeDuration) { this.writeDuration = writeDuration; return this; @@ -123,13 +116,13 @@ public void timeWrite(Runnable runnable) { * @param fun nodeId to communityId mapping function * @return result */ - public T build(long nodes, LongFunction fun) { + public T build(IdMapping nodes, LongFunction fun) { final Histogram histogram = new Histogram(2); final LongLongMap communityMap = new LongLongScatterMap(); - + final long nodeCount = nodes.nodeCount(); final ProgressTimer timer = ProgressTimer.start(); - for (int i = 0; i < nodes; i++) { + for (int i = 0; i < nodeCount; i++) { // map to community id final long r = fun.apply(i); // aggregate community size @@ -137,24 +130,18 @@ public T build(long nodes, LongFunction fun) { // fill histogram histogram.recordValue(r); } + final List top3Communities = top3(nodes, communityMap); timer.stop(); return build(loadDuration, evalDuration, writeDuration, timer.getDuration(), - nodes, + nodeCount, communityMap.size(), - histogram.getValueAtPercentile(.99), - histogram.getValueAtPercentile(.95), - histogram.getValueAtPercentile(.9), - histogram.getValueAtPercentile(.75), - histogram.getValueAtPercentile(.5), - histogram.getValueAtPercentile(.25), - histogram.getValueAtPercentile(.1), - histogram.getValueAtPercentile(.05), - histogram.getValueAtPercentile(.01), - top3(communityMap)); + communityMap, + histogram, + top3Communities); } protected abstract T build( @@ -164,18 +151,11 @@ protected abstract T build( long postProcessingMillis, long nodeCount, long communityCount, - long p99, - long p95, - long p90, - long p75, - long p50, - long p25, - long p10, - long p05, - long p01, + LongLongMap communitySizeMap, + Histogram communityHistogram, List top3Communities); - private List top3(LongLongMap assignment) { + private List top3(IdMapping idMapping, LongLongMap assignment) { // index of top 3 biggest communities final Long[] t3idx = new Long[]{-1L, -1L, -1L}; // size of the top 3 communities @@ -198,10 +178,6 @@ private List top3(LongLongMap assignment) { t3idx[2] = cursor.key; } } - - Arrays.stream(t3idx) - .map(l -> idMapping.toOriginalNodeId(((Integer) l))) - .collect(Collectors.toList()) - + return Arrays.asList(t3idx); } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java new file mode 100644 index 000000000..3b8883011 --- /dev/null +++ b/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java @@ -0,0 +1,92 @@ +package org.neo4j.graphalgo.results; + +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; + +import java.util.Collections; +import java.util.List; + +/** + * @author mknblch + */ +public class DefaultCommunityResult { + + public static final DefaultCommunityResult EMPTY = new DefaultCommunityResult( + 0, 0,0,0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, Collections.emptyList() + ); + + public final long loadMillis; + public final long computeMillis; + public final long postProcessingMillis; + public final long writeMillis; + public final long nodes; + public final long communityCount; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; + public final List top3; + + public DefaultCommunityResult(long loadMillis, + long computeMillis, + long postProcessingMillis, + long writeMillis, + long nodes, + long communityCount, + long p99, + long p95, + long p90, + long p75, + long p50, + long p25, + long p10, + long p05, + long p01, + List top3) { + this.loadMillis = loadMillis; + this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; + this.writeMillis = writeMillis; + this.nodes = nodes; + this.communityCount = communityCount; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; + this.top3 = top3; + } + + public static class DefaultCommunityResultBuilder extends AbstractCommunityResultBuilder { + + @Override + protected DefaultCommunityResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, List top3Communities) { + return new DefaultCommunityResult( + loadMillis, + evalDuration, + postProcessingMillis, + writeMillis, + nodeCount, + communityCount, + communityHistogram.getValueAtPercentile(.99), + communityHistogram.getValueAtPercentile(.95), + communityHistogram.getValueAtPercentile(.9), + communityHistogram.getValueAtPercentile(.75), + communityHistogram.getValueAtPercentile(.5), + communityHistogram.getValueAtPercentile(.25), + communityHistogram.getValueAtPercentile(.1), + communityHistogram.getValueAtPercentile(.05), + communityHistogram.getValueAtPercentile(.01), + top3Communities); + } + } +} diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java index bb208ad63..db8762af5 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java @@ -18,27 +18,83 @@ */ package org.neo4j.graphalgo.results; +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; + import java.util.Collections; import java.util.List; -public class LabelPropagationStats extends AbstractCommunityResult { +public class LabelPropagationStats { - public final long iterations; + public static final LabelPropagationStats EMPTY = new LabelPropagationStats( + 0, + 0, + 0, + 0, + 0, + 0, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + Collections.emptyList(), + 0, + false, + false, + "", + ""); + + public final long loadMillis; + public final long computeMillis; + public final long postProcessingMillis; + public final long writeMillis; public final long nodes; + public final long communityCount; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; + public final List top3; + public final long iterations; public final boolean write, didConverge; public final String weightProperty, partitionProperty; - public LabelPropagationStats(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List biggestCommunities, long iterations, boolean write, boolean didConverge, String weightProperty, String partitionProperty) { - super(loadMillis, computeMillis, writeMillis, postProcessingMillis, nodes, communityCount, p99, p95, p90, p75, p50, p25, p10, p05, p01, biggestCommunities); + public LabelPropagationStats(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations, boolean write, boolean didConverge, String weightProperty, String partitionProperty) { + this.loadMillis = loadMillis; + this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; + this.writeMillis = writeMillis; + this.nodes = nodes; + this.communityCount = communityCount; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; + this.top3 = top3; this.iterations = iterations; this.write = write; this.didConverge = didConverge; this.weightProperty = weightProperty; this.partitionProperty = partitionProperty; - this.nodes = nodes; } - public static class Builder extends CommunityResultBuilder { + + public static class Builder extends AbstractCommunityResultBuilder { private long iterations = 0; private boolean didConverge = false; @@ -71,22 +127,8 @@ public Builder partitionProperty(final String partitionProperty) { return this; } - protected LabelPropagationStats build(long loadMillis, - long computeMillis, - long writeMillis, - long postProcessingMillis, - long nodeCount, - long communityCount, - long p99, - long p95, - long p90, - long p75, - long p50, - long p25, - long p10, - long p05, - long p01, - List top3Communities) { + @Override + protected LabelPropagationStats build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, List top3Communities) { return new LabelPropagationStats( loadMillis, computeMillis, @@ -94,15 +136,15 @@ protected LabelPropagationStats build(long loadMillis, postProcessingMillis, nodeCount, communityCount, - p99, - p95, - p90, - p75, - p50, - p25, - p10, - p05, - p01, + communityHistogram.getValueAtPercentile(.99), + communityHistogram.getValueAtPercentile(.95), + communityHistogram.getValueAtPercentile(.9), + communityHistogram.getValueAtPercentile(.75), + communityHistogram.getValueAtPercentile(.5), + communityHistogram.getValueAtPercentile(.25), + communityHistogram.getValueAtPercentile(.1), + communityHistogram.getValueAtPercentile(.05), + communityHistogram.getValueAtPercentile(.01), top3Communities, iterations, write, @@ -112,29 +154,5 @@ protected LabelPropagationStats build(long loadMillis, ); } - public LabelPropagationStats emptyResult() { - return new LabelPropagationStats( - loadDuration, - evalDuration, - writeDuration, - -1, - 0, - 0, - -1, - -1, - -1, - -1, - -1, - -1, - -1, - -1, - -1, - Collections.emptyList(), - iterations, - write, - didConverge, - weightProperty, - partitionProperty); - } } } diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java index a7b856f8b..3b77c9ed3 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/LabelPropagationProcIntegrationTest.java @@ -115,7 +115,7 @@ public void shouldTakeParametersFromConfig() { @Test public void shouldRunLabelPropagation() { - String query = "CALL algo.labelPropagation(null, 'X', 'OUTGOING', {batchSize:$batchSize,concurrency:$concurrency}) RETURN loadMillis"; + String query = "CALL algo.labelPropagation(null, 'X', 'OUTGOING', {batchSize:$batchSize,concurrency:$concurrency})"; String check = "MATCH (n) WHERE n.id IN [0,1] RETURN n.partition AS partition"; runQuery(query, parParams(), row -> { diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java index 0a75112bc..6a53c2e5e 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java @@ -97,18 +97,17 @@ public void clearCommunities() { @Test public void test() { final String cypher = "CALL algo.louvain('', '', {concurrency:1}) " + - "YIELD nodes, communityCount, iterations, loadMillis, computeMillis, writeMillis, top3, p99"; + "YIELD nodes, communityCount, loadMillis, computeMillis, writeMillis, postProcessingMillis, top3, p99"; DB.execute(cypher).accept(row -> { final long nodes = row.getNumber("nodes").longValue(); final long communityCount = row.getNumber("communityCount").longValue(); - final long iterations = row.getNumber("iterations").longValue(); final long loadMillis = row.getNumber("loadMillis").longValue(); final long computeMillis = row.getNumber("computeMillis").longValue(); final long writeMillis = row.getNumber("writeMillis").longValue(); + System.out.println("postProcessingMillis = " + row.getNumber("postProcessingMillis")); System.out.println("nodes = " + nodes); System.out.println("communityCount = " + communityCount); - System.out.println("iterations = " + iterations); System.out.println("p99 = " + row.get("p99")); System.out.println("top3 = " + row.get("top3")); diff --git a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainTest1.java b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainTest1.java index f6e3b7d8e..054fb7cdb 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainTest1.java +++ b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainTest1.java @@ -25,6 +25,7 @@ import org.junit.rules.ErrorCollector; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.neo4j.graphalgo.LouvainProc; import org.neo4j.graphalgo.TestProgressLogger; import org.neo4j.graphalgo.api.Graph; import org.neo4j.graphalgo.api.GraphFactory; @@ -32,6 +33,7 @@ import org.neo4j.graphalgo.core.heavyweight.HeavyGraphFactory; import org.neo4j.graphalgo.core.huge.HugeGraphFactory; import org.neo4j.graphalgo.core.utils.Pools; +import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.core.utils.paged.HugeLongArray; import org.neo4j.graphalgo.impl.louvain.Louvain; @@ -132,6 +134,7 @@ public void testRunner() throws Exception { setup(unidirectional); final Louvain algorithm = new Louvain(graph, Pools.DEFAULT, 1, AllocationTracker.EMPTY) .withProgressLogger(TestProgressLogger.INSTANCE) + .withTerminationFlag(TerminationFlag.RUNNING_TRUE) .compute(10, 10); final int[][] dendogram = algorithm.getDendrogram(); for (int i = 1; i <= dendogram.length; i++) { From 266457936cdcf0fe911a38a1cbaf8a2c969644b9 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Fri, 25 Jan 2019 10:42:59 +0100 Subject: [PATCH 11/26] adapt SCC --- .../java/org/neo4j/graphalgo/LouvainProc.java | 23 --- .../StronglyConnectedComponentsProc.java | 71 +++---- .../org/neo4j/graphalgo/impl/Algorithm.java | 4 +- .../graphalgo/impl/scc/SCCAlgorithm.java | 61 ------ .../neo4j/graphalgo/impl/scc/SCCTarjan.java | 182 ++++++------------ .../neo4j/graphalgo/results/SCCResult.java | 105 ++++++---- ...onnectedComponentsProcIntegrationTest.java | 8 +- 7 files changed, 155 insertions(+), 299 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index 4ba56e0b7..5bbe23b31 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -241,29 +241,6 @@ public LouvainResult(long loadMillis, long computeMillis, long postProcessingMil this.top3 = top3; this.iterations = iterations; } - - @Override - public String toString() { - return "LouvainResult{" + - "loadMillis=" + loadMillis + - ", computeMillis=" + computeMillis + - ", postProcessingMillis=" + postProcessingMillis + - ", writeMillis=" + writeMillis + - ", nodes=" + nodes + - ", communityCount=" + communityCount + - ", p99=" + p99 + - ", p95=" + p95 + - ", p90=" + p90 + - ", p75=" + p75 + - ", p50=" + p50 + - ", p25=" + p25 + - ", p10=" + p10 + - ", p05=" + p05 + - ", p01=" + p01 + - ", top3=" + top3 + - ", iterations=" + iterations + - '}'; - } } public static class Builder extends AbstractCommunityResultBuilder { diff --git a/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java b/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java index c464107f3..a2e2c771c 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java @@ -49,6 +49,7 @@ import org.neo4j.values.storable.IntValue; import org.neo4j.values.storable.Values; +import java.util.Arrays; import java.util.Map; import java.util.stream.Stream; @@ -117,7 +118,7 @@ public Stream sccTarjan( loadTimer.stop(); if (graph.nodeCount() == 0) { - return Stream.of(builder.build()); + return Stream.of(SCCResult.EMPTY); } final TerminationFlag terminationFlag = TerminationFlag.wrap(transaction); @@ -125,38 +126,25 @@ public Stream sccTarjan( .withProgressLogger(ProgressLogger.wrap(log, "SCC(Tarjan)")) .withTerminationFlag(terminationFlag); - builder.timeEval(() -> { - tarjan.compute(); - builder.withMaxSetSize(tarjan.getMaxSetSize()) - .withMinSetSize(tarjan.getMinSetSize()) - .withSetCount(tarjan.getConnectedComponents().size()); - }); + builder.timeEval(tarjan::compute); + final int[] connectedComponents = tarjan.getConnectedComponents(); if (configuration.isWriteFlag()) { builder.timeWrite(() -> { - final ObjectArrayList connectedComponents = tarjan.getConnectedComponents(); + graph.release(); tarjan.release(); - Exporter.of(new DirectIdMapping(connectedComponents.size()), api) + Exporter.of(api, graph) .withLog(log) .parallel(Pools.DEFAULT, configuration.getConcurrency(), terminationFlag) .build() .write( configuration.get(CONFIG_WRITE_PROPERTY, CONFIG_CLUSTER), - propertyId -> (ops, id) -> { - final int setId = (int) (id); - IntValue property = Values.intValue(setId + 1); - for (final IntCursor iCursor : connectedComponents.get(setId)) { - ops.nodeSetProperty( - graph.toOriginalNodeId(iCursor.value), - propertyId, - property); - } - } + connectedComponents, + Translators.OPTIONAL_INT_ARRAY_TRANSLATOR ); }); } - - return Stream.of(builder.build()); + return Stream.of(builder.build(graph, l -> (long) connectedComponents[((int) l)])); } // algo.scc.tunedTarjan @@ -181,7 +169,7 @@ public Stream sccTunedTarjan( loadTimer.stop(); if (graph.nodeCount() == 0) { - return Stream.of(builder.build()); + return Stream.of(SCCResult.EMPTY); } final TerminationFlag terminationFlag = TerminationFlag.wrap(transaction); @@ -191,10 +179,6 @@ public Stream sccTunedTarjan( builder.timeEval(tarjan::compute); - builder.withMaxSetSize(tarjan.getMaxSetSize()) - .withMinSetSize(tarjan.getMinSetSize()) - .withSetCount(tarjan.getSetCount()); - if (configuration.isWriteFlag()) { builder.timeWrite(() -> Exporter .of(api, graph) @@ -208,7 +192,8 @@ public Stream sccTunedTarjan( )); } - return Stream.of(builder.build()); + final int[] connectedComponents = tarjan.getConnectedComponents(); + return Stream.of(builder.build(graph, l -> (long) connectedComponents[((int) l)])); } // algo.scc.tunedTarjan.stream @@ -261,7 +246,7 @@ public Stream sccIterativeTarjan( loadTimer.stop(); if (graph.nodeCount() == 0) { - return Stream.of(builder.build()); + return Stream.of(SCCResult.EMPTY); } final AllocationTracker tracker = AllocationTracker.create(); @@ -272,15 +257,17 @@ public Stream sccIterativeTarjan( builder.timeEval(tarjan::compute); - builder.withSetCount(tarjan.getSetCount()) - .withMinSetSize(tarjan.getMinSetSize()) - .withMaxSetSize(tarjan.getMaxSetSize()); - if (configuration.isWriteFlag()) { builder.timeWrite(() -> write(configuration, graph, terminationFlag, tarjan)); } - return Stream.of(builder.build()); + if (graph instanceof HugeGraph) { + final HugeLongArray connectedComponents = tarjan.getConnectedComponents(); + return Stream.of(builder.build(graph, connectedComponents::get)); + } + final int[] connectedComponents = tarjan.getConnectedComponents(); + tarjan.release(); + return Stream.of(builder.build(graph, l -> (long) connectedComponents[((int) l)])); } private void write(ProcedureConfiguration configuration, Graph graph, TerminationFlag terminationFlag, SCCAlgorithm tarjan) { @@ -302,8 +289,6 @@ private void write(ProcedureConfiguration configuration, Graph graph, Terminatio } final int[] connectedComponents = tarjan.getConnectedComponents(); - graph.release(); - tarjan.release(); Exporter.of(api, graph) .withLog(log) .parallel(Pools.DEFAULT, configuration.getConcurrency(), terminationFlag) @@ -372,7 +357,7 @@ public Stream multistep( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder.build()); + return Stream.of(SCCResult.EMPTY); } final TerminationFlag terminationFlag = TerminationFlag.wrap(transaction); @@ -384,12 +369,9 @@ public Stream multistep( builder.timeEval(multistep::compute); - builder.withMaxSetSize(multistep.getMaxSetSize()) - .withMinSetSize(multistep.getMinSetSize()) - .withSetCount(multistep.getSetCount()); + final int[] connectedComponents = multistep.getConnectedComponents(); if (configuration.isWriteFlag()) { - final int[] connectedComponents = multistep.getConnectedComponents(); graph.release(); multistep.release(); builder.timeWrite(() -> Exporter @@ -404,7 +386,7 @@ public Stream multistep( )); } - return Stream.of(builder.build()); + return Stream.of(builder.build(graph, l -> (long) connectedComponents[((int) l)])); } // algo.scc.multistep.stream @@ -417,23 +399,19 @@ public Stream multistepStream( @Name(value = "config", defaultValue = "{}") Map config) { ProcedureConfiguration configuration = ProcedureConfiguration.create(config); - Graph graph = new GraphLoader(api, Pools.DEFAULT) .init(log, label, relationship, configuration) .withoutRelationshipWeights() .load(configuration.getGraphImpl()); - if (graph.nodeCount() == 0) { graph.release(); return Stream.empty(); } - final MultistepSCC multistep = new MultistepSCC(graph, org.neo4j.graphalgo.core.utils.Pools.DEFAULT, configuration.getConcurrency(), configuration.getNumber("cutoff", 100_000).intValue()) .withProgressLogger(ProgressLogger.wrap(log, "SCC(MultiStep)")) .withTerminationFlag(TerminationFlag.wrap(transaction)); - multistep.compute(); graph.release(); return multistep.resultStream(); @@ -450,17 +428,14 @@ public Stream fwbwStream( @Name(value = "config", defaultValue = "{}") Map config) { ProcedureConfiguration configuration = ProcedureConfiguration.create(config); - Graph graph = new GraphLoader(api, Pools.DEFAULT) .init(log, label, relationship, configuration) .withoutRelationshipWeights() .load(configuration.getGraphImpl()); - if (graph.nodeCount() == 0) { graph.release(); return Stream.empty(); } - final ForwardBackwardScc algo = new ForwardBackwardScc(graph, Pools.DEFAULT, configuration.getConcurrency()) .withProgressLogger(ProgressLogger.wrap(log, "SCC(ForwardBackward)")) diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/Algorithm.java b/algo/src/main/java/org/neo4j/graphalgo/impl/Algorithm.java index 09ee40c94..83db2c31a 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/Algorithm.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/Algorithm.java @@ -27,9 +27,9 @@ */ public abstract class Algorithm> implements TerminationFlag { - private ProgressLogger progressLogger = ProgressLogger.NULL_LOGGER; + protected ProgressLogger progressLogger = ProgressLogger.NULL_LOGGER; - private TerminationFlag terminationFlag = TerminationFlag.RUNNING_TRUE; + protected TerminationFlag terminationFlag = TerminationFlag.RUNNING_TRUE; public abstract ME me(); diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCAlgorithm.java b/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCAlgorithm.java index 522f9c6e9..e549a54e0 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCAlgorithm.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCAlgorithm.java @@ -62,67 +62,6 @@ public StreamResult(long nodeId, long partition) { } } - class Result { - - public final Long loadMillis; - public final Long computeMillis; - public final Long writeMillis; - public final Long setCount; - public final Long minSetSize; - public final Long maxSetSize; - - public Result(Long loadMillis, - Long computeMillis, - Long writeMillis, - Long setCount, - Long minSetSize, - Long maxSetSize) { - this.loadMillis = loadMillis; - this.computeMillis = computeMillis; - this.writeMillis = writeMillis; - this.setCount = setCount; - this.minSetSize = minSetSize; - this.maxSetSize = maxSetSize; - } - - public static Result.Builder builder() { - return new Result.Builder(); - } - - public static final class Builder extends AbstractResultBuilder { - - private long setCount; - private long minSetSize; - private long maxSetSize; - - public Result.Builder withSetCount(long setCount) { - this.setCount = setCount; - return this; - } - - public Result.Builder withMinSetSize(long minSetSize) { - this.minSetSize = minSetSize; - return this; - } - - public Result.Builder withMaxSetSize(long maxSetSize) { - this.maxSetSize = maxSetSize; - return this; - } - - @Override - public Result build() { - return new Result(loadDuration, - evalDuration, - writeDuration, - setCount, - minSetSize, - maxSetSize); - } - } - - } - static SCCAlgorithm iterativeTarjan(Graph graph, AllocationTracker tracker) { if (graph instanceof HugeGraph) { return new HugeSCCIterativeTarjan((HugeGraph) graph, tracker); diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCTarjan.java b/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCTarjan.java index b63bd2acc..9486a885f 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCTarjan.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCTarjan.java @@ -1,36 +1,31 @@ /** * Copyright (c) 2017 "Neo4j, Inc." - * + *

* This file is part of Neo4j Graph Algorithms . - * + *

* Neo4j Graph Algorithms is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + *

* This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + *

* You should have received a copy of the GNU General Public License * along with this program. If not, see . */ package org.neo4j.graphalgo.impl.scc; -import com.carrotsearch.hppc.IntHashSet; -import com.carrotsearch.hppc.IntSet; import com.carrotsearch.hppc.IntStack; -import com.carrotsearch.hppc.ObjectArrayList; import org.neo4j.graphalgo.api.Graph; -import org.neo4j.graphalgo.api.RelationshipConsumer; import org.neo4j.graphalgo.core.utils.ProgressLogger; import org.neo4j.graphalgo.impl.Algorithm; import org.neo4j.graphdb.Direction; import java.util.Arrays; import java.util.BitSet; -import java.util.function.IntPredicate; /** * Sequential strongly connected components algorithm (Tarjan). @@ -43,51 +38,32 @@ public class SCCTarjan extends Algorithm { private Graph graph; private final int nodeCount; - private Aggregator aggregator; + private int[] communities; + private int[] indices; + private int[] lowLink; + private final BitSet onStack; + private final IntStack stack; + private int index; + public SCCTarjan(Graph graph) { this.graph = graph; nodeCount = Math.toIntExact(graph.nodeCount()); - - aggregator = new Aggregator(graph, - new int[nodeCount], - new int[nodeCount], - new ObjectArrayList<>(), - new BitSet(nodeCount), - new IntStack(nodeCount)); + indices = new int[nodeCount]; + lowLink = new int[nodeCount]; + onStack = new BitSet(nodeCount); + stack = new IntStack(nodeCount); + communities = new int[nodeCount]; + Arrays.setAll(communities, i -> i); } public SCCTarjan compute() { - aggregator.reset(); - graph.forEachNode(aggregator); + graph.forEachNode(this::test); return this; } - /** - * get connected components list - * - * @return list of sets of strongly connected component ID's - */ - public ObjectArrayList getConnectedComponents() { - return aggregator.connectedComponents; - } - - /** - * return the maximum set size - * - * @return the maximum set size - */ - public long getMaxSetSize() { - return graph.nodeCount() == 0 ? 0 : aggregator.maxSetSize; - } - - /** - * return the minimum set size - * - * @return minimum set size - */ - public long getMinSetSize() { - return graph.nodeCount() == 0 ? 0 : aggregator.minSetSize; + public int[] getConnectedComponents() { + return communities; } @Override @@ -97,93 +73,57 @@ public SCCTarjan me() { @Override public SCCTarjan release() { - aggregator = null; + stack.clear(); + communities = null; + indices = null; + lowLink = null; graph = null; return this; } - private final class Aggregator implements IntPredicate, RelationshipConsumer { - - private final Graph graph; - private final int[] indices; - private final int[] lowLink; - private final ObjectArrayList connectedComponents; - private final BitSet onStack; - private final IntStack stack; - private int index; - private long minSetSize = Long.MAX_VALUE; - private long maxSetSize = 0; - private ProgressLogger progressLogger; - - private Aggregator(Graph graph, int[] indices, int[] lowLink, ObjectArrayList connectedComponents, BitSet onStack, IntStack stack) { - this.graph = graph; - this.indices = indices; - this.lowLink = lowLink; - this.connectedComponents = connectedComponents; - this.onStack = onStack; - this.stack = stack; - this.progressLogger = getProgressLogger(); - } - - public void reset() { - connectedComponents.clear(); - Arrays.fill(indices, -1); - Arrays.fill(lowLink, -1); - onStack.clear(); - stack.clear(); - index = 0; - minSetSize = Long.MAX_VALUE; - maxSetSize = 0; - } + public void reset() { + Arrays.fill(indices, -1); + Arrays.fill(lowLink, -1); + onStack.clear(); + stack.clear(); + index = 0; + } - private void strongConnect(int node) { - lowLink[node] = index; - indices[node] = index; - index++; - stack.push(node); - onStack.set(node); - graph.forEachRelationship(node, Direction.OUTGOING, this); - if (indices[node] == lowLink[node]) { - relax(node); - } + private void strongConnect(int node) { + lowLink[node] = index; + indices[node] = index++; + stack.push(node); + onStack.set(node); + graph.forEachRelationship(node, Direction.OUTGOING, this::accept); + if (indices[node] == lowLink[node]) { + relax(node); } + } - private void relax(int nodeId) { - IntHashSet connected = new IntHashSet(); - int w; - do { - w = stack.pop(); - onStack.clear(w); - connected.add(w); - } while (w != nodeId); - connectedComponents.add(connected); - int size = connected.size(); - if (size < minSetSize) { - minSetSize = size; - } - if (size > maxSetSize) { - maxSetSize = size; - } - } + private void relax(int nodeId) { + int w; + do { + w = stack.pop(); + onStack.clear(w); + communities[w] = nodeId; + } while (w != nodeId); + } - @Override - public boolean accept(int source, int target, long edgeId) { - if (indices[target] == -1) { - strongConnect(target); - lowLink[source] = Math.min(lowLink[source], lowLink[target]); - } else if (onStack.get(target)) { - lowLink[source] = Math.min(lowLink[source], indices[target]); - } - return true; + private boolean accept(int source, int target, long unused) { + if (indices[target] == -1) { + strongConnect(target); + lowLink[source] = Math.min(lowLink[source], lowLink[target]); + } else if (onStack.get(target)) { + lowLink[source] = Math.min(lowLink[source], indices[target]); } + return true; + } - @Override - public boolean test(int node) { - if (indices[node] == -1) { - strongConnect(node); - } - progressLogger.logProgress((double) node / (nodeCount - 1)); - return running(); + private boolean test(int node) { + if (indices[node] == -1) { + strongConnect(node); } + progressLogger.logProgress((double) node / (nodeCount - 1)); + return running(); } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java index d040bdbcb..3bb99f417 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java @@ -18,65 +18,94 @@ */ package org.neo4j.graphalgo.results; +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; + +import java.util.Collections; +import java.util.List; + /** * @author mknblch */ public class SCCResult { - public final Long loadMillis; - public final Long computeMillis; - public final Long writeMillis; - public final Long setCount; - public final Long minSetSize; - public final Long maxSetSize; + public static SCCResult EMPTY = new SCCResult( + 0, 0, 0, 0,0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, Collections.emptyList(), 0 + ); + + public final long loadMillis; + public final long computeMillis; + public final long postProcessingMillis; + public final long writeMillis; + public final long nodes; + public final long communityCount; + public final long setCount; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; + public final List top3; + public final long iterations; - public SCCResult(Long loadMillis, - Long computeMillis, - Long writeMillis, - Long setCount, - Long minSetSize, - Long maxSetSize) { + public SCCResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; this.writeMillis = writeMillis; - this.setCount = setCount; - this.minSetSize = minSetSize; - this.maxSetSize = maxSetSize; + this.nodes = nodes; + this.setCount = this.communityCount = communityCount; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; + this.top3 = top3; + this.iterations = iterations; } public static Builder builder() { return new Builder(); } - public static final class Builder extends AbstractResultBuilder { + public static final class Builder extends AbstractCommunityResultBuilder { - private long setCount; - private long minSetSize; - private long maxSetSize; - - public Builder withSetCount(long setCount) { - this.setCount = setCount; - return this; - } - - public Builder withMinSetSize(long minSetSize) { - this.minSetSize = minSetSize; - return this; - } + private int iterations = -1; - public Builder withMaxSetSize(long maxSetSize) { - this.maxSetSize = maxSetSize; + public Builder withIterations(int iterations) { + this.iterations = iterations; return this; } @Override - public SCCResult build() { - return new SCCResult(loadDuration, - evalDuration, - writeDuration, - setCount, - minSetSize, - maxSetSize); + protected SCCResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, List top3Communities) { + return new SCCResult( + loadMillis, + computeMillis, + writeMillis, + postProcessingMillis, + nodeCount, + communityCount, + communityHistogram.getValueAtPercentile(.99), + communityHistogram.getValueAtPercentile(.95), + communityHistogram.getValueAtPercentile(.9), + communityHistogram.getValueAtPercentile(.75), + communityHistogram.getValueAtPercentile(.5), + communityHistogram.getValueAtPercentile(.25), + communityHistogram.getValueAtPercentile(.1), + communityHistogram.getValueAtPercentile(.05), + communityHistogram.getValueAtPercentile(.01), + top3Communities, + iterations + ); } } diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java index eeaba8f69..157af4e01 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java @@ -97,22 +97,18 @@ public static Collection data() { @Test public void testScc() throws Exception { - db.execute("CALL algo.scc('Node', 'TYPE', {write:true, graph:'"+graphImpl+"'}) YIELD loadMillis, computeMillis, writeMillis, setCount, maxSetSize, minSetSize") + db.execute("CALL algo.scc('Node', 'TYPE', {write:true, graph:'"+graphImpl+"'}) YIELD loadMillis, computeMillis, writeMillis, setCount, top3") .accept(row -> { System.out.println(row.getNumber("loadMillis").longValue()); System.out.println(row.getNumber("computeMillis").longValue()); System.out.println(row.getNumber("writeMillis").longValue()); System.out.println(row.getNumber("setCount").longValue()); - System.out.println(row.getNumber("maxSetSize").longValue()); - System.out.println(row.getNumber("minSetSize").longValue()); + System.out.println(row.get("top3")); assertNotEquals(-1L, row.getNumber("computeMillis").longValue()); assertNotEquals(-1L, row.getNumber("writeMillis").longValue()); assertEquals(2, row.getNumber("setCount").longValue()); - assertEquals(2, row.getNumber("minSetSize").longValue()); - assertEquals(3, row.getNumber("maxSetSize").longValue()); - return true; }); } From 3c17c1ac7028a891a130ffc16bd07c8983aacd39 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Fri, 25 Jan 2019 11:37:47 +0100 Subject: [PATCH 12/26] WIP --- .../StronglyConnectedComponentsProc.java | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java b/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java index a2e2c771c..8d4520fcd 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java @@ -1,31 +1,27 @@ /** * Copyright (c) 2017 "Neo4j, Inc." - * + *

* This file is part of Neo4j Graph Algorithms . - * + *

* Neo4j Graph Algorithms is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + *

* This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + *

* You should have received a copy of the GNU General Public License * along with this program. If not, see . */ package org.neo4j.graphalgo; -import com.carrotsearch.hppc.IntSet; -import com.carrotsearch.hppc.ObjectArrayList; -import com.carrotsearch.hppc.cursors.IntCursor; import org.neo4j.graphalgo.api.Graph; import org.neo4j.graphalgo.api.HugeGraph; import org.neo4j.graphalgo.core.GraphLoader; import org.neo4j.graphalgo.core.ProcedureConfiguration; -import org.neo4j.graphalgo.core.neo4jview.DirectIdMapping; import org.neo4j.graphalgo.core.utils.Pools; import org.neo4j.graphalgo.core.utils.ProgressLogger; import org.neo4j.graphalgo.core.utils.ProgressTimer; @@ -34,7 +30,7 @@ import org.neo4j.graphalgo.core.utils.paged.HugeLongArray; import org.neo4j.graphalgo.core.write.Exporter; import org.neo4j.graphalgo.core.write.Translators; -import org.neo4j.graphalgo.impl.*; +import org.neo4j.graphalgo.impl.ForwardBackwardScc; import org.neo4j.graphalgo.impl.multistepscc.MultistepSCC; import org.neo4j.graphalgo.impl.scc.SCCAlgorithm; import org.neo4j.graphalgo.impl.scc.SCCTarjan; @@ -46,10 +42,7 @@ import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; import org.neo4j.procedure.*; -import org.neo4j.values.storable.IntValue; -import org.neo4j.values.storable.Values; -import java.util.Arrays; import java.util.Map; import java.util.stream.Stream; @@ -258,7 +251,7 @@ public Stream sccIterativeTarjan( builder.timeEval(tarjan::compute); if (configuration.isWriteFlag()) { - builder.timeWrite(() -> write(configuration, graph, terminationFlag, tarjan)); + builder.timeWrite(() -> write(configuration, graph, terminationFlag, tarjan)); } if (graph instanceof HugeGraph) { From dd4d4b2ff7024c7d8d190b2c10cd9c316474564b Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Fri, 25 Jan 2019 14:29:15 +0100 Subject: [PATCH 13/26] add syntactic sugar methods in builder, adapt triangle count --- .../org/neo4j/graphalgo/TriangleProc.java | 126 +++++++++++++----- .../AbstractCommunityResultBuilder.java | 16 +++ 2 files changed, 109 insertions(+), 33 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java index 4af94d276..8dc9f5dfb 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java @@ -18,6 +18,8 @@ */ package org.neo4j.graphalgo; +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; import org.neo4j.graphalgo.api.Graph; import org.neo4j.graphalgo.api.HugeGraph; import org.neo4j.graphalgo.core.GraphLoader; @@ -30,12 +32,14 @@ import org.neo4j.graphalgo.core.write.Exporter; import org.neo4j.graphalgo.core.write.Translators; import org.neo4j.graphalgo.impl.triangle.*; -import org.neo4j.graphalgo.results.AbstractResultBuilder; +import org.neo4j.graphalgo.results.AbstractCommunityResultBuilder; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; import org.neo4j.procedure.*; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ForkJoinPool; @@ -221,11 +225,23 @@ public Stream triangleCountQueue( } } - builder.withNodeCount(graph.nodeCount()) - .withTriangleCount(triangleCount.getTriangleCount()) - .withAverageClusteringCoefficient(triangleCount.getAverageCoefficient()); - return Stream.of(builder.build()); + builder.withAvergaeClusteringCoefficient(triangleCount.getAverageCoefficient()) + .withTriangleCount(triangleCount.getTriangleCount()); + + return buildResult(builder, graph, triangleCount); + } + + private Stream buildResult(TriangleCountResultBuilder builder, Graph graph, TriangleCountAlgorithm algorithm) { + + if (algorithm instanceof IntersectingTriangleCount) { + final PagedAtomicIntegerArray triangles = ((IntersectingTriangleCount) algorithm).getTriangles(); + return Stream.of(builder.buildLI(graph, triangles::get)); + } else if (algorithm instanceof TriangleCountQueue){ + final AtomicIntegerArray triangles = ((TriangleCountQueue) algorithm).getTriangles(); + return Stream.of(builder.buildII(graph, triangles::get)); + } + throw new UnsupportedOperationException("unknown algorithm"); } /** @@ -364,11 +380,11 @@ public Stream triangleCountExp3( } } - builder.withNodeCount(graph.nodeCount()) - .withTriangleCount(triangleCount.getTriangleCount()) - .withAverageClusteringCoefficient(triangleCount.getAverageClusteringCoefficient()); + builder.withAvergaeClusteringCoefficient(triangleCount.getAverageClusteringCoefficient()) + .withTriangleCount(triangleCount.getTriangleCount()); - return Stream.of(builder.build()); + final AtomicIntegerArray triangles = triangleCount.getTriangles(); + return Stream.of(builder.buildII(graph, triangles::get)); } @@ -377,42 +393,72 @@ public Stream triangleCountExp3( */ public static class Result { + + public static final Result EMPTY = new Result( + 0, + 0, + 0, + 0, + 0, + 0, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + Collections.emptyList(), + .0); + public final long loadMillis; public final long computeMillis; + public final long postProcessingMillis; public final long writeMillis; public final long nodeCount; public final long triangleCount; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; + public final List top3; public final double averageClusteringCoefficient; - public Result( - long loadMillis, - long computeMillis, - long writeMillis, - long nodeCount, - long triangleCount, - double averageClusteringCoefficient) { + public Result(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodeCount, long triangleCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, double averageClusteringCoefficient) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; this.writeMillis = writeMillis; this.nodeCount = nodeCount; - this.triangleCount = triangleCount; this.averageClusteringCoefficient = averageClusteringCoefficient; + this.triangleCount = triangleCount; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; + this.top3 = top3; } } - public class TriangleCountResultBuilder extends AbstractResultBuilder { + public class TriangleCountResultBuilder extends AbstractCommunityResultBuilder { - private long nodeCount = -1L; - private long triangleCount = -1L; - private double averageClusteringCoefficient = -1d; - - public TriangleCountResultBuilder withAverageClusteringCoefficient(double averageClusteringCoefficient) { - this.averageClusteringCoefficient = averageClusteringCoefficient; - return this; - } + private double avergaeClusteringCoefficient = .0; + private long triangleCount = 0; - public TriangleCountResultBuilder withNodeCount(long nodeCount) { - this.nodeCount = nodeCount; + public TriangleCountResultBuilder withAvergaeClusteringCoefficient(double avergaeClusteringCoefficient) { + this.avergaeClusteringCoefficient = avergaeClusteringCoefficient; return this; } @@ -421,15 +467,29 @@ public TriangleCountResultBuilder withTriangleCount(long triangleCount) { return this; } + + // communityCount is not used here @Override - public Result build() { + protected Result build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, List top3Communities) { return new Result( - loadDuration, - evalDuration, - writeDuration, + loadMillis, + computeMillis, + writeMillis, + postProcessingMillis, nodeCount, triangleCount, - averageClusteringCoefficient); + communityHistogram.getValueAtPercentile(.99), + communityHistogram.getValueAtPercentile(.95), + communityHistogram.getValueAtPercentile(.9), + communityHistogram.getValueAtPercentile(.75), + communityHistogram.getValueAtPercentile(.5), + communityHistogram.getValueAtPercentile(.25), + communityHistogram.getValueAtPercentile(.1), + communityHistogram.getValueAtPercentile(.05), + communityHistogram.getValueAtPercentile(.01), + top3Communities, + avergaeClusteringCoefficient + ); } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java index f204aac18..e57cb9780 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java @@ -3,13 +3,17 @@ import com.carrotsearch.hppc.LongLongMap; import com.carrotsearch.hppc.LongLongScatterMap; import com.carrotsearch.hppc.cursors.LongLongCursor; +import com.carrotsearch.hppc.procedures.IntIntProcedure; import org.HdrHistogram.Histogram; import org.neo4j.graphalgo.api.IdMapping; import org.neo4j.graphalgo.core.utils.ProgressTimer; import java.util.Arrays; import java.util.List; +import java.util.function.IntFunction; +import java.util.function.IntToLongFunction; import java.util.function.LongFunction; +import java.util.function.LongToIntFunction; /** * @author mknblch @@ -109,6 +113,18 @@ public void timeWrite(Runnable runnable) { } } + public T buildII(IdMapping nodes, IntFunction fun) { + return build(nodes, value -> (long) fun.apply((int) value)); + } + + public T buildLI(IdMapping nodes, LongToIntFunction fun) { + return build(nodes, value -> (long) fun.applyAsInt(value)); + } + + public T buildIL(IdMapping nodes, IntToLongFunction fun) { + return build(nodes, value -> fun.applyAsLong((int) value)); + } + /** * build result * From e33ecbcc87dc14c0a5f4ae287cfe881b5b244a64 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Tue, 29 Jan 2019 14:00:19 +0100 Subject: [PATCH 14/26] map resulting nodeIds back to graphId --- .../AbstractCommunityResultBuilder.java | 46 +++++++++---------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java index e57cb9780..08153ec25 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java @@ -3,15 +3,13 @@ import com.carrotsearch.hppc.LongLongMap; import com.carrotsearch.hppc.LongLongScatterMap; import com.carrotsearch.hppc.cursors.LongLongCursor; -import com.carrotsearch.hppc.procedures.IntIntProcedure; import org.HdrHistogram.Histogram; import org.neo4j.graphalgo.api.IdMapping; import org.neo4j.graphalgo.core.utils.ProgressTimer; -import java.util.Arrays; +import java.util.LinkedList; import java.util.List; import java.util.function.IntFunction; -import java.util.function.IntToLongFunction; import java.util.function.LongFunction; import java.util.function.LongToIntFunction; @@ -25,21 +23,18 @@ public abstract class AbstractCommunityResultBuilder { protected long writeDuration = -1; public AbstractCommunityResultBuilder withLoadDuration(long loadDuration) { - this.loadDuration = loadDuration; return this; } public AbstractCommunityResultBuilder withEvalDuration(long evalDuration) { - this.evalDuration = evalDuration; return this; } public AbstractCommunityResultBuilder withWriteDuration(long writeDuration) { - this.writeDuration = writeDuration; return this; } @@ -51,7 +46,6 @@ public AbstractCommunityResultBuilder withWriteDuration(long writeDuration) { * @return */ public ProgressTimer timeLoad() { - return ProgressTimer.start(this::withLoadDuration); } @@ -62,7 +56,6 @@ public ProgressTimer timeLoad() { * @return */ public ProgressTimer timeEval() { - return ProgressTimer.start(this::withEvalDuration); } @@ -73,7 +66,6 @@ public ProgressTimer timeEval() { * @return */ public ProgressTimer timeWrite() { - return ProgressTimer.start(this::withWriteDuration); } @@ -83,7 +75,6 @@ public ProgressTimer timeWrite() { * @param runnable */ public void timeLoad(Runnable runnable) { - try (ProgressTimer timer = timeLoad()) { runnable.run(); } @@ -95,7 +86,6 @@ public void timeLoad(Runnable runnable) { * @param runnable */ public void timeEval(Runnable runnable) { - try (ProgressTimer timer = timeEval()) { runnable.run(); } @@ -107,7 +97,6 @@ public void timeEval(Runnable runnable) { * @param runnable */ public void timeWrite(Runnable runnable) { - try (ProgressTimer timer = timeWrite()) { runnable.run(); } @@ -121,10 +110,6 @@ public T buildLI(IdMapping nodes, LongToIntFunction fun) { return build(nodes, value -> (long) fun.applyAsInt(value)); } - public T buildIL(IdMapping nodes, IntToLongFunction fun) { - return build(nodes, value -> fun.applyAsLong((int) value)); - } - /** * build result * @@ -135,18 +120,18 @@ public T buildIL(IdMapping nodes, IntToLongFunction fun) { public T build(IdMapping nodes, LongFunction fun) { final Histogram histogram = new Histogram(2); - final LongLongMap communityMap = new LongLongScatterMap(); + final LongLongMap communitySizeMap = new LongLongScatterMap(); final long nodeCount = nodes.nodeCount(); final ProgressTimer timer = ProgressTimer.start(); for (int i = 0; i < nodeCount; i++) { // map to community id - final long r = fun.apply(i); + final long cId = fun.apply(i); // aggregate community size - communityMap.addTo(r, 1); + communitySizeMap.addTo(cId, 1); // fill histogram - histogram.recordValue(r); + histogram.recordValue(cId); } - final List top3Communities = top3(nodes, communityMap); + final List top3Communities = top3(nodes, communitySizeMap); timer.stop(); return build(loadDuration, @@ -154,8 +139,8 @@ public T build(IdMapping nodes, LongFunction fun) { writeDuration, timer.getDuration(), nodeCount, - communityMap.size(), - communityMap, + communitySizeMap.size(), + communitySizeMap, histogram, top3Communities); } @@ -173,9 +158,10 @@ protected abstract T build( private List top3(IdMapping idMapping, LongLongMap assignment) { // index of top 3 biggest communities - final Long[] t3idx = new Long[]{-1L, -1L, -1L}; + final long[] t3idx = new long[]{-1L, -1L, -1L}; // size of the top 3 communities final long[] top3 = new long[]{-1L, -1L, -1L}; + for (LongLongCursor cursor : assignment) { if (cursor.value > top3[0]) { top3[2] = top3[1]; @@ -194,6 +180,16 @@ private List top3(IdMapping idMapping, LongLongMap assignment) { t3idx[2] = cursor.key; } } - return Arrays.asList(t3idx); + return new LinkedList() {{ + if (t3idx[0] != -1) { + add(idMapping.toOriginalNodeId(((int) t3idx[0]))); + } + if (t3idx[1] != -1) { + add(idMapping.toOriginalNodeId(((int) t3idx[1]))); + } + if (t3idx[2] != -1) { + add(idMapping.toOriginalNodeId(((int) t3idx[2]))); + } + }}; } } From 4764c993507564f589ad5d7283ae4684e0660025 Mon Sep 17 00:00:00 2001 From: Martin Knobloch Date: Tue, 29 Jan 2019 14:25:07 +0100 Subject: [PATCH 15/26] fix mapping --- .../neo4j/graphalgo/LabelPropagationProc.java | 2 +- .../java/org/neo4j/graphalgo/LouvainProc.java | 2 +- .../org/neo4j/graphalgo/MSColoringProc.java | 2 +- .../StronglyConnectedComponentsProc.java | 10 +++---- .../org/neo4j/graphalgo/TriangleProc.java | 6 ++-- .../graphalgo/impl/UnionFindProcExec.java | 4 +-- .../AbstractCommunityResultBuilder.java | 30 +++++-------------- 7 files changed, 21 insertions(+), 35 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java b/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java index df2dcdc18..c48b1e753 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java @@ -113,7 +113,7 @@ public Stream labelPropagation( write(concurrency, partitionProperty, graph, labels, stats); } - return Stream.of(stats.build(graph, l -> (long) labels[(int) l])); + return Stream.of(stats.build(graph.nodeCount(), l -> (long) labels[(int) l])); } @Procedure(value = "algo.labelPropagation.stream") diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index 5bbe23b31..0936d792d 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -110,7 +110,7 @@ public Stream louvain( } final int[] communityIds = louvain.getCommunityIds(); - return Stream.of(builder.build(graph, n -> (long) communityIds[(int) n])); + return Stream.of(builder.build(graph.nodeCount(), n -> (long) communityIds[(int) n])); } @Procedure(value = "algo.louvain.stream") diff --git a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java index 86273f72d..7a7999316 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java @@ -88,7 +88,7 @@ public Stream unionFind( write(graph, struct, configuration)); } - return Stream.of(builder.build(graph, n -> (long) struct.get((int) n))); + return Stream.of(builder.build(graph.nodeCount(), n -> (long) struct.get((int) n))); } @Procedure(value = "algo.unionFind.mscoloring.stream") diff --git a/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java b/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java index 8d4520fcd..10367e7fb 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java @@ -137,7 +137,7 @@ public Stream sccTarjan( ); }); } - return Stream.of(builder.build(graph, l -> (long) connectedComponents[((int) l)])); + return Stream.of(builder.build(graph.nodeCount(), l -> (long) connectedComponents[((int) l)])); } // algo.scc.tunedTarjan @@ -186,7 +186,7 @@ public Stream sccTunedTarjan( } final int[] connectedComponents = tarjan.getConnectedComponents(); - return Stream.of(builder.build(graph, l -> (long) connectedComponents[((int) l)])); + return Stream.of(builder.build(graph.nodeCount(), l -> (long) connectedComponents[((int) l)])); } // algo.scc.tunedTarjan.stream @@ -256,11 +256,11 @@ public Stream sccIterativeTarjan( if (graph instanceof HugeGraph) { final HugeLongArray connectedComponents = tarjan.getConnectedComponents(); - return Stream.of(builder.build(graph, connectedComponents::get)); + return Stream.of(builder.build(graph.nodeCount(), connectedComponents::get)); } final int[] connectedComponents = tarjan.getConnectedComponents(); tarjan.release(); - return Stream.of(builder.build(graph, l -> (long) connectedComponents[((int) l)])); + return Stream.of(builder.build(graph.nodeCount(), l -> (long) connectedComponents[((int) l)])); } private void write(ProcedureConfiguration configuration, Graph graph, TerminationFlag terminationFlag, SCCAlgorithm tarjan) { @@ -379,7 +379,7 @@ public Stream multistep( )); } - return Stream.of(builder.build(graph, l -> (long) connectedComponents[((int) l)])); + return Stream.of(builder.build(graph.nodeCount(), l -> (long) connectedComponents[((int) l)])); } // algo.scc.multistep.stream diff --git a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java index 8dc9f5dfb..b453ea30c 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java @@ -236,10 +236,10 @@ private Stream buildResult(TriangleCountResultBuilder builder, Graph gra if (algorithm instanceof IntersectingTriangleCount) { final PagedAtomicIntegerArray triangles = ((IntersectingTriangleCount) algorithm).getTriangles(); - return Stream.of(builder.buildLI(graph, triangles::get)); + return Stream.of(builder.buildLI(graph.nodeCount(), triangles::get)); } else if (algorithm instanceof TriangleCountQueue){ final AtomicIntegerArray triangles = ((TriangleCountQueue) algorithm).getTriangles(); - return Stream.of(builder.buildII(graph, triangles::get)); + return Stream.of(builder.buildII(graph.nodeCount(), triangles::get)); } throw new UnsupportedOperationException("unknown algorithm"); } @@ -384,7 +384,7 @@ public Stream triangleCountExp3( .withTriangleCount(triangleCount.getTriangleCount()); final AtomicIntegerArray triangles = triangleCount.getTriangles(); - return Stream.of(builder.buildII(graph, triangles::get)); + return Stream.of(builder.buildII(graph.nodeCount(), triangles::get)); } diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java index ba5e3d670..9328d6653 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java @@ -88,9 +88,9 @@ public static Stream run( } if (dssResult.isHuge) { - return Stream.of(builder.build(graph, dssResult.hugeStruct::find)); + return Stream.of(builder.build(graph.nodeCount(), dssResult.hugeStruct::find)); } else { - return Stream.of(builder.build(graph, l -> (long) dssResult.struct.find((int) l))); + return Stream.of(builder.build(graph.nodeCount(), l -> (long) dssResult.struct.find((int) l))); } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java index 08153ec25..69652937c 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java @@ -7,6 +7,7 @@ import org.neo4j.graphalgo.api.IdMapping; import org.neo4j.graphalgo.core.utils.ProgressTimer; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.function.IntFunction; @@ -102,26 +103,21 @@ public void timeWrite(Runnable runnable) { } } - public T buildII(IdMapping nodes, IntFunction fun) { + public T buildII(long nodes, IntFunction fun) { return build(nodes, value -> (long) fun.apply((int) value)); } - public T buildLI(IdMapping nodes, LongToIntFunction fun) { + public T buildLI(long nodes, LongToIntFunction fun) { return build(nodes, value -> (long) fun.applyAsInt(value)); } /** * build result - * - * @param nodes number of nodes in the graph - * @param fun nodeId to communityId mapping function - * @return result */ - public T build(IdMapping nodes, LongFunction fun) { + public T build(long nodeCount, LongFunction fun) { final Histogram histogram = new Histogram(2); final LongLongMap communitySizeMap = new LongLongScatterMap(); - final long nodeCount = nodes.nodeCount(); final ProgressTimer timer = ProgressTimer.start(); for (int i = 0; i < nodeCount; i++) { // map to community id @@ -131,7 +127,7 @@ public T build(IdMapping nodes, LongFunction fun) { // fill histogram histogram.recordValue(cId); } - final List top3Communities = top3(nodes, communitySizeMap); + final List top3Communities = top3(communitySizeMap); timer.stop(); return build(loadDuration, @@ -156,9 +152,9 @@ protected abstract T build( Histogram communityHistogram, List top3Communities); - private List top3(IdMapping idMapping, LongLongMap assignment) { + private List top3(LongLongMap assignment) { // index of top 3 biggest communities - final long[] t3idx = new long[]{-1L, -1L, -1L}; + final Long[] t3idx = new Long[]{-1L, -1L, -1L}; // size of the top 3 communities final long[] top3 = new long[]{-1L, -1L, -1L}; @@ -180,16 +176,6 @@ private List top3(IdMapping idMapping, LongLongMap assignment) { t3idx[2] = cursor.key; } } - return new LinkedList() {{ - if (t3idx[0] != -1) { - add(idMapping.toOriginalNodeId(((int) t3idx[0]))); - } - if (t3idx[1] != -1) { - add(idMapping.toOriginalNodeId(((int) t3idx[1]))); - } - if (t3idx[2] != -1) { - add(idMapping.toOriginalNodeId(((int) t3idx[2]))); - } - }}; + return Arrays.asList(t3idx); } } From 48b198008dcd3fda8da0a0659ce41136db58175f Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Thu, 31 Jan 2019 14:57:47 +0000 Subject: [PATCH 16/26] update percentiles --- .../java/org/neo4j/graphalgo/LouvainProc.java | 24 +++++++------ .../AbstractCommunityResultBuilder.java | 6 ++-- .../graphalgo/results/CommunityHistogram.java | 18 ++++++++++ .../results/DefaultCommunityResult.java | 24 +++++++------ .../results/LabelPropagationStats.java | 24 +++++++------ .../neo4j/graphalgo/results/SCCResult.java | 25 +++++++------ .../results/CommunityHistogramTest.java | 35 +++++++++++++++++++ 7 files changed, 112 insertions(+), 44 deletions(-) create mode 100644 algo/src/main/java/org/neo4j/graphalgo/results/CommunityHistogram.java create mode 100644 algo/src/test/java/org/neo4j/graphalgo/results/CommunityHistogramTest.java diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index 0936d792d..72a44a7f6 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -200,6 +200,7 @@ public static class LouvainResult { -1, -1, -1, + -1, Collections.emptyList(), 0 ); @@ -210,6 +211,7 @@ public static class LouvainResult { public final long writeMillis; public final long nodes; public final long communityCount; + public final long p100; public final long p99; public final long p95; public final long p90; @@ -222,13 +224,14 @@ public static class LouvainResult { public final List top3; public final long iterations; - public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations) { + public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.postProcessingMillis = postProcessingMillis; this.writeMillis = writeMillis; this.nodes = nodes; this.communityCount = communityCount; + this.p100 = p100; this.p99 = p99; this.p95 = p95; this.p90 = p90; @@ -261,15 +264,16 @@ protected LouvainResult build(long loadMillis, long computeMillis, long writeMil writeMillis, nodeCount, communityCount, - communityHistogram.getValueAtPercentile(.99), - communityHistogram.getValueAtPercentile(.95), - communityHistogram.getValueAtPercentile(.9), - communityHistogram.getValueAtPercentile(.75), - communityHistogram.getValueAtPercentile(.5), - communityHistogram.getValueAtPercentile(.25), - communityHistogram.getValueAtPercentile(.1), - communityHistogram.getValueAtPercentile(.05), - communityHistogram.getValueAtPercentile(.01), + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1), top3Communities, iterations ); diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java index 69652937c..7c09240b9 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java @@ -116,7 +116,6 @@ public T buildLI(long nodes, LongToIntFunction fun) { */ public T build(long nodeCount, LongFunction fun) { - final Histogram histogram = new Histogram(2); final LongLongMap communitySizeMap = new LongLongScatterMap(); final ProgressTimer timer = ProgressTimer.start(); for (int i = 0; i < nodeCount; i++) { @@ -124,9 +123,10 @@ public T build(long nodeCount, LongFunction fun) { final long cId = fun.apply(i); // aggregate community size communitySizeMap.addTo(cId, 1); - // fill histogram - histogram.recordValue(cId); } + + Histogram histogram = CommunityHistogram.buildFrom(communitySizeMap); + final List top3Communities = top3(communitySizeMap); timer.stop(); diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityHistogram.java b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityHistogram.java new file mode 100644 index 000000000..04a67b8d2 --- /dev/null +++ b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityHistogram.java @@ -0,0 +1,18 @@ +package org.neo4j.graphalgo.results; + +import com.carrotsearch.hppc.LongLongMap; +import com.carrotsearch.hppc.cursors.LongLongCursor; +import org.HdrHistogram.Histogram; + +public class CommunityHistogram { + + public static Histogram buildFrom(LongLongMap communitySizeMap) { + final Histogram histogram = new Histogram(2); + + for (LongLongCursor cursor : communitySizeMap) { + histogram.recordValue(cursor.value); + } + + return histogram; + } +} diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java index 3b8883011..323e609c3 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java @@ -12,7 +12,7 @@ public class DefaultCommunityResult { public static final DefaultCommunityResult EMPTY = new DefaultCommunityResult( - 0, 0,0,0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, Collections.emptyList() + 0, 0,0,0, 0, 0, -1,-1, -1, -1, -1, -1, -1, -1, -1, -1, Collections.emptyList() ); public final long loadMillis; @@ -21,6 +21,7 @@ public class DefaultCommunityResult { public final long writeMillis; public final long nodes; public final long communityCount; + public final long p100; public final long p99; public final long p95; public final long p90; @@ -38,6 +39,7 @@ public DefaultCommunityResult(long loadMillis, long writeMillis, long nodes, long communityCount, + long p100, long p99, long p95, long p90, @@ -54,6 +56,7 @@ public DefaultCommunityResult(long loadMillis, this.writeMillis = writeMillis; this.nodes = nodes; this.communityCount = communityCount; + this.p100 = p100; this.p99 = p99; this.p95 = p95; this.p90 = p90; @@ -77,15 +80,16 @@ protected DefaultCommunityResult build(long loadMillis, long computeMillis, long writeMillis, nodeCount, communityCount, - communityHistogram.getValueAtPercentile(.99), - communityHistogram.getValueAtPercentile(.95), - communityHistogram.getValueAtPercentile(.9), - communityHistogram.getValueAtPercentile(.75), - communityHistogram.getValueAtPercentile(.5), - communityHistogram.getValueAtPercentile(.25), - communityHistogram.getValueAtPercentile(.1), - communityHistogram.getValueAtPercentile(.05), - communityHistogram.getValueAtPercentile(.01), + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1), top3Communities); } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java index db8762af5..9fc8409f4 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java @@ -42,6 +42,7 @@ public class LabelPropagationStats { -1, -1, -1, + -1, Collections.emptyList(), 0, false, @@ -55,6 +56,7 @@ public class LabelPropagationStats { public final long writeMillis; public final long nodes; public final long communityCount; + public final long p100; public final long p99; public final long p95; public final long p90; @@ -69,13 +71,14 @@ public class LabelPropagationStats { public final boolean write, didConverge; public final String weightProperty, partitionProperty; - public LabelPropagationStats(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations, boolean write, boolean didConverge, String weightProperty, String partitionProperty) { + public LabelPropagationStats(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations, boolean write, boolean didConverge, String weightProperty, String partitionProperty) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.postProcessingMillis = postProcessingMillis; this.writeMillis = writeMillis; this.nodes = nodes; this.communityCount = communityCount; + this.p100 = p100; this.p99 = p99; this.p95 = p95; this.p90 = p90; @@ -136,15 +139,16 @@ protected LabelPropagationStats build(long loadMillis, long computeMillis, long postProcessingMillis, nodeCount, communityCount, - communityHistogram.getValueAtPercentile(.99), - communityHistogram.getValueAtPercentile(.95), - communityHistogram.getValueAtPercentile(.9), - communityHistogram.getValueAtPercentile(.75), - communityHistogram.getValueAtPercentile(.5), - communityHistogram.getValueAtPercentile(.25), - communityHistogram.getValueAtPercentile(.1), - communityHistogram.getValueAtPercentile(.05), - communityHistogram.getValueAtPercentile(.01), + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1), top3Communities, iterations, write, diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java index 3bb99f417..a4c37cc41 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java @@ -30,7 +30,7 @@ public class SCCResult { public static SCCResult EMPTY = new SCCResult( - 0, 0, 0, 0,0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, Collections.emptyList(), 0 + 0, 0, 0, 0,0, 0, -1,-1, -1, -1, -1, -1, -1, -1, -1, -1, Collections.emptyList(), 0 ); public final long loadMillis; @@ -40,6 +40,7 @@ public class SCCResult { public final long nodes; public final long communityCount; public final long setCount; + public final long p100; public final long p99; public final long p95; public final long p90; @@ -52,13 +53,14 @@ public class SCCResult { public final List top3; public final long iterations; - public SCCResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations) { + public SCCResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.postProcessingMillis = postProcessingMillis; this.writeMillis = writeMillis; this.nodes = nodes; this.setCount = this.communityCount = communityCount; + this.p100 = p100; this.p99 = p99; this.p95 = p95; this.p90 = p90; @@ -94,15 +96,16 @@ protected SCCResult build(long loadMillis, long computeMillis, long writeMillis, postProcessingMillis, nodeCount, communityCount, - communityHistogram.getValueAtPercentile(.99), - communityHistogram.getValueAtPercentile(.95), - communityHistogram.getValueAtPercentile(.9), - communityHistogram.getValueAtPercentile(.75), - communityHistogram.getValueAtPercentile(.5), - communityHistogram.getValueAtPercentile(.25), - communityHistogram.getValueAtPercentile(.1), - communityHistogram.getValueAtPercentile(.05), - communityHistogram.getValueAtPercentile(.01), + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1), top3Communities, iterations ); diff --git a/algo/src/test/java/org/neo4j/graphalgo/results/CommunityHistogramTest.java b/algo/src/test/java/org/neo4j/graphalgo/results/CommunityHistogramTest.java new file mode 100644 index 000000000..d64e41797 --- /dev/null +++ b/algo/src/test/java/org/neo4j/graphalgo/results/CommunityHistogramTest.java @@ -0,0 +1,35 @@ +package org.neo4j.graphalgo.results; + +import com.carrotsearch.hppc.LongLongMap; +import com.carrotsearch.hppc.LongLongScatterMap; +import org.HdrHistogram.Histogram; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class CommunityHistogramTest { + @Test + public void oneCommunity() { + final LongLongMap communitySizeMap = new LongLongScatterMap(); + communitySizeMap.addTo(1, 4); + + Histogram histogram = CommunityHistogram.buildFrom(communitySizeMap); + + assertEquals(4.0, histogram.getValueAtPercentile(100D), 0.01); + } + + @Test + public void multipleCommunities() { + final LongLongMap communitySizeMap = new LongLongScatterMap(); + communitySizeMap.addTo(1, 4); + communitySizeMap.addTo(2, 10); + communitySizeMap.addTo(3, 9); + communitySizeMap.addTo(4, 8); + communitySizeMap.addTo(5, 7); + + Histogram histogram = CommunityHistogram.buildFrom(communitySizeMap); + + assertEquals(10.0, histogram.getValueAtPercentile(100D), 0.01); + assertEquals(8.0, histogram.getValueAtPercentile(50D), 0.01); + } +} \ No newline at end of file From c468bafdd3b1789ff6ce44dcbf3a3277ebabfc1a Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Thu, 31 Jan 2019 15:06:24 +0000 Subject: [PATCH 17/26] fix triangle count percentiles --- .../org/neo4j/graphalgo/TriangleProc.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java index b453ea30c..2873a8b8b 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java @@ -410,6 +410,7 @@ public static class Result { -1, -1, -1, + -1, Collections.emptyList(), .0); @@ -419,6 +420,7 @@ public static class Result { public final long writeMillis; public final long nodeCount; public final long triangleCount; + public final long p100; public final long p99; public final long p95; public final long p90; @@ -431,7 +433,7 @@ public static class Result { public final List top3; public final double averageClusteringCoefficient; - public Result(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodeCount, long triangleCount, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, double averageClusteringCoefficient) { + public Result(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodeCount, long triangleCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, double averageClusteringCoefficient) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.postProcessingMillis = postProcessingMillis; @@ -439,6 +441,7 @@ public Result(long loadMillis, long computeMillis, long postProcessingMillis, lo this.nodeCount = nodeCount; this.averageClusteringCoefficient = averageClusteringCoefficient; this.triangleCount = triangleCount; + this.p100 = p100; this.p99 = p99; this.p95 = p95; this.p90 = p90; @@ -478,15 +481,16 @@ protected Result build(long loadMillis, long computeMillis, long writeMillis, lo postProcessingMillis, nodeCount, triangleCount, - communityHistogram.getValueAtPercentile(.99), - communityHistogram.getValueAtPercentile(.95), - communityHistogram.getValueAtPercentile(.9), - communityHistogram.getValueAtPercentile(.75), - communityHistogram.getValueAtPercentile(.5), - communityHistogram.getValueAtPercentile(.25), - communityHistogram.getValueAtPercentile(.1), - communityHistogram.getValueAtPercentile(.05), - communityHistogram.getValueAtPercentile(.01), + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1), top3Communities, avergaeClusteringCoefficient ); From b2af041dadff11600268b65a28d8ac33401ed777 Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Thu, 31 Jan 2019 15:25:12 +0000 Subject: [PATCH 18/26] don't think we need top3 --- .../java/org/neo4j/graphalgo/LouvainProc.java | 10 +---- .../org/neo4j/graphalgo/TriangleProc.java | 9 +---- .../AbstractCommunityResultBuilder.java | 38 ++----------------- .../results/DefaultCommunityResult.java | 12 ++---- .../results/LabelPropagationStats.java | 9 +---- .../neo4j/graphalgo/results/SCCResult.java | 12 ++---- 6 files changed, 17 insertions(+), 73 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index 72a44a7f6..9ef5b67ee 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -30,13 +30,11 @@ import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.impl.louvain.*; import org.neo4j.graphalgo.results.AbstractCommunityResultBuilder; -import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; import org.neo4j.procedure.*; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -201,7 +199,6 @@ public static class LouvainResult { -1, -1, -1, - Collections.emptyList(), 0 ); @@ -221,10 +218,9 @@ public static class LouvainResult { public final long p10; public final long p05; public final long p01; - public final List top3; public final long iterations; - public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations) { + public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.postProcessingMillis = postProcessingMillis; @@ -241,7 +237,6 @@ public LouvainResult(long loadMillis, long computeMillis, long postProcessingMil this.p10 = p10; this.p05 = p05; this.p01 = p01; - this.top3 = top3; this.iterations = iterations; } } @@ -256,7 +251,7 @@ public Builder withIterations(long iterations) { } @Override - protected LouvainResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, List top3Communities) { + protected LouvainResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { return new LouvainResult( loadMillis, computeMillis, @@ -274,7 +269,6 @@ protected LouvainResult build(long loadMillis, long computeMillis, long writeMil communityHistogram.getValueAtPercentile(10), communityHistogram.getValueAtPercentile(5), communityHistogram.getValueAtPercentile(1), - top3Communities, iterations ); } diff --git a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java index 2873a8b8b..5d89eaca2 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java @@ -38,7 +38,6 @@ import org.neo4j.logging.Log; import org.neo4j.procedure.*; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -411,7 +410,6 @@ public static class Result { -1, -1, -1, - Collections.emptyList(), .0); public final long loadMillis; @@ -430,10 +428,9 @@ public static class Result { public final long p10; public final long p05; public final long p01; - public final List top3; public final double averageClusteringCoefficient; - public Result(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodeCount, long triangleCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, double averageClusteringCoefficient) { + public Result(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodeCount, long triangleCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, double averageClusteringCoefficient) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.postProcessingMillis = postProcessingMillis; @@ -451,7 +448,6 @@ public Result(long loadMillis, long computeMillis, long postProcessingMillis, lo this.p10 = p10; this.p05 = p05; this.p01 = p01; - this.top3 = top3; } } @@ -473,7 +469,7 @@ public TriangleCountResultBuilder withTriangleCount(long triangleCount) { // communityCount is not used here @Override - protected Result build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, List top3Communities) { + protected Result build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { return new Result( loadMillis, computeMillis, @@ -491,7 +487,6 @@ protected Result build(long loadMillis, long computeMillis, long writeMillis, lo communityHistogram.getValueAtPercentile(10), communityHistogram.getValueAtPercentile(5), communityHistogram.getValueAtPercentile(1), - top3Communities, avergaeClusteringCoefficient ); } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java index 7c09240b9..f7241eaac 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java @@ -4,11 +4,9 @@ import com.carrotsearch.hppc.LongLongScatterMap; import com.carrotsearch.hppc.cursors.LongLongCursor; import org.HdrHistogram.Histogram; -import org.neo4j.graphalgo.api.IdMapping; import org.neo4j.graphalgo.core.utils.ProgressTimer; import java.util.Arrays; -import java.util.LinkedList; import java.util.List; import java.util.function.IntFunction; import java.util.function.LongFunction; @@ -127,7 +125,6 @@ public T build(long nodeCount, LongFunction fun) { Histogram histogram = CommunityHistogram.buildFrom(communitySizeMap); - final List top3Communities = top3(communitySizeMap); timer.stop(); return build(loadDuration, @@ -137,8 +134,8 @@ public T build(long nodeCount, LongFunction fun) { nodeCount, communitySizeMap.size(), communitySizeMap, - histogram, - top3Communities); + histogram + ); } protected abstract T build( @@ -149,33 +146,6 @@ protected abstract T build( long nodeCount, long communityCount, LongLongMap communitySizeMap, - Histogram communityHistogram, - List top3Communities); - - private List top3(LongLongMap assignment) { - // index of top 3 biggest communities - final Long[] t3idx = new Long[]{-1L, -1L, -1L}; - // size of the top 3 communities - final long[] top3 = new long[]{-1L, -1L, -1L}; - - for (LongLongCursor cursor : assignment) { - if (cursor.value > top3[0]) { - top3[2] = top3[1]; - top3[1] = top3[0]; - top3[0] = cursor.value; - t3idx[2] = t3idx[1]; - t3idx[1] = t3idx[0]; - t3idx[0] = cursor.key; - } else if (cursor.value > top3[1]) { - top3[2] = top3[1]; - top3[1] = cursor.value; - t3idx[2] = t3idx[1]; - t3idx[1] = cursor.key; - } else if (cursor.value > top3[2]) { - top3[2] = cursor.value; - t3idx[2] = cursor.key; - } - } - return Arrays.asList(t3idx); - } + Histogram communityHistogram); + } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java index 323e609c3..4eb0e8a17 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java @@ -12,7 +12,7 @@ public class DefaultCommunityResult { public static final DefaultCommunityResult EMPTY = new DefaultCommunityResult( - 0, 0,0,0, 0, 0, -1,-1, -1, -1, -1, -1, -1, -1, -1, -1, Collections.emptyList() + 0, 0,0,0, 0, 0, -1,-1, -1, -1, -1, -1, -1, -1, -1, -1 ); public final long loadMillis; @@ -31,7 +31,6 @@ public class DefaultCommunityResult { public final long p10; public final long p05; public final long p01; - public final List top3; public DefaultCommunityResult(long loadMillis, long computeMillis, @@ -48,8 +47,7 @@ public DefaultCommunityResult(long loadMillis, long p25, long p10, long p05, - long p01, - List top3) { + long p01) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.postProcessingMillis = postProcessingMillis; @@ -66,13 +64,12 @@ public DefaultCommunityResult(long loadMillis, this.p10 = p10; this.p05 = p05; this.p01 = p01; - this.top3 = top3; } public static class DefaultCommunityResultBuilder extends AbstractCommunityResultBuilder { @Override - protected DefaultCommunityResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, List top3Communities) { + protected DefaultCommunityResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { return new DefaultCommunityResult( loadMillis, evalDuration, @@ -89,8 +86,7 @@ protected DefaultCommunityResult build(long loadMillis, long computeMillis, long communityHistogram.getValueAtPercentile(25), communityHistogram.getValueAtPercentile(10), communityHistogram.getValueAtPercentile(5), - communityHistogram.getValueAtPercentile(1), - top3Communities); + communityHistogram.getValueAtPercentile(1)); } } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java index 9fc8409f4..992378fb9 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java @@ -21,7 +21,6 @@ import com.carrotsearch.hppc.LongLongMap; import org.HdrHistogram.Histogram; -import java.util.Collections; import java.util.List; public class LabelPropagationStats { @@ -43,7 +42,6 @@ public class LabelPropagationStats { -1, -1, -1, - Collections.emptyList(), 0, false, false, @@ -66,12 +64,11 @@ public class LabelPropagationStats { public final long p10; public final long p05; public final long p01; - public final List top3; public final long iterations; public final boolean write, didConverge; public final String weightProperty, partitionProperty; - public LabelPropagationStats(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations, boolean write, boolean didConverge, String weightProperty, String partitionProperty) { + public LabelPropagationStats(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations, boolean write, boolean didConverge, String weightProperty, String partitionProperty) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.postProcessingMillis = postProcessingMillis; @@ -88,7 +85,6 @@ public LabelPropagationStats(long loadMillis, long computeMillis, long postProce this.p10 = p10; this.p05 = p05; this.p01 = p01; - this.top3 = top3; this.iterations = iterations; this.write = write; this.didConverge = didConverge; @@ -131,7 +127,7 @@ public Builder partitionProperty(final String partitionProperty) { } @Override - protected LabelPropagationStats build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, List top3Communities) { + protected LabelPropagationStats build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { return new LabelPropagationStats( loadMillis, computeMillis, @@ -149,7 +145,6 @@ protected LabelPropagationStats build(long loadMillis, long computeMillis, long communityHistogram.getValueAtPercentile(10), communityHistogram.getValueAtPercentile(5), communityHistogram.getValueAtPercentile(1), - top3Communities, iterations, write, didConverge, diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java index a4c37cc41..2adbc84f0 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java @@ -21,16 +21,13 @@ import com.carrotsearch.hppc.LongLongMap; import org.HdrHistogram.Histogram; -import java.util.Collections; -import java.util.List; - /** * @author mknblch */ public class SCCResult { public static SCCResult EMPTY = new SCCResult( - 0, 0, 0, 0,0, 0, -1,-1, -1, -1, -1, -1, -1, -1, -1, -1, Collections.emptyList(), 0 + 0, 0, 0, 0,0, 0, -1,-1, -1, -1, -1, -1, -1, -1, -1, -1, 0 ); public final long loadMillis; @@ -50,10 +47,9 @@ public class SCCResult { public final long p10; public final long p05; public final long p01; - public final List top3; public final long iterations; - public SCCResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, List top3, long iterations) { + public SCCResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.postProcessingMillis = postProcessingMillis; @@ -70,7 +66,6 @@ public SCCResult(long loadMillis, long computeMillis, long postProcessingMillis, this.p10 = p10; this.p05 = p05; this.p01 = p01; - this.top3 = top3; this.iterations = iterations; } @@ -88,7 +83,7 @@ public Builder withIterations(int iterations) { } @Override - protected SCCResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, List top3Communities) { + protected SCCResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { return new SCCResult( loadMillis, computeMillis, @@ -106,7 +101,6 @@ protected SCCResult build(long loadMillis, long computeMillis, long writeMillis, communityHistogram.getValueAtPercentile(10), communityHistogram.getValueAtPercentile(5), communityHistogram.getValueAtPercentile(1), - top3Communities, iterations ); } From 2b03e105fc57138f0803dbf1a9d4f53c68ebce47 Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Mon, 4 Feb 2019 14:15:11 +0000 Subject: [PATCH 19/26] add modularity back --- .../java/org/neo4j/graphalgo/LouvainProc.java | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index 9ef5b67ee..fd5399fc7 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -35,7 +35,7 @@ import org.neo4j.logging.Log; import org.neo4j.procedure.*; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -93,13 +93,14 @@ public Stream louvain( .withTerminationFlag(TerminationFlag.wrap(transaction)); // evaluation + int iterations = configuration.getIterations(10); try (ProgressTimer timer = builder.timeEval()) { if (configuration.getString(DEFAULT_CLUSTER_PROPERTY).isPresent()) { // use predefined clustering final WeightMapping communityMap = ((NodeProperties) graph).nodeProperties(CLUSTERING_IDENTIFIER); - louvain.compute(communityMap, configuration.getIterations(10), configuration.get("innerIterations", 10)); + louvain.compute(communityMap, iterations, configuration.get("innerIterations", 10)); } else { - louvain.compute(configuration.getIterations(10), configuration.get("innerIterations", 10)); + louvain.compute(iterations, configuration.get("innerIterations", 10)); } } @@ -107,6 +108,10 @@ public Stream louvain( builder.timeWrite(() -> write(graph, louvain.getDendrogram(), louvain.getCommunityIds(), configuration)); } + builder.withIterations(louvain.getLevel()); + builder.withModularities(louvain.getModularities() ); + builder.withFinalModularity(louvain.getFinalModularity()); + final int[] communityIds = louvain.getCommunityIds(); return Stream.of(builder.build(graph.nodeCount(), n -> (long) communityIds[(int) n])); } @@ -199,8 +204,8 @@ public static class LouvainResult { -1, -1, -1, - 0 - ); + 0, + new double[] {}, -1); public final long loadMillis; public final long computeMillis; @@ -219,8 +224,10 @@ public static class LouvainResult { public final long p05; public final long p01; public final long iterations; + public final List modularities; + public final double modularity; - public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations) { + public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations, double[] modularities, double finalModularity) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.postProcessingMillis = postProcessingMillis; @@ -238,12 +245,17 @@ public LouvainResult(long loadMillis, long computeMillis, long postProcessingMil this.p05 = p05; this.p01 = p01; this.iterations = iterations; + this.modularities = new ArrayList<>(modularities.length); + for (double mod : modularities) this.modularities.add(mod); + this.modularity = finalModularity; } } public static class Builder extends AbstractCommunityResultBuilder { private long iterations = -1; + private double[] modularities = new double[] {}; + private double finalModularity = -1; public Builder withIterations(long iterations) { this.iterations = iterations; @@ -269,9 +281,19 @@ protected LouvainResult build(long loadMillis, long computeMillis, long writeMil communityHistogram.getValueAtPercentile(10), communityHistogram.getValueAtPercentile(5), communityHistogram.getValueAtPercentile(1), - iterations + iterations, modularities, finalModularity ); } + + public Builder withModularities(double[] modularities) { + this.modularities = modularities; + return this; + } + + public Builder withFinalModularity(double finalModularity) { + this.finalModularity = finalModularity; + return null; + } } From b124fabe9a0a63051723b9f9da3ef3e9fcdab0b5 Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Mon, 4 Feb 2019 14:22:40 +0000 Subject: [PATCH 20/26] min and max size for SCC --- .../StronglyConnectedComponentsProc.java | 1 + .../org/neo4j/graphalgo/results/SCCResult.java | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java b/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java index 10367e7fb..b83881ff6 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java @@ -137,6 +137,7 @@ public Stream sccTarjan( ); }); } + return Stream.of(builder.build(graph.nodeCount(), l -> (long) connectedComponents[((int) l)])); } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java index 2adbc84f0..5588a9488 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java @@ -27,7 +27,7 @@ public class SCCResult { public static SCCResult EMPTY = new SCCResult( - 0, 0, 0, 0,0, 0, -1,-1, -1, -1, -1, -1, -1, -1, -1, -1, 0 + 0, 0, 0, 0,0, 0, -1,-1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 0, 0 ); public final long loadMillis; @@ -48,8 +48,10 @@ public class SCCResult { public final long p05; public final long p01; public final long iterations; + public final long minSetSize; + public final long maxSetSize; - public SCCResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations) { + public SCCResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations, long minSetSize, long maxSetSize) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; this.postProcessingMillis = postProcessingMillis; @@ -67,6 +69,8 @@ public SCCResult(long loadMillis, long computeMillis, long postProcessingMillis, this.p05 = p05; this.p01 = p01; this.iterations = iterations; + this.minSetSize = minSetSize; + this.maxSetSize = maxSetSize; } public static Builder builder() { @@ -74,14 +78,8 @@ public static Builder builder() { } public static final class Builder extends AbstractCommunityResultBuilder { - private int iterations = -1; - public Builder withIterations(int iterations) { - this.iterations = iterations; - return this; - } - @Override protected SCCResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { return new SCCResult( @@ -101,7 +99,9 @@ protected SCCResult build(long loadMillis, long computeMillis, long writeMillis, communityHistogram.getValueAtPercentile(10), communityHistogram.getValueAtPercentile(5), communityHistogram.getValueAtPercentile(1), - iterations + iterations, + communityHistogram.getMinNonZeroValue(), + communityHistogram.getMaxValue() ); } } From 9a4ce601d3f2f270512d7b6efa69254ad12b8633 Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Mon, 4 Feb 2019 14:25:39 +0000 Subject: [PATCH 21/26] remove top3 and bring back min/max set size --- .../StronglyConnectedComponentsProcIntegrationTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java index 157af4e01..546b8cb62 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java @@ -97,18 +97,20 @@ public static Collection data() { @Test public void testScc() throws Exception { - db.execute("CALL algo.scc('Node', 'TYPE', {write:true, graph:'"+graphImpl+"'}) YIELD loadMillis, computeMillis, writeMillis, setCount, top3") + db.execute("CALL algo.scc('Node', 'TYPE', {write:true, graph:'"+graphImpl+"'}) YIELD loadMillis, computeMillis, writeMillis, setCount, maxSetSize, minSetSize") .accept(row -> { System.out.println(row.getNumber("loadMillis").longValue()); System.out.println(row.getNumber("computeMillis").longValue()); System.out.println(row.getNumber("writeMillis").longValue()); System.out.println(row.getNumber("setCount").longValue()); - System.out.println(row.get("top3")); assertNotEquals(-1L, row.getNumber("computeMillis").longValue()); assertNotEquals(-1L, row.getNumber("writeMillis").longValue()); assertEquals(2, row.getNumber("setCount").longValue()); + assertEquals(2, row.getNumber("minSetSize").longValue()); + assertEquals(3, row.getNumber("maxSetSize").longValue()); + return true; }); } From 51b312e4ebd445838d6588dd31dfa825eef00c9f Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Mon, 4 Feb 2019 14:34:25 +0000 Subject: [PATCH 22/26] We need to keep around setCount in case someone relies on it --- .../org/neo4j/graphalgo/UnionFindProc.java | 2 +- .../org/neo4j/graphalgo/UnionFindProc2.java | 2 +- .../org/neo4j/graphalgo/UnionFindProc3.java | 2 +- .../org/neo4j/graphalgo/UnionFindProc4.java | 2 +- .../graphalgo/impl/UnionFindProcExec.java | 94 ++++++++++++++++++- .../algo/UnionFindProcIntegrationTest.java | 9 +- 6 files changed, 100 insertions(+), 11 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java index 93919d4f5..2770d2e29 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java @@ -52,7 +52,7 @@ public class UnionFindProc { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{weightProperty:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition'}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java index 74191c419..47e3ca4e9 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java @@ -52,7 +52,7 @@ public class UnionFindProc2 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition',concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java index 71ed14c08..d7d392a99 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java @@ -52,7 +52,7 @@ public class UnionFindProc3 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition', concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java index 2ad3a3b93..14cdf1a0f 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java @@ -52,7 +52,7 @@ public class UnionFindProc4 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition',concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java index 9328d6653..5fea95e09 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java @@ -18,6 +18,8 @@ */ package org.neo4j.graphalgo.impl; +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; import org.neo4j.graphalgo.api.Graph; import org.neo4j.graphalgo.core.GraphLoader; import org.neo4j.graphalgo.core.ProcedureConfiguration; @@ -29,7 +31,7 @@ import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.core.utils.paged.PagedDisjointSetStruct; import org.neo4j.graphalgo.core.write.Exporter; -import org.neo4j.graphalgo.results.DefaultCommunityResult; +import org.neo4j.graphalgo.results.AbstractCommunityResultBuilder; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -53,7 +55,7 @@ public final class UnionFindProcExec implements BiConsumer> private final UnionFindAlgo sequential; private final UnionFindAlgo parallel; - public static Stream run( + public static Stream run( Map config, String label, String relationship, @@ -65,7 +67,7 @@ public static Stream run( AllocationTracker tracker = AllocationTracker.create(); - final DefaultCommunityResult.DefaultCommunityResultBuilder builder = new DefaultCommunityResult.DefaultCommunityResultBuilder(); + final Builder builder = new Builder(); UnionFindProcExec uf = unionFind.get(); @@ -73,7 +75,7 @@ public static Stream run( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(DefaultCommunityResult.EMPTY); + return Stream.of(UnionFindResult.EMPTY); } final DSSResult dssResult = uf.evaluate( @@ -94,6 +96,90 @@ public static Stream run( } } + public static class UnionFindResult { + + public static final UnionFindProcExec.UnionFindResult EMPTY = new UnionFindProcExec.UnionFindResult( + 0, + 0, + 0, + 0, + 0, + 0, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1 + ); + + public final long loadMillis; + public final long computeMillis; + public final long postProcessingMillis; + public final long writeMillis; + public final long nodes; + public final long communityCount; + public final long setCount; + public final long p100; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; + + public UnionFindResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01) { + this.loadMillis = loadMillis; + this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; + this.writeMillis = writeMillis; + this.nodes = nodes; + this.communityCount = communityCount; + this.p100 = p100; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; + this.setCount = communityCount; + } + } + + public static class Builder extends AbstractCommunityResultBuilder { + @Override + protected UnionFindResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { + return new UnionFindResult( + loadMillis, + computeMillis, + postProcessingMillis, + writeMillis, + nodeCount, + communityCount, + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1) + ); + } + } + public static Stream stream( Map config, String label, diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java index 5ba32d3b9..2ee3cee0d 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java @@ -110,29 +110,32 @@ public static Collection data() { @Test public void testUnionFind() throws Exception { - db.execute("CALL algo.unionFind('', '',{graph:'"+graphImpl+"'}) YIELD communityCount") + db.execute("CALL algo.unionFind('', '',{graph:'"+graphImpl+"'}) YIELD setCount, communityCount") .accept((Result.ResultVisitor) row -> { assertEquals(3L, row.getNumber("communityCount")); + assertEquals(3L, row.getNumber("setCount")); return true; }); } @Test public void testUnionFindWithLabel() throws Exception { - db.execute("CALL algo.unionFind('Label', '',{graph:'"+graphImpl+"'}) YIELD communityCount") + db.execute("CALL algo.unionFind('Label', '',{graph:'"+graphImpl+"'}) YIELD setCount, communityCount") .accept((Result.ResultVisitor) row -> { assertEquals(1L, row.getNumber("communityCount")); + assertEquals(1L, row.getNumber("setCount")); return true; }); } @Test public void testUnionFindWriteBack() throws Exception { - db.execute("CALL algo.unionFind('', 'TYPE', {write:true,graph:'"+graphImpl+"'}) YIELD communityCount, writeMillis, nodes") + db.execute("CALL algo.unionFind('', 'TYPE', {write:true,graph:'"+graphImpl+"'}) YIELD setCount, communityCount, writeMillis, nodes") .accept((Result.ResultVisitor) row -> { assertNotEquals(-1L, row.getNumber("writeMillis")); assertNotEquals(-1L, row.getNumber("nodes")); assertEquals(3L, row.getNumber("communityCount")); + assertEquals(3L, row.getNumber("setCount")); return false; }); } From 59184008fb93baf6e8d4a780fb84eefbff0c427c Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Mon, 4 Feb 2019 14:38:05 +0000 Subject: [PATCH 23/26] remove top3 from test --- .../neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java index 6a53c2e5e..fd01ee65c 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java @@ -97,7 +97,7 @@ public void clearCommunities() { @Test public void test() { final String cypher = "CALL algo.louvain('', '', {concurrency:1}) " + - "YIELD nodes, communityCount, loadMillis, computeMillis, writeMillis, postProcessingMillis, top3, p99"; + "YIELD nodes, communityCount, loadMillis, computeMillis, writeMillis, postProcessingMillis, p99"; DB.execute(cypher).accept(row -> { final long nodes = row.getNumber("nodes").longValue(); @@ -109,7 +109,6 @@ public void test() { System.out.println("nodes = " + nodes); System.out.println("communityCount = " + communityCount); System.out.println("p99 = " + row.get("p99")); - System.out.println("top3 = " + row.get("top3")); assertEquals("invalid node count",9, nodes); assertEquals("wrong community count", 3, communityCount); From 0f8a3c446814ac3eb11500e6b5e444de7007ed85 Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Mon, 4 Feb 2019 14:40:05 +0000 Subject: [PATCH 24/26] we need setCount here too --- algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java index 7a7999316..029521338 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java @@ -26,6 +26,7 @@ import org.neo4j.graphalgo.core.write.Exporter; import org.neo4j.graphalgo.core.write.Translators; import org.neo4j.graphalgo.impl.MSColoring; +import org.neo4j.graphalgo.impl.UnionFindProcExec; import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -54,7 +55,7 @@ public class MSColoringProc { @Description("CALL algo.unionFind.mscoloring(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition', concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { @@ -63,7 +64,7 @@ public Stream unionFind( .overrideNodeLabelOrQuery(label) .overrideRelationshipTypeOrQuery(relationship); - final DefaultCommunityResult.DefaultCommunityResultBuilder builder = new DefaultCommunityResult.DefaultCommunityResultBuilder(); + final UnionFindProcExec.Builder builder = new UnionFindProcExec.Builder(); // loading final Graph graph; @@ -73,7 +74,7 @@ public Stream unionFind( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(DefaultCommunityResult.EMPTY); + return Stream.of(UnionFindProcExec.UnionFindResult.EMPTY); } // evaluation From 54c9bd3e888781e8cf4f77d932a7a5daa9b1cb13 Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Mon, 4 Feb 2019 14:40:13 +0000 Subject: [PATCH 25/26] unused exports --- algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java | 1 - 1 file changed, 1 deletion(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java index 029521338..544084e3e 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java @@ -27,7 +27,6 @@ import org.neo4j.graphalgo.core.write.Translators; import org.neo4j.graphalgo.impl.MSColoring; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; From a0a68f720ceaf492275014bbc27d85b2f6d660ae Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Mon, 4 Feb 2019 14:47:37 +0000 Subject: [PATCH 26/26] typo --- .../main/java/org/neo4j/graphalgo/TriangleProc.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java index 5d89eaca2..1adb52aed 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java @@ -38,7 +38,6 @@ import org.neo4j.logging.Log; import org.neo4j.procedure.*; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ForkJoinPool; @@ -225,7 +224,7 @@ public Stream triangleCountQueue( } - builder.withAvergaeClusteringCoefficient(triangleCount.getAverageCoefficient()) + builder.withAverageClusteringCoefficient(triangleCount.getAverageCoefficient()) .withTriangleCount(triangleCount.getTriangleCount()); return buildResult(builder, graph, triangleCount); @@ -379,7 +378,7 @@ public Stream triangleCountExp3( } } - builder.withAvergaeClusteringCoefficient(triangleCount.getAverageClusteringCoefficient()) + builder.withAverageClusteringCoefficient(triangleCount.getAverageClusteringCoefficient()) .withTriangleCount(triangleCount.getTriangleCount()); final AtomicIntegerArray triangles = triangleCount.getTriangles(); @@ -453,11 +452,11 @@ public Result(long loadMillis, long computeMillis, long postProcessingMillis, lo public class TriangleCountResultBuilder extends AbstractCommunityResultBuilder { - private double avergaeClusteringCoefficient = .0; + private double averageClusteringCoefficient = .0; private long triangleCount = 0; - public TriangleCountResultBuilder withAvergaeClusteringCoefficient(double avergaeClusteringCoefficient) { - this.avergaeClusteringCoefficient = avergaeClusteringCoefficient; + public TriangleCountResultBuilder withAverageClusteringCoefficient(double averageClusteringCoefficient) { + this.averageClusteringCoefficient = averageClusteringCoefficient; return this; } @@ -487,7 +486,7 @@ protected Result build(long loadMillis, long computeMillis, long writeMillis, lo communityHistogram.getValueAtPercentile(10), communityHistogram.getValueAtPercentile(5), communityHistogram.getValueAtPercentile(1), - avergaeClusteringCoefficient + averageClusteringCoefficient ); } }