diff --git a/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java b/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java index 0faf44f11..c48b1e753 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java @@ -105,15 +105,15 @@ public Stream labelPropagation( if(graph.nodeCount() == 0) { graph.release(); - return Stream.of(stats.build()); + return Stream.of(LabelPropagationStats.EMPTY); } - int[] labels = compute(direction, iterations, batchSize, concurrency, graph, stats); + final int[] labels = compute(direction, iterations, batchSize, concurrency, graph, stats); if (configuration.isWriteFlag(DEFAULT_WRITE) && partitionProperty != null) { write(concurrency, partitionProperty, graph, labels, stats); } - return Stream.of(stats.build()); + return Stream.of(stats.build(graph.nodeCount(), l -> (long) labels[(int) l])); } @Procedure(value = "algo.labelPropagation.stream") @@ -205,7 +205,6 @@ private int[] compute( stats.iterations(labelPropagation.ranIterations()); stats.didConverge(labelPropagation.didConverge()); - stats.nodes(result.length); labelPropagation.release(); graph.release(); diff --git a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java index c79df1a2e..fd5399fc7 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java @@ -18,25 +18,25 @@ */ package org.neo4j.graphalgo; +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; import org.neo4j.graphalgo.api.*; import org.neo4j.graphalgo.core.GraphLoader; import org.neo4j.graphalgo.core.ProcedureConfiguration; -import org.neo4j.graphalgo.core.ProcedureConstants; -import org.neo4j.graphalgo.core.heavyweight.HeavyGraph; -import org.neo4j.graphalgo.core.heavyweight.HeavyGraphFactory; import org.neo4j.graphalgo.core.utils.Pools; import org.neo4j.graphalgo.core.utils.ProgressLogger; import org.neo4j.graphalgo.core.utils.ProgressTimer; import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.impl.louvain.*; -import org.neo4j.graphalgo.results.LouvainResult; +import org.neo4j.graphalgo.results.AbstractCommunityResultBuilder; import org.neo4j.kernel.api.KernelTransaction; -import org.neo4j.kernel.impl.store.PropertyType; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; import org.neo4j.procedure.*; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -75,18 +75,17 @@ public Stream louvain( .overrideNodeLabelOrQuery(label) .overrideRelationshipTypeOrQuery(relationship); - LouvainResult.Builder builder = LouvainResult.builder(); + + final Builder builder = new Builder(); final Graph graph; try (ProgressTimer timer = builder.timeLoad()) { graph = graph(label, relationship, configuration); } - builder.withNodeCount(graph.nodeCount()); - if(graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder.build()); + return Stream.of(LouvainResult.EMPTY); } final Louvain louvain = new Louvain(graph, Pools.DEFAULT, configuration.getConcurrency(), AllocationTracker.create()) @@ -94,25 +93,27 @@ public Stream louvain( .withTerminationFlag(TerminationFlag.wrap(transaction)); // evaluation + int iterations = configuration.getIterations(10); try (ProgressTimer timer = builder.timeEval()) { if (configuration.getString(DEFAULT_CLUSTER_PROPERTY).isPresent()) { // use predefined clustering final WeightMapping communityMap = ((NodeProperties) graph).nodeProperties(CLUSTERING_IDENTIFIER); - louvain.compute(communityMap, configuration.getIterations(10), configuration.get("innerIterations", 10)); + louvain.compute(communityMap, iterations, configuration.get("innerIterations", 10)); } else { - louvain.compute(configuration.getIterations(10), configuration.get("innerIterations", 10)); + louvain.compute(iterations, configuration.get("innerIterations", 10)); } - builder.withIterations(louvain.getLevel()) - .withCommunityCount(louvain.getCommunityCount()) - .withModularities(louvain.getModularities()) - .withFinalModularity(louvain.getFinalModularity()); } if (configuration.isWriteFlag()) { builder.timeWrite(() -> write(graph, louvain.getDendrogram(), louvain.getCommunityIds(), configuration)); } - return Stream.of(builder.build()); + builder.withIterations(louvain.getLevel()); + builder.withModularities(louvain.getModularities() ); + builder.withFinalModularity(louvain.getFinalModularity()); + + final int[] communityIds = louvain.getCommunityIds(); + return Stream.of(builder.build(graph.nodeCount(), n -> (long) communityIds[(int) n])); } @Procedure(value = "algo.louvain.stream") @@ -183,4 +184,117 @@ private void write(Graph graph, int[][] allCommunities, int[] finalCommunities, configuration.get(INTERMEDIATE_COMMUNITIES_WRITE_PROPERTY, "communities")) .export(allCommunities, finalCommunities, includeIntermediateCommunities); } + + public static class LouvainResult { + + public static final LouvainResult EMPTY = new LouvainResult( + 0, + 0, + 0, + 0, + 0, + 0, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + 0, + new double[] {}, -1); + + public final long loadMillis; + public final long computeMillis; + public final long postProcessingMillis; + public final long writeMillis; + public final long nodes; + public final long communityCount; + public final long p100; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; + public final long iterations; + public final List modularities; + public final double modularity; + + public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations, double[] modularities, double finalModularity) { + this.loadMillis = loadMillis; + this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; + this.writeMillis = writeMillis; + this.nodes = nodes; + this.communityCount = communityCount; + this.p100 = p100; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; + this.iterations = iterations; + this.modularities = new ArrayList<>(modularities.length); + for (double mod : modularities) this.modularities.add(mod); + this.modularity = finalModularity; + } + } + + public static class Builder extends AbstractCommunityResultBuilder { + + private long iterations = -1; + private double[] modularities = new double[] {}; + private double finalModularity = -1; + + public Builder withIterations(long iterations) { + this.iterations = iterations; + return this; + } + + @Override + protected LouvainResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { + return new LouvainResult( + loadMillis, + computeMillis, + postProcessingMillis, + writeMillis, + nodeCount, + communityCount, + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1), + iterations, modularities, finalModularity + ); + } + + public Builder withModularities(double[] modularities) { + this.modularities = modularities; + return this; + } + + public Builder withFinalModularity(double finalModularity) { + this.finalModularity = finalModularity; + return null; + } + } + + } diff --git a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java index 3b7bd6103..544084e3e 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/MSColoringProc.java @@ -26,7 +26,7 @@ import org.neo4j.graphalgo.core.write.Exporter; import org.neo4j.graphalgo.core.write.Translators; import org.neo4j.graphalgo.impl.MSColoring; -import org.neo4j.graphalgo.results.UnionFindResult; +import org.neo4j.graphalgo.impl.UnionFindProcExec; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -54,7 +54,7 @@ public class MSColoringProc { @Description("CALL algo.unionFind.mscoloring(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition', concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { @@ -63,7 +63,7 @@ public Stream unionFind( .overrideNodeLabelOrQuery(label) .overrideRelationshipTypeOrQuery(relationship); - UnionFindResult.Builder builder = UnionFindResult.builder(); + final UnionFindProcExec.Builder builder = new UnionFindProcExec.Builder(); // loading final Graph graph; @@ -73,7 +73,7 @@ public Stream unionFind( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder.build()); + return Stream.of(UnionFindProcExec.UnionFindResult.EMPTY); } // evaluation @@ -88,7 +88,7 @@ public Stream unionFind( write(graph, struct, configuration)); } - return Stream.of(builder.build()); + return Stream.of(builder.build(graph.nodeCount(), n -> (long) struct.get((int) n))); } @Procedure(value = "algo.unionFind.mscoloring.stream") diff --git a/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java b/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java index c464107f3..b83881ff6 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/StronglyConnectedComponentsProc.java @@ -1,31 +1,27 @@ /** * Copyright (c) 2017 "Neo4j, Inc." - * + *

* This file is part of Neo4j Graph Algorithms . - * + *

* Neo4j Graph Algorithms is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + *

* This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + *

* You should have received a copy of the GNU General Public License * along with this program. If not, see . */ package org.neo4j.graphalgo; -import com.carrotsearch.hppc.IntSet; -import com.carrotsearch.hppc.ObjectArrayList; -import com.carrotsearch.hppc.cursors.IntCursor; import org.neo4j.graphalgo.api.Graph; import org.neo4j.graphalgo.api.HugeGraph; import org.neo4j.graphalgo.core.GraphLoader; import org.neo4j.graphalgo.core.ProcedureConfiguration; -import org.neo4j.graphalgo.core.neo4jview.DirectIdMapping; import org.neo4j.graphalgo.core.utils.Pools; import org.neo4j.graphalgo.core.utils.ProgressLogger; import org.neo4j.graphalgo.core.utils.ProgressTimer; @@ -34,7 +30,7 @@ import org.neo4j.graphalgo.core.utils.paged.HugeLongArray; import org.neo4j.graphalgo.core.write.Exporter; import org.neo4j.graphalgo.core.write.Translators; -import org.neo4j.graphalgo.impl.*; +import org.neo4j.graphalgo.impl.ForwardBackwardScc; import org.neo4j.graphalgo.impl.multistepscc.MultistepSCC; import org.neo4j.graphalgo.impl.scc.SCCAlgorithm; import org.neo4j.graphalgo.impl.scc.SCCTarjan; @@ -46,8 +42,6 @@ import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; import org.neo4j.procedure.*; -import org.neo4j.values.storable.IntValue; -import org.neo4j.values.storable.Values; import java.util.Map; import java.util.stream.Stream; @@ -117,7 +111,7 @@ public Stream sccTarjan( loadTimer.stop(); if (graph.nodeCount() == 0) { - return Stream.of(builder.build()); + return Stream.of(SCCResult.EMPTY); } final TerminationFlag terminationFlag = TerminationFlag.wrap(transaction); @@ -125,38 +119,26 @@ public Stream sccTarjan( .withProgressLogger(ProgressLogger.wrap(log, "SCC(Tarjan)")) .withTerminationFlag(terminationFlag); - builder.timeEval(() -> { - tarjan.compute(); - builder.withMaxSetSize(tarjan.getMaxSetSize()) - .withMinSetSize(tarjan.getMinSetSize()) - .withSetCount(tarjan.getConnectedComponents().size()); - }); + builder.timeEval(tarjan::compute); + final int[] connectedComponents = tarjan.getConnectedComponents(); if (configuration.isWriteFlag()) { builder.timeWrite(() -> { - final ObjectArrayList connectedComponents = tarjan.getConnectedComponents(); + graph.release(); tarjan.release(); - Exporter.of(new DirectIdMapping(connectedComponents.size()), api) + Exporter.of(api, graph) .withLog(log) .parallel(Pools.DEFAULT, configuration.getConcurrency(), terminationFlag) .build() .write( configuration.get(CONFIG_WRITE_PROPERTY, CONFIG_CLUSTER), - propertyId -> (ops, id) -> { - final int setId = (int) (id); - IntValue property = Values.intValue(setId + 1); - for (final IntCursor iCursor : connectedComponents.get(setId)) { - ops.nodeSetProperty( - graph.toOriginalNodeId(iCursor.value), - propertyId, - property); - } - } + connectedComponents, + Translators.OPTIONAL_INT_ARRAY_TRANSLATOR ); }); } - return Stream.of(builder.build()); + return Stream.of(builder.build(graph.nodeCount(), l -> (long) connectedComponents[((int) l)])); } // algo.scc.tunedTarjan @@ -181,7 +163,7 @@ public Stream sccTunedTarjan( loadTimer.stop(); if (graph.nodeCount() == 0) { - return Stream.of(builder.build()); + return Stream.of(SCCResult.EMPTY); } final TerminationFlag terminationFlag = TerminationFlag.wrap(transaction); @@ -191,10 +173,6 @@ public Stream sccTunedTarjan( builder.timeEval(tarjan::compute); - builder.withMaxSetSize(tarjan.getMaxSetSize()) - .withMinSetSize(tarjan.getMinSetSize()) - .withSetCount(tarjan.getSetCount()); - if (configuration.isWriteFlag()) { builder.timeWrite(() -> Exporter .of(api, graph) @@ -208,7 +186,8 @@ public Stream sccTunedTarjan( )); } - return Stream.of(builder.build()); + final int[] connectedComponents = tarjan.getConnectedComponents(); + return Stream.of(builder.build(graph.nodeCount(), l -> (long) connectedComponents[((int) l)])); } // algo.scc.tunedTarjan.stream @@ -261,7 +240,7 @@ public Stream sccIterativeTarjan( loadTimer.stop(); if (graph.nodeCount() == 0) { - return Stream.of(builder.build()); + return Stream.of(SCCResult.EMPTY); } final AllocationTracker tracker = AllocationTracker.create(); @@ -272,15 +251,17 @@ public Stream sccIterativeTarjan( builder.timeEval(tarjan::compute); - builder.withSetCount(tarjan.getSetCount()) - .withMinSetSize(tarjan.getMinSetSize()) - .withMaxSetSize(tarjan.getMaxSetSize()); - if (configuration.isWriteFlag()) { - builder.timeWrite(() -> write(configuration, graph, terminationFlag, tarjan)); + builder.timeWrite(() -> write(configuration, graph, terminationFlag, tarjan)); } - return Stream.of(builder.build()); + if (graph instanceof HugeGraph) { + final HugeLongArray connectedComponents = tarjan.getConnectedComponents(); + return Stream.of(builder.build(graph.nodeCount(), connectedComponents::get)); + } + final int[] connectedComponents = tarjan.getConnectedComponents(); + tarjan.release(); + return Stream.of(builder.build(graph.nodeCount(), l -> (long) connectedComponents[((int) l)])); } private void write(ProcedureConfiguration configuration, Graph graph, TerminationFlag terminationFlag, SCCAlgorithm tarjan) { @@ -302,8 +283,6 @@ private void write(ProcedureConfiguration configuration, Graph graph, Terminatio } final int[] connectedComponents = tarjan.getConnectedComponents(); - graph.release(); - tarjan.release(); Exporter.of(api, graph) .withLog(log) .parallel(Pools.DEFAULT, configuration.getConcurrency(), terminationFlag) @@ -372,7 +351,7 @@ public Stream multistep( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder.build()); + return Stream.of(SCCResult.EMPTY); } final TerminationFlag terminationFlag = TerminationFlag.wrap(transaction); @@ -384,12 +363,9 @@ public Stream multistep( builder.timeEval(multistep::compute); - builder.withMaxSetSize(multistep.getMaxSetSize()) - .withMinSetSize(multistep.getMinSetSize()) - .withSetCount(multistep.getSetCount()); + final int[] connectedComponents = multistep.getConnectedComponents(); if (configuration.isWriteFlag()) { - final int[] connectedComponents = multistep.getConnectedComponents(); graph.release(); multistep.release(); builder.timeWrite(() -> Exporter @@ -404,7 +380,7 @@ public Stream multistep( )); } - return Stream.of(builder.build()); + return Stream.of(builder.build(graph.nodeCount(), l -> (long) connectedComponents[((int) l)])); } // algo.scc.multistep.stream @@ -417,23 +393,19 @@ public Stream multistepStream( @Name(value = "config", defaultValue = "{}") Map config) { ProcedureConfiguration configuration = ProcedureConfiguration.create(config); - Graph graph = new GraphLoader(api, Pools.DEFAULT) .init(log, label, relationship, configuration) .withoutRelationshipWeights() .load(configuration.getGraphImpl()); - if (graph.nodeCount() == 0) { graph.release(); return Stream.empty(); } - final MultistepSCC multistep = new MultistepSCC(graph, org.neo4j.graphalgo.core.utils.Pools.DEFAULT, configuration.getConcurrency(), configuration.getNumber("cutoff", 100_000).intValue()) .withProgressLogger(ProgressLogger.wrap(log, "SCC(MultiStep)")) .withTerminationFlag(TerminationFlag.wrap(transaction)); - multistep.compute(); graph.release(); return multistep.resultStream(); @@ -450,17 +422,14 @@ public Stream fwbwStream( @Name(value = "config", defaultValue = "{}") Map config) { ProcedureConfiguration configuration = ProcedureConfiguration.create(config); - Graph graph = new GraphLoader(api, Pools.DEFAULT) .init(log, label, relationship, configuration) .withoutRelationshipWeights() .load(configuration.getGraphImpl()); - if (graph.nodeCount() == 0) { graph.release(); return Stream.empty(); } - final ForwardBackwardScc algo = new ForwardBackwardScc(graph, Pools.DEFAULT, configuration.getConcurrency()) .withProgressLogger(ProgressLogger.wrap(log, "SCC(ForwardBackward)")) diff --git a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java index 4af94d276..1adb52aed 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/TriangleProc.java @@ -18,6 +18,8 @@ */ package org.neo4j.graphalgo; +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; import org.neo4j.graphalgo.api.Graph; import org.neo4j.graphalgo.api.HugeGraph; import org.neo4j.graphalgo.core.GraphLoader; @@ -30,7 +32,7 @@ import org.neo4j.graphalgo.core.write.Exporter; import org.neo4j.graphalgo.core.write.Translators; import org.neo4j.graphalgo.impl.triangle.*; -import org.neo4j.graphalgo.results.AbstractResultBuilder; +import org.neo4j.graphalgo.results.AbstractCommunityResultBuilder; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -221,11 +223,23 @@ public Stream triangleCountQueue( } } - builder.withNodeCount(graph.nodeCount()) - .withTriangleCount(triangleCount.getTriangleCount()) - .withAverageClusteringCoefficient(triangleCount.getAverageCoefficient()); - return Stream.of(builder.build()); + builder.withAverageClusteringCoefficient(triangleCount.getAverageCoefficient()) + .withTriangleCount(triangleCount.getTriangleCount()); + + return buildResult(builder, graph, triangleCount); + } + + private Stream buildResult(TriangleCountResultBuilder builder, Graph graph, TriangleCountAlgorithm algorithm) { + + if (algorithm instanceof IntersectingTriangleCount) { + final PagedAtomicIntegerArray triangles = ((IntersectingTriangleCount) algorithm).getTriangles(); + return Stream.of(builder.buildLI(graph.nodeCount(), triangles::get)); + } else if (algorithm instanceof TriangleCountQueue){ + final AtomicIntegerArray triangles = ((TriangleCountQueue) algorithm).getTriangles(); + return Stream.of(builder.buildII(graph.nodeCount(), triangles::get)); + } + throw new UnsupportedOperationException("unknown algorithm"); } /** @@ -364,11 +378,11 @@ public Stream triangleCountExp3( } } - builder.withNodeCount(graph.nodeCount()) - .withTriangleCount(triangleCount.getTriangleCount()) - .withAverageClusteringCoefficient(triangleCount.getAverageClusteringCoefficient()); + builder.withAverageClusteringCoefficient(triangleCount.getAverageClusteringCoefficient()) + .withTriangleCount(triangleCount.getTriangleCount()); - return Stream.of(builder.build()); + final AtomicIntegerArray triangles = triangleCount.getTriangles(); + return Stream.of(builder.buildII(graph.nodeCount(), triangles::get)); } @@ -377,59 +391,103 @@ public Stream triangleCountExp3( */ public static class Result { + + public static final Result EMPTY = new Result( + 0, + 0, + 0, + 0, + 0, + 0, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + .0); + public final long loadMillis; public final long computeMillis; + public final long postProcessingMillis; public final long writeMillis; public final long nodeCount; public final long triangleCount; + public final long p100; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; public final double averageClusteringCoefficient; - public Result( - long loadMillis, - long computeMillis, - long writeMillis, - long nodeCount, - long triangleCount, - double averageClusteringCoefficient) { + public Result(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodeCount, long triangleCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, double averageClusteringCoefficient) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; this.writeMillis = writeMillis; this.nodeCount = nodeCount; - this.triangleCount = triangleCount; this.averageClusteringCoefficient = averageClusteringCoefficient; + this.triangleCount = triangleCount; + this.p100 = p100; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; } } - public class TriangleCountResultBuilder extends AbstractResultBuilder { + public class TriangleCountResultBuilder extends AbstractCommunityResultBuilder { - private long nodeCount = -1L; - private long triangleCount = -1L; - private double averageClusteringCoefficient = -1d; + private double averageClusteringCoefficient = .0; + private long triangleCount = 0; public TriangleCountResultBuilder withAverageClusteringCoefficient(double averageClusteringCoefficient) { this.averageClusteringCoefficient = averageClusteringCoefficient; return this; } - public TriangleCountResultBuilder withNodeCount(long nodeCount) { - this.nodeCount = nodeCount; - return this; - } - public TriangleCountResultBuilder withTriangleCount(long triangleCount) { this.triangleCount = triangleCount; return this; } + + // communityCount is not used here @Override - public Result build() { + protected Result build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { return new Result( - loadDuration, - evalDuration, - writeDuration, + loadMillis, + computeMillis, + writeMillis, + postProcessingMillis, nodeCount, triangleCount, - averageClusteringCoefficient); + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1), + averageClusteringCoefficient + ); } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java index 899803288..2770d2e29 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.UnionFindResult; +import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{weightProperty:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition'}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java index 1de2545cf..47e3ca4e9 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc2.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.UnionFindResult; +import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc2 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition',concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java index 5cd9ec59a..d7d392a99 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc3.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.UnionFindResult; +import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc3 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition', concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java index 7ff7b1249..14cdf1a0f 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java +++ b/algo/src/main/java/org/neo4j/graphalgo/UnionFindProc4.java @@ -21,7 +21,7 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.impl.UnionFindAlgo; import org.neo4j.graphalgo.impl.UnionFindProcExec; -import org.neo4j.graphalgo.results.UnionFindResult; +import org.neo4j.graphalgo.results.DefaultCommunityResult; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.logging.Log; @@ -52,7 +52,7 @@ public class UnionFindProc4 { @Description("CALL algo.unionFind(label:String, relationship:String, " + "{property:'weight', threshold:0.42, defaultValue:1.0, write: true, partitionProperty:'partition',concurrency:4}) " + "YIELD nodes, setCount, loadMillis, computeMillis, writeMillis") - public Stream unionFind( + public Stream unionFind( @Name(value = "label", defaultValue = "") String label, @Name(value = "relationship", defaultValue = "") String relationship, @Name(value = "config", defaultValue = "{}") Map config) { diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/Algorithm.java b/algo/src/main/java/org/neo4j/graphalgo/impl/Algorithm.java index 09ee40c94..83db2c31a 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/Algorithm.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/Algorithm.java @@ -27,9 +27,9 @@ */ public abstract class Algorithm> implements TerminationFlag { - private ProgressLogger progressLogger = ProgressLogger.NULL_LOGGER; + protected ProgressLogger progressLogger = ProgressLogger.NULL_LOGGER; - private TerminationFlag terminationFlag = TerminationFlag.RUNNING_TRUE; + protected TerminationFlag terminationFlag = TerminationFlag.RUNNING_TRUE; public abstract ME me(); diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/DSSResult.java b/algo/src/main/java/org/neo4j/graphalgo/impl/DSSResult.java index c3352e408..94e0bff8e 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/DSSResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/DSSResult.java @@ -27,9 +27,12 @@ import org.neo4j.graphalgo.core.utils.dss.DisjointSetStruct; import org.neo4j.graphalgo.core.utils.paged.PagedDisjointSetStruct; +import java.util.Arrays; import java.util.stream.Stream; public final class DSSResult { + + public final boolean isHuge; public final DisjointSetStruct struct; public final PagedDisjointSetStruct hugeStruct; @@ -45,6 +48,19 @@ private DSSResult(DisjointSetStruct struct, PagedDisjointSetStruct hugeStruct) { assert (struct != null && hugeStruct == null) || (struct == null && hugeStruct != null); this.struct = struct; this.hugeStruct = hugeStruct; + isHuge = hugeStruct != null; + } + + public int[] getCommunities() { + + if (isHuge) { + return new int[0]; // not supported + } + + final int size = struct.capacity(); + final int[] communities = new int[size]; + Arrays.parallelSetAll(communities, struct::findNoOpt); + return communities; } public int getSetCount() { diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java index 8dd4d80c1..5fea95e09 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/UnionFindProcExec.java @@ -18,6 +18,8 @@ */ package org.neo4j.graphalgo.impl; +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; import org.neo4j.graphalgo.api.Graph; import org.neo4j.graphalgo.core.GraphLoader; import org.neo4j.graphalgo.core.ProcedureConfiguration; @@ -29,7 +31,7 @@ import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.core.utils.paged.PagedDisjointSetStruct; import org.neo4j.graphalgo.core.write.Exporter; -import org.neo4j.graphalgo.results.UnionFindResult; +import org.neo4j.graphalgo.results.AbstractCommunityResultBuilder; import org.neo4j.graphdb.Direction; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -50,7 +52,7 @@ public final class UnionFindProcExec implements BiConsumer> private final GraphDatabaseAPI api; private final Log log; private final KernelTransaction transaction; - private final UnionFindAlgo sequential; + private final UnionFindAlgo sequential; private final UnionFindAlgo parallel; public static Stream run( @@ -64,7 +66,8 @@ public static Stream run( .overrideRelationshipTypeOrQuery(relationship); AllocationTracker tracker = AllocationTracker.create(); - UnionFindResult.Builder builder = UnionFindResult.builder(); + + final Builder builder = new Builder(); UnionFindProcExec uf = unionFind.get(); @@ -72,13 +75,10 @@ public static Stream run( if (graph.nodeCount() == 0) { graph.release(); - return Stream.of(builder - .withNodeCount(graph.nodeCount()) - .withSetCount(0) - .build()); + return Stream.of(UnionFindResult.EMPTY); } - DSSResult dssResult = uf.evaluate( + final DSSResult dssResult = uf.evaluate( builder::timeEval, graph, configuration, @@ -89,10 +89,95 @@ public static Stream run( uf.write(builder::timeWrite, graph, dssResult, configuration); } - return Stream.of(builder - .withNodeCount(graph.nodeCount()) - .withSetCount(dssResult.getSetCount()) - .build()); + if (dssResult.isHuge) { + return Stream.of(builder.build(graph.nodeCount(), dssResult.hugeStruct::find)); + } else { + return Stream.of(builder.build(graph.nodeCount(), l -> (long) dssResult.struct.find((int) l))); + } + } + + public static class UnionFindResult { + + public static final UnionFindProcExec.UnionFindResult EMPTY = new UnionFindProcExec.UnionFindResult( + 0, + 0, + 0, + 0, + 0, + 0, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1 + ); + + public final long loadMillis; + public final long computeMillis; + public final long postProcessingMillis; + public final long writeMillis; + public final long nodes; + public final long communityCount; + public final long setCount; + public final long p100; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; + + public UnionFindResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01) { + this.loadMillis = loadMillis; + this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; + this.writeMillis = writeMillis; + this.nodes = nodes; + this.communityCount = communityCount; + this.p100 = p100; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; + this.setCount = communityCount; + } + } + + public static class Builder extends AbstractCommunityResultBuilder { + @Override + protected UnionFindResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { + return new UnionFindResult( + loadMillis, + computeMillis, + postProcessingMillis, + writeMillis, + nodeCount, + communityCount, + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1) + ); + } } public static Stream stream( @@ -242,4 +327,6 @@ private void write( PagedDisjointSetStruct.Translator.INSTANCE); } + + } diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/Louvain.java b/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/Louvain.java index fdbb552e1..692c677d4 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/Louvain.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/Louvain.java @@ -64,7 +64,7 @@ public class Louvain extends Algorithm { private int[][] dendrogram; private double[] nodeWeights; private Graph root; - private int communityCount = 0; + private int communityCount; public Louvain(Graph graph, ExecutorService pool, @@ -137,7 +137,7 @@ public Louvain compute(WeightMapping communityMap, int maxLevel, int maxIteratio dendrogram = new int[maxLevel][]; modularities = new double[maxLevel]; - for (level = 0; level < maxLevel; level++) { + for (level = 0; level < maxLevel && terminationFlag.running(); level++) { // start modularity optimization final ModularityOptimization modularityOptimization = new ModularityOptimization(graph, @@ -213,9 +213,7 @@ private int[] rebuildCommunityStructure(int[] communityIds) { // rebuild community array assert rootNodeCount == communities.length; final int[] ints = new int[rootNodeCount]; - Arrays.setAll(ints, i -> { - return communityIds[communities[i]]; - }); + Arrays.setAll(ints, i -> communityIds[communities[i]]); communities = ints; return communities; } diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/ModularityOptimization.java b/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/ModularityOptimization.java index e96009d02..037f8f855 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/ModularityOptimization.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/louvain/ModularityOptimization.java @@ -29,6 +29,7 @@ import org.neo4j.graphalgo.core.utils.ParallelUtil; import org.neo4j.graphalgo.core.utils.Pointer; import org.neo4j.graphalgo.core.utils.ProgressLogger; +import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.impl.Algorithm; import org.neo4j.graphdb.Direction; @@ -170,6 +171,7 @@ private void init() { * @return */ public ModularityOptimization compute(int maxIterations) { + final TerminationFlag terminationFlag = getTerminationFlag(); // init helper values & initial community structure init(); // create an array of tasks for parallel exec @@ -180,7 +182,7 @@ public ModularityOptimization compute(int maxIterations) { // (2x double + 1x int) * N * threads tracker.add(20 * nodeCount * concurrency); // as long as maxIterations is not reached - for (iterations = 0; iterations < maxIterations; iterations++) { + for (iterations = 0; iterations < maxIterations && terminationFlag.running(); iterations++) { // reset node counter (for logging) counter.set(0); // run all tasks @@ -267,6 +269,7 @@ private class Task implements Runnable { final double[] sTot, sIn; final int[] localCommunities; + private final TerminationFlag terminationFlag; double bestGain, bestWeight, q = MINIMUM_MODULARITY; int bestCommunity; boolean improvement = false; @@ -276,6 +279,7 @@ private class Task implements Runnable { * and initializes its helper arrays */ Task() { + terminationFlag = getTerminationFlag(); sTot = new double[nodeCount]; System.arraycopy(ki, 0, sTot, 0, nodeCount); // ki -> sTot localCommunities = new int[nodeCount]; @@ -307,7 +311,7 @@ public void run() { counter.getAndIncrement(), denominator, () -> String.format("round %d", iterations + 1)); - return true; + return terminationFlag.running(); }); this.q = calcModularity(); } @@ -383,29 +387,6 @@ private void removeWeightForSelfRelationships(int node, IntDoubleMap communityWe }); } - - /** - * apply consumer to each connected community one time - * - * @param node node nodeId - * @param consumer community nodeId consumer - */ - private void forEachConnectedCommunity(int node, IntConsumer consumer) { - final BitSet visited = new BitSet(nodeCount); - graph.forEachRelationship(node, D, (s, t, r) -> { - final int c = localCommunities[t]; - if (c == NONE) { - return true; - } - if (visited.get(c)) { - return true; - } - visited.set(c); - consumer.accept(c); - return true; - }); - } - /** * sum weights from node into community c * diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCAlgorithm.java b/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCAlgorithm.java index 522f9c6e9..e549a54e0 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCAlgorithm.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCAlgorithm.java @@ -62,67 +62,6 @@ public StreamResult(long nodeId, long partition) { } } - class Result { - - public final Long loadMillis; - public final Long computeMillis; - public final Long writeMillis; - public final Long setCount; - public final Long minSetSize; - public final Long maxSetSize; - - public Result(Long loadMillis, - Long computeMillis, - Long writeMillis, - Long setCount, - Long minSetSize, - Long maxSetSize) { - this.loadMillis = loadMillis; - this.computeMillis = computeMillis; - this.writeMillis = writeMillis; - this.setCount = setCount; - this.minSetSize = minSetSize; - this.maxSetSize = maxSetSize; - } - - public static Result.Builder builder() { - return new Result.Builder(); - } - - public static final class Builder extends AbstractResultBuilder { - - private long setCount; - private long minSetSize; - private long maxSetSize; - - public Result.Builder withSetCount(long setCount) { - this.setCount = setCount; - return this; - } - - public Result.Builder withMinSetSize(long minSetSize) { - this.minSetSize = minSetSize; - return this; - } - - public Result.Builder withMaxSetSize(long maxSetSize) { - this.maxSetSize = maxSetSize; - return this; - } - - @Override - public Result build() { - return new Result(loadDuration, - evalDuration, - writeDuration, - setCount, - minSetSize, - maxSetSize); - } - } - - } - static SCCAlgorithm iterativeTarjan(Graph graph, AllocationTracker tracker) { if (graph instanceof HugeGraph) { return new HugeSCCIterativeTarjan((HugeGraph) graph, tracker); diff --git a/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCTarjan.java b/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCTarjan.java index b63bd2acc..9486a885f 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCTarjan.java +++ b/algo/src/main/java/org/neo4j/graphalgo/impl/scc/SCCTarjan.java @@ -1,36 +1,31 @@ /** * Copyright (c) 2017 "Neo4j, Inc." - * + *

* This file is part of Neo4j Graph Algorithms . - * + *

* Neo4j Graph Algorithms is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + *

* This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + *

* You should have received a copy of the GNU General Public License * along with this program. If not, see . */ package org.neo4j.graphalgo.impl.scc; -import com.carrotsearch.hppc.IntHashSet; -import com.carrotsearch.hppc.IntSet; import com.carrotsearch.hppc.IntStack; -import com.carrotsearch.hppc.ObjectArrayList; import org.neo4j.graphalgo.api.Graph; -import org.neo4j.graphalgo.api.RelationshipConsumer; import org.neo4j.graphalgo.core.utils.ProgressLogger; import org.neo4j.graphalgo.impl.Algorithm; import org.neo4j.graphdb.Direction; import java.util.Arrays; import java.util.BitSet; -import java.util.function.IntPredicate; /** * Sequential strongly connected components algorithm (Tarjan). @@ -43,51 +38,32 @@ public class SCCTarjan extends Algorithm { private Graph graph; private final int nodeCount; - private Aggregator aggregator; + private int[] communities; + private int[] indices; + private int[] lowLink; + private final BitSet onStack; + private final IntStack stack; + private int index; + public SCCTarjan(Graph graph) { this.graph = graph; nodeCount = Math.toIntExact(graph.nodeCount()); - - aggregator = new Aggregator(graph, - new int[nodeCount], - new int[nodeCount], - new ObjectArrayList<>(), - new BitSet(nodeCount), - new IntStack(nodeCount)); + indices = new int[nodeCount]; + lowLink = new int[nodeCount]; + onStack = new BitSet(nodeCount); + stack = new IntStack(nodeCount); + communities = new int[nodeCount]; + Arrays.setAll(communities, i -> i); } public SCCTarjan compute() { - aggregator.reset(); - graph.forEachNode(aggregator); + graph.forEachNode(this::test); return this; } - /** - * get connected components list - * - * @return list of sets of strongly connected component ID's - */ - public ObjectArrayList getConnectedComponents() { - return aggregator.connectedComponents; - } - - /** - * return the maximum set size - * - * @return the maximum set size - */ - public long getMaxSetSize() { - return graph.nodeCount() == 0 ? 0 : aggregator.maxSetSize; - } - - /** - * return the minimum set size - * - * @return minimum set size - */ - public long getMinSetSize() { - return graph.nodeCount() == 0 ? 0 : aggregator.minSetSize; + public int[] getConnectedComponents() { + return communities; } @Override @@ -97,93 +73,57 @@ public SCCTarjan me() { @Override public SCCTarjan release() { - aggregator = null; + stack.clear(); + communities = null; + indices = null; + lowLink = null; graph = null; return this; } - private final class Aggregator implements IntPredicate, RelationshipConsumer { - - private final Graph graph; - private final int[] indices; - private final int[] lowLink; - private final ObjectArrayList connectedComponents; - private final BitSet onStack; - private final IntStack stack; - private int index; - private long minSetSize = Long.MAX_VALUE; - private long maxSetSize = 0; - private ProgressLogger progressLogger; - - private Aggregator(Graph graph, int[] indices, int[] lowLink, ObjectArrayList connectedComponents, BitSet onStack, IntStack stack) { - this.graph = graph; - this.indices = indices; - this.lowLink = lowLink; - this.connectedComponents = connectedComponents; - this.onStack = onStack; - this.stack = stack; - this.progressLogger = getProgressLogger(); - } - - public void reset() { - connectedComponents.clear(); - Arrays.fill(indices, -1); - Arrays.fill(lowLink, -1); - onStack.clear(); - stack.clear(); - index = 0; - minSetSize = Long.MAX_VALUE; - maxSetSize = 0; - } + public void reset() { + Arrays.fill(indices, -1); + Arrays.fill(lowLink, -1); + onStack.clear(); + stack.clear(); + index = 0; + } - private void strongConnect(int node) { - lowLink[node] = index; - indices[node] = index; - index++; - stack.push(node); - onStack.set(node); - graph.forEachRelationship(node, Direction.OUTGOING, this); - if (indices[node] == lowLink[node]) { - relax(node); - } + private void strongConnect(int node) { + lowLink[node] = index; + indices[node] = index++; + stack.push(node); + onStack.set(node); + graph.forEachRelationship(node, Direction.OUTGOING, this::accept); + if (indices[node] == lowLink[node]) { + relax(node); } + } - private void relax(int nodeId) { - IntHashSet connected = new IntHashSet(); - int w; - do { - w = stack.pop(); - onStack.clear(w); - connected.add(w); - } while (w != nodeId); - connectedComponents.add(connected); - int size = connected.size(); - if (size < minSetSize) { - minSetSize = size; - } - if (size > maxSetSize) { - maxSetSize = size; - } - } + private void relax(int nodeId) { + int w; + do { + w = stack.pop(); + onStack.clear(w); + communities[w] = nodeId; + } while (w != nodeId); + } - @Override - public boolean accept(int source, int target, long edgeId) { - if (indices[target] == -1) { - strongConnect(target); - lowLink[source] = Math.min(lowLink[source], lowLink[target]); - } else if (onStack.get(target)) { - lowLink[source] = Math.min(lowLink[source], indices[target]); - } - return true; + private boolean accept(int source, int target, long unused) { + if (indices[target] == -1) { + strongConnect(target); + lowLink[source] = Math.min(lowLink[source], lowLink[target]); + } else if (onStack.get(target)) { + lowLink[source] = Math.min(lowLink[source], indices[target]); } + return true; + } - @Override - public boolean test(int node) { - if (indices[node] == -1) { - strongConnect(node); - } - progressLogger.logProgress((double) node / (nodeCount - 1)); - return running(); + private boolean test(int node) { + if (indices[node] == -1) { + strongConnect(node); } + progressLogger.logProgress((double) node / (nodeCount - 1)); + return running(); } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java new file mode 100644 index 000000000..f7241eaac --- /dev/null +++ b/algo/src/main/java/org/neo4j/graphalgo/results/AbstractCommunityResultBuilder.java @@ -0,0 +1,151 @@ +package org.neo4j.graphalgo.results; + +import com.carrotsearch.hppc.LongLongMap; +import com.carrotsearch.hppc.LongLongScatterMap; +import com.carrotsearch.hppc.cursors.LongLongCursor; +import org.HdrHistogram.Histogram; +import org.neo4j.graphalgo.core.utils.ProgressTimer; + +import java.util.Arrays; +import java.util.List; +import java.util.function.IntFunction; +import java.util.function.LongFunction; +import java.util.function.LongToIntFunction; + +/** + * @author mknblch + */ +public abstract class AbstractCommunityResultBuilder { + + protected long loadDuration = -1; + protected long evalDuration = -1; + protected long writeDuration = -1; + + public AbstractCommunityResultBuilder withLoadDuration(long loadDuration) { + this.loadDuration = loadDuration; + return this; + } + + + public AbstractCommunityResultBuilder withEvalDuration(long evalDuration) { + this.evalDuration = evalDuration; + return this; + } + + + public AbstractCommunityResultBuilder withWriteDuration(long writeDuration) { + this.writeDuration = writeDuration; + return this; + } + + /** + * returns an AutoClosable which measures the time + * until it gets closed. Saves the duration as loadMillis + * + * @return + */ + public ProgressTimer timeLoad() { + return ProgressTimer.start(this::withLoadDuration); + } + + /** + * returns an AutoClosable which measures the time + * until it gets closed. Saves the duration as evalMillis + * + * @return + */ + public ProgressTimer timeEval() { + return ProgressTimer.start(this::withEvalDuration); + } + + /** + * returns an AutoClosable which measures the time + * until it gets closed. Saves the duration as writeMillis + * + * @return + */ + public ProgressTimer timeWrite() { + return ProgressTimer.start(this::withWriteDuration); + } + + /** + * evaluates loadMillis + * + * @param runnable + */ + public void timeLoad(Runnable runnable) { + try (ProgressTimer timer = timeLoad()) { + runnable.run(); + } + } + + /** + * evaluates comuteMillis + * + * @param runnable + */ + public void timeEval(Runnable runnable) { + try (ProgressTimer timer = timeEval()) { + runnable.run(); + } + } + + /** + * evaluates writeMillis + * + * @param runnable + */ + public void timeWrite(Runnable runnable) { + try (ProgressTimer timer = timeWrite()) { + runnable.run(); + } + } + + public T buildII(long nodes, IntFunction fun) { + return build(nodes, value -> (long) fun.apply((int) value)); + } + + public T buildLI(long nodes, LongToIntFunction fun) { + return build(nodes, value -> (long) fun.applyAsInt(value)); + } + + /** + * build result + */ + public T build(long nodeCount, LongFunction fun) { + + final LongLongMap communitySizeMap = new LongLongScatterMap(); + final ProgressTimer timer = ProgressTimer.start(); + for (int i = 0; i < nodeCount; i++) { + // map to community id + final long cId = fun.apply(i); + // aggregate community size + communitySizeMap.addTo(cId, 1); + } + + Histogram histogram = CommunityHistogram.buildFrom(communitySizeMap); + + timer.stop(); + + return build(loadDuration, + evalDuration, + writeDuration, + timer.getDuration(), + nodeCount, + communitySizeMap.size(), + communitySizeMap, + histogram + ); + } + + protected abstract T build( + long loadMillis, + long computeMillis, + long writeMillis, + long postProcessingMillis, + long nodeCount, + long communityCount, + LongLongMap communitySizeMap, + Histogram communityHistogram); + +} diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/CommunityHistogram.java b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityHistogram.java new file mode 100644 index 000000000..04a67b8d2 --- /dev/null +++ b/algo/src/main/java/org/neo4j/graphalgo/results/CommunityHistogram.java @@ -0,0 +1,18 @@ +package org.neo4j.graphalgo.results; + +import com.carrotsearch.hppc.LongLongMap; +import com.carrotsearch.hppc.cursors.LongLongCursor; +import org.HdrHistogram.Histogram; + +public class CommunityHistogram { + + public static Histogram buildFrom(LongLongMap communitySizeMap) { + final Histogram histogram = new Histogram(2); + + for (LongLongCursor cursor : communitySizeMap) { + histogram.recordValue(cursor.value); + } + + return histogram; + } +} diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java new file mode 100644 index 000000000..4eb0e8a17 --- /dev/null +++ b/algo/src/main/java/org/neo4j/graphalgo/results/DefaultCommunityResult.java @@ -0,0 +1,92 @@ +package org.neo4j.graphalgo.results; + +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; + +import java.util.Collections; +import java.util.List; + +/** + * @author mknblch + */ +public class DefaultCommunityResult { + + public static final DefaultCommunityResult EMPTY = new DefaultCommunityResult( + 0, 0,0,0, 0, 0, -1,-1, -1, -1, -1, -1, -1, -1, -1, -1 + ); + + public final long loadMillis; + public final long computeMillis; + public final long postProcessingMillis; + public final long writeMillis; + public final long nodes; + public final long communityCount; + public final long p100; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; + + public DefaultCommunityResult(long loadMillis, + long computeMillis, + long postProcessingMillis, + long writeMillis, + long nodes, + long communityCount, + long p100, + long p99, + long p95, + long p90, + long p75, + long p50, + long p25, + long p10, + long p05, + long p01) { + this.loadMillis = loadMillis; + this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; + this.writeMillis = writeMillis; + this.nodes = nodes; + this.communityCount = communityCount; + this.p100 = p100; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; + } + + public static class DefaultCommunityResultBuilder extends AbstractCommunityResultBuilder { + + @Override + protected DefaultCommunityResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { + return new DefaultCommunityResult( + loadMillis, + evalDuration, + postProcessingMillis, + writeMillis, + nodeCount, + communityCount, + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1)); + } + } +} diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java b/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java new file mode 100644 index 000000000..d3b1169dc --- /dev/null +++ b/algo/src/main/java/org/neo4j/graphalgo/results/HppcMapComparator.java @@ -0,0 +1,56 @@ +package org.neo4j.graphalgo.results; + +import com.carrotsearch.hppc.IntIntHashMap; +import com.carrotsearch.hppc.sorting.IndirectComparator; + +public final class HppcMapComparator implements IndirectComparator { + + private final int[] values; + private final int[] keys; + private final int max; + private boolean hasEmptyKey; + + HppcMapComparator(IntIntHashMap map) { + values = map.values; + keys = map.keys; + max = keys.length - 1; + int lastValue = values[max]; + hasEmptyKey = map.getOrDefault(0, ~lastValue) == lastValue; + } + + @Override + public int compare(final int indexA, final int indexB) { + assert validIndex(indexA); + assert validIndex(indexB); + + // assigned slot for A + if ((keys[indexA] != 0) || (indexA == max && hasEmptyKey)) { + + // assigned slot for B + if ((keys[indexB] != 0) || (indexB == max && hasEmptyKey)) { + + // reverse order because descending + return Integer.compare(values[indexB], values[indexA]); + } + + // empty slot for B, put it at the end, A is "smaller" + return -1; + } + + // assigned slot for B + if ((keys[indexB] != 0) || (indexB == max && hasEmptyKey)) { + + // empty slot for A, put it at the end, B is "smaller" + return 1; + } + + // empty slot for A and B, both are equal + return 0; + } + + private boolean validIndex(int index) { + assert index >= 0 : "The index " + index + " must point at an existing key."; + assert index <= max; + return true; + } + } \ No newline at end of file diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java index 89aa0548a..992378fb9 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/LabelPropagationStats.java @@ -1,64 +1,106 @@ /** * Copyright (c) 2017 "Neo4j, Inc." - * + *

* This file is part of Neo4j Graph Algorithms . - * + *

* Neo4j Graph Algorithms is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + *

* This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + *

* You should have received a copy of the GNU General Public License * along with this program. If not, see . */ package org.neo4j.graphalgo.results; +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; + +import java.util.List; + public class LabelPropagationStats { - public final long nodes, iterations, loadMillis, computeMillis, writeMillis; + public static final LabelPropagationStats EMPTY = new LabelPropagationStats( + 0, + 0, + 0, + 0, + 0, + 0, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + 0, + false, + false, + "", + ""); + + public final long loadMillis; + public final long computeMillis; + public final long postProcessingMillis; + public final long writeMillis; + public final long nodes; + public final long communityCount; + public final long p100; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; + public final long iterations; public final boolean write, didConverge; public final String weightProperty, partitionProperty; - public LabelPropagationStats( - final long nodes, - final long iterations, - final long loadMillis, - final long computeMillis, - final long writeMillis, - final boolean write, - final boolean didConverge, - final String weightProperty, - final String partitionProperty) { - this.nodes = nodes; - this.iterations = iterations; + public LabelPropagationStats(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations, boolean write, boolean didConverge, String weightProperty, String partitionProperty) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; this.writeMillis = writeMillis; + this.nodes = nodes; + this.communityCount = communityCount; + this.p100 = p100; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; + this.iterations = iterations; this.write = write; this.didConverge = didConverge; this.weightProperty = weightProperty; this.partitionProperty = partitionProperty; } - public static class Builder extends AbstractResultBuilder { - private long nodes = 0; + public static class Builder extends AbstractCommunityResultBuilder { + private long iterations = 0; private boolean didConverge = false; private boolean write; private String weightProperty; private String partitionProperty; - public Builder nodes(final long nodes) { - this.nodes = nodes; - return this; - } - public Builder iterations(final long iterations) { this.iterations = iterations; return this; @@ -84,17 +126,32 @@ public Builder partitionProperty(final String partitionProperty) { return this; } - public LabelPropagationStats build() { + @Override + protected LabelPropagationStats build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { return new LabelPropagationStats( - nodes, + loadMillis, + computeMillis, + writeMillis, + postProcessingMillis, + nodeCount, + communityCount, + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1), iterations, - loadDuration, - evalDuration, - writeDuration, write, didConverge, weightProperty, - partitionProperty); + partitionProperty + ); } + } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/LouvainResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/LouvainResult.java deleted file mode 100644 index 77896a3c0..000000000 --- a/algo/src/main/java/org/neo4j/graphalgo/results/LouvainResult.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Copyright (c) 2017 "Neo4j, Inc." - * - * This file is part of Neo4j Graph Algorithms . - * - * Neo4j Graph Algorithms is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.graphalgo.results; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author mknblch - */ -public class LouvainResult { - - public final long loadMillis; - public final long computeMillis; - public final long writeMillis; - public final long nodes; - public final long iterations; - public final long communityCount; - public final List modularities; - public final double modularity; - - private LouvainResult(long loadMillis, long computeMillis, long writeMillis, long nodes, long iterations, - long communityCount, double[] modularities, double modularity) { - this.loadMillis = loadMillis; - this.computeMillis = computeMillis; - this.writeMillis = writeMillis; - this.nodes = nodes; - this.iterations = iterations; - this.communityCount = communityCount; - - this.modularities = new ArrayList<>(modularities.length); - for (double mod : modularities) this.modularities.add(mod); - - - this.modularity = modularity; - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder extends AbstractResultBuilder { - - private long nodes = 0; - private long communityCount = 0; - private long iterations = 1; - private double[] modularities = new double[]{}; - private double modularity = -1.0; - - public Builder withIterations(long iterations) { - this.iterations = iterations; - return this; - } - - public Builder withCommunityCount(long setCount) { - this.communityCount = setCount; - return this; - } - - public Builder withNodeCount(long nodes) { - this.nodes = nodes; - return this; - } - - public Builder withModularities(double[] modularities) { - this.modularities = modularities; - return this; - } - - public Builder withFinalModularity(double modularity) { - this.modularity = modularity; - return this; - } - - public LouvainResult build() { - return new LouvainResult(loadDuration, evalDuration, writeDuration, nodes, iterations, communityCount, - modularities, modularity); - } - } -} diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java index d040bdbcb..5588a9488 100644 --- a/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java +++ b/algo/src/main/java/org/neo4j/graphalgo/results/SCCResult.java @@ -18,28 +18,57 @@ */ package org.neo4j.graphalgo.results; +import com.carrotsearch.hppc.LongLongMap; +import org.HdrHistogram.Histogram; + /** * @author mknblch */ public class SCCResult { - public final Long loadMillis; - public final Long computeMillis; - public final Long writeMillis; - public final Long setCount; - public final Long minSetSize; - public final Long maxSetSize; + public static SCCResult EMPTY = new SCCResult( + 0, 0, 0, 0,0, 0, -1,-1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 0, 0 + ); + + public final long loadMillis; + public final long computeMillis; + public final long postProcessingMillis; + public final long writeMillis; + public final long nodes; + public final long communityCount; + public final long setCount; + public final long p100; + public final long p99; + public final long p95; + public final long p90; + public final long p75; + public final long p50; + public final long p25; + public final long p10; + public final long p05; + public final long p01; + public final long iterations; + public final long minSetSize; + public final long maxSetSize; - public SCCResult(Long loadMillis, - Long computeMillis, - Long writeMillis, - Long setCount, - Long minSetSize, - Long maxSetSize) { + public SCCResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations, long minSetSize, long maxSetSize) { this.loadMillis = loadMillis; this.computeMillis = computeMillis; + this.postProcessingMillis = postProcessingMillis; this.writeMillis = writeMillis; - this.setCount = setCount; + this.nodes = nodes; + this.setCount = this.communityCount = communityCount; + this.p100 = p100; + this.p99 = p99; + this.p95 = p95; + this.p90 = p90; + this.p75 = p75; + this.p50 = p50; + this.p25 = p25; + this.p10 = p10; + this.p05 = p05; + this.p01 = p01; + this.iterations = iterations; this.minSetSize = minSetSize; this.maxSetSize = maxSetSize; } @@ -48,35 +77,32 @@ public static Builder builder() { return new Builder(); } - public static final class Builder extends AbstractResultBuilder { - - private long setCount; - private long minSetSize; - private long maxSetSize; - - public Builder withSetCount(long setCount) { - this.setCount = setCount; - return this; - } - - public Builder withMinSetSize(long minSetSize) { - this.minSetSize = minSetSize; - return this; - } - - public Builder withMaxSetSize(long maxSetSize) { - this.maxSetSize = maxSetSize; - return this; - } + public static final class Builder extends AbstractCommunityResultBuilder { + private int iterations = -1; @Override - public SCCResult build() { - return new SCCResult(loadDuration, - evalDuration, - writeDuration, - setCount, - minSetSize, - maxSetSize); + protected SCCResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) { + return new SCCResult( + loadMillis, + computeMillis, + writeMillis, + postProcessingMillis, + nodeCount, + communityCount, + communityHistogram.getValueAtPercentile(100), + communityHistogram.getValueAtPercentile(99), + communityHistogram.getValueAtPercentile(95), + communityHistogram.getValueAtPercentile(90), + communityHistogram.getValueAtPercentile(75), + communityHistogram.getValueAtPercentile(50), + communityHistogram.getValueAtPercentile(25), + communityHistogram.getValueAtPercentile(10), + communityHistogram.getValueAtPercentile(5), + communityHistogram.getValueAtPercentile(1), + iterations, + communityHistogram.getMinNonZeroValue(), + communityHistogram.getMaxValue() + ); } } diff --git a/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java b/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java deleted file mode 100644 index d4ac91970..000000000 --- a/algo/src/main/java/org/neo4j/graphalgo/results/UnionFindResult.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright (c) 2017 "Neo4j, Inc." - * - * This file is part of Neo4j Graph Algorithms . - * - * Neo4j Graph Algorithms is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.graphalgo.results; - -/** - * @author mknblch - */ -public class UnionFindResult { - - public final Long loadMillis; - public final Long computeMillis; - public final Long writeMillis; - public final Long nodes; - public final Long setCount; - - private UnionFindResult(Long loadMillis, Long computeMillis, Long writeMillis, Long nodes, Long setCount) { - this.loadMillis = loadMillis; - this.computeMillis = computeMillis; - this.writeMillis = writeMillis; - this.nodes = nodes; - this.setCount = setCount; - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder extends AbstractResultBuilder { - - private long nodes = 0; - private long setCount = 0; - - public Builder withSetCount(long setCount) { - this.setCount = setCount; - return this; - } - - public Builder withNodeCount(long nodes) { - this.nodes = nodes; - return this; - } - - public UnionFindResult build() { - return new UnionFindResult(loadDuration, evalDuration, writeDuration, nodes, setCount); - } - } -} diff --git a/algo/src/test/java/org/neo4j/graphalgo/results/CommunityHistogramTest.java b/algo/src/test/java/org/neo4j/graphalgo/results/CommunityHistogramTest.java new file mode 100644 index 000000000..d64e41797 --- /dev/null +++ b/algo/src/test/java/org/neo4j/graphalgo/results/CommunityHistogramTest.java @@ -0,0 +1,35 @@ +package org.neo4j.graphalgo.results; + +import com.carrotsearch.hppc.LongLongMap; +import com.carrotsearch.hppc.LongLongScatterMap; +import org.HdrHistogram.Histogram; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class CommunityHistogramTest { + @Test + public void oneCommunity() { + final LongLongMap communitySizeMap = new LongLongScatterMap(); + communitySizeMap.addTo(1, 4); + + Histogram histogram = CommunityHistogram.buildFrom(communitySizeMap); + + assertEquals(4.0, histogram.getValueAtPercentile(100D), 0.01); + } + + @Test + public void multipleCommunities() { + final LongLongMap communitySizeMap = new LongLongScatterMap(); + communitySizeMap.addTo(1, 4); + communitySizeMap.addTo(2, 10); + communitySizeMap.addTo(3, 9); + communitySizeMap.addTo(4, 8); + communitySizeMap.addTo(5, 7); + + Histogram histogram = CommunityHistogram.buildFrom(communitySizeMap); + + assertEquals(10.0, histogram.getValueAtPercentile(100D), 0.01); + assertEquals(8.0, histogram.getValueAtPercentile(50D), 0.01); + } +} \ No newline at end of file diff --git a/core/src/main/java/org/neo4j/graphalgo/core/utils/dss/DisjointSetStruct.java b/core/src/main/java/org/neo4j/graphalgo/core/utils/dss/DisjointSetStruct.java index 27f6affd7..8bca215b4 100644 --- a/core/src/main/java/org/neo4j/graphalgo/core/utils/dss/DisjointSetStruct.java +++ b/core/src/main/java/org/neo4j/graphalgo/core/utils/dss/DisjointSetStruct.java @@ -122,11 +122,11 @@ public Stream resultStream(IdMapping idMapping) { } /** - * element count + * element (node) count * * @return the element count */ - public int count() { + public int capacity() { return parent.length; } @@ -234,11 +234,11 @@ public IntIntMap getSetSize() { public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("\n"); - for (int i = 0; i < count(); i++) { + for (int i = 0; i < capacity(); i++) { builder.append(String.format(" %d ", i)); } builder.append("\n"); - for (int i = 0; i < count(); i++) { + for (int i = 0; i < capacity(); i++) { builder.append(String.format("[%d]", find(i))); } return builder.toString(); @@ -282,7 +282,7 @@ private static class NodeSetIterator implements Iterator { private NodeSetIterator(DisjointSetStruct struct) { this.struct = struct; - this.length = struct.count(); + this.length = struct.capacity(); } @Override @@ -311,8 +311,8 @@ private static class ConcurrentNodeSetIterator implements Iterator { private ConcurrentNodeSetIterator(DisjointSetStruct struct, int startOffset, int length) { this.struct = struct; - this.length = length + offset > struct.count() - ? struct.count() - offset + this.length = length + offset > struct.capacity() + ? struct.capacity() - offset : length; this.offset = startOffset; } diff --git a/core/src/main/java/org/neo4j/graphalgo/core/utils/paged/PagedDisjointSetStruct.java b/core/src/main/java/org/neo4j/graphalgo/core/utils/paged/PagedDisjointSetStruct.java index 3b1bcfdb1..f00b30617 100644 --- a/core/src/main/java/org/neo4j/graphalgo/core/utils/paged/PagedDisjointSetStruct.java +++ b/core/src/main/java/org/neo4j/graphalgo/core/utils/paged/PagedDisjointSetStruct.java @@ -44,6 +44,10 @@ public PagedDisjointSetStruct reset() { return this; } + public long capacity() { + return capacity; + } + public boolean connected(long p, long q) { return find(p) == find(q); } diff --git a/pom.xml b/pom.xml index f0531a1c0..d4fc9a390 100644 --- a/pom.xml +++ b/pom.xml @@ -104,7 +104,7 @@ net.biville.florent neo4j-sproc-compiler - 1.2 + 1.2.1 provided true diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java index 0d431c11b..fd01ee65c 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/LouvainClusteringIntegrationTest.java @@ -28,7 +28,6 @@ import java.util.Arrays; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import static junit.framework.TestCase.assertNull; @@ -98,18 +97,19 @@ public void clearCommunities() { @Test public void test() { final String cypher = "CALL algo.louvain('', '', {concurrency:1}) " + - "YIELD nodes, communityCount, iterations, loadMillis, computeMillis, writeMillis"; + "YIELD nodes, communityCount, loadMillis, computeMillis, writeMillis, postProcessingMillis, p99"; DB.execute(cypher).accept(row -> { final long nodes = row.getNumber("nodes").longValue(); final long communityCount = row.getNumber("communityCount").longValue(); - final long iterations = row.getNumber("iterations").longValue(); final long loadMillis = row.getNumber("loadMillis").longValue(); final long computeMillis = row.getNumber("computeMillis").longValue(); final long writeMillis = row.getNumber("writeMillis").longValue(); + System.out.println("postProcessingMillis = " + row.getNumber("postProcessingMillis")); System.out.println("nodes = " + nodes); System.out.println("communityCount = " + communityCount); - System.out.println("iterations = " + iterations); + System.out.println("p99 = " + row.get("p99")); + assertEquals("invalid node count",9, nodes); assertEquals("wrong community count", 3, communityCount); assertTrue("invalid loadTime", loadMillis >= 0); @@ -333,5 +333,4 @@ public int[] getClusterId(String nodeName) { }); return id; } - } diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java index eeaba8f69..546b8cb62 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/StronglyConnectedComponentsProcIntegrationTest.java @@ -104,8 +104,6 @@ public void testScc() throws Exception { System.out.println(row.getNumber("computeMillis").longValue()); System.out.println(row.getNumber("writeMillis").longValue()); System.out.println(row.getNumber("setCount").longValue()); - System.out.println(row.getNumber("maxSetSize").longValue()); - System.out.println(row.getNumber("minSetSize").longValue()); assertNotEquals(-1L, row.getNumber("computeMillis").longValue()); assertNotEquals(-1L, row.getNumber("writeMillis").longValue()); diff --git a/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java b/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java index 0a57f9716..2ee3cee0d 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/algo/UnionFindProcIntegrationTest.java @@ -110,8 +110,9 @@ public static Collection data() { @Test public void testUnionFind() throws Exception { - db.execute("CALL algo.unionFind('', '',{graph:'"+graphImpl+"'}) YIELD setCount") + db.execute("CALL algo.unionFind('', '',{graph:'"+graphImpl+"'}) YIELD setCount, communityCount") .accept((Result.ResultVisitor) row -> { + assertEquals(3L, row.getNumber("communityCount")); assertEquals(3L, row.getNumber("setCount")); return true; }); @@ -119,8 +120,9 @@ public void testUnionFind() throws Exception { @Test public void testUnionFindWithLabel() throws Exception { - db.execute("CALL algo.unionFind('Label', '',{graph:'"+graphImpl+"'}) YIELD setCount") + db.execute("CALL algo.unionFind('Label', '',{graph:'"+graphImpl+"'}) YIELD setCount, communityCount") .accept((Result.ResultVisitor) row -> { + assertEquals(1L, row.getNumber("communityCount")); assertEquals(1L, row.getNumber("setCount")); return true; }); @@ -128,10 +130,11 @@ public void testUnionFindWithLabel() throws Exception { @Test public void testUnionFindWriteBack() throws Exception { - db.execute("CALL algo.unionFind('', 'TYPE', {write:true,graph:'"+graphImpl+"'}) YIELD setCount, writeMillis, nodes") + db.execute("CALL algo.unionFind('', 'TYPE', {write:true,graph:'"+graphImpl+"'}) YIELD setCount, communityCount, writeMillis, nodes") .accept((Result.ResultVisitor) row -> { assertNotEquals(-1L, row.getNumber("writeMillis")); assertNotEquals(-1L, row.getNumber("nodes")); + assertEquals(3L, row.getNumber("communityCount")); assertEquals(3L, row.getNumber("setCount")); return false; }); diff --git a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainMultiLevelTest.java b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainMultiLevelTest.java index 0ab2a837b..c2d69aedc 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainMultiLevelTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainMultiLevelTest.java @@ -30,6 +30,7 @@ import org.neo4j.graphalgo.core.heavyweight.HeavyGraphFactory; import org.neo4j.graphalgo.core.huge.HugeGraphFactory; import org.neo4j.graphalgo.core.utils.Pools; +import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.impl.louvain.Louvain; import org.neo4j.graphdb.Label; @@ -129,6 +130,7 @@ public void testComplex() throws Exception { setup(COMPLEX_CYPHER); final Louvain algorithm = new Louvain(graph, Pools.DEFAULT, 1, AllocationTracker.EMPTY) .withProgressLogger(TestProgressLogger.INSTANCE) + .withTerminationFlag(TerminationFlag.RUNNING_TRUE) .compute(10, 10); final int[][] dendogram = algorithm.getDendrogram(); for (int i = 1; i <= dendogram.length; i++) { diff --git a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainTest1.java b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainTest1.java index f6e3b7d8e..054fb7cdb 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainTest1.java +++ b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainTest1.java @@ -25,6 +25,7 @@ import org.junit.rules.ErrorCollector; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.neo4j.graphalgo.LouvainProc; import org.neo4j.graphalgo.TestProgressLogger; import org.neo4j.graphalgo.api.Graph; import org.neo4j.graphalgo.api.GraphFactory; @@ -32,6 +33,7 @@ import org.neo4j.graphalgo.core.heavyweight.HeavyGraphFactory; import org.neo4j.graphalgo.core.huge.HugeGraphFactory; import org.neo4j.graphalgo.core.utils.Pools; +import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.core.utils.paged.HugeLongArray; import org.neo4j.graphalgo.impl.louvain.Louvain; @@ -132,6 +134,7 @@ public void testRunner() throws Exception { setup(unidirectional); final Louvain algorithm = new Louvain(graph, Pools.DEFAULT, 1, AllocationTracker.EMPTY) .withProgressLogger(TestProgressLogger.INSTANCE) + .withTerminationFlag(TerminationFlag.RUNNING_TRUE) .compute(10, 10); final int[][] dendogram = algorithm.getDendrogram(); for (int i = 1; i <= dendogram.length; i++) { diff --git a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainWeightedGraphTest.java b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainWeightedGraphTest.java index 19d387915..207023686 100644 --- a/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainWeightedGraphTest.java +++ b/tests/src/test/java/org/neo4j/graphalgo/impl/LouvainWeightedGraphTest.java @@ -31,6 +31,7 @@ import org.neo4j.graphalgo.core.heavyweight.HeavyGraphFactory; import org.neo4j.graphalgo.core.huge.HugeGraphFactory; import org.neo4j.graphalgo.core.utils.Pools; +import org.neo4j.graphalgo.core.utils.TerminationFlag; import org.neo4j.graphalgo.core.utils.paged.AllocationTracker; import org.neo4j.graphalgo.core.utils.paged.HugeLongArray; import org.neo4j.graphalgo.impl.louvain.*; @@ -148,6 +149,7 @@ public void testWeightedLouvain() throws Exception { final Louvain louvain = new Louvain(graph,Pools.DEFAULT, 1, AllocationTracker.EMPTY) .withProgressLogger(TestProgressLogger.INSTANCE) + .withTerminationFlag(TerminationFlag.RUNNING_TRUE) .compute(10, 10); final int[][] dendogram = louvain.getDendrogram();