Skip to content
This repository was archived by the owner on Apr 22, 2020. It is now read-only.

Docs unified community detection #807

Merged
merged 17 commits into from
Feb 5, 2019
Merged

Docs unified community detection #807

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
9298408
Louvain - unified write result fields
mneedham Feb 4, 2019
dbe65cd
LPA - unified write result fields
mneedham Feb 4, 2019
383cd78
add write properties to louvain output
mneedham Feb 4, 2019
362fcf4
uniform ordering of results
mneedham Feb 5, 2019
2baea24
pull 'write' up to the abstract class because all of them have that
mneedham Feb 5, 2019
ead026e
add write properties for triangle count
mneedham Feb 5, 2019
a3265dd
fix the percentiles for triangle count - the result turned by the fun…
mneedham Feb 5, 2019
5b03c4a
store write property for SCC
mneedham Feb 5, 2019
835c21c
tidy up louvain order
mneedham Feb 5, 2019
d6b30ae
add write properties to SCC docs
mneedham Feb 5, 2019
2339c7a
tidy up union find code
mneedham Feb 5, 2019
490825d
union find docs
mneedham Feb 5, 2019
d62839e
triangle count docs
mneedham Feb 5, 2019
f691ef3
update label prop algo
mneedham Feb 5, 2019
b80bb6b
lpa docs
mneedham Feb 5, 2019
71c5939
Balanced Triads - use a default weight if no weight property passed in
mneedham Feb 5, 2019
d72403e
Balanced Triads - compute percentiles for balanced triads
mneedham Feb 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 83 additions & 23 deletions algo/src/main/java/org/neo4j/graphalgo/BalancedTriadsProc.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
*/
package org.neo4j.graphalgo;

import com.carrotsearch.hppc.LongLongMap;
import org.HdrHistogram.Histogram;
import org.neo4j.graphalgo.api.HugeGraph;
import org.neo4j.graphalgo.core.GraphLoader;
import org.neo4j.graphalgo.core.ProcedureConfiguration;
Expand All @@ -26,7 +28,7 @@
import org.neo4j.graphalgo.core.utils.paged.PagedAtomicIntegerArray;
import org.neo4j.graphalgo.core.write.Exporter;
import org.neo4j.graphalgo.impl.triangle.*;
import org.neo4j.graphalgo.results.AbstractResultBuilder;
import org.neo4j.graphalgo.results.AbstractCommunityResultBuilder;
import org.neo4j.kernel.api.KernelTransaction;
import org.neo4j.kernel.internal.GraphDatabaseAPI;
import org.neo4j.logging.Log;
Expand Down Expand Up @@ -68,7 +70,7 @@ public Stream<HugeBalancedTriads.Result> balancedTriadsStream(
final HugeGraph graph = (HugeGraph) new GraphLoader(api, Pools.DEFAULT)
.withOptionalLabel(configuration.getNodeLabelOrQuery())
.withOptionalRelationshipType(configuration.getRelationshipOrQuery())
.withRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
.withOptionalRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
.withoutNodeWeights()
.withSort(true)
.withLog(log)
Expand Down Expand Up @@ -113,7 +115,7 @@ public Stream<Result> balancedTriads(
graph = (HugeGraph) new GraphLoader(api, Pools.DEFAULT)
.withOptionalLabel(configuration.getNodeLabelOrQuery())
.withOptionalRelationshipType(configuration.getRelationshipOrQuery())
.withRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
.withOptionalRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
.withoutNodeWeights()
.withSort(true)
.withLog(log)
Expand All @@ -132,26 +134,33 @@ public Stream<Result> balancedTriads(

// write
if (configuration.isWriteFlag()) {
builder.withWrite(true);

String balancedProperty = configuration.get("balancedProperty", DEFAULT_BALANCED_PROPERTY);
builder.withBalancedProperty(balancedProperty);

String unbalancedProperty = configuration.get("unbalancedProperty", DEFAULT_UNBALANCED_PROPERTY);
builder.withUnbalancedProperty(unbalancedProperty);

try (ProgressTimer timer = builder.timeWrite()) {
Exporter.of(api, graph)
.withLog(log)
.parallel(Pools.DEFAULT, configuration.getConcurrency(), terminationFlag)
.build()
.write(
configuration.get("balancedProperty", DEFAULT_BALANCED_PROPERTY),
balancedProperty,
balancedTriads.getBalancedTriangles(),
PagedAtomicIntegerArray.Translator.INSTANCE,
configuration.get("unbalancedProperty", DEFAULT_UNBALANCED_PROPERTY),
unbalancedProperty,
balancedTriads.getUnbalancedTriangles(),
PagedAtomicIntegerArray.Translator.INSTANCE);
}
}

// result
return Stream.of(builder.withNodeCount(graph.nodeCount())
.withBalancedTriadCount(balancedTriads.getBalancedTriangleCount())
.withUnbalancedTriadCount(balancedTriads.getUnbalancedTriangleCount())
.build());
builder.withBalancedTriadCount(balancedTriads.getBalancedTriangleCount())
.withUnbalancedTriadCount(balancedTriads.getUnbalancedTriangleCount());
return Stream.of(builder.buildII(graph.nodeCount(), balancedTriads.getBalancedTriangles()::get));
}

/**
Expand All @@ -162,54 +171,105 @@ public static class Result {
public final long loadMillis;
public final long computeMillis;
public final long writeMillis;
public final long postProcessingMillis;

public final long nodeCount;
public final long balancedTriadCount;
public final long unbalancedTriadCount;

public final long p1;
public final long p5;
public final long p10;
public final long p25;
public final long p50;
public final long p75;
public final long p90;
public final long p95;
public final long p99;
public final long p100;

public final boolean write;
public final String balancedProperty;
public final String unbalancedProperty;


public Result(
long loadMillis,
long computeMillis,
long writeMillis,
long nodeCount, long balancedTriadCount, long unbalancedTriadCount) {
long postProcessingMillis,
long nodeCount, long balancedTriadCount,
long unbalancedTriadCount,
long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p5, long p1, boolean write, String balancedProperty, String unbalancedProperty) {
this.loadMillis = loadMillis;
this.computeMillis = computeMillis;
this.writeMillis = writeMillis;
this.postProcessingMillis = postProcessingMillis;
this.nodeCount = nodeCount;
this.balancedTriadCount = balancedTriadCount;
this.unbalancedTriadCount = unbalancedTriadCount;
this.p100 = p100;
this.p99 = p99;
this.p95 = p95;
this.p90 = p90;
this.p75 = p75;
this.p50 = p50;
this.p25 = p25;
this.p10 = p10;
this.p5 = p5;
this.p1 = p1;
this.write = write;
this.balancedProperty = balancedProperty;
this.unbalancedProperty = unbalancedProperty;
}
}

public class BalancedTriadsResultBuilder extends AbstractResultBuilder<Result> {
public class BalancedTriadsResultBuilder extends AbstractCommunityResultBuilder<Result> {

private long nodeCount = 0;
private long balancedTriadCount = 0;
private long unbalancedTriadCount = 0;
private String balancedProperty;
private String unbalancedProperty;

public BalancedTriadsResultBuilder withNodeCount(long nodeCount) {
this.nodeCount = nodeCount;
return this;
}

public BalancedTriadsResultBuilder withBalancedTriadCount(long balancedTriadCount) {
this.balancedTriadCount = balancedTriadCount;
return this;
}

public BalancedTriadsResultBuilder withBalancedProperty(String property) {
this.balancedProperty = property;
return this;
}

public BalancedTriadsResultBuilder withUnbalancedProperty(String property) {
this.unbalancedProperty = property;
return this;
}

public BalancedTriadsResultBuilder withUnbalancedTriadCount(long unbalancedTriadCount) {
this.unbalancedTriadCount = unbalancedTriadCount;
return this;
}

@Override
public Result build() {
protected Result build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, boolean write) {
return new Result(
loadDuration,
evalDuration,
writeDuration,
nodeCount,
balancedTriadCount,
unbalancedTriadCount);
loadMillis, computeMillis, writeMillis, postProcessingMillis, nodeCount, balancedTriadCount, unbalancedTriadCount,
communityHistogram.getValueAtPercentile(100),
communityHistogram.getValueAtPercentile(99),
communityHistogram.getValueAtPercentile(95),
communityHistogram.getValueAtPercentile(90),
communityHistogram.getValueAtPercentile(75),
communityHistogram.getValueAtPercentile(50),
communityHistogram.getValueAtPercentile(25),
communityHistogram.getValueAtPercentile(10),
communityHistogram.getValueAtPercentile(5),
communityHistogram.getValueAtPercentile(1),
write,
balancedProperty,
unbalancedProperty
);
}
}

Expand Down
2 changes: 1 addition & 1 deletion algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ public Stream<LabelPropagationStats> labelPropagation(

final int[] labels = compute(direction, iterations, batchSize, concurrency, graph, stats);
if (configuration.isWriteFlag(DEFAULT_WRITE) && partitionProperty != null) {
stats.withWrite(true);
write(concurrency, partitionProperty, graph, labels, stats);
}

Expand Down Expand Up @@ -218,7 +219,6 @@ private void write(
HeavyGraph graph,
int[] labels,
LabelPropagationStats.Builder stats) {
stats.write(true);
try (ProgressTimer timer = stats.timeWrite()) {
Exporter.of(dbAPI, graph)
.withLog(log)
Expand Down
87 changes: 64 additions & 23 deletions algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,18 @@ public Stream<LouvainResult> louvain(
}

if (configuration.isWriteFlag()) {
builder.timeWrite(() -> write(graph, louvain.getDendrogram(), louvain.getCommunityIds(), configuration));
builder.timeWrite(() -> {
String writeProperty = configuration.getWriteProperty("community");
boolean includeIntermediateCommunities = configuration.get(INCLUDE_INTERMEDIATE_COMMUNITIES, false);
String intermediateCommunitiesWriteProperty = configuration.get(INTERMEDIATE_COMMUNITIES_WRITE_PROPERTY, "communities");

builder.withWrite(true);
builder.withWriteProperty(writeProperty);
builder.withIntermediateCommunities(includeIntermediateCommunities);
builder.withIntermediateCommunitiesWriteProperty(intermediateCommunitiesWriteProperty);

write(graph, louvain.getDendrogram(), louvain.getCommunityIds(), configuration, writeProperty, includeIntermediateCommunities, intermediateCommunitiesWriteProperty);
});
}

builder.withIterations(louvain.getLevel());
Expand Down Expand Up @@ -170,18 +181,17 @@ public Graph graph(String label, String relationship, ProcedureConfiguration con
.load(config.getGraphImpl());
}

private void write(Graph graph, int[][] allCommunities, int[] finalCommunities, ProcedureConfiguration configuration) {
private void write(Graph graph, int[][] allCommunities, int[] finalCommunities, ProcedureConfiguration configuration, String writeProperty, boolean includeIntermediateCommunities, String intermediateCommunitiesPropertyName) {
log.debug("Writing results");
boolean includeIntermediateCommunities = configuration.get(INCLUDE_INTERMEDIATE_COMMUNITIES, false);

new LouvainCommunityExporter(
api,
Pools.DEFAULT,
configuration.getConcurrency(),
graph,
finalCommunities.length,
configuration.getWriteProperty("community"),
configuration.get(INTERMEDIATE_COMMUNITIES_WRITE_PROPERTY, "communities"))
writeProperty,
intermediateCommunitiesPropertyName)
.export(allCommunities, finalCommunities, includeIntermediateCommunities);
}

Expand All @@ -205,29 +215,37 @@ public static class LouvainResult {
-1,
-1,
0,
new double[] {}, -1);
new double[] {}, -1, false, null, false, null);

public final long loadMillis;
public final long computeMillis;
public final long postProcessingMillis;
public final long writeMillis;
public final long postProcessingMillis;
public final long nodes;
public final long communityCount;
public final long p100;
public final long p99;
public final long p95;
public final long p90;
public final long p75;
public final long p50;
public final long p25;
public final long p10;
public final long p05;
public final long p01;
public final long iterations;
public final List<Double> modularities;
public final double modularity;

public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations, double[] modularities, double finalModularity) {
public final long p1;
public final long p5;
public final long p10;
public final long p25;
public final long p50;
public final long p75;
public final long p90;
public final long p95;
public final long p99;
public final long p100;
public final boolean write;
public final String writeProperty;
public final boolean includeIntermediateCommunities;
public final String intermediateCommunitiesWriteProperty;

public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes,
long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p5, long p1,
long iterations, double[] modularities, double finalModularity,
boolean write, String writeProperty,
boolean includeIntermediateCommunities, String intermediateCommunitiesWriteProperty) {
this.loadMillis = loadMillis;
this.computeMillis = computeMillis;
this.postProcessingMillis = postProcessingMillis;
Expand All @@ -242,12 +260,16 @@ public LouvainResult(long loadMillis, long computeMillis, long postProcessingMil
this.p50 = p50;
this.p25 = p25;
this.p10 = p10;
this.p05 = p05;
this.p01 = p01;
this.p5 = p5;
this.p1 = p1;
this.iterations = iterations;
this.modularities = new ArrayList<>(modularities.length);
this.write = write;
this.includeIntermediateCommunities = includeIntermediateCommunities;
for (double mod : modularities) this.modularities.add(mod);
this.modularity = finalModularity;
this.writeProperty = writeProperty;
this.intermediateCommunitiesWriteProperty = intermediateCommunitiesWriteProperty;
}
}

Expand All @@ -256,14 +278,22 @@ public static class Builder extends AbstractCommunityResultBuilder<LouvainResult
private long iterations = -1;
private double[] modularities = new double[] {};
private double finalModularity = -1;
private String writeProperty;
private String intermediateCommunitiesWriteProperty;
private boolean includeIntermediateCommunities;

public Builder withWriteProperty(String writeProperty) {
this.writeProperty = writeProperty;
return this;
}

public Builder withIterations(long iterations) {
this.iterations = iterations;
return this;
}

@Override
protected LouvainResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) {
protected LouvainResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, boolean write) {
return new LouvainResult(
loadMillis,
computeMillis,
Expand All @@ -281,7 +311,8 @@ protected LouvainResult build(long loadMillis, long computeMillis, long writeMil
communityHistogram.getValueAtPercentile(10),
communityHistogram.getValueAtPercentile(5),
communityHistogram.getValueAtPercentile(1),
iterations, modularities, finalModularity
iterations, modularities, finalModularity,
write, writeProperty, includeIntermediateCommunities, intermediateCommunitiesWriteProperty
);
}

Expand All @@ -294,6 +325,16 @@ public Builder withFinalModularity(double finalModularity) {
this.finalModularity = finalModularity;
return null;
}

public Builder withIntermediateCommunitiesWriteProperty(String intermediateCommunitiesWriteProperty) {
this.intermediateCommunitiesWriteProperty = intermediateCommunitiesWriteProperty;
return null;
}

public Builder withIntermediateCommunities(boolean includeIntermediateCommunities) {
this.includeIntermediateCommunities = includeIntermediateCommunities;
return this;
}
}


Expand Down
Loading