Skip to content

Commit

Permalink
Replace Elasticsearch facets with aggregations
Browse files Browse the repository at this point in the history
  • Loading branch information
Jochen Schalanda committed Mar 11, 2015
1 parent 28094cb commit a58d06c
Show file tree
Hide file tree
Showing 8 changed files with 222 additions and 171 deletions.
43 changes: 19 additions & 24 deletions graylog2-server/src/main/java/org/graylog2/indexer/results/DateHistogramResult.java
View file
Expand Up @@ -19,40 +19,35 @@
import com.google.common.collect.Maps;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.facet.datehistogram.DateHistogramFacet;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram;
import org.graylog2.indexer.searches.Searches;

import java.util.Map;

/**
* @author Lennart Koopmann <lennart@socketfeed.com>
*/
public class DateHistogramResult extends HistogramResult {

private final DateHistogramFacet result;
private final Searches.DateHistogramInterval interval;
private final DateHistogram result;
private final Searches.DateHistogramInterval interval;

public DateHistogramResult(DateHistogramFacet result, String originalQuery, BytesReference builtQuery, Searches.DateHistogramInterval interval, TimeValue took) {
public DateHistogramResult(DateHistogram result, String originalQuery, BytesReference builtQuery, Searches.DateHistogramInterval interval, TimeValue took) {
super(originalQuery, builtQuery, took);

this.result = result;
this.interval = interval;
}
this.result = result;
this.interval = interval;
}

@Override
public Searches.DateHistogramInterval getInterval() {
return interval;
}
public Searches.DateHistogramInterval getInterval() {
return interval;
}

@Override
public Map<Long, Long> getResults() {
Map<Long, Long> results = Maps.newTreeMap();

for (DateHistogramFacet.Entry e : result) {
results.put(e.getTime()/1000, e.getCount());
}

return results;
}

public Map<Long, Long> getResults() {
Map<Long, Long> results = Maps.newTreeMap();

for (DateHistogram.Bucket bucket : result.getBuckets()) {
results.put(bucket.getKeyAsDate().getMillis() / 1000L, bucket.getDocCount());
}

return results;
}
}
68 changes: 35 additions & 33 deletions graylog2-server/src/main/java/org/graylog2/indexer/results/FieldHistogramResult.java
View file
Expand Up @@ -16,23 +16,30 @@
*/
package org.graylog2.indexer.results;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.facet.datehistogram.DateHistogramFacet;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram;
import org.elasticsearch.search.aggregations.metrics.stats.Stats;
import org.graylog2.indexer.searches.Searches;

import java.util.Map;

/**
* @author Lennart Koopmann <lennart@torch.sh>
*/
public class FieldHistogramResult extends HistogramResult {
private static final Map<String, Number> EMPTY_RESULT = ImmutableMap.<String, Number>builder()
.put("count", 0)
.put("min", 0)
.put("max", 0)
.put("total", 0)
.put("total_count", 0)
.put("mean", 0)
.build();

private final DateHistogramFacet result;
private final DateHistogram result;
private final Searches.DateHistogramInterval interval;

public FieldHistogramResult(DateHistogramFacet result, String originalQuery, BytesReference builtQuery, Searches.DateHistogramInterval interval, TimeValue took) {
public FieldHistogramResult(DateHistogram result, String originalQuery, BytesReference builtQuery, Searches.DateHistogramInterval interval, TimeValue took) {
super(originalQuery, builtQuery, took);

this.result = result;
Expand All @@ -43,45 +50,40 @@ public Searches.DateHistogramInterval getInterval() {
return interval;
}

public Map<Long, Map<String, Object>> getResults() {
Map<Long, Map<String, Object>> results = Maps.newTreeMap();
Long minTimestamp = Long.MAX_VALUE;
Long maxTimestamp = Long.MIN_VALUE;
for (DateHistogramFacet.Entry e : result) {
Map<String, Object> resultMap = Maps.newHashMap();

resultMap.put("count", e.getCount());
resultMap.put("min", e.getMin());
resultMap.put("max", e.getMax());
resultMap.put("total", e.getTotal());
resultMap.put("total_count", e.getTotalCount());
resultMap.put("mean", e.getMean());

final long timestamp = e.getTime() / 1000;
public Map<Long, Map<String, Number>> getResults() {
Map<Long, Map<String, Number>> results = Maps.newTreeMap();
long minTimestamp = Long.MAX_VALUE;
long maxTimestamp = Long.MIN_VALUE;

for (DateHistogram.Bucket b : result.getBuckets()) {
Map<String, Number> resultMap = Maps.newHashMap();

resultMap.put("total_count", b.getDocCount());

Stats stats = b.getAggregations().get(Searches.AGG_STATS);
resultMap.put("count", stats.getCount());
resultMap.put("min", stats.getMin());
resultMap.put("max", stats.getMax());
resultMap.put("total", stats.getSum());
resultMap.put("mean", stats.getAvg());

final long timestamp = b.getKeyAsDate().getMillis() / 1000L;
if (timestamp < minTimestamp) minTimestamp = timestamp;
if (timestamp > maxTimestamp) maxTimestamp = timestamp;

results.put(timestamp, resultMap);
}
long curTimestamp = minTimestamp;
while (curTimestamp < maxTimestamp) {
Map<String, Object> entry = results.get(curTimestamp);
Map<String, Number> entry = results.get(curTimestamp);

// advance timestamp by the interval's seconds value
curTimestamp += interval.getPeriod().toStandardSeconds().getSeconds();

if (entry != null) {
continue;
if (entry == null) {
// synthesize a 0 value for this timestamp
results.put(curTimestamp, EMPTY_RESULT);
}
// synthesize a 0 value for this timestamp
entry = Maps.newHashMap();
entry.put("count", 0);
entry.put("min", 0);
entry.put("max", 0);
entry.put("total", 0);
entry.put("total_count", 0);
entry.put("mean", 0);
results.put(curTimestamp, entry);
}
return results;
}
Expand Down
13 changes: 5 additions & 8 deletions graylog2-server/src/main/java/org/graylog2/indexer/results/FieldStatsResult.java
View file
Expand Up @@ -19,13 +19,10 @@
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.facet.statistical.StatisticalFacet;
import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats;

import java.util.List;

/**
* @author Lennart Koopmann <lennart@torch.sh>
*/
public class FieldStatsResult extends IndexQueryResult {

private final long count;
Expand All @@ -38,20 +35,20 @@ public class FieldStatsResult extends IndexQueryResult {
private final double stdDeviation;
private List<ResultMessage> searchHits;

public FieldStatsResult(StatisticalFacet f, String originalQuery, BytesReference builtQuery, TimeValue took) {
public FieldStatsResult(ExtendedStats f, String originalQuery, BytesReference builtQuery, TimeValue took) {
super(originalQuery, builtQuery, took);

this.count = f.getCount();
this.sum = f.getTotal();
this.sum = f.getSum();
this.sumOfSquares = f.getSumOfSquares();
this.mean = f.getMean();
this.mean = f.getAvg();
this.min = f.getMin();
this.max = f.getMax();
this.variance = f.getVariance();
this.stdDeviation = f.getStdDeviation();
}

public FieldStatsResult(StatisticalFacet facet, SearchHits searchHits, String query, BytesReference source, TimeValue took) {
public FieldStatsResult(ExtendedStats facet, SearchHits searchHits, String query, BytesReference source, TimeValue took) {
this(facet, query, source, took);
this.searchHits = buildResults(searchHits);
}
Expand Down
29 changes: 13 additions & 16 deletions graylog2-server/src/main/java/org/graylog2/indexer/results/TermsResult.java
View file
Expand Up @@ -19,36 +19,33 @@
import com.google.common.collect.Maps;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.facet.terms.TermsFacet;
import org.elasticsearch.search.aggregations.bucket.missing.Missing;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;

import java.util.List;
import java.util.Map;

/**
* @author Lennart Koopmann <lennart@torch.sh>
*/
public class TermsResult extends IndexQueryResult {

private final long total;
private final long missing;
private final long other;
private final Map<String, Integer> terms;
private final Map<String, Long> terms;

public TermsResult(TermsFacet f, String originalQuery, BytesReference builtQuery, TimeValue took) {
public TermsResult(Terms f, Missing m, long totalCount, String originalQuery, BytesReference builtQuery, TimeValue took) {
super(originalQuery, builtQuery, took);

this.total = f.getTotalCount();
this.missing = f.getMissingCount();
this.other = f.getOtherCount();

this.terms = buildTermsMap(f.getEntries());
this.total = totalCount;
this.missing = m.getDocCount();
this.other = f.getSumOfOtherDocCounts();
this.terms = buildTermsMap(f.getBuckets());
}

private Map<String, Integer> buildTermsMap(List<? extends TermsFacet.Entry> entries) {
Map<String, Integer> terms = Maps.newHashMap();
private Map<String, Long> buildTermsMap(List<Terms.Bucket> entries) {
Map<String, Long> terms = Maps.newHashMap();

for(TermsFacet.Entry term : entries) {
terms.put(term.getTerm().string(), term.getCount());
for(Terms.Bucket bucket : entries) {
terms.put(bucket.getKey(), bucket.getDocCount());
}

return terms;
Expand All @@ -66,7 +63,7 @@ public long getOther() {
return other;
}

public Map<String, Integer> getTerms() {
public Map<String, Long> getTerms() {
return terms;
}

Expand Down
58 changes: 29 additions & 29 deletions graylog2-server/src/main/java/org/graylog2/indexer/results/TermsStatsResult.java
View file
Expand Up @@ -20,21 +20,32 @@
import com.google.common.collect.Maps;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.facet.termsstats.TermsStatsFacet;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.stats.Stats;
import org.graylog2.indexer.searches.Searches;

import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;

/**
* @author Lennart Koopmann <lennart@torch.sh>
*/
public class TermsStatsResult extends IndexQueryResult {
private static final Comparator<Map<String, Object>> COMPARATOR = new Comparator<Map<String, Object>>() {
@Override
public int compare(Map<String, Object> o1, Map<String, Object> o2) {
double o1Mean = (double) o1.get("mean");
double o2Mean = (double) o2.get("mean");
if (o1Mean > o2Mean) {
return -1;
} else if (o1Mean < o2Mean) {
return 1;
}
return 0;
}
};
private final Terms facet;

private final TermsStatsFacet facet;

public TermsStatsResult(TermsStatsFacet facet, String originalQuery, BytesReference builtQuery, TimeValue took) {
public TermsStatsResult(Terms facet, String originalQuery, BytesReference builtQuery, TimeValue took) {
super(originalQuery, builtQuery, took);

this.facet = facet;
Expand All @@ -43,37 +54,26 @@ public TermsStatsResult(TermsStatsFacet facet, String originalQuery, BytesRefere
public List<Map<String, Object>> getResults() {
List<Map<String, Object>> results = Lists.newArrayList();

for (TermsStatsFacet.Entry e : facet.getEntries()) {
for (Terms.Bucket e : facet.getBuckets()) {
Map<String, Object> resultMap = Maps.newHashMap();

resultMap.put("key_field", e.getTerm().toString());
resultMap.put("key_field", e.getKey());

resultMap.put("count", e.getDocCount());

resultMap.put("count", e.getCount());
resultMap.put("min", e.getMin());
resultMap.put("max", e.getMax());
resultMap.put("total", e.getTotal());
resultMap.put("total_count", e.getTotalCount());
resultMap.put("mean", e.getMean());
final Stats stats = e.getAggregations().get(Searches.AGG_STATS);
resultMap.put("min", stats.getMin());
resultMap.put("max", stats.getMax());
resultMap.put("total", stats.getSum());
resultMap.put("total_count", stats.getCount());
resultMap.put("mean", stats.getAvg());

results.add(resultMap);
}

// Sort results by descending mean value
Collections.sort(results, new Comparator<Map<String, Object>>() {
@Override
public int compare(Map<String, Object> o1, Map<String, Object> o2) {
double o1Mean = (double)o1.get("mean");
double o2Mean = (double)o2.get("mean");
if (o1Mean > o2Mean) {
return -1;
} else if (o1Mean < o2Mean) {
return 1;
}
return 0;
}
});
Collections.sort(results, COMPARATOR);

return results;
}

}

0 comments on commit a58d06c

Please sign in to comment.