Skip to content

Commit

Permalink
SONAR-6137 Apply feedback on PR (refactoring of facet processing, tes…
Browse files Browse the repository at this point in the history
…ts split)
  • Loading branch information
jblievremont committed Feb 4, 2015
1 parent f45419c commit 59b15fc
Show file tree
Hide file tree
Showing 6 changed files with 369 additions and 101 deletions.
Expand Up @@ -47,7 +47,11 @@ public boolean equals(Object o) {
}

FacetValue that = (FacetValue) o;
return key == null ? that.key == null : key.equals(that.key);
if (key == null) {
return that.key == null;
} else {
return key.equals(that.key);
}
}

@Override
Expand All @@ -58,8 +62,8 @@ public int hashCode() {
@Override
public String toString() {
return "FacetValue{" +
"key='" + key + '\'' +
", value=" + value +
"key='" + getKey() + '\'' +
", value=" + getValue() +
'}';
}
}
126 changes: 126 additions & 0 deletions server/sonar-server/src/main/java/org/sonar/server/search/Facets.java
@@ -0,0 +1,126 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.search;

import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Multimap;
import org.apache.commons.lang.builder.ReflectionToStringBuilder;
import org.apache.commons.lang.builder.ToStringStyle;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.HasAggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram;
import org.elasticsearch.search.aggregations.bucket.missing.Missing;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;

class Facets {

private static final Logger LOGGER = LoggerFactory.getLogger(Facets.class);

private final Multimap<String, FacetValue> facetValues;

public Facets(SearchResponse response) {
facetValues = LinkedHashMultimap.create();

if (response.getAggregations() != null) {
for (Aggregation facet : response.getAggregations()) {
this.processAggregation(facet);
}
}
}

private void processAggregation(Aggregation aggregation) {
if (Missing.class.isAssignableFrom(aggregation.getClass())) {
processMissingAggregation(aggregation);
} else if (Terms.class.isAssignableFrom(aggregation.getClass())) {
processTermsAggregation(aggregation);
} else if (HasAggregations.class.isAssignableFrom(aggregation.getClass())) {
processSubAggregations(aggregation);
} else if (DateHistogram.class.isAssignableFrom(aggregation.getClass())) {
processDateHistogram(aggregation);
} else {
LOGGER.warn("Cannot process {} type of aggregation", aggregation.getClass());
}
}

private void processMissingAggregation(Aggregation aggregation) {
Missing missing = (Missing) aggregation;
long docCount = missing.getDocCount();
if (docCount > 0L) {
this.facetValues.put(aggregation.getName().replace("_missing", ""), new FacetValue("", docCount));
}
}

private void processTermsAggregation(Aggregation aggregation) {
Terms termAggregation = (Terms) aggregation;
for (Terms.Bucket value : termAggregation.getBuckets()) {
String facetName = aggregation.getName();
if (facetName.contains("__") && !facetName.startsWith("__")) {
facetName = facetName.substring(0, facetName.indexOf("__"));
}
facetName = facetName.replace("_selected", "");
this.facetValues.put(facetName, new FacetValue(value.getKey(), value.getDocCount()));
}
}

private void processSubAggregations(Aggregation aggregation) {
HasAggregations hasAggregations = (HasAggregations) aggregation;
for (Aggregation internalAggregation : hasAggregations.getAggregations()) {
this.processAggregation(internalAggregation);
}
}

private void processDateHistogram(Aggregation aggregation) {
DateHistogram dateHistogram = (DateHistogram) aggregation;
for (DateHistogram.Bucket value : dateHistogram.getBuckets()) {
this.facetValues.put(dateHistogram.getName(), new FacetValue(value.getKeyAsText().toString(), value.getDocCount()));
}
}

public Map<String, Collection<FacetValue>> getFacets() {
return this.facetValues.asMap();
}

public Collection<FacetValue> getFacetValues(String facetName) {
return this.facetValues.get(facetName);
}

public List<String> getFacetKeys(String facetName) {
List<String> keys = new ArrayList<String>();
if (this.facetValues.containsKey(facetName)) {
for (FacetValue facetValue : facetValues.get(facetName)) {
keys.add(facetValue.getKey());
}
}
return keys;
}

@Override
public String toString() {
return ReflectionToStringBuilder.toString(this, ToStringStyle.SIMPLE_STYLE);
}
}
Expand Up @@ -20,18 +20,9 @@
package org.sonar.server.search;

import com.google.common.base.Preconditions;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Multimap;
import org.apache.commons.lang.builder.ReflectionToStringBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.HasAggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram;
import org.elasticsearch.search.aggregations.bucket.missing.Missing;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
Expand All @@ -40,10 +31,8 @@

public class Result<K> {

private static final Logger LOGGER = LoggerFactory.getLogger(Result.class);

private final List<K> hits;
private final Multimap<String, FacetValue> facets;
private final Facets facets;
private final long total;
private final String scrollId;
private final BaseIndex<K, ?, ?> index;
Expand All @@ -55,67 +44,14 @@ public Result(SearchResponse response) {
public Result(@Nullable BaseIndex<K, ?, ?> index, SearchResponse response) {
this.index = index;
this.scrollId = response.getScrollId();
this.facets = LinkedHashMultimap.create();
this.facets = new Facets(response);
this.total = (int) response.getHits().totalHits();
this.hits = new ArrayList<K>();
if (index != null) {
for (SearchHit hit : response.getHits()) {
this.hits.add(index.toDoc(hit.getSource()));
}
}
if (response.getAggregations() != null) {
for (Map.Entry<String, Aggregation> facet : response.getAggregations().asMap().entrySet()) {
this.processAggregation(facet.getValue());
}
}
}

private void processAggregation(Aggregation aggregation) {
if (Missing.class.isAssignableFrom(aggregation.getClass())) {
processMissingAggregation(aggregation);
} else if (Terms.class.isAssignableFrom(aggregation.getClass())) {
processTermsAggregation(aggregation);
} else if (HasAggregations.class.isAssignableFrom(aggregation.getClass())) {
processSubAggregations(aggregation);
} else if (DateHistogram.class.isAssignableFrom(aggregation.getClass())) {
processDateHistogram(aggregation);
} else {
LOGGER.warn("Cannot process {} type of aggregation", aggregation.getClass());
}
}

private void processMissingAggregation(Aggregation aggregation) {
Missing missing = (Missing) aggregation;
long docCount = missing.getDocCount();
if (docCount > 0L) {
this.facets.put(aggregation.getName().replace("_missing",""), new FacetValue("", docCount));
}
}

private void processTermsAggregation(Aggregation aggregation) {
Terms termAggregation = (Terms) aggregation;
for (Terms.Bucket value : termAggregation.getBuckets()) {
String facetName = aggregation.getName();
if (facetName.contains("__") && !facetName.startsWith("__")) {
facetName = facetName.substring(0, facetName.indexOf("__"));
}
facetName = facetName.replace("_selected", "");
this.facets.put(facetName, new FacetValue(value.getKey(), value.getDocCount()));
}
}

private void processSubAggregations(Aggregation aggregation) {
HasAggregations hasAggregations = (HasAggregations) aggregation;
for (Aggregation internalAggregation : hasAggregations.getAggregations()) {
this.processAggregation(internalAggregation);
}
}

private void processDateHistogram(Aggregation aggregation) {
DateHistogram dateHistogram = (DateHistogram) aggregation;
for (DateHistogram.Bucket value : dateHistogram.getBuckets()) {
this.facets.put(dateHistogram.getName(), new FacetValue(value.getKeyAsText().toString(), value.getDocCount()));
}
}

public Iterator<K> scroll() {
Expand All @@ -132,24 +68,17 @@ public long getTotal() {
}

public Map<String, Collection<FacetValue>> getFacets() {
return this.facets.asMap();
return this.facets.getFacets();
}

@CheckForNull
public Collection<FacetValue> getFacetValues(String facetName) {
return this.facets.get(facetName);
return this.facets.getFacetValues(facetName);
}

@CheckForNull
public List<String> getFacetKeys(String facetName) {
if (this.facets.containsKey(facetName)) {
List<String> keys = new ArrayList<String>();
for (FacetValue facetValue : facets.get(facetName)) {
keys.add(facetValue.getKey());
}
return keys;
}
return null;
return this.facets.getFacetKeys(facetName);
}

@Override
Expand Down
Expand Up @@ -652,24 +652,9 @@ public void filter_by_created_at() throws Exception {
}

@Test
public void facet_on_created_at() throws Exception {
public void facet_on_created_at_with_less_than_20_days() throws Exception {

ComponentDto project = ComponentTesting.newProjectDto();
ComponentDto file = ComponentTesting.newFileDto(project);

TimeZone.setDefault(TimeZone.getTimeZone("GMT+04:00"));

IssueDoc issue0 = IssueTesting.newDoc("ISSUE0", file).setFuncCreationDate(DateUtils.parseDateTime("2011-04-25T01:05:13+0400"));
IssueDoc issue1 = IssueTesting.newDoc("ISSUE1", file).setFuncCreationDate(DateUtils.parseDateTime("2014-09-01T12:34:56+0400"));
IssueDoc issue2 = IssueTesting.newDoc("ISSUE2", file).setFuncCreationDate(DateUtils.parseDateTime("2014-09-01T23:45:60+0400"));
IssueDoc issue3 = IssueTesting.newDoc("ISSUE3", file).setFuncCreationDate(DateUtils.parseDateTime("2014-09-02T12:34:56+0400"));
IssueDoc issue4 = IssueTesting.newDoc("ISSUE4", file).setFuncCreationDate(DateUtils.parseDateTime("2014-09-05T12:34:56+0400"));
IssueDoc issue5 = IssueTesting.newDoc("ISSUE5", file).setFuncCreationDate(DateUtils.parseDateTime("2014-09-20T12:34:56+0400"));
IssueDoc issue6 = IssueTesting.newDoc("ISSUE6", file).setFuncCreationDate(DateUtils.parseDateTime("2015-01-18T12:34:56+0400"));

indexIssues(issue0, issue1, issue2, issue3, issue4, issue5, issue6);

QueryContext queryContext = new QueryContext().addFacets(Arrays.asList("createdAt"));
QueryContext queryContext = fixtureForCreatedAtFacet();

Collection<FacetValue> createdAt = index.search(IssueQuery.builder().createdAfter(DateUtils.parseDate("2014-09-01")).createdBefore(DateUtils.parseDate("2014-09-08")).build(),
queryContext).getFacets().get("createdAt");
Expand All @@ -680,16 +665,28 @@ public void facet_on_created_at() throws Exception {
new FacetValue("2014-09-03T04:00:00+0000", 0),
new FacetValue("2014-09-04T04:00:00+0000", 0),
new FacetValue("2014-09-05T04:00:00+0000", 1));
}

@Test
public void facet_on_created_at_with_less_than_20_weeks() throws Exception {

QueryContext queryContext = fixtureForCreatedAtFacet();

createdAt = index.search(IssueQuery.builder().createdAfter(DateUtils.parseDate("2014-09-01")).createdBefore(DateUtils.parseDate("2014-09-21")).build(),
Collection<FacetValue> createdAt = index.search(IssueQuery.builder().createdAfter(DateUtils.parseDate("2014-09-01")).createdBefore(DateUtils.parseDate("2014-09-21")).build(),
queryContext).getFacets().get("createdAt");
assertThat(createdAt).hasSize(3)
.containsOnly(
new FacetValue("2014-09-01T04:00:00+0000", 4),
new FacetValue("2014-09-08T04:00:00+0000", 0),
new FacetValue("2014-09-15T04:00:00+0000", 1));
}

@Test
public void facet_on_created_at_with_less_than_20_months() throws Exception {

createdAt = index.search(IssueQuery.builder().createdAfter(DateUtils.parseDate("2014-09-01")).createdBefore(DateUtils.parseDate("2015-01-19")).build(),
QueryContext queryContext = fixtureForCreatedAtFacet();

Collection<FacetValue> createdAt = index.search(IssueQuery.builder().createdAfter(DateUtils.parseDate("2014-09-01")).createdBefore(DateUtils.parseDate("2015-01-19")).build(),
queryContext).getFacets().get("createdAt");
assertThat(createdAt).hasSize(5)
.containsOnly(
Expand All @@ -698,8 +695,14 @@ public void facet_on_created_at() throws Exception {
new FacetValue("2014-11-01T04:00:00+0000", 0),
new FacetValue("2014-12-01T04:00:00+0000", 0),
new FacetValue("2015-01-01T04:00:00+0000", 1));
}

createdAt = index.search(IssueQuery.builder().createdAfter(DateUtils.parseDate("2011-01-01")).createdBefore(DateUtils.parseDate("2016-01-01")).build(),
@Test
public void facet_on_created_at_with_more_than_20_months() throws Exception {

QueryContext queryContext = fixtureForCreatedAtFacet();

Collection<FacetValue> createdAt = index.search(IssueQuery.builder().createdAfter(DateUtils.parseDate("2011-01-01")).createdBefore(DateUtils.parseDate("2016-01-01")).build(),
queryContext).getFacets().get("createdAt");
assertThat(createdAt).hasSize(5)
.containsOnly(
Expand All @@ -709,8 +712,14 @@ public void facet_on_created_at() throws Exception {
new FacetValue("2014-01-01T04:00:00+0000", 5),
new FacetValue("2015-01-01T04:00:00+0000", 1));

// createdAfter not set: taking min value
createdAt = index.search(IssueQuery.builder().createdBefore(DateUtils.parseDate("2016-01-01")).build(),
}

@Test
public void facet_on_created_at_without_start_bound() throws Exception {

QueryContext queryContext = fixtureForCreatedAtFacet();

Collection<FacetValue> createdAt = index.search(IssueQuery.builder().createdBefore(DateUtils.parseDate("2016-01-01")).build(),
queryContext).getFacets().get("createdAt");
assertThat(createdAt).hasSize(5)
.containsOnly(
Expand All @@ -721,6 +730,26 @@ public void facet_on_created_at() throws Exception {
new FacetValue("2015-01-01T04:00:00+0000", 1));
}

protected QueryContext fixtureForCreatedAtFacet() {
ComponentDto project = ComponentTesting.newProjectDto();
ComponentDto file = ComponentTesting.newFileDto(project);

TimeZone.setDefault(TimeZone.getTimeZone("GMT+04:00"));

IssueDoc issue0 = IssueTesting.newDoc("ISSUE0", file).setFuncCreationDate(DateUtils.parseDateTime("2011-04-25T01:05:13+0400"));
IssueDoc issue1 = IssueTesting.newDoc("ISSUE1", file).setFuncCreationDate(DateUtils.parseDateTime("2014-09-01T12:34:56+0400"));
IssueDoc issue2 = IssueTesting.newDoc("ISSUE2", file).setFuncCreationDate(DateUtils.parseDateTime("2014-09-01T23:45:60+0400"));
IssueDoc issue3 = IssueTesting.newDoc("ISSUE3", file).setFuncCreationDate(DateUtils.parseDateTime("2014-09-02T12:34:56+0400"));
IssueDoc issue4 = IssueTesting.newDoc("ISSUE4", file).setFuncCreationDate(DateUtils.parseDateTime("2014-09-05T12:34:56+0400"));
IssueDoc issue5 = IssueTesting.newDoc("ISSUE5", file).setFuncCreationDate(DateUtils.parseDateTime("2014-09-20T12:34:56+0400"));
IssueDoc issue6 = IssueTesting.newDoc("ISSUE6", file).setFuncCreationDate(DateUtils.parseDateTime("2015-01-18T12:34:56+0400"));

indexIssues(issue0, issue1, issue2, issue3, issue4, issue5, issue6);

QueryContext queryContext = new QueryContext().addFacets(Arrays.asList("createdAt"));
return queryContext;
}

@Test
public void paging() throws Exception {
ComponentDto project = ComponentTesting.newProjectDto();
Expand Down

0 comments on commit 59b15fc

Please sign in to comment.