Skip to content

Commit

Permalink
SONAR-6078 Aggregate technical debt on issues search WS
Browse files Browse the repository at this point in the history
  • Loading branch information
@jblievremont
jblievremont committed Jun 25, 2015
1 parent a77b83f commit 8f7068c
Show file tree
Hide file tree
Showing 10 changed files with 602 additions and 38 deletions.
39 changes: 30 additions & 9 deletions server/sonar-server/src/main/java/org/sonar/server/es/Facets.java
View file
Expand Up @@ -19,6 +19,11 @@
*/
package org.sonar.server.es;

import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import javax.annotation.CheckForNull;
import org.apache.commons.lang.builder.ReflectionToStringBuilder;
import org.apache.commons.lang.builder.ToStringStyle;
import org.elasticsearch.action.search.SearchResponse;
Expand All @@ -27,15 +32,12 @@
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram;
import org.elasticsearch.search.aggregations.bucket.missing.Missing;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;

import javax.annotation.CheckForNull;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;

public class Facets {

public static final String TOTAL = "total";

private final Map<String, LinkedHashMap<String, Long>> facetsByName = new LinkedHashMap<>();

public Facets(SearchResponse response) {
Expand All @@ -59,6 +61,8 @@ private void processAggregation(Aggregation aggregation) {
processSubAggregations((HasAggregations) aggregation);
} else if (DateHistogram.class.isAssignableFrom(aggregation.getClass())) {
processDateHistogram((DateHistogram) aggregation);
} else if (Sum.class.isAssignableFrom(aggregation.getClass())) {
processSum((Sum) aggregation);
} else {
throw new IllegalArgumentException("Aggregation type not supported yet: " + aggregation.getClass());
}
Expand All @@ -67,7 +71,12 @@ private void processAggregation(Aggregation aggregation) {
private void processMissingAggregation(Missing aggregation) {
long docCount = aggregation.getDocCount();
if (docCount > 0L) {
getOrCreateFacet(aggregation.getName().replace("_missing", "")).put("", docCount);
LinkedHashMap<String, Long> facet = getOrCreateFacet(aggregation.getName().replace("_missing", ""));
if (aggregation.getAggregations().getAsMap().containsKey("debt")) {
facet.put("", Math.round(((Sum) aggregation.getAggregations().get("debt")).getValue()));
} else {
facet.put("", docCount);
}
}
}

Expand All @@ -80,7 +89,11 @@ private void processTermsAggregation(Terms aggregation) {
facetName = facetName.replace("_selected", "");
LinkedHashMap<String, Long> facet = getOrCreateFacet(facetName);
for (Terms.Bucket value : aggregation.getBuckets()) {
facet.put(value.getKey(), value.getDocCount());
if (value.getAggregations().getAsMap().containsKey("debt")) {
facet.put(value.getKey(), Math.round(((Sum) value.getAggregations().get("debt")).getValue()));
} else {
facet.put(value.getKey(), value.getDocCount());
}
}
}

Expand All @@ -93,10 +106,18 @@ private void processSubAggregations(HasAggregations aggregation) {
private void processDateHistogram(DateHistogram aggregation) {
LinkedHashMap<String, Long> facet = getOrCreateFacet(aggregation.getName());
for (DateHistogram.Bucket value : aggregation.getBuckets()) {
facet.put(value.getKeyAsText().toString(), value.getDocCount());
if (value.getAggregations().getAsMap().containsKey("debt")) {
facet.put(value.getKey(), Math.round(((Sum) value.getAggregations().get("debt")).getValue()));
} else {
facet.put(value.getKey(), value.getDocCount());
}
}
}

private void processSum(Sum aggregation) {
getOrCreateFacet(aggregation.getName()).put(TOTAL, Math.round(aggregation.getValue()));
}

public boolean contains(String facetName) {
return facetsByName.containsKey(facetName);
}
Expand Down
11 changes: 11 additions & 0 deletions server/sonar-server/src/main/java/org/sonar/server/issue/IssueQuery.java
View file
Expand Up @@ -83,6 +83,7 @@ public class IssueQuery {
private final Date createdBefore;
private final String sort;
private final Boolean asc;
private final String facetMode;

private final String userLogin;
private final Set<String> userGroups;
Expand Down Expand Up @@ -121,6 +122,7 @@ private IssueQuery(Builder builder) {
this.userLogin = builder.userLogin;
this.userGroups = builder.userGroups;
this.checkAuthorization = builder.checkAuthorization;
this.facetMode = builder.facetMode;
}

public Collection<String> issueKeys() {
Expand Down Expand Up @@ -269,6 +271,10 @@ public boolean checkAuthorization() {
return checkAuthorization;
}

public String facetMode() {
return facetMode;
}

@Override
public String toString() {
return ReflectionToStringBuilder.toString(this);
Expand Down Expand Up @@ -311,6 +317,7 @@ public static class Builder {
private String userLogin;
private Set<String> userGroups;
private boolean checkAuthorization = true;
private String facetMode;

private Builder(UserSession userSession) {
this.userLogin = userSession.getLogin();
Expand Down Expand Up @@ -518,6 +525,10 @@ public Builder checkAuthorization(boolean checkAuthorization) {
return this;
}

public Builder facetMode(String facetMode) {
this.facetMode = facetMode;
return this;
}
}

private static <T> Collection<T> defaultCollection(@Nullable Collection<T> c) {
Expand Down
9 changes: 8 additions & 1 deletion server/sonar-server/src/main/java/org/sonar/server/issue/IssueQueryService.java
View file
Expand Up @@ -132,6 +132,12 @@ public IssueQuery createFromMap(Map<String, Object> params) {
builder.sort(sort);
builder.asc(RubyUtils.toBoolean(params.get(IssueFilterParameters.ASC)));
}
String facetMode = (String) params.get(IssueFilterParameters.FACET_MODE);
if (!Strings.isNullOrEmpty(facetMode)) {
builder.facetMode(facetMode);
} else {
builder.facetMode(IssueFilterParameters.FACET_MODE_COUNT);
}
return builder.build();

} finally {
Expand Down Expand Up @@ -168,7 +174,8 @@ public IssueQuery createFromRequest(Request request) {
.planned(request.paramAsBoolean(IssueFilterParameters.PLANNED))
.createdAt(request.paramAsDateTime(IssueFilterParameters.CREATED_AT))
.createdAfter(buildCreatedAfter(request.paramAsDateTime(IssueFilterParameters.CREATED_AFTER), request.param(IssueFilterParameters.CREATED_IN_LAST)))
.createdBefore(request.paramAsDateTime(IssueFilterParameters.CREATED_BEFORE));
.createdBefore(request.paramAsDateTime(IssueFilterParameters.CREATED_BEFORE))
.facetMode(request.mandatoryParam(IssueFilterParameters.FACET_MODE));

Set<String> allComponentUuids = Sets.newHashSet();
boolean effectiveOnComponentOnly = mergeDeprecatedComponentParameters(session,
Expand Down
6 changes: 5 additions & 1 deletion server/sonar-server/src/main/java/org/sonar/server/issue/filter/IssueFilterParameters.java
View file
Expand Up @@ -68,11 +68,15 @@ public class IssueFilterParameters {
public static final String PAGE_INDEX = "pageIndex";
public static final String SORT = "sort";
public static final String ASC = "asc";
public static final String FACET_MODE = "facetMode";

public static final String FACET_MODE_COUNT = "count";
public static final String FACET_MODE_DEBT = "debt";

public static final String FACET_ASSIGNED_TO_ME = "assigned_to_me";

public static final List<String> ALL = ImmutableList.of(ISSUES, SEVERITIES, STATUSES, RESOLUTIONS, RESOLVED, COMPONENTS, COMPONENT_ROOTS, RULES, ACTION_PLANS, REPORTERS, TAGS,
ASSIGNEES, LANGUAGES, ASSIGNED, PLANNED, HIDE_RULES, CREATED_AT, CREATED_AFTER, CREATED_BEFORE, CREATED_IN_LAST, COMPONENT_UUIDS, COMPONENT_ROOT_UUIDS,
ASSIGNEES, LANGUAGES, ASSIGNED, PLANNED, HIDE_RULES, CREATED_AT, CREATED_AFTER, CREATED_BEFORE, CREATED_IN_LAST, COMPONENT_UUIDS, COMPONENT_ROOT_UUIDS, FACET_MODE,
PROJECTS, PROJECT_UUIDS, PROJECT_KEYS, COMPONENT_KEYS, MODULE_UUIDS, DIRECTORIES, FILE_UUIDS, AUTHORS, HIDE_COMMENTS, PAGE_SIZE, PAGE_INDEX, SORT, ASC);

public static final List<String> ALL_WITHOUT_PAGINATION = newArrayList(Iterables.filter(ALL, new Predicate<String>() {
Expand Down
60 changes: 45 additions & 15 deletions server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java
View file
Expand Up @@ -57,8 +57,10 @@
import org.elasticsearch.search.aggregations.bucket.global.GlobalBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.elasticsearch.search.aggregations.metrics.min.Min;
import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder;
import org.joda.time.Duration;
import org.sonar.api.issue.Issue;
import org.sonar.api.resources.Scopes;
Expand Down Expand Up @@ -90,6 +92,7 @@
*/
public class IssueIndex extends BaseIndex {


private static final String SUBSTRING_MATCH_REGEXP = ".*%s.*";

public static final List<String> SUPPORTED_FACETS = ImmutableList.of(
Expand All @@ -116,6 +119,10 @@ public class IssueIndex extends BaseIndex {

private static final String IS_ASSIGNED_FILTER = "__isAssigned";

public static final String DEBT_AGGREGATION_NAME = "debt";
private static final SumBuilder DEBT_AGGREGATION = AggregationBuilders.sum(DEBT_AGGREGATION_NAME).field(IssueIndexDefinition.FIELD_ISSUE_DEBT);
private static final Order DEBT_AGGREGATION_ORDER = Order.aggregation(DEBT_AGGREGATION_NAME, false);

private static final int DEFAULT_FACET_SIZE = 15;
private static final Duration TWENTY_DAYS = Duration.standardDays(20L);
private static final Duration TWENTY_WEEKS = Duration.standardDays(20L * 7L);
Expand Down Expand Up @@ -373,7 +380,7 @@ private void validateCreationDateBounds(Date createdBefore, Date createdAfter) {

private void configureStickyFacets(IssueQuery query, SearchOptions options, Map<String, FilterBuilder> filters, QueryBuilder esQuery, SearchRequestBuilder esSearch) {
if (!options.getFacets().isEmpty()) {
StickyFacetBuilder stickyFacetBuilder = new StickyFacetBuilder(esQuery, filters);
StickyFacetBuilder stickyFacetBuilder = newStickyFacetBuilder(query, filters, esQuery);
// Execute Term aggregations
addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch,
IssueFilterParameters.SEVERITIES, IssueIndexDefinition.FIELD_ISSUE_SEVERITY, Severity.ALL.toArray());
Expand Down Expand Up @@ -402,7 +409,7 @@ private void configureStickyFacets(IssueQuery query, SearchOptions options, Map<
}

if (options.getFacets().contains(IssueFilterParameters.RESOLUTIONS)) {
esSearch.addAggregation(createResolutionFacet(filters, esQuery));
esSearch.addAggregation(createResolutionFacet(query, filters, esQuery));
}
if (options.getFacets().contains(IssueFilterParameters.ASSIGNEES)) {
esSearch.addAggregation(createAssigneesFacet(query, filters, esQuery));
Expand All @@ -415,6 +422,20 @@ private void configureStickyFacets(IssueQuery query, SearchOptions options, Map<
esSearch.addAggregation(getCreatedAtFacet(query, filters, esQuery));
}
}

if (IssueFilterParameters.FACET_MODE_DEBT.equals(query.facetMode())) {
esSearch.addAggregation(DEBT_AGGREGATION);
}
}

private StickyFacetBuilder newStickyFacetBuilder(IssueQuery query, Map<String, FilterBuilder> filters, QueryBuilder esQuery) {
StickyFacetBuilder stickyFacetBuilder;
if (IssueFilterParameters.FACET_MODE_DEBT.equals(query.facetMode())) {
stickyFacetBuilder = new StickyFacetBuilder(esQuery, filters, DEBT_AGGREGATION, DEBT_AGGREGATION_ORDER);
} else {
stickyFacetBuilder = new StickyFacetBuilder(esQuery, filters);
}
return stickyFacetBuilder;
}

private void addSimpleStickyFacetIfNeeded(SearchOptions options, StickyFacetBuilder stickyFacetBuilder, SearchRequestBuilder esSearch,
Expand All @@ -424,6 +445,13 @@ private void addSimpleStickyFacetIfNeeded(SearchOptions options, StickyFacetBuil
}
}

private AggregationBuilder addDebtAggregationIfNeeded(IssueQuery query, AggregationBuilder aggregation) {
if (IssueFilterParameters.FACET_MODE_DEBT.equals(query.facetMode())) {
aggregation.subAggregation(DEBT_AGGREGATION);
}
return aggregation;
}

private AggregationBuilder getCreatedAtFacet(IssueQuery query, Map<String, FilterBuilder> filters, QueryBuilder esQuery) {
long now = system.now();

Expand All @@ -445,14 +473,16 @@ private AggregationBuilder getCreatedAtFacet(IssueQuery query, Map<String, Filte
bucketSize = DateHistogram.Interval.MONTH;
}

return AggregationBuilders.dateHistogram(IssueFilterParameters.CREATED_AT)
AggregationBuilder dateHistogram = AggregationBuilders.dateHistogram(IssueFilterParameters.CREATED_AT)
.field(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT)
.interval(bucketSize)
.minDocCount(0L)
.format(DateUtils.DATETIME_FORMAT)
.preZone(timeZoneString)
.postZone(timeZoneString)
.extendedBounds(startTime, endTime);
dateHistogram = addDebtAggregationIfNeeded(query, dateHistogram);
return dateHistogram;
}

private long getMinCreatedAt(Map<String, FilterBuilder> filters, QueryBuilder esQuery) {
Expand Down Expand Up @@ -490,7 +520,7 @@ private AggregationBuilder createAssigneesFacet(IssueQuery query, Map<String, Fi
Map<String, FilterBuilder> assigneeFilters = Maps.newHashMap(filters);
assigneeFilters.remove(IS_ASSIGNED_FILTER);
assigneeFilters.remove(fieldName);
StickyFacetBuilder assigneeFacetBuilder = new StickyFacetBuilder(queryBuilder, assigneeFilters);
StickyFacetBuilder assigneeFacetBuilder = newStickyFacetBuilder(query, assigneeFilters, queryBuilder);
BoolFilterBuilder facetFilter = assigneeFacetBuilder.getStickyFacetFilter(fieldName);
FilterAggregationBuilder facetTopAggregation = assigneeFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, DEFAULT_FACET_SIZE);

Expand All @@ -501,9 +531,9 @@ private AggregationBuilder createAssigneesFacet(IssueQuery query, Map<String, Fi

// Add missing facet for unassigned issues
facetTopAggregation.subAggregation(
AggregationBuilders
addDebtAggregationIfNeeded(query, AggregationBuilders
.missing(facetName + FACET_SUFFIX_MISSING)
.field(fieldName)
.field(fieldName))
);

return AggregationBuilders
Expand Down Expand Up @@ -531,37 +561,37 @@ private void addAssignedToMeFacetIfNeeded(SearchRequestBuilder builder, SearchOp
String facetName = IssueFilterParameters.FACET_ASSIGNED_TO_ME;

// Same as in super.stickyFacetBuilder
StickyFacetBuilder assignedToMeFacetBuilder = new StickyFacetBuilder(queryBuilder, filters);
StickyFacetBuilder assignedToMeFacetBuilder = newStickyFacetBuilder(query, filters, queryBuilder);
BoolFilterBuilder facetFilter = assignedToMeFacetBuilder.getStickyFacetFilter(IS_ASSIGNED_FILTER, fieldName);

FilterAggregationBuilder facetTopAggregation = AggregationBuilders
.filter(facetName + "__filter")
.filter(facetFilter)
.subAggregation(AggregationBuilders.terms(facetName + "__terms").field(fieldName).include(login));
.subAggregation(addDebtAggregationIfNeeded(query, AggregationBuilders.terms(facetName + "__terms").field(fieldName).include(login)));

builder.addAggregation(
AggregationBuilders.global(facetName)
.subAggregation(facetTopAggregation));
}

private AggregationBuilder createResolutionFacet(Map<String, FilterBuilder> filters, QueryBuilder esQuery) {
private AggregationBuilder createResolutionFacet(IssueQuery query, Map<String, FilterBuilder> filters, QueryBuilder esQuery) {
String fieldName = IssueIndexDefinition.FIELD_ISSUE_RESOLUTION;
String facetName = IssueFilterParameters.RESOLUTIONS;

// Same as in super.stickyFacetBuilder
Map<String, FilterBuilder> resolutionFilters = Maps.newHashMap(filters);
resolutionFilters.remove("__isResolved");
resolutionFilters.remove(fieldName);
StickyFacetBuilder assigneeFacetBuilder = new StickyFacetBuilder(esQuery, resolutionFilters);
StickyFacetBuilder assigneeFacetBuilder = newStickyFacetBuilder(query, resolutionFilters, esQuery);
BoolFilterBuilder facetFilter = assigneeFacetBuilder.getStickyFacetFilter(fieldName);
FilterAggregationBuilder facetTopAggregation = assigneeFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, DEFAULT_FACET_SIZE);
facetTopAggregation = assigneeFacetBuilder.addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation, Issue.RESOLUTIONS.toArray());

// Add missing facet for unresolved issues
facetTopAggregation.subAggregation(
AggregationBuilders
addDebtAggregationIfNeeded(query, AggregationBuilders
.missing(facetName + FACET_SUFFIX_MISSING)
.field(fieldName)
.field(fieldName))
);

return AggregationBuilders
Expand All @@ -577,16 +607,16 @@ private AggregationBuilder createActionPlansFacet(IssueQuery query, Map<String,
Map<String, FilterBuilder> actionPlanFilters = Maps.newHashMap(filters);
actionPlanFilters.remove("__isPlanned");
actionPlanFilters.remove(fieldName);
StickyFacetBuilder actionPlanFacetBuilder = new StickyFacetBuilder(esQuery, actionPlanFilters);
StickyFacetBuilder actionPlanFacetBuilder = newStickyFacetBuilder(query, actionPlanFilters, esQuery);
BoolFilterBuilder facetFilter = actionPlanFacetBuilder.getStickyFacetFilter(fieldName);
FilterAggregationBuilder facetTopAggregation = actionPlanFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, DEFAULT_FACET_SIZE);
facetTopAggregation = actionPlanFacetBuilder.addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation, query.actionPlans().toArray());

// Add missing facet for unresolved issues
facetTopAggregation.subAggregation(
AggregationBuilders
addDebtAggregationIfNeeded(query, AggregationBuilders
.missing(facetName + FACET_SUFFIX_MISSING)
.field(fieldName)
.field(fieldName))
);

return AggregationBuilders
Expand Down
9 changes: 9 additions & 0 deletions server/sonar-server/src/main/java/org/sonar/server/issue/ws/SearchAction.java
View file
Expand Up @@ -55,6 +55,7 @@
import org.sonar.core.persistence.DbSession;
import org.sonar.server.component.ws.ComponentJsonWriter;
import org.sonar.server.db.DbClient;
import org.sonar.server.es.Facets;
import org.sonar.server.es.SearchOptions;
import org.sonar.server.es.SearchResult;
import org.sonar.server.issue.IssueQuery;
Expand Down Expand Up @@ -126,6 +127,10 @@ public void define(WebService.NewController controller) {
action.createParam(WebService.Param.FACETS)
.setDescription("Comma-separated list of the facets to be computed. No facet is computed by default.")
.setPossibleValues(IssueIndex.SUPPORTED_FACETS);
action.createParam(IssueFilterParameters.FACET_MODE)
.setDefaultValue(IssueFilterParameters.FACET_MODE_COUNT)
.setDescription("Choose the returned value for facet items, either count of issues or sum of debt.")
.setPossibleValues(IssueFilterParameters.FACET_MODE_COUNT, IssueFilterParameters.FACET_MODE_DEBT);
action.addSortParams(IssueQuery.SORTS, null, true);
action.addFieldsParam(IssueJsonWriter.SELECTABLE_FIELDS);

Expand Down Expand Up @@ -286,6 +291,10 @@ private SearchResult<IssueDoc> execute(IssueQuery query, SearchOptions options)
}

private void writeResponse(Request request, SearchResult<IssueDoc> result, JsonWriter json) {
if (result.getFacets().contains(IssueIndex.DEBT_AGGREGATION_NAME)) {
json.prop("debtTotal", result.getFacets().get(IssueIndex.DEBT_AGGREGATION_NAME).get(Facets.TOTAL));
}

List<String> issueKeys = newArrayList();
Set<RuleKey> ruleKeys = newHashSet();
Set<String> projectUuids = newHashSet();
Expand Down

0 comments on commit 8f7068c

Please sign in to comment.