SONAR-6623 count issues

server/sonar-server/src/main/java/org/sonar/server/computation/container/ComputeEngineContainerImpl.java

@@ -45,6 +45,7 @@
 import org.sonar.server.computation.event.EventRepositoryImpl;
 import org.sonar.server.computation.formula.CoreFormulaRepositoryImpl;
 import org.sonar.server.computation.issue.BaseIssuesLoader;
+import org.sonar.server.computation.issue.CountIssuesListener;
 import org.sonar.server.computation.issue.DebtCalculator;
 import org.sonar.server.computation.issue.DefaultAssignee;
 import org.sonar.server.computation.issue.IssueAssigner;
@@ -176,6 +177,7 @@ private static List componentClasses() {
       DebtCalculator.class,
       IssueListeners.class,
       IssueLifecycle.class,
+      CountIssuesListener.class,
       UpdateConflictResolver.class,
       TrackerBaseInputFactory.class,
       TrackerRawInputFactory.class,

server/sonar-server/src/main/java/org/sonar/server/computation/issue/CountIssuesListener.java

@@ -0,0 +1,273 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.computation.issue;
+
+import com.google.common.collect.HashMultiset;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Multiset;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.sonar.api.issue.Issue;
+import org.sonar.core.issue.DefaultIssue;
+import org.sonar.core.issue.tracking.Tracking;
+import org.sonar.server.computation.component.Component;
+import org.sonar.server.computation.measure.Measure;
+import org.sonar.server.computation.measure.MeasureRepository;
+import org.sonar.server.computation.measure.MeasureVariations;
+import org.sonar.server.computation.metric.Metric;
+import org.sonar.server.computation.metric.MetricRepository;
+import org.sonar.server.computation.period.Period;
+import org.sonar.server.computation.period.PeriodsHolder;
+
+import static org.sonar.api.issue.Issue.STATUS_CONFIRMED;
+import static org.sonar.api.issue.Issue.STATUS_OPEN;
+import static org.sonar.api.issue.Issue.STATUS_REOPENED;
+import static org.sonar.api.measures.CoreMetrics.BLOCKER_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.CONFIRMED_ISSUES_KEY;
+import static org.sonar.api.measures.CoreMetrics.CRITICAL_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.FALSE_POSITIVE_ISSUES_KEY;
+import static org.sonar.api.measures.CoreMetrics.INFO_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.MAJOR_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.MINOR_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.NEW_BLOCKER_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.NEW_CRITICAL_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.NEW_INFO_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.NEW_MAJOR_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.NEW_MINOR_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.NEW_VIOLATIONS_KEY;
+import static org.sonar.api.measures.CoreMetrics.OPEN_ISSUES_KEY;
+import static org.sonar.api.measures.CoreMetrics.REOPENED_ISSUES_KEY;
+import static org.sonar.api.measures.CoreMetrics.VIOLATIONS_KEY;
+import static org.sonar.api.rule.Severity.BLOCKER;
+import static org.sonar.api.rule.Severity.CRITICAL;
+import static org.sonar.api.rule.Severity.INFO;
+import static org.sonar.api.rule.Severity.MAJOR;
+import static org.sonar.api.rule.Severity.MINOR;
+
+/**
+ * For each component, computes the measures related to number of issues:
+ * <ul>
+ *   <li>unresolved issues</li>
+ *   <li>false-positives</li>
+ *   <li>open issues</li>
+ *   <li>issues per status (open, reopen, confirmed)</li>
+ *   <li>issues per severity (from info to blocker)</li>
+ * </ul>
+ * For each value, the variation on configured periods is also computed.
+ */
+public class CountIssuesListener extends IssueListener {
+
+  private final static Map<String, String> SEVERITY_TO_METRIC_KEY = ImmutableMap.of(
+    BLOCKER, BLOCKER_VIOLATIONS_KEY,
+    CRITICAL, CRITICAL_VIOLATIONS_KEY,
+    MAJOR, MAJOR_VIOLATIONS_KEY,
+    MINOR, MINOR_VIOLATIONS_KEY,
+    INFO, INFO_VIOLATIONS_KEY
+    );
+
+  private final static Map<String, String> SEVERITY_TO_NEW_METRIC_KEY = ImmutableMap.of(
+    BLOCKER, NEW_BLOCKER_VIOLATIONS_KEY,
+    CRITICAL, NEW_CRITICAL_VIOLATIONS_KEY,
+    MAJOR, NEW_MAJOR_VIOLATIONS_KEY,
+    MINOR, NEW_MINOR_VIOLATIONS_KEY,
+    INFO, NEW_INFO_VIOLATIONS_KEY
+    );
+
+  private final PeriodsHolder periodsHolder;
+  private final MetricRepository metricRepository;
+  private final MeasureRepository measureRepository;
+
+  private final Map<Integer, Counters> countersByComponentRef = new HashMap<>();
+  private Counters currentCounters;
+
+  public CountIssuesListener(PeriodsHolder periodsHolder,
+    MetricRepository metricRepository, MeasureRepository measureRepository) {
+    this.periodsHolder = periodsHolder;
+    this.metricRepository = metricRepository;
+    this.measureRepository = measureRepository;
+  }
+
+  @Override
+  public void beforeComponent(Component component, Tracking tracking) {
+    // TODO optimization no need to instantiate counter if no open issues
+    currentCounters = new Counters();
+    countersByComponentRef.put(component.getRef(), currentCounters);
+  }
+
+  @Override
+  public void onIssue(Component component, DefaultIssue issue) {
+    currentCounters.add(issue);
+    for (Period period : periodsHolder.getPeriods()) {
+      // Add one second to not take into account issues created during current analysis
+      if (issue.creationDate().getTime() >= period.getSnapshotDate() + 1000L) {
+        currentCounters.addOnPeriod(issue, period.getIndex());
+      }
+    }
+  }
+
+  @Override
+  public void afterComponent(Component component) {
+    // aggregate children counters
+    for (Component child : component.getChildren()) {
+      // no need to keep the children in memory. They can be garbage-collected.
+      Counters childCounters = countersByComponentRef.remove(child.getRef());
+      currentCounters.add(childCounters);
+    }
+
+    addMeasuresByStatus(component);
+    addMeasuresBySeverity(component);
+    addMeasuresByPeriod(component);
+  }
+
+  private void addMeasuresBySeverity(Component component) {
+    for (Map.Entry<String, String> entry : SEVERITY_TO_METRIC_KEY.entrySet()) {
+      String severity = entry.getKey();
+      String metricKey = entry.getValue();
+      addMeasure(component, metricKey, currentCounters.counter().severityBag.count(severity));
+    }
+  }
+
+  private void addMeasuresByStatus(Component component) {
+    addMeasure(component, VIOLATIONS_KEY, currentCounters.counter().unresolved);
+    addMeasure(component, OPEN_ISSUES_KEY, currentCounters.counter().open);
+    addMeasure(component, REOPENED_ISSUES_KEY, currentCounters.counter().reopened);
+    addMeasure(component, CONFIRMED_ISSUES_KEY, currentCounters.counter().confirmed);
+    addMeasure(component, FALSE_POSITIVE_ISSUES_KEY, currentCounters.counter().falsePositives);
+  }
+
+  private void addMeasure(Component component, String metricKey, int value) {
+    Metric metric = metricRepository.getByKey(metricKey);
+    measureRepository.add(component, metric, Measure.newMeasureBuilder().create(value));
+  }
+
+  private void addMeasuresByPeriod(Component component) {
+    if (!periodsHolder.getPeriods().isEmpty()) {
+      Double[] unresolvedVariations = new Double[PeriodsHolder.MAX_NUMBER_OF_PERIODS];
+      for (Period period : periodsHolder.getPeriods()) {
+        unresolvedVariations[period.getIndex() - 1] = new Double(currentCounters.counterForPeriod(period.getIndex()).unresolved);
+      }
+      measureRepository.add(component, metricRepository.getByKey(NEW_VIOLATIONS_KEY), Measure.newMeasureBuilder()
+        .setVariations(new MeasureVariations(unresolvedVariations))
+        .createNoValue());
+
+      for (Map.Entry<String, String> entry : SEVERITY_TO_NEW_METRIC_KEY.entrySet()) {
+        String severity = entry.getKey();
+        String metricKey = entry.getValue();
+        Double[] variations = new Double[PeriodsHolder.MAX_NUMBER_OF_PERIODS];
+        boolean set = false;
+        for (Period period : periodsHolder.getPeriods()) {
+          Multiset<String> bag = currentCounters.counterForPeriod(period.getIndex()).severityBag;
+          if (bag.contains(severity)) {
+            variations[period.getIndex() - 1] = new Double(bag.count(severity));
+            set = true;
+          }
+        }
+        if (set) {
+          Metric metric = metricRepository.getByKey(metricKey);
+          measureRepository.add(component, metric, Measure.newMeasureBuilder()
+            .setVariations(new MeasureVariations(variations))
+            .createNoValue());
+        }
+      }
+    }
+  }
+
+  /**
+   * Count issues by status, resolutions, rules and severities
+   */
+  private static class Counter {
+    private int unresolved = 0;
+    private int open = 0;
+    private int reopened = 0;
+    private int confirmed = 0;
+    private int falsePositives = 0;
+    private Multiset<String> severityBag = HashMultiset.create();
+
+    void add(Counter counter) {
+      unresolved += counter.unresolved;
+      open += counter.open;
+      reopened += counter.reopened;
+      confirmed += counter.confirmed;
+      falsePositives += counter.falsePositives;
+      severityBag.addAll(counter.severityBag);
+    }
+
+    void add(Issue issue) {
+      if (issue.resolution() == null) {
+        unresolved++;
+        severityBag.add(issue.severity());
+      } else if (Issue.RESOLUTION_FALSE_POSITIVE.equals(issue.resolution())) {
+        falsePositives++;
+      }
+      switch (issue.status()) {
+        case STATUS_OPEN:
+          open++;
+          break;
+        case STATUS_REOPENED:
+          reopened++;
+          break;
+        case STATUS_CONFIRMED:
+          confirmed++;
+          break;
+        default:
+          // Other statuses are ignored
+      }
+    }
+  }
+
+  /**
+   * List of {@link Counter} for regular value and periods.
+   */
+  private static class Counters {
+    private final Counter[] array = new Counter[1 + PeriodsHolder.MAX_NUMBER_OF_PERIODS];
+
+    Counters() {
+      array[0] = new Counter();
+      for (int i = 1; i <= PeriodsHolder.MAX_NUMBER_OF_PERIODS; i++) {
+        array[i] = new Counter();
+      }
+    }
+
+    void add(@Nullable Counters other) {
+      if (other != null) {
+        for (int i = 0; i < array.length; i++) {
+          array[i].add(other.array[i]);
+        }
+      }
+    }
+
+    void addOnPeriod(Issue issue, int periodIndex) {
+      array[periodIndex].add(issue);
+    }
+
+    void add(Issue issue) {
+      array[0].add(issue);
+    }
+
+    Counter counter() {
+      return array[0];
+    }
+
+    Counter counterForPeriod(int periodIndex) {
+      return array[periodIndex];
+    }
+  }
+}

server/sonar-server/src/main/java/org/sonar/server/computation/issue/DebtCalculator.java

@@ -25,7 +25,7 @@
 public class DebtCalculator extends IssueListener {
 
   @Override
-  public void beforeIssue(Component component, DefaultIssue issue) {
+  public void onOpenIssueInitialization(Component component, DefaultIssue issue) {
     if (issue.resolution() == null) {
       // TODO
     }

server/sonar-server/src/main/java/org/sonar/server/computation/issue/IssueListener.java

@@ -25,11 +25,23 @@
 
 public abstract class IssueListener {
 
+  /**
+   * This method is called for each component before processing its issues.
+   * The component does not necessarily have issues.
+   */
   public void beforeComponent(Component component, Tracking tracking) {
 
   }
 
-  public void beforeIssue(Component component, DefaultIssue issue) {
+  /**
+   * This method is called when initializing an open issue. At that time
+   * any information related to tracking step are not available (line, assignee,
+   * resolution, status, creation date, uuid, ...).
+   * <p/>
+   * The need for this method is for example to calculate the issue debt
+   * before merging with base issue
+   */
+  public void onOpenIssueInitialization(Component component, DefaultIssue issue) {
 
   }
 

server/sonar-server/src/main/java/org/sonar/server/computation/issue/IssueListeners.java

@@ -37,9 +37,9 @@ public void beforeComponent(Component component, Tracking tracking) {
     }
   }
 
-  public void beforeIssue(Component component, DefaultIssue issue) {
+  public void onOpenIssueInitialization(Component component, DefaultIssue issue) {
     for (IssueListener listener : listeners) {
-      listener.beforeIssue(component, issue);
+      listener.onOpenIssueInitialization(component, issue);
     }
   }
 

server/sonar-server/src/main/java/org/sonar/server/computation/period/PeriodsHolder.java

@@ -32,6 +32,8 @@
  */
 public interface PeriodsHolder {
 
+  int MAX_NUMBER_OF_PERIODS = 5;
+
   /**
    * Return the list of differential periods, ordered by increasing index.
    *

server/sonar-server/src/main/java/org/sonar/server/computation/period/PeriodsHolderImpl.java

@@ -50,10 +50,10 @@ public class PeriodsHolderImpl implements PeriodsHolder {
    */
   public void setPeriods(Iterable<Period> periods) {
     requireNonNull(periods, "Periods cannot be null");
-    checkArgument(Iterables.size(periods) <= 5, "There can not be more than 5 periods");
-    checkState(this.periods == null, "Periods have already been initialized");
+    checkArgument(Iterables.size(periods) <= MAX_NUMBER_OF_PERIODS, String.format("There can not be more than %d periods", MAX_NUMBER_OF_PERIODS));
+      checkState(this.periods == null, "Periods have already been initialized");
 
-    Period[] newPeriods = new Period[5];
+    Period[] newPeriods = new Period[MAX_NUMBER_OF_PERIODS];
     for (Period period : from(periods).filter(CheckNotNull.INSTANCE)) {
       int arrayIndex = period.getIndex() - 1;
       checkArgument(newPeriods[arrayIndex] == null, "More than one period has the index " + period.getIndex());

server/sonar-server/src/main/java/org/sonar/server/computation/step/ComputationSteps.java

@@ -24,7 +24,6 @@
 import java.util.Arrays;
 import java.util.List;
 import org.sonar.server.computation.container.ComputeEngineContainer;
-import org.sonar.server.computation.issue.IntegrateIssuesStep;
 
 /**
  * Ordered list of steps to be executed
@@ -48,14 +47,14 @@ public List<Class<? extends ComputationStep>> orderedStepClasses() {
       FeedDebtModelStep.class,
 
       // load project related stuffs
-      IntegrateIssuesStep.class,
       QualityGateLoadingStep.class,
       FeedPeriodsStep.class,
 
       // data computation
-      ComputeFormulaMeasuresStep.class,
-      CustomMeasuresCopyStep.class,
+      IntegrateIssuesStep.class,
       ComputeIssueMeasuresStep.class,
+      CustomMeasuresCopyStep.class,
+      ComputeFormulaMeasuresStep.class,      
       SqaleMeasuresStep.class,
       NewCoverageMeasuresStep.class,
       NewCoverageAggregationStep.class,