SONAR-11513 backdate issues on new files

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java

@@ -43,6 +43,7 @@
 import org.sonar.ce.task.projectanalysis.duplication.DuplicationRepositoryImpl;
 import org.sonar.ce.task.projectanalysis.duplication.IntegrateCrossProjectDuplications;
 import org.sonar.ce.task.projectanalysis.event.EventRepositoryImpl;
+import org.sonar.ce.task.projectanalysis.filemove.AddedFileRepositoryImpl;
 import org.sonar.ce.task.projectanalysis.filemove.FileSimilarityImpl;
 import org.sonar.ce.task.projectanalysis.filemove.MutableMovedFilesRepositoryImpl;
 import org.sonar.ce.task.projectanalysis.filemove.ScoreMatrixDumperImpl;
@@ -284,6 +285,7 @@ private static List<Object> componentClasses() {
       SourceSimilarityImpl.class,
       FileSimilarityImpl.class,
       MutableMovedFilesRepositoryImpl.class,
+      AddedFileRepositoryImpl.class,
 
       // duplication
       IntegrateCrossProjectDuplications.class,

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/filemove/AddedFileRepository.java

@@ -0,0 +1,30 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.filemove;
+
+import org.sonar.ce.task.projectanalysis.component.Component;
+
+public interface AddedFileRepository {
+  /**
+   * @return {@code true} for any component on first analysis, otherwise {@code true} only if the specified component is
+   *         a {@link Component.Type#FILE file} registered to the repository.
+   */
+  boolean isAdded(Component component);
+}

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/filemove/AddedFileRepositoryImpl.java

@@ -0,0 +1,60 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.filemove;
+
+import java.util.HashSet;
+import java.util.Set;
+import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
+import org.sonar.ce.task.projectanalysis.component.Component;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+public class AddedFileRepositoryImpl implements MutableAddedFileRepository {
+  private final Set<Component> addedComponents = new HashSet<>();
+  private final AnalysisMetadataHolder analysisMetadataHolder;
+
+  public AddedFileRepositoryImpl(AnalysisMetadataHolder analysisMetadataHolder) {
+    this.analysisMetadataHolder = analysisMetadataHolder;
+  }
+
+  @Override
+  public boolean isAdded(Component component) {
+    checkComponent(component);
+    if (analysisMetadataHolder.isFirstAnalysis()) {
+      return true;
+    }
+    return addedComponents.contains(component);
+  }
+
+  @Override
+  public void register(Component component) {
+    checkComponent(component);
+    checkArgument(component.getType() == Component.Type.FILE, "component must be a file");
+    checkState(!analysisMetadataHolder.isFirstAnalysis(), "No file can be registered on first analysis");
+
+    addedComponents.add(component);
+  }
+
+  private static void checkComponent(Component component) {
+    checkNotNull(component, "component can't be null");
+  }
+}

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/filemove/FileMoveDetectionStep.java

@@ -22,7 +22,6 @@
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 import java.util.ArrayList;
@@ -62,7 +61,7 @@
 import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.POST_ORDER;
 
 public class FileMoveDetectionStep implements ComputationStep {
-  protected static final int MIN_REQUIRED_SCORE = 85;
+  static final int MIN_REQUIRED_SCORE = 85;
   private static final Logger LOG = Loggers.get(FileMoveDetectionStep.class);
   private static final Comparator<ScoreMatrix.ScoreFile> SCORE_FILE_COMPARATOR = (o1, o2) -> -1 * Integer.compare(o1.getLineCount(), o2.getLineCount());
   private static final double LOWER_BOUND_RATIO = 0.84;
@@ -75,17 +74,19 @@ public class FileMoveDetectionStep implements ComputationStep {
   private final MutableMovedFilesRepository movedFilesRepository;
   private final SourceLinesHashRepository sourceLinesHash;
   private final ScoreMatrixDumper scoreMatrixDumper;
+  private final MutableAddedFileRepository addedFileRepository;
 
   public FileMoveDetectionStep(AnalysisMetadataHolder analysisMetadataHolder, TreeRootHolder rootHolder, DbClient dbClient,
     FileSimilarity fileSimilarity, MutableMovedFilesRepository movedFilesRepository, SourceLinesHashRepository sourceLinesHash,
-    ScoreMatrixDumper scoreMatrixDumper) {
+    ScoreMatrixDumper scoreMatrixDumper, MutableAddedFileRepository addedFileRepository) {
     this.analysisMetadataHolder = analysisMetadataHolder;
     this.rootHolder = rootHolder;
     this.dbClient = dbClient;
     this.fileSimilarity = fileSimilarity;
     this.movedFilesRepository = movedFilesRepository;
     this.sourceLinesHash = sourceLinesHash;
     this.scoreMatrixDumper = scoreMatrixDumper;
+    this.addedFileRepository = addedFileRepository;
   }
 
   @Override
@@ -103,25 +104,30 @@ public void execute(ComputationStep.Context context) {
     Profiler p = Profiler.createIfTrace(LOG);
 
     p.start();
-    Map<String, DbComponent> dbFilesByKey = getDbFilesByKey();
-    context.getStatistics().add("dbFiles", dbFilesByKey.size());
-    if (dbFilesByKey.isEmpty()) {
-      LOG.debug("Previous snapshot has no file. Do nothing.");
-      return;
-    }
-
     Map<String, Component> reportFilesByKey = getReportFilesByKey(this.rootHolder.getRoot());
+    context.getStatistics().add("reportFiles", reportFilesByKey.size());
     if (reportFilesByKey.isEmpty()) {
-      LOG.debug("No files in report. Do nothing.");
+      LOG.debug("No files in report. No file move detection.");
       return;
     }
 
-    Set<String> addedFileKeys = ImmutableSet.copyOf(Sets.difference(reportFilesByKey.keySet(), dbFilesByKey.keySet()));
+    Map<String, DbComponent> dbFilesByKey = getDbFilesByKey();
+    context.getStatistics().add("dbFiles", dbFilesByKey.size());
+
+    Set<String> addedFileKeys = difference(reportFilesByKey.keySet(), dbFilesByKey.keySet());
     context.getStatistics().add("addedFiles", addedFileKeys.size());
-    Set<String> removedFileKeys = ImmutableSet.copyOf(Sets.difference(dbFilesByKey.keySet(), reportFilesByKey.keySet()));
+
+    if (dbFilesByKey.isEmpty()) {
+      registerAddedFiles(addedFileKeys, reportFilesByKey, null);
+      LOG.debug("Previous snapshot has no file. No file move detection.");
+      return;
+    }
+
+    Set<String> removedFileKeys = difference(dbFilesByKey.keySet(), reportFilesByKey.keySet());
 
     // can find matches if at least one of the added or removed files groups is empty => abort
     if (addedFileKeys.isEmpty() || removedFileKeys.isEmpty()) {
+      registerAddedFiles(addedFileKeys, reportFilesByKey, null);
       LOG.debug("Either no files added or no files removed. Do nothing.");
       return;
     }
@@ -138,6 +144,8 @@ public void execute(ComputationStep.Context context) {
 
     // not a single match with score higher than MIN_REQUIRED_SCORE => abort
     if (scoreMatrix.getMaxScore() < MIN_REQUIRED_SCORE) {
+      context.getStatistics().add("movedFiles", 0);
+      registerAddedFiles(addedFileKeys, reportFilesByKey, null);
       LOG.debug("max score in matrix is less than min required score ({}). Do nothing.", MIN_REQUIRED_SCORE);
       return;
     }
@@ -148,7 +156,16 @@ public void execute(ComputationStep.Context context) {
     ElectedMatches electedMatches = electMatches(removedFileKeys, reportFileSourcesByKey, matchesByScore);
     p.stopTrace("Matches elected");
 
+    context.getStatistics().add("movedFiles", electedMatches.size());
     registerMatches(dbFilesByKey, reportFilesByKey, electedMatches);
+    registerAddedFiles(addedFileKeys, reportFilesByKey, electedMatches);
+  }
+
+  public Set<String> difference(Set<String> set1, Set<String> set2) {
+    if (set1.isEmpty() || set2.isEmpty()) {
+      return set1;
+    }
+    return Sets.difference(set1, set2).immutableCopy();
   }
 
   private void registerMatches(Map<String, DbComponent> dbFilesByKey, Map<String, Component> reportFilesByKey, ElectedMatches electedMatches) {
@@ -161,6 +178,22 @@ private void registerMatches(Map<String, DbComponent> dbFilesByKey, Map<String,
     }
   }
 
+  private void registerAddedFiles(Set<String> addedFileKeys, Map<String, Component> reportFilesByKey, @Nullable ElectedMatches electedMatches) {
+    if (electedMatches == null || electedMatches.isEmpty()) {
+      addedFileKeys.stream()
+        .map(reportFilesByKey::get)
+        .forEach(addedFileRepository::register);
+    } else {
+      Set<String> reallyAddedFileKeys = new HashSet<>(addedFileKeys);
+      for (Match electedMatch : electedMatches) {
+        reallyAddedFileKeys.remove(electedMatch.getReportKey());
+      }
+      reallyAddedFileKeys.stream()
+        .map(reportFilesByKey::get)
+        .forEach(addedFileRepository::register);
+    }
+  }
+
   private Map<String, DbComponent> getDbFilesByKey() {
     try (DbSession dbSession = dbClient.openSession(false)) {
       ImmutableList.Builder<DbComponent> builder = ImmutableList.builder();
@@ -402,5 +435,9 @@ public Iterator<Match> iterator() {
     public int size() {
       return matches.size();
     }
+
+    public boolean isEmpty() {
+      return matches.isEmpty();
+    }
   }
 }

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/filemove/MutableAddedFileRepository.java

@@ -0,0 +1,32 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.filemove;
+
+import org.sonar.ce.task.projectanalysis.component.Component;
+
+public interface MutableAddedFileRepository extends AddedFileRepository {
+
+  /**
+   * @throws IllegalArgumentException if the specified component is not a {@link Component.Type#FILE File}
+   * @throws IllegalStateException on first analysis as all components are added on first analysis, none should be
+   *         registered for performance reasons.
+   */
+  void register(Component file);
+}

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IssueCreationDateCalculator.java

@@ -30,6 +30,7 @@
 import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
 import org.sonar.ce.task.projectanalysis.analysis.ScannerPlugin;
 import org.sonar.ce.task.projectanalysis.component.Component;
+import org.sonar.ce.task.projectanalysis.filemove.AddedFileRepository;
 import org.sonar.ce.task.projectanalysis.qualityprofile.ActiveRule;
 import org.sonar.ce.task.projectanalysis.qualityprofile.ActiveRulesHolder;
 import org.sonar.ce.task.projectanalysis.scm.Changeset;
@@ -53,39 +54,55 @@ public class IssueCreationDateCalculator extends IssueVisitor {
   private final IssueChangeContext changeContext;
   private final ActiveRulesHolder activeRulesHolder;
   private final RuleRepository ruleRepository;
+  private final AddedFileRepository addedFileRepository;
 
   public IssueCreationDateCalculator(AnalysisMetadataHolder analysisMetadataHolder, ScmInfoRepository scmInfoRepository,
-    IssueFieldsSetter issueUpdater, ActiveRulesHolder activeRulesHolder, RuleRepository ruleRepository) {
+    IssueFieldsSetter issueUpdater, ActiveRulesHolder activeRulesHolder, RuleRepository ruleRepository,
+    AddedFileRepository addedFileRepository) {
     this.scmInfoRepository = scmInfoRepository;
     this.issueUpdater = issueUpdater;
     this.analysisMetadataHolder = analysisMetadataHolder;
     this.ruleRepository = ruleRepository;
     this.changeContext = createScan(new Date(analysisMetadataHolder.getAnalysisDate()));
     this.activeRulesHolder = activeRulesHolder;
+    this.addedFileRepository = addedFileRepository;
   }
 
   @Override
   public void onIssue(Component component, DefaultIssue issue) {
     if (!issue.isNew()) {
       return;
     }
+
     Optional<Long> lastAnalysisOptional = lastAnalysis();
     boolean firstAnalysis = !lastAnalysisOptional.isPresent();
+    if (firstAnalysis || isNewFile(component)) {
+      backdateIssue(component, issue);
+      return;
+    }
+
     Rule rule = ruleRepository.findByKey(issue.getRuleKey())
       .orElseThrow(illegalStateException("The rule with key '%s' raised an issue, but no rule with that key was found", issue.getRuleKey()));
-
     if (rule.isExternal()) {
-      getDateOfLatestChange(component, issue).ifPresent(changeDate -> updateDate(issue, changeDate));
+      backdateIssue(component, issue);
     } else {
       // Rule can't be inactive (see contract of IssueVisitor)
       ActiveRule activeRule = activeRulesHolder.get(issue.getRuleKey()).get();
-      if (firstAnalysis || activeRuleIsNew(activeRule, lastAnalysisOptional.get())
+      if (activeRuleIsNew(activeRule, lastAnalysisOptional.get())
         || ruleImplementationChanged(activeRule.getRuleKey(), activeRule.getPluginKey(), lastAnalysisOptional.get())) {
-        getDateOfLatestChange(component, issue).ifPresent(changeDate -> updateDate(issue, changeDate));
+        backdateIssue(component, issue);
       }
     }
   }
 
+  private boolean isNewFile(Component component) {
+    return component.getType() == Component.Type.FILE && addedFileRepository.isAdded(component);
+  }
+
+  private void backdateIssue(Component component, DefaultIssue issue) {
+    getDateOfLatestChange(component, issue).ifPresent(changeDate -> updateDate(issue, changeDate));
+  }
+
   private boolean ruleImplementationChanged(RuleKey ruleKey, @Nullable String pluginKey, long lastAnalysisDate) {
     if (pluginKey == null) {
       return false;