SONAR-10257 Generate SCM Info for changed lines

server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java

@@ -114,6 +114,7 @@
 import org.sonar.server.computation.task.projectanalysis.scm.ScmInfoRepositoryImpl;
 import org.sonar.server.computation.task.projectanalysis.source.LastCommitVisitor;
 import org.sonar.server.computation.task.projectanalysis.source.SourceHashRepositoryImpl;
+import org.sonar.server.computation.task.projectanalysis.source.SourceLinesDiffImpl;
 import org.sonar.server.computation.task.projectanalysis.source.SourceLinesRepositoryImpl;
 import org.sonar.server.computation.task.projectanalysis.step.ReportComputationSteps;
 import org.sonar.server.computation.task.projectanalysis.step.SmallChangesetQualityGateSpecialCase;
@@ -189,6 +190,7 @@ private static List<Object> componentClasses() {
       EvaluationResultTextConverterImpl.class,
       SourceLinesRepositoryImpl.class,
       SourceHashRepositoryImpl.class,
+      SourceLinesDiffImpl.class,
       ScmInfoRepositoryImpl.class,
       ScmInfoDbLoader.class,
       DuplicationRepositoryImpl.class,

server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/DbScmInfo.java

@@ -36,12 +36,14 @@
 class DbScmInfo implements ScmInfo {
 
   private final ScmInfo delegate;
+  private final String fileHash;
 
-  private DbScmInfo(ScmInfo delegate) {
+  private DbScmInfo(ScmInfo delegate, String fileHash) {
     this.delegate = delegate;
+    this.fileHash = fileHash;
   }
 
-  static Optional<ScmInfo> create(Iterable<DbFileSources.Line> lines) {
+  public static Optional<DbScmInfo> create(Iterable<DbFileSources.Line> lines, String fileHash) {
     LineToChangeset lineToChangeset = new LineToChangeset();
     Map<Integer, Changeset> lineChanges = new LinkedHashMap<>();
 
@@ -55,7 +57,11 @@ static Optional<ScmInfo> create(Iterable<DbFileSources.Line> lines) {
     if (lineChanges.isEmpty()) {
       return Optional.empty();
     }
-    return Optional.of(new DbScmInfo(new ScmInfoImpl(lineChanges)));
+    return Optional.of(new DbScmInfo(new ScmInfoImpl(lineChanges), fileHash));
+  }
+
+  public String fileHash() {
+    return fileHash;
   }
 
   @Override

server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/GeneratedScmInfo.java

@@ -0,0 +1,82 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.computation.task.projectanalysis.scm;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Preconditions.checkState;
+
+public class GeneratedScmInfo implements ScmInfo {
+  private final ScmInfoImpl delegate;
+
+  public GeneratedScmInfo(Map<Integer, Changeset> changesets) {
+    delegate = new ScmInfoImpl(changesets);
+  }
+
+  public static ScmInfo create(long analysisDate, Set<Integer> lines) {
+    checkState(!lines.isEmpty(), "No changesets");
+
+    Changeset changeset = Changeset.newChangesetBuilder()
+      .setDate(analysisDate)
+      .build();
+    Map<Integer, Changeset> changesets = lines.stream()
+      .collect(Collectors.toMap(x -> x, i -> changeset));
+    return new GeneratedScmInfo(changesets);
+  }
+
+  public static ScmInfo create(long analysisDate, Set<Integer> lines, ScmInfo dbScmInfo) {
+    checkState(!lines.isEmpty(), "No changesets");
+
+    Changeset changeset = Changeset.newChangesetBuilder()
+      .setDate(analysisDate)
+      .build();
+    Map<Integer, Changeset> changesets = lines.stream()
+      .collect(Collectors.toMap(x -> x, i -> changeset));
+
+    dbScmInfo.getAllChangesets().entrySet().stream()
+      .filter(e -> !lines.contains(e.getKey()))
+      .forEach(e -> changesets.put(e.getKey(), e.getValue()));
+
+    return new GeneratedScmInfo(changesets);
+  }
+
+  @Override
+  public Changeset getLatestChangeset() {
+    return delegate.getLatestChangeset();
+  }
+
+  @Override
+  public Changeset getChangesetForLine(int lineNumber) {
+    return delegate.getChangesetForLine(lineNumber);
+  }
+
+  @Override
+  public boolean hasChangesetForLine(int lineNumber) {
+    return delegate.hasChangesetForLine(lineNumber);
+  }
+
+  @Override
+  public Map<Integer, Changeset> getAllChangesets() {
+    return delegate.getAllChangesets();
+  }
+
+}

server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoDbLoader.java

@@ -28,41 +28,34 @@
 import org.sonar.server.computation.task.projectanalysis.analysis.AnalysisMetadataHolder;
 import org.sonar.server.computation.task.projectanalysis.analysis.Branch;
 import org.sonar.server.computation.task.projectanalysis.component.Component;
-import org.sonar.server.computation.task.projectanalysis.component.Component.Status;
 import org.sonar.server.computation.task.projectanalysis.component.MergeBranchComponentUuids;
-import org.sonar.server.computation.task.projectanalysis.scm.ScmInfoRepositoryImpl.NoScmInfo;
-import org.sonar.server.computation.task.projectanalysis.source.SourceHashRepository;
 
 public class ScmInfoDbLoader {
   private static final Logger LOGGER = Loggers.get(ScmInfoDbLoader.class);
 
   private final AnalysisMetadataHolder analysisMetadataHolder;
   private final DbClient dbClient;
-  private final SourceHashRepository sourceHashRepository;
   private final MergeBranchComponentUuids mergeBranchComponentUuid;
 
-  public ScmInfoDbLoader(AnalysisMetadataHolder analysisMetadataHolder, DbClient dbClient,
-    SourceHashRepository sourceHashRepository, MergeBranchComponentUuids mergeBranchComponentUuid) {
+  public ScmInfoDbLoader(AnalysisMetadataHolder analysisMetadataHolder, DbClient dbClient, MergeBranchComponentUuids mergeBranchComponentUuid) {
     this.analysisMetadataHolder = analysisMetadataHolder;
     this.dbClient = dbClient;
-    this.sourceHashRepository = sourceHashRepository;
     this.mergeBranchComponentUuid = mergeBranchComponentUuid;
   }
 
-  public ScmInfo getScmInfoFromDb(Component file) {
+  public Optional<DbScmInfo> getScmInfo(Component file) {
     Optional<String> uuid = getFileUUid(file);
-
     if (!uuid.isPresent()) {
-      return NoScmInfo.INSTANCE;
+      return Optional.empty();
     }
 
     LOGGER.trace("Reading SCM info from db for file '{}'", uuid.get());
     try (DbSession dbSession = dbClient.openSession(false)) {
       FileSourceDto dto = dbClient.fileSourceDao().selectSourceByFileUuid(dbSession, uuid.get());
-      if (dto == null || !isDtoValid(file, dto)) {
-        return NoScmInfo.INSTANCE;
+      if (dto == null) {
+        return Optional.empty();
       }
-      return DbScmInfo.create(dto.getSourceData().getLinesList()).orElse(NoScmInfo.INSTANCE);
+      return DbScmInfo.create(dto.getSourceData().getLinesList(), dto.getSrcHash());
     }
   }
 
@@ -80,10 +73,4 @@ private Optional<String> getFileUUid(Component file) {
     return Optional.empty();
   }
 
-  private boolean isDtoValid(Component file, FileSourceDto dto) {
-    if (file.getStatus() == Status.SAME) {
-      return true;
-    }
-    return sourceHashRepository.getRawSourceHash(file).equals(dto.getSrcHash());
-  }
 }

server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoImpl.java

@@ -24,6 +24,7 @@
 import java.util.stream.Collectors;
 import javax.annotation.CheckForNull;
 import javax.annotation.concurrent.Immutable;
+import static com.google.common.base.Preconditions.checkState;
 
 @Immutable
 public class ScmInfoImpl implements ScmInfo {
@@ -33,6 +34,7 @@ public class ScmInfoImpl implements ScmInfo {
   private final Map<Integer, Changeset> lineChangesets;
 
   public ScmInfoImpl(Map<Integer, Changeset> lineChangesets) {
+    checkState(!lineChangesets.isEmpty(), "A ScmInfo must have at least one Changeset and does not support any null one");
     this.lineChangesets = lineChangesets;
     this.latestChangeset = computeLatestChangeset(lineChangesets);
   }
@@ -54,7 +56,7 @@ public Changeset getChangesetForLine(int lineNumber) {
     if (changeset != null) {
       return changeset;
     }
-    throw new IllegalArgumentException("Line " + lineNumber + " doesn't have a changeset");
+    throw new IllegalArgumentException("There's no changeset on line " + lineNumber);
   }
 
   @Override

server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImpl.java

@@ -19,102 +19,112 @@
  */
 package org.sonar.server.computation.task.projectanalysis.scm;
 
-import com.google.common.base.Optional;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
 import org.sonar.scanner.protocol.output.ScannerReport;
+import org.sonar.server.computation.task.projectanalysis.analysis.AnalysisMetadataHolder;
 import org.sonar.server.computation.task.projectanalysis.batch.BatchReportReader;
 import org.sonar.server.computation.task.projectanalysis.component.Component;
+import org.sonar.server.computation.task.projectanalysis.component.Component.Status;
+import org.sonar.server.computation.task.projectanalysis.source.SourceHashRepository;
+import org.sonar.server.computation.task.projectanalysis.source.SourceLinesDiff;
 
 import static java.util.Objects.requireNonNull;
 
 public class ScmInfoRepositoryImpl implements ScmInfoRepository {
 
   private static final Logger LOGGER = Loggers.get(ScmInfoRepositoryImpl.class);
 
-  private final BatchReportReader batchReportReader;
-  private final Map<Component, ScmInfo> scmInfoCache = new HashMap<>();
+  private final BatchReportReader scannerReportReader;
+  private final Map<Component, Optional<ScmInfo>> scmInfoCache = new HashMap<>();
   private final ScmInfoDbLoader scmInfoDbLoader;
-
-  public ScmInfoRepositoryImpl(BatchReportReader batchReportReader, ScmInfoDbLoader scmInfoDbLoader) {
-    this.batchReportReader = batchReportReader;
+  private final AnalysisMetadataHolder analysisMetadata;
+  private final SourceLinesDiff sourceLinesDiff;
+  private final SourceHashRepository sourceHashRepository;
+
+  public ScmInfoRepositoryImpl(BatchReportReader scannerReportReader, AnalysisMetadataHolder analysisMetadata, ScmInfoDbLoader scmInfoDbLoader,
+    SourceLinesDiff sourceLinesDiff, SourceHashRepository sourceHashRepository) {
+    this.scannerReportReader = scannerReportReader;
+    this.analysisMetadata = analysisMetadata;
     this.scmInfoDbLoader = scmInfoDbLoader;
+    this.sourceLinesDiff = sourceLinesDiff;
+    this.sourceHashRepository = sourceHashRepository;
   }
 
   @Override
-  public Optional<ScmInfo> getScmInfo(Component component) {
+  public com.google.common.base.Optional<ScmInfo> getScmInfo(Component component) {
     requireNonNull(component, "Component cannot be null");
-    return initializeScmInfoForComponent(component);
-  }
 
-  private Optional<ScmInfo> initializeScmInfoForComponent(Component component) {
     if (component.getType() != Component.Type.FILE) {
-      return Optional.absent();
-    }
-    ScmInfo scmInfo = scmInfoCache.get(component);
-    if (scmInfo != null) {
-      return optionalOf(scmInfo);
+      return com.google.common.base.Optional.absent();
     }
 
-    scmInfo = getScmInfoForComponent(component);
-    scmInfoCache.put(component, scmInfo);
-    return optionalOf(scmInfo);
+    return toGuavaOptional(scmInfoCache.computeIfAbsent(component, this::getScmInfoForComponent));
   }
 
-  private static Optional<ScmInfo> optionalOf(ScmInfo scmInfo) {
-    if (scmInfo == NoScmInfo.INSTANCE) {
-      return Optional.absent();
-    }
-    return Optional.of(scmInfo);
+  private static com.google.common.base.Optional<ScmInfo> toGuavaOptional(Optional<ScmInfo> scmInfo) {
+    return com.google.common.base.Optional.fromNullable(scmInfo.orElse(null));
   }
 
-  private ScmInfo getScmInfoForComponent(Component component) {
-    ScannerReport.Changesets changesets = batchReportReader.readChangesets(component.getReportAttributes().getRef());
-    if (changesets == null) {
-      LOGGER.trace("No SCM info for file '{}'", component.getKey());
-      return NoScmInfo.INSTANCE;
-    }
-    if (changesets.getCopyFromPrevious()) {
-      return scmInfoDbLoader.getScmInfoFromDb(component);
+  private Optional<ScmInfo> getScmInfoForComponent(Component component) {
+    ScannerReport.Changesets changesets = scannerReportReader.readChangesets(component.getReportAttributes().getRef());
+
+    if (changesets != null) {
+      if (changesets.getChangesetCount() == 0) {
+        return generateAndMergeDb(component, changesets.getCopyFromPrevious());
+      }
+      return getScmInfoFromReport(component, changesets);
     }
-    return getScmInfoFromReport(component, changesets);
+
+    LOGGER.trace("No SCM info for file '{}'", component.getKey());
+    return generateAndMergeDb(component, false);
   }
 
-  private static ScmInfo getScmInfoFromReport(Component file, ScannerReport.Changesets changesets) {
+  private static Optional<ScmInfo> getScmInfoFromReport(Component file, ScannerReport.Changesets changesets) {
     LOGGER.trace("Reading SCM info from report for file '{}'", file.getKey());
-    return new ReportScmInfo(changesets);
+    return Optional.of(new ReportScmInfo(changesets));
   }
 
-  /**
-   * Internally used to populate cache when no ScmInfo exist.
-   */
-  enum NoScmInfo implements ScmInfo {
-    INSTANCE {
-      @Override
-      public Changeset getLatestChangeset() {
-        return notImplemented();
-      }
+  private Optional<ScmInfo> generateScmInfoForAllFile(Component file) {
+    Set<Integer> newOrChangedLines = IntStream.rangeClosed(1, file.getFileAttributes().getLines()).boxed().collect(Collectors.toSet());
+    return Optional.of(GeneratedScmInfo.create(analysisMetadata.getAnalysisDate(), newOrChangedLines));
+  }
 
-      @Override
-      public Changeset getChangesetForLine(int lineNumber) {
-        return notImplemented();
-      }
+  private ScmInfo removeAuthorAndRevision(ScmInfo info) {
+    Map<Integer, Changeset> cleanedScmInfo = info.getAllChangesets().entrySet().stream()
+      .collect(Collectors.toMap(Map.Entry::getKey, e -> removeAuthorAndRevision(e.getValue())));
+    return new ScmInfoImpl(cleanedScmInfo);
+  }
 
-      @Override
-      public boolean hasChangesetForLine(int lineNumber) {
-        return notImplemented();
-      }
+  private static Changeset removeAuthorAndRevision(Changeset changeset) {
+    return Changeset.newChangesetBuilder().setDate(changeset.getDate()).build();
+  }
 
-      @Override
-      public Iterable<Changeset> getAllChangesets() {
-        return notImplemented();
-      }
+  private Optional<ScmInfo> generateAndMergeDb(Component file, boolean copyFromPrevious) {
+    Optional<DbScmInfo> dbInfoOpt = scmInfoDbLoader.getScmInfo(file);
+    if (!dbInfoOpt.isPresent()) {
+      return generateScmInfoForAllFile(file);
+    }
 
-      private <T> T notImplemented() {
-        throw new UnsupportedOperationException("NoScmInfo does not implement any method");
-      }
+    ScmInfo scmInfo = copyFromPrevious ? dbInfoOpt.get() : removeAuthorAndRevision(dbInfoOpt.get());
+    boolean fileUnchanged = file.getStatus() == Status.SAME && sourceHashRepository.getRawSourceHash(file).equals(dbInfoOpt.get().fileHash());
+
+    if (fileUnchanged) {
+      return Optional.of(scmInfo);
     }
+
+    // generate date for new/changed lines
+    Set<Integer> newOrChangedLines = sourceLinesDiff.getNewOrChangedLines(file);
+    if (newOrChangedLines.isEmpty()) {
+      return Optional.of(scmInfo);
+    }
+    return Optional.of(GeneratedScmInfo.create(analysisMetadata.getAnalysisDate(), newOrChangedLines, scmInfo));
   }
+
 }