SONAR-11463 Relocate issues from modules/dirs to root project

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java

@@ -31,6 +31,7 @@
 import javax.annotation.Nullable;
 import org.sonar.api.internal.apachecommons.lang.StringUtils;
 import org.sonar.ce.task.projectanalysis.analysis.Branch;
+import org.sonar.ce.task.projectanalysis.issue.IssueRelocationToRoot;
 import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.db.component.SnapshotDto;
 import org.sonar.scanner.protocol.output.ScannerReport;
@@ -67,6 +68,7 @@ public class ComponentTreeBuilder {
   private final Branch branch;
   @Nullable
   private final SnapshotDto baseAnalysis;
+  private final IssueRelocationToRoot issueRelocationToRoot;
 
   private ScannerReport.Component rootComponent;
   private String scmBasePath;
@@ -77,7 +79,7 @@ public ComponentTreeBuilder(
     Function<String, String> uuidSupplier,
     Function<Integer, ScannerReport.Component> scannerComponentSupplier,
     Project project,
-    Branch branch, @Nullable SnapshotDto baseAnalysis) {
+    Branch branch, @Nullable SnapshotDto baseAnalysis, IssueRelocationToRoot issueRelocationToRoot) {
 
     this.keyGenerator = keyGenerator;
     this.publicKeyGenerator = publicKeyGenerator;
@@ -86,17 +88,18 @@ public ComponentTreeBuilder(
     this.project = project;
     this.branch = branch;
     this.baseAnalysis = baseAnalysis;
+    this.issueRelocationToRoot = issueRelocationToRoot;
   }
 
   public Component buildProject(ScannerReport.Component project, String scmBasePath) {
     this.rootComponent = project;
     this.scmBasePath = trimToNull(scmBasePath);
 
-    Node root = buildFolderHierarchy(project);
-    return buildNode(root, "");
+    Node root = createProjectHierarchy(project);
+    return buildComponent(root, "");
   }
 
-  private Node buildFolderHierarchy(ScannerReport.Component rootComponent) {
+  private Node createProjectHierarchy(ScannerReport.Component rootComponent) {
     Preconditions.checkArgument(rootComponent.getType() == ScannerReport.Component.ComponentType.PROJECT, "Expected root component of type 'PROJECT'");
 
     LinkedList<ScannerReport.Component> queue = new LinkedList<>();
@@ -112,16 +115,11 @@ private Node buildFolderHierarchy(ScannerReport.Component rootComponent) {
       ScannerReport.Component component = queue.removeFirst();
       switch (component.getType()) {
         case FILE:
-          addFileOrDirectory(root, component);
+          addFile(root, component);
           break;
         case MODULE:
-
-          component.getChildRefList().stream()
-            .map(scannerComponentSupplier)
-            .forEach(queue::addLast);
-          break;
         case DIRECTORY:
-          addFileOrDirectory(root, component);
+          issueRelocationToRoot.relocate(rootComponent, component);
           component.getChildRefList().stream()
             .map(scannerComponentSupplier)
             .forEach(queue::addLast);
@@ -133,9 +131,8 @@ private Node buildFolderHierarchy(ScannerReport.Component rootComponent) {
     return root;
   }
 
-  private static void addFileOrDirectory(Node root, ScannerReport.Component file) {
-    Preconditions.checkArgument(file.getType() != ScannerReport.Component.ComponentType.FILE || !StringUtils.isEmpty(file.getProjectRelativePath()),
-      "Files should have a relative path: " + file);
+  private static void addFile(Node root, ScannerReport.Component file) {
+    Preconditions.checkArgument(!StringUtils.isEmpty(file.getProjectRelativePath()), "Files should have a project relative path: " + file);
     String[] split = StringUtils.split(file.getProjectRelativePath(), '/');
     Node currentNode = root.children().computeIfAbsent("", k -> new Node());
 
@@ -145,7 +142,7 @@ private static void addFileOrDirectory(Node root, ScannerReport.Component file)
     currentNode.reportComponent = file;
   }
 
-  private Component buildNode(Node node, String currentPath) {
+  private Component buildComponent(Node node, String currentPath) {
     List<Component> childComponents = buildChildren(node, currentPath);
     ScannerReport.Component component = node.reportComponent();
 
@@ -157,22 +154,23 @@ private Component buildNode(Node node, String currentPath) {
       }
     }
 
-    return buildDirectory(currentPath, component, childComponents);
+    return buildDirectory(currentPath, childComponents);
   }
 
   private List<Component> buildChildren(Node node, String currentPath) {
     List<Component> children = new ArrayList<>();
 
     for (Map.Entry<String, Node> e : node.children().entrySet()) {
       String path = buildPath(currentPath, e.getKey());
-      Node n = e.getValue();
+      Node childNode = e.getValue();
 
-      while (n.children().size() == 1 && n.children().values().iterator().next().children().size() > 0) {
-        Map.Entry<String, Node> childEntry = n.children().entrySet().iterator().next();
+      // collapse folders that only contain one folder
+      while (childNode.children().size() == 1 && childNode.children().values().iterator().next().children().size() > 0) {
+        Map.Entry<String, Node> childEntry = childNode.children().entrySet().iterator().next();
         path = buildPath(path, childEntry.getKey());
-        n = childEntry.getValue();
+        childNode = childEntry.getValue();
       }
-      children.add(buildNode(n, path));
+      children.add(buildComponent(childNode, path));
     }
     return children;
   }
@@ -216,18 +214,17 @@ private ComponentImpl buildFile(ScannerReport.Component component) {
       .build();
   }
 
-  private ComponentImpl buildDirectory(String path, @Nullable ScannerReport.Component scannerComponent, List<Component> children) {
+  private ComponentImpl buildDirectory(String path, List<Component> children) {
     String nonEmptyPath = path.isEmpty() ? "/" : path;
     String key = keyGenerator.generateKey(rootComponent, nonEmptyPath);
     String publicKey = publicKeyGenerator.generateKey(rootComponent, nonEmptyPath);
-    Integer ref = scannerComponent != null ? scannerComponent.getRef() : null;
     return ComponentImpl.builder(Component.Type.DIRECTORY)
       .setUuid(uuidSupplier.apply(key))
       .setDbKey(key)
       .setKey(publicKey)
       .setName(publicKey)
       .setStatus(convertStatus(FileStatus.UNAVAILABLE))
-      .setReportAttributes(createAttributesBuilder(ref, nonEmptyPath, scmBasePath, path).build())
+      .setReportAttributes(createAttributesBuilder(null, nonEmptyPath, scmBasePath, path).build())
       .addChildren(children)
       .build();
   }
@@ -359,7 +356,7 @@ private static ReportAttributes.Builder createAttributesBuilder(@Nullable Intege
   @CheckForNull
   private static String computeScmPath(@Nullable String scmBasePath, String scmRelativePath) {
     if (scmRelativePath.isEmpty()) {
-      return null;
+      return scmBasePath;
     }
     if (scmBasePath == null) {
       return scmRelativePath;

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactory.java

@@ -22,20 +22,69 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.commons.lang.StringUtils;
+import org.sonar.api.resources.Qualifiers;
+import org.sonar.core.component.ComponentKeys;
 import org.sonar.core.util.Uuids;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
+import org.sonar.db.component.ComponentDto;
+import org.sonar.db.component.ComponentWithModuleUuidDto;
 import org.sonar.db.component.KeyWithUuidDto;
 
 public class ComponentUuidFactory {
-
   private final Map<String, String> uuidsByKey = new HashMap<>();
 
   public ComponentUuidFactory(DbClient dbClient, DbSession dbSession, String rootKey) {
-    List<KeyWithUuidDto> keys = dbClient.componentDao().selectUuidsByKeyFromProjectKey(dbSession, rootKey);
-    for (KeyWithUuidDto dto : keys) {
-      uuidsByKey.put(dto.key(), dto.uuid());
+    Map<String, String> modulePathsByUuid = loadModulePathsByUuid(dbClient, dbSession, rootKey);
+
+    if (modulePathsByUuid.isEmpty()) {
+      // only contains root project
+      List<KeyWithUuidDto> keys = dbClient.componentDao().selectUuidsByKeyFromProjectKey(dbSession, rootKey);
+      keys.forEach(dto -> uuidsByKey.put(dto.key(), dto.uuid()));
+    } else {
+      List<ComponentWithModuleUuidDto> dtos = loadComponentsWithModuleUuid(dbClient, dbSession, rootKey);
+      for (ComponentWithModuleUuidDto dto : dtos) {
+        String pathFromRootProject = modulePathsByUuid.get(dto.moduleUuid());
+        String componentPath = StringUtils.isEmpty(pathFromRootProject) ? dto.path() : (pathFromRootProject + "/" + dto.path());
+        uuidsByKey.put(ComponentKeys.createEffectiveKey(rootKey, componentPath), dto.uuid());
+      }
+    }
+  }
+
+  private static List<ComponentWithModuleUuidDto> loadComponentsWithModuleUuid(DbClient dbClient, DbSession dbSession, String rootKey) {
+    return dbClient.componentDao().selectComponentsWithModuleUuidFromProjectKey(dbSession, rootKey);
+  }
+
+  private static Map<String, String> loadModulePathsByUuid(DbClient dbClient, DbSession dbSession, String rootKey) {
+    List<ComponentDto> moduleDtos = dbClient.componentDao()
+      .selectEnabledModulesFromProjectKey(dbSession, rootKey, false).stream()
+      .filter(c -> Qualifiers.MODULE.equals(c.qualifier()))
+      .collect(Collectors.toList());
+
+    Map<String, ComponentDto> dtoByUuid = moduleDtos.stream()
+      .collect(Collectors.toMap(ComponentDto::uuid, dto -> dto));
+
+    Map<String, String> modulePathByUuid = new HashMap<>();
+
+    for (ComponentDto dto : moduleDtos) {
+      String modulePath = null;
+      ComponentDto currentDto = dto;
+      while (currentDto != null && currentDto.moduleUuid() != null) {
+        String path = currentDto.path();
+        if (modulePath == null) {
+          modulePath = path;
+        } else {
+          modulePath = path + "/" + modulePath;
+        }
+        currentDto = dtoByUuid.get(currentDto.moduleUuid());
+      }
+
+      modulePathByUuid.put(dto.uuid(), modulePath);
     }
+
+    return modulePathByUuid;
   }
 
   /**

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java

@@ -64,6 +64,7 @@
 import org.sonar.ce.task.projectanalysis.issue.IssueCounter;
 import org.sonar.ce.task.projectanalysis.issue.IssueCreationDateCalculator;
 import org.sonar.ce.task.projectanalysis.issue.IssueLifecycle;
+import org.sonar.ce.task.projectanalysis.issue.IssueRelocationToRoot;
 import org.sonar.ce.task.projectanalysis.issue.IssueTrackingDelegator;
 import org.sonar.ce.task.projectanalysis.issue.IssueVisitors;
 import org.sonar.ce.task.projectanalysis.issue.IssuesRepositoryVisitor;
@@ -232,6 +233,7 @@ private static List<Object> componentClasses() {
       ComponentsWithUnprocessedIssues.class,
       ComponentIssuesRepositoryImpl.class,
       IssueFilter.class,
+      IssueRelocationToRoot.class,
 
       // common rules
       CommonRuleEngineImpl.class,

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/duplication/DuplicationMeasures.java

@@ -156,10 +156,7 @@ private void initializeForProjectView(CounterInitializationContext context) {
 
     private static int getMeasure(CounterInitializationContext context, String metricKey) {
       Optional<Measure> files = context.getMeasure(metricKey);
-      if (files.isPresent()) {
-        return files.get().getIntValue();
-      }
-      return 0;
+      return files.map(Measure::getIntValue).orElse(0);
     }
   }
 

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IssueRelocationToRoot.java

@@ -0,0 +1,75 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.issue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.sonar.ce.task.projectanalysis.batch.BatchReportReader;
+import org.sonar.core.util.CloseableIterator;
+import org.sonar.scanner.protocol.output.ScannerReport;
+
+public class IssueRelocationToRoot {
+  private final BatchReportReader reader;
+  private final List<ScannerReport.Issue> movedIssues = new ArrayList<>();
+
+  public IssueRelocationToRoot(BatchReportReader reader) {
+    this.reader = reader;
+  }
+
+  public List<ScannerReport.Issue> getMovedIssues() {
+    return Collections.unmodifiableList(movedIssues);
+  }
+
+  public void relocate(ScannerReport.Component root, ScannerReport.Component component) {
+    CloseableIterator<ScannerReport.Issue> issueIt = reader.readComponentIssues(component.getRef());
+    while (issueIt.hasNext()) {
+      ScannerReport.Issue issue = issueIt.next();
+      movedIssues.add(moveIssueToRoot(root, component, issue));
+    }
+  }
+
+  private static ScannerReport.Issue moveIssueToRoot(ScannerReport.Component root, ScannerReport.Component component, ScannerReport.Issue issue) {
+    return ScannerReport.Issue.newBuilder(issue)
+      .clearFlow()
+      .clearTextRange()
+      .addAllFlow(issue.getFlowList().stream()
+        .map(flow -> moveFlow(root, component, flow))
+        .collect(Collectors.toList()))
+      .build();
+  }
+
+  private static ScannerReport.IssueLocation moveLocation(ScannerReport.Component root, ScannerReport.Component component, ScannerReport.IssueLocation location) {
+    String msg = "[" + component.getProjectRelativePath() + "] " + location.getMsg();
+    return ScannerReport.IssueLocation.newBuilder()
+      .setComponentRef(root.getRef())
+      .setMsg(msg)
+      .build();
+  }
+
+  private static ScannerReport.Flow moveFlow(ScannerReport.Component root, ScannerReport.Component component, ScannerReport.Flow flow) {
+    return ScannerReport.Flow.newBuilder()
+      .addAllLocation(flow.getLocationList().stream()
+        .map(location -> moveLocation(root, component, location))
+        .collect(Collectors.toList()))
+      .build();
+  }
+}

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/TrackerRawInputFactory.java

@@ -61,17 +61,19 @@ public class TrackerRawInputFactory {
   private final SourceLinesHashRepository sourceLinesHash;
   private final RuleRepository ruleRepository;
   private final ActiveRulesHolder activeRulesHolder;
+  private final IssueRelocationToRoot issueRelocationToRoot;
 
   public TrackerRawInputFactory(TreeRootHolder treeRootHolder, BatchReportReader reportReader,
     SourceLinesHashRepository sourceLinesHash, CommonRuleEngine commonRuleEngine, IssueFilter issueFilter, RuleRepository ruleRepository,
-    ActiveRulesHolder activeRulesHolder) {
+    ActiveRulesHolder activeRulesHolder, IssueRelocationToRoot issueRelocationToRoot) {
     this.treeRootHolder = treeRootHolder;
     this.reportReader = reportReader;
     this.sourceLinesHash = sourceLinesHash;
     this.commonRuleEngine = commonRuleEngine;
     this.issueFilter = issueFilter;
     this.ruleRepository = ruleRepository;
     this.activeRulesHolder = activeRulesHolder;
+    this.issueRelocationToRoot = issueRelocationToRoot;
   }
 
   public Input<DefaultIssue> create(Component component) {
@@ -144,6 +146,13 @@ protected List<DefaultIssue> loadIssues() {
         }
       }
 
+      if (component.getType() == Component.Type.PROJECT) {
+        // TODO apply filters?
+        issueRelocationToRoot.getMovedIssues().stream()
+          .map(i -> toIssue(getLineHashSequence(), i))
+          .forEach(result::add);
+      }
+
       return result;
     }
 

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStep.java

@@ -31,6 +31,7 @@
 import org.sonar.ce.task.projectanalysis.component.ComponentUuidFactory;
 import org.sonar.ce.task.projectanalysis.component.DefaultBranchImpl;
 import org.sonar.ce.task.projectanalysis.component.MutableTreeRootHolder;
+import org.sonar.ce.task.projectanalysis.issue.IssueRelocationToRoot;
 import org.sonar.ce.task.step.ComputationStep;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
@@ -47,13 +48,15 @@ public class BuildComponentTreeStep implements ComputationStep {
   private final BatchReportReader reportReader;
   private final MutableTreeRootHolder treeRootHolder;
   private final MutableAnalysisMetadataHolder analysisMetadataHolder;
+  private final IssueRelocationToRoot issueRelocationToRoot;
 
   public BuildComponentTreeStep(DbClient dbClient, BatchReportReader reportReader,
-    MutableTreeRootHolder treeRootHolder, MutableAnalysisMetadataHolder analysisMetadataHolder) {
+    MutableTreeRootHolder treeRootHolder, MutableAnalysisMetadataHolder analysisMetadataHolder, IssueRelocationToRoot issueRelocationToRoot) {
     this.dbClient = dbClient;
     this.reportReader = reportReader;
     this.treeRootHolder = treeRootHolder;
     this.analysisMetadataHolder = analysisMetadataHolder;
+    this.issueRelocationToRoot = issueRelocationToRoot;
   }
 
   @Override
@@ -82,7 +85,7 @@ public void execute(ComputationStep.Context context) {
         reportReader::readComponent,
         analysisMetadataHolder.getProject(),
         analysisMetadataHolder.getBranch(),
-        baseAnalysis);
+        baseAnalysis, issueRelocationToRoot);
       String relativePathFromScmRoot = reportReader.readMetadata().getRelativePathFromScmRoot();
 
       Component reportTreeRoot = builder.buildProject(reportProject, relativePathFromScmRoot);