SONAR-6458 Persist file dependencies in the compute engine

server/sonar-server/src/main/java/org/sonar/server/computation/step/ComponentUuidsCache.java

@@ -0,0 +1,56 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.computation.step;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import org.sonar.batch.protocol.output.BatchReportReader;
+
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Waiting for having all components persisted by the Compute Engine, this class contains a cache of component uuids by their report reference
+ */
+public class ComponentUuidsCache {
+
+  private final Cache<Integer, String> componentRefToUuidCache;
+
+  public ComponentUuidsCache(final BatchReportReader reader) {
+    this.componentRefToUuidCache = CacheBuilder.newBuilder()
+      .maximumSize(500_000)
+      .build(
+        new CacheLoader<Integer, String>() {
+          @Override
+          public String load(Integer ref) {
+            return reader.readComponent(ref).getUuid();
+          }
+        });
+  }
+
+  public String getUuidFromRef(int componentRef) {
+    try {
+      return componentRefToUuidCache.get(componentRef);
+    } catch (ExecutionException e) {
+      throw new IllegalStateException(String.format("Error while retrieving uuid of component ref '%d'", componentRef), e);
+    }
+  }
+}

server/sonar-server/src/main/java/org/sonar/server/computation/step/ComputationSteps.java

@@ -48,6 +48,7 @@ public static List<Class<? extends ComputationStep>> orderedStepClasses() {
       PersistEventsStep.class,
       PersistDuplicationMeasuresStep.class,
       PersistFileSourcesStep.class,
+      PersistFileDependenciesStep.class,
 
       // Switch snapshot and purge
       SwitchSnapshotStep.class,

server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileDependenciesStep.java

@@ -0,0 +1,143 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.computation.step;
+
+import org.sonar.api.resources.Qualifiers;
+import org.sonar.api.utils.System2;
+import org.sonar.batch.protocol.Constants;
+import org.sonar.batch.protocol.output.BatchReport;
+import org.sonar.batch.protocol.output.BatchReportReader;
+import org.sonar.core.design.FileDependencyDto;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.core.persistence.MyBatis;
+import org.sonar.server.computation.ComputationContext;
+import org.sonar.server.computation.source.ReportIterator;
+import org.sonar.server.db.DbClient;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+public class PersistFileDependenciesStep implements ComputationStep {
+
+  private final DbClient dbClient;
+  private final System2 system2;
+
+  public PersistFileDependenciesStep(DbClient dbClient, System2 system2) {
+    this.dbClient = dbClient;
+    this.system2 = system2;
+  }
+
+  @Override
+  public String[] supportedProjectQualifiers() {
+    return new String[] {Qualifiers.PROJECT};
+  }
+
+  @Override
+  public void execute(ComputationContext context) {
+    DbSession session = dbClient.openSession(true);
+    try {
+      FileDependenciesContext fileDependenciesContext = new FileDependenciesContext(context, session);
+      int rootComponentRef = context.getReportMetadata().getRootComponentRef();
+      recursivelyProcessComponent(fileDependenciesContext, rootComponentRef, rootComponentRef);
+      session.commit();
+    } finally {
+      MyBatis.closeQuietly(session);
+    }
+  }
+
+  private void recursivelyProcessComponent(FileDependenciesContext fileDependenciesContext, int componentRef, int parentModuleRef) {
+    BatchReportReader reportReader = fileDependenciesContext.context.getReportReader();
+    BatchReport.Component component = reportReader.readComponent(componentRef);
+    if (component.getType().equals(Constants.ComponentType.FILE)) {
+      processFileDependenciesReport(fileDependenciesContext, component);
+    }
+
+    for (Integer childRef : component.getChildRefList()) {
+      recursivelyProcessComponent(fileDependenciesContext, childRef, componentRef);
+    }
+  }
+
+  private void processFileDependenciesReport(FileDependenciesContext fileDependenciesContext, BatchReport.Component component){
+    File fileDependencyReport = fileDependenciesContext.context.getReportReader().readFileDependencies(component.getRef());
+    if (fileDependencyReport != null) {
+      ReportIterator<BatchReport.FileDependency> fileDependenciesIterator = new ReportIterator<>(fileDependencyReport, BatchReport.FileDependency.PARSER);
+      try {
+        while (fileDependenciesIterator.hasNext()) {
+          BatchReport.FileDependency fileDependency = fileDependenciesIterator.next();
+          persistFileDependency(fileDependenciesContext, fileDependency, component.getRef());
+        }
+      } finally {
+        fileDependenciesIterator.close();
+      }
+    }
+  }
+
+  private void persistFileDependency(FileDependenciesContext fileDependenciesContext, BatchReport.FileDependency fileDependency, int fromRef){
+    int toFileRef = fileDependency.getToFileRef();
+    String fromComponentUuid = fileDependenciesContext.uuidsByRef.getUuidFromRef(fromRef);
+    String toComponentUuid = fileDependenciesContext.uuidsByRef.getUuidFromRef(toFileRef);
+    dbClient.fileDependencyDao().insert(fileDependenciesContext.session, new FileDependencyDto()
+        .setFromComponentUuid(fileDependenciesContext.uuidsByRef.getUuidFromRef(fromRef))
+        .setToComponentUuid(fileDependenciesContext.uuidsByRef.getUuidFromRef(toFileRef))
+        .setFromParentUuid(fileDependenciesContext.parentUuidsByComponentUuid.get(fromComponentUuid))
+        .setToParentUuid(fileDependenciesContext.parentUuidsByComponentUuid.get(toComponentUuid))
+        .setRootProjectSnapshotId(fileDependenciesContext.rootSnapshotId)
+        .setWeight(fileDependency.getWeight())
+        .setCreatedAt(system2.now())
+    );
+  }
+
+  private static class FileDependenciesContext {
+    private final Long rootSnapshotId;
+    private final ComponentUuidsCache uuidsByRef;
+    private final ComputationContext context;
+    private final Map<String, String> parentUuidsByComponentUuid = new HashMap<>();
+    private final DbSession session;
+
+    public FileDependenciesContext(ComputationContext context, DbSession session) {
+      this.context = context;
+      this.rootSnapshotId = context.getReportMetadata().getSnapshotId();
+      this.session = session;
+      this.uuidsByRef = new ComponentUuidsCache(context.getReportReader());
+      int rootComponentRef = context.getReportMetadata().getRootComponentRef();
+      recursivelyProcessParentByComponentCache(rootComponentRef, rootComponentRef);
+    }
+
+    private void recursivelyProcessParentByComponentCache(int componentRef, int parentModuleRef){
+      BatchReportReader reportReader = context.getReportReader();
+      BatchReport.Component component = reportReader.readComponent(componentRef);
+      BatchReport.Component parent = reportReader.readComponent(parentModuleRef);
+      if (component.getType().equals(Constants.ComponentType.FILE)) {
+        parentUuidsByComponentUuid.put(component.getUuid(), parent.getUuid());
+      }
+
+      for (Integer childRef : component.getChildRefList()) {
+        recursivelyProcessParentByComponentCache(childRef, componentRef);
+      }
+    }
+  }
+
+  @Override
+  public String getDescription() {
+    return "Persist file dependencies";
+  }
+}

server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistTestsStep.java

@@ -21,9 +21,6 @@
 package org.sonar.server.computation.step;
 
 import com.google.common.base.Joiner;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.HashBasedTable;
 import com.google.common.collect.ImmutableMap;
@@ -55,7 +52,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ExecutionException;
 
 public class PersistTestsStep implements ComputationStep {
 
@@ -245,24 +241,16 @@ private static class TestContext {
     final DbSession session;
     final ComputationContext context;
     final BatchReportReader reader;
-    final Cache<Integer, String> componentRefToUuidCache;
+    final ComponentUuidsCache componentRefToUuidCache;
     final Map<String, FileSourceDto> existingFileSourcesByUuid;
     boolean hasUnprocessedCoverageDetails = false;
 
     TestContext(ComputationContext context, DbSession session) {
       this.session = session;
       this.context = context;
       this.reader = context.getReportReader();
-      this.componentRefToUuidCache = CacheBuilder.newBuilder()
-        .maximumSize(500_000)
-        .build(
-          new CacheLoader<Integer, String>() {
-            @Override
-            public String load(Integer key) {
-              return reader.readComponent(key).getUuid();
-            }
-          });
-      existingFileSourcesByUuid = new HashMap<>();
+      this.componentRefToUuidCache = new ComponentUuidsCache(context.getReportReader());
+      this.existingFileSourcesByUuid = new HashMap<>();
       session.select("org.sonar.core.source.db.FileSourceMapper.selectHashesForProject",
         ImmutableMap.of("projectUuid", context.getProject().uuid(), "dataType", Type.TEST),
         new ResultHandler() {
@@ -275,11 +263,7 @@ public void handleResult(ResultContext context) {
     }
 
     public String getUuid(int fileRef) {
-      try {
-        return componentRefToUuidCache.get(fileRef);
-      } catch (ExecutionException e) {
-        throw new IllegalStateException(String.format("Error while retrieving uuid of component file ref '%d'", fileRef), e);
-      }
+      return componentRefToUuidCache.getUuidFromRef(fileRef);
     }
   }
 }

server/sonar-server/src/main/java/org/sonar/server/design/db/FileDependencyDao.java

@@ -34,6 +34,11 @@ public List<FileDependencyDto> selectFromParents(DbSession session, String fromP
     return session.getMapper(FileDependencyMapper.class).selectFromParents(fromParentUuid, toParentUuid, projectId);
   }
 
+  public List<FileDependencyDto> selectAll(DbSession session) {
+    return session.getMapper(FileDependencyMapper.class).selectAll();
+  }
+
+
   public void insert(DbSession session, FileDependencyDto dto) {
     session.getMapper(FileDependencyMapper.class).insert(dto);
   }

server/sonar-server/src/main/java/org/sonar/server/design/db/package-info.java

@@ -0,0 +1,24 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+@ParametersAreNonnullByDefault
+package org.sonar.server.design.db;
+
+import javax.annotation.ParametersAreNonnullByDefault;

server/sonar-server/src/main/java/org/sonar/server/design/package-info.java

@@ -0,0 +1,24 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+@ParametersAreNonnullByDefault
+package org.sonar.server.design;
+
+import javax.annotation.ParametersAreNonnullByDefault;

server/sonar-server/src/test/java/org/sonar/server/computation/step/ComponentUuidsCacheTest.java

@@ -0,0 +1,60 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.computation.step;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.batch.protocol.Constants;
+import org.sonar.batch.protocol.output.BatchReport;
+import org.sonar.batch.protocol.output.BatchReportReader;
+import org.sonar.batch.protocol.output.BatchReportWriter;
+
+import java.io.File;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ComponentUuidsCacheTest {
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  File reportDir;
+
+  @Before
+  public void setUp() throws Exception {
+    reportDir = temp.newFolder();
+  }
+
+  @Test
+  public void get_uuid_from_ref() throws Exception {
+    BatchReportWriter writer = new BatchReportWriter(reportDir);
+    writer.writeComponent(BatchReport.Component.newBuilder()
+      .setRef(1)
+      .setType(Constants.ComponentType.PROJECT)
+      .setUuid("ABCD")
+      .build());
+
+    ComponentUuidsCache cache = new ComponentUuidsCache(new BatchReportReader(reportDir));
+    assertThat(cache.getUuidFromRef(1)).isEqualTo("ABCD");
+  }
+}