Computation - add some tests and filter ComputationStep by project qu…

…alifier

server/sonar-server/src/main/java/org/sonar/server/computation/ComputationContext.java

@@ -20,27 +20,24 @@
 
 package org.sonar.server.computation;
 
-import org.sonar.batch.protocol.output.BatchOutputReader;
+import org.sonar.batch.protocol.output.BatchOutput;
 import org.sonar.core.component.ComponentDto;
 import org.sonar.core.computation.db.AnalysisReportDto;
-
-import java.util.Date;
+import org.sonar.server.computation.step.ParseReportStep;
 
 public class ComputationContext {
 
   private final AnalysisReportDto reportDto;
   private final ComponentDto project;
-  private final BatchOutputReader reportReader;
 
   /**
    * Cache of analysis date as it can be accessed several times
    */
-  private Date analysisDate = null;
+  private BatchOutput.ReportMetadata reportMetadata = null;
 
-  public ComputationContext(AnalysisReportDto reportDto, ComponentDto project, BatchOutputReader reportReader) {
+  public ComputationContext(AnalysisReportDto reportDto, ComponentDto project) {
     this.reportDto = reportDto;
     this.project = project;
-    this.reportReader = reportReader;
   }
 
   public AnalysisReportDto getReportDto() {
@@ -51,14 +48,14 @@ public ComponentDto getProject() {
     return project;
   }
 
-  public BatchOutputReader getReportReader() {
-    return reportReader;
+  public BatchOutput.ReportMetadata getReportMetadata() {
+    if (reportMetadata == null) {
+      throw new IllegalStateException("Report metadata is available after execution of " + ParseReportStep.class);
+    }
+    return reportMetadata;
   }
 
-  public Date getAnalysisDate() {
-    if (analysisDate == null) {
-      analysisDate = new Date(reportReader.readMetadata().getAnalysisDate());
-    }
-    return analysisDate;
+  public void setReportMetadata(BatchOutput.ReportMetadata m) {
+    this.reportMetadata = m;
   }
 }

server/sonar-server/src/main/java/org/sonar/server/computation/ComputationService.java

@@ -21,14 +21,12 @@
 package org.sonar.server.computation;
 
 import com.google.common.base.Throwables;
-import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.ArrayUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.ServerComponent;
 import org.sonar.api.utils.System2;
-import org.sonar.api.utils.TempFolder;
 import org.sonar.api.utils.TimeProfiler;
-import org.sonar.batch.protocol.output.BatchOutputReader;
 import org.sonar.core.activity.Activity;
 import org.sonar.core.component.ComponentDto;
 import org.sonar.core.computation.db.AnalysisReportDto;
@@ -39,38 +37,33 @@
 import org.sonar.server.computation.step.ComputationSteps;
 import org.sonar.server.db.DbClient;
 
-import java.io.File;
-
 public class ComputationService implements ServerComponent {
 
   private static final Logger LOG = LoggerFactory.getLogger(ComputationService.class);
 
   private final DbClient dbClient;
   private final ComputationSteps steps;
   private final ActivityService activityService;
-  private final TempFolder tempFolder;
 
-  public ComputationService(DbClient dbClient, ComputationSteps steps, ActivityService activityService,
-    TempFolder tempFolder) {
+  public ComputationService(DbClient dbClient, ComputationSteps steps, ActivityService activityService) {
     this.dbClient = dbClient;
     this.steps = steps;
     this.activityService = activityService;
-    this.tempFolder = tempFolder;
   }
 
   public void process(AnalysisReportDto report) {
     TimeProfiler profiler = new TimeProfiler(LOG).start(String.format(
       "#%s - %s - processing analysis report", report.getId(), report.getProjectKey()));
 
     ComponentDto project = loadProject(report);
-    File reportDir = tempFolder.newDir();
     try {
-      decompressReport(report, reportDir);
-      ComputationContext context = new ComputationContext(report, project, new BatchOutputReader(reportDir));
+      ComputationContext context = new ComputationContext(report, project);
       for (ComputationStep step : steps.orderedSteps()) {
-        TimeProfiler stepProfiler = new TimeProfiler(LOG).start(step.getDescription());
-        step.execute(context);
-        stepProfiler.stop();
+        if (ArrayUtils.contains(step.supportedProjectQualifiers(), context.getProject().qualifier())) {
+          TimeProfiler stepProfiler = new TimeProfiler(LOG).start(step.getDescription());
+          step.execute(context);
+          stepProfiler.stop();
+        }
       }
       report.succeed();
 
@@ -79,7 +72,6 @@ public void process(AnalysisReportDto report) {
       throw Throwables.propagate(e);
 
     } finally {
-      FileUtils.deleteQuietly(reportDir);
       logActivity(report, project);
       profiler.stop();
     }
@@ -104,16 +96,4 @@ private void logActivity(AnalysisReportDto report, ComponentDto project) {
       MyBatis.closeQuietly(session);
     }
   }
-
-  private void decompressReport(AnalysisReportDto report, File toDir) {
-    long startTime = System.currentTimeMillis();
-    DbSession session = dbClient.openSession(false);
-    try {
-      dbClient.analysisReportDao().selectAndDecompressToDir(session, report.getId(), toDir);
-    } finally {
-      MyBatis.closeQuietly(session);
-    }
-    long stopTime = System.currentTimeMillis();
-    LOG.info("Analysis report loaded and uncompressed in " + (stopTime - startTime) + "ms (project=" + report.getProjectKey() + ")");
-  }
 }

server/sonar-server/src/main/java/org/sonar/server/computation/db/AnalysisReportDao.java

@@ -37,7 +37,12 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.sql.*;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Types;
 import java.util.List;
 
 import static org.sonar.core.computation.db.AnalysisReportDto.Status.PENDING;
@@ -110,11 +115,12 @@ public AnalysisReportDto insert(DbSession session, AnalysisReportDto report) {
 
     Connection connection = session.getConnection();
     PreparedStatement ps = null;
+    ResultSet generatedIdRs = null;
     try {
       ps = connection.prepareStatement(
         "insert into analysis_reports " +
           " (project_key, snapshot_id, report_status, report_data, created_at, updated_at, started_at, finished_at)" +
-          " values (?, ?, ?, ?, ?, ?, ?, ?)");
+          " values (?, ?, ?, ?, ?, ?, ?, ?)", Statement.RETURN_GENERATED_KEYS);
       ps.setString(1, report.getProjectKey());
       ps.setLong(2, report.getSnapshotId());
       ps.setString(3, report.getStatus().toString());
@@ -124,11 +130,16 @@ public AnalysisReportDto insert(DbSession session, AnalysisReportDto report) {
       setLong(ps, 7, report.getStartedAt());
       setLong(ps, 8, report.getFinishedAt());
 
-      ps.executeUpdate();
+      if (ps.executeUpdate() == 1) {
+        generatedIdRs = ps.getGeneratedKeys();
+        generatedIdRs.next();
+        report.setId(generatedIdRs.getLong(1));
+      }
       connection.commit();
     } catch (SQLException | IOException e) {
       throw new IllegalStateException(String.format("Failed to insert %s in the database", report), e);
     } finally {
+      DatabaseUtils.closeQuietly(generatedIdRs);
       DatabaseUtils.closeQuietly(ps);
     }
 

server/sonar-server/src/main/java/org/sonar/server/computation/issue/IssueComputation.java

@@ -77,7 +77,7 @@ private DefaultIssue toDefaultIssue(ComputationContext context, String component
     target.setDebt(issue.hasDebtInMinutes() ? Duration.create(issue.getDebtInMinutes()) : null);
     if (issue.hasDiffFields()) {
       FieldDiffs fieldDiffs = FieldDiffs.parse(issue.getDiffFields());
-      fieldDiffs.setCreationDate(context.getAnalysisDate());
+      fieldDiffs.setCreationDate(new Date(context.getReportMetadata().getAnalysisDate()));
       target.setCurrentChange(fieldDiffs);
     }
     target.setStatus(issue.getStatus());

server/sonar-server/src/main/java/org/sonar/server/computation/step/ApplyPermissionsStep.java

@@ -20,6 +20,7 @@
 
 package org.sonar.server.computation.step;
 
+import org.sonar.api.resources.Qualifiers;
 import org.sonar.server.computation.ComputationContext;
 import org.sonar.server.issue.index.IssueAuthorizationIndexer;
 
@@ -31,6 +32,11 @@ public ApplyPermissionsStep(IssueAuthorizationIndexer indexer) {
     this.indexer = indexer;
   }
 
+  @Override
+  public String[] supportedProjectQualifiers() {
+    return new String[] {Qualifiers.PROJECT, Qualifiers.VIEW};
+  }
+
   @Override
   public void execute(ComputationContext context) {
     indexer.index();

server/sonar-server/src/main/java/org/sonar/server/computation/step/ComputationStep.java

@@ -27,6 +27,11 @@
  */
 public interface ComputationStep {
 
+  /**
+   * Allows to distinguish standard projects from view projects
+   */
+  String[] supportedProjectQualifiers();
+
   void execute(ComputationContext context);
 
   String getDescription();

server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexComponentsStep.java

@@ -20,6 +20,7 @@
 
 package org.sonar.server.computation.step;
 
+import org.sonar.api.resources.Qualifiers;
 import org.sonar.core.resource.ResourceIndexerDao;
 import org.sonar.server.computation.ComputationContext;
 
@@ -33,6 +34,11 @@ public IndexComponentsStep(ResourceIndexerDao resourceIndexerDao) {
     this.resourceIndexerDao = resourceIndexerDao;
   }
 
+  @Override
+  public String[] supportedProjectQualifiers() {
+    return new String[] {Qualifiers.PROJECT, Qualifiers.VIEW};
+  }
+
   @Override
   public void execute(ComputationContext context) {
     resourceIndexerDao.indexProject(context.getProject().getId());

server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexIssuesStep.java

@@ -20,6 +20,7 @@
 
 package org.sonar.server.computation.step;
 
+import org.sonar.api.resources.Qualifiers;
 import org.sonar.server.computation.ComputationContext;
 import org.sonar.server.issue.index.IssueAuthorizationIndexer;
 import org.sonar.server.issue.index.IssueIndexer;
@@ -34,6 +35,11 @@ public IndexIssuesStep(IssueAuthorizationIndexer authorizationIndexer, IssueInde
     this.indexer = indexer;
   }
 
+  @Override
+  public String[] supportedProjectQualifiers() {
+    return new String[] {Qualifiers.PROJECT};
+  }
+
   @Override
   public void execute(ComputationContext context) {
     authorizationIndexer.index();

server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexSourceLinesStep.java

@@ -19,6 +19,7 @@
  */
 package org.sonar.server.computation.step;
 
+import org.sonar.api.resources.Qualifiers;
 import org.sonar.server.computation.ComputationContext;
 import org.sonar.server.source.index.SourceLineIndexer;
 
@@ -30,6 +31,11 @@ public IndexSourceLinesStep(SourceLineIndexer indexer) {
     this.indexer = indexer;
   }
 
+  @Override
+  public String[] supportedProjectQualifiers() {
+    return new String[] {Qualifiers.PROJECT};
+  }
+
   @Override
   public void execute(ComputationContext context) {
     indexer.index();

server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexViewsStep.java

@@ -32,11 +32,14 @@ public IndexViewsStep(ViewIndexer indexer) {
     this.indexer = indexer;
   }
 
+  @Override
+  public String[] supportedProjectQualifiers() {
+    return new String[] {Qualifiers.VIEW};
+  }
+
   @Override
   public void execute(ComputationContext context) {
-    if (context.getProject().qualifier().equals(Qualifiers.VIEW)) {
-      indexer.index(context.getProject().uuid());
-    }
+    indexer.index(context.getProject().uuid());
   }
 
   @Override

server/sonar-server/src/main/java/org/sonar/server/computation/step/ParseReportStep.java

@@ -20,33 +20,82 @@
 
 package org.sonar.server.computation.step;
 
-import org.sonar.batch.protocol.output.BatchOutputReader;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.resources.Qualifiers;
+import org.sonar.api.utils.TempFolder;
 import org.sonar.batch.protocol.output.BatchOutput;
+import org.sonar.batch.protocol.output.BatchOutputReader;
+import org.sonar.core.computation.db.AnalysisReportDto;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.core.persistence.MyBatis;
 import org.sonar.server.computation.ComputationContext;
 import org.sonar.server.computation.issue.IssueComputation;
+import org.sonar.server.db.DbClient;
+
+import java.io.File;
 
 public class ParseReportStep implements ComputationStep {
 
+  private static final Logger LOG = LoggerFactory.getLogger(ParseReportStep.class);
+
   private final IssueComputation issueComputation;
+  private final DbClient dbClient;
+  private final TempFolder tempFolder;
 
-  public ParseReportStep(IssueComputation issueComputation) {
+  public ParseReportStep(IssueComputation issueComputation, DbClient dbClient, TempFolder tempFolder) {
     this.issueComputation = issueComputation;
+    this.dbClient = dbClient;
+    this.tempFolder = tempFolder;
+  }
+
+  @Override
+  public String[] supportedProjectQualifiers() {
+    return new String[] {Qualifiers.PROJECT, Qualifiers.VIEW};
   }
 
   @Override
   public void execute(ComputationContext context) {
-    int rootComponentRef = context.getReportReader().readMetadata().getRootComponentRef();
-    processComponent(context, rootComponentRef);
-    issueComputation.afterReportProcessing();
+    File reportDir = tempFolder.newDir();
+    try {
+      // extract compressed report from database and uncompress it in temporary directory
+      extractReport(context.getReportDto(), reportDir);
+
+      // prepare parsing of report
+      BatchOutputReader reader = new BatchOutputReader(reportDir);
+      BatchOutput.ReportMetadata reportMetadata = reader.readMetadata();
+      context.setReportMetadata(reportMetadata);
+
+      // and parse!
+      int rootComponentRef = reportMetadata.getRootComponentRef();
+      recursivelyProcessComponent(reader, context, rootComponentRef);
+      issueComputation.afterReportProcessing();
+
+    } finally {
+      FileUtils.deleteQuietly(reportDir);
+    }
+  }
+
+  private void extractReport(AnalysisReportDto report, File toDir) {
+    long startTime = System.currentTimeMillis();
+    DbSession session = dbClient.openSession(false);
+    try {
+      dbClient.analysisReportDao().selectAndDecompressToDir(session, report.getId(), toDir);
+    } finally {
+      MyBatis.closeQuietly(session);
+    }
+    long stopTime = System.currentTimeMillis();
+    LOG.info(String.format("Report extracted in %dms | uncompressed size=%s | project=%s",
+      stopTime - startTime, FileUtils.byteCountToDisplaySize(FileUtils.sizeOf(toDir)), report.getProjectKey()));
   }
 
-  private void processComponent(ComputationContext context, int componentRef) {
-    BatchOutputReader reader = context.getReportReader();
-    BatchOutput.ReportComponent component = reader.readComponent(componentRef);
-    issueComputation.processComponentIssues(context, component.getUuid(), reader.readComponentIssues(componentRef));
+  private void recursivelyProcessComponent(BatchOutputReader reportReader, ComputationContext context, int componentRef) {
+    BatchOutput.ReportComponent component = reportReader.readComponent(componentRef);
+    issueComputation.processComponentIssues(context, component.getUuid(), reportReader.readComponentIssues(componentRef));
 
     for (Integer childRef : component.getChildRefsList()) {
-      processComponent(context, childRef);
+      recursivelyProcessComponent(reportReader, context, childRef);
     }
   }