SONAR-7903 persist analysis reports in db

instead of file system (data/ce/reports). That allows
support of clustering.

server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java

@@ -89,9 +89,9 @@ public void real_start() throws IOException {
       .hasSize(
         CONTAINER_ITSELF
           + 75 // level 4
-          + 7 // content of CeModule
+          + 6 // content of CeModule
           + 7 // content of CeQueueModule
-          + 4 // content of ProjectAnalysisTaskModule
+          + 3 // content of ProjectAnalysisTaskModule
           + 4 // content of CeTaskProcessorModule
     );
     assertThat(picoContainer.getParent().getComponentAdapters()).hasSize(
@@ -105,7 +105,7 @@ public void real_start() throws IOException {
     assertThat(picoContainer.getParent().getParent().getParent().getComponentAdapters()).hasSize(
       COMPONENTS_IN_LEVEL_1_AT_CONSTRUCTION
         + 26 // level 1
-        + 46 // content of DaoModule
+        + 47 // content of DaoModule
         + 2 // content of EsSearchModule
         + 55 // content of CorePropertyDefinitions
         + 1 // content of CePropertyDefinitions

server/sonar-server/src/main/java/org/sonar/ce/CeModule.java

@@ -21,10 +21,9 @@
 
 import org.sonar.ce.log.CeLogging;
 import org.sonar.ce.queue.CeQueueImpl;
-import org.sonar.ce.queue.report.ReportFiles;
-import org.sonar.ce.queue.report.ReportSubmitter;
 import org.sonar.ce.taskprocessor.ReportTaskProcessorDeclaration;
 import org.sonar.core.platform.Module;
+import org.sonar.server.computation.queue.ReportSubmitter;
 
 public class CeModule extends Module {
   @Override
@@ -34,8 +33,7 @@ protected void configureModule() {
       // Queue
       CeQueueImpl.class,
       ReportSubmitter.class,
-      ReportFiles.class,
-
+
       // Core tasks processors
       ReportTaskProcessorDeclaration.class);
   }

server/sonar-server/src/main/java/org/sonar/ce/queue/CeQueueImpl.java

@@ -41,29 +41,21 @@
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Predicates.notNull;
 import static com.google.common.collect.FluentIterable.from;
+import static java.util.Collections.singletonList;
 import static org.sonar.db.component.ComponentDtoFunctions.toUuid;
 
 @ComputeEngineSide
 public class CeQueueImpl implements CeQueue {
 
   private final DbClient dbClient;
   private final UuidFactory uuidFactory;
-  private final CeQueueListener[] listeners;
 
   // state
   private AtomicBoolean submitPaused = new AtomicBoolean(false);
 
-  /**
-   * Constructor in case there is no CeQueueListener
-   */
   public CeQueueImpl(DbClient dbClient, UuidFactory uuidFactory) {
-    this(dbClient, uuidFactory, new CeQueueListener[] {});
-  }
-
-  public CeQueueImpl(DbClient dbClient, UuidFactory uuidFactory, CeQueueListener[] listeners) {
     this.dbClient = dbClient;
     this.uuidFactory = uuidFactory;
-    this.listeners = listeners;
   }
 
   @Override
@@ -126,7 +118,7 @@ private List<CeTask> loadTasks(DbSession dbSession, List<CeQueueDto> dtos) {
       .toSet();
     Map<String, ComponentDto> componentDtoByUuid = from(dbClient.componentDao()
       .selectByUuids(dbSession, componentUuids))
-      .uniqueIndex(toUuid());
+        .uniqueIndex(toUuid());
 
     return from(dtos)
       .transform(new CeQueueDtoToCeTask(componentDtoByUuid))
@@ -150,10 +142,9 @@ public boolean cancel(String taskUuid) {
   }
 
   protected void cancelImpl(DbSession dbSession, CeQueueDto q) {
-    CeTask task = loadTask(dbSession, q);
     CeActivityDto activityDto = new CeActivityDto(q);
     activityDto.setStatus(CeActivityDto.Status.CANCELED);
-    remove(dbSession, task, q, activityDto);
+    remove(dbSession, q, activityDto);
   }
 
   @Override
@@ -177,13 +168,11 @@ protected int cancelAll(boolean includeInProgress) {
     }
   }
 
-  protected void remove(DbSession dbSession, CeTask task, CeQueueDto queueDto, CeActivityDto activityDto) {
+  protected void remove(DbSession dbSession, CeQueueDto queueDto, CeActivityDto activityDto) {
     dbClient.ceActivityDao().insert(dbSession, activityDto);
     dbClient.ceQueueDao().deleteByUuid(dbSession, queueDto.getUuid());
+    dbClient.ceTaskDataDao().deleteByUuids(dbSession, singletonList(queueDto.getUuid()));
     dbSession.commit();
-    for (CeQueueListener listener : listeners) {
-      listener.onRemoved(task, activityDto.getStatus());
-    }
   }
 
   @Override

server/sonar-server/src/main/java/org/sonar/server/ce/ws/SubmitAction.java

@@ -25,7 +25,7 @@
 import org.sonar.api.server.ws.Response;
 import org.sonar.api.server.ws.WebService;
 import org.sonar.ce.queue.CeTask;
-import org.sonar.ce.queue.report.ReportSubmitter;
+import org.sonar.server.computation.queue.ReportSubmitter;
 import org.sonar.server.ws.WsUtils;
 import org.sonarqube.ws.WsCe;
 

server/sonar-server/src/main/java/org/sonar/server/computation/CeQueueModule.java

@@ -19,7 +19,6 @@
  */
 package org.sonar.server.computation;
 
-import org.sonar.ce.queue.report.ReportFiles;
 import org.sonar.core.platform.Module;
 import org.sonar.server.computation.monitoring.CEQueueStatusImpl;
 import org.sonar.server.computation.monitoring.CeTasksMBeanImpl;
@@ -40,8 +39,7 @@ protected void configureModule() {
 
       // queue cleaning
       CeQueueCleaner.class,
-      ReportFiles.class,
-
+
       // init queue state and queue processing
       CeQueueInitializer.class);
   }

server/sonar-server/src/main/java/org/sonar/server/computation/queue/CeQueueCleaner.java

@@ -19,17 +19,13 @@
  */
 package org.sonar.server.computation.queue;
 
-import java.util.HashSet;
-import java.util.Set;
-import org.sonar.api.platform.ServerUpgradeStatus;
+import java.util.List;
 import org.sonar.api.ce.ComputeEngineSide;
+import org.sonar.api.platform.ServerUpgradeStatus;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
-import org.sonar.ce.queue.report.ReportFiles;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
-import org.sonar.db.ce.CeQueueDto;
-import org.sonar.db.ce.CeTaskTypes;
 
 /**
  * Cleans-up the Compute Engine queue and resets the JMX counters.
@@ -42,13 +38,11 @@ public class CeQueueCleaner {
 
   private final DbClient dbClient;
   private final ServerUpgradeStatus serverUpgradeStatus;
-  private final ReportFiles reportFiles;
   private final InternalCeQueue queue;
 
-  public CeQueueCleaner(DbClient dbClient, ServerUpgradeStatus serverUpgradeStatus, ReportFiles reportFiles, InternalCeQueue queue) {
+  public CeQueueCleaner(DbClient dbClient, ServerUpgradeStatus serverUpgradeStatus, InternalCeQueue queue) {
     this.dbClient = dbClient;
     this.serverUpgradeStatus = serverUpgradeStatus;
-    this.reportFiles = reportFiles;
     this.queue = queue;
   }
 
@@ -72,21 +66,11 @@ private void verifyConsistency(DbSession dbSession) {
     dbClient.ceQueueDao().resetAllToPendingStatus(dbSession);
     dbSession.commit();
 
-    // verify that the report files are available for the tasks in queue
-    Set<String> uuidsInQueue = new HashSet<>();
-    for (CeQueueDto queueDto : dbClient.ceQueueDao().selectAllInAscOrder(dbSession)) {
-      uuidsInQueue.add(queueDto.getUuid());
-      if (CeTaskTypes.REPORT.equals(queueDto.getTaskType()) && !reportFiles.fileForUuid(queueDto.getUuid()).exists()) {
-        // the report is not available on file system
-        queue.cancel(dbSession, queueDto);
-      }
-    }
-
-    // clean-up filesystem
-    for (String uuid : reportFiles.listUuids()) {
-      if (!uuidsInQueue.contains(uuid)) {
-        reportFiles.deleteIfExists(uuid);
-      }
-    }
+    // Reports that have been processed are not kept in database yet.
+    // They are supposed to be systematically dropped.
+    // Let's clean-up orphans if any.
+    List<String> uuids = dbClient.ceTaskDataDao().selectUuidsNotInQueue(dbSession);
+    dbClient.ceTaskDataDao().deleteByUuids(dbSession, uuids);
+    dbSession.commit();
   }
 }

server/sonar-server/src/main/java/org/sonar/server/computation/queue/InternalCeQueueImpl.java

@@ -26,7 +26,6 @@
 import org.sonar.api.utils.System2;
 import org.sonar.ce.monitoring.CEQueueStatus;
 import org.sonar.ce.queue.CeQueueImpl;
-import org.sonar.ce.queue.CeQueueListener;
 import org.sonar.ce.queue.CeTask;
 import org.sonar.ce.queue.CeTaskResult;
 import org.sonar.core.util.UuidFactory;
@@ -48,8 +47,8 @@ public class InternalCeQueueImpl extends CeQueueImpl implements InternalCeQueue
   private AtomicBoolean peekPaused = new AtomicBoolean(false);
 
   public InternalCeQueueImpl(System2 system2, DbClient dbClient, UuidFactory uuidFactory,
-    CEQueueStatus queueStatus, CeQueueListener[] listeners) {
-    super(dbClient, uuidFactory, listeners);
+    CEQueueStatus queueStatus) {
+    super(dbClient, uuidFactory);
     this.system2 = system2;
     this.dbClient = dbClient;
     this.queueStatus = queueStatus;
@@ -92,7 +91,7 @@ public void remove(CeTask task, CeActivityDto.Status status, CeTaskResult taskRe
       activityDto.setStatus(status);
       updateQueueStatus(status, activityDto);
       updateTaskResult(activityDto, taskResult);
-      remove(dbSession, task, queueDto.get(), activityDto);
+      remove(dbSession, queueDto.get(), activityDto);
 
     } finally {
       dbClient.closeSession(dbSession);