SONAR-8221 Fix project measures indexing on MySQL

Replace streaming of projects by first loading all projects once, then load measures project by project

server/sonar-server/src/main/java/org/sonar/server/component/es/ProjectMeasuresDoc.java

@@ -94,10 +94,10 @@ public ProjectMeasuresDoc setMeasures(Collection<Map<String, Object>> measures)
     return this;
   }
 
-  public ProjectMeasuresDoc setMeasuresFromMap(Map<String, Object> measures) {
+  public ProjectMeasuresDoc setMeasuresFromMap(Map<String, Double> measures) {
     setMeasures(
       measures.entrySet().stream()
-        .map(entry -> ImmutableMap.of(
+        .map(entry -> ImmutableMap.<String, Object>of(
           ProjectMeasuresIndexDefinition.FIELD_MEASURES_KEY, entry.getKey(),
           ProjectMeasuresIndexDefinition.FIELD_MEASURES_VALUE, entry.getValue()))
         .collect(Collectors.toList()));

server/sonar-server/src/main/java/org/sonar/server/component/es/ProjectMeasuresIndexer.java

@@ -26,6 +26,8 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
+import org.sonar.db.measure.ProjectMeasuresIndexerIterator;
+import org.sonar.db.measure.ProjectMeasuresIndexerIterator.ProjectMeasures;
 import org.sonar.server.es.BaseIndexer;
 import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.EsClient;
@@ -67,24 +69,22 @@ public void deleteProject(String uuid) {
   }
 
   private long doIndex(BulkIndexer bulk, long lastUpdatedAt, @Nullable String projectUuid) {
-    try (DbSession dbSession = dbClient.openSession(false)) {
-      ProjectMeasuresResultSetIterator rowIt = ProjectMeasuresResultSetIterator.create(dbClient, dbSession, lastUpdatedAt, projectUuid);
-      long maxDate = doIndex(bulk, rowIt);
-      rowIt.close();
-      return maxDate;
+    try (DbSession dbSession = dbClient.openSession(false);
+      ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, lastUpdatedAt, projectUuid)) {
+      return doIndex(bulk, rowIt);
     }
   }
 
-  private static long doIndex(BulkIndexer bulk, Iterator<ProjectMeasuresDoc> docs) {
+  private static long doIndex(BulkIndexer bulk, Iterator<ProjectMeasures> docs) {
     bulk.start();
     long maxDate = 0L;
     while (docs.hasNext()) {
-      ProjectMeasuresDoc doc = docs.next();
-      bulk.add(newIndexRequest(doc));
+      ProjectMeasures doc = docs.next();
+      bulk.add(newIndexRequest(toProjectMeasuresDoc(doc)));
 
-      Date analysisDate = doc.getAnalysedAt();
+      Long analysisDate = doc.getProject().getAnalysisDate();
       // it's more efficient to sort programmatically than in SQL on some databases (MySQL for instance)
-      maxDate = Math.max(maxDate, analysisDate == null ? 0L : analysisDate.getTime());
+      maxDate = Math.max(maxDate, analysisDate == null ? 0L : analysisDate);
     }
     bulk.stop();
     return maxDate;
@@ -97,9 +97,21 @@ private BulkIndexer createBulkIndexer(boolean large) {
   }
 
   private static IndexRequest newIndexRequest(ProjectMeasuresDoc doc) {
-    return new IndexRequest(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES, doc.getId())
-      .routing(doc.getId())
-      .parent(doc.getId())
+    String projectUuid = doc.getId();
+    return new IndexRequest(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES, projectUuid)
+      .routing(projectUuid)
+      .parent(projectUuid)
       .source(doc.getFields());
   }
+
+  private static ProjectMeasuresDoc toProjectMeasuresDoc(ProjectMeasures projectMeasures) {
+    Long analysisDate = projectMeasures.getProject().getAnalysisDate();
+    return new ProjectMeasuresDoc()
+      .setId(projectMeasures.getProject().getUuid())
+      .setKey(projectMeasures.getProject().getKey())
+      .setName(projectMeasures.getProject().getName())
+      .setQualityGate(projectMeasures.getMeasures().getQualityGateStatus())
+      .setAnalysedAt(analysisDate == null ? null : new Date(analysisDate))
+      .setMeasuresFromMap(projectMeasures.getMeasures().getNumericMeasures());
+  }
 }