SONAR-7705 purge: delete measures of FILE and DIR via analysis uuid

it/it-tests/src/test/java/it/dbCleaner/PurgeTest.java

@@ -49,8 +49,12 @@
 
 public class PurgeTest {
 
-  static final String PROJECT_KEY = "com.sonarsource.it.samples:multi-modules-sample";
-  static final String PROJECT_SAMPLE_PATH = "dbCleaner/xoo-multi-modules-sample";
+  private static final String COUNT_FILE_SNAPSHOTS = "snapshots where scope='FIL'";
+  private static final String COUNT_FILE_MEASURES = "project_measures pm, snapshots s where pm.snapshot_id = s.id and s.scope='FIL'";
+  private static final String COUNT_DIR_SNAPSHOTS = "snapshots where scope='DIR'";
+  private static final String COUNT_DIR_MEASURES = "project_measures pm, snapshots s where pm.snapshot_id = s.id and s.scope='DIR'";
+  private static final String PROJECT_KEY = "com.sonarsource.it.samples:multi-modules-sample";
+  private static final String PROJECT_SAMPLE_PATH = "dbCleaner/xoo-multi-modules-sample";
 
   @ClassRule
   public static final Orchestrator orchestrator = Category4Suite.ORCHESTRATOR;
@@ -247,11 +251,18 @@ public void should_delete_removed_files() {
   @Test
   public void should_delete_historical_data_of_directories_by_default() {
     scan(PROJECT_SAMPLE_PATH, "2012-01-01");
-    String select = "snapshots where scope='DIR'";
-    int directorySnapshots = count(select);
+
+    int fileSnapshots = count(COUNT_FILE_SNAPSHOTS);
+    int fileMeasures = count(COUNT_FILE_MEASURES);
+    int directorySnapshots = count(COUNT_DIR_SNAPSHOTS);
+    int dirMeasures = count(COUNT_DIR_MEASURES);
 
     scan(PROJECT_SAMPLE_PATH, "2012-02-02");
-    assertThat(count(select)).isEqualTo(directorySnapshots);
+
+    assertThat(count(COUNT_FILE_SNAPSHOTS)).isEqualTo(fileSnapshots);
+    assertThat(count(COUNT_FILE_MEASURES)).isEqualTo(fileMeasures);
+    assertThat(count(COUNT_DIR_SNAPSHOTS)).isEqualTo(directorySnapshots);
+    assertThat(count(COUNT_DIR_MEASURES)).isLessThan(2 * dirMeasures); // second analysis as NEW_* metrics
   }
 
   /**
@@ -261,14 +272,19 @@ public void should_delete_historical_data_of_directories_by_default() {
   public void should_not_delete_historical_data_of_directories() {
     scan(PROJECT_SAMPLE_PATH, "2012-01-01");
 
-    String select = "snapshots where scope='DIR'";
-    int directorySnapshots = count(select);
+    int fileSnapshots = count(COUNT_FILE_SNAPSHOTS);
+    int fileMeasures = count(COUNT_FILE_MEASURES);
+    int directorySnapshots = count(COUNT_DIR_SNAPSHOTS);
+    int dirMeasures = count(COUNT_DIR_MEASURES);
 
     setServerProperty(orchestrator, "sonar.dbcleaner.cleanDirectory", "false");
 
     scan(PROJECT_SAMPLE_PATH, "2012-02-02");
 
-    assertThat(count(select)).isEqualTo(2 * directorySnapshots);
+    assertThat(count(COUNT_FILE_SNAPSHOTS)).isEqualTo(fileSnapshots);
+    assertThat(count(COUNT_FILE_MEASURES)).isEqualTo(fileMeasures);
+    assertThat(count(COUNT_DIR_SNAPSHOTS)).isEqualTo(2 * directorySnapshots);
+    assertThat(count(COUNT_DIR_MEASURES)).isGreaterThan(2 * dirMeasures); // second analysis as NEW_* metrics
   }
 
   /**
@@ -331,7 +347,7 @@ private int countMeasures(String qualifier) {
   }
 
   private void logMeasures(String title, String qualifier) {
-    String sql = "SELECT m.name as metricName, pm.value as value, pm.text_value as textValue, pm.variation_value_1, pm.variation_value_2, pm.variation_value_3, pm.rule_id "
+    String sql = "SELECT m.name as metricName, pm.value as value, pm.text_value as textValue, pm.variation_value_1, pm.variation_value_2, pm.variation_value_3 "
       +
       "FROM project_measures pm, snapshots s, metrics m " +
       "WHERE pm.snapshot_id=s.id and pm.metric_id=m.id and s.qualifier='"

sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java

@@ -175,10 +175,28 @@ protected void deleteSnapshots(List<IdUuidPair> snapshotIds) {
     profiler.stop();
   }
 
+  public void deleteComponentMeasures(List<String> analysisUuids, List<String> componentUuids) {
+    if (analysisUuids.isEmpty() || componentUuids.isEmpty()) {
+      return;
+    }
+
+    List<List<String>> analysisUuidsPartitions = Lists.partition(analysisUuids, MAX_SNAPSHOTS_PER_QUERY);
+    List<List<String>> componentUuidsPartitions = Lists.partition(componentUuids, MAX_RESOURCES_PER_QUERY);
+
+    profiler.start("deleteComponentMeasures");
+    for (List<String> analysisUuidsPartition : analysisUuidsPartitions) {
+      for (List<String> componentUuidsPartition : componentUuidsPartitions) {
+        purgeMapper.deleteComponentMeasures(analysisUuidsPartition, componentUuidsPartition);
+      }
+    }
+    session.commit();
+    profiler.stop();
+  }
+
   void deleteAnalyses(PurgeSnapshotQuery... queries) {
     List<IdUuidPair> snapshotIds = from(asList(queries))
-        .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids)
-        .toList();
+      .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids)
+      .toList();
     deleteAnalyses(snapshotIds);
   }
 
@@ -209,23 +227,22 @@ protected void deleteAnalyses(List<IdUuidPair> analysisIdUuids) {
   void purgeSnapshots(PurgeSnapshotQuery... queries) {
     // use LinkedHashSet to keep order by remove duplicated ids
     LinkedHashSet<IdUuidPair> snapshotIds = Sets.newLinkedHashSet(from(asList(queries))
-        .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids));
+      .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids));
     purgeSnapshots(snapshotIds);
   }
 
   @VisibleForTesting
   protected void purgeSnapshots(Iterable<IdUuidPair> snapshotIdUuidPairs) {
     // note that events are not deleted
-    List<List<Long>> snapshotIdsPartition = Lists.partition(IdUuidPairs.ids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY);
-    List<List<String>> snapshotUuidsPartition = Lists.partition(IdUuidPairs.uuids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY);
+    List<List<Long>> snapshotIdsPartitions = Lists.partition(IdUuidPairs.ids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY);
+    List<List<String>> snapshotUuidsPartitions = Lists.partition(IdUuidPairs.uuids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY);
 
-    deleteSnapshotDuplications(snapshotUuidsPartition);
+    deleteSnapshotDuplications(snapshotUuidsPartitions);
 
     profiler.start("deleteSnapshotWastedMeasures (project_measures)");
     List<Long> metricIdsWithoutHistoricalData = purgeMapper.selectMetricIdsWithoutHistoricalData();
-    for (List<Long> partSnapshotIds : snapshotIdsPartition) {
-      purgeMapper.deleteSnapshotWastedMeasures(partSnapshotIds, metricIdsWithoutHistoricalData);
-    }
+    snapshotIdsPartitions.stream()
+        .forEach(snapshotIdsPartition -> purgeMapper.deleteSnapshotWastedMeasures(snapshotIdsPartition, metricIdsWithoutHistoricalData));
     session.commit();
     profiler.stop();
 
@@ -235,9 +252,9 @@ protected void purgeSnapshots(Iterable<IdUuidPair> snapshotIdUuidPairs) {
     profiler.stop();
   }
 
-  private void deleteSnapshotDuplications(List<List<String>> snapshotUuidsPartition) {
+  private void deleteSnapshotDuplications(List<List<String>> snapshotUuidsPartitions) {
     profiler.start("deleteSnapshotDuplications (duplications_index)");
-    snapshotUuidsPartition.forEach(purgeMapper::deleteSnapshotDuplications);
+    snapshotUuidsPartitions.forEach(purgeMapper::deleteSnapshotDuplications);
     session.commit();
     profiler.stop();
   }

sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java

@@ -21,17 +21,21 @@
 
 import com.google.common.collect.Lists;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
-import org.apache.commons.lang.ArrayUtils;
 import org.apache.ibatis.session.SqlSession;
 import org.sonar.api.resources.Scopes;
 import org.sonar.api.utils.System2;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
+import org.sonar.core.util.stream.GuavaCollectors;
 import org.sonar.db.Dao;
 import org.sonar.db.DbSession;
+import org.sonar.db.component.ComponentDao;
+import org.sonar.db.component.ComponentDto;
+import org.sonar.db.component.ComponentTreeQuery;
 import org.sonar.db.component.ResourceDao;
 import org.sonar.db.component.ResourceDto;
 
@@ -45,23 +49,28 @@
 public class PurgeDao implements Dao {
   private static final Logger LOG = Loggers.get(PurgeDao.class);
   private static final String[] UNPROCESSED_STATUS = new String[] {"U"};
+  private static final List<String> UUID_FIELD_SORT = Collections.singletonList("uuid");
 
   private final ResourceDao resourceDao;
+  private final ComponentDao componentDao;
   private final System2 system2;
 
-  public PurgeDao(ResourceDao resourceDao, System2 system2) {
+  public PurgeDao(ResourceDao resourceDao, ComponentDao componentDao, System2 system2) {
     this.resourceDao = resourceDao;
+    this.componentDao = componentDao;
     this.system2 = system2;
   }
 
   public void purge(DbSession session, PurgeConfiguration conf, PurgeListener listener, PurgeProfiler profiler) {
     PurgeMapper mapper = session.getMapper(PurgeMapper.class);
     PurgeCommands commands = new PurgeCommands(session, mapper, profiler);
     deleteAbortedAnalyses(conf.rootProjectIdUuid().getUuid(), commands);
+    deleteDataOfComponentsWithoutHistoricalData(session, conf.rootProjectIdUuid().getUuid(), conf.scopesWithoutHistoricalData(), commands);
+    // retrieve all nodes in the tree (including root) with scope=PROJECT
     List<ResourceDto> projects = getProjects(conf.rootProjectIdUuid().getId(), session);
     for (ResourceDto project : projects) {
       LOG.debug("-> Clean " + project.getLongName() + " [id=" + project.getId() + "]");
-      purge(project, conf.scopesWithoutHistoricalData(), commands);
+      purge(project.getUuid(), commands);
     }
     for (ResourceDto project : projects) {
       disableOrphanResources(project, session, mapper, listener);
@@ -92,21 +101,53 @@ private static void deleteAbortedAnalyses(String rootUuid, PurgeCommands command
     commands.deleteAnalyses(query);
   }
 
-  private static void purge(ResourceDto project, String[] scopesWithoutHistoricalData, PurgeCommands purgeCommands) {
+  private void deleteDataOfComponentsWithoutHistoricalData(DbSession dbSession, String rootUuid, String[] scopesWithoutHistoricalData, PurgeCommands purgeCommands) {
+    if (scopesWithoutHistoricalData.length == 0) {
+      return;
+    }
+
+    List<String> analysisUuids = purgeCommands.selectSnapshotUuids(
+      PurgeSnapshotQuery.create()
+        .setComponentUuid(rootUuid)
+        .setIslast(false)
+        .setNotPurged(true));
+    List<String> componentWithoutHistoricalDataUuids = componentDao
+      .selectDescendants(
+        dbSession,
+        newComponentTreeQuery()
+          .setBaseUuid(rootUuid)
+          .setQualifiers(Arrays.asList(scopesWithoutHistoricalData))
+          .build())
+      .stream().map(ComponentDto::uuid)
+      .collect(GuavaCollectors.toList());
+
+    purgeCommands.deleteComponentMeasures(analysisUuids, componentWithoutHistoricalDataUuids);
+    // FIXME remove this when cardinality of snapshots has been changed
+    for (String componentUuid : componentWithoutHistoricalDataUuids) {
+      purgeCommands.deleteSnapshots(PurgeSnapshotQuery.create()
+        .setIslast(false)
+        .setComponentUuid(componentUuid));
+    }
+  }
+
+  /**
+   * Creates a new ComponentTreeQuery.Builder with properties that don't matter here but are mandatory populated.
+   */
+  private static ComponentTreeQuery.Builder newComponentTreeQuery() {
+    return ComponentTreeQuery.builder()
+      .setPage(1)
+      .setPageSize(Integer.MAX_VALUE)
+      .setSortFields(UUID_FIELD_SORT);
+  }
+
+  private static void purge(String componentUuid, PurgeCommands purgeCommands) {
     List<String> projectSnapshotUuids = purgeCommands.selectSnapshotUuids(
-        PurgeSnapshotQuery.create()
-            .setComponentUuid(project.getUuid())
-            .setIslast(false)
-            .setNotPurged(true));
+      PurgeSnapshotQuery.create()
+        .setComponentUuid(componentUuid)
+        .setIslast(false)
+        .setNotPurged(true));
     for (String snapshotUuid : projectSnapshotUuids) {
       LOG.debug("<- Clean analysis " + snapshotUuid);
-      if (!ArrayUtils.isEmpty(scopesWithoutHistoricalData)) {
-        PurgeSnapshotQuery query = PurgeSnapshotQuery.create()
-          .setIslast(false)
-          .setScopes(scopesWithoutHistoricalData)
-          .setAnalysisUuid(snapshotUuid);
-        purgeCommands.deleteSnapshots(query);
-      }
 
       // must be executed at the end for reentrance
       purgeCommands.purgeSnapshots(

sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java

@@ -46,6 +46,8 @@ public interface PurgeMapper {
 
   void deleteSnapshotMeasures(@Param("snapshotIds") List<Long> snapshotIds);
 
+  void deleteComponentMeasures(@Param("analysisUuids") List<String> analysisUuids, @Param("componentUuids") List<String> componentUuids);
+
   List<Long> selectMetricIdsWithoutHistoricalData();
 
   void deleteSnapshotWastedMeasures(@Param("snapshotIds") List<Long> snapshotIds, @Param("mids") List<Long> metricIds);
@@ -95,5 +97,4 @@ public interface PurgeMapper {
   void deleteFileSourcesByUuid(String fileUuid);
 
   void deleteCeActivityByProjectUuid(String projectUuid);
-
 }

sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml

@@ -109,6 +109,19 @@
       </foreach>
   </delete>
 
+  <delete id="deleteComponentMeasures" parameterType="map">
+    delete from project_measures
+    where
+      analysis_uuid in
+      <foreach collection="analysisUuids" open="(" close=")" item="analysisUuid" separator=",">
+        #{analysisUuid}
+      </foreach>
+      and component_uuid in
+      <foreach collection="componentUuids" open="(" close=")" item="componentUuid" separator=",">
+        #{componentUuid}
+      </foreach>
+  </delete>
+
   <delete id="deleteSnapshotDuplications" parameterType="map">
     delete from duplications_index where analysis_uuid in
     <foreach collection="analysisUuids" open="(" close=")" item="analysisUuid" separator=",">

sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java

@@ -84,7 +84,7 @@ public void delete_file_sources_of_disabled_resources() {
   @Test
   public void shouldDeleteHistoricalDataOfDirectoriesAndFiles() {
     dbTester.prepareDbUnit(getClass(), "shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml");
-    underTest.purge(dbSession, new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "1"), new String[] {Scopes.DIRECTORY, Scopes.FILE}, 30), PurgeListener.EMPTY,
+    underTest.purge(dbSession, new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "ABCD"), new String[] {Scopes.DIRECTORY, Scopes.FILE}, 30), PurgeListener.EMPTY,
       new PurgeProfiler());
     dbSession.commit();
     dbTester.assertDbUnit(getClass(), "shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml", "projects", "snapshots");

sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml

@@ -8,7 +8,7 @@ What has been changed : purge_status=1 on snapshot 4 (PRJ) and snapshots 5 and 6
 
   <!-- the project -->
   <projects uuid="ABCD"
-            uuid_path="NOT_USED"
+            uuid_path="."
             project_uuid="ABCD"
             module_uuid="[null]"
             module_uuid_path="."
@@ -31,7 +31,7 @@ What has been changed : purge_status=1 on snapshot 4 (PRJ) and snapshots 5 and 6
 
   <!-- the directory -->
   <projects uuid="EFGH"
-            uuid_path="NOT_USED"
+            uuid_path=".ABCD."
             project_uuid="ABCD"
             module_uuid="ABCD"
             module_uuid_path="."
@@ -54,7 +54,7 @@ What has been changed : purge_status=1 on snapshot 4 (PRJ) and snapshots 5 and 6
 
   <!-- the file -->
   <projects uuid="GHIJ"
-            uuid_path="NOT_USED"
+            uuid_path=".ABCD.EFGH."
             project_uuid="ABCD"
             module_uuid="ABCD"
             module_uuid_path=".ABCD."

sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml

@@ -2,7 +2,7 @@
 
   <!-- the project -->
   <projects uuid="ABCD"
-            uuid_path="NOT_USED"
+            uuid_path="."
             project_uuid="ABCD"
             module_uuid="[null]"
             module_uuid_path="."
@@ -25,7 +25,7 @@
 
   <!-- the directory -->
   <projects uuid="EFGH"
-            uuid_path="NOT_USED"
+            uuid_path=".ABCD."
             project_uuid="ABCD"
             module_uuid="ABCD"
             module_uuid_path="."
@@ -48,7 +48,7 @@
 
   <!-- the file -->
   <projects uuid="GHIJ"
-            uuid_path="NOT_USED"
+            uuid_path=".ABCD.EFGH."
             project_uuid="ABCD"
             module_uuid="ABCD"
             module_uuid_path=".ABCD."