From c8f5214778ac4c3e16e35d9c47a055a2667fb39a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Lesaint?= Date: Thu, 30 Jun 2016 09:57:13 +0200 Subject: [PATCH] SONAR-7705 purge: delete measures of FILE and DIR via analysis uuid --- .../src/test/java/it/dbCleaner/PurgeTest.java | 34 ++++++--- .../org/sonar/db/purge/PurgeCommands.java | 39 +++++++--- .../java/org/sonar/db/purge/PurgeDao.java | 71 +++++++++++++++---- .../java/org/sonar/db/purge/PurgeMapper.java | 3 +- .../org/sonar/db/purge/PurgeMapper.xml | 13 ++++ .../java/org/sonar/db/purge/PurgeDaoTest.java | 2 +- ...oricalDataOfDirectoriesAndFiles-result.xml | 6 +- ...eteHistoricalDataOfDirectoriesAndFiles.xml | 6 +- 8 files changed, 131 insertions(+), 43 deletions(-) diff --git a/it/it-tests/src/test/java/it/dbCleaner/PurgeTest.java b/it/it-tests/src/test/java/it/dbCleaner/PurgeTest.java index bd7683fb5a85..2ed7b2125a50 100644 --- a/it/it-tests/src/test/java/it/dbCleaner/PurgeTest.java +++ b/it/it-tests/src/test/java/it/dbCleaner/PurgeTest.java @@ -49,8 +49,12 @@ public class PurgeTest { - static final String PROJECT_KEY = "com.sonarsource.it.samples:multi-modules-sample"; - static final String PROJECT_SAMPLE_PATH = "dbCleaner/xoo-multi-modules-sample"; + private static final String COUNT_FILE_SNAPSHOTS = "snapshots where scope='FIL'"; + private static final String COUNT_FILE_MEASURES = "project_measures pm, snapshots s where pm.snapshot_id = s.id and s.scope='FIL'"; + private static final String COUNT_DIR_SNAPSHOTS = "snapshots where scope='DIR'"; + private static final String COUNT_DIR_MEASURES = "project_measures pm, snapshots s where pm.snapshot_id = s.id and s.scope='DIR'"; + private static final String PROJECT_KEY = "com.sonarsource.it.samples:multi-modules-sample"; + private static final String PROJECT_SAMPLE_PATH = "dbCleaner/xoo-multi-modules-sample"; @ClassRule public static final Orchestrator orchestrator = Category4Suite.ORCHESTRATOR; @@ -247,11 +251,18 @@ public void should_delete_removed_files() { @Test public void should_delete_historical_data_of_directories_by_default() { scan(PROJECT_SAMPLE_PATH, "2012-01-01"); - String select = "snapshots where scope='DIR'"; - int directorySnapshots = count(select); + + int fileSnapshots = count(COUNT_FILE_SNAPSHOTS); + int fileMeasures = count(COUNT_FILE_MEASURES); + int directorySnapshots = count(COUNT_DIR_SNAPSHOTS); + int dirMeasures = count(COUNT_DIR_MEASURES); scan(PROJECT_SAMPLE_PATH, "2012-02-02"); - assertThat(count(select)).isEqualTo(directorySnapshots); + + assertThat(count(COUNT_FILE_SNAPSHOTS)).isEqualTo(fileSnapshots); + assertThat(count(COUNT_FILE_MEASURES)).isEqualTo(fileMeasures); + assertThat(count(COUNT_DIR_SNAPSHOTS)).isEqualTo(directorySnapshots); + assertThat(count(COUNT_DIR_MEASURES)).isLessThan(2 * dirMeasures); // second analysis as NEW_* metrics } /** @@ -261,14 +272,19 @@ public void should_delete_historical_data_of_directories_by_default() { public void should_not_delete_historical_data_of_directories() { scan(PROJECT_SAMPLE_PATH, "2012-01-01"); - String select = "snapshots where scope='DIR'"; - int directorySnapshots = count(select); + int fileSnapshots = count(COUNT_FILE_SNAPSHOTS); + int fileMeasures = count(COUNT_FILE_MEASURES); + int directorySnapshots = count(COUNT_DIR_SNAPSHOTS); + int dirMeasures = count(COUNT_DIR_MEASURES); setServerProperty(orchestrator, "sonar.dbcleaner.cleanDirectory", "false"); scan(PROJECT_SAMPLE_PATH, "2012-02-02"); - assertThat(count(select)).isEqualTo(2 * directorySnapshots); + assertThat(count(COUNT_FILE_SNAPSHOTS)).isEqualTo(fileSnapshots); + assertThat(count(COUNT_FILE_MEASURES)).isEqualTo(fileMeasures); + assertThat(count(COUNT_DIR_SNAPSHOTS)).isEqualTo(2 * directorySnapshots); + assertThat(count(COUNT_DIR_MEASURES)).isGreaterThan(2 * dirMeasures); // second analysis as NEW_* metrics } /** @@ -331,7 +347,7 @@ private int countMeasures(String qualifier) { } private void logMeasures(String title, String qualifier) { - String sql = "SELECT m.name as metricName, pm.value as value, pm.text_value as textValue, pm.variation_value_1, pm.variation_value_2, pm.variation_value_3, pm.rule_id " + String sql = "SELECT m.name as metricName, pm.value as value, pm.text_value as textValue, pm.variation_value_1, pm.variation_value_2, pm.variation_value_3 " + "FROM project_measures pm, snapshots s, metrics m " + "WHERE pm.snapshot_id=s.id and pm.metric_id=m.id and s.qualifier='" diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java index c959a9497953..3273834f56ac 100644 --- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java +++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeCommands.java @@ -175,10 +175,28 @@ protected void deleteSnapshots(List snapshotIds) { profiler.stop(); } + public void deleteComponentMeasures(List analysisUuids, List componentUuids) { + if (analysisUuids.isEmpty() || componentUuids.isEmpty()) { + return; + } + + List> analysisUuidsPartitions = Lists.partition(analysisUuids, MAX_SNAPSHOTS_PER_QUERY); + List> componentUuidsPartitions = Lists.partition(componentUuids, MAX_RESOURCES_PER_QUERY); + + profiler.start("deleteComponentMeasures"); + for (List analysisUuidsPartition : analysisUuidsPartitions) { + for (List componentUuidsPartition : componentUuidsPartitions) { + purgeMapper.deleteComponentMeasures(analysisUuidsPartition, componentUuidsPartition); + } + } + session.commit(); + profiler.stop(); + } + void deleteAnalyses(PurgeSnapshotQuery... queries) { List snapshotIds = from(asList(queries)) - .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids) - .toList(); + .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids) + .toList(); deleteAnalyses(snapshotIds); } @@ -209,23 +227,22 @@ protected void deleteAnalyses(List analysisIdUuids) { void purgeSnapshots(PurgeSnapshotQuery... queries) { // use LinkedHashSet to keep order by remove duplicated ids LinkedHashSet snapshotIds = Sets.newLinkedHashSet(from(asList(queries)) - .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids)); + .transformAndConcat(purgeMapper::selectSnapshotIdsAndUuids)); purgeSnapshots(snapshotIds); } @VisibleForTesting protected void purgeSnapshots(Iterable snapshotIdUuidPairs) { // note that events are not deleted - List> snapshotIdsPartition = Lists.partition(IdUuidPairs.ids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY); - List> snapshotUuidsPartition = Lists.partition(IdUuidPairs.uuids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY); + List> snapshotIdsPartitions = Lists.partition(IdUuidPairs.ids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY); + List> snapshotUuidsPartitions = Lists.partition(IdUuidPairs.uuids(snapshotIdUuidPairs), MAX_SNAPSHOTS_PER_QUERY); - deleteSnapshotDuplications(snapshotUuidsPartition); + deleteSnapshotDuplications(snapshotUuidsPartitions); profiler.start("deleteSnapshotWastedMeasures (project_measures)"); List metricIdsWithoutHistoricalData = purgeMapper.selectMetricIdsWithoutHistoricalData(); - for (List partSnapshotIds : snapshotIdsPartition) { - purgeMapper.deleteSnapshotWastedMeasures(partSnapshotIds, metricIdsWithoutHistoricalData); - } + snapshotIdsPartitions.stream() + .forEach(snapshotIdsPartition -> purgeMapper.deleteSnapshotWastedMeasures(snapshotIdsPartition, metricIdsWithoutHistoricalData)); session.commit(); profiler.stop(); @@ -235,9 +252,9 @@ protected void purgeSnapshots(Iterable snapshotIdUuidPairs) { profiler.stop(); } - private void deleteSnapshotDuplications(List> snapshotUuidsPartition) { + private void deleteSnapshotDuplications(List> snapshotUuidsPartitions) { profiler.start("deleteSnapshotDuplications (duplications_index)"); - snapshotUuidsPartition.forEach(purgeMapper::deleteSnapshotDuplications); + snapshotUuidsPartitions.forEach(purgeMapper::deleteSnapshotDuplications); session.commit(); profiler.stop(); } diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java index 96df1ac2db17..79fdd11aed4c 100644 --- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -21,17 +21,21 @@ import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; -import org.apache.commons.lang.ArrayUtils; import org.apache.ibatis.session.SqlSession; import org.sonar.api.resources.Scopes; import org.sonar.api.utils.System2; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; +import org.sonar.core.util.stream.GuavaCollectors; import org.sonar.db.Dao; import org.sonar.db.DbSession; +import org.sonar.db.component.ComponentDao; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.ComponentTreeQuery; import org.sonar.db.component.ResourceDao; import org.sonar.db.component.ResourceDto; @@ -45,12 +49,15 @@ public class PurgeDao implements Dao { private static final Logger LOG = Loggers.get(PurgeDao.class); private static final String[] UNPROCESSED_STATUS = new String[] {"U"}; + private static final List UUID_FIELD_SORT = Collections.singletonList("uuid"); private final ResourceDao resourceDao; + private final ComponentDao componentDao; private final System2 system2; - public PurgeDao(ResourceDao resourceDao, System2 system2) { + public PurgeDao(ResourceDao resourceDao, ComponentDao componentDao, System2 system2) { this.resourceDao = resourceDao; + this.componentDao = componentDao; this.system2 = system2; } @@ -58,10 +65,12 @@ public void purge(DbSession session, PurgeConfiguration conf, PurgeListener list PurgeMapper mapper = session.getMapper(PurgeMapper.class); PurgeCommands commands = new PurgeCommands(session, mapper, profiler); deleteAbortedAnalyses(conf.rootProjectIdUuid().getUuid(), commands); + deleteDataOfComponentsWithoutHistoricalData(session, conf.rootProjectIdUuid().getUuid(), conf.scopesWithoutHistoricalData(), commands); + // retrieve all nodes in the tree (including root) with scope=PROJECT List projects = getProjects(conf.rootProjectIdUuid().getId(), session); for (ResourceDto project : projects) { LOG.debug("-> Clean " + project.getLongName() + " [id=" + project.getId() + "]"); - purge(project, conf.scopesWithoutHistoricalData(), commands); + purge(project.getUuid(), commands); } for (ResourceDto project : projects) { disableOrphanResources(project, session, mapper, listener); @@ -92,21 +101,53 @@ private static void deleteAbortedAnalyses(String rootUuid, PurgeCommands command commands.deleteAnalyses(query); } - private static void purge(ResourceDto project, String[] scopesWithoutHistoricalData, PurgeCommands purgeCommands) { + private void deleteDataOfComponentsWithoutHistoricalData(DbSession dbSession, String rootUuid, String[] scopesWithoutHistoricalData, PurgeCommands purgeCommands) { + if (scopesWithoutHistoricalData.length == 0) { + return; + } + + List analysisUuids = purgeCommands.selectSnapshotUuids( + PurgeSnapshotQuery.create() + .setComponentUuid(rootUuid) + .setIslast(false) + .setNotPurged(true)); + List componentWithoutHistoricalDataUuids = componentDao + .selectDescendants( + dbSession, + newComponentTreeQuery() + .setBaseUuid(rootUuid) + .setQualifiers(Arrays.asList(scopesWithoutHistoricalData)) + .build()) + .stream().map(ComponentDto::uuid) + .collect(GuavaCollectors.toList()); + + purgeCommands.deleteComponentMeasures(analysisUuids, componentWithoutHistoricalDataUuids); + // FIXME remove this when cardinality of snapshots has been changed + for (String componentUuid : componentWithoutHistoricalDataUuids) { + purgeCommands.deleteSnapshots(PurgeSnapshotQuery.create() + .setIslast(false) + .setComponentUuid(componentUuid)); + } + } + + /** + * Creates a new ComponentTreeQuery.Builder with properties that don't matter here but are mandatory populated. + */ + private static ComponentTreeQuery.Builder newComponentTreeQuery() { + return ComponentTreeQuery.builder() + .setPage(1) + .setPageSize(Integer.MAX_VALUE) + .setSortFields(UUID_FIELD_SORT); + } + + private static void purge(String componentUuid, PurgeCommands purgeCommands) { List projectSnapshotUuids = purgeCommands.selectSnapshotUuids( - PurgeSnapshotQuery.create() - .setComponentUuid(project.getUuid()) - .setIslast(false) - .setNotPurged(true)); + PurgeSnapshotQuery.create() + .setComponentUuid(componentUuid) + .setIslast(false) + .setNotPurged(true)); for (String snapshotUuid : projectSnapshotUuids) { LOG.debug("<- Clean analysis " + snapshotUuid); - if (!ArrayUtils.isEmpty(scopesWithoutHistoricalData)) { - PurgeSnapshotQuery query = PurgeSnapshotQuery.create() - .setIslast(false) - .setScopes(scopesWithoutHistoricalData) - .setAnalysisUuid(snapshotUuid); - purgeCommands.deleteSnapshots(query); - } // must be executed at the end for reentrance purgeCommands.purgeSnapshots( diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java index fc2ec93eff3b..5ebe7ded3016 100644 --- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java +++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeMapper.java @@ -46,6 +46,8 @@ public interface PurgeMapper { void deleteSnapshotMeasures(@Param("snapshotIds") List snapshotIds); + void deleteComponentMeasures(@Param("analysisUuids") List analysisUuids, @Param("componentUuids") List componentUuids); + List selectMetricIdsWithoutHistoricalData(); void deleteSnapshotWastedMeasures(@Param("snapshotIds") List snapshotIds, @Param("mids") List metricIds); @@ -95,5 +97,4 @@ public interface PurgeMapper { void deleteFileSourcesByUuid(String fileUuid); void deleteCeActivityByProjectUuid(String projectUuid); - } diff --git a/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml index a1bcc6315075..b087ab53da1d 100644 --- a/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml +++ b/sonar-db/src/main/resources/org/sonar/db/purge/PurgeMapper.xml @@ -109,6 +109,19 @@ + + delete from project_measures + where + analysis_uuid in + + #{analysisUuid} + + and component_uuid in + + #{componentUuid} + + + delete from duplications_index where analysis_uuid in diff --git a/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java b/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java index 154dda8bc0a3..05aa18219a07 100644 --- a/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java +++ b/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java @@ -84,7 +84,7 @@ public void delete_file_sources_of_disabled_resources() { @Test public void shouldDeleteHistoricalDataOfDirectoriesAndFiles() { dbTester.prepareDbUnit(getClass(), "shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml"); - underTest.purge(dbSession, new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "1"), new String[] {Scopes.DIRECTORY, Scopes.FILE}, 30), PurgeListener.EMPTY, + underTest.purge(dbSession, new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "ABCD"), new String[] {Scopes.DIRECTORY, Scopes.FILE}, 30), PurgeListener.EMPTY, new PurgeProfiler()); dbSession.commit(); dbTester.assertDbUnit(getClass(), "shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml", "projects", "snapshots"); diff --git a/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml b/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml index 2b56c7bb8836..4c119f1b43d7 100644 --- a/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml +++ b/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml @@ -8,7 +8,7 @@ What has been changed : purge_status=1 on snapshot 4 (PRJ) and snapshots 5 and 6