From c4ba2c0d17cd5514c48280c96f18509561e13730 Mon Sep 17 00:00:00 2001 From: Julien HENRY Date: Wed, 28 Jan 2015 16:50:15 +0100 Subject: [PATCH] SONAR-5831 Provide the ability to explicitly ask the SCM sensor to reload all blame information --- .../bootstrap/BatchPluginRepository.java | 4 +- .../bootstrap/DefaultPluginsReferential.java | 4 +- .../batch/bootstrap/GlobalContainer.java | 2 +- ...eferential.java => PluginsRepository.java} | 2 +- .../batch/mediumtest/BatchMediumTester.java | 42 +++++++----- .../org/sonar/batch/scm/ScmConfiguration.java | 22 ++++++ .../java/org/sonar/batch/scm/ScmSensor.java | 20 ++++-- .../batch/mediumtest/scm/ScmMediumTest.java | 67 +++++++++++++++++-- 8 files changed, 132 insertions(+), 31 deletions(-) rename sonar-batch/src/main/java/org/sonar/batch/bootstrap/{PluginsReferential.java => PluginsRepository.java} (97%) diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java index c62ec99bd1c5..42f21fe91044 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java @@ -51,7 +51,7 @@ public class BatchPluginRepository implements PluginRepository { private static final Logger LOG = LoggerFactory.getLogger(BatchPluginRepository.class); private static final String CORE_PLUGIN = "core"; - private PluginsReferential pluginsReferential; + private PluginsRepository pluginsReferential; private Map pluginsByKey; private Map metadataByKey; private Settings settings; @@ -59,7 +59,7 @@ public class BatchPluginRepository implements PluginRepository { private final DefaultAnalysisMode analysisMode; private final BatchPluginJarInstaller pluginInstaller; - public BatchPluginRepository(PluginsReferential pluginsReferential, Settings settings, DefaultAnalysisMode analysisMode, + public BatchPluginRepository(PluginsRepository pluginsReferential, Settings settings, DefaultAnalysisMode analysisMode, BatchPluginJarInstaller pluginInstaller) { this.pluginsReferential = pluginsReferential; this.settings = settings; diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/DefaultPluginsReferential.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/DefaultPluginsReferential.java index 7d611676d2cb..d023b5e75e0d 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/DefaultPluginsReferential.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/DefaultPluginsReferential.java @@ -37,9 +37,9 @@ import java.util.Map; /** - * A {@link PluginsReferential} implementation that put downloaded plugins in a FS cache. + * A {@link PluginsRepository} implementation that put downloaded plugins in a FS cache. */ -public class DefaultPluginsReferential implements PluginsReferential { +public class DefaultPluginsReferential implements PluginsRepository { private static final Logger LOG = LoggerFactory.getLogger(DefaultPluginsReferential.class); diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java index f30a891be30c..f55308c348ce 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java @@ -107,7 +107,7 @@ private void addBootstrapComponents() { DefaultI18n.class, new GlobalRepositoriesProvider(), UserRepository.class); - if (getComponentByType(PluginsReferential.class) == null) { + if (getComponentByType(PluginsRepository.class) == null) { add(DefaultPluginsReferential.class); } if (getComponentByType(GlobalRepositoriesLoader.class) == null) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginsReferential.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginsRepository.java similarity index 97% rename from sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginsReferential.java rename to sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginsRepository.java index 330c8bfb3377..58473fd6af10 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginsReferential.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginsRepository.java @@ -31,7 +31,7 @@ * Plugin referential. * @since 4.4 */ -public interface PluginsReferential { +public interface PluginsRepository { /** * Return list of remote plugins to be installed diff --git a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java index 8930a759b91d..68a58b5df6e7 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java +++ b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java @@ -28,11 +28,12 @@ import org.sonar.api.measures.CoreMetrics; import org.sonar.api.measures.Metric; import org.sonar.api.platform.PluginMetadata; -import org.sonar.batch.bootstrap.PluginsReferential; +import org.sonar.batch.bootstrap.PluginsRepository; import org.sonar.batch.bootstrap.TaskProperties; import org.sonar.batch.bootstrapper.Batch; import org.sonar.batch.bootstrapper.EnvironmentInformation; import org.sonar.batch.protocol.input.ActiveRule; +import org.sonar.batch.protocol.input.FileData; import org.sonar.batch.protocol.input.GlobalRepositories; import org.sonar.batch.protocol.input.ProjectRepositories; import org.sonar.batch.protocol.input.issues.PreviousIssue; @@ -68,9 +69,9 @@ public static BatchMediumTesterBuilder builder() { } public static class BatchMediumTesterBuilder { - private final FakeGlobalReferentialsLoader globalRefProvider = new FakeGlobalReferentialsLoader(); - private final FakeProjectReferentialsLoader projectRefProvider = new FakeProjectReferentialsLoader(); - private final FakePluginsReferential pluginsReferential = new FakePluginsReferential(); + private final FakeGlobalRepositoriesLoader globalRefProvider = new FakeGlobalRepositoriesLoader(); + private final FakeProjectRepositoriesLoader projectRefProvider = new FakeProjectRepositoriesLoader(); + private final FakePluginsRepository pluginsReferential = new FakePluginsRepository(); private final FakePreviousIssuesLoader previousIssues = new FakePreviousIssuesLoader(); private final Map bootstrapProperties = new HashMap(); @@ -121,6 +122,11 @@ public BatchMediumTesterBuilder activateRule(ActiveRule activeRule) { return this; } + public BatchMediumTesterBuilder addFileData(String moduleKey, String path, FileData fileData) { + projectRefProvider.addFileData(moduleKey, path, fileData); + return this; + } + public BatchMediumTesterBuilder addPreviousIssue(PreviousIssue issue) { previousIssues.getPreviousIssues().add(issue); return this; @@ -180,9 +186,9 @@ public TaskBuilder(BatchMediumTester tester) { public TaskResult start() { TaskResult result = new TaskResult(); - tester.batch.executeTask(taskProperties, - result - ); + Map props = new HashMap<>(); + props.putAll(taskProperties); + tester.batch.executeTask(props, result); return result; } @@ -197,7 +203,7 @@ public TaskBuilder property(String key, String value) { } } - private static class FakeGlobalReferentialsLoader implements GlobalRepositoriesLoader { + private static class FakeGlobalRepositoriesLoader implements GlobalRepositoriesLoader { private int metricId = 1; @@ -212,7 +218,7 @@ public Map globalSettings() { return ref.globalSettings(); } - public FakeGlobalReferentialsLoader add(Metric metric) { + public FakeGlobalRepositoriesLoader add(Metric metric) { Boolean optimizedBestValue = metric.isOptimizedBestValue(); ref.metrics().add(new org.sonar.batch.protocol.input.Metric(metricId, metric.key(), @@ -230,7 +236,7 @@ public FakeGlobalReferentialsLoader add(Metric metric) { } } - private static class FakeProjectReferentialsLoader implements ProjectRepositoriesLoader { + private static class FakeProjectRepositoriesLoader implements ProjectRepositoriesLoader { private ProjectRepositories ref = new ProjectRepositories(); @@ -239,18 +245,24 @@ public ProjectRepositories load(ProjectReactor reactor, TaskProperties taskPrope return ref; } - public FakeProjectReferentialsLoader addQProfile(String language, String name) { + public FakeProjectRepositoriesLoader addQProfile(String language, String name) { ref.addQProfile(new org.sonar.batch.protocol.input.QProfile(name, name, language, new Date())); return this; } - public FakeProjectReferentialsLoader addActiveRule(ActiveRule activeRule) { + public FakeProjectRepositoriesLoader addActiveRule(ActiveRule activeRule) { ref.addActiveRule(activeRule); return this; } + + public FakeProjectRepositoriesLoader addFileData(String moduleKey, String path, FileData fileData) { + ref.addFileData(moduleKey, path, fileData); + return this; + } + } - private static class FakePluginsReferential implements PluginsReferential { + private static class FakePluginsRepository implements PluginsRepository { private List pluginList = new ArrayList(); private Map pluginFiles = new HashMap(); @@ -266,14 +278,14 @@ public File pluginFile(RemotePlugin remote) { return pluginFiles.get(remote); } - public FakePluginsReferential addPlugin(String pluginKey, File location) { + public FakePluginsRepository addPlugin(String pluginKey, File location) { RemotePlugin plugin = new RemotePlugin(pluginKey, false); pluginList.add(plugin); pluginFiles.put(plugin, location); return this; } - public FakePluginsReferential addPlugin(String pluginKey, SonarPlugin pluginInstance) { + public FakePluginsRepository addPlugin(String pluginKey, SonarPlugin pluginInstance) { localPlugins.put(DefaultPluginMetadata.create(pluginKey), pluginInstance); return this; } diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java index d95f870b5ac2..e81e4d5f6cc9 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java @@ -25,6 +25,9 @@ import org.slf4j.LoggerFactory; import org.sonar.api.BatchComponent; import org.sonar.api.CoreProperties; +import org.sonar.api.Properties; +import org.sonar.api.Property; +import org.sonar.api.PropertyType; import org.sonar.api.batch.AnalysisMode; import org.sonar.api.batch.InstantiationStrategy; import org.sonar.api.batch.bootstrap.ProjectReactor; @@ -37,10 +40,25 @@ import java.util.LinkedHashMap; import java.util.Map; +@Properties({ + @Property( + key = ScmConfiguration.FORCE_RELOAD_KEY, + defaultValue = "false", + type = PropertyType.BOOLEAN, + name = "Force reloading of SCM information for all files", + description = "By default only files modified since previous analysis are inspected. Set this parameter to true to force the reloading.", + module = false, + project = false, + global = false, + category = CoreProperties.CATEGORY_SCM + ) +}) @InstantiationStrategy(InstantiationStrategy.PER_BATCH) public final class ScmConfiguration implements BatchComponent, Startable { private static final Logger LOG = LoggerFactory.getLogger(ScmConfiguration.class); + public static final String FORCE_RELOAD_KEY = "sonar.scm.forceReloadAll"; + private final ProjectReactor projectReactor; private final Settings settings; private final Map providerPerKey = new LinkedHashMap(); @@ -140,6 +158,10 @@ public boolean isDisabled() { return settings.getBoolean(CoreProperties.SCM_DISABLED_KEY); } + public boolean forceReloadAll() { + return settings.getBoolean(FORCE_RELOAD_KEY); + } + @Override public void stop() { // Nothing to do diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java index 6e4207d410b7..2a33ed3b3fb9 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java @@ -73,10 +73,7 @@ public void execute(final SensorContext context) { return; } - List filesToBlame = new LinkedList(); - for (InputFile f : fs.inputFiles(fs.predicates().all())) { - copyPreviousMeasuresForUnmodifiedFiles(context, filesToBlame, f); - } + List filesToBlame = collectFilesToBlame(context); if (!filesToBlame.isEmpty()) { LOG.info("SCM provider for this project is: " + configuration.provider().key()); TimeProfiler profiler = new TimeProfiler().start("Retrieve SCM blame information"); @@ -87,6 +84,21 @@ public void execute(final SensorContext context) { } } + private List collectFilesToBlame(final SensorContext context) { + if (configuration.forceReloadAll()) { + LOG.warn("Forced reloading of SCM data for all files."); + } + List filesToBlame = new LinkedList(); + for (InputFile f : fs.inputFiles(fs.predicates().all())) { + if (!configuration.forceReloadAll()) { + copyPreviousMeasuresForUnmodifiedFiles(context, filesToBlame, f); + } else { + filesToBlame.add(f); + } + } + return filesToBlame; + } + private void copyPreviousMeasuresForUnmodifiedFiles(final SensorContext context, List filesToBlame, InputFile f) { FileData fileData = projectReferentials.fileData(projectDefinition.getKeyWithBranch(), f.relativePath()); diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java index 13994e0081c9..fb9601f182d8 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java @@ -20,6 +20,7 @@ package org.sonar.batch.mediumtest.scm; import com.google.common.collect.ImmutableMap; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Before; @@ -31,7 +32,9 @@ import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure; import org.sonar.api.measures.CoreMetrics; import org.sonar.batch.mediumtest.BatchMediumTester; +import org.sonar.batch.mediumtest.BatchMediumTester.TaskBuilder; import org.sonar.batch.mediumtest.TaskResult; +import org.sonar.batch.protocol.input.FileData; import org.sonar.xoo.XooPlugin; import java.io.File; @@ -41,6 +44,8 @@ public class ScmMediumTest { + private static final String SAMPLE_XOO_CONTENT = "Sample xoo\ncontent"; + @org.junit.Rule public TemporaryFolder temp = new TemporaryFolder(); @@ -50,6 +55,7 @@ public class ScmMediumTest { public BatchMediumTester tester = BatchMediumTester.builder() .registerPlugin("xoo", new XooPlugin()) .addDefaultQProfile("xoo", "Sonar Way") + .addFileData("com.foo.project", "src/sample2.xoo", new FileData(DigestUtils.md5Hex(SAMPLE_XOO_CONTENT), false, "1=;2=", "1=;2=", "1=;2=")) .build(); @Before @@ -145,7 +151,57 @@ public void failIfMissingFile() throws IOException { .put("sonar.scm.provider", "xoo") .build()) .start(); + } + + @Test + public void copyPreviousMeasuresOrForceReload() throws IOException { + + File baseDir = prepareProject(); + File xooFileNoScm = new File(baseDir, "src/sample2.xoo"); + FileUtils.write(xooFileNoScm, SAMPLE_XOO_CONTENT); + + TaskBuilder taskBuilder = tester.newTask() + .properties(ImmutableMap.builder() + .put("sonar.task", "scan") + .put("sonar.projectBaseDir", baseDir.getAbsolutePath()) + .put("sonar.projectKey", "com.foo.project") + .put("sonar.projectName", "Foo Project") + .put("sonar.projectVersion", "1.0-SNAPSHOT") + .put("sonar.projectDescription", "Description of Foo Project") + .put("sonar.sources", "src") + .put("sonar.scm.provider", "xoo") + .build()); + + TaskResult result = taskBuilder.start(); + + assertThat(result.measures()).hasSize(1 + 2 * 4); + assertThat(result.measures()).contains(new DefaultMeasure() + .forMetric(CoreMetrics.LINES) + .onFile(new DefaultInputFile("com.foo.project", "src/sample2.xoo")) + .withValue(2)); + + assertThat(result.measures()).contains(new DefaultMeasure() + .forMetric(CoreMetrics.SCM_AUTHORS_BY_LINE) + .onFile(new DefaultInputFile("com.foo.project", "src/sample2.xoo")) + .withValue("1=;2=")); + + // Force reload + File xooScmFile = new File(baseDir, "src/sample2.xoo.scm"); + FileUtils.write(xooScmFile, + // revision,author,dateTime + "1,foo,2013-01-04\n" + + "1,bar,2013-01-04\n" + ); + + result = taskBuilder + .property("sonar.scm.forceReloadAll", "true") + .start(); + + assertThat(result.measures()).contains(new DefaultMeasure() + .forMetric(CoreMetrics.SCM_AUTHORS_BY_LINE) + .onFile(new DefaultInputFile("com.foo.project", "src/sample2.xoo")) + .withValue("1=foo;2=bar")); } @Test @@ -215,12 +271,10 @@ private File prepareProject() throws IOException { File srcDir = new File(baseDir, "src"); srcDir.mkdir(); - File xooFile = new File(srcDir, "sample.xoo"); - File xooMeasureFile = new File(srcDir, "sample.xoo.measures"); - File xooScmFile = new File(srcDir, "sample.xoo.scm"); - FileUtils.write(xooFile, "Sample xoo\ncontent\n3\n4\n5"); - FileUtils.write(xooMeasureFile, "lines:5"); - FileUtils.write(xooScmFile, + File xooFile1 = new File(srcDir, "sample.xoo"); + FileUtils.write(xooFile1, "Sample xoo\ncontent\n3\n4\n5"); + File xooScmFile1 = new File(srcDir, "sample.xoo.scm"); + FileUtils.write(xooScmFile1, // revision,author,dateTime "1,,2013-01-04\n" + "1,julien,2013-01-04\n" + @@ -228,6 +282,7 @@ private File prepareProject() throws IOException { "2,julien,2013-02-03\n" + "3,simon,2013-03-04\n" ); + return baseDir; }