SONAR-6777 Project cache sync

SonarSource · Sep 30, 2015 · 0847774 · 0847774
1 parent 1dae583
commit 0847774
Show file tree

Hide file tree

Showing 42 changed files with 1,108 additions and 607 deletions.
diff --git a/it/it-tests/src/test/java/batch/IssuesModeTest.java b/it/it-tests/src/test/java/batch/IssuesModeTest.java
@@ -71,7 +71,7 @@ public void issues_analysis_on_new_project() throws IOException {
     restoreProfile("one-issue-per-line.xml");
     orchestrator.getServer().provisionProject("sample", "xoo-sample");
     orchestrator.getServer().associateProjectToQualityProfile("sample", "xoo", "one-issue-per-line");
-    SonarRunner runner = configureRunnerIssues("shared/xoo-sample");
+    SonarRunner runner = configureRunnerIssues("shared/xoo-sample", "sonar.verbose", "true");
     BuildResult result = orchestrator.executeBuild(runner);
     assertThat(ItUtils.countIssuesInJsonReport(result, true)).isEqualTo(17);
   }

diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java
@@ -28,7 +28,7 @@
 import org.sonar.api.utils.UriReader;
 import org.sonar.batch.analysis.AnalysisProperties;
 import org.sonar.batch.analysis.DefaultAnalysisMode;
-import org.sonar.batch.cache.PersistentCacheProvider;
+import org.sonar.batch.cache.GlobalPersistentCacheProvider;
 import org.sonar.batch.cache.ProjectSyncContainer;
 import org.sonar.batch.cache.StrategyWSLoaderProvider;
 import org.sonar.batch.cache.WSLoader.LoadStrategy;
@@ -85,7 +85,7 @@ private void addBootstrapComponents() {
       BatchPluginPredicate.class,
       ExtensionInstaller.class,
 
-    CachesManager.class,
+      CachesManager.class,
       GlobalMode.class,
       GlobalSettings.class,
       new RulesProvider(),
@@ -95,7 +95,7 @@ private void addBootstrapComponents() {
       DefaultHttpDownloader.class,
       UriReader.class,
       new FileCacheProvider(),
-      new PersistentCacheProvider(),
+      new GlobalPersistentCacheProvider(),
       System2.INSTANCE,
       new GlobalRepositoriesProvider(),
       UuidFactoryImpl.INSTANCE);

diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ServerClient.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ServerClient.java
@@ -19,9 +19,6 @@
  */
 package org.sonar.batch.bootstrap;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
@@ -30,13 +27,10 @@
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.net.URI;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
@@ -62,7 +56,6 @@
 @BatchSide
 public class ServerClient {
   private static final String GET = "GET";
-  private static final Logger LOG = LoggerFactory.getLogger(ServerClient.class);
   private GlobalProperties props;
   private DefaultHttpDownloader.BaseHttpDownloader downloader;
 
@@ -74,20 +67,6 @@ public ServerClient(GlobalProperties settings, EnvironmentInformation env) {
   public String getURL() {
     return StringUtils.removeEnd(StringUtils.defaultIfBlank(props.property("sonar.host.url"), "http://localhost:9000"), "/");
   }
-
-  public String getServerVersion() {
-    InputStream is = this.getClass().getClassLoader().getResourceAsStream("sq-version.txt");
-    if (is == null) {
-      LOG.warn("Failed to get SQ version");
-      return null;
-    }
-    try (BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
-      return br.readLine();
-    } catch (IOException e) {
-      LOG.warn("Failed to get SQ version", e);
-      return null;
-    }
-  }
 
   public URI getURI(String pathStartingWithSlash) {
     Preconditions.checkArgument(pathStartingWithSlash.startsWith("/"), "Path must start with slash /: " + pathStartingWithSlash);

diff --git a/sonar-batch/src/main/java/org/sonar/batch/cache/DefaultProjectCacheStatus.java b/sonar-batch/src/main/java/org/sonar/batch/cache/DefaultProjectCacheStatus.java
@@ -19,73 +19,67 @@
  */
 package org.sonar.batch.cache;
 
-import javax.annotation.Nullable;
+import org.apache.commons.io.FileUtils;
 
-import org.sonar.batch.bootstrap.ServerClient;
 import org.sonar.home.cache.PersistentCache;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Date;
 
 public class DefaultProjectCacheStatus implements ProjectCacheStatus {
-  private static final String STATUS_PREFIX = "cache-sync-status-";
+  private static final String STATUS_FILENAME = "cache-sync-status";
   private PersistentCache cache;
-  private ServerClient client;
 
-  public DefaultProjectCacheStatus(PersistentCache cache, ServerClient client) {
+  public DefaultProjectCacheStatus(PersistentCache cache) {
     this.cache = cache;
-    this.client = client;
   }
 
   @Override
-  public void save(@Nullable String projectKey) {
+  public void save() {
     Date now = new Date();
 
     try {
-      ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
-      try (ObjectOutputStream objOutput = new ObjectOutputStream(byteOutput)) {
+      FileOutputStream fos = new FileOutputStream(getStatusFilePath().toFile());
+      try (ObjectOutputStream objOutput = new ObjectOutputStream(fos)) {
         objOutput.writeObject(now);
       }
-      cache.put(getKey(projectKey), byteOutput.toByteArray());
+
     } catch (IOException e) {
       throw new IllegalStateException("Failed to write cache sync status", e);
     }
   }
 
   @Override
-  public void delete(@Nullable String projectKey) {
-    try {
-      cache.put(getKey(projectKey), new byte[0]);
-    } catch (IOException e) {
-      throw new IllegalStateException("Failed to delete cache sync status", e);
-    }
+  public void delete() {
+    cache.clear();
+    FileUtils.deleteQuietly(getStatusFilePath().toFile());
   }
 
   @Override
-  public Date getSyncStatus(@Nullable String projectKey) {
+  public Date getSyncStatus() {
+    Path p = getStatusFilePath();
     try {
-      byte[] status = cache.get(getKey(projectKey), null);
-      if (status == null || status.length == 0) {
+      if (!Files.isRegularFile(p)) {
         return null;
       }
-      ByteArrayInputStream byteInput = new ByteArrayInputStream(status);
-      try (ObjectInputStream objInput = new ObjectInputStream(byteInput)) {
+      InputStream is = new FileInputStream(p.toFile());
+      try (ObjectInputStream objInput = new ObjectInputStream(is)) {
         return (Date) objInput.readObject();
       }
     } catch (IOException | ClassNotFoundException e) {
+      FileUtils.deleteQuietly(p.toFile());
       throw new IllegalStateException("Failed to read cache sync status", e);
     }
   }
 
-  private String getKey(@Nullable String projectKey) {
-    if (projectKey != null) {
-      return STATUS_PREFIX + client.getURL() + "-" + client.getServerVersion() + "-" + projectKey;
-    } else {
-      return STATUS_PREFIX + client.getURL() + "-" + client.getServerVersion();
-    }
+  private Path getStatusFilePath() {
+    return cache.getDirectory().resolve(STATUS_FILENAME);
   }
 }
diff --git a/.../batch/cache/PersistentCacheProvider.java → .../cache/GlobalPersistentCacheProvider.java b/.../batch/cache/PersistentCacheProvider.java → .../cache/GlobalPersistentCacheProvider.java
@@ -19,54 +19,38 @@
  */
 package org.sonar.batch.cache;
 
+import org.apache.commons.lang.StringUtils;
 import org.sonar.batch.bootstrap.Slf4jLogger;
-import org.sonar.batch.bootstrap.UserProperties;
-
-import org.sonar.api.utils.log.Logger;
-import org.sonar.api.utils.log.Loggers;
+import org.sonar.batch.util.BatchUtils;
+import org.sonar.home.cache.PersistentCacheBuilder;
 
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Paths;
 
-import org.picocontainer.injectors.ProviderAdapter;
+import org.sonar.batch.bootstrap.GlobalProperties;
 import org.sonar.home.cache.PersistentCache;
-import org.sonar.home.cache.PersistentCacheBuilder;
+import org.picocontainer.injectors.ProviderAdapter;
 
-public class PersistentCacheProvider extends ProviderAdapter {
-  private static final Logger LOG = Loggers.get(PersistentCacheProvider.class);
+public class GlobalPersistentCacheProvider extends ProviderAdapter {
   private PersistentCache cache;
 
-  public PersistentCache provide(UserProperties props) {
+  public PersistentCache provide(GlobalProperties props) {
     if (cache == null) {
       PersistentCacheBuilder builder = new PersistentCacheBuilder(new Slf4jLogger());
-
       String home = props.property("sonar.userHome");
+      String serverUrl = getServerUrl(props);
+
       if (home != null) {
         builder.setSonarHome(Paths.get(home));
       }
-
-      builder.setVersion(getServerVersion());
+      
+      builder.setAreaForGlobal(serverUrl, BatchUtils.getServerVersion());
       cache = builder.build();
     }
 
     return cache;
   }
 
-  private String getServerVersion() {
-    InputStream is = this.getClass().getClassLoader().getResourceAsStream("sq-version.txt");
-    if (is == null) {
-      LOG.warn("Failed to get SQ version");
-      return null;
-    }
-    try (BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
-      return br.readLine();
-    } catch (IOException e) {
-      LOG.warn("Failed to get SQ version", e);
-      return null;
-    }
+  private String getServerUrl(GlobalProperties props) {
+    return StringUtils.removeEnd(StringUtils.defaultIfBlank(props.property("sonar.host.url"), "http://localhost:9000"), "/");
   }
 }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/cache/NonAssociatedCacheSynchronizer.java b/sonar-batch/src/main/java/org/sonar/batch/cache/NonAssociatedCacheSynchronizer.java
@@ -46,7 +46,7 @@ public NonAssociatedCacheSynchronizer(QualityProfileLoader qualityProfileLoader,
   }
 
   public void execute(boolean force) {
-    Date lastSync = cacheStatus.getSyncStatus(null);
+    Date lastSync = cacheStatus.getSyncStatus();
 
     if (lastSync != null) {
       if (!force) {
@@ -55,13 +55,15 @@ public void execute(boolean force) {
       } else {
         LOG.info("-- Found cache [{}], synchronizing data..", lastSync);
       }
-      cacheStatus.delete(null);
+      cacheStatus.delete();
     } else {
       LOG.info("-- Cache not found, synchronizing data..");
     }
 
     loadData();
-    saveStatus();
+
+    cacheStatus.save();
+    LOG.info("-- Succesfully synchronized cache");
   }
 
   private static Collection<String> getKeys(Collection<QProfile> qProfiles) {
@@ -73,11 +75,6 @@ private static Collection<String> getKeys(Collection<QProfile> qProfiles) {
     return list;
   }
 
-  private void saveStatus() {
-    cacheStatus.save(null);
-    LOG.info("-- Succesfully synchronized cache");
-  }
-
   private void loadData() {
     Profiler profiler = Profiler.create(Loggers.get(ProjectCacheSynchronizer.class));
 

diff --git a/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectCacheStatus.java b/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectCacheStatus.java
@@ -19,14 +19,12 @@
  */
 package org.sonar.batch.cache;
 
-import javax.annotation.Nullable;
-
 import java.util.Date;
 
 public interface ProjectCacheStatus {
-  void save(@Nullable String projectKey);
+  void save();
 
-  void delete(@Nullable String projectKey);
+  void delete();
 
-  Date getSyncStatus(@Nullable String projectKey);
+  Date getSyncStatus();
 }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectCacheSynchronizer.java b/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectCacheSynchronizer.java
@@ -63,7 +63,7 @@ public ProjectCacheSynchronizer(QualityProfileLoader qualityProfileLoader, Proje
   }
 
   public void load(String projectKey, boolean force) {
-    Date lastSync = cacheStatus.getSyncStatus(projectKey);
+    Date lastSync = cacheStatus.getSyncStatus();
 
     if (lastSync != null) {
       if (!force) {
@@ -72,17 +72,17 @@ public void load(String projectKey, boolean force) {
       } else {
         LOG.info("-- Found project [{}] cache [{}], synchronizing data..", projectKey, lastSync);
       }
-      cacheStatus.delete(projectKey);
+      cacheStatus.delete();
     } else {
       LOG.info("-- Cache for project [{}] not found, synchronizing data..", projectKey);
     }
 
     loadData(projectKey);
-    saveStatus(projectKey);
+    saveStatus();
   }
 
-  private void saveStatus(String projectKey) {
-    cacheStatus.save(projectKey);
+  private void saveStatus() {
+    cacheStatus.save();
     LOG.info("-- Succesfully synchronized project cache");
   }
 
@@ -103,7 +103,7 @@ private void loadData(String projectKey) {
     profiler.stopInfo();
 
     Collection<String> profileKeys = getKeys(qProfiles);
-    
+
     profiler.startInfo("Load project active rules");
     activeRulesLoader.load(profileKeys, projectKey);
     profiler.stopInfo();

diff --git a/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectKeySupplier.java b/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectKeySupplier.java
@@ -0,0 +1,36 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.cache;
+
+import org.sonar.api.batch.bootstrap.ProjectKey;
+
+public class ProjectKeySupplier implements ProjectKey {
+  private final String key;
+
+  ProjectKeySupplier(String key) {
+    this.key = key;
+  }
+
+  @Override
+  public String get() {
+    return key;
+  }
+
+}