Merge pull request #61 from jenkins-infra/develop

[WEBSITE-639, WEBSITE-643] - Promote the #50, #60, #62 to production
jenkins-infra · Sep 3, 2019 · 9194b92 · 9194b92
2 parents b93d692 + a8c84e7
commit 9194b92
Show file tree

Hide file tree

Showing 14 changed files with 357 additions and 26 deletions.
diff --git a/src/main/java/io/jenkins/plugins/endpoints/HealthEndpoint.java b/src/main/java/io/jenkins/plugins/endpoints/HealthEndpoint.java
@@ -2,22 +2,18 @@
 
 import io.jenkins.plugins.services.PrepareDatastoreService;
 import io.jenkins.plugins.utils.VersionUtils;
-import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.inject.Inject;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Properties;
 
 /**
  * <p>Endpoint for retrieving health about the application</p>

diff --git a/src/main/java/io/jenkins/plugins/endpoints/PluginEndpoint.java b/src/main/java/io/jenkins/plugins/endpoints/PluginEndpoint.java
@@ -4,7 +4,6 @@
 import io.jenkins.plugins.services.DatastoreService;
 import io.jenkins.plugins.services.ServiceException;
 import io.jenkins.plugins.services.WikiService;
-import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

diff --git a/src/main/java/io/jenkins/plugins/endpoints/PluginsEndpoint.java b/src/main/java/io/jenkins/plugins/endpoints/PluginsEndpoint.java
@@ -12,7 +12,6 @@
 import javax.ws.rs.*;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
-import java.util.Collections;
 import java.util.Set;
 
 /**

diff --git a/src/main/java/io/jenkins/plugins/generate/parsers/ImpliedDependenciesCoreResourceParser.java b/src/main/java/io/jenkins/plugins/generate/parsers/ImpliedDependenciesCoreResourceParser.java
@@ -5,7 +5,6 @@
 import io.jenkins.plugins.models.Dependency;
 import io.jenkins.plugins.models.Plugin;
 import org.apache.commons.io.IOUtils;
-import org.apache.http.client.HttpClient;
 import org.json.JSONObject;
 
 import java.io.IOException;
@@ -15,7 +14,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
 
 /**
  * Adds implied dependencies to detached plugins if a plugin's core dependency is old.

diff --git a/src/main/java/io/jenkins/plugins/generate/parsers/MaintainersPluginDataParser.java b/src/main/java/io/jenkins/plugins/generate/parsers/MaintainersPluginDataParser.java
@@ -6,7 +6,6 @@
 import org.apache.commons.lang3.StringUtils;
 import org.json.JSONObject;
 
-import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 

diff --git a/src/main/java/io/jenkins/plugins/services/SearchOptions.java b/src/main/java/io/jenkins/plugins/services/SearchOptions.java
@@ -1,6 +1,5 @@
 package io.jenkins.plugins.services;
 
-import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 

diff --git a/src/main/java/io/jenkins/plugins/services/impl/DefaultConfigurationService.java b/src/main/java/io/jenkins/plugins/services/impl/DefaultConfigurationService.java
@@ -17,7 +17,6 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
-import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;

diff --git a/src/main/java/io/jenkins/plugins/services/impl/ElasticsearchPrepareDatastoreService.java b/src/main/java/io/jenkins/plugins/services/impl/ElasticsearchPrepareDatastoreService.java
@@ -59,8 +59,8 @@ public void populateDataStore() {
 
   @Override
   public void schedulePopulateDataStore() {
-    final int interval = Integer.valueOf(System.getenv().getOrDefault("POPULATE_DATASTORE_INTERVAL", "1"));
-    scheduledExecutorService.scheduleWithFixedDelay(this::populateDataStore, interval, interval, TimeUnit.HOURS);
+    final int interval = Integer.valueOf(System.getenv().getOrDefault("POPULATE_DATASTORE_INTERVAL", "30"));
+    scheduledExecutorService.scheduleWithFixedDelay(this::populateDataStore, interval, interval, TimeUnit.MINUTES);
   }
 
   private boolean shouldIndex(GeneratedPluginData data) {

diff --git a/src/main/java/io/jenkins/plugins/services/impl/GithubExtractor.java b/src/main/java/io/jenkins/plugins/services/impl/GithubExtractor.java
@@ -60,13 +60,13 @@ public String extractHtml(String apiContent, String url, HttpClientWikiService s
 
   private void convertLinksToAbsolute(HttpClientWikiService service, Element wikiContent, String orgName, String repoName, String branch) {
     String documentationHost = String.format("https://github.com/%s/%s/blob/%s", orgName, repoName, branch);
-    String imageHost = String.format("https://raw.githubusercontent.com/%s/%s/%s", orgName, repoName, branch);
+    String imageHost = String.format("https://cdn.jsdelivr.net/gh/%s/%s@%s", orgName, repoName, branch);
 
     // Relative hyperlinks, we resolve "/docs/rest-api.adoc" as https://github.com/jenkinsci/folder-auth-plugin/blob/master/docs/rest-api.adoc
-    wikiContent.getElementsByAttribute("href").forEach(element -> service.replaceAttribute(element, "href", documentationHost, ""));
-    //TODO: Should we host images from our infrastructure? What are the GitHub terms here?
-    // Relative image inclusions, we resolve /docs/images/screenshot.png as https://raw.githubusercontent.com/jenkinsci/folder-auth-plugin/master/docs/images/screenshot.png
-    wikiContent.getElementsByAttribute("src").forEach(element -> service.replaceAttribute(element, "src", imageHost, ""));
+    wikiContent.getElementsByAttribute("href").forEach(element -> service.replaceAttribute(element, "href", documentationHost, "/"));
+
+    // Relative image inclusions, we resolve /docs/images/screenshot.png as https://cdn.jsdelivr.net/gh/jenkinsci/folder-auth-plugin@master/docs/images/screenshot.png
+    wikiContent.getElementsByAttribute("src").forEach(element -> service.replaceAttribute(element, "src", imageHost, "/"));
   }
 
   @Override

diff --git a/src/test/java/io/jenkins/plugins/RestAppIntegrationTest.java b/src/test/java/io/jenkins/plugins/RestAppIntegrationTest.java
@@ -1,8 +1,6 @@
 package io.jenkins.plugins;
 
 import io.jenkins.plugins.models.*;
-import io.jenkins.plugins.services.SearchOptions;
-import io.jenkins.plugins.services.SortBy;
 import org.apache.commons.lang3.StringUtils;
 import org.glassfish.jersey.test.JerseyTest;
 import org.glassfish.jersey.test.jetty.JettyTestContainerFactory;

diff --git a/src/test/java/io/jenkins/plugins/services/DatastoreServiceIntegrationTest.java b/src/test/java/io/jenkins/plugins/services/DatastoreServiceIntegrationTest.java
@@ -12,8 +12,6 @@
 import org.mockito.Mockito;
 
 import javax.inject.Singleton;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.concurrent.ScheduledExecutorService;
 
 public class DatastoreServiceIntegrationTest {

diff --git a/src/test/java/io/jenkins/plugins/services/WikiServiceTest.java b/src/test/java/io/jenkins/plugins/services/WikiServiceTest.java
@@ -69,19 +69,46 @@ public void testGetWikiContentNoUrl() {
   }
 
   @Test
-  public void testCleanWikiContent() throws IOException {
+  public void testCleanWikiContentConfluence() throws IOException {
     final File file = new File("src/test/resources/wiki_content.html");
     final String content = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
     final String cleanContent = ConfluenceDirectExtractor.cleanWikiContent(content, wikiService);
     Assert.assertNotNull("Wiki content is null", cleanContent);
-    final Document html = Jsoup.parseBodyFragment(cleanContent);
+    assertAllLinksMatch(cleanContent, "https?://.*", "https://.*");
+  }
+
+  @Test
+  public void testCleanWikiContentGithub() throws IOException {
+    final File file = new File("src/test/resources/github_content.html");
+    final String content = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
+    final String cleanContent = new GithubExtractor().extractHtml(content, 
+        "https://github.com/jenkinsci/configuration-as-code-plugin", wikiService);
+    Assert.assertNotNull("Wiki content is null", cleanContent);
+    assertAllLinksMatch(cleanContent, "(#|https?://).*", "https?://.*");
+  }
+
+  @Test
+  public void testCleanWikiExcerptGithub() throws IOException {
+    final File file = new File("src/test/resources/github_excerpt.html");
+    final String content = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
+    final String cleanContent = new GithubExtractor().extractHtml(content, 
+        "https://github.com/jenkinsci/configuration-as-code-plugin", wikiService);
+    Assert.assertNotNull("Wiki content is null", cleanContent);
+    String hrefRegexp = "#getting-started|https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/.*";
+    String srcRegexp = "https://cdn.jsdelivr.net/gh/jenkinsci/configuration-as-code-plugin@master/.*"
+         + "|https://camo.githubusercontent.com/[a-z0-9]*/[a-z0-9]*";
+    assertAllLinksMatch(cleanContent, hrefRegexp, srcRegexp);
+  }
+
+  private void assertAllLinksMatch(String content, String hrefRegexp, String srcRegexp) {
+    final Document html = Jsoup.parseBodyFragment(content);
     html.getElementsByAttribute("href").forEach(element -> {
       final String value = element.attr("href");
-      Assert.assertFalse("Wiki content not clean - href references to root : " + value, value.startsWith("/"));
+      Assert.assertTrue("Wiki content not clean - href references to root : " + value, value.matches(hrefRegexp));
     });
     html.getElementsByAttribute("src").forEach(element -> {
       final String value = element.attr("src");
-      Assert.assertFalse("Wiki content not clean - src references to root : " + value, value.startsWith("/"));
+      Assert.assertTrue("Wiki content not clean - src references to root : " + value, value.matches(srcRegexp));
     });
   }