Skip to content

Commit

Permalink
Exclude plugins without a valid wiki page from the Update Centre JSON…
Browse files Browse the repository at this point in the history 
… files.

The criteria for inclusion are:
- Plugins must specify a Jenkins wiki URL in their pom.xml, via the <url> tag
- The wiki page must exist on the Jenkins wiki
- The wiki page must be a child of the "Plugins" page

Also accepted are:
- Wiki short URLs (e.g. /x/GYCGAQ)
- Wiki URLs with a numeric ID (e.g. /pages/viewpage.action?pageId=60915753)
- URLs which have legacy URL prefixes (e.g. non-HTTPS, wiki.hudson-ci.org)
- URLs ending with a trailing slash

Plugins not meeting these criteria will not be included in the Update Centre
JSON file, nor in the release history JSON file.

As before, plugins may also be excluded via the "artifact-ignores.properties"
file, or if a plugin does specify a valid wiki page, but that wiki page has the
"plugin-deprecated" label.

While building the plugin list, more informational output is now printed to the
console regarding why plugins are being excluded, or whether their wiki URL was
rewritten to the canonical wiki URL format.

A summary of the total number of included, excluded and deprecated plugins is
also shown at the end of building the Update Centre plugin list.

A number of test cases were added to test the basic URL resolution process, as
well as some of the rarer types of URL found in currently-released plugins.
  • Loading branch information
@orrc
orrc committed May 18, 2015
1 parent b507798 commit 6c44a90
Show file tree
Hide file tree
Showing 8 changed files with 336 additions and 115 deletions.
6 changes: 6 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,12 @@
<artifactId>jetty-util</artifactId>
<version>6.0.0rc1</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>1.10.19</version>
<scope>test</scope>
</dependency>
</dependencies>

<repositories>
Expand Down
216 changes: 143 additions & 73 deletions src/main/java/org/jvnet/hudson/update_center/ConfluencePluginList.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@
*/
package org.jvnet.hudson.update_center;

import com.sun.xml.bind.v2.util.EditDistance;
import hudson.plugins.jira.soap.ConfluenceSoapService;
import hudson.plugins.jira.soap.RemoteLabel;
import hudson.plugins.jira.soap.RemotePage;
import hudson.plugins.jira.soap.RemotePageSummary;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.jvnet.hudson.confluence.Confluence;

import javax.xml.rpc.ServiceException;
Expand All @@ -40,10 +40,11 @@
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
Expand All @@ -59,101 +60,155 @@
* @author Kohsuke Kawaguchi
*/
public class ConfluencePluginList {
private final ConfluenceSoapService service;
private final Map<String,RemotePageSummary> children = new HashMap<String, RemotePageSummary>();
private final String[] normalizedTitles;

private String wikiSessionId;
private final String WIKI_URL = "https://wiki.jenkins-ci.org/";
private static final Pattern TINYLINK_PATTERN = Pattern.compile(".*/x/(\\w+)");

/** Base URL of the wiki. */
private static final String WIKI_URL = "https://wiki.jenkins-ci.org/";

/** List of old wiki base URLs, which plugins may still have in their POM. */
private static final String[] OLD_URL_PREFIXES = {
"http://wiki.jenkins-ci.org/",
"http://wiki.hudson-ci.org/",
"http://hudson.gotdns.com/wiki/",
};

private final File cacheDir = new File(System.getProperty("user.home"),".wiki.jenkins-ci.org-cache");
private final ConfluenceSoapService service;
private final Map<String, String> pluginPages = new HashMap<String, String>();

private String wikiSessionId;

public ConfluencePluginList() throws IOException, ServiceException {
this(Confluence.connect(new URL(WIKI_URL)));
}

ConfluencePluginList(ConfluenceSoapService service) throws IOException, ServiceException {
this.service = service;

cacheDir.mkdirs();

service = Confluence.connect(new URL(WIKI_URL));
System.out.println("Fetching the 'Plugins' page and child info from the wiki...");
RemotePage page = service.getPage("", "JENKINS", "Plugins");

for (RemotePageSummary child : service.getChildren("", page.getId()))
children.put(normalize(child.getTitle()),child);
normalizedTitles = children.keySet().toArray(new String[children.size()]);
// Note the URL of each child page of the "Plugins" page on the wiki
for (RemotePageSummary child : service.getChildren("", page.getId())) {
// Normalise URLs coming from the Confluence API, so that when we later check whether a certain URL is in
// this list, we don't get a false negative due to differences in how the URL was encoded
pluginPages.put(getKeyForUrl(child.getUrl()), child.getUrl());
}
}

/**
* Make the page title as close to artifactId as possible.
*/
private String normalize(String title) {
title = title.toLowerCase().trim();
if(title.endsWith("plugin")) title=title.substring(0,title.length()-6).trim();
return title.replace(" ","-");
/** @return A wiki URL if the given URL is a child page of the "Plugins" wiki page, otherwise {@code null}. */
private String getCanonicalUrl(String url) {
return pluginPages.get(getKeyForUrl(url));
}

private String getKeyForUrl(String url) {
// We call `getPath()` to ensure that the path is URL-encoded in a consistent way.
// Confluence is case-insensitive when it comes to the URL path, hence `toLowerCase()`
URI uri = URI.create(url.replace(' ', '+'));
return String.format("%s?%s", uri.getPath().toLowerCase(Locale.ROOT), uri.getQuery());
}

/**
* Finds the closest match, if any. Otherwise null.
* Attempts to determine the canonical wiki URL for a given URL.
*
* @param url Any URL.
* @return A canonical URL to a wiki page, or {@code null} if the URL is not a child of the "Plugins" wiki page.
* @throws IOException If resolving a short URL fails.
*/
public WikiPage findNearest(String pluginArtifactId) throws IOException {
// comparison is case insensitive
pluginArtifactId = pluginArtifactId.toLowerCase();

String nearest = EditDistance.findNearest(pluginArtifactId, normalizedTitles);
if (EditDistance.editDistance(nearest,pluginArtifactId) <= 1) {
System.out.println("** No wiki page specified.. picking one with similar name."
+ "\nUsing '"+nearest+"' for "+pluginArtifactId);
return loadPage(children.get(nearest).getTitle());
} else
return null; // too far
}
public String resolveWikiUrl(String url) throws IOException {
// Empty or null values can't be good
if (url == null || url.isEmpty()) {
System.out.println("** Wiki URL is missing");
return null;
}

public WikiPage getPage(String url) throws IOException {
// If the URL is a short URL (e.g. "/x/tgeIAg"), then resolve the target URL
Matcher tinylink = TINYLINK_PATTERN.matcher(url);
if (tinylink.matches()) {
String id = tinylink.group(1);
url = resolveLink(tinylink.group(1));
}

File cache = new File(cacheDir,id+".link");
if (cache.exists()) {
url = FileUtils.readFileToString(cache);
} else {
try {
// Avoid creating lots of sessions on wiki server.. get a session and reuse it.
if (wikiSessionId == null)
wikiSessionId = initSession(WIKI_URL);
url = checkRedirect(
WIKI_URL + "pages/tinyurl.action?urlIdentifier=" + id,
wikiSessionId);
FileUtils.writeStringToFile(cache,url);
} catch (IOException e) {
throw new RemoteException("Failed to lookup tinylink redirect", e);
}
// Fix up URLs with old hostnames or paths
for (String p : OLD_URL_PREFIXES) {
if (url.startsWith(p)) {
url = url.replace(p, WIKI_URL).replaceAll("(?i)/HUDSON/", "/JENKINS/");
}
}

for( String p : WIKI_PREFIXES ) {
if (!url.startsWith(p))
continue;
// Reject the URL if it's not on the wiki at all
if (!url.startsWith(WIKI_URL)) {
System.out.println("** Wiki URLs should start with "+ WIKI_URL);
return null;
}

// Strip trailing slashes (e.g.
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}

String pageName = url.substring(p.length()).replace('+',' '); // poor hack for URL escape
// If the page exists in the child list we fetched, get the canonical URL
String canonicalUrl = getCanonicalUrl(url);
if (canonicalUrl == null) {
System.out.println("** Wiki page does not exist, or is not a child of the Plugins wiki page: "+ url);
}
return canonicalUrl;
}

// trim off the trailing '/'
if (pageName.endsWith("/"))
pageName = pageName.substring(0,pageName.length()-1);
/**
* Determines the full wiki URL for a given Confluence short URL.
*
* @param id Short URL ID.
* @return The full wiki URL.
* @throws IOException If accessing the wiki fails.
*/
private String resolveLink(String id) throws IOException {
File cache = new File(cacheDir,id+".link");
if (cache.exists()) {
return FileUtils.readFileToString(cache);
}

return loadPage(pageName);
String url;
try {
// Avoid creating lots of sessions on wiki server.. get a session and reuse it.
if (wikiSessionId == null)
wikiSessionId = initSession(WIKI_URL);
url = checkRedirect(WIKI_URL + "pages/tinyurl.action?urlIdentifier=" + id, wikiSessionId);
FileUtils.writeStringToFile(cache, url);
} catch (IOException e) {
throw new RemoteException("Failed to lookup tinylink redirect", e);
}
throw new IllegalArgumentException("** Failed to resolve "+url);
return url;
}

/**
* Loads the page from Wiki after consulting with the cache.
* Attempts to fetch a page from the wiki, possibly returning from local disk cache.
*
* @param pomUrl URL from the POM.
* @return Wiki page object if the page exists, otherwise {@code null}.
* @throws IOException If accessing the wiki fails.
*/
private WikiPage loadPage(String title) throws IOException {
File cache = new File(cacheDir,title+".page");
if (cache.exists() && cache.lastModified() >= System.currentTimeMillis()- TimeUnit.DAYS.toMillis(1)) {
// load from cache
public WikiPage getPage(String pomUrl) throws IOException {
String url = resolveWikiUrl(pomUrl);
if (url == null) {
return null;
}

// Determine the page identifier for the given wiki URL
String cacheKey = getIdentifierForUrl(url);

// Load the serialised page from the cache, if we retrieved it within the last day
File cache = new File(cacheDir, cacheKey + ".page");
if (cache.exists() && cache.lastModified() >= System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1)) {
try {
FileInputStream f = new FileInputStream(cache);
try {
Object o = new ObjectInputStream(f).readObject();
if (o==null) return null;
if (o == null) {
return null;
}
if (o instanceof WikiPage) {
return (WikiPage) o;
}
Expand All @@ -162,17 +217,24 @@ private WikiPage loadPage(String title) throws IOException {
f.close();
}
} catch (ClassNotFoundException e) {
throw (IOException)new IOException("Failed to retrieve from cache: "+cache).initCause(e);
throw (IOException) new IOException("Failed to retrieve from cache: " + cache).initCause(e);
}
}

// Otherwise fetch it from the wiki and cache the page
try {
RemotePage page = service.getPage("", "JENKINS", title);
RemotePage page;
if (NumberUtils.isDigits(cacheKey)) {
page = service.getPage("", Long.parseLong(cacheKey));
} else {
page = service.getPage("", "JENKINS", cacheKey);
}
RemoteLabel[] labels = service.getLabelsById("", page.getId());
WikiPage p = new WikiPage(page, labels);
writeToCache(cache, p);
return p;
} catch (RemoteException e) {
// Something went wrong; invalidate the cache for this page
writeToCache(cache, null);
throw e;
}
Expand All @@ -196,6 +258,22 @@ private void writeToCache(File cache, Object o) throws IOException {
tmp.delete();
}

/**
* @param url Wiki URL, in the canonical URL format.
* @return The identifier we need to fetch the given URL via the Confluence API.
*/
private static String getIdentifierForUrl(String url) {
URI pageUri = URI.create(url);
String path = pageUri.getPath();
if (path.equals("/pages/viewpage.action")) {
// This is the canonical URL format for titles with odd characters, e.g. "Anything Goes" Formatter Plugin
return pageUri.getQuery().replace("pageId=", "");
}

// In all other cases, we can just take the title straight from the URL
return path.replaceAll("(?i)/display/JENKINS/", "").replace("+", " ");
}

private static String checkRedirect(String url, String sessionId) throws IOException {
return connect(url, sessionId).getHeaderField("Location");
}
Expand All @@ -215,12 +293,4 @@ private static HttpURLConnection connect(String url, String sessionId) throws IO
return huc;
}

private static final String[] WIKI_PREFIXES = {
"https://wiki.jenkins-ci.org/display/JENKINS/",
"http://wiki.jenkins-ci.org/display/JENKINS/",
"http://wiki.hudson-ci.org/display/HUDSON/",
"http://hudson.gotdns.com/wiki/display/HUDSON/",
};

private static final Pattern TINYLINK_PATTERN = Pattern.compile(".*/x/(\\w+)");
}
2 changes: 2 additions & 0 deletions src/main/java/org/jvnet/hudson/update_center/LatestLinkBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ public class LatestLinkBuilder implements Closeable {
private final PrintWriter htaccess;

public LatestLinkBuilder(File dir) throws IOException {
System.out.println(String.format("Writing plugin symlinks and redirects to dir: %s", dir));

index = new IndexHtmlBuilder(dir,"Permalinks to latest files");
htaccess = new PrintWriter(new FileWriter(new File(dir,".htaccess")),true);

Expand Down

0 comments on commit 6c44a90

Please sign in to comment.