[INFRA-1809] Private update center improvement

README.md

@@ -110,3 +110,35 @@ you can try to use the appassembler plugin as described below. The exec:java plu
     # warning this may take quite a bit of time, so you might want to add the -maxPlugins 1 option
     mvn package appassembler:assemble
     sh target/appassembler/bin/app -id com.example.jenkins -www www
+
+Private Update Center
+---------------------
+
+Jenkins offical update center has lots of plugins. You could follow below steps to create your own
+private update center:
+
+1. Create your own certificate. For example:
+
+```shell script
+openssl genrsa -out rootCA/demo.key 1024
+openssl req -new -x509 -days 1095 -key rootCA/demo.key \
+    -out rootCA/demo.crt \
+    -subj "/C=CN/ST=GD/L=SZ/O=vihoo/OU=dev/CN=demo.com/emailAddress=demo@demo.com"
+```
+2. Fetch plugins information then generate update.json
+
+```shell script
+echo "localization-zh-cn=" > whiteList.properties
+mvn package appassembler:assemble
+sh target/appassembler/bin/app -id default -www www \
+    -skip-release-history -cache plugins -whitelist whiteList.properties \
+    -key rootCA/demo.key -certificate rootCA/emo.crt \
+    -root-certificate rootCA/demo.crt \
+    -cache-server http://localhost:9090/plugins/ \
+    -connectionCheckUrl http://localhost:9090/
+```
+3. Start your update center server (e.g. Nginx). Copy `www` into publish directory.
+4. Start your Jenkins server then copy demo.crt into `$JENKINS_HOME/war/WEB-INF/update-center-rootCAs/`.
+5. Change the update center url in `http://localhost:8080/pluginManager/advanced`.
+
+Finally, you could see your favour plugins.

src/main/java/org/jvnet/hudson/update_center/HPI.java

@@ -46,6 +46,7 @@ public class HPI extends MavenArtifact {
      * Which of the lineage did this come from?
      */
     public final PluginHistory history;
+    private String pluginSite = "http://updates.jenkins-ci.org/download/plugins/";
 
     private final Pattern developersPattern = Pattern.compile("([^:]*):([^:]*):([^,]*),?");
 
@@ -58,7 +59,7 @@ public HPI(MavenRepository repository, PluginHistory history, ArtifactInfo artif
      * Download a plugin via more intuitive URL. This also helps us track download counts.
      */
     public URL getURL() throws MalformedURLException {
-        return new URL("http://updates.jenkins-ci.org/download/plugins/"+artifact.artifactId+"/"+version+"/"+artifact.artifactId+".hpi");
+        return new URL(pluginSite+artifact.artifactId+"/"+version+"/"+artifact.artifactId+".hpi");
     }
 
     /**
@@ -145,6 +146,14 @@ public List<Developer> getDevelopers() throws IOException {
         return r;
     }
 
+    public String getPluginSite() {
+        return pluginSite;
+    }
+
+    public void setPluginSite(String pluginSite) {
+        this.pluginSite = pluginSite;
+    }
+
     boolean isEqualsTo(String groupId, String artifactId, String version) {
         return artifact.artifactId.equals(artifactId)
             && artifact.groupId.equals(groupId)

src/main/java/org/jvnet/hudson/update_center/Main.java

@@ -35,12 +35,16 @@
 import org.kohsuke.args4j.spi.OptionHandler;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.lang.reflect.Field;
+import java.net.MalformedURLException;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Collection;
@@ -51,6 +55,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.TreeMap;
 
 /**
@@ -68,6 +73,7 @@ public class Main {
     public File urlmap = new File("plugin-to-documentation-url.json");
 
     private Map<String, String> pluginToDocumentationUrl = new HashMap<>();
+    private Properties whitelistProperties = new Properties();
 
     /**
      * This file defines all the convenient symlinks in the form of
@@ -87,6 +93,15 @@ public class Main {
     @Option(name="-download",usage="Build mirrors.jenkins-ci.org layout")
     public File download = null;
 
+    @Option(name="-cache",usage="Cache directory used for caching plugin files")
+    public File cache = null;
+
+    @Option(name="-cache-all",usage="Cache all version of plugin instead of only the latest")
+    public boolean cacheAll;
+
+    @Option(name="-cache-server",usage="Cache server will replace the orignal")
+    public String cacheServer;
+
     /**
      * This option generates a directory layout containing htaccess files redirecting to Artifactory
      * for all files contained therein. This can be used for the 'fallback' mirror server.
@@ -158,6 +173,9 @@ public class Main {
     @Option(name="-arguments-file",usage="Specify invocation arguments in a file, with each line being a separate update site build")
     public File argumentsFile;
 
+    @Option(name="-whitelist",usage="If set to a file using the Java properties format, only plugins listed as keys will be included in the generated update site")
+    public File whitelist = null;
+
     private Signer signer = new Signer();
 
     public static final String EOL = System.getProperty("line.separator");
@@ -248,6 +266,13 @@ public void run() throws Exception {
 
         LatestLinkBuilder latest = createHtaccessWriter();
 
+        // load white list of plugins
+        if(whitelist != null && whitelist.isFile()) {
+            try(InputStream input = new FileInputStream(whitelist)) {
+                whitelistProperties.load(input);
+            }
+        }
+
         JSONObject ucRoot = buildUpdateCenterJson(repo, latest);
         writeToFile(mapPluginToDocumentationUrl(), urlmap);
         writeToFile(updateCenterPostCallJson(ucRoot), jsonp);
@@ -364,6 +389,9 @@ private JSONObject buildPluginVersions(MavenRepository repository) throws Except
         System.out.println("Build plugin versions index from the maven repo...");
 
         for (PluginHistory plugin : repository.listHudsonPlugins()) {
+                if(whitelistProperties.size() > 0 && whitelistProperties.get(plugin.artifactId) == null) {
+                    continue;
+                }
                 System.out.println(plugin.artifactId);
 
                 JSONObject versions = new JSONObject();
@@ -419,6 +447,10 @@ protected JSONObject buildPlugins(MavenRepository repository, LatestLinkBuilder
         }
         System.out.println("Gathering list of plugins and versions from the maven repo...");
         for (PluginHistory hpi : repository.listHudsonPlugins()) {
+            if(whitelistProperties.size() > 0 && whitelistProperties.get(hpi.artifactId) == null) {
+                continue;
+            }
+
             try {
                 System.out.println(hpi.artifactId);
 
@@ -427,12 +459,30 @@ protected JSONObject buildPlugins(MavenRepository repository, LatestLinkBuilder
 
                 pluginToDocumentationUrl.put(plugin.artifactId, plugin.getPluginUrl());
 
+                if (cache!=null) {
+                    if(cacheAll) {
+                        for (HPI v : hpi.artifacts.values()) {
+                            cachePlugin(v, new File(cache, "plugins/" + hpi.artifactId + "/" + v.version + "/" + hpi.artifactId + ".hpi"));
+                        }
+                    }
+
+                    HPI latestHpi = plugin.latest;
+                    cachePlugin(latestHpi, new File(cache, "plugins/" + hpi.artifactId + "/" + latestHpi.version + "/" + hpi.artifactId + ".hpi"));
+                }
+
+                if (cacheServer!=null) {
+                    for (HPI v : hpi.artifacts.values()) {
+                        v.setPluginSite(cacheServer);
+                    }
+                }
+
                 JSONObject json = plugin.toJSON();
                 if (json == null) {
                     System.out.println("Skipping due to lack of checksums: " + plugin.getName());
                     continue;
                 }
                 System.out.println("=> " + hpi.latest().getGavId());
+
                 plugins.put(plugin.artifactId, json);
                 latest.add(plugin.artifactId+".hpi", plugin.latest.getURL().getPath());
 
@@ -472,6 +522,35 @@ protected JSONObject buildPlugins(MavenRepository repository, LatestLinkBuilder
         return plugins;
     }
 
+    /**
+     * Cache target plugin file into local
+     * @param v plugin
+     * @param file target location
+     */
+    private void cachePlugin(HPI v, File file) {
+        if(file.exists()) {
+            System.out.println("Plugin file " + file.getName() + " already exists, skip download.");
+            return;
+        }
+
+        File parentDir = file.getParentFile();
+        if(!parentDir.exists() && !parentDir.mkdirs()) {
+            System.err.println("Can't create directory: " + parentDir.getAbsolutePath());
+            return;
+        }
+
+        try(InputStream input = v.getURL().openStream();
+            OutputStream output = new FileOutputStream(file)) {
+            System.out.println("Prepare download file: " + file.getName());
+
+            IOUtils.copy(input, output);
+        } catch (MalformedURLException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
     /**
      * Generates symlink to the latest version.
      */
@@ -537,6 +616,9 @@ protected JSONArray buildReleaseHistory(MavenRepository repository) throws Excep
                 JSONObject o = new JSONObject();
                 try {
                     Plugin plugin = new Plugin(h);
+                    if(whitelistProperties.size() > 0 && whitelistProperties.get(plugin.artifactId) == null) {
+                        continue;
+                    }
 
                     if (h.getTimestampAsDate().after(oldestDate.getTime())) {
                         String title = plugin.getName();