Add support for cache plugins

jenkins-infra · Oct 15, 2018 · 4bfd4f3 · 4bfd4f3
1 parent 7809901
commit 4bfd4f3
Show file tree

Hide file tree

Showing 5 changed files with 196 additions and 1 deletion.
diff --git a/list.properties b/list.properties
@@ -0,0 +1,81 @@
+durable-task=1.26
+project-inheritance=2.0.0
+pipeline-rest-api=2.10
+jsch=0.1.54.2
+parameterized-trigger=2.35.2
+job-dsl=1.70
+jackson2-api=2.8.11.3
+structs=1.17
+junit=1.26.1
+pipeline-model-api=1.3.2
+workflow-basic-steps=2.11
+pipeline-stage-tags-metadata=1.3.2
+branch-api=2.0.20
+alauda-devops-sync=0.2.19
+script-security=1.46
+credentials=2.1.18
+workflow-support=2.21
+plain-credentials=1.4
+pipeline-stage-step=2.3
+vsphere-cloud=2.18
+workflow-api=2.30
+pipeline-graph-analysis=1.7
+token-macro=2.5
+ace-editor=1.1
+git=3.9.1
+pipeline-model-definition=1.3.2
+maven-plugin=3.1.2
+authentication-tokens=1.3
+display-url-api=2.2.0
+variant=1.1
+metrics=4.0.2.2
+jquery-detached=1.2.1
+favorite=2.3.2
+ssh-credentials=1.14
+docker-workflow=1.17
+jdk-tool=1.1
+cloudbees-folder=6.6
+support-core=2.50
+async-http-client=1.9.40.0
+managed-scripts=1.4
+subversion=2.12.1
+mapdb-api=1.0.9.0
+blueocean-rest-impl=1.9.0
+blueocean-core-js=1.9.0
+antisamy-markup-formatter=1.5
+node-iterator-api=1.5.0
+jenkins-design-language=1.9.0
+workflow-scm-step=2.7
+bouncycastle-api=2.17
+workflow-cps=2.58
+blueocean-web=1.9.0
+scm-api=2.2.8
+kubernetes=1.12.7
+workflow-job=2.26
+git-client=2.7.3
+javadoc=1.4
+rebuild=1.29
+command-launcher=1.2
+mailer=1.21
+pipeline-model-extensions=1.3.2
+pipeline-input-step=2.8
+workflow-multibranch=2.20
+conditional-buildstep=1.3.6
+promoted-builds=3.2
+ssh-slaves=1.28.1
+workflow-step-api=2.16
+apache-httpcomponents-client-4-api=4.5.5-3.0
+kubernetes-credentials=0.4.0
+workflow-durable-task-step=2.22
+workflow-cps-global-lib=2.12
+git-server=1.7
+docker-commons=1.13
+credentials-binding=1.16
+badge=1.6
+run-condition=1.2
+config-file-provider=3.3
+blueocean-jwt=1.9.0
+blueocean-commons=1.9.0
+blueocean-rest=1.9.0
+matrix-project=1.13
+pipeline-model-declarative-agent=1.1.1
diff --git a/src/main/java/org/jvnet/hudson/update_center/HPI.java b/src/main/java/org/jvnet/hudson/update_center/HPI.java
@@ -46,6 +46,7 @@ public class HPI extends MavenArtifact {
      * Which of the lineage did this come from?
      */
     public final PluginHistory history;
+    private String pluginSite = "http://updates.jenkins-ci.org/download/plugins/";
 
     private final Pattern developersPattern = Pattern.compile("([^:]*):([^:]*):([^,]*),?");
 
@@ -58,7 +59,7 @@ public HPI(MavenRepository repository, PluginHistory history, ArtifactInfo artif
      * Download a plugin via more intuitive URL. This also helps us track download counts.
      */
     public URL getURL() throws MalformedURLException {
-        return new URL("http://updates.jenkins-ci.org/download/plugins/"+artifact.artifactId+"/"+version+"/"+artifact.artifactId+".hpi");
+        return new URL(pluginSite+artifact.artifactId+"/"+version+"/"+artifact.artifactId+".hpi");
     }
 
     /**
@@ -145,6 +146,14 @@ public List<Developer> getDevelopers() throws IOException {
         return r;
     }
 
+    public String getPluginSite() {
+        return pluginSite;
+    }
+
+    public void setPluginSite(String pluginSite) {
+        this.pluginSite = pluginSite;
+    }
+
     boolean isEqualsTo(String groupId, String artifactId, String version) {
         return artifact.artifactId.equals(artifactId)
             && artifact.groupId.equals(groupId)

diff --git a/src/main/java/org/jvnet/hudson/update_center/Main.java b/src/main/java/org/jvnet/hudson/update_center/Main.java
@@ -35,12 +35,16 @@
 import org.kohsuke.args4j.spi.OptionHandler;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.lang.reflect.Field;
+import java.net.MalformedURLException;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Collection;
@@ -51,6 +55,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.TreeMap;
 
 /**
@@ -87,6 +92,15 @@ public class Main {
     @Option(name="-download",usage="Build mirrors.jenkins-ci.org layout")
     public File download = null;
 
+    @Option(name="-cache",usage="Cache target plugin")
+    public File cache = null;
+
+    @Option(name="-cacheAll",usage="Cache all version of plugin")
+    public boolean cacheAll;
+
+    @Option(name="-cacheServer",usage="Cache server will replace the orignal")
+    public String cacheServer;
+
     /**
      * This option generates a directory layout containing htaccess files redirecting to Artifactory
      * for all files contained therein. This can be used for the 'fallback' mirror server.
@@ -158,6 +172,9 @@ public class Main {
     @Option(name="-arguments-file",usage="Specify invocation arguments in a file, with each line being a separate update site build")
     public File argumentsFile;
 
+    @Option(name="-whitelist",usage="White list for plugins")
+    public File whitelist = null;
+
     private Signer signer = new Signer();
 
     public static final String EOL = System.getProperty("line.separator");
@@ -412,13 +429,26 @@ protected JSONObject buildPlugins(MavenRepository repository, LatestLinkBuilder
 
         int validCount = 0;
 
+        Properties whitelistPro = new Properties();
+        if(whitelist != null && whitelist.isFile()) {
+            try(InputStream input = new FileInputStream(whitelist)) {
+                whitelistPro.load(input);
+            }
+        }
+
         JSONObject plugins = new JSONObject();
         ArtifactoryRedirector redirector = null;
         if (downloadFallback != null) {
             redirector = new ArtifactoryRedirector(downloadFallback);
         }
         System.out.println("Gathering list of plugins and versions from the maven repo...");
         for (PluginHistory hpi : repository.listHudsonPlugins()) {
+            if(whitelistPro.size() > 0) {
+                if(whitelistPro.get(hpi.artifactId) == null) {
+                    continue;
+                }
+            }
+
             try {
                 System.out.println(hpi.artifactId);
 
@@ -427,12 +457,30 @@ protected JSONObject buildPlugins(MavenRepository repository, LatestLinkBuilder
 
                 pluginToDocumentationUrl.put(plugin.artifactId, plugin.getPluginUrl());
 
+                if (cache!=null) {
+                    if(cacheAll) {
+                        for (HPI v : hpi.artifacts.values()) {
+                            cachePlugin(v, new File(cache, "plugins/" + hpi.artifactId + "/" + v.version + "/" + hpi.artifactId + ".hpi"));
+                        }
+                    }
+
+                    HPI latestHpi = plugin.latest;
+                    cachePlugin(latestHpi, new File(cache, "plugins/" + hpi.artifactId + "/" + latestHpi.version + "/" + hpi.artifactId + ".hpi"));
+                }
+
+                if (cacheServer!=null) {
+                    for (HPI v : hpi.artifacts.values()) {
+                        v.setPluginSite(cacheServer);
+                    }
+                }
+
                 JSONObject json = plugin.toJSON();
                 if (json == null) {
                     System.out.println("Skipping due to lack of checksums: " + plugin.getName());
                     continue;
                 }
                 System.out.println("=> " + hpi.latest().getGavId());
+
                 plugins.put(plugin.artifactId, json);
                 latest.add(plugin.artifactId+".hpi", plugin.latest.getURL().getPath());
 
@@ -472,6 +520,35 @@ protected JSONObject buildPlugins(MavenRepository repository, LatestLinkBuilder
         return plugins;
     }
 
+    /**
+     * Cache target plugin file into local
+     * @param v plugin
+     * @param file target location
+     */
+    private void cachePlugin(HPI v, File file) {
+        if(file.exists()) {
+            System.out.println("Plugin file " + file.getName() + " already exists, skip download.");
+            return;
+        }
+
+        File parentDir = file.getParentFile();
+        if(!parentDir.exists() && !parentDir.mkdirs()) {
+            System.err.println("Can't create directory: " + parentDir.getAbsolutePath());
+            return;
+        }
+
+        try(InputStream input = v.getURL().openStream();
+            OutputStream output = new FileOutputStream(file)) {
+            System.out.println("Prepare download file: " + file.getName());
+
+            IOUtils.copy(input, output);
+        } catch (MalformedURLException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
     /**
      * Generates symlink to the latest version.
      */

diff --git a/surenpi.crt b/surenpi.crt
@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIIB8zCCAVwCCQC9v6hxT0pAbjANBgkqhkiG9w0BAQsFADA+MQswCQYDVQQGEwJj
+bjEQMA4GA1UECgwHc3VyZW5waTEdMBsGCSqGSIb3DQEJARYOenhqbHd0QDEyNi5j
+b20wHhcNMTgxMDEyMTMxNTQ5WhcNMjExMDExMTMxNTQ5WjA+MQswCQYDVQQGEwJj
+bjEQMA4GA1UECgwHc3VyZW5waTEdMBsGCSqGSIb3DQEJARYOenhqbHd0QDEyNi5j
+b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKzjBFaYsyoNocbpaQG5yWhn
+v6QrzTw9JjtaMgc87uL7iuHUAay5wRi0GHFgfaG8dKTsaWeYq7hv8iNVvUc60eHA
+ZvMyMGOvY46yEsGgdupERxDjRqglngN41d8VB4sW4DBN4a894xBVsIfIqgn1Ktp1
+Y+tLqgkdP6ox7gv0CSu3AgMBAAEwDQYJKoZIhvcNAQELBQADgYEAm5vOvF1PG0aG
+hbzupFAEiAtekpBaXNzVA0rl0/NyN8ky5xkWwP/u7Hn4kOaX2TwdefamHimOAggP
+KEJ+Wb4ZzyUMw9HkTTWCACP5gqB8Hf/bVQ+f2zLNZgHXT1v09TS+G5MPvb9Atl0K
+gbraIptomDqkmRS0ER9fWf2ze/I4CnU=
+-----END CERTIFICATE-----
diff --git a/surenpi.key b/surenpi.key
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICWwIBAAKBgQCs4wRWmLMqDaHG6WkBucloZ7+kK808PSY7WjIHPO7i+4rh1AGs
+ucEYtBhxYH2hvHSk7GlnmKu4b/IjVb1HOtHhwGbzMjBjr2OOshLBoHbqREcQ40ao
+JZ4DeNXfFQeLFuAwTeGvPeMQVbCHyKoJ9SradWPrS6oJHT+qMe4L9AkrtwIDAQAB
+AoGAMw78qgKA70uFWrR2YZ8X33C8hWS2QTMiBmdV3RNezcRSOO2e7P8TCxOtIFbJ
+46k/nbGAU6sgE+Fj1pGsy8uQTipV0SAoe0gegdGNVSLTyRnR7447s5FSWfayE6gn
+2WgnmqbI90QM3+g6r+tWc7zKnmNp0MXhz49cAp/tzotfwykCQQDc8MyuNKL+2v3d
+LtoXTZx16qkmEr4rzBPh6F4qtUyxS+ey10EFTp6JynO5270jG3tMDwf4xgsstvmY
+i0Kh5zlDAkEAyFIkRHK/Bz/i+yBQclwbq2BQuP5Lz/jpKSxZ/IpvmXrnh+FqEVJT
+VZeyIMoAHdSNBHrOWbiTmIe2Fsf5/f6SfQJAbqkpS4PPs+iJyJPj5lIYMLNlH025
+9IPV3YueRrPiAtdq6JeJ2JelLoy5Y/jchc4I9nJ84qyHAlFvOGlf6Wm/swJAXq50
+deXGZKbNoeSkIWlnqpjbjpSjQKllixBrd/H2zZRJ2u+wMn+2473Pc7AGXhMYlAV3
+gvC60l1Ss3HRtzucAQJAfbOXrdulU0QjuvV9homNfOH35comgeLVDRgKfrTDBxqt
+X/EfH2G8N9PEH6O4obemPK/MW+4eKA7mD9OI/P/K8A==
+-----END RSA PRIVATE KEY-----