[JENKINS-58147] Update yaml support to include groupId for incrementals

README.md

@@ -41,24 +41,33 @@ The following custom version specifiers can also be used:
 
 Plugins can also be entered in a Jenkins yaml file with the following format:
 
-```
+```yaml
 jenkins:
   ...
 plugins:
-  - artifactId: plugin1_artifactId
-    source: 
-      version: plugin1_version
-      url: plugin1_url
-  - artifactId: plugin2_artifactId
+- artifactId: git
+    source:
+      version: latest
+  - artifactId: job-import-plugin
+    source:
+      version: 2.1
+  - artifactId: docker
+  - artifactId: cloudbees-bitbucket-branch-source
+    source:
+      version: 2.4.4
+  - artifactId: script-security
     source:
-      version: plugin2_version
-  - artifactId: plugin3_artifactId
+      url: http://ftp-chi.osuosl.org/pub/jenkins/plugins/script-security/1.56/script-security.hpi
+  - artifactId: workflow-step-api
+    groupId: org.jenkins-ci.plugins.workflow
+    source:
+      version: 2.19-rc289.d09828a05a74
   ...
 tool:
   ...
 ```
 
-Any root object other than `plugins` will be ignored. As with the plugins.txt file, version and url are optional, and if no version is entered, the latest version is the default.
+Any root object other than `plugins` will be ignored by the plugin installation manager tool. As with the plugins.txt file, version and url are optional, and if no version is entered, the latest version is the default. If a groupId is entered, the tool will try to download the plugin from the incrementals repository.
 
 
 #### Examples
@@ -80,4 +89,7 @@ java -jar plugin-management-cli/target/plugin-management-tool.jar -p "workflow-s
 
 #### Other Information
 The plugin manager tries to use update center data to get the latest information about a plugin's dependencies. If this information is unavailable, it will use the dependency information from the downloaded plugin's MANIFEST.MF file.
+
 For plugins listed in a .txt file, each plugin must be listed on a new line. Comments beginning with `#` will be filtered out.
+
+Support for downloading plugins from maven is not currently supported. [JENKINS-58217](https://issues.jenkins-ci.org/browse/JENKINS-58217)

plugin-management-cli/src/main/java/io/jenkins/tools/pluginmanager/cli/PluginInputFormatException.java

@@ -0,0 +1,12 @@
+package io.jenkins.tools.pluginmanager.cli;
+
+public class PluginInputFormatException extends RuntimeException {
+
+    public PluginInputFormatException(String message) {
+        super(message);
+    }
+
+    public PluginInputFormatException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

plugin-management-cli/src/main/java/io/jenkins/tools/pluginmanager/cli/PluginParser.java

@@ -1,5 +1,6 @@
 package io.jenkins.tools.pluginmanager.cli;
 
+import io.jenkins.tools.pluginmanager.cli.PluginInputFormatException;
 import io.jenkins.tools.pluginmanager.impl.Plugin;
 import java.io.BufferedReader;
 import java.io.File;
@@ -16,6 +17,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.yaml.snakeyaml.Yaml;
 
+
 import static java.util.stream.Collectors.toList;
 
 public class PluginParser {
@@ -76,15 +78,22 @@ public List<Plugin> parsePluginYamlFile(File pluginYamlFile) {
                     Object nameObject = pluginInfo.get("artifactId");
                     String name = nameObject == null ? null : nameObject.toString();
                     if (StringUtils.isEmpty(name)) {
-                        System.out.println("artifactId is required, skipping...");
-                        continue;
+                        throw new PluginInputFormatException("ArtifactId is required");
                     }
+                    Object groupIdObject = pluginInfo.get("groupId");
+                    String groupId = groupIdObject == null ? null : groupIdObject.toString();
                     Map pluginSource = (Map) pluginInfo.get("source");
+                    String incrementalsVersion = null;
                     Plugin plugin;
-                    if (pluginSource == null) {
-                        plugin = new Plugin(name, "latest", null);
+                    if (pluginSource == null && !StringUtils.isEmpty(groupId)) {
+                        throw new PluginInputFormatException("Version must be input for " + name);
+                    } else if (pluginSource == null) {
+                        plugin = new Plugin(name, "latest", null, null);
                     } else {
                         Object versionObject = pluginSource.get("version");
+                        if (!StringUtils.isEmpty(groupId) && versionObject == null) {
+                            throw new PluginInputFormatException("Version must be input for " + name);
+                        }
                         String version = versionObject == null ? "latest" : versionObject.toString();
                         Object urlObject = pluginSource.get("url");
                         String url;
@@ -93,7 +102,7 @@ public List<Plugin> parsePluginYamlFile(File pluginYamlFile) {
                         } else {
                             url = null;
                         }
-                        plugin = new Plugin(name, version, url);
+                        plugin = new Plugin(name, version, url, groupId);
                     }
                     pluginsFromYaml.add(plugin);
                 }
@@ -134,22 +143,27 @@ private Plugin parsePluginLine(String pluginLine) {
         String pluginName = pluginInfo[0];
         String pluginVersion = "latest";
         String pluginUrl = null;
+        String groupId = null;
 
         // "http, https, ftp" are valid
 
-        if (pluginInfo.length >= 2 && !StringUtils.isEmpty(pluginInfo[1])) {
+        if (pluginInfo.length >= 2) {
             pluginVersion = pluginInfo[1];
+            if (pluginVersion.contains("incrementals")) {
+                String[] incrementalsVersionInfo = pluginVersion.split(";");
+                groupId = incrementalsVersionInfo[1];
+                pluginVersion = incrementalsVersionInfo[2];
+            }
         }
 
         if (pluginInfo.length >= 3) {
-            pluginVersion = pluginInfo[1];
             if (isURL(pluginInfo[2])) {
                 pluginUrl = pluginInfo[2];
             } else {
                 System.out.println("Invalid URL "+ pluginInfo[2] +" , will ignore");
             }
         }
-        return new Plugin(pluginName, pluginVersion, pluginUrl);
+        return new Plugin(pluginName, pluginVersion, pluginUrl, groupId);
     }
 
     public static boolean isURL(String url) {

plugin-management-cli/src/test/java/io/jenkins/tools/pluginmanager/cli/CliOptionsTest.java

@@ -54,20 +54,20 @@ public void createParser() throws CmdLineException {
         //corresponds to plugins in plugin.txt
         txtRequestedPlugins = new ArrayList<>();
         txtRequestedPlugins.add(new Plugin("google-api-client-plugin",
-                "latest", "https://updates.jenkins.io/latest/google-api-client-plugin.hpi"));
-        txtRequestedPlugins.add(new Plugin("git", "latest", null));
-        txtRequestedPlugins.add(new Plugin("job-import-plugin", "2.1", null));
-        txtRequestedPlugins.add(new Plugin("docker", "latest", null));
-        txtRequestedPlugins.add(new Plugin("cloudbees-bitbucket-branch-source", "2.4.4", null));
+                "latest", "https://updates.jenkins.io/latest/google-api-client-plugin.hpi", null));
+        txtRequestedPlugins.add(new Plugin("git", "latest", null, null));
+        txtRequestedPlugins.add(new Plugin("job-import-plugin", "2.1", null, null));
+        txtRequestedPlugins.add(new Plugin("docker", "latest", null, null));
+        txtRequestedPlugins.add(new Plugin("cloudbees-bitbucket-branch-source", "2.4.4", null, null));
         txtRequestedPlugins.add(new Plugin("script-security", "latest",
-                "http://ftp-chi.osuosl.org/pub/jenkins/plugins/script-security/1.56/script-security.hpi"));
+                "http://ftp-chi.osuosl.org/pub/jenkins/plugins/script-security/1.56/script-security.hpi", null));
         txtRequestedPlugins.add(new Plugin("workflow-step-api",
-                "incrementals;org.jenkins-ci.plugins.workflow;2.19-rc289.d09828a05a74", null));
-        txtRequestedPlugins.add(new Plugin("matrix-project", "latest", null));
-        txtRequestedPlugins.add(new Plugin("junit", "experimental", null));
+                "2.19-rc289.d09828a05a74", null, "org.jenkins-ci.plugins.workflow"));
+        txtRequestedPlugins.add(new Plugin("matrix-project", "latest", null, null));
+        txtRequestedPlugins.add(new Plugin("junit", "experimental", null, null));
         txtRequestedPlugins.add(new Plugin("credentials", "2.2.0",
-                "http://ftp-chi.osuosl.org/pub/jenkins/plugins/credentials/2.2.0/credentials.hpi"));
-        txtRequestedPlugins.add(new Plugin("blueocean", "latest", null));
+                "http://ftp-chi.osuosl.org/pub/jenkins/plugins/credentials/2.2.0/credentials.hpi", null));
+        txtRequestedPlugins.add(new Plugin("blueocean", "latest", null, null));
 
         System.setOut(new PrintStream(outContent));
     }
@@ -111,7 +111,7 @@ public void setupAliasTest() throws CmdLineException, IOException, URISyntaxExce
                 "display-url-api::https://updates.jenkins.io/download/plugins/display-url-api/1.0/display-url-api.hpi");
 
         Plugin displayUrlPlugin = new Plugin("display-url-api", "latest",
-                "https://updates.jenkins.io/download/plugins/display-url-api/1.0/display-url-api.hpi");
+                "https://updates.jenkins.io/download/plugins/display-url-api/1.0/display-url-api.hpi", null);
 
         Config cfg = options.setup();
 
@@ -132,9 +132,9 @@ public void setupPluginsTest() throws CmdLineException, IOException, URISyntaxEx
                 "--plugins", "ssh-slaves:1.10 mailer cobertura:experimental");
 
         List<Plugin> requestedPlugins = new ArrayList<>(txtRequestedPlugins);
-        requestedPlugins.add(new Plugin("ssh-slaves", "1.10", null));
-        requestedPlugins.add(new Plugin("mailer", "latest", null));
-        requestedPlugins.add(new Plugin("cobertura", "experimental", null));
+        requestedPlugins.add(new Plugin("ssh-slaves", "1.10", null, null));
+        requestedPlugins.add(new Plugin("mailer", "latest", null, null));
+        requestedPlugins.add(new Plugin("cobertura", "experimental", null, null));
 
         Config cfg = options.setup();
 

plugin-management-cli/src/test/java/io/jenkins/tools/pluginmanager/cli/PluginParserTest.java

@@ -1,5 +1,6 @@
 package io.jenkins.tools.pluginmanager.cli;
 
+import io.jenkins.tools.pluginmanager.cli.PluginInputFormatException;
 import io.jenkins.tools.pluginmanager.impl.Plugin;
 import java.io.File;
 import java.net.URISyntaxException;
@@ -30,21 +31,22 @@ public void setup() {
         pluginParser = new PluginParser();
 
         expectedPluginInfo = new ArrayList<>();
-        expectedPluginInfo.add(new Plugin("git", "latest", null).toString());
-        expectedPluginInfo.add(new Plugin("job-import-plugin", "2.1", null).toString());
-        expectedPluginInfo.add(new Plugin("docker", "latest", null).toString());
-        expectedPluginInfo.add(new Plugin("cloudbees-bitbucket-branch-source", "2.4.4", null).toString());
+        expectedPluginInfo.add(new Plugin("git", "latest", null, null).toString());
+        expectedPluginInfo.add(new Plugin("job-import-plugin", "2.1", null, null).toString());
+        expectedPluginInfo.add(new Plugin("docker", "latest", null, null).toString());
+        expectedPluginInfo.add(new Plugin("cloudbees-bitbucket-branch-source", "2.4.4", null, null).toString());
         expectedPluginInfo.add(new Plugin("script-security", "latest",
-                "http://ftp-chi.osuosl.org/pub/jenkins/plugins/script-security/1.56/script-security.hpi").toString());
+                "http://ftp-chi.osuosl.org/pub/jenkins/plugins/script-security/1.56/script-security.hpi", null)
+                .toString());
         expectedPluginInfo.add(new Plugin("workflow-step-api",
-                "incrementals;org.jenkins-ci.plugins.workflow;2.19-rc289.d09828a05a74", null).toString());
-        expectedPluginInfo.add(new Plugin("matrix-project", "latest", null).toString());
-        expectedPluginInfo.add(new Plugin("junit", "experimental", null).toString());
+                "2.19-rc289.d09828a05a74", null, "org.jenkins-ci.plugins.workflow").toString());
+        expectedPluginInfo.add(new Plugin("matrix-project", "latest", null, null).toString());
+        expectedPluginInfo.add(new Plugin("junit", "experimental", null, null).toString());
         expectedPluginInfo.add(new Plugin("credentials", "2.2.0",
-                "http://ftp-chi.osuosl.org/pub/jenkins/plugins/credentials/2.2.0/credentials.hpi").toString());
-        expectedPluginInfo.add(new Plugin("blueocean", "latest", null).toString());
+                "http://ftp-chi.osuosl.org/pub/jenkins/plugins/credentials/2.2.0/credentials.hpi", null).toString());
+        expectedPluginInfo.add(new Plugin("blueocean", "latest", null, null).toString());
         expectedPluginInfo.add(new Plugin("google-api-client-plugin", "latest",
-                "https://updates.jenkins.io/latest/google-api-client-plugin.hpi").toString());
+                "https://updates.jenkins.io/latest/google-api-client-plugin.hpi", null).toString());
 
         Collections.sort(expectedPluginInfo);
 
@@ -119,6 +121,25 @@ public void parsePluginYamlFileTest() throws URISyntaxException {
         assertEquals(expectedPluginInfo, pluginInfo);
     }
 
+    @Test(expected = PluginInputFormatException.class)
+    public void badFormatYamlNoArtifactIdTest() throws URISyntaxException {
+        File pluginYmlFile = new File(this.getClass().getResource("/badformat1.yaml").toURI());
+        List<Plugin> pluginsFromYamlFile = pluginParser.parsePluginYamlFile(pluginYmlFile);
+    }
+
+    @Test(expected = PluginInputFormatException.class)
+    public void badFormatYamlGroupIdNoVersion() throws URISyntaxException {
+        File pluginYmlFile = new File(this.getClass().getResource("/badformat2.yaml").toURI());
+        List<Plugin> pluginsFromYamlFile = pluginParser.parsePluginYamlFile(pluginYmlFile);
+    }
+
+    @Test(expected = PluginInputFormatException.class)
+    public void badFormatYamlGroupIdNoVersion2() throws URISyntaxException {
+        File pluginYmlFile = new File(this.getClass().getResource("/badformat3.yaml").toURI());
+        List<Plugin> pluginsFromYamlFile = pluginParser.parsePluginYamlFile(pluginYmlFile);
+    }
+
+    @Test
     public void fileExistsTest() throws URISyntaxException {
         assertEquals(false, pluginParser.fileExists(null));
 

plugin-management-cli/src/test/resources/badformat1.yaml

@@ -0,0 +1,7 @@
+plugins:
+  - artifactId: job-import-plugin
+    source:
+      version: 2.1
+  - artifactId:
+    source:
+      version: latest

plugin-management-cli/src/test/resources/badformat2.yaml

@@ -0,0 +1,15 @@
+jenkins:
+  systemMessage: "Testing"
+plugins:
+  - artifactId: git
+    source:
+      version: latest
+  - artifactId: workflow-step-api
+    groupId: org.jenkins-ci.plugins.workflow
+    source:
+      version:
+tool:
+  git:
+    installations:
+    - home: "git"
+      name: "Default"

plugin-management-cli/src/test/resources/badformat3.yaml

@@ -0,0 +1,13 @@
+jenkins:
+  systemMessage: "Testing"
+plugins:
+  - artifactId: git
+    source:
+      version: latest
+  - artifactId: workflow-step-api
+    groupId: org.jenkins-ci.plugins.workflow
+tool:
+  git:
+    installations:
+    - home: "git"
+      name: "Default"

plugin-management-cli/src/test/resources/plugins.yaml

@@ -15,8 +15,9 @@ plugins:
     source:
       url: http://ftp-chi.osuosl.org/pub/jenkins/plugins/script-security/1.56/script-security.hpi
   - artifactId: workflow-step-api
+    groupId: org.jenkins-ci.plugins.workflow
     source:
-      version: "incrementals;org.jenkins-ci.plugins.workflow;2.19-rc289.d09828a05a74"
+      version: 2.19-rc289.d09828a05a74
   - artifactId: matrix-project
     source:
       url: httttp://ftp-chi.osuosl.org/pub/jenkins/plugins/matrix-project/1.7.1/matrix-project.hpi

plugin-management-library/src/main/java/io/jenkins/tools/pluginmanager/impl/Plugin.java

@@ -10,23 +10,25 @@ public class Plugin {
     private String name;
     private String originalName;
     private VersionNumber version;
+    private String groupId;
     private String url;
     private File file;
     private boolean isPluginOptional;
     private List<Plugin> dependencies;
     private List<Plugin> dependents;
 
-    public Plugin(String name, String version, String url) {
+
+    public Plugin(String name, String version, String url, String groupId) {
         this.originalName = name;
         this.name = name;
         if (StringUtils.isEmpty(version)) {
             version = "latest";
         }
         this.version = new VersionNumber(version);
         this.url = url;
-        this.dependencies = new ArrayList<>();
-        this.dependents = new ArrayList<>();
-
+        this.groupId = groupId;
+        dependencies = new ArrayList<>();
+        dependents = new ArrayList<>();
     }
 
     public Plugin(String name, String version, boolean isPluginOptional) {
@@ -59,6 +61,10 @@ public void setPluginOptional(boolean isPluginOptional) {
         this.isPluginOptional = isPluginOptional;
     }
 
+    public void setGroupId(String groupId) {
+        this.groupId = groupId;
+    }
+
     public String getName() {
         return name;
     }
@@ -87,6 +93,10 @@ public String getOriginalName() {
         return originalName;
     }
 
+    public String getGroupId() {
+        return groupId;
+    }
+
     public void setDependencies(List<Plugin> dependencies) {
         this.dependencies = dependencies;
     }

plugin-management-library/src/main/java/io/jenkins/tools/pluginmanager/impl/PluginManager.java

@@ -438,14 +438,11 @@ public String getPluginDownloadUrl(Plugin plugin) {
             urlString = String.format("%s/latest/%s.hpi", jenkinsUcLatest, pluginName);
         } else if (pluginVersion.equals("experimental")) {
             urlString = String.format("%s/latest/%s.hpi", cfg.getJenkinsUcExperimental(), pluginName);
-        } else if (pluginVersion.contains("incrementals")) {
-            System.out.println("plugin version " + pluginVersion);
-            String[] incrementalsVersionInfo = pluginVersion.split(";");
-            String groupId = incrementalsVersionInfo[1];
-            String incrementalsVersion = incrementalsVersionInfo[2];
+        } else if (!StringUtils.isEmpty(plugin.getGroupId())) {
+            String groupId = plugin.getGroupId();
             groupId = groupId.replace(".", "/");
             String incrementalsVersionPath =
-                    String.format("%s/%s/%s-%s.hpi", pluginName, incrementalsVersion, pluginName, incrementalsVersion);
+                    String.format("%s/%s/%s-%s.hpi", pluginName, pluginVersion, pluginName, pluginVersion);
             urlString =
                     String.format("%s/%s/%s", cfg.getJenkinsIncrementalsRepoMirror(), groupId, incrementalsVersionPath);
         } else {