feat(travis/gradle): configurable regexes to parse artifacts (#142)

Offer configurable regexes to find RPM/DEB artifacts pushed to
artifactory as the default ones only match on the output of the "art"
CLI, but wouldn't on that of any other tools.  The result is a failed
Spinnaker pipeline when no artifact is found.

With this change users may add a list of arbitrary regexes that will be
used to parse build logs.  An example is provided that corresponds to
Gradle maven-publish, which would match on the following line (taken
from a real build, with project names changed):

    Upload https://artifacts.my.org/artifactory/yum-local/some/project/package-0.1.0.20161216.g1b1c767.rpm

README.md

@@ -45,10 +45,16 @@ travis:
     baseUrl: https://travis-ci.org
     address: https://api.travis-ci.org
     githubToken: 6a7729bdba8c4f9abc58b175213d83f072d1d832
+  regexes:
+  - /Upload https?:\/\/.+\/(.+\.(deb|rpm))/
 ```
 
 Currently Travis is used within Spinnaker to trigger pipelines and provide artifact information for the bake stages.
 
+When parsing artifact information from Travis builds, Igor uses a default regex
+that will match on output from the `art` CLI tool.  Different regexes than the
+default may be configured using the `regexes` list.
+
 ## Git Repositories
 
 ```

igor-web/config/igor.yml

@@ -19,12 +19,21 @@ endpoints.health.sensitive: false
 
 #travis:
 #  enabled: true
+#
 #  Repository sync makes a call to travis telling travis to sync repos against github.
 #  This makes new repos with builds in travis the github user has access to tracked by spinnaker.
 #  repositorySyncEnabled: true
+#
 #  Travis names are prefixed with travis- inside igor.
 #  masters:
 #    - name: ci # This will show as travis-ci inside spinnaker.
 #      baseUrl: https://travis-ci.org
 #      address: https://api.travis-ci.org
 #      githubToken: XXXXXX
+#
+#  Additional regexes to use when parsing build logs in search for uploaded
+#  artifacts, besides the default regexes which match on the output of the
+#  `art` CLI.  The provided example would match on the output from Gradle's
+#  maven-publish plugin.
+#  regexes:
+#  - Upload https?:\/\/.+\/(.+\.(deb|rpm))$

igor-web/src/main/groovy/com/netflix/spinnaker/igor/config/TravisConfig.groovy

@@ -54,14 +54,14 @@ class TravisConfig {
             String travisName = "travis-${host.name}"
             log.info "bootstrapping ${host.address} as ${travisName}"
 
-            [(travisName): travisService(travisName, host.baseUrl, host.githubToken, travisClient(host.address, igorConfigurationProperties.client.timeout), travisCache)]
+            [(travisName): travisService(travisName, host.baseUrl, host.githubToken, travisClient(host.address, igorConfigurationProperties.client.timeout), travisCache, travisProperties?.regexes)]
         })
         buildMasters.map.putAll travisMasters
         travisMasters
     }
 
-    static TravisService travisService(String travisHostId, String baseUrl, String githubToken, TravisClient travisClient, TravisCache travisCache) {
-        return new TravisService(travisHostId, baseUrl, githubToken, travisClient, travisCache)
+    static TravisService travisService(String travisHostId, String baseUrl, String githubToken, TravisClient travisClient, TravisCache travisCache, Iterable<String> artifactRexeges) {
+        return new TravisService(travisHostId, baseUrl, githubToken, travisClient, travisCache, artifactRexeges)
     }
 
     static TravisClient travisClient(String address, int timeout = 30000) {

igor-web/src/main/groovy/com/netflix/spinnaker/igor/config/TravisProperties.groovy

@@ -31,6 +31,9 @@ class TravisProperties {
     @Valid
     List<TravisHost> masters
 
+    @Valid
+    List<String> regexes
+
     static class TravisHost {
         @NotEmpty
         String name

igor-web/src/main/groovy/com/netflix/spinnaker/igor/travis/client/logparser/ArtifactParser.groovy

@@ -20,22 +20,34 @@ import com.netflix.spinnaker.igor.build.model.GenericArtifact
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter
+import java.util.regex.Matcher
+
 @CompileStatic
 @Slf4j
 class ArtifactParser {
 
-    static def REGEXES = [ /Uploading artifact: https?:\/\/.+\/(.+\.(deb|rpm)).*/,
-                          /Successfully pushed (.+\.(deb|rpm)) to .*/ ]
+    static List<String> DEFAULT_REGEXES = [
+        /Uploading artifact: https?:\/\/.+\/(.+\.(deb|rpm)).*$/,
+        /Successfully pushed (.+\\.(deb|rpm)) to .*/].toList()
 
-    static List<GenericArtifact> getArtifactsFromLog(String buildLog) {
-        List<GenericArtifact> artifacts = new ArrayList<GenericArtifact>()
+    /**
+     * Parse the build log using the given regular expressions.  If they are
+     * null, or empty, then DEFAULT_REGEXES will be used, matching on artifacts
+     * uploading from the `art` CLI tool.
+     */
+    static List<GenericArtifact> getArtifactsFromLog(String buildLog, Iterable<String> regexes) {
+        final List<GenericArtifact> artifacts = new ArrayList<GenericArtifact>()
+        if (regexes == null || regexes.size() <= 0) {
+            regexes = DEFAULT_REGEXES
+        }
         buildLog.split('\n').each { line ->
-            def m
-            for (def regex : REGEXES) {
+            Matcher m
+            for (def regex : regexes) {
                 if ((m = line =~ regex)) {
                     def match = m.group(1)
                     log.debug "Found artifact: ${match}"
-                    artifacts.add new GenericArtifact(match,match,match)
+                    artifacts.add new GenericArtifact(match, match, match)
                 }
             }
         }

igor-web/src/main/groovy/com/netflix/spinnaker/igor/travis/service/TravisService.groovy

@@ -54,15 +54,17 @@ class TravisService implements BuildService {
     final GithubAuth gitHubAuth
     final TravisClient travisClient
     final TravisCache travisCache
+    final private Set<String> artifactRegexes
     protected AccessToken accessToken
     private Accounts accounts
 
-    TravisService(String travisHostId, String baseUrl, String githubToken, TravisClient travisClient, TravisCache travisCache) {
+    TravisService(String travisHostId, String baseUrl, String githubToken, TravisClient travisClient, TravisCache travisCache, Iterable<String> artifactRegexes) {
         this.groupKey     = "${travisHostId}"
         this.gitHubAuth   = new GithubAuth(githubToken)
         this.travisClient = travisClient
         this.baseUrl      = baseUrl
         this.travisCache  = travisCache
+        this.artifactRegexes = artifactRegexes == null ? Collections.EMPTY_LIST : new HashSet<>(artifactRegexes)
     }
 
     @Override
@@ -253,7 +255,7 @@ class TravisService implements BuildService {
 
     GenericBuild getGenericBuild(Build build, String repoSlug) {
         GenericBuild genericBuild = TravisBuildConverter.genericBuild(build, repoSlug, baseUrl)
-        genericBuild.artifacts = ArtifactParser.getArtifactsFromLog(getLog(build))
+        genericBuild.artifacts = ArtifactParser.getArtifactsFromLog(getLog(build), artifactRegexes)
         return genericBuild
     }
 

igor-web/src/test/groovy/com/netflix/spinnaker/igor/travis/client/logparser/ArtifactParserTest.groovy

@@ -21,23 +21,48 @@ import spock.lang.Specification
 import spock.lang.Unroll
 
 class ArtifactParserTest extends Specification {
-    def "get multiple artifactory deb from log"() {
+
+    /**
+     * Covers projects pushing to artifactory with the Gradle plugin.
+     */
+    def "get multiple artifactory files from gradle log using custom regexes"() {
+        String buildLog =
+            ":publishPublishRpmPublicationToIvyRepository\n" +
+            "Upload https://foo.host/artifactory/yum-local/theorg/theprj/some-package-1.2.3-4.noarch.rpm\n" +
+            "Upload https://foo.host/artifactory/yum-local/theorg/theprj/another-package-4.3.2.deb\n" +
+            // add some noise, we expect none will match
+            "[Thread 0] Uploading artifact: https://foo.host/artifactory/debian-local/some/nice/path/some-package_0.0.7_amd64.deb;deb.distribution=trusty;deb.component=main;deb.architecture=amd64\n" +
+            "[Thread 0] Artifactory response: 201 Created\n" +
+            "[Thread 0] Uploading artifact: https://foo.host/artifactory/debian-local/some/other/path/some-other-package_1.3.3.7_amd64.deb;deb.distribution=trusty;deb.component=main;deb.architecture=amd64\n" +
+            "[Thread 0] Artifactory response: 201 Created"
+
+        List<String> gradleRegex = [/Upload https?:\/\/.+\/(.+\.(deb|rpm))$/].toList()
+
+        when:
+        List<GenericArtifact> artifacts = ArtifactParser.getArtifactsFromLog(buildLog, gradleRegex)
+
+        then:
+        artifacts.first().fileName == "some-package-1.2.3-4.noarch.rpm"
+        artifacts.last().fileName == "another-package-4.3.2.deb"
+    }
+
+    def "get multiple artifactory deb from log using default regexes"() {
         String buildLog = "[Thread 0] Uploading artifact: https://foo.host/artifactory/debian-local/some/nice/path/some-package_0.0.7_amd64.deb;deb.distribution=trusty;deb.component=main;deb.architecture=amd64\n" +
             "[Thread 0] Artifactory response: 201 Created\n" +
             "[Thread 0] Uploading artifact: https://foo.host/artifactory/debian-local/some/other/path/some-other-package_1.3.3.7_amd64.deb;deb.distribution=trusty;deb.component=main;deb.architecture=amd64\n" +
             "[Thread 0] Artifactory response: 201 Created"
         when:
-        List<GenericArtifact> artifacts = ArtifactParser.getArtifactsFromLog(buildLog)
+        List<GenericArtifact> artifacts = ArtifactParser.getArtifactsFromLog(buildLog, null)
 
         then:
         artifacts.first().fileName == "some-package_0.0.7_amd64.deb"
         artifacts.last().fileName == "some-other-package_1.3.3.7_amd64.deb"
     }
 
     @Unroll
-    def "get single deb or rpm from log"() {
+    def "get single deb or rpm from log using default regexes"() {
         expect:
-        ArtifactParser.getArtifactsFromLog(buildLog).first().fileName == packageName
+        List<GenericArtifact> artifacts = ArtifactParser.getArtifactsFromLog(buildLog, Collections.emptyList())
 
         where:
         buildLog                     || packageName

igor-web/src/test/groovy/com/netflix/spinnaker/igor/travis/service/TravisServiceSpec.groovy

@@ -37,7 +37,7 @@ class TravisServiceSpec extends Specification{
 
     void setup() {
         client = Mock(TravisClient)
-        service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', client, null)
+        service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', client, null, Collections.emptyList())
 
         AccessToken accessToken = new AccessToken()
         accessToken.accessToken = "someToken"