feat(travis): Refactor and simplify Travis polling (#467)

* chore(travis): Refactor and simplify Travis polling

Previously, the polling, using the Travis v2 API, asked Travis for ~50 repos with recently built builds, and then, _for each repo_, asked for the 5 most recent builds. In other words, each polling cycle hits Travis with ~51 potentially heavy API calls (plus more to fetch logs). This commit brings that number down to 1, using the v3 API, and also removes a lot of code that belongs to the v2 API or was just band-aid for the quirks in the v2 API. One of those quirks was that some builds sometimes disappeared from the API as soon as they were built. I have not been able to verify 100% that this bug is gone in the v3 API, and because this commit removes the band-aid, we should proceed with caution before deploying to production.

* Increase default number of jobs retrieved to 300

* Add a couple of @deprecated annotations and fix a comment

* Add back the tracking cache due to limitations in the Travis API

Should give us the best of both worlds; no more missed builds and still quite a bit easier on the Travis API. I still need to update tests, will fix ASAP.

* Fix tests

* Add back getJobConfig to TravisService that I erroneously removed in 890bb44

* Add back repo sync and the duration logging

* Add toMillis() to duration logging

igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/BuildCache.java

@@ -17,7 +17,11 @@
 
 import com.netflix.spinnaker.igor.IgorConfigurationProperties;
 import com.netflix.spinnaker.kork.jedis.RedisClientDelegate;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -141,13 +145,15 @@ public Map<String, Object> getDeprecatedLastBuild(String master, String job) {
   }
 
   public List<Map<String, String>> getTrackedBuilds(String master) {
-    List<Map<String, String>> builds =
-        redisClientDelegate.withMultiClient(
-            c -> {
-              return c.keys(baseKey() + ":track:" + master + ":*").stream()
-                  .map(BuildCache::getTrackedBuild)
-                  .collect(Collectors.toList());
-            });
+    List<Map<String, String>> builds = new ArrayList<>();
+    redisClientDelegate.withKeyScan(
+        baseKey() + ":track:" + master + ":*",
+        1000,
+        page ->
+            builds.addAll(
+                page.getResults().stream()
+                    .map(BuildCache::getTrackedBuild)
+                    .collect(Collectors.toList())));
     return builds;
   }
 

igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/model/GenericJobConfiguration.java

@@ -0,0 +1,37 @@
+package com.netflix.spinnaker.igor.build.model;
+
+import java.util.List;
+import lombok.Data;
+
+@Data
+public class GenericJobConfiguration implements JobConfiguration {
+  private String description;
+  private String displayName;
+  private String name;
+  private boolean buildable;
+  private String url;
+  private boolean concurrentBuild;
+  private List<GenericParameterDefinition> parameterDefinitionList;
+
+  public GenericJobConfiguration(
+      String description,
+      String displayName,
+      String name,
+      boolean buildable,
+      String url,
+      boolean concurrentBuild,
+      List<GenericParameterDefinition> genericParameterDefinition) {
+    this.description = description;
+    this.displayName = displayName;
+    this.name = name;
+    this.buildable = buildable;
+    this.url = url;
+    this.concurrentBuild = concurrentBuild;
+    this.parameterDefinitionList = genericParameterDefinition;
+  }
+
+  public GenericJobConfiguration(String description, String name) {
+    this.description = description;
+    this.name = name;
+  }
+}

igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/model/GenericParameterDefinition.java

@@ -0,0 +1,20 @@
+package com.netflix.spinnaker.igor.build.model;
+
+import lombok.Data;
+
+@Data
+public class GenericParameterDefinition implements ParameterDefinition {
+  private String name;
+  private String defaultValue;
+  private String description = "";
+
+  public GenericParameterDefinition(String name, String defaultValue) {
+    this.name = name;
+    this.defaultValue = defaultValue;
+  }
+
+  public GenericParameterDefinition(String name, String defaultValue, String description) {
+    this(name, defaultValue);
+    this.description = description;
+  }
+}

igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/model/GenericParameterDefinition.groovy → igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/model/JobConfiguration.java

@@ -1,8 +1,9 @@
 /*
- * Copyright 2016 Schibsted ASA.
+ * Copyright 2019 Schibsted ASA.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
+ *
  * You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
@@ -14,16 +15,22 @@
  * limitations under the License.
  */
 
-package com.netflix.spinnaker.igor.build.model
+package com.netflix.spinnaker.igor.build.model;
+
+import java.util.List;
+
+public interface JobConfiguration {
+  String getDescription();
+
+  String getDisplayName();
+
+  String getName();
+
+  boolean isBuildable();
+
+  String getUrl();
 
-class GenericParameterDefinition {
-    String name
-    String defaultValue
-    String description
+  boolean isConcurrentBuild();
 
-    GenericParameterDefinition(String name, String defaultValue, String description = "") {
-        this.name = name
-        this.defaultValue = defaultValue
-        this.description = description
-    }
+  List<? extends ParameterDefinition> getParameterDefinitionList();
 }

igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/model/ParameterDefinition.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 Schibsted ASA.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ *
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.igor.build.model;
+
+public interface ParameterDefinition {
+  String getName();
+
+  String getDefaultValue();
+
+  String getDescription();
+}

igor-web/src/main/groovy/com/netflix/spinnaker/igor/gitlabci/service/GitlabCiService.java

@@ -18,6 +18,7 @@
 import com.netflix.spinnaker.fiat.model.resources.Permissions;
 import com.netflix.spinnaker.igor.build.model.GenericBuild;
 import com.netflix.spinnaker.igor.build.model.GenericGitRevision;
+import com.netflix.spinnaker.igor.build.model.JobConfiguration;
 import com.netflix.spinnaker.igor.gitlabci.client.GitlabCiClient;
 import com.netflix.spinnaker.igor.gitlabci.client.model.Pipeline;
 import com.netflix.spinnaker.igor.gitlabci.client.model.PipelineSummary;
@@ -77,6 +78,11 @@ public List<GenericBuild> getBuilds(String job) {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public JobConfiguration getJobConfig(String jobName) {
+    throw new UnsupportedOperationException("getJobConfig is not yet implemented for Gitlab CI");
+  }
+
   @Override
   public int triggerBuildWithParameters(String job, Map<String, String> queryParameters) {
     throw new UnsupportedOperationException();

igor-web/src/main/groovy/com/netflix/spinnaker/igor/jenkins/client/model/JobConfig.groovy

@@ -17,9 +17,7 @@
 package com.netflix.spinnaker.igor.jenkins.client.model
 
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper
-import com.netflix.spinnaker.igor.jenkins.client.model.DownstreamProject
-import com.netflix.spinnaker.igor.jenkins.client.model.ParameterDefinition
-import com.netflix.spinnaker.igor.jenkins.client.model.UpstreamProject
+import com.netflix.spinnaker.igor.build.model.JobConfiguration
 
 import javax.xml.bind.annotation.XmlElement
 import javax.xml.bind.annotation.XmlElementWrapper
@@ -29,7 +27,7 @@ import javax.xml.bind.annotation.XmlRootElement
  * Represents the basic Jenkins job configuration information
  */
 @XmlRootElement
-class JobConfig {
+class JobConfig implements JobConfiguration {
     @XmlElement(required = false)
     String description
 

igor-web/src/main/groovy/com/netflix/spinnaker/igor/jenkins/client/model/ParameterDefinition.java

@@ -17,7 +17,8 @@
       "defaultValue",
       "defaultName"
     })
-public class ParameterDefinition {
+public class ParameterDefinition
+    implements com.netflix.spinnaker.igor.build.model.ParameterDefinition {
   @XmlElement public DefaultParameterValue defaultParameterValue;
 
   @XmlElement public String name;
@@ -32,6 +33,11 @@ public class ParameterDefinition {
   @JsonInclude(JsonInclude.Include.NON_NULL)
   public List<String> choices;
 
+  @Override
+  public String getName() {
+    return name;
+  }
+
   @XmlElement(name = "defaultValue")
   public String getDefaultValue() {
     if (defaultParameterValue == null) {
@@ -40,6 +46,11 @@ public String getDefaultValue() {
     return defaultParameterValue.getValue();
   }
 
+  @Override
+  public String getDescription() {
+    return description;
+  }
+
   @XmlElement(name = "defaultName")
   public String getDefaultName() {
     if (defaultParameterValue == null) {

igor-web/src/main/groovy/com/netflix/spinnaker/igor/jenkins/service/JenkinsService.java

@@ -249,6 +249,7 @@ public Response buildWithParameters(String jobName, Map<String, String> queryPar
     return jenkinsClient.buildWithParameters(encode(jobName), queryParams, "", getCrumb());
   }
 
+  @Override
   public JobConfig getJobConfig(String jobName) {
     return jenkinsClient.getJobConfig(encode(jobName));
   }

igor-web/src/main/groovy/com/netflix/spinnaker/igor/service/BuildOperations.java

@@ -19,6 +19,7 @@
 
 import com.netflix.spinnaker.igor.build.model.GenericBuild;
 import com.netflix.spinnaker.igor.build.model.GenericGitRevision;
+import com.netflix.spinnaker.igor.build.model.JobConfiguration;
 import java.util.List;
 import java.util.Map;
 
@@ -61,4 +62,6 @@ public interface BuildOperations extends BuildService {
    * @return A list of builds
    */
   List<?> getBuilds(String job);
+
+  JobConfiguration getJobConfig(String jobName);
 }

igor-web/src/main/groovy/com/netflix/spinnaker/igor/wercker/WerckerService.groovy

@@ -13,10 +13,11 @@ import com.netflix.spinnaker.hystrix.SimpleHystrixCommand
 import com.netflix.spinnaker.igor.build.BuildController
 import com.netflix.spinnaker.igor.build.model.GenericBuild
 import com.netflix.spinnaker.igor.build.model.GenericGitRevision
+import com.netflix.spinnaker.igor.build.model.GenericJobConfiguration
+import com.netflix.spinnaker.igor.build.model.JobConfiguration
 import com.netflix.spinnaker.igor.build.model.Result
 import com.netflix.spinnaker.igor.config.WerckerProperties.WerckerHost
 import com.netflix.spinnaker.igor.exceptions.BuildJobError
-import com.netflix.spinnaker.igor.jenkins.client.model.JobConfig
 import com.netflix.spinnaker.igor.model.BuildServiceProvider
 import com.netflix.spinnaker.igor.service.BuildOperations
 import com.netflix.spinnaker.igor.wercker.model.Application
@@ -349,9 +350,8 @@ class WerckerService implements BuildOperations {
     }
 
     //TODO investigate if Wercker needs the JobConfig implementation
-    JobConfig getJobConfig(String jobName) {
-        return new JobConfig(
-                description: 'WerckerPipeline ' + jobName,
-                name: jobName)
+    @Override
+    JobConfiguration getJobConfig(String jobName) {
+        return new GenericJobConfiguration('WerckerPipeline ' + jobName, jobName)
     }
 }

igor-web/src/main/java/com/netflix/spinnaker/igor/concourse/service/ConcourseService.java

@@ -26,6 +26,7 @@
 import com.netflix.spinnaker.fiat.model.resources.Permissions;
 import com.netflix.spinnaker.igor.build.model.GenericBuild;
 import com.netflix.spinnaker.igor.build.model.GenericGitRevision;
+import com.netflix.spinnaker.igor.build.model.JobConfiguration;
 import com.netflix.spinnaker.igor.concourse.client.ConcourseClient;
 import com.netflix.spinnaker.igor.concourse.client.model.Build;
 import com.netflix.spinnaker.igor.concourse.client.model.Event;
@@ -279,6 +280,11 @@ public List<GenericBuild> getBuilds(String jobPath) {
         .collect(Collectors.toList());
   }
 
+  @Override
+  public JobConfiguration getJobConfig(String jobName) {
+    throw new UnsupportedOperationException("getJobConfig is not yet implemented for Concourse");
+  }
+
   public List<Build> getBuilds(String jobPath, @Nullable Long since) {
     Job job = toJob(jobPath);