From 890bb441d63cdd73be499cebe7e20c06f0070c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Jervidalo?= Date: Thu, 27 Jun 2019 21:11:11 +0200 Subject: [PATCH] feat(travis): Support for log_complete from Travis API (#449) * feat(travis): Support for log_complete from Travis API Travis added a new property in version 2.2.9 (also available on travis-ci.org) that tells us if the log is complete or not. This allows us to be more efficient in the way we fetch artifacts and properties from Travis. It is not backwards compatible, so we have to keep the old way of doing it around for a while. Each Travis master will be queried at Igor startup for some simple feature detection to determine wether to use the legacy method or the new method. A few interfaces has been changed to use GenericBuild instead of `buildNumber`. This allows for some reduction in complexity for a few of the providers that don't use `buildNumber` internally, as we don't have to query the master for `buildId` multiple times. If log_complete is false, we fall back to use the legacy log fetching. The reason is that the travis log endpoint actually returns the complete logs for a while before the flag turns to `true`. If we only wait for the flag, the delay between a finished build and the Spinnaker pipeline triggering can become very long. On the flip side, we can use the flag to avoid downloading the logs to check for builds where we know the log is completed. This gives us the best of both worlds (ideally Travis should fix this in the API, but I'm not holding my breath). Also contains some spring cleaning of the TravisClient and the TravisService, it contained a lot of unused methods. * Don't purge builds from the tracking cache before the log is ready --- .../igor/build/BuildController.groovy | 7 +- .../igor/build/model/GenericGitRevision.java | 2 + .../gitlabci/service/GitlabCiService.java | 2 +- .../igor/jenkins/service/JenkinsService.java | 13 +- .../igor/service/BuildOperations.java | 4 +- .../igor/service/BuildProperties.java | 5 +- .../igor/wercker/WerckerService.groovy | 2 +- .../concourse/service/ConcourseService.java | 6 +- .../spinnaker/igor/config/TravisConfig.java | 29 +- .../igor/travis/TravisBuildMonitor.java | 33 +- .../igor/travis/client/TravisClient.java | 68 +-- .../igor/travis/client/model/v3/Root.java | 53 ++ .../igor/travis/client/model/v3/V3Build.java | 13 + .../igor/travis/client/model/v3/V3Commit.java | 68 ++- .../igor/travis/client/model/v3/V3Log.java | 2 +- .../travis/service/TravisBuildConverter.java | 2 + .../igor/travis/service/TravisService.java | 216 +++----- .../igor/build/BuildControllerSpec.groovy | 13 +- .../igor/build/InfoControllerSpec.groovy | 2 +- .../igor/gcb/GoogleCloudBuildTest.java | 11 +- .../jenkins/service/JenkinsServiceSpec.groovy | 29 +- .../travis/client/TravisClientSpec.groovy | 479 ++++++++++-------- .../travis/service/TravisServiceSpec.groovy | 119 +++-- .../com/netflix/spinnaker/igor/MainTest.java | 18 +- 24 files changed, 638 insertions(+), 558 deletions(-) create mode 100644 igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/Root.java diff --git a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/BuildController.groovy b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/BuildController.groovy index 999de8cce..c5fe5d7c3 100644 --- a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/BuildController.groovy +++ b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/BuildController.groovy @@ -74,7 +74,7 @@ class BuildController { return null try { - build.genericGitRevisions = buildService.getGenericGitRevisions(job, buildNumber) + build.genericGitRevisions = buildService.getGenericGitRevisions(job, build) } catch (Exception e) { log.error("could not get scm results for {} / {} / {}", kv("master", master), kv("job", job), kv("buildNumber", buildNumber), e) } @@ -108,7 +108,7 @@ class BuildController { def buildService = getBuildService(master) GenericBuild build = jobStatus(buildService, master, job, buildNumber) if (build && buildService instanceof BuildProperties && artifactExtractor != null) { - build.properties = buildService.getBuildProperties(job, buildNumber, propertyFile) + build.properties = buildService.getBuildProperties(job, build, propertyFile) return artifactExtractor.extractArtifacts(build) } return Collections.emptyList() @@ -240,7 +240,8 @@ class BuildController { def buildService = getBuildService(master) if (buildService instanceof BuildProperties) { BuildProperties buildProperties = (BuildProperties) buildService - return buildProperties.getBuildProperties(job, buildNumber, fileName) + def genericBuild = buildService.getGenericBuild(job, buildNumber) + return buildProperties.getBuildProperties(job, genericBuild, fileName) } return Collections.emptyMap() } diff --git a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/model/GenericGitRevision.java b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/model/GenericGitRevision.java index 245e896c1..e8856c1ff 100644 --- a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/model/GenericGitRevision.java +++ b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/build/model/GenericGitRevision.java @@ -5,10 +5,12 @@ import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.experimental.Wither; @Getter @EqualsAndHashCode(of = "sha1") @Builder +@Wither @JsonInclude(JsonInclude.Include.NON_NULL) public class GenericGitRevision { private String name; diff --git a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/gitlabci/service/GitlabCiService.java b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/gitlabci/service/GitlabCiService.java index 8dcced177..b13340e2f 100644 --- a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/gitlabci/service/GitlabCiService.java +++ b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/gitlabci/service/GitlabCiService.java @@ -63,7 +63,7 @@ public BuildServiceProvider getBuildServiceProvider() { } @Override - public List getGenericGitRevisions(String job, int buildNumber) { + public List getGenericGitRevisions(String job, GenericBuild build) { throw new UnsupportedOperationException(); } diff --git a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/jenkins/service/JenkinsService.java b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/jenkins/service/JenkinsService.java index 1d9bbbe91..0b604f4e5 100644 --- a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/jenkins/service/JenkinsService.java +++ b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/jenkins/service/JenkinsService.java @@ -254,19 +254,18 @@ public JobConfig getJobConfig(String jobName) { } @Override - @SuppressWarnings("unchecked") - public Map getBuildProperties(String job, int buildNumber, String fileName) { + public Map getBuildProperties(String job, GenericBuild build, String fileName) { if (StringUtils.isEmpty(fileName)) { return new HashMap<>(); } Map map = new HashMap<>(); try { - String path = getArtifactPathFromBuild(job, buildNumber, fileName); + String path = getArtifactPathFromBuild(job, build.getNumber(), fileName); try (InputStream propertyStream = - this.getPropertyFile(job, buildNumber, path).getBody().in()) { + this.getPropertyFile(job, build.getNumber(), path).getBody().in()) { if (fileName.endsWith(".yml") || fileName.endsWith(".yaml")) { Yaml yml = new Yaml(new SafeConstructor()); - map = (Map) yml.load(propertyStream); + map = yml.load(propertyStream); } else if (fileName.endsWith(".json")) { map = objectMapper.readValue(propertyStream, new TypeReference>() {}); } else { @@ -331,8 +330,8 @@ public BuildServiceProvider getBuildServiceProvider() { } @Override - public List getGenericGitRevisions(String job, int buildNumber) { - ScmDetails scmDetails = getGitDetails(job, buildNumber); + public List getGenericGitRevisions(String job, GenericBuild build) { + ScmDetails scmDetails = getGitDetails(job, build.getNumber()); return scmDetails.genericGitRevisions(); } } diff --git a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/service/BuildOperations.java b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/service/BuildOperations.java index 88e13eddc..ee5cf0b4d 100644 --- a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/service/BuildOperations.java +++ b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/service/BuildOperations.java @@ -31,10 +31,10 @@ public interface BuildOperations extends BuildService { * Get a list of the Spinnaker representation of the Git commits relevant for the given build * * @param job The name of the job - * @param buildNumber The build number + * @param build The build * @return A list of git revisions relevant for the build */ - List getGenericGitRevisions(String job, int buildNumber); + List getGenericGitRevisions(String job, GenericBuild build); /** * Return all information of a given build diff --git a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/service/BuildProperties.java b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/service/BuildProperties.java index 41a279fbb..e7dc9e400 100644 --- a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/service/BuildProperties.java +++ b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/service/BuildProperties.java @@ -16,6 +16,7 @@ package com.netflix.spinnaker.igor.service; +import com.netflix.spinnaker.igor.build.model.GenericBuild; import java.util.Map; import javax.annotation.Nullable; @@ -29,10 +30,10 @@ public interface BuildProperties { * Get build properties defined in the given build * * @param job The name of the job - * @param buildNumber The build number + * @param build The build * @param fileName The file name containing the properties. Not all providers require this * parameter. * @return A map of properties */ - Map getBuildProperties(String job, int buildNumber, @Nullable String fileName); + Map getBuildProperties(String job, GenericBuild build, @Nullable String fileName); } diff --git a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/wercker/WerckerService.groovy b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/wercker/WerckerService.groovy index ddf0b23ba..f8edd57e1 100644 --- a/igor-web/src/main/groovy/com/netflix/spinnaker/igor/wercker/WerckerService.groovy +++ b/igor-web/src/main/groovy/com/netflix/spinnaker/igor/wercker/WerckerService.groovy @@ -80,7 +80,7 @@ class WerckerService implements BuildOperations { } @Override - List getGenericGitRevisions(final String job, final int buildNumber) { + List getGenericGitRevisions(final String job, final GenericBuild build) { return null } diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/concourse/service/ConcourseService.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/concourse/service/ConcourseService.java index c7728385e..fc859c0ce 100644 --- a/igor-web/src/main/java/com/netflix/spinnaker/igor/concourse/service/ConcourseService.java +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/concourse/service/ConcourseService.java @@ -113,14 +113,12 @@ public Collection getJobs() { } @Override - public List getGenericGitRevisions(String jobPath, int buildNumber) { - GenericBuild build = getGenericBuild(jobPath, buildNumber); + public List getGenericGitRevisions(String jobPath, GenericBuild build) { return build == null ? emptyList() : build.getGenericGitRevisions(); } @Override - public Map getBuildProperties(String jobPath, int buildNumber, String fileName) { - GenericBuild build = getGenericBuild(jobPath, buildNumber); + public Map getBuildProperties(String jobPath, GenericBuild build, String fileName) { return build == null ? emptyMap() : build.getProperties(); } diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/config/TravisConfig.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/config/TravisConfig.java index 9889d09a3..f2bbaa04e 100644 --- a/igor-web/src/main/java/com/netflix/spinnaker/igor/config/TravisConfig.java +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/config/TravisConfig.java @@ -24,6 +24,7 @@ import com.netflix.spinnaker.igor.service.BuildServices; import com.netflix.spinnaker.igor.travis.TravisCache; import com.netflix.spinnaker.igor.travis.client.TravisClient; +import com.netflix.spinnaker.igor.travis.client.model.v3.Root; import com.netflix.spinnaker.igor.travis.service.TravisService; import com.squareup.okhttp.OkHttpClient; import java.util.ArrayList; @@ -34,8 +35,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import javax.validation.Valid; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -53,8 +53,8 @@ @Configuration @ConditionalOnProperty("travis.enabled") @EnableConfigurationProperties(com.netflix.spinnaker.igor.config.TravisProperties.class) +@Slf4j public class TravisConfig { - private Logger log = LoggerFactory.getLogger(getClass()); @Bean public Map travisMasters( @@ -81,6 +81,18 @@ public Map travisMasters( host.getAddress(), igorConfigurationProperties.getClient().getTimeout(), objectMapper); + + boolean useLegacyLogFetching = false; + try { + Root root = client.getRoot(); + useLegacyLogFetching = !root.hasLogCompleteAttribute(); + if (useLegacyLogFetching) { + log.info( + "It seems Travis Enterprise is older than version 2.2.9. Will use legacy log fetching."); + } + } catch (Exception e) { + log.warn("Could not query Travis API to check API compability", e); + } return travisService( travisName, host.getBaseUrl(), @@ -89,9 +101,10 @@ public Map travisMasters( client, travisCache, artifactDecorator, - (travisProperties == null ? null : travisProperties.getRegexes()), + travisProperties.getRegexes(), travisProperties.getBuildMessageKey(), - host.getPermissions().build()); + host.getPermissions().build(), + useLegacyLogFetching); }) .collect(Collectors.toMap(TravisService::getGroupKey, Function.identity())); @@ -109,7 +122,8 @@ private static TravisService travisService( Optional artifactDecorator, Collection artifactRexeges, String buildMessageKey, - Permissions permissions) { + Permissions permissions, + boolean legacyLogFetching) { return new TravisService( travisHostId, baseUrl, @@ -120,7 +134,8 @@ private static TravisService travisService( artifactDecorator, artifactRexeges, buildMessageKey, - permissions); + permissions, + legacyLogFetching); } public static TravisClient travisClient(String address, int timeout, ObjectMapper objectMapper) { diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/TravisBuildMonitor.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/TravisBuildMonitor.java index 2a99be150..e49f5cc4c 100644 --- a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/TravisBuildMonitor.java +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/TravisBuildMonitor.java @@ -30,19 +30,21 @@ import com.netflix.spinnaker.igor.history.model.GenericBuildContent; import com.netflix.spinnaker.igor.history.model.GenericBuildEvent; import com.netflix.spinnaker.igor.model.BuildServiceProvider; -import com.netflix.spinnaker.igor.polling.*; +import com.netflix.spinnaker.igor.polling.CommonPollingMonitor; +import com.netflix.spinnaker.igor.polling.DeltaItem; +import com.netflix.spinnaker.igor.polling.LockService; +import com.netflix.spinnaker.igor.polling.PollContext; +import com.netflix.spinnaker.igor.polling.PollingDelta; import com.netflix.spinnaker.igor.service.BuildServices; import com.netflix.spinnaker.igor.travis.client.model.Repo; import com.netflix.spinnaker.igor.travis.client.model.v3.TravisBuildState; import com.netflix.spinnaker.igor.travis.client.model.v3.V3Build; -import com.netflix.spinnaker.igor.travis.client.model.v3.V3Job; import com.netflix.spinnaker.igor.travis.service.TravisBuildConverter; import com.netflix.spinnaker.igor.travis.service.TravisService; import com.netflix.spinnaker.security.AuthenticatedRequest; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -212,7 +214,7 @@ private List processBuilds( List results = new ArrayList<>(); builds.stream() .map(build -> setTracking(build, master)) - .filter(filterNewBuildsPredicate()) + .filter(filterNewBuildsPredicate(master)) .forEach( build -> { String branchedRepoSlug = build.branchedRepoSlug(); @@ -220,13 +222,7 @@ private List processBuilds( buildCache.getLastBuild(master, branchedRepoSlug, build.getState().isRunning()); GenericBuild genericBuild = TravisBuildConverter.genericBuild(build, travisService.getBaseUrl()); - List jobIds = - build.getJobs() != null - ? build.getJobs().stream().map(V3Job::getId).collect(Collectors.toList()) - : Collections.emptyList(); - if (build.getNumber() > cachedBuild - && !build.spinnakerTriggered() - && travisService.isLogReady(jobIds)) { + if (build.getNumber() > cachedBuild && !build.spinnakerTriggered()) { BuildDelta delta = new BuildDelta() .setBranchedRepoSlug(branchedRepoSlug) @@ -248,7 +244,7 @@ private List processBuilds( } private V3Build setTracking(V3Build build, String master) { - if (!filterNewBuildsPredicate().test(build)) { + if (!filterNewBuildsPredicate(master).test(build)) { buildCache.setTracking( master, build.getRepository().getSlug(), build.getId(), getPollInterval() * 5); log.debug("({}) tracking set up for {}", kv("master", master), build.toString()); @@ -264,10 +260,7 @@ private void sendEventForBuild( if (!buildDelta.getBuild().spinnakerTriggered()) { if (echoService.isPresent()) { log.info( - "({}) pushing event for :" - + branchedSlug - + ":" - + String.valueOf(buildDelta.getBuild().getNumber()), + "({}) pushing event for :" + branchedSlug + ":" + buildDelta.getBuild().getNumber(), kv("master", master)); GenericProject project = new GenericProject(branchedSlug, buildDelta.getGenericBuild()); @@ -334,7 +327,8 @@ private int buildCacheJobTTLSeconds() { return (int) TimeUnit.DAYS.toSeconds(travisProperties.getCachedJobTTLDays()); } - private Predicate filterNewBuildsPredicate() { + private Predicate filterNewBuildsPredicate(String master) { + final TravisService travisService = (TravisService) buildServices.getService(master); /* NewBuildGracePeriodSeconds is here because the travis API needs some time in order to fully represent the build in the api. This can be overridden by travis.newBuildGracePeriod. @@ -342,8 +336,9 @@ private Predicate filterNewBuildsPredicate() { Instant threshold = Instant.now().minus(travisProperties.getNewBuildGracePeriodSeconds(), ChronoUnit.SECONDS); return build -> - !build.getState().isRunning() - || (build.getFinishedAt() != null && build.getFinishedAt().isBefore(threshold)); + (!build.getState().isRunning() + || (build.getFinishedAt() != null && build.getFinishedAt().isBefore(threshold))) + && travisService.isLogReady(build); } private List filterOutOldBuilds(List repos) { diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/TravisClient.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/TravisClient.java index c0889aea2..8ce0f3cf8 100644 --- a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/TravisClient.java +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/TravisClient.java @@ -19,23 +19,19 @@ import com.netflix.spinnaker.igor.travis.client.model.AccessToken; import com.netflix.spinnaker.igor.travis.client.model.Accounts; -import com.netflix.spinnaker.igor.travis.client.model.Build; import com.netflix.spinnaker.igor.travis.client.model.Builds; import com.netflix.spinnaker.igor.travis.client.model.EmptyObject; import com.netflix.spinnaker.igor.travis.client.model.GithubAuth; -import com.netflix.spinnaker.igor.travis.client.model.Jobs; -import com.netflix.spinnaker.igor.travis.client.model.Repo; import com.netflix.spinnaker.igor.travis.client.model.RepoRequest; -import com.netflix.spinnaker.igor.travis.client.model.RepoWrapper; import com.netflix.spinnaker.igor.travis.client.model.Repos; import com.netflix.spinnaker.igor.travis.client.model.TriggerResponse; import com.netflix.spinnaker.igor.travis.client.model.v3.Request; +import com.netflix.spinnaker.igor.travis.client.model.v3.Root; import com.netflix.spinnaker.igor.travis.client.model.v3.V3Build; import com.netflix.spinnaker.igor.travis.client.model.v3.V3Builds; import com.netflix.spinnaker.igor.travis.client.model.v3.V3Log; import retrofit.client.Response; import retrofit.http.Body; -import retrofit.http.EncodedPath; import retrofit.http.GET; import retrofit.http.Header; import retrofit.http.Headers; @@ -44,33 +40,23 @@ import retrofit.http.Query; public interface TravisClient { + /** + * Root endpoint (developer.travis-ci.com/resource/home), + * describes the API and its capabilities + * + * @return The root object, describing the API + */ + @GET("/v3/") + @Headers("Travis-API-Version: 3") + public Root getRoot(); + @POST("/auth/github") public abstract AccessToken accessToken(@Body GithubAuth gitHubAuth); @GET("/accounts") public abstract Accounts accounts(@Header("Authorization") String accessToken); - @GET("/builds") - public abstract Builds builds(@Header("Authorization") String accessToken); - - @GET("/builds") - public abstract Builds builds( - @Header("Authorization") String accessToken, @Query("repository_id") int repositoryId); - - @GET("/builds") - public abstract Builds builds( - @Header("Authorization") String accessToken, @Query("slug") String repoSlug); - - @GET("/builds/{build_id}") - public abstract Build build( - @Header("Authorization") String accessToken, @Path("build_id") int buildId); - - @GET("/builds") - public abstract Build build( - @Header("Authorization") String accessToken, - @Query("repository_id") int repositoryId, - @Query("number") int buildNumber); - @GET("/builds") public abstract Builds builds( @Header("Authorization") String accessToken, @@ -85,14 +71,6 @@ public abstract Repos repos( @Query("limit") int limit, @Query("offset") int offset); - @GET("/repos/{repositoryId}") - public abstract Repo repo( - @Header("Authorization") String accessToken, @Path("repositoryId") int repositoryId); - - @GET("/repos/{repo_slug}") - public abstract RepoWrapper repoWrapper( - @Header("Authorization") String accessToken, @EncodedPath("repo_slug") String repoSlug); - @POST("/repo/{repoSlug}/requests") @Headers("Travis-API-Version: 3") public abstract TriggerResponse triggerBuild( @@ -104,9 +82,6 @@ public abstract TriggerResponse triggerBuild( public abstract Response usersSync( @Header("Authorization") String accessToken, @Body EmptyObject emptyObject); - @GET("/jobs/{job_id}") - public abstract Jobs jobs(@Header("Authorization") String accessToken, @Path("job_id") int jobId); - @Headers({"Travis-API-Version: 3", "Accept: text/plain"}) @GET("/job/{jobId}/log") public abstract V3Log jobLog( @@ -115,21 +90,25 @@ public abstract V3Log jobLog( @GET("/build/{build_id}") @Headers("Travis-API-Version: 3") public abstract V3Build v3build( - @Header("Authorization") String accessToken, @Path("build_id") int repositoryId); + @Header("Authorization") String accessToken, + @Path("build_id") int buildId, + @Query("include") String include); @GET("/repo/{repository_id}/builds") @Headers("Travis-API-Version: 3") public abstract V3Builds builds( @Header("Authorization") String accessToken, @Path("repository_id") int repositoryId, - @Query("limit") int limit); + @Query("limit") int limit, + @Query("include") String include); @GET("/repo/{repository_slug}/builds") @Headers("Travis-API-Version: 3") public abstract V3Builds v3builds( @Header("Authorization") String accessToken, @Path("repository_slug") String repositorySlug, - @Query("limit") int limit); + @Query("limit") int limit, + @Query("include") String include); @GET("/repo/{repository_slug}/builds") @Headers("Travis-API-Version: 3") @@ -137,7 +116,8 @@ public abstract V3Builds v3builds( @Header("Authorization") String accessToken, @Path("repository_slug") String repositorySlug, @Query("branch.name") String branchName, - @Query("limit") int limit); + @Query("limit") int limit, + @Query("include") String include); @GET("/repo/{repository_slug}/builds") @Headers("Travis-API-Version: 3") @@ -146,7 +126,8 @@ public abstract V3Builds v3builds( @Path("repository_slug") String repositorySlug, @Query("branch.name") String branchName, @Query("event_type") String eventType, - @Query("limit") int limit); + @Query("limit") int limit, + @Query("include") String include); @GET("/repo/{repository_slug}/builds") @Headers("Travis-API-Version: 3") @@ -154,7 +135,8 @@ public abstract V3Builds v3buildsByEventType( @Header("Authorization") String accessToken, @Path("repository_slug") String repositorySlug, @Query("event_type") String EventType, - @Query("limit") int limit); + @Query("limit") int limit, + @Query("include") String include); @GET("/repo/{repository_id}/request/{request_id}") @Headers("Travis-API-Version: 3") diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/Root.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/Root.java new file mode 100644 index 000000000..2f0e2d67b --- /dev/null +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/Root.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 Schibsted ASA. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.igor.travis.client.model.v3; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import java.util.List; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Class representing the root endpoint of the Travis API. Currently only used for feature + * detection, as the API can differ a bit between versions. Only the elements required for the + * feature detection are represented in the class. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@Data +@NoArgsConstructor +public class Root { + private Resources resources; + + public boolean hasLogCompleteAttribute() { + return resources.build.attributes.contains("log_complete"); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + @Data + @NoArgsConstructor + private static class Resources { + private Build build; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + @Data + @NoArgsConstructor + private static class Build { + private List attributes; + } +} diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Build.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Build.java index 515d788a0..5ebd606e3 100644 --- a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Build.java +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Build.java @@ -24,6 +24,7 @@ import com.netflix.spinnaker.igor.travis.client.model.Config; import java.time.Instant; import java.util.List; +import lombok.extern.slf4j.Slf4j; import org.simpleframework.xml.Default; import org.simpleframework.xml.Root; @@ -31,6 +32,7 @@ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) @Root(name = "builds") +@Slf4j public class V3Build { private V3Branch branch; @@ -55,6 +57,9 @@ public class V3Build { @JsonProperty("finished_at") private Instant finishedAt; + @JsonProperty("log_complete") + private Boolean logComplete; + private List jobs; private Config config; @@ -202,4 +207,12 @@ public Config getConfig() { public void setConfig(Config config) { this.config = config; } + + public Boolean getLogComplete() { + return logComplete; + } + + public void setLogComplete(Boolean logComplete) { + this.logComplete = logComplete; + } } diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Commit.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Commit.java index 5442c4ad8..a292c8e53 100644 --- a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Commit.java +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Commit.java @@ -17,22 +17,32 @@ package com.netflix.spinnaker.igor.travis.client.model.v3; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import com.netflix.spinnaker.igor.build.model.GenericGitRevision; +import java.time.Instant; +import lombok.Data; +import lombok.NoArgsConstructor; import org.simpleframework.xml.Default; import org.simpleframework.xml.Root; @Default @Root(name = "commits") +@Data +@NoArgsConstructor +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) @JsonIgnoreProperties(ignoreUnknown = true) public class V3Commit { private int id; private String sha; private String ref; private String message; - - @JsonProperty("compare_url") private String compareUrl; + private Instant committedAt; + private Person author; + private Person committer; public boolean isTag() { return ref != null && ref.split("/")[1].equals("tags"); @@ -42,43 +52,23 @@ public boolean isPullRequest() { return ref != null && ref.split("/")[1].equals("pull"); } - public int getId() { - return id; - } - - public void setId(int id) { - this.id = id; - } - - public String getSha() { - return sha; - } - - public void setSha(String sha) { - this.sha = sha; - } - - public String getRef() { - return ref; - } - - public void setRef(String ref) { - this.ref = ref; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - public String getCompareUrl() { - return compareUrl; + @Data + @NoArgsConstructor + @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) + @JsonIgnoreProperties(ignoreUnknown = true) + private static class Person { + private String name; + private String avatarUrl; } - public void setCompareUrl(String compareUrl) { - this.compareUrl = compareUrl; + @JsonIgnore + public GenericGitRevision getGenericGitRevision() { + return GenericGitRevision.builder() + .sha1(sha) + .committer(getAuthor().getName()) + .compareUrl(compareUrl) + .message(message) + .timestamp(committedAt) + .build(); } } diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Log.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Log.java index 1bae36fc8..893876512 100644 --- a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Log.java +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/client/model/v3/V3Log.java @@ -66,7 +66,7 @@ public boolean isReady() { } @JsonIgnoreProperties(ignoreUnknown = true) - static class V3LogPart { + public static class V3LogPart { private String content; private Integer number; private boolean isFinal; diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/service/TravisBuildConverter.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/service/TravisBuildConverter.java index 99fd49191..2d3196452 100644 --- a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/service/TravisBuildConverter.java +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/service/TravisBuildConverter.java @@ -31,6 +31,7 @@ public static GenericBuild genericBuild(Build build, String repoSlug, String bas genericBuild.setResult(build.getState().getResult()); genericBuild.setName(repoSlug); genericBuild.setUrl(url(repoSlug, baseUrl, build.getId())); + genericBuild.setId(String.valueOf(build.getId())); if (build.getFinishedAt() != null) { genericBuild.setTimestamp(String.valueOf(build.getTimestamp())); } @@ -45,6 +46,7 @@ public static GenericBuild genericBuild(V3Build build, String baseUrl) { genericBuild.setResult(build.getState().getResult()); genericBuild.setName(build.getRepository().getSlug()); genericBuild.setUrl(url(build.getRepository().getSlug(), baseUrl, build.getId())); + genericBuild.setId(String.valueOf(build.getId())); if (build.getFinishedAt() != null) { genericBuild.setTimestamp(String.valueOf(build.getTimestamp())); } diff --git a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/service/TravisService.java b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/service/TravisService.java index fbfb8cf46..b87bcf136 100644 --- a/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/service/TravisService.java +++ b/igor-web/src/main/java/com/netflix/spinnaker/igor/travis/service/TravisService.java @@ -23,7 +23,6 @@ import com.netflix.spinnaker.hystrix.SimpleJava8HystrixCommand; import com.netflix.spinnaker.igor.build.model.GenericBuild; import com.netflix.spinnaker.igor.build.model.GenericGitRevision; -import com.netflix.spinnaker.igor.build.model.GenericJobConfiguration; import com.netflix.spinnaker.igor.build.model.Result; import com.netflix.spinnaker.igor.model.BuildServiceProvider; import com.netflix.spinnaker.igor.service.ArtifactDecorator; @@ -42,8 +41,6 @@ import com.netflix.spinnaker.igor.travis.client.model.Config; import com.netflix.spinnaker.igor.travis.client.model.EmptyObject; import com.netflix.spinnaker.igor.travis.client.model.GithubAuth; -import com.netflix.spinnaker.igor.travis.client.model.Job; -import com.netflix.spinnaker.igor.travis.client.model.Jobs; import com.netflix.spinnaker.igor.travis.client.model.Repo; import com.netflix.spinnaker.igor.travis.client.model.RepoRequest; import com.netflix.spinnaker.igor.travis.client.model.TriggerResponse; @@ -60,19 +57,19 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; import net.logstash.logback.argument.StructuredArguments; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import retrofit.RetrofitError; public class TravisService implements BuildOperations, BuildProperties { - private static final int TRAVIS_BUILD_RESULT_LIMIT = 25; + static final int TRAVIS_BUILD_RESULT_LIMIT = 25; private final Logger log = LoggerFactory.getLogger(getClass()); private final String baseUrl; @@ -85,6 +82,7 @@ public class TravisService implements BuildOperations, BuildProperties { private final Optional artifactDecorator; private final String buildMessageKey; private final Permissions permissions; + private final boolean legacyLogFetching; protected AccessToken accessToken; private Accounts accounts; @@ -98,7 +96,8 @@ public TravisService( Optional artifactDecorator, Collection artifactRegexes, String buildMessageKey, - Permissions permissions) { + Permissions permissions, + boolean legacyLogFetching) { this.numberOfRepositories = numberOfRepositories; this.groupKey = travisHostId; this.gitHubAuth = new GithubAuth(githubToken); @@ -110,6 +109,7 @@ public TravisService( artifactRegexes != null ? new HashSet<>(artifactRegexes) : Collections.emptySet(); this.buildMessageKey = buildMessageKey; this.permissions = permissions; + this.legacyLogFetching = legacyLogFetching; } @Override @@ -123,19 +123,35 @@ public BuildServiceProvider getBuildServiceProvider() { } @Override - public List getGenericGitRevisions(String inputRepoSlug, int buildNumber) { + public List getGenericGitRevisions(String inputRepoSlug, GenericBuild build) { return new SimpleJava8HystrixCommand<>( groupKey, buildCommandKey("getGenericGitRevisions"), () -> { String repoSlug = cleanRepoSlug(inputRepoSlug); - Builds builds = getBuilds(repoSlug, buildNumber); - final List commits = builds.getCommits(); - if (commits != null) { - return commits.stream() - .filter(commit -> commit.getBranch() != null) - .map(Commit::getGenericGitRevision) - .collect(Collectors.toList()); + if (StringUtils.isNumeric(build.getId())) { + V3Build v3build = getV3Build(Integer.parseInt(build.getId())); + if (v3build.getCommit() != null) { + return Collections.singletonList( + v3build + .getCommit() + .getGenericGitRevision() + .withName(v3build.getBranch().getName()) + .withBranch(v3build.getBranch().getName())); + } + } else { + log.info( + "Getting getGenericGitRevisions for build {}:{} using deprecated V2 API", + inputRepoSlug, + build.getNumber()); + Builds builds = getBuilds(repoSlug, build.getNumber()); + final List commits = builds.getCommits(); + if (commits != null) { + return commits.stream() + .filter(commit -> commit.getBranch() != null) + .map(Commit::getGenericGitRevision) + .collect(Collectors.toList()); + } } return null; }) @@ -191,25 +207,15 @@ public Permissions getPermissions() { return permissions; } - public List getBuilds() { - Builds builds = travisClient.builds(getAccessToken()); - log.debug("fetched {} builds", builds.getBuilds().size()); - return builds.getBuilds(); - } - - public Build getBuild(Repo repo, int buildNumber) { - return new SimpleJava8HystrixCommand<>( - groupKey, - buildCommandKey("getBuild"), - () -> travisClient.build(getAccessToken(), repo.getId(), buildNumber)) - .execute(); - } - public V3Build getV3Build(int buildId) { return new SimpleJava8HystrixCommand<>( groupKey, buildCommandKey("getV3Build"), - () -> travisClient.v3build(getAccessToken(), buildId)) + () -> + travisClient.v3build( + getAccessToken(), + buildId, + "build.commit" + (legacyLogFetching ? "" : "," + includeLogFetching()))) .execute(); } @@ -228,25 +234,19 @@ public Build getBuild(String repoSlug, int buildNumber) { @Override public Map getBuildProperties( - String inputRepoSlug, int buildNumber, String fileName) { + String inputRepoSlug, GenericBuild build, String fileName) { try { - String repoSlug = cleanRepoSlug(inputRepoSlug); - Build build = getBuild(repoSlug, buildNumber); - return PropertyParser.extractPropertiesFromLog(getLog(build)); + V3Build v3build = getV3Build(Integer.valueOf(build.getId())); + return PropertyParser.extractPropertiesFromLog(getLog(v3build)); } catch (Exception e) { log.error("Unable to get igorProperties '{}'", kv("job", inputRepoSlug), e); return Collections.emptyMap(); } } - public List getBuilds(Repo repo) { - Builds builds = travisClient.builds(getAccessToken(), repo.getId()); - log.debug("fetched {} builds", builds.getBuilds().size()); - return builds.getBuilds(); - } - public List getBuilds(Repo repo, int limit) { - final V3Builds builds = travisClient.builds(getAccessToken(), repo.getId(), limit); + final V3Builds builds = + travisClient.builds(getAccessToken(), repo.getId(), limit, includeLogFetching()); log.debug("fetched {} builds", builds.getBuilds().size()); return builds.getBuilds(); } @@ -256,9 +256,14 @@ public List getTagBuilds(String repoSlug) { // Increasing the limit to increase the odds for finding some tag builds. V3Builds builds = travisClient.v3buildsByEventType( - getAccessToken(), repoSlug, "push", TRAVIS_BUILD_RESULT_LIMIT * 2); + getAccessToken(), + repoSlug, + "push", + TRAVIS_BUILD_RESULT_LIMIT * 2, + includeLogFetching()); return builds.getBuilds().stream() .filter(build -> build.getCommit().isTag()) + .filter(V3Build::getLogComplete) .map(this::getGenericBuild) .collect(Collectors.toList()); } @@ -276,28 +281,34 @@ public List getBuilds(String inputRepoSlug) { case branch: builds = travisClient.v3builds( - getAccessToken(), repoSlug, branch, "push", TRAVIS_BUILD_RESULT_LIMIT); + getAccessToken(), + repoSlug, + branch, + "push", + TRAVIS_BUILD_RESULT_LIMIT, + includeLogFetching()); break; case pull_request: builds = travisClient.v3builds( - getAccessToken(), repoSlug, branch, "pull_request", TRAVIS_BUILD_RESULT_LIMIT); + getAccessToken(), + repoSlug, + branch, + "pull_request", + TRAVIS_BUILD_RESULT_LIMIT, + includeLogFetching()); break; case unknown: default: - builds = travisClient.v3builds(getAccessToken(), repoSlug, TRAVIS_BUILD_RESULT_LIMIT); - } - - return builds.getBuilds().stream().map(this::getGenericBuild).collect(Collectors.toList()); - } - - public Commit getCommit(String repoSlug, int buildNumber) { - Builds builds = getBuilds(repoSlug, buildNumber); - if (builds != null && builds.getCommits() != null && !builds.getCommits().isEmpty()) { - return builds.getCommits().get(0); + builds = + travisClient.v3builds( + getAccessToken(), repoSlug, TRAVIS_BUILD_RESULT_LIMIT, includeLogFetching()); } - throw new NoSuchElementException("No commit found for " + repoSlug + ":" + buildNumber); + return builds.getBuilds().stream() + .filter(this::isLogReady) + .map(this::getGenericBuild) + .collect(Collectors.toList()); } public List getReposForAccounts() { @@ -328,17 +339,6 @@ public List getReposForAccounts() { .execute(); } - public Job getJob(int jobId) { - log.debug("fetching job for {}", jobId); - Jobs jobs = - new SimpleJava8HystrixCommand<>( - groupKey, - buildCommandKey("getJob"), - () -> travisClient.jobs(getAccessToken(), jobId)) - .execute(); - return jobs.getJob(); - } - public String getLog(Build build) { List jobIds = build.getJob_ids(); if (jobIds == null) { @@ -372,6 +372,19 @@ public String getLog(V3Build build) { return travisLog; } + public boolean isLogReady(V3Build build) { + if (!legacyLogFetching) { + boolean logComplete = build.getLogComplete(); + if (logComplete) { + return true; + // If not complete, we still want to try to fetch the log to check, because the logs are + // always completed for a + // while before the log_complete flag turns true + } + } + return isLogReady(build.getJobs().stream().map(V3Job::getId).collect(Collectors.toList())); + } + public boolean isLogReady(List jobIds) { return jobIds.stream().map(this::getAndCacheJobLog).allMatch(Optional::isPresent); } @@ -407,14 +420,6 @@ public String getJobLog(int jobId) { } } - public Repo getRepo(int repositoryId) { - return travisClient.repo(getAccessToken(), repositoryId); - } - - public Repo getRepo(String repoSlug) { - return travisClient.repoWrapper(getAccessToken(), repoSlug).getRepo(); - } - public GenericBuild getGenericBuild(Build build, String repoSlug) { GenericBuild genericBuild = TravisBuildConverter.genericBuild(build, repoSlug, baseUrl); boolean logReady = isLogReady(build.getJob_ids()); @@ -438,24 +443,6 @@ public GenericBuild getGenericBuild(V3Build build, boolean fetchLogs) { return genericBuild; } - public GenericJobConfiguration getJobConfig(String inputRepoSlug) { - String repoSlug = cleanRepoSlug(inputRepoSlug); - Builds builds = travisClient.builds(getAccessToken(), repoSlug); - final Config config = builds.getBuilds().get(0).getConfig(); - return new GenericJobConfiguration( - extractRepoFromRepoSlug(repoSlug), - extractRepoFromRepoSlug(repoSlug), - repoSlug, - true, - getUrl(repoSlug), - false, - (config == null ? null : config.getParameterDefinitionList())); - } - - public String getUrl(String repoSlug) { - return baseUrl + "/" + repoSlug; - } - public Map queuedBuild(int queueId) { Map queuedJob = travisCache.getQueuedJob(groupKey, queueId); Request requestResponse = @@ -477,45 +464,6 @@ public Map queuedBuild(int queueId) { return null; } - public boolean hasRepo(String inputRepoSlug) { - return new SimpleJava8HystrixCommand<>( - groupKey, - buildCommandKey("hasRepo"), - () -> { - String repoSlug = cleanRepoSlug(inputRepoSlug); - try { - getRepo(repoSlug); - } catch (RetrofitError error) { - if (error.getResponse() != null) { - if (error.getResponse().getStatus() == 404) { - log.debug("repo not found {}", repoSlug); - return false; - } - } - log.info("Error requesting repo {}: {}", repoSlug, error.getMessage()); - return true; - } - return true; - }) - .execute(); - } - - public String branchedRepoSlug(String repoSlug, int buildNumber, Commit commit) { - return new SimpleJava8HystrixCommand<>( - groupKey, - buildCommandKey("branchedRepoSlug"), - () -> { - Build build = getBuild(repoSlug, buildNumber); - String branchedRepoSlug = repoSlug + "/"; - if (build.getPullRequest()) { - branchedRepoSlug = branchedRepoSlug + "pull_request_"; - } - return branchedRepoSlug + commit.getBranchNameWithTagHandling(); - }, - (ignored) -> repoSlug) - .execute(); - } - public void syncRepos() { try { travisClient.usersSync(getAccessToken(), new EmptyObject()); @@ -574,10 +522,6 @@ private static String extractBranchFromRepoSlug(String inputRepoSlug) { return parts.subList(2, parts.size()).stream().collect(Collectors.joining("/")); } - private static String extractRepoFromRepoSlug(String repoSlug) { - return repoSlug.split("/")[1]; - } - private void setAccessToken() { this.accessToken = travisClient.accessToken(gitHubAuth); } @@ -644,4 +588,8 @@ public final TravisClient getTravisClient() { public final TravisCache getTravisCache() { return travisCache; } + + private String includeLogFetching() { + return legacyLogFetching ? null : "build.log_complete"; + } } diff --git a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/build/BuildControllerSpec.groovy b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/build/BuildControllerSpec.groovy index ffab3e15d..2d621ccbb 100644 --- a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/build/BuildControllerSpec.groovy +++ b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/build/BuildControllerSpec.groovy @@ -70,11 +70,14 @@ class BuildControllerSpec extends Specification { final TRAVIS_SERVICE = 'TRAVIS_SERVICE' final HTTP_201 = 201 final BUILD_NUMBER = 123 + final BUILD_ID = 654321 final QUEUED_JOB_NUMBER = 123456 final JOB_NAME = "job/name/can/have/slashes" final JOB_NAME_LEGACY = "job" final FILE_NAME = "test.yml" + GenericBuild genericBuild + void cleanup() { server.shutdown() } @@ -91,6 +94,10 @@ class BuildControllerSpec extends Specification { (JENKINS_SERVICE): jenkinsService, (TRAVIS_SERVICE): travisService, ]) + genericBuild = new GenericBuild() + genericBuild.number = BUILD_NUMBER + genericBuild.id = BUILD_ID + cache = Mock(BuildCache) server = new MockWebServer() @@ -193,7 +200,8 @@ class BuildControllerSpec extends Specification { void 'get properties of a build with a bad filename'() { given: - jenkinsService.getBuildProperties(JOB_NAME, BUILD_NUMBER, FILE_NAME) >> { + 1 * jenkinsService.getGenericBuild(JOB_NAME, BUILD_NUMBER) >> genericBuild + 1 * jenkinsService.getBuildProperties(JOB_NAME, genericBuild, FILE_NAME) >> { throw new NotFoundException() } @@ -207,7 +215,8 @@ class BuildControllerSpec extends Specification { void 'get properties of a travis build'() { given: - 1 * travisService.getBuildProperties(JOB_NAME, BUILD_NUMBER, _) >> ['foo': 'bar'] + 1 * travisService.getGenericBuild(JOB_NAME, BUILD_NUMBER) >> genericBuild + 1 * travisService.getBuildProperties(JOB_NAME, genericBuild, _) >> ['foo': 'bar'] when: MockHttpServletResponse response = mockMvc.perform( diff --git a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/build/InfoControllerSpec.groovy b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/build/InfoControllerSpec.groovy index 60332a159..b893848f0 100644 --- a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/build/InfoControllerSpec.groovy +++ b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/build/InfoControllerSpec.groovy @@ -108,7 +108,7 @@ class InfoControllerSpec extends Specification { TravisService travisService = new TravisService('travis-baz', null, null, 100, null, null, Optional.empty(), [], null, new Permissions.Builder() .add(Authorization.READ, ['group-3', 'group-4']) - .add(Authorization.WRITE, 'group-3').build()) + .add(Authorization.WRITE, 'group-3').build(), false) createMocks([ 'jenkins-foo': jenkinsService1, 'jenkins-bar': jenkinsService2, diff --git a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/gcb/GoogleCloudBuildTest.java b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/gcb/GoogleCloudBuildTest.java index 499b0506b..6f3409a2f 100644 --- a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/gcb/GoogleCloudBuildTest.java +++ b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/gcb/GoogleCloudBuildTest.java @@ -16,7 +16,10 @@ package com.netflix.spinnaker.igor.gcb; -import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; @@ -35,7 +38,11 @@ import com.netflix.spinnaker.igor.config.GoogleCloudBuildConfig; import com.netflix.spinnaker.igor.config.LockManagerConfig; import com.netflix.spinnaker.kork.web.exceptions.GenericExceptionHandlers; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; diff --git a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/jenkins/service/JenkinsServiceSpec.groovy b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/jenkins/service/JenkinsServiceSpec.groovy index adf702940..2212034a0 100644 --- a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/jenkins/service/JenkinsServiceSpec.groovy +++ b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/jenkins/service/JenkinsServiceSpec.groovy @@ -17,6 +17,7 @@ package com.netflix.spinnaker.igor.jenkins.service import com.netflix.spinnaker.fiat.model.resources.Permissions +import com.netflix.spinnaker.igor.build.model.GenericBuild import com.netflix.spinnaker.igor.build.model.GenericGitRevision import com.netflix.spinnaker.igor.config.JenkinsConfig import com.netflix.spinnaker.igor.config.JenkinsProperties @@ -279,9 +280,11 @@ class JenkinsServiceSpec extends Specification { password: 'password') client = new JenkinsConfig().jenkinsClient(host) service = new JenkinsService('http://my.jenkins.net', client, false, Permissions.EMPTY) + def genericBuild = new GenericBuild() + genericBuild.number = 1 when: - List genericGitRevision = service.getGenericGitRevisions('test', 1) + List genericGitRevision = service.getGenericGitRevisions('test', genericBuild) then: genericGitRevision.size() == 1 @@ -332,9 +335,11 @@ class JenkinsServiceSpec extends Specification { password: 'password') client = new JenkinsConfig().jenkinsClient(host) service = new JenkinsService('http://my.jenkins.net', client, false, Permissions.EMPTY) + def genericBuild = new GenericBuild() + genericBuild.number = 1 when: - List genericGitRevision = service.getGenericGitRevisions('test', 1) + List genericGitRevision = service.getGenericGitRevisions('test', genericBuild) then: genericGitRevision.size() == 2 @@ -385,9 +390,11 @@ class JenkinsServiceSpec extends Specification { password: 'password') client = new JenkinsConfig().jenkinsClient(host) service = new JenkinsService('http://my.jenkins.net', client, false, Permissions.EMPTY) + def genericBuild = new GenericBuild() + genericBuild.number = 1 when: - List genericGitRevision = service.getGenericGitRevisions('test', 1) + List genericGitRevision = service.getGenericGitRevisions('test', genericBuild) then: genericGitRevision.size() == 1 @@ -444,9 +451,11 @@ class JenkinsServiceSpec extends Specification { password: 'password') client = new JenkinsConfig().jenkinsClient(host) service = new JenkinsService('http://my.jenkins.net', client, false, Permissions.EMPTY) + def genericBuild = new GenericBuild() + genericBuild.number = 1 when: - List genericGitRevision = service.getGenericGitRevisions('test', 1) + List genericGitRevision = service.getGenericGitRevisions('test', genericBuild) then: genericGitRevision.size() == 2 @@ -506,9 +515,11 @@ class JenkinsServiceSpec extends Specification { password: 'password') client = new JenkinsConfig().jenkinsClient(host) service = new JenkinsService('http://my.jenkins.net', client, false, Permissions.EMPTY) + def genericBuild = new GenericBuild() + genericBuild.number = 1 when: - List genericGitRevision = service.getGenericGitRevisions('test', 1) + List genericGitRevision = service.getGenericGitRevisions('test', genericBuild) then: genericGitRevision.size() == 2 @@ -568,9 +579,11 @@ class JenkinsServiceSpec extends Specification { password: 'password') client = new JenkinsConfig().jenkinsClient(host) service = new JenkinsService('http://my.jenkins.net', client, false, Permissions.EMPTY) + def genericBuild = new GenericBuild() + genericBuild.number = 1 when: - List genericGitRevision = service.getGenericGitRevisions('test', 1) + List genericGitRevision = service.getGenericGitRevisions('test', genericBuild) then: genericGitRevision.size() == 1 @@ -620,9 +633,11 @@ class JenkinsServiceSpec extends Specification { password: 'password') client = new JenkinsConfig().jenkinsClient(host) service = new JenkinsService('http://my.jenkins.net', client, false, Permissions.EMPTY) + def genericBuild = new GenericBuild() + genericBuild.number = 1 expect: - service.getBuildProperties("PropertiesTest", 1, "props$extension") == testCase.result + service.getBuildProperties("PropertiesTest", genericBuild, "props$extension") == testCase.result cleanup: server.shutdown() diff --git a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/travis/client/TravisClientSpec.groovy b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/travis/client/TravisClientSpec.groovy index 08a88a603..b49ec0ccb 100644 --- a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/travis/client/TravisClientSpec.groovy +++ b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/travis/client/TravisClientSpec.groovy @@ -24,12 +24,10 @@ import com.netflix.spinnaker.igor.travis.client.model.Accounts import com.netflix.spinnaker.igor.travis.client.model.Build import com.netflix.spinnaker.igor.travis.client.model.Builds import com.netflix.spinnaker.igor.travis.client.model.GithubAuth -import com.netflix.spinnaker.igor.travis.client.model.Job -import com.netflix.spinnaker.igor.travis.client.model.Jobs import com.netflix.spinnaker.igor.travis.client.model.RepoRequest -import com.netflix.spinnaker.igor.travis.client.model.RepoWrapper import com.netflix.spinnaker.igor.travis.client.model.Repos import com.netflix.spinnaker.igor.travis.client.model.TriggerResponse +import com.netflix.spinnaker.igor.travis.client.model.v3.V3Build import com.netflix.spinnaker.igor.travis.client.model.v3.V3Builds import com.netflix.spinnaker.igor.travis.client.model.v3.V3Log import com.squareup.okhttp.mockwebserver.MockResponse @@ -87,41 +85,6 @@ class TravisClientSpec extends Specification { account.login == 'gardalize' } - def "Builds"() { - given: - setResponse '''{ - "builds": [ - { - "commit_id": 6534711, - "config": { }, - "duration": 2648, - "finished_at": "2014-04-08T19:52:56Z", - "id": 22555277, - "job_ids": [22555278, 22555279, 22555280, 22555281], - "number": "784", - "pull_request": true, - "pull_request_number": "1912", - "pull_request_title": "Example PR", - "repository_id": 82, - "started_at": "2014-04-08T19:37:44Z", - "state": "failed" - } - ], - "jobs": [ ], - "commits": [ ] - }''' - - when: - Builds builds = client.builds("someToken") - - then: - builds.builds.first().commitId == 6534711 - builds.builds.first().job_ids == [22555278, 22555279, 22555280, 22555281] - - } - - - def "Repos"() { given: setResponse '''{"repos":[{"id":8059977,"slug":"gardalize/travistest","description":"testing travis stuff","last_build_id":118583435,"last_build_number":"5","last_build_state":"passed","last_build_duration":39,"last_build_language":null,"last_build_started_at":"2016-03-25T22:29:44Z","last_build_finished_at":"2016-03-25T22:30:23Z","active":true,"github_language":"Ruby"}]}''' @@ -135,49 +98,6 @@ class TravisClientSpec extends Specification { repos.repos.first().lastBuildId == 118583435 } - def "repoWrapper(accessToken, repoSlug)"() { - given: - setResponse ''' - { - "repo": { - "id": 8059977, - "slug": "gardalize/travistest", - "active": true, - "description": "testing travis stuff", - "last_build_id": 123621158, - "last_build_number": "51", - "last_build_state": "passed", - "last_build_duration": 38, - "last_build_language": null, - "last_build_started_at": "2016-04-16T21:26:35Z", - "last_build_finished_at": "2016-04-16T21:27:13Z", - "github_language": "Ruby" - } - } - ''' - - when: - RepoWrapper repoWrapper = client.repoWrapper("someToken", "gardalize/travistest") - - then: - repoWrapper.repo.slug == "gardalize/travistest" - repoWrapper.repo.lastBuildDuration == 38 - } - - - def "Job"() { - given: - setResponse '''{"job":{"id":118582578,"repository_id":8059977,"repository_slug":"gardalize/travistest","build_id":118582577,"commit_id":33511919,"log_id":85633918,"number":"4.1","config":{"language":"ruby","rvm":"1.9.3",".result":"configured","group":"stable","dist":"precise","os":"linux"},"state":"passed","started_at":"2016-03-25T22:26:13Z","finished_at":"2016-03-25T22:26:48Z","queue":"builds.docker","allow_failure":false,"tags":null,"annotation_ids":[]},"commit":{"id":33511919,"sha":"f8f8a8defe21ddc337185850cf894fe40ada2b9f","branch":"master","branch_is_default":true,"message":"rake","committed_at":"2016-03-25T22:25:19Z","author_name":"Gard Rimestad","author_email":"gardalize@gurters.com","committer_name":"Gard Rimestad","committer_email":"gardalize@gurters.com","compare_url":"https://github.com/gardalize/travistest/compare/2a9956914194...f8f8a8defe21"},"annotations":[]}''' - - when: - Jobs jobs = client.jobs("someToken", 118582578) - - then: - Job job = jobs.job - job.id == 118582578 - job.logId == 85633918 - } - def "triggerBuild()" () { given: setResponse '''{ @@ -293,7 +213,41 @@ class TravisClientSpec extends Specification { !v3Log.ready } - def "getBuilds(accessToken, repoSlug, buildNumber"() { + def "detect incomplete log due to missing part"() { + given: + setResponse ''' + { + "@type": "log", + "@href": "/job/123/log", + "@representation": "standard", + "@permissions": { + "read": true, + "debug": false, + "cancel": false, + "restart": false, + "delete_log": false + }, + "id": 1337, + "content": "ERROR: An error occured while trying to parse your .travis.yml file.\\n\\nPlease make sure that the file is valid YAML.\\n\\nhttp://lint.travis-ci.org can check your .travis.yml.\\n\\nThe error was \\"undefined method `merge' for false:FalseClass\\".", + "log_parts": [ + { + "content": "http://lint.travis-ci.org can check your .travis.yml.\\n\\nThe error was \\"undefined method `merge' for false:FalseClass\\".", + "final": true, + "number": 1 + } + ], + "@raw_log_href": "/v3/job/123/log.txt?log.token=hfeus7seyfhfe8" + }'''.stripIndent() + + when: + V3Log v3Log = client.jobLog("someToken", 123) + + then: + v3Log.content.contains "false:FalseClass" + !v3Log.ready + } + + def "getBuilds(accessToken, repoSlug, buildNumber)"() { given: setResponse ''' { @@ -497,149 +451,236 @@ class TravisClientSpec extends Specification { def "Parse builds from the v3 api"() { given: setResponse ''' -{ - "@type": "builds", - "@href": "/api/repo/spt-infrastructure%2Forca/builds?branch.name=sch_master&limit=2", - "@representation": "standard", - "@pagination": { - "limit": 2, - "offset": 0, - "count": 160, - "is_first": true, - "is_last": false, - "next": { - "@href": "/api/repo/spt-infrastructure%2Forca/builds?branch.name=sch_master&limit=2&offset=2", - "offset": 2, - "limit": 2 - }, - "prev": null, - "first": { - "@href": "/api/repo/spt-infrastructure%2Forca/builds?branch.name=sch_master&limit=2", - "offset": 0, - "limit": 2 - }, - "last": { - "@href": "/api/repo/spt-infrastructure%2Forca/builds?branch.name=sch_master&limit=2&offset=158", - "offset": 158, - "limit": 2 - } - }, - "builds": [ - { - "@type": "build", - "@href": "/api/build/1386282", - "@representation": "standard", - "@permissions": { - "read": true, - "cancel": true, - "restart": true - }, - "id": 1386282, - "number": "389", - "state": "passed", - "duration": 413, - "event_type": "push", - "previous_state": "passed", - "pull_request_title": null, - "pull_request_number": null, - "started_at": "2017-06-06T18:06:56Z", - "finished_at": "2017-06-06T18:13:49Z", - "repository": { - "@type": "repository", - "@href": "/api/repo/1996", - "@representation": "minimal", - "id": 1996, - "name": "orca", - "slug": "spt-infrastructure/orca" - }, - "branch": { - "@type": "branch", - "@href": "/api/repo/1996/branch/sch_master", - "@representation": "minimal", - "name": "sch_master" - }, - "commit": { - "@type": "commit", - "@representation": "minimal", - "id": 802307, - "sha": "3d38456a3656a65032c4db9c08d3648abb696b58", - "ref": "refs/heads/sch_master", - "message": "Merge pull request #54 from spt-infrastructure/DOCD-1025\\n\\nDocd 1025", - "compare_url": "https://github.schibsted.io/spt-infrastructure/orca/compare/0b373922a733...3d38456a3656", - "committed_at": "2017-06-06T18:06:51Z" - }, - "jobs": [ - { - "@type": "job", - "@href": "/api/job/1386283", - "@representation": "minimal", - "id": 1386283 - } - ] - }, - { - "@type": "build", - "@href": "/api/build/1384443", - "@representation": "standard", - "@permissions": { - "read": true, - "cancel": true, - "restart": true - }, - "id": 1384443, - "number": "388", - "state": "passed", - "duration": 717, - "event_type": "pull_request", - "previous_state": "passed", - "pull_request_title": "Docd 1025", - "pull_request_number": 54, - "started_at": "2017-06-06T13:46:06Z", - "finished_at": "2017-06-06T13:58:03Z", - "repository": { - "@type": "repository", - "@href": "/api/repo/1996", - "@representation": "minimal", - "id": 1996, - "name": "orca", - "slug": "spt-infrastructure/orca" - }, - "branch": { - "@type": "branch", - "@href": "/api/repo/1996/branch/sch_master", - "@representation": "minimal", - "name": "sch_master" - }, - "commit": { - "@type": "commit", - "@representation": "minimal", - "id": 801226, - "sha": "40e8099ba9cab03febe17b38a650b8868434a068", - "ref": "refs/pull/54/merge", - "message": "DOCD-1025 enable pipelineTemplates and add proxy", - "compare_url": "https://github.schibsted.io/spt-infrastructure/orca/pull/54", - "committed_at": "2017-06-06T13:34:13Z" - }, - "jobs": [ - { - "@type": "job", - "@href": "/api/job/1384444", - "@representation": "minimal", - "id": 1384444 - } - ] - } - ] -} -''' + { + "@type": "builds", + "@href": "/api/repo/spt-infrastructure%2Forca/builds?branch.name=sch_master&limit=2", + "@representation": "standard", + "@pagination": { + "limit": 2, + "offset": 0, + "count": 160, + "is_first": true, + "is_last": false, + "next": { + "@href": "/api/repo/spt-infrastructure%2Forca/builds?branch.name=sch_master&limit=2&offset=2", + "offset": 2, + "limit": 2 + }, + "prev": null, + "first": { + "@href": "/api/repo/spt-infrastructure%2Forca/builds?branch.name=sch_master&limit=2", + "offset": 0, + "limit": 2 + }, + "last": { + "@href": "/api/repo/spt-infrastructure%2Forca/builds?branch.name=sch_master&limit=2&offset=158", + "offset": 158, + "limit": 2 + } + }, + "builds": [ + { + "@type": "build", + "@href": "/api/build/1386282", + "@representation": "standard", + "@permissions": { + "read": true, + "cancel": true, + "restart": true + }, + "id": 1386282, + "number": "389", + "state": "passed", + "duration": 413, + "event_type": "push", + "previous_state": "passed", + "pull_request_title": null, + "pull_request_number": null, + "started_at": "2017-06-06T18:06:56Z", + "finished_at": "2017-06-06T18:13:49Z", + "repository": { + "@type": "repository", + "@href": "/api/repo/1996", + "@representation": "minimal", + "id": 1996, + "name": "orca", + "slug": "spt-infrastructure/orca" + }, + "branch": { + "@type": "branch", + "@href": "/api/repo/1996/branch/sch_master", + "@representation": "minimal", + "name": "sch_master" + }, + "commit": { + "@type": "commit", + "@representation": "minimal", + "id": 802307, + "sha": "3d38456a3656a65032c4db9c08d3648abb696b58", + "ref": "refs/heads/sch_master", + "message": "Merge pull request #54 from spt-infrastructure/DOCD-1025\\n\\nDocd 1025", + "compare_url": "https://github.schibsted.io/spt-infrastructure/orca/compare/0b373922a733...3d38456a3656", + "committed_at": "2017-06-06T18:06:51Z" + }, + "jobs": [ + { + "@type": "job", + "@href": "/api/job/1386283", + "@representation": "minimal", + "id": 1386283 + } + ] + }, + { + "@type": "build", + "@href": "/api/build/1384443", + "@representation": "standard", + "@permissions": { + "read": true, + "cancel": true, + "restart": true + }, + "id": 1384443, + "number": "388", + "state": "passed", + "duration": 717, + "event_type": "pull_request", + "previous_state": "passed", + "pull_request_title": "Docd 1025", + "pull_request_number": 54, + "started_at": "2017-06-06T13:46:06Z", + "finished_at": "2017-06-06T13:58:03Z", + "repository": { + "@type": "repository", + "@href": "/api/repo/1996", + "@representation": "minimal", + "id": 1996, + "name": "orca", + "slug": "spt-infrastructure/orca" + }, + "branch": { + "@type": "branch", + "@href": "/api/repo/1996/branch/sch_master", + "@representation": "minimal", + "name": "sch_master" + }, + "commit": { + "@type": "commit", + "@representation": "minimal", + "id": 801226, + "sha": "40e8099ba9cab03febe17b38a650b8868434a068", + "ref": "refs/pull/54/merge", + "message": "DOCD-1025 enable pipelineTemplates and add proxy", + "compare_url": "https://github.schibsted.io/spt-infrastructure/orca/pull/54", + "committed_at": "2017-06-06T13:34:13Z" + }, + "jobs": [ + { + "@type": "job", + "@href": "/api/job/1384444", + "@representation": "minimal", + "id": 1384444 + } + ] + } + ] + } + ''' when: - V3Builds builds = client.v3builds("someToken", "org/repo","bah", 2) + V3Builds builds = client.v3builds("someToken", "org/repo","bah", 2, null) then: builds.builds.size() == 2 } + def "Fetch a single V3 build"() { + given: + setResponse ''' + { + "@type": "build", + "@href": "/api/build/7128433", + "@representation": "standard", + "@permissions": { + "read": true, + "cancel": true, + "restart": true + }, + "id": 7128433, + "number": "265", + "state": "passed", + "duration": 19, + "event_type": "api", + "previous_state": "passed", + "pull_request_title": null, + "pull_request_number": null, + "started_at": "2019-05-31T10:27:47Z", + "finished_at": "2019-05-31T10:28:06Z", + "repository": { + "@type": "repository", + "@href": "/api/repo/8881", + "@representation": "minimal", + "id": 8881, + "name": "jervi-is-testing", + "slug": "arrested-developers/jervi-is-testing" + }, + "branch": { + "@type": "branch", + "@href": "/api/repo/8881/branch/master", + "@representation": "minimal", + "name": "master" + }, + "tag": null, + "commit": { + "@type": "commit", + "@representation": "standard", + "id": 4113219, + "sha": "fc726afaaf5d3892dae017e2dbe6fa8534f4423a", + "ref": null, + "message": "Triggered from Spinnaker: Hello world! 🐥 awesomeapp:Testing trigger", + "compare_url": "https://github.acme.io/arrested-developers/jervi-is-testing/compare/5e8a99bbc0e189dfbe1cec4b8da2a48aaa2f672e...fc726afaaf5d3892dae017e2dbe6fa8534f4423a", + "committed_at": "2019-05-31T10:26:30Z", + "committer": { + "name": "Alice", + "avatar_url": "https://0.gravatar.com/avatar/xxxxxxxxxxxxxxxxxx" + }, + "author": { + "name": "Bob", + "avatar_url": "https://0.gravatar.com/avatar/xxxxxxxxxxxxxxxxxx" + } + }, + "jobs": [ + { + "@type": "job", + "@href": "/api/job/7128434", + "@representation": "minimal", + "id": 7128434 + } + ], + "stages": [], + "created_by": { + "@type": "user", + "@href": "/api/user/574", + "@representation": "minimal", + "id": 574, + "login": "spinnaker" + }, + "updated_at": "2019-05-31T10:28:06.312Z", + "log_complete": true + } + ''' + + when: + V3Build build = client.v3build("someToken", 7128433, "build.log_complete") + + then: + build.number == 265 + build.commit.committer.name == "Alice" + build.commit.author.name == "Bob" + build.jobs*.id == [7128434] + build.getLogComplete() + } + def "commits, dont mark regular branches as tags"() { given: setResponse ''' @@ -678,7 +719,7 @@ class TravisClientSpec extends Specification { Builds builds = client.builds("someToken", "org/repo", 38) then: - builds.commits.first().isTag() == false + !builds.commits.first().isTag() } diff --git a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/travis/service/TravisServiceSpec.groovy b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/travis/service/TravisServiceSpec.groovy index 55d08110a..922c7d540 100644 --- a/igor-web/src/test/groovy/com/netflix/spinnaker/igor/travis/service/TravisServiceSpec.groovy +++ b/igor-web/src/test/groovy/com/netflix/spinnaker/igor/travis/service/TravisServiceSpec.groovy @@ -26,12 +26,14 @@ import com.netflix.spinnaker.igor.travis.TravisCache import com.netflix.spinnaker.igor.travis.client.TravisClient import com.netflix.spinnaker.igor.travis.client.model.AccessToken import com.netflix.spinnaker.igor.travis.client.model.Build -import com.netflix.spinnaker.igor.travis.client.model.Builds -import com.netflix.spinnaker.igor.travis.client.model.Commit import com.netflix.spinnaker.igor.travis.client.model.RepoRequest import com.netflix.spinnaker.igor.travis.client.model.TriggerResponse import com.netflix.spinnaker.igor.travis.client.model.v3.Request +import com.netflix.spinnaker.igor.travis.client.model.v3.TravisBuildState import com.netflix.spinnaker.igor.travis.client.model.v3.TravisBuildType +import com.netflix.spinnaker.igor.travis.client.model.v3.V3Build +import com.netflix.spinnaker.igor.travis.client.model.v3.V3Builds +import com.netflix.spinnaker.igor.travis.client.model.v3.V3Job import com.netflix.spinnaker.igor.travis.client.model.v3.V3Log import com.netflix.spinnaker.igor.travis.client.model.v3.V3Repository import spock.lang.Shared @@ -57,7 +59,7 @@ class TravisServiceSpec extends Specification{ client = Mock() travisCache = Mock() artifactDecorator = Optional.of(new ArtifactDecorator([new DebDetailsDecorator(), new RpmDetailsDecorator()], null)) - service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', 25, client, travisCache, artifactDecorator, [], "travis.buildMessage", Permissions.EMPTY) + service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', 25, client, travisCache, artifactDecorator, [], "travis.buildMessage", Permissions.EMPTY, false) AccessToken accessToken = new AccessToken() accessToken.accessToken = "someToken" @@ -151,64 +153,6 @@ class TravisServiceSpec extends Specification{ "my-org/repo/tags" || false } - def "getCommit(repoSlug, buildNumber)"() { - given: - Commit commit = new Commit() - commit.branch = "1.0" - commit.compareUrl = "https://github.domain/org/repo/compare/1.0" - Builds builds = Mock(Builds) - - when: - Commit fetchedCommit = service.getCommit("org/repo", 38) - - then: - fetchedCommit.isTag() - 1 * client.builds("token someToken", "org/repo", 38) >> builds - 3 * builds.commits >> [commit] - } - - def "getCommit(repoSlug, buildNumber) when no commit is found"() { - given: - Builds builds = Mock(Builds) - - when: - service.getCommit("org/repo", 38) - - then: - thrown NoSuchElementException - 1 * client.builds("token someToken", "org/repo", 38) >> builds - 2 * builds.commits >> [] - } - - def "branchedRepoSlug should return branch prefixed with pull_request if it is a pull request"() { - given: - Builds builds = Mock(Builds) - Build build = Mock(Build) - Commit commit = Mock(Commit) - - when: - String branchedRepoSlug = service.branchedRepoSlug("my/slug", 21, commit) - - then: - branchedRepoSlug == "my/slug/pull_request_master" - 1 * client.builds("token someToken", "my/slug", 21) >> builds - 2 * builds.builds >> [build] - 1 * build.pullRequest >> true - 1 * commit.getBranchNameWithTagHandling() >> "master" - - } - - def "branchedRepoSlug should fallback to input repoSlug if hystrix kicks in"() { - given: - Commit commit = Mock(Commit) - - when: - String branchedRepoSlug = service.branchedRepoSlug("my/slug", 21, commit) - - then: - branchedRepoSlug == "my/slug" - } - @Unroll def "calculate pagination correctly"() { expect: @@ -264,4 +208,57 @@ class TravisServiceSpec extends Specification{ buildNumber == 1 } + + @Unroll + def "use correct way of checking if logs are completed when legacyLogFetching is #legacyLogFetching and log_complete flag is #isLogCompleteFlag"() { + given: + def job = new V3Job().with { v3job -> + v3job.id = 2 + return v3job + } + def build = new V3Build([ + id: 1, + jobs: [ job ], + state: TravisBuildState.passed, + repository: new V3Repository([slug: "my/slug"]) + ]) + if (!legacyLogFetching) { + build.logComplete = isLogCompleteFlag + } + def v3log = new V3Log([ + logParts: [ + new V3Log.V3LogPart([ + number: 0, + content: "log", + isFinal: isLogReallyComplete + ])], + content: "log" + ]) + service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', 25, client, travisCache, artifactDecorator, [], "travis.buildMessage", Permissions.EMPTY, legacyLogFetching) + AccessToken accessToken = new AccessToken() + accessToken.accessToken = "someToken" + service.accessToken = accessToken + + when: + def genericBuilds = service.getBuilds("my/slug") + + then: + 1 * client.v3builds("token someToken", "my/slug", TravisService.TRAVIS_BUILD_RESULT_LIMIT, + legacyLogFetching ? null : "build.log_complete") >> new V3Builds([builds: [build]]) + (isLogCompleteFlag ? 0 : 1) * travisCache.getJobLog("travis-ci", 2) >> (isLogCached ? "log" : null) + (!isLogCompleteFlag && !isLogCached ? 1 : 0) * client.jobLog("token someToken", 2) >> v3log + (!isLogCompleteFlag && !isLogCached && isLogReallyComplete ? 1 : 0) * travisCache.setJobLog("travis-ci", 2, "log") + genericBuilds.size() == expectedNumberOfBuilds + + where: + legacyLogFetching | isLogCompleteFlag | isLogCached | isLogReallyComplete | expectedNumberOfBuilds + false | true | true | true | 1 + false | false | true | true | 1 + false | false | false | true | 1 + false | false | false | false | 0 + true | null | true | true | 1 + true | null | true | true | 1 + true | null | false | true | 1 + true | null | false | false | 0 + } } diff --git a/igor-web/src/test/java/com/netflix/spinnaker/igor/MainTest.java b/igor-web/src/test/java/com/netflix/spinnaker/igor/MainTest.java index 26469d48a..7e1774ea4 100644 --- a/igor-web/src/test/java/com/netflix/spinnaker/igor/MainTest.java +++ b/igor-web/src/test/java/com/netflix/spinnaker/igor/MainTest.java @@ -16,16 +16,28 @@ package com.netflix.spinnaker.igor; +import com.netflix.hystrix.Hystrix; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; @RunWith(SpringRunner.class) -@SpringBootTest(classes = {Main.class}) -@ContextConfiguration(classes = {RedisConfig.class}) +@SpringBootTest(classes = {RedisConfig.class, Main.class}) public class MainTest { + + @BeforeClass + public static void setUp() { + Hystrix.reset(); + } + @Test public void startupTest() {} + + @AfterClass + public static void tearDown() { + Hystrix.reset(); + } }