From 2e07fcc7ecb5bf77e3a3a6a5df9e5d1be9c5eabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Jervidalo?= Date: Tue, 18 Jun 2019 18:54:10 +0200 Subject: [PATCH] fix(pipelines): Fix pipeline triggers for some templated pipelines (#2963) Pipeline triggers and pipeline stages were broken for templated pipelines that use a dynamic source that references an artifact. Resolved by resolving the artifacts a bit earlier (before the preprocessors). --- orca-front50/orca-front50.gradle | 1 + .../front50/DependentPipelineStarter.groovy | 25 ++-- .../DependentPipelineStarterSpec.groovy | 134 ++++++++++++++++++ 3 files changed, 150 insertions(+), 10 deletions(-) diff --git a/orca-front50/orca-front50.gradle b/orca-front50/orca-front50.gradle index 45af498573..0ded2f56f8 100644 --- a/orca-front50/orca-front50.gradle +++ b/orca-front50/orca-front50.gradle @@ -30,5 +30,6 @@ dependencies { annotationProcessor("org.projectlombok:lombok") testImplementation(project(":orca-test-groovy")) + testImplementation(project(":orca-pipelinetemplate")) testImplementation("com.github.ben-manes.caffeine:guava") } diff --git a/orca-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/DependentPipelineStarter.groovy b/orca-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/DependentPipelineStarter.groovy index 5aa56b9957..71ae1bf208 100644 --- a/orca-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/DependentPipelineStarter.groovy +++ b/orca-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/DependentPipelineStarter.groovy @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.netflix.spectator.api.Id import com.netflix.spectator.api.Registry import com.netflix.spinnaker.kork.web.exceptions.InvalidRequestException +import com.netflix.spinnaker.kork.web.exceptions.ValidationException import com.netflix.spinnaker.orca.extensionpoint.pipeline.ExecutionPreprocessor import com.netflix.spinnaker.orca.pipeline.ExecutionLauncher import com.netflix.spinnaker.orca.pipeline.model.Execution @@ -87,7 +88,7 @@ class DependentPipelineStarter implements ApplicationContextAware { strategy : suppliedParameters.strategy == true ] - if (pipelineConfig.parameterConfig || !suppliedParameters.empty) { + if (pipelineConfig.parameterConfig || !suppliedParameters.isEmpty()) { def pipelineParameters = suppliedParameters ?: [:] pipelineConfig.parameterConfig.each { pipelineConfig.trigger.parameters[it.name] = pipelineParameters.containsKey(it.name) ? pipelineParameters[it.name] : it.default @@ -101,21 +102,12 @@ class DependentPipelineStarter implements ApplicationContextAware { //keep the trigger as the preprocessor removes it. def expectedArtifacts = pipelineConfig.expectedArtifacts - for (ExecutionPreprocessor preprocessor : executionPreprocessors.findAll { - it.supports(pipelineConfig, ExecutionPreprocessor.Type.PIPELINE) - }) { - pipelineConfig = preprocessor.process(pipelineConfig) - } - if (parentPipelineStageId != null) { pipelineConfig.receivedArtifacts = artifactResolver?.getArtifacts(parentPipeline.stageById(parentPipelineStageId)) } else { pipelineConfig.receivedArtifacts = artifactResolver?.getAllArtifacts(parentPipeline) } - pipelineConfig.trigger = trigger - pipelineConfig.expectedArtifacts = expectedArtifacts - def artifactError = null try { artifactResolver?.resolveArtifacts(pipelineConfig) @@ -123,6 +115,19 @@ class DependentPipelineStarter implements ApplicationContextAware { artifactError = e } + for (ExecutionPreprocessor preprocessor : executionPreprocessors.findAll { + it.supports(pipelineConfig, ExecutionPreprocessor.Type.PIPELINE) + }) { + pipelineConfig = preprocessor.process(pipelineConfig) + } + + if (pipelineConfig.errors != null) { + throw new ValidationException("Pipeline template is invalid", pipelineConfig.errors as List>) + } + + pipelineConfig.trigger = trigger + pipelineConfig.expectedArtifacts = expectedArtifacts + // Process the raw trigger to resolve any expressions before converting it to a Trigger object, which will not be // processed by the contextParameterProcessor (it only handles Maps, Lists, and Strings) Map processedTrigger = contextParameterProcessor.process([trigger: pipelineConfig.trigger], [trigger: pipelineConfig.trigger], false).trigger diff --git a/orca-front50/src/test/groovy/com/netflix/spinnaker/orca/front50/DependentPipelineStarterSpec.groovy b/orca-front50/src/test/groovy/com/netflix/spinnaker/orca/front50/DependentPipelineStarterSpec.groovy index 94a8051f10..0531d81fa9 100644 --- a/orca-front50/src/test/groovy/com/netflix/spinnaker/orca/front50/DependentPipelineStarterSpec.groovy +++ b/orca-front50/src/test/groovy/com/netflix/spinnaker/orca/front50/DependentPipelineStarterSpec.groovy @@ -16,21 +16,34 @@ package com.netflix.spinnaker.orca.front50 +import com.fasterxml.jackson.databind.JavaType import com.fasterxml.jackson.databind.ObjectMapper import com.netflix.spectator.api.NoopRegistry import com.netflix.spinnaker.kork.artifacts.model.Artifact +import com.netflix.spinnaker.orca.extensionpoint.pipeline.ExecutionPreprocessor import com.netflix.spinnaker.orca.jackson.OrcaObjectMapper import com.netflix.spinnaker.orca.pipeline.ExecutionLauncher import com.netflix.spinnaker.orca.pipeline.model.DefaultTrigger import com.netflix.spinnaker.orca.pipeline.model.Execution +import com.netflix.spinnaker.orca.pipeline.model.Stage import com.netflix.spinnaker.orca.pipeline.model.Trigger import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository import com.netflix.spinnaker.orca.pipeline.util.ArtifactResolver import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor +import com.netflix.spinnaker.orca.pipelinetemplate.PipelineTemplatePreprocessor +import com.netflix.spinnaker.orca.pipelinetemplate.handler.PipelineTemplateErrorHandler +import com.netflix.spinnaker.orca.pipelinetemplate.handler.SchemaVersionHandler +import com.netflix.spinnaker.orca.pipelinetemplate.loader.TemplateLoader +import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.handler.V1SchemaHandlerGroup +import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.handler.v2.V2SchemaHandlerGroup +import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.model.PipelineTemplate +import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.model.TemplateConfiguration +import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.render.JinjaRenderer import com.netflix.spinnaker.security.User import org.slf4j.MDC import org.springframework.context.ApplicationContext import org.springframework.context.support.StaticApplicationContext +import rx.Observable import spock.lang.Specification import spock.lang.Subject @@ -450,4 +463,125 @@ class DependentPipelineStarterSpec extends Specification { then: result.trigger.parameters.a == "pipeline" } + + def "should trigger v1 templated pipelines with dynamic source using prior artifact"() { + given: + def triggeredPipelineConfig = [ + id: "triggered", + type: "templatedPipeline", + config: [ + pipeline: [ + application: "covfefe", + name: "Templated pipeline", + template: [ + source: "{% for artifact in trigger.artifacts %}{% if artifact.type == 'spinnaker-pac' && artifact.name == 'wait' %}{{ artifact.reference }}{% endif %}{% endfor %}" + ] + ], + schema: "1" + ], + expectedArtifacts: [[ + defaultArtifact: [ + customKind: true, + id: "091b682c-10ac-441a-97f2-659113128960", + ], + displayName: "friendly-gecko-6", + id: "28907e3a-e529-473d-bf2d-b3737c9d6dc6", + matchArtifact: [ + customKind: true, + id: "daef2911-ea5c-4098-aa07-ee2535b2788d", + name: "wait", + type: "spinnaker-pac" + ], + useDefaultArtifact: false, + usePriorArtifact: true + ]], + ] + def triggeredPipelineTemplate = mapper.convertValue([ + schema: "1", + id: "barebones", + stages: [[ + id: "wait1", + type: "wait", + name: "Wait for 5 seconds", + config: [ + waitTime: 5 + ] + ]] + ], PipelineTemplate) + def priorExecution = pipeline { + id = "01DCKTEZPRCMFV1H35EDFC62RG" + trigger = new DefaultTrigger("manual", null, "user@acme.com", [:], [ + new Artifact([ + customKind: false, + metadata: [ + fileName: "wait.0.1.yml", + ], + name: "wait", + reference: "https://artifactory.acme.com/spinnaker-pac/wait.0.1.yml", + type: "spinnaker-pac", + version: "0.1" + ]) + ]) + } + def parentPipeline = pipeline { + name = "parent" + trigger = new DefaultTrigger("manual", null, "user@schibsted.com", [:], [], [], false, true) + authentication = new Execution.AuthenticationDetails("parentUser", "acct1", "acct2") + } + def executionLauncher = Mock(ExecutionLauncher) + def templateLoader = Mock(TemplateLoader) + def applicationContext = new StaticApplicationContext() + def renderer = new JinjaRenderer(mapper, Mock(Front50Service), []) + def registry = new NoopRegistry() + def parameterProcessor = new ContextParameterProcessor() + def pipelineTemplatePreprocessor = new PipelineTemplatePreprocessor( + mapper, + new SchemaVersionHandler( + new V1SchemaHandlerGroup( + templateLoader, + renderer, + mapper, + registry), + Mock(V2SchemaHandlerGroup)), + new PipelineTemplateErrorHandler(), + registry) + applicationContext.beanFactory.registerSingleton("pipelineLauncher", executionLauncher) + dependentPipelineStarter = new DependentPipelineStarter( + applicationContext, + mapper, + parameterProcessor, + Optional.of([pipelineTemplatePreprocessor] as List), + Optional.of(artifactResolver), + registry + ) + + and: + 1 * executionLauncher.start(*_) >> { + def p = mapper.readValue(it[1], Map) + return pipeline { + JavaType type = mapper.getTypeFactory().constructCollectionType(List, Stage) + trigger = mapper.convertValue(p.trigger, Trigger) + stages.addAll(mapper.convertValue(p.stages, type)) + } + } + 1 * templateLoader.load(_ as TemplateConfiguration.TemplateSource) >> [triggeredPipelineTemplate] + 1 * executionRepository.retrievePipelinesForPipelineConfigId("triggered", _ as ExecutionRepository.ExecutionCriteria) >> + Observable.just(priorExecution) + + when: + def result = dependentPipelineStarter.trigger( + triggeredPipelineConfig, + null /*user*/, + parentPipeline, + [:], + null, + buildAuthenticatedUser("user", []) + ) + + then: + result.stages.size() == 1 + result.stages[0].type == "wait" + result.stages[0].refId == "wait1" + result.stages[0].name == "Wait for 5 seconds" + } }