fix(pipelines): Fix pipeline triggers for some templated pipelines (#…

…2963)

Pipeline triggers and pipeline stages were broken for templated pipelines that use a dynamic source that references an artifact. Resolved by resolving the artifacts a bit earlier (before the preprocessors).

orca-front50/orca-front50.gradle

@@ -30,5 +30,6 @@ dependencies {
   annotationProcessor("org.projectlombok:lombok")
 
   testImplementation(project(":orca-test-groovy"))
+  testImplementation(project(":orca-pipelinetemplate"))
   testImplementation("com.github.ben-manes.caffeine:guava")
 }

orca-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/DependentPipelineStarter.groovy

@@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import com.netflix.spectator.api.Id
 import com.netflix.spectator.api.Registry
 import com.netflix.spinnaker.kork.web.exceptions.InvalidRequestException
+import com.netflix.spinnaker.kork.web.exceptions.ValidationException
 import com.netflix.spinnaker.orca.extensionpoint.pipeline.ExecutionPreprocessor
 import com.netflix.spinnaker.orca.pipeline.ExecutionLauncher
 import com.netflix.spinnaker.orca.pipeline.model.Execution
@@ -87,7 +88,7 @@ class DependentPipelineStarter implements ApplicationContextAware {
       strategy             : suppliedParameters.strategy == true
     ]
 
-    if (pipelineConfig.parameterConfig || !suppliedParameters.empty) {
+    if (pipelineConfig.parameterConfig || !suppliedParameters.isEmpty()) {
       def pipelineParameters = suppliedParameters ?: [:]
       pipelineConfig.parameterConfig.each {
         pipelineConfig.trigger.parameters[it.name] = pipelineParameters.containsKey(it.name) ? pipelineParameters[it.name] : it.default
@@ -101,28 +102,32 @@ class DependentPipelineStarter implements ApplicationContextAware {
     //keep the trigger as the preprocessor removes it.
     def expectedArtifacts = pipelineConfig.expectedArtifacts
 
-    for (ExecutionPreprocessor preprocessor : executionPreprocessors.findAll {
-      it.supports(pipelineConfig, ExecutionPreprocessor.Type.PIPELINE)
-    }) {
-      pipelineConfig = preprocessor.process(pipelineConfig)
-    }
-
     if (parentPipelineStageId != null) {
       pipelineConfig.receivedArtifacts = artifactResolver?.getArtifacts(parentPipeline.stageById(parentPipelineStageId))
     } else {
       pipelineConfig.receivedArtifacts = artifactResolver?.getAllArtifacts(parentPipeline)
     }
 
-    pipelineConfig.trigger = trigger
-    pipelineConfig.expectedArtifacts = expectedArtifacts
-
     def artifactError = null
     try {
       artifactResolver?.resolveArtifacts(pipelineConfig)
     } catch (Exception e) {
       artifactError = e
     }
 
+    for (ExecutionPreprocessor preprocessor : executionPreprocessors.findAll {
+      it.supports(pipelineConfig, ExecutionPreprocessor.Type.PIPELINE)
+    }) {
+      pipelineConfig = preprocessor.process(pipelineConfig)
+    }
+
+    if (pipelineConfig.errors != null) {
+      throw new ValidationException("Pipeline template is invalid", pipelineConfig.errors as List<Map<String, Object>>)
+    }
+
+    pipelineConfig.trigger = trigger
+    pipelineConfig.expectedArtifacts = expectedArtifacts
+
     // Process the raw trigger to resolve any expressions before converting it to a Trigger object, which will not be
     // processed by the contextParameterProcessor (it only handles Maps, Lists, and Strings)
     Map processedTrigger = contextParameterProcessor.process([trigger: pipelineConfig.trigger], [trigger: pipelineConfig.trigger], false).trigger

orca-front50/src/test/groovy/com/netflix/spinnaker/orca/front50/DependentPipelineStarterSpec.groovy

@@ -16,21 +16,34 @@
 
 package com.netflix.spinnaker.orca.front50
 
+import com.fasterxml.jackson.databind.JavaType
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.netflix.spectator.api.NoopRegistry
 import com.netflix.spinnaker.kork.artifacts.model.Artifact
+import com.netflix.spinnaker.orca.extensionpoint.pipeline.ExecutionPreprocessor
 import com.netflix.spinnaker.orca.jackson.OrcaObjectMapper
 import com.netflix.spinnaker.orca.pipeline.ExecutionLauncher
 import com.netflix.spinnaker.orca.pipeline.model.DefaultTrigger
 import com.netflix.spinnaker.orca.pipeline.model.Execution
+import com.netflix.spinnaker.orca.pipeline.model.Stage
 import com.netflix.spinnaker.orca.pipeline.model.Trigger
 import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository
 import com.netflix.spinnaker.orca.pipeline.util.ArtifactResolver
 import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor
+import com.netflix.spinnaker.orca.pipelinetemplate.PipelineTemplatePreprocessor
+import com.netflix.spinnaker.orca.pipelinetemplate.handler.PipelineTemplateErrorHandler
+import com.netflix.spinnaker.orca.pipelinetemplate.handler.SchemaVersionHandler
+import com.netflix.spinnaker.orca.pipelinetemplate.loader.TemplateLoader
+import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.handler.V1SchemaHandlerGroup
+import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.handler.v2.V2SchemaHandlerGroup
+import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.model.PipelineTemplate
+import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.model.TemplateConfiguration
+import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.render.JinjaRenderer
 import com.netflix.spinnaker.security.User
 import org.slf4j.MDC
 import org.springframework.context.ApplicationContext
 import org.springframework.context.support.StaticApplicationContext
+import rx.Observable
 import spock.lang.Specification
 import spock.lang.Subject
 
@@ -450,4 +463,125 @@ class DependentPipelineStarterSpec extends Specification {
     then:
     result.trigger.parameters.a == "pipeline"
   }
+
+  def "should trigger v1 templated pipelines with dynamic source using prior artifact"() {
+    given:
+    def triggeredPipelineConfig = [
+      id: "triggered",
+      type: "templatedPipeline",
+      config: [
+        pipeline: [
+          application: "covfefe",
+          name: "Templated pipeline",
+          template: [
+            source: "{% for artifact in trigger.artifacts %}{% if artifact.type == 'spinnaker-pac' && artifact.name == 'wait' %}{{ artifact.reference }}{% endif %}{% endfor %}"
+          ]
+        ],
+        schema: "1"
+      ],
+      expectedArtifacts: [[
+        defaultArtifact: [
+          customKind: true,
+          id: "091b682c-10ac-441a-97f2-659113128960",
+        ],
+        displayName: "friendly-gecko-6",
+        id: "28907e3a-e529-473d-bf2d-b3737c9d6dc6",
+        matchArtifact: [
+          customKind: true,
+          id: "daef2911-ea5c-4098-aa07-ee2535b2788d",
+          name: "wait",
+          type: "spinnaker-pac"
+        ],
+        useDefaultArtifact: false,
+        usePriorArtifact: true
+      ]],
+    ]
+    def triggeredPipelineTemplate = mapper.convertValue([
+      schema: "1",
+      id: "barebones",
+      stages: [[
+        id: "wait1",
+        type: "wait",
+        name: "Wait for 5 seconds",
+        config: [
+          waitTime: 5
+        ]
+      ]]
+    ], PipelineTemplate)
+    def priorExecution = pipeline {
+      id = "01DCKTEZPRCMFV1H35EDFC62RG"
+      trigger = new DefaultTrigger("manual", null, "user@acme.com", [:], [
+        new Artifact([
+          customKind: false,
+          metadata: [
+            fileName: "wait.0.1.yml",
+          ],
+          name: "wait",
+          reference: "https://artifactory.acme.com/spinnaker-pac/wait.0.1.yml",
+          type: "spinnaker-pac",
+          version: "0.1"
+        ])
+      ])
+    }
+    def parentPipeline = pipeline {
+      name = "parent"
+      trigger = new DefaultTrigger("manual", null, "user@schibsted.com", [:], [], [], false, true)
+      authentication = new Execution.AuthenticationDetails("parentUser", "acct1", "acct2")
+    }
+    def executionLauncher = Mock(ExecutionLauncher)
+    def templateLoader = Mock(TemplateLoader)
+    def applicationContext = new StaticApplicationContext()
+    def renderer = new JinjaRenderer(mapper, Mock(Front50Service), [])
+    def registry = new NoopRegistry()
+    def parameterProcessor = new ContextParameterProcessor()
+    def pipelineTemplatePreprocessor = new PipelineTemplatePreprocessor(
+      mapper,
+      new SchemaVersionHandler(
+        new V1SchemaHandlerGroup(
+          templateLoader,
+          renderer,
+          mapper,
+          registry),
+        Mock(V2SchemaHandlerGroup)),
+      new PipelineTemplateErrorHandler(),
+      registry)
+    applicationContext.beanFactory.registerSingleton("pipelineLauncher", executionLauncher)
+    dependentPipelineStarter = new DependentPipelineStarter(
+      applicationContext,
+      mapper,
+      parameterProcessor,
+      Optional.of([pipelineTemplatePreprocessor] as List<ExecutionPreprocessor>),
+      Optional.of(artifactResolver),
+      registry
+    )
+
+    and:
+    1 * executionLauncher.start(*_) >> {
+      def p = mapper.readValue(it[1], Map)
+      return pipeline {
+        JavaType type = mapper.getTypeFactory().constructCollectionType(List, Stage)
+        trigger = mapper.convertValue(p.trigger, Trigger)
+        stages.addAll(mapper.convertValue(p.stages, type))
+      }
+    }
+    1 * templateLoader.load(_ as TemplateConfiguration.TemplateSource) >> [triggeredPipelineTemplate]
+    1 * executionRepository.retrievePipelinesForPipelineConfigId("triggered", _ as ExecutionRepository.ExecutionCriteria) >>
+      Observable.just(priorExecution)
+
+    when:
+    def result = dependentPipelineStarter.trigger(
+      triggeredPipelineConfig,
+      null /*user*/,
+      parentPipeline,
+      [:],
+      null,
+      buildAuthenticatedUser("user", [])
+    )
+
+    then:
+    result.stages.size() == 1
+    result.stages[0].type == "wait"
+    result.stages[0].refId == "wait1"
+    result.stages[0].name == "Wait for 5 seconds"
+  }
 }