Add support for unique keys and depends_on (#76)

* Add support for unique keys and depends_on

* Fix syntax

* Address feedback

* Address feedback

* A little bit cleaner

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+0.9.3 (boyntoni)
+----------------
+
+1. Generate unique keys for project steps, and add support for depends_on.
+
+
 0.9.2 (mowies)
 --------------
 1. Add option to set pipeline and project scoped environment variables.

README.md

@@ -15,7 +15,7 @@ Example
 steps:
   - label: ":buildkite:"
     plugins:
-      - jwplayer/buildpipe#v0.9.2:
+      - jwplayer/buildpipe#v0.9.3:
           dynamic_pipeline: dynamic_pipeline.yml
 ```
 

hooks/command

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euo pipefail
 
-buildpipe_version="${BUILDKITE_PLUGIN_BUILDPIPE_VERSION:-0.9.2}"
+buildpipe_version="${BUILDKITE_PLUGIN_BUILDPIPE_VERSION:-0.9.3}"
 is_test="${BUILDKITE_PLUGIN_BUILDPIPE_TEST_MODE:-false}"
 
 if [[ "$is_test" == "false" ]]; then

pipeline.go

@@ -14,29 +14,79 @@ type Pipeline struct {
 	Steps []interface{} `yaml:"steps"`
 }
 
-func generateProjectSteps(step interface{}, projects []Project) []interface{} {
+func generateProjectSteps(steps []interface{}, step interface{}, projects []Project) []interface{} {
 	projectSteps := make([]interface{}, 0)
+
 	for _, project := range projects {
 		stepCopy := deepcopy.Copy(step)
 		stepCopyMap := stepCopy.(map[interface{}]interface{})
 
 		if project.checkProjectRules(stepCopyMap) {
+			// Unique project level label
 			stepCopyMap["label"] = fmt.Sprintf("%s %s", stepCopyMap["label"], project.Label)
+
+			// Set default buildpipe environment variables and
+			// set project env vars as step env vars
 			env := stepCopyMap["env"].(map[interface{}]interface{})
 			env["BUILDPIPE_PROJECT_LABEL"] = project.Label
 			env["BUILDPIPE_PROJECT_PATH"] = project.getMainPath()
-
 			for envVarName, envVarValue := range project.Env {
 				env[envVarName] = envVarValue
 			}
 
+			// Unique project level key, if present
+			if val, ok := stepCopyMap["key"]; ok {
+				stepCopyMap["key"] = fmt.Sprintf("%s %s", val, project.Label)
+			}
+
+			// If the step includes a depends_on clause, we need to validate whether each dependency
+			// is a project-scoped step. If so, the dependency has the current project name added
+			// to it to match the unique key given above.
+			if val, ok := stepCopyMap["depends_on"]; ok {
+				dependencyList := val.([]interface{})
+
+				for i, dependency := range dependencyList {
+					depStr := dependency.(string)
+
+					if step := findStepByKey(steps, depStr); step != nil {
+						if isProjectScopeStep(step) {
+							dependencyList[i] = fmt.Sprintf("%s %s", depStr, project.Label)
+						}
+					}
+				}
+			}
 			projectSteps = append(projectSteps, stepCopy)
 		}
 	}
 
 	return projectSteps
 }
 
+func isProjectScopeStep(step map[interface{}]interface{}) bool {
+	if env, ok := step["env"].(map[interface{}]interface{}); ok {
+		if value, ok := env["BUILDPIPE_SCOPE"]; ok {
+			return value == "project"
+		}
+	}
+	return false
+}
+
+func findStepByKey(steps []interface{}, stepKey string) map[interface{}]interface{} {
+	for _, step := range steps {
+		// skip wait commands
+		stepMap, ok := step.(map[interface{}]interface{})
+		if !ok {
+			continue
+		}
+		// grab key if it has one and check whether it is project scoped
+		foundStepKey, ok := stepMap["key"]
+		if ok && stepKey == foundStepKey {
+			return stepMap
+		}
+	}
+	return nil
+}
+
 func generatePipeline(steps []interface{}, pipelineEnv map[string]string, projects []Project) *Pipeline {
 	generatedSteps := make([]interface{}, 0)
 
@@ -63,7 +113,7 @@ func generatePipeline(steps []interface{}, pipelineEnv map[string]string, projec
 
 		value, ok := env["BUILDPIPE_SCOPE"]
 		if ok && value == "project" {
-			projectSteps := generateProjectSteps(step, projects)
+			projectSteps := generateProjectSteps(steps, step, projects)
 			generatedSteps = append(generatedSteps, projectSteps...)
 		} else {
 			generatedSteps = append(generatedSteps, step)

tests/dynamic_pipeline.yml

@@ -12,12 +12,17 @@ projects:
       - project2/
       - project1 # you can trigger a project using multiple paths
   - label: project3
-    skip: # you can skip a list of projects
+    skip: # you can skip a list of steps for a project
       - test
       - deploy-stg
     path: project3/somedir/ # subpaths can also be triggered
 steps: # the same schema as regular buildkite pipeline steps
+  - label: bootstrap # a non project scoped step (to test depends_on handling)
+    key: bootstrap
+    command:
+      - make bootstrap
   - label: test
+    key: test
     env:
       BUILDPIPE_SCOPE: project # this variable ensures a test step is generated for each project
     command:
@@ -35,13 +40,17 @@ steps: # the same schema as regular buildkite pipeline steps
       - make publish-image
     agents:
       - queue=build
+    depends_on:
+      - bootstrap # the rendered template should not include the project name for a non-project step
+      - test # the rendered template should include the project name for a project-scoped step
   - wait
   - label: tag
     branches: "master"
     command:
       - make tag-release
   - wait
   - label: deploy-stg
+    key: deploy-stg
     branches: "master"
     concurrency: 1
     concurrency_group: deploy-stg
@@ -58,6 +67,8 @@ steps: # the same schema as regular buildkite pipeline steps
     branches: "master"
     concurrency: 1
     concurrency_group: deploy-prd
+    depends_on:
+      - deploy-stg
     env:
       BUILDPIPE_SCOPE: project
     command:

tests/post-command.bats

@@ -34,6 +34,12 @@ teardown() {
 
   assert_output --partial << EOM
 steps:
+- command:
+  - make bootstrap
+  env:
+    TEST_ENV_PIPELINE: test-pipeline
+  key: bootstrap
+  label: bootstrap
 - command:
   - cd \$\$BUILDPIPE_PROJECT_PATH
   - make test
@@ -43,6 +49,7 @@ steps:
     BUILDPIPE_SCOPE: project
     TEST_ENV_PIPELINE: test-pipeline
     TEST_ENV_PROJECT: test-project
+  key: test project1
   label: test project1
 - wait
 - agents:
@@ -52,6 +59,9 @@ steps:
   - cd \$\$BUILDPIPE_PROJECT_PATH
   - make build
   - make publish-image
+  depends_on:
+  - bootstrap
+  - test project1
   env:
     BUILDPIPE_PROJECT_LABEL: project1
     BUILDPIPE_PROJECT_PATH: project1/
@@ -67,6 +77,9 @@ steps:
   - cd \$\$BUILDPIPE_PROJECT_PATH
   - make build
   - make publish-image
+  depends_on:
+  - bootstrap
+  - test project2
   env:
     BUILDPIPE_PROJECT_LABEL: project2
     BUILDPIPE_PROJECT_PATH: project2/
@@ -93,6 +106,7 @@ steps:
     BUILDPIPE_PROJECT_PATH: project2/
     BUILDPIPE_SCOPE: project
     TEST_ENV_PIPELINE: test-pipeline
+  key: deploy-stg project2
   label: deploy-stg project2
 - wait
 - block: ':rocket: Release!'
@@ -104,6 +118,8 @@ steps:
   - make deploy-prod
   concurrency: 1
   concurrency_group: deploy-prd
+  depends_on:
+  - deploy-stg project2
   env:
     BUILDPIPE_PROJECT_LABEL: project2
     BUILDPIPE_PROJECT_PATH: project2/