Add support for specific Docker tag in spawn_data_pipeline.py

infra/spawn_data_pipeline.py

@@ -69,6 +69,8 @@ def main():
 
     id_mapping = {}
 
+    docker_tag = os.getenv("TAG", None)
+
     # First pass, do the template rendering and dependencies resolution
     tasks = []
 
@@ -105,6 +107,24 @@ def main():
 
         payload["dependencies"] = new_dependencies
 
+        # Override the Docker image tag if needed
+        if docker_tag:
+            base_image = payload["payload"]["image"]
+            splitted_image = base_image.rsplit(":", 1)
+
+            # If we have already a Docker tag
+            if len(splitted_image) > 1:
+                docker_tag = splitted_image
+
+                if docker_tag != "latest":
+                    msg = "Don't update image {} for task {} as its already has a tag"
+                    print(msg.format(base_image, task_internal_id))
+
+            tagless_image = splitted_image[0]
+
+            new_image = "{}:{}".format(tagless_image, docker_tag)
+            payload["payload"]["image"] = new_image
+
         tasks.append((task_id, payload))
 
     # Now sends them