Add support for pulling the image secret (kubeflow#88)

* initial code

* address comments

* address comments

* generate service account name based on pipelinerun name

* re-generate the yaml

sdk/python/kfp_tekton/compiler/compiler.py

@@ -388,6 +388,21 @@ def _create_pipeline_workflow(self, args, pipeline, op_transformers=None, pipeli
       if pipeline_conf.timeout:
         pipelinerun['spec']['timeout'] = '%ds' % pipeline_conf.timeout
 
+      # generate the Tekton service account template
+      service_template = {}
+      if len(pipeline_conf.image_pull_secrets) > 0:
+        service_template = {
+          'apiVersion': 'v1',
+          'kind': 'ServiceAccount',
+          'metadata': {'name': pipelinerun['metadata']['name'] + '-sa'}
+        }
+      for image_pull_secret in pipeline_conf.image_pull_secrets:
+        service_template['imagePullSecrets'] = [{'name': image_pull_secret.name}]
+
+      if service_template:
+        workflow = workflow + [service_template]
+        pipelinerun['spec']['serviceAccountName'] = service_template['metadata']['name']
+
       workflow = workflow + [pipelinerun]
 
     # Use regex to replace all the Argo variables to Tekton variables. For variables that are unique to Argo,

sdk/python/tests/compiler/compiler_tests.py

@@ -165,6 +165,13 @@ def test_katib_workflow(self):
     """
     from .testdata.katib import mnist_hpo
     self._test_pipeline_workflow(mnist_hpo, 'katib.yaml')
+
+  def test_imagepullsecrets_workflow(self):
+    """ 
+    Test compiling a imagepullsecrets workflow.
+    """
+    from .testdata.imagepullsecrets import imagepullsecrets_pipeline
+    self._test_pipeline_workflow(imagepullsecrets_pipeline, 'imagepullsecrets.yaml', generate_pipelinerun=True)
 
   def _test_pipeline_workflow(self, 
                               pipeline_function,

sdk/python/tests/compiler/testdata/imagepullsecrets.py

@@ -0,0 +1,61 @@
+# Copyright 2020 kubeflow.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Toy example demonstrating how to specify imagepullsecrets to access protected
+container registry.
+"""
+
+import kfp
+import kfp.dsl as dsl
+from kubernetes import client as k8s_client
+
+
+class GetFrequentWordOp(dsl.ContainerOp):
+  """A get frequent word class representing a component in ML Pipelines.
+
+  The class provides a nice interface to users by hiding details such as container,
+  command, arguments.
+  """
+  def __init__(self, name, message):
+    """Args:
+         name: An identifier of the step which needs to be unique within a pipeline.
+         message: a dsl.PipelineParam object representing an input message.
+    """
+    super(GetFrequentWordOp, self).__init__(
+        name=name,
+        image='python:3.5-jessie',
+        command=['sh', '-c'],
+        arguments=['python -c "from collections import Counter; '
+                   'words = Counter(\'%s\'.split()); print(max(words, key=words.get))" '
+                   '| tee /tmp/message.txt' % message],
+        file_outputs={'word': '/tmp/message.txt'})
+
+@dsl.pipeline(
+  name='Save Most Frequent',
+  description='Get Most Frequent Word and Save to GCS'
+)
+# def save_most_frequent_word(message: str):
+def imagepullsecrets_pipeline(message="This is testing"):
+  """A pipeline function describing the orchestration of the workflow."""
+
+  counter = GetFrequentWordOp(
+          name='get-Frequent',
+          message=message)
+  # Call set_image_pull_secrets after get_pipeline_conf().
+  dsl.get_pipeline_conf()\
+    .set_image_pull_secrets([k8s_client.V1ObjectReference(name="secretA")])
+
+if __name__ == '__main__':
+  # don't use top-level import of TektonCompiler to prevent monkey-patching KFP compiler when using KFP's dsl-compile
+  from kfp_tekton.compiler import TektonCompiler
+  TektonCompiler().compile(imagepullsecrets_pipeline, __file__.replace('.py', '.yaml'), generate_pipelinerun=True)

sdk/python/tests/compiler/testdata/imagepullsecrets.yaml

@@ -0,0 +1,72 @@
+# Copyright 2020 kubeflow.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: get-frequent
+spec:
+  params:
+  - name: message
+  results:
+  - description: /tmp/message.txt
+    name: word
+  steps:
+  - args:
+    - python -c "from collections import Counter; words = Counter('$(inputs.params.message)'.split());
+      print(max(words, key=words.get))" | tee $(results.word.path)
+    command:
+    - sh
+    - -c
+    image: python:3.5-jessie
+    name: get-frequent
+---
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word
+      and Save to GCS", "inputs": [{"default": "This is testing", "name": "message",
+      "optional": true}], "name": "Save Most Frequent"}'
+  name: save-most-frequent
+spec:
+  params:
+  - default: This is testing
+    name: message
+  tasks:
+  - name: get-frequent
+    params:
+    - name: message
+      value: $(params.message)
+    taskRef:
+      name: get-frequent
+---
+apiVersion: v1
+imagePullSecrets:
+- name: secretA
+kind: ServiceAccount
+metadata:
+  name: save-most-frequent-run-sa
+---
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: save-most-frequent-run
+spec:
+  params:
+  - name: message
+    value: This is testing
+  pipelineRef:
+    name: save-most-frequent
+  serviceAccountName: save-most-frequent-run-sa