From cf8e99866d29299c908a8ec907f2636a17a8736e Mon Sep 17 00:00:00 2001 From: Kevin Yu Date: Fri, 10 Apr 2020 11:31:47 -0700 Subject: [PATCH] Add support for pulling the image secret (#88) * initial code * address comments * address comments * generate service account name based on pipelinerun name * re-generate the yaml --- sdk/python/kfp_tekton/compiler/compiler.py | 15 ++++ sdk/python/tests/compiler/compiler_tests.py | 7 ++ .../compiler/testdata/imagepullsecrets.py | 61 ++++++++++++++++ .../compiler/testdata/imagepullsecrets.yaml | 72 +++++++++++++++++++ 4 files changed, 155 insertions(+) create mode 100644 sdk/python/tests/compiler/testdata/imagepullsecrets.py create mode 100644 sdk/python/tests/compiler/testdata/imagepullsecrets.yaml diff --git a/sdk/python/kfp_tekton/compiler/compiler.py b/sdk/python/kfp_tekton/compiler/compiler.py index 9767fdaa81f..af0e1962dde 100644 --- a/sdk/python/kfp_tekton/compiler/compiler.py +++ b/sdk/python/kfp_tekton/compiler/compiler.py @@ -388,6 +388,21 @@ def _create_pipeline_workflow(self, args, pipeline, op_transformers=None, pipeli if pipeline_conf.timeout: pipelinerun['spec']['timeout'] = '%ds' % pipeline_conf.timeout + # generate the Tekton service account template + service_template = {} + if len(pipeline_conf.image_pull_secrets) > 0: + service_template = { + 'apiVersion': 'v1', + 'kind': 'ServiceAccount', + 'metadata': {'name': pipelinerun['metadata']['name'] + '-sa'} + } + for image_pull_secret in pipeline_conf.image_pull_secrets: + service_template['imagePullSecrets'] = [{'name': image_pull_secret.name}] + + if service_template: + workflow = workflow + [service_template] + pipelinerun['spec']['serviceAccountName'] = service_template['metadata']['name'] + workflow = workflow + [pipelinerun] # Use regex to replace all the Argo variables to Tekton variables. For variables that are unique to Argo, diff --git a/sdk/python/tests/compiler/compiler_tests.py b/sdk/python/tests/compiler/compiler_tests.py index 21f0fcba786..cb880854d57 100644 --- a/sdk/python/tests/compiler/compiler_tests.py +++ b/sdk/python/tests/compiler/compiler_tests.py @@ -165,6 +165,13 @@ def test_katib_workflow(self): """ from .testdata.katib import mnist_hpo self._test_pipeline_workflow(mnist_hpo, 'katib.yaml') + + def test_imagepullsecrets_workflow(self): + """ + Test compiling a imagepullsecrets workflow. + """ + from .testdata.imagepullsecrets import imagepullsecrets_pipeline + self._test_pipeline_workflow(imagepullsecrets_pipeline, 'imagepullsecrets.yaml', generate_pipelinerun=True) def _test_pipeline_workflow(self, pipeline_function, diff --git a/sdk/python/tests/compiler/testdata/imagepullsecrets.py b/sdk/python/tests/compiler/testdata/imagepullsecrets.py new file mode 100644 index 00000000000..2721e39ca9d --- /dev/null +++ b/sdk/python/tests/compiler/testdata/imagepullsecrets.py @@ -0,0 +1,61 @@ +# Copyright 2020 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Toy example demonstrating how to specify imagepullsecrets to access protected +container registry. +""" + +import kfp +import kfp.dsl as dsl +from kubernetes import client as k8s_client + + +class GetFrequentWordOp(dsl.ContainerOp): + """A get frequent word class representing a component in ML Pipelines. + + The class provides a nice interface to users by hiding details such as container, + command, arguments. + """ + def __init__(self, name, message): + """Args: + name: An identifier of the step which needs to be unique within a pipeline. + message: a dsl.PipelineParam object representing an input message. + """ + super(GetFrequentWordOp, self).__init__( + name=name, + image='python:3.5-jessie', + command=['sh', '-c'], + arguments=['python -c "from collections import Counter; ' + 'words = Counter(\'%s\'.split()); print(max(words, key=words.get))" ' + '| tee /tmp/message.txt' % message], + file_outputs={'word': '/tmp/message.txt'}) + +@dsl.pipeline( + name='Save Most Frequent', + description='Get Most Frequent Word and Save to GCS' +) +# def save_most_frequent_word(message: str): +def imagepullsecrets_pipeline(message="This is testing"): + """A pipeline function describing the orchestration of the workflow.""" + + counter = GetFrequentWordOp( + name='get-Frequent', + message=message) + # Call set_image_pull_secrets after get_pipeline_conf(). + dsl.get_pipeline_conf()\ + .set_image_pull_secrets([k8s_client.V1ObjectReference(name="secretA")]) + +if __name__ == '__main__': + # don't use top-level import of TektonCompiler to prevent monkey-patching KFP compiler when using KFP's dsl-compile + from kfp_tekton.compiler import TektonCompiler + TektonCompiler().compile(imagepullsecrets_pipeline, __file__.replace('.py', '.yaml'), generate_pipelinerun=True) diff --git a/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml b/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml new file mode 100644 index 00000000000..360b6a655fe --- /dev/null +++ b/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml @@ -0,0 +1,72 @@ +# Copyright 2020 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: get-frequent +spec: + params: + - name: message + results: + - description: /tmp/message.txt + name: word + steps: + - args: + - python -c "from collections import Counter; words = Counter('$(inputs.params.message)'.split()); + print(max(words, key=words.get))" | tee $(results.word.path) + command: + - sh + - -c + image: python:3.5-jessie + name: get-frequent +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word + and Save to GCS", "inputs": [{"default": "This is testing", "name": "message", + "optional": true}], "name": "Save Most Frequent"}' + name: save-most-frequent +spec: + params: + - default: This is testing + name: message + tasks: + - name: get-frequent + params: + - name: message + value: $(params.message) + taskRef: + name: get-frequent +--- +apiVersion: v1 +imagePullSecrets: +- name: secretA +kind: ServiceAccount +metadata: + name: save-most-frequent-run-sa +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: save-most-frequent-run +spec: + params: + - name: message + value: This is testing + pipelineRef: + name: save-most-frequent + serviceAccountName: save-most-frequent-run-sa