New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(Backend + SDK): Update kfp backend and kubernetes sdk to support ConfigMaps as volumes and as env variables #10483
Changes from 7 commits
d42bca0
dcde0c0
b395d10
03015f3
c703e3d
ca26f84
8dff487
fc44104
e989d04
811d887
36daba6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
# Copyright 2023 The Kubeflow Authors | ||
roytman marked this conversation as resolved.
Show resolved
Hide resolved
|
||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
from typing import Dict | ||
|
||
from google.protobuf import json_format | ||
from kfp.dsl import PipelineTask | ||
from kfp.kubernetes import common | ||
from kfp.kubernetes import kubernetes_executor_config_pb2 as pb | ||
|
||
|
||
def use_config_map_as_env( | ||
task: PipelineTask, | ||
config_map_name: str, | ||
config_map_key_to_env: Dict[str, str], | ||
) -> PipelineTask: | ||
"""Use a Kubernetes ConfigMap as an environment variable as described in | ||
https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we style the link using RST for clean rendering in the reference docs? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. done, thanks |
||
|
||
Args: | ||
task: Pipeline task. | ||
config_map_name: Name of the ConfigMap. | ||
config_map_key_to_env: Dictionary of ConfigMap key to environment variable name. For example, ``{'foo': 'FOO'}`` sets the value of the ConfigMap's foo field to the environment variable ``FOO``. | ||
|
||
Returns: | ||
Task object with updated ConfigMap configuration. | ||
""" | ||
|
||
msg = common.get_existing_kubernetes_config_as_message(task) | ||
|
||
key_to_env = [ | ||
pb.ConfigMapAsEnv.ConfigMapKeyToEnvMap( | ||
config_map_key=config_map_key, | ||
env_var=env_var, | ||
) for config_map_key, env_var in config_map_key_to_env.items() | ||
] | ||
config_map_as_env = pb.ConfigMapAsEnv( | ||
config_map_name=config_map_name, | ||
key_to_env=key_to_env, | ||
) | ||
|
||
msg.config_map_as_env.append(config_map_as_env) | ||
|
||
task.platform_config['kubernetes'] = json_format.MessageToDict(msg) | ||
|
||
return task | ||
|
||
|
||
def use_config_map_as_volume( | ||
task: PipelineTask, | ||
config_map_name: str, | ||
mount_path: str, | ||
) -> PipelineTask: | ||
"""Use a Kubernetes ConfigMap by mounting its data to the task's container as | ||
described by the `Kubernetes documentation <https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#add-configmap-data-to-a-volume>`_. | ||
|
||
Args: | ||
task: Pipeline task. | ||
config_map_name: Name of the ConfigMap. | ||
mount_path: Path to which to mount the ConfigMap data. | ||
|
||
Returns: | ||
Task object with updated ConfigMap configuration. | ||
""" | ||
|
||
msg = common.get_existing_kubernetes_config_as_message(task) | ||
|
||
config_map_as_vol = pb.ConfigMapAsVolume( | ||
config_map_name=config_map_name, | ||
mount_path=mount_path, | ||
) | ||
msg.config_map_as_volume.append(config_map_as_vol) | ||
|
||
task.platform_config['kubernetes'] = json_format.MessageToDict(msg) | ||
|
||
return task |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
# Copyright 2023 The Kubeflow Authors | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
from kfp import dsl | ||
from kfp import kubernetes | ||
|
||
|
||
@dsl.component | ||
def comp(): | ||
pass | ||
|
||
|
||
@dsl.pipeline | ||
def my_pipeline(): | ||
task = comp() | ||
kubernetes.use_config_map_as_env( | ||
task, | ||
config_map_name='my-cm', | ||
config_map_key_to_env={'foo': 'CONFIG_MAP_VAR'}) | ||
|
||
|
||
if __name__ == '__main__': | ||
from kfp import compiler | ||
compiler.Compiler().compile(my_pipeline, __file__.replace('.py', '.yaml')) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: maybe we can refactor these two for loops into functions so that we can also reuse them for secret mount and secret env