Skip to content
This repository has been archived by the owner on Mar 11, 2022. It is now read-only.

Commit

Permalink
Add labels and annotations to Elyra pods (#83)
Browse files Browse the repository at this point in the history
Properly label Elyra pods to enable cluster administrators 
to identify which pods are associated with Elyra pipelines (in general)
as a well as enabling correlation with the artifact associated with the pod

Labels metadata:
elyra-node-type: static string identifying the node type as notebook-script, 
which covers file-based Elyra container ops for notebooks and Python scripts

elyra-pipeline-name: the pipeline's experiment name, 
e.g. my-labeled-pipeline3-0208165033 which is not the same as the pipeline name

elyra-node-name: the pipeline node name (see also elyra-ai/elyra#971)

Annotations metadata:
elyra-node-file-name: the notebook or Python script name, e.g. notebook1.ipynb
  • Loading branch information
ptitzler committed Feb 22, 2021
1 parent b361f2e commit 21ebe68
Show file tree
Hide file tree
Showing 2 changed files with 291 additions and 8 deletions.
100 changes: 94 additions & 6 deletions kfp_notebook/pipeline/_notebook_op.py
Expand Up @@ -16,6 +16,7 @@
#

import os
import string

from kfp.dsl import ContainerOp
from kfp_notebook import __version__
Expand Down Expand Up @@ -52,23 +53,28 @@
class NotebookOp(ContainerOp):

def __init__(self,
pipeline_name: str,
experiment_name: str,
notebook: str,
cos_endpoint: str,
cos_bucket: str,
cos_directory: str,
cos_dependencies_archive: str,
pipeline_version: Optional[str] = '',
pipeline_outputs: Optional[List[str]] = None,
pipeline_inputs: Optional[List[str]] = None,
pipeline_envs: Optional[Dict[str, str]] = None,
requirements_url: str = None,
bootstrap_script_url: str = None,
emptydir_volume_size: str = None,
cpu_request: str = None,
mem_request: str = None,
gpu_limit: str = None,
requirements_url: Optional[str] = None,
bootstrap_script_url: Optional[str] = None,
emptydir_volume_size: Optional[str] = None,
cpu_request: Optional[str] = None,
mem_request: Optional[str] = None,
gpu_limit: Optional[str] = None,
**kwargs):
"""Create a new instance of ContainerOp.
Args:
pipeline_name: pipeline that this op belongs to
experiment_name: the experiment where pipeline_name is executed
notebook: name of the notebook that will be executed per this operation
cos_endpoint: object storage endpoint e.g weaikish1.fyre.ibm.com:30442
cos_bucket: bucket to retrieve archive from
Expand All @@ -88,6 +94,9 @@ def __init__(self,
https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.dsl.html#kfp.dsl.ContainerOp
"""

self.pipeline_name = pipeline_name
self.pipeline_version = pipeline_version
self.experiment_name = experiment_name
self.notebook = notebook
self.notebook_name = os.path.basename(notebook)
self.cos_endpoint = cos_endpoint
Expand Down Expand Up @@ -228,6 +237,27 @@ def __init__(self,
gpu_vendor = self.pipeline_envs.get('GPU_VENDOR', 'nvidia')
self.container.set_gpu_limit(gpu=str(gpu_limit), vendor=gpu_vendor)

# Attach metadata to the pod
# Node type (a static type for this op)
self.add_pod_label('elyra/node-type',
NotebookOp._normalize_label_value(
'notebook-script'))
# Pipeline name
self.add_pod_label('elyra/pipeline-name',
NotebookOp._normalize_label_value(self.pipeline_name))
# Pipeline version
self.add_pod_label('elyra/pipeline-version',
NotebookOp._normalize_label_value(self.pipeline_version))
# Experiment name
self.add_pod_label('elyra/experiment-name',
NotebookOp._normalize_label_value(self.experiment_name))
# Pipeline node name
self.add_pod_label('elyra/node-name',
NotebookOp._normalize_label_value(kwargs.get('name')))
# Pipeline node file
self.add_pod_annotation('elyra/node-file-name',
self.notebook)

def _artifact_list_to_str(self, pipeline_array):
trimmed_artifact_list = []
for artifact_name in pipeline_array:
Expand All @@ -236,3 +266,61 @@ def _artifact_list_to_str(self, pipeline_array):
ValueError("Illegal character ({}) found in filename '{}'.".format(INOUT_SEPARATOR, artifact_name))
trimmed_artifact_list.append(artifact_name.strip())
return INOUT_SEPARATOR.join(trimmed_artifact_list)

@staticmethod
def _normalize_label_value(value):

"""Produce a Kubernetes-compliant label from value
Valid label values must be 63 characters or less and
must be empty or begin and end with an alphanumeric
character ([a-z0-9A-Z]) with dashes (-), underscores
(_), dots (.), and alphanumerics between.
"""

if value is None or len(value) == 0:
return '' # nothing to do

max_length = 63
# This char is added at the front and/or back
# of value, if the first and/or last character
# is invalid. For example a value of "-abc"
# is converted to "a-abc". The specified character
# must meet the label value constraints.
valid_char = 'a'
# This char is used to replace invalid characters
# that are in the "middle" of value. For example
# a value of "abc%def" is converted to "abc_def".
# The specified character must meet the label value
# constraints.
valid_middle_char = '_'

# must begin with [0-9a-zA-Z]
valid_chars = string.ascii_letters + string.digits
if value[0] not in valid_chars:
value = valid_char + value

value = value[:max_length] # enforce max length

# must end with [0-9a-zA-Z]
if value[-1] not in valid_chars:
if len(value) <= max_length - 1:
# append valid character if max length
# would not be exceeded
value = value + valid_char
else:
# replace with valid character
value = value[:-1] + valid_char

# middle chars must be [0-9a-zA-Z\-_.]
valid_chars = valid_chars + '-_.'

newstr = ''
for c in range(len(value)):
if value[c] not in valid_chars:
newstr = newstr + valid_middle_char
else:
newstr = newstr + value[c]
value = newstr

return value

0 comments on commit 21ebe68

Please sign in to comment.