diff --git a/src/codeflare_sdk/templates/base-template.yaml b/src/codeflare_sdk/templates/base-template.yaml index 960d7383..d2c6074f 100644 --- a/src/codeflare_sdk/templates/base-template.yaml +++ b/src/codeflare_sdk/templates/base-template.yaml @@ -157,6 +157,18 @@ spec: - name: server-cert mountPath: "/home/ray/workspace/tls" readOnly: true + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt initContainers: - command: - sh @@ -181,6 +193,20 @@ spec: optional: false - name: server-cert emptyDir: {} + - name: odh-trusted-ca-cert + configMap: + name: odh-trusted-ca-bundle + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + optional: true + - name: odh-ca-cert + configMap: + name: odh-trusted-ca-bundle + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + optional: true workerGroupSpecs: # the pod replicas in this group typed worker - replicas: 3 @@ -277,6 +303,18 @@ spec: - name: server-cert mountPath: "/home/ray/workspace/tls" readOnly: true + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt volumes: - name: ca-vol secret: @@ -284,6 +322,20 @@ spec: optional: false - name: server-cert emptyDir: {} + - name: odh-trusted-ca-cert + configMap: + name: odh-trusted-ca-bundle + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + optional: true + - name: odh-ca-cert + configMap: + name: odh-trusted-ca-bundle + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + optional: true - replicas: 1 generictemplate: apiVersion: networking.k8s.io/v1 diff --git a/src/codeflare_sdk/utils/generate_yaml.py b/src/codeflare_sdk/utils/generate_yaml.py index bfbce859..49380523 100755 --- a/src/codeflare_sdk/utils/generate_yaml.py +++ b/src/codeflare_sdk/utils/generate_yaml.py @@ -17,6 +17,7 @@ (in the cluster sub-module) for AppWrapper generation. """ +import typing import yaml import sys import os @@ -466,35 +467,38 @@ def enable_local_interactive(resources, cluster_name, namespace, ingress_domain) ][0].get("command")[2] = command +def del_from_list_by_name(l: list, target: typing.List[str]) -> list: + return [x for x in l if x["name"] not in target] + + def disable_raycluster_tls(resources): generic_template_spec = resources["GenericItems"][0]["generictemplate"]["spec"] - if "volumes" in generic_template_spec["headGroupSpec"]["template"]["spec"]: - del generic_template_spec["headGroupSpec"]["template"]["spec"]["volumes"] + headGroupTemplateSpec = generic_template_spec["headGroupSpec"]["template"]["spec"] + headGroupTemplateSpec["volumes"] = del_from_list_by_name( + headGroupTemplateSpec.get("volumes", []), + ["ca-vol", "server-cert"], + ) - if ( - "volumeMounts" - in generic_template_spec["headGroupSpec"]["template"]["spec"]["containers"][0] - ): - del generic_template_spec["headGroupSpec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + c: dict + for c in generic_template_spec["headGroupSpec"]["template"]["spec"]["containers"]: + c["volumeMounts"] = del_from_list_by_name( + c.get("volumeMounts", []), ["ca-vol", "server-cert"] + ) if "initContainers" in generic_template_spec["headGroupSpec"]["template"]["spec"]: del generic_template_spec["headGroupSpec"]["template"]["spec"]["initContainers"] - if "volumes" in generic_template_spec["workerGroupSpecs"][0]["template"]["spec"]: - del generic_template_spec["workerGroupSpecs"][0]["template"]["spec"]["volumes"] - - if ( - "volumeMounts" - in generic_template_spec["workerGroupSpecs"][0]["template"]["spec"][ - "containers" - ][0] - ): - del generic_template_spec["workerGroupSpecs"][0]["template"]["spec"][ - "containers" - ][0]["volumeMounts"] + for workerGroup in generic_template_spec.get("workerGroupSpecs", []): + workerGroupSpec = workerGroup["template"]["spec"] + workerGroupSpec["volumes"] = del_from_list_by_name( + workerGroupSpec.get("volumes", []), + ["ca-vol", "server-cert"], + ) + for c in workerGroup["template"]["spec"].get("containers", []): + c["volumeMounts"] = del_from_list_by_name( + c.get("volumeMounts", []), ["ca-vol", "server-cert"] + ) del generic_template_spec["workerGroupSpecs"][0]["template"]["spec"][ "initContainers" diff --git a/tests/test-case-no-mcad.yamls b/tests/test-case-no-mcad.yamls index 38109dfc..299ff9a8 100644 --- a/tests/test-case-no-mcad.yamls +++ b/tests/test-case-no-mcad.yamls @@ -77,8 +77,36 @@ spec: cpu: 2 memory: 8G nvidia.com/gpu: 0 + volumeMounts: + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt imagePullSecrets: - name: unit-test-pull-secret + volumes: + - configMap: + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-trusted-ca-cert + - configMap: + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-ca-cert rayVersion: 2.7.0 workerGroupSpecs: - groupName: small-group-unit-test-cluster-ray @@ -136,8 +164,36 @@ spec: cpu: 3 memory: 5G nvidia.com/gpu: 7 + volumeMounts: + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt imagePullSecrets: - name: unit-test-pull-secret + volumes: + - configMap: + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-trusted-ca-cert + - configMap: + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-ca-cert --- apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/tests/test-case-prio.yaml b/tests/test-case-prio.yaml index 9f907135..1e72c442 100644 --- a/tests/test-case-prio.yaml +++ b/tests/test-case-prio.yaml @@ -107,9 +107,37 @@ spec: cpu: 2 memory: 8G nvidia.com/gpu: 0 + volumeMounts: + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt imagePullSecrets: - name: unit-test-pull-secret priorityClassName: default + volumes: + - configMap: + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-trusted-ca-cert + - configMap: + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-ca-cert rayVersion: 2.7.0 workerGroupSpecs: - groupName: small-group-prio-test-cluster @@ -167,9 +195,37 @@ spec: cpu: 3 memory: 5G nvidia.com/gpu: 7 + volumeMounts: + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt imagePullSecrets: - name: unit-test-pull-secret priorityClassName: default + volumes: + - configMap: + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-trusted-ca-cert + - configMap: + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-ca-cert replicas: 1 - generictemplate: apiVersion: networking.k8s.io/v1 diff --git a/tests/test-case.yaml b/tests/test-case.yaml index f7e287cf..76285209 100644 --- a/tests/test-case.yaml +++ b/tests/test-case.yaml @@ -106,8 +106,36 @@ spec: cpu: 2 memory: 8G nvidia.com/gpu: 0 + volumeMounts: + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt imagePullSecrets: - name: unit-test-pull-secret + volumes: + - configMap: + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-trusted-ca-cert + - configMap: + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-ca-cert rayVersion: 2.7.0 workerGroupSpecs: - groupName: small-group-unit-test-cluster @@ -165,8 +193,36 @@ spec: cpu: 3 memory: 5G nvidia.com/gpu: 7 + volumeMounts: + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt imagePullSecrets: - name: unit-test-pull-secret + volumes: + - configMap: + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-trusted-ca-cert + - configMap: + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-ca-cert replicas: 1 - generictemplate: apiVersion: networking.k8s.io/v1 diff --git a/tests/unit_test.py b/tests/unit_test.py index fc9ecde2..3fde4960 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -2709,10 +2709,28 @@ def test_enable_local_interactive(mocker): volumes = [ { "name": "ca-vol", - "secret": {"secretName": f"ca-secret-{cluster_name}"}, + "secret": {"secretName": "ca-secret-test-enable-local"}, "optional": False, }, {"name": "server-cert", "emptyDir": {}}, + { + "name": "odh-trusted-ca-cert", + "configMap": { + "name": "odh-trusted-ca-bundle", + "items": [ + {"key": "ca-bundle.crt", "path": "odh-trusted-ca-bundle.crt"} + ], + "optional": True, + }, + }, + { + "name": "odh-ca-cert", + "configMap": { + "name": "odh-trusted-ca-bundle", + "items": [{"key": "odh-ca-bundle.crt", "path": "odh-ca-bundle.crt"}], + "optional": True, + }, + }, ] tls_env = [ {"name": "RAY_USE_TLS", "value": "1"},