From 37d25141c876d2e4d07a167a0ac89e3314bf196b Mon Sep 17 00:00:00 2001 From: sauagarwa Date: Tue, 3 Dec 2024 14:16:56 -0500 Subject: [PATCH 1/2] Reverting back to using jobs for fixing race condition. The issue has been fixed in rhods 2.16 --- .../templates/accelerator-profile.yaml | 32 ++-- .../templates/inference-service.yaml | 72 ++++---- .../templates/serving-runtime.yaml | 138 +++++++------- .../templates/serving-service-setup.yaml | 169 ++++++++++++++++++ charts/all/rhods/templates/dsc.yaml | 60 +++---- .../all/rhods/templates/job-create-dsc.yaml | 134 ++++++++++++++ charts/all/rhods/templates/sub-rhods.yaml | 12 ++ values-hub.yaml | 135 +------------- 8 files changed, 474 insertions(+), 278 deletions(-) create mode 100644 charts/all/llm-serving-service/templates/serving-service-setup.yaml create mode 100644 charts/all/rhods/templates/job-create-dsc.yaml create mode 100644 charts/all/rhods/templates/sub-rhods.yaml diff --git a/charts/all/llm-serving-service/templates/accelerator-profile.yaml b/charts/all/llm-serving-service/templates/accelerator-profile.yaml index 212b7a84..f8be751c 100644 --- a/charts/all/llm-serving-service/templates/accelerator-profile.yaml +++ b/charts/all/llm-serving-service/templates/accelerator-profile.yaml @@ -1,16 +1,16 @@ ---- -apiVersion: dashboard.opendatahub.io/v1 -kind: AcceleratorProfile -metadata: - name: nvidia-gpu - namespace: redhat-ods-applications - annotations: - argocd.argoproj.io/sync-wave: "40" -spec: - displayName: NVIDIA GPU - enabled: true - identifier: nvidia.com/gpu - tolerations: - - effect: NoSchedule - key: odh-notebook - operator: Exists \ No newline at end of file +# --- +# apiVersion: dashboard.opendatahub.io/v1 +# kind: AcceleratorProfile +# metadata: +# name: nvidia-gpu +# namespace: redhat-ods-applications +# annotations: +# argocd.argoproj.io/sync-wave: "40" +# spec: +# displayName: NVIDIA GPU +# enabled: true +# identifier: nvidia.com/gpu +# tolerations: +# - effect: NoSchedule +# key: odh-notebook +# operator: Exists \ No newline at end of file diff --git a/charts/all/llm-serving-service/templates/inference-service.yaml b/charts/all/llm-serving-service/templates/inference-service.yaml index 0bfe9eb4..55142575 100644 --- a/charts/all/llm-serving-service/templates/inference-service.yaml +++ b/charts/all/llm-serving-service/templates/inference-service.yaml @@ -1,36 +1,36 @@ -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - annotations: - openshift.io/display-name: mistral-7b-instruct - serving.knative.openshift.io/enablePassthrough: 'true' - sidecar.istio.io/inject: 'true' - sidecar.istio.io/rewriteAppHTTPProbers: 'true' - argocd.argoproj.io/sync-wave: "60" - name: mistral-7b-instruct - namespace: rag-llm - labels: - opendatahub.io/dashboard: 'true' -spec: - predictor: - restartPolicy: OnFailure - maxReplicas: 1 - minReplicas: 1 - model: - modelFormat: - name: vLLM - name: '' - resources: - limits: - cpu: '8' - memory: 10Gi - nvidia.com/gpu: '1' - requests: - cpu: '2' - memory: 8Gi - nvidia.com/gpu: '1' - runtime: mistral-7b-instruct - tolerations: - - effect: NoSchedule - key: odh-notebook - operator: Exists \ No newline at end of file +# apiVersion: serving.kserve.io/v1beta1 +# kind: InferenceService +# metadata: +# annotations: +# openshift.io/display-name: mistral-7b-instruct +# serving.knative.openshift.io/enablePassthrough: 'true' +# sidecar.istio.io/inject: 'true' +# sidecar.istio.io/rewriteAppHTTPProbers: 'true' +# argocd.argoproj.io/sync-wave: "60" +# name: mistral-7b-instruct +# namespace: rag-llm +# labels: +# opendatahub.io/dashboard: 'true' +# spec: +# predictor: +# restartPolicy: OnFailure +# maxReplicas: 1 +# minReplicas: 1 +# model: +# modelFormat: +# name: vLLM +# name: '' +# resources: +# limits: +# cpu: '8' +# memory: 10Gi +# nvidia.com/gpu: '1' +# requests: +# cpu: '2' +# memory: 8Gi +# nvidia.com/gpu: '1' +# runtime: mistral-7b-instruct +# tolerations: +# - effect: NoSchedule +# key: odh-notebook +# operator: Exists \ No newline at end of file diff --git a/charts/all/llm-serving-service/templates/serving-runtime.yaml b/charts/all/llm-serving-service/templates/serving-runtime.yaml index b5065204..d5f575c8 100644 --- a/charts/all/llm-serving-service/templates/serving-runtime.yaml +++ b/charts/all/llm-serving-service/templates/serving-runtime.yaml @@ -1,69 +1,69 @@ -apiVersion: serving.kserve.io/v1alpha1 -kind: ServingRuntime -metadata: - annotations: - opendatahub.io/accelerator-name: nvidia-gpu - opendatahub.io/apiProtocol: REST - opendatahub.io/recommended-accelerators: '["nvidia.com/gpu"]' - openshift.io/display-name: mistral-7b-instruct - argocd.argoproj.io/sync-wave: "50" - name: mistral-7b-instruct - namespace: rag-llm - labels: - opendatahub.io/dashboard: 'true' -spec: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: '8080' - containers: - - args: - - '--port=8080' - - '--model=$(MODEL_ID)' - - '--download-dir=/cache' - - '--distributed-executor-backend=mp' - - '--served-model-name=mistral-7b-instruct' - - '--max-model-len=4096' - - '--dtype=half' - - '--gpu-memory-utilization' - - '0.98' - - '--enforce-eager' - command: - - python - - '-m' - - vllm.entrypoints.openai.api_server - env: - - name: HF_HOME - value: /tmp/hf_home - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: huggingface-secret - key: hftoken - - name: MODEL_ID - valueFrom: - secretKeyRef: - name: huggingface-secret - key: modelId - - name: HF_HUB_OFFLINE - value: "0" - image: 'quay.io/modh/vllm@sha256:b51fde66f162f1a78e8c027320dddf214732d5345953b1599a84fe0f0168c619' - name: kserve-container - ports: - - containerPort: 8080 - protocol: TCP - volumeMounts: - - mountPath: /dev/shm - name: shm - - mountPath: /cache - name: cache - multiModel: false - supportedModelFormats: - - autoSelect: true - name: vLLM - volumes: - - emptyDir: - medium: Memory - sizeLimit: 2Gi - name: shm - - emptyDir: {} - name: cache \ No newline at end of file +# apiVersion: serving.kserve.io/v1alpha1 +# kind: ServingRuntime +# metadata: +# annotations: +# opendatahub.io/accelerator-name: nvidia-gpu +# opendatahub.io/apiProtocol: REST +# opendatahub.io/recommended-accelerators: '["nvidia.com/gpu"]' +# openshift.io/display-name: mistral-7b-instruct +# argocd.argoproj.io/sync-wave: "50" +# name: mistral-7b-instruct +# namespace: rag-llm +# labels: +# opendatahub.io/dashboard: 'true' +# spec: +# annotations: +# prometheus.io/path: /metrics +# prometheus.io/port: '8080' +# containers: +# - args: +# - '--port=8080' +# - '--model=$(MODEL_ID)' +# - '--download-dir=/cache' +# - '--distributed-executor-backend=mp' +# - '--served-model-name=mistral-7b-instruct' +# - '--max-model-len=4096' +# - '--dtype=half' +# - '--gpu-memory-utilization' +# - '0.98' +# - '--enforce-eager' +# command: +# - python +# - '-m' +# - vllm.entrypoints.openai.api_server +# env: +# - name: HF_HOME +# value: /tmp/hf_home +# - name: HF_TOKEN +# valueFrom: +# secretKeyRef: +# name: huggingface-secret +# key: hftoken +# - name: MODEL_ID +# valueFrom: +# secretKeyRef: +# name: huggingface-secret +# key: modelId +# - name: HF_HUB_OFFLINE +# value: "0" +# image: 'quay.io/modh/vllm@sha256:b51fde66f162f1a78e8c027320dddf214732d5345953b1599a84fe0f0168c619' +# name: kserve-container +# ports: +# - containerPort: 8080 +# protocol: TCP +# volumeMounts: +# - mountPath: /dev/shm +# name: shm +# - mountPath: /cache +# name: cache +# multiModel: false +# supportedModelFormats: +# - autoSelect: true +# name: vLLM +# volumes: +# - emptyDir: +# medium: Memory +# sizeLimit: 2Gi +# name: shm +# - emptyDir: {} +# name: cache \ No newline at end of file diff --git a/charts/all/llm-serving-service/templates/serving-service-setup.yaml b/charts/all/llm-serving-service/templates/serving-service-setup.yaml new file mode 100644 index 00000000..886a0d10 --- /dev/null +++ b/charts/all/llm-serving-service/templates/serving-service-setup.yaml @@ -0,0 +1,169 @@ +--- +apiVersion: dashboard.opendatahub.io/v1 +kind: AcceleratorProfile +metadata: + name: nvidia-gpu + namespace: redhat-ods-applications +spec: + displayName: NVIDIA GPU + enabled: true + identifier: nvidia.com/gpu + tolerations: + - effect: NoSchedule + key: odh-notebook + operator: Exists +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: create-vllm +spec: + selector: {} + template: + spec: + containers: + - args: + - -ec + - |- + export MINIO_SECRET=$(oc get secret minio-secret -o jsonpath='{.data.MINIO_ROOT_PASSWORD}' | base64 --decode) + export MODEL_NAME=$(echo "${modelId##*/}" | sed 's/-[^-]*$//' | tr '[:upper:]' '[:lower:]') + export MODEL_ID=$(echo "${modelId}" | awk -F'/' '{print $NF}') + cat << EOF | oc apply -f- + kind: Secret + apiVersion: v1 + metadata: + name: aws-connection-rag-llm-bucket + namespace: rag-llm + labels: + opendatahub.io/dashboard: 'true' + opendatahub.io/managed: 'true' + annotations: + opendatahub.io/connection-type: s3 + openshift.io/display-name: rag-llm-bucket + stringData: + AWS_ACCESS_KEY_ID: minio + AWS_SECRET_ACCESS_KEY: $MINIO_SECRET + AWS_DEFAULT_REGION: us-east-1 + AWS_S3_BUCKET: models + AWS_S3_ENDPOINT: http://minio-service:9000 + type: Opaque + EOF + cat << EOF | oc apply -f- + apiVersion: serving.kserve.io/v1alpha1 + kind: ServingRuntime + metadata: + annotations: + opendatahub.io/accelerator-name: nvidia-gpu + opendatahub.io/apiProtocol: REST + opendatahub.io/recommended-accelerators: '["nvidia.com/gpu"]' + opendatahub.io/template-display-name: vLLM ServingRuntime for KServe + opendatahub.io/template-name: vllm-runtime + openshift.io/display-name: $MODEL_NAME + name: $MODEL_NAME + namespace: rag-llm + labels: + opendatahub.io/dashboard: 'true' + spec: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '8080' + containers: + - args: + - '--port=8080' + - '--model=/mnt/models' + - '--served-model-name={{`{{.Name}}`}}' + - '--distributed-executor-backend=mp' + - '--max-model-len=4096' + - '--dtype=half' + - '--gpu-memory-utilization' + - '0.98' + - '--enforce-eager' + command: + - python + - '-m' + - vllm.entrypoints.openai.api_server + env: + - name: HF_HOME + value: /tmp/hf_home + image: 'quay.io/modh/vllm@sha256:b51fde66f162f1a78e8c027320dddf214732d5345953b1599a84fe0f0168c619' + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + volumeMounts: + - mountPath: /dev/shm + name: shm + multiModel: false + supportedModelFormats: + - autoSelect: true + name: vLLM + volumes: + - emptyDir: + medium: Memory + sizeLimit: 2Gi + name: shm + EOF + cat << EOF | oc apply -f- + apiVersion: serving.kserve.io/v1beta1 + kind: InferenceService + metadata: + annotations: + openshift.io/display-name: $MODEL_NAME + serving.knative.openshift.io/enablePassthrough: 'true' + sidecar.istio.io/inject: 'true' + sidecar.istio.io/rewriteAppHTTPProbers: 'true' + name: $MODEL_NAME + namespace: rag-llm + labels: + opendatahub.io/dashboard: 'true' + spec: + predictor: + restartPolicy: OnFailure + maxReplicas: 1 + minReplicas: 1 + model: + modelFormat: + name: vLLM + name: '' + resources: + limits: + cpu: '8' + memory: 10Gi + nvidia.com/gpu: '1' + requests: + cpu: '2' + memory: 8Gi + nvidia.com/gpu: '1' + runtime: $MODEL_NAME + storage: + key: aws-connection-rag-llm-bucket + path: llm-models/$MODEL_ID + tolerations: + - effect: NoSchedule + key: odh-notebook + operator: Exists + EOF + command: + - /bin/bash + image: image-registry.openshift-image-registry.svc:5000/openshift/tools:latest + imagePullPolicy: IfNotPresent + name: create-vllm + envFrom: + - secretRef: + name: minio-secret + - secretRef: + name: huggingface-secret + initContainers: + - args: + - -ec + - |- + oc wait --for=condition=complete job/load-model-set -n rag-llm --timeout=10m + sleep 10 + command: + - /bin/bash + image: image-registry.openshift-image-registry.svc:5000/openshift/tools:latest + imagePullPolicy: IfNotPresent + name: wait-for-openshift + restartPolicy: Never + serviceAccount: demo-setup + serviceAccountName: demo-setup \ No newline at end of file diff --git a/charts/all/rhods/templates/dsc.yaml b/charts/all/rhods/templates/dsc.yaml index af226cda..572b3d1a 100644 --- a/charts/all/rhods/templates/dsc.yaml +++ b/charts/all/rhods/templates/dsc.yaml @@ -1,30 +1,30 @@ -apiVersion: datasciencecluster.opendatahub.io/v1 -kind: DataScienceCluster -metadata: - name: default-dsc - annotations: - argocd.argoproj.io/sync-wave: "20" -spec: - components: - dashboard: - managementState: Managed - workbenches: - managementState: Managed - datasciencepipelines: - managementState: Managed - kueue: - managementState: Removed - codeflare: - managementState: Removed - ray: - managementState: Removed - modelmeshserving: - managementState: Managed - kserve: - managementState: Managed - serving: - ingressGateway: - certificate: - type: SelfSigned - managementState: Managed - name: knative-serving +# apiVersion: datasciencecluster.opendatahub.io/v1 +# kind: DataScienceCluster +# metadata: +# name: default-dsc +# annotations: +# argocd.argoproj.io/sync-wave: "20" +# spec: +# components: +# dashboard: +# managementState: Managed +# workbenches: +# managementState: Managed +# datasciencepipelines: +# managementState: Managed +# kueue: +# managementState: Removed +# codeflare: +# managementState: Removed +# ray: +# managementState: Removed +# modelmeshserving: +# managementState: Managed +# kserve: +# managementState: Managed +# serving: +# ingressGateway: +# certificate: +# type: SelfSigned +# managementState: Managed +# name: knative-serving diff --git a/charts/all/rhods/templates/job-create-dsc.yaml b/charts/all/rhods/templates/job-create-dsc.yaml new file mode 100644 index 00000000..10822f56 --- /dev/null +++ b/charts/all/rhods/templates/job-create-dsc.yaml @@ -0,0 +1,134 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dsc-setup +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: datasciencecluster-manager +rules: +- apiGroups: ["datasciencecluster.opendatahub.io"] + resources: ["datascienceclusters"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: datasciencecluster-manager-binding +subjects: +- kind: ServiceAccount + name: dsc-setup + namespace: redhat-ods-operator +roleRef: + kind: ClusterRole + name: datasciencecluster-manager + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: dsc-setup-cluster-reader +subjects: +- kind: ServiceAccount + name: dsc-setup + namespace: redhat-ods-operator +roleRef: + kind: ClusterRole + name: cluster-reader + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: installplan-modifier + namespace: redhat-ods-operator +rules: +- apiGroups: ["operators.coreos.com"] + resources: ["installplans"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: installplan-modifier-binding + namespace: redhat-ods-operator +subjects: +- kind: ServiceAccount + name: dsc-setup + namespace: redhat-ods-operator +roleRef: + kind: Role + name: installplan-modifier + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: create-dsc-connections +spec: + selector: {} + template: + spec: + containers: + - args: + - -ec + - |- + echo -n 'Waiting for openshift ai initialize' + while ! oc describe sub rhods-operator -n redhat-ods-operator 2>/dev/null | grep -qF rhods-operator; do + echo -n . + sleep 5 + done; echo + sleep 20 + + export INSTALL_PLAN_NAME=$(oc get installplan -o jsonpath='{.items[0].metadata.name}') + oc patch installplan $INSTALL_PLAN_NAME --namespace redhat-ods-operator --type merge --patch '{"spec":{"approved":true}}' + + cat < Date: Tue, 3 Dec 2024 17:41:19 -0500 Subject: [PATCH 2/2] Remove minio dependency --- .../templates/accelerator-profile.yaml | 31 ++++--- .../templates/serving-service-setup.yaml | 84 ++++++++----------- 2 files changed, 49 insertions(+), 66 deletions(-) diff --git a/charts/all/llm-serving-service/templates/accelerator-profile.yaml b/charts/all/llm-serving-service/templates/accelerator-profile.yaml index f8be751c..7a6be47c 100644 --- a/charts/all/llm-serving-service/templates/accelerator-profile.yaml +++ b/charts/all/llm-serving-service/templates/accelerator-profile.yaml @@ -1,16 +1,15 @@ -# --- -# apiVersion: dashboard.opendatahub.io/v1 -# kind: AcceleratorProfile -# metadata: -# name: nvidia-gpu -# namespace: redhat-ods-applications -# annotations: -# argocd.argoproj.io/sync-wave: "40" -# spec: -# displayName: NVIDIA GPU -# enabled: true -# identifier: nvidia.com/gpu -# tolerations: -# - effect: NoSchedule -# key: odh-notebook -# operator: Exists \ No newline at end of file +apiVersion: dashboard.opendatahub.io/v1 +kind: AcceleratorProfile +metadata: + name: nvidia-gpu + namespace: redhat-ods-applications + annotations: + argocd.argoproj.io/sync-wave: "40" +spec: + displayName: NVIDIA GPU + enabled: true + identifier: nvidia.com/gpu + tolerations: + - effect: NoSchedule + key: odh-notebook + operator: Exists \ No newline at end of file diff --git a/charts/all/llm-serving-service/templates/serving-service-setup.yaml b/charts/all/llm-serving-service/templates/serving-service-setup.yaml index 886a0d10..e5f115bd 100644 --- a/charts/all/llm-serving-service/templates/serving-service-setup.yaml +++ b/charts/all/llm-serving-service/templates/serving-service-setup.yaml @@ -1,18 +1,4 @@ --- -apiVersion: dashboard.opendatahub.io/v1 -kind: AcceleratorProfile -metadata: - name: nvidia-gpu - namespace: redhat-ods-applications -spec: - displayName: NVIDIA GPU - enabled: true - identifier: nvidia.com/gpu - tolerations: - - effect: NoSchedule - key: odh-notebook - operator: Exists ---- apiVersion: batch/v1 kind: Job metadata: @@ -25,29 +11,6 @@ spec: - args: - -ec - |- - export MINIO_SECRET=$(oc get secret minio-secret -o jsonpath='{.data.MINIO_ROOT_PASSWORD}' | base64 --decode) - export MODEL_NAME=$(echo "${modelId##*/}" | sed 's/-[^-]*$//' | tr '[:upper:]' '[:lower:]') - export MODEL_ID=$(echo "${modelId}" | awk -F'/' '{print $NF}') - cat << EOF | oc apply -f- - kind: Secret - apiVersion: v1 - metadata: - name: aws-connection-rag-llm-bucket - namespace: rag-llm - labels: - opendatahub.io/dashboard: 'true' - opendatahub.io/managed: 'true' - annotations: - opendatahub.io/connection-type: s3 - openshift.io/display-name: rag-llm-bucket - stringData: - AWS_ACCESS_KEY_ID: minio - AWS_SECRET_ACCESS_KEY: $MINIO_SECRET - AWS_DEFAULT_REGION: us-east-1 - AWS_S3_BUCKET: models - AWS_S3_ENDPOINT: http://minio-service:9000 - type: Opaque - EOF cat << EOF | oc apply -f- apiVersion: serving.kserve.io/v1alpha1 kind: ServingRuntime @@ -56,10 +19,8 @@ spec: opendatahub.io/accelerator-name: nvidia-gpu opendatahub.io/apiProtocol: REST opendatahub.io/recommended-accelerators: '["nvidia.com/gpu"]' - opendatahub.io/template-display-name: vLLM ServingRuntime for KServe - opendatahub.io/template-name: vllm-runtime - openshift.io/display-name: $MODEL_NAME - name: $MODEL_NAME + openshift.io/display-name: mistral-7b-instruct + name: mistral-7b-instruct namespace: rag-llm labels: opendatahub.io/dashboard: 'true' @@ -70,9 +31,10 @@ spec: containers: - args: - '--port=8080' - - '--model=/mnt/models' - - '--served-model-name={{`{{.Name}}`}}' + - '--model=\$(MODEL_ID)' + - '--download-dir=/cache' - '--distributed-executor-backend=mp' + - '--served-model-name=mistral-7b-instruct' - '--max-model-len=4096' - '--dtype=half' - '--gpu-memory-utilization' @@ -85,6 +47,18 @@ spec: env: - name: HF_HOME value: /tmp/hf_home + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: huggingface-secret + key: hftoken + - name: MODEL_ID + valueFrom: + secretKeyRef: + name: huggingface-secret + key: modelId + - name: HF_HUB_OFFLINE + value: "0" image: 'quay.io/modh/vllm@sha256:b51fde66f162f1a78e8c027320dddf214732d5345953b1599a84fe0f0168c619' name: kserve-container ports: @@ -93,6 +67,8 @@ spec: volumeMounts: - mountPath: /dev/shm name: shm + - mountPath: /cache + name: cache multiModel: false supportedModelFormats: - autoSelect: true @@ -102,17 +78,19 @@ spec: medium: Memory sizeLimit: 2Gi name: shm + - emptyDir: {} + name: cache EOF cat << EOF | oc apply -f- apiVersion: serving.kserve.io/v1beta1 kind: InferenceService metadata: annotations: - openshift.io/display-name: $MODEL_NAME + openshift.io/display-name: mistral-7b-instruct serving.knative.openshift.io/enablePassthrough: 'true' sidecar.istio.io/inject: 'true' sidecar.istio.io/rewriteAppHTTPProbers: 'true' - name: $MODEL_NAME + name: mistral-7b-instruct namespace: rag-llm labels: opendatahub.io/dashboard: 'true' @@ -134,10 +112,7 @@ spec: cpu: '2' memory: 8Gi nvidia.com/gpu: '1' - runtime: $MODEL_NAME - storage: - key: aws-connection-rag-llm-bucket - path: llm-models/$MODEL_ID + runtime: mistral-7b-instruct tolerations: - effect: NoSchedule key: odh-notebook @@ -157,8 +132,17 @@ spec: - args: - -ec - |- - oc wait --for=condition=complete job/load-model-set -n rag-llm --timeout=10m + echo -n 'Waiting for openshift-ai initialize' + while ! oc describe sub rhods-operator -n redhat-ods-operator 2>/dev/null | grep -qF rhods-operator; do + echo -n . + sleep 15 + done; echo + echo -n 'openshift-ai initialized';echo + echo -n 'Waiting for dscinitialization/default-dsci to initialize' + echo + oc wait --for=jsonpath='{.status.phase}'=Ready --timeout=600s -n redhat-ods-operator dscinitialization/default-dsci sleep 10 + echo -n 'dscinitialization/default-dsci initialized';echo command: - /bin/bash image: image-registry.openshift-image-registry.svc:5000/openshift/tools:latest