From dbd6f74d3125613fe298ca3aa35516785ffd21fa Mon Sep 17 00:00:00 2001 From: Yunfeng Bai Date: Fri, 25 Aug 2023 00:54:50 +0000 Subject: [PATCH 1/2] Update service template --- .../service_template_config_map.yaml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/charts/llm-engine/templates/service_template_config_map.yaml b/charts/llm-engine/templates/service_template_config_map.yaml index 0f277c45..e41769f3 100644 --- a/charts/llm-engine/templates/service_template_config_map.yaml +++ b/charts/llm-engine/templates/service_template_config_map.yaml @@ -483,6 +483,62 @@ data: protocol: TCP name: http ${NODE_PORT_DICT} + virtual-service.yaml: |- + apiVersion: networking.istio.io/v1alpha3 + kind: VirtualService + metadata: + name: ${RESOURCE_NAME} + namespace: ${NAMESPACE} + labels: + user_id: ${OWNER} + team: ${TEAM} + product: ${PRODUCT} + created_by: ${CREATED_BY} + owner: ${OWNER} + env: circleci + managed-by: model-engine + use_scale_launch_endpoint_network_policy: "true" + tags.datadoghq.com/env: circleci + tags.datadoghq.com/version: ${GIT_TAG} + tags.datadoghq.com/service: ${ENDPOINT_NAME} + endpoint_id: ${ENDPOINT_ID} + endpoint_name: ${ENDPOINT_NAME} + spec: + hosts: + - ${RESOURCE_NAME}.${DNS_HOST_DOMAIN} + gateways: + - default/internal-gateway + http: + - route: + - destination: + host: "${RESOURCE_NAME}.${NAMESPACE}.svc.cluster.local" + port: + number: 80 + destination-rule.yaml: |- + apiVersion: networking.istio.io/v1beta1 + kind: DestinationRule + metadata: + name: ${RESOURCE_NAME} + namespace: ${NAMESPACE} + labels: + user_id: ${OWNER} + team: ${TEAM} + product: ${PRODUCT} + created_by: ${CREATED_BY} + owner: ${OWNER} + env: circleci + managed-by: model-engine + use_scale_launch_endpoint_network_policy: "true" + tags.datadoghq.com/env: circleci + tags.datadoghq.com/version: ${GIT_TAG} + tags.datadoghq.com/service: ${ENDPOINT_NAME} + endpoint_id: ${ENDPOINT_ID} + endpoint_name: ${ENDPOINT_NAME} + spec: + host: "${RESOURCE_NAME}.${NAMESPACE}.svc.cluster.local" + trafficPolicy: + loadBalancer: + simple: LEAST_REQUEST vertical-pod-autoscaler.yaml: |- apiVersion: "autoscaling.k8s.io/v1" kind: VerticalPodAutoscaler From 311a0dcec6ba2ab169cdba999a9be6f5fda8c60f Mon Sep 17 00:00:00 2001 From: Yunfeng Bai Date: Fri, 25 Aug 2023 05:04:44 +0000 Subject: [PATCH 2/2] fix --- charts/llm-engine/templates/_helpers.tpl | 8 +-- .../service_template_config_map.yaml | 56 ++++++++++++++++++- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/charts/llm-engine/templates/_helpers.tpl b/charts/llm-engine/templates/_helpers.tpl index 08af45f4..01b63b8d 100644 --- a/charts/llm-engine/templates/_helpers.tpl +++ b/charts/llm-engine/templates/_helpers.tpl @@ -330,21 +330,21 @@ volumeMounts: mountPath: /workspace/llm_engine/llm_engine/infra/gateways/resources/templates {{- if .Values.aws }} - name: config-volume - mountPath: /home/user/.aws/config + mountPath: /root/.aws/config subPath: config {{- end }} {{- if .Values.config.values }} - name: llm-engine-service-config-volume mountPath: /workspace/llm_engine/service_configs - name: infra-service-config-volume - mountPath: /workspace/ml_infra_core/llm_engine.core/llm_engine.core/configs + mountPath: /workspace/model-engine/model_engine_server/core/configs {{- end }} {{- end }} {{- define "llmEngine.forwarderVolumeMounts" }} volumeMounts: - name: config-volume - mountPath: /home/user/.aws/config + mountPath: /root/.aws/config subPath: config - name: user-config mountPath: /workspace/user_config @@ -354,7 +354,7 @@ volumeMounts: subPath: raw_data {{- if .Values.config.values }} - name: infra-service-config-volume - mountPath: /workspace/ml_infra_core/llm_engine.core/llm_engine.core/configs + mountPath: /workspace/model-engine/model_engine_server/core/configs {{- end }} {{- end }} diff --git a/charts/llm-engine/templates/service_template_config_map.yaml b/charts/llm-engine/templates/service_template_config_map.yaml index e41769f3..af78b38f 100644 --- a/charts/llm-engine/templates/service_template_config_map.yaml +++ b/charts/llm-engine/templates/service_template_config_map.yaml @@ -180,7 +180,7 @@ data: - ddtrace-run - run-service - --config - - /workspace/server/model_engine_server/inference/configs/${FORWARDER_CONFIG_FILE_NAME} + - /workspace/model-engine/model_engine_server/inference/configs/${FORWARDER_CONFIG_FILE_NAME} - --http - production_threads - --port @@ -223,7 +223,7 @@ data: - -m - model_engine_server.inference.forwarding.http_forwarder - --config - - /workspace/server/model_engine_server/inference/configs/service--http_forwarder.yaml + - /workspace/model-engine/model_engine_server/inference/configs/service--http_forwarder.yaml - --port - "${FORWARDER_PORT}" - --num-workers @@ -266,7 +266,7 @@ data: - ddtrace-run - run-service - --config - - /workspace/server/model_engine_server/inference/configs/${FORWARDER_CONFIG_FILE_NAME} + - /workspace/model-engine/model_engine_server/inference/configs/${FORWARDER_CONFIG_FILE_NAME} - --queue - "${QUEUE}" - --task-visibility @@ -798,3 +798,53 @@ data: command: ["/bin/sh", "-ec", "while : ; do sleep 30 ; done"] terminationGracePeriodSeconds: 0 {{- end }} + cron-trigger.yaml: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: ${NAME} + labels: + user_id: ${OWNER} + team: ${TEAM} + product: ${PRODUCT} + created_by: ${CREATED_BY} + owner: ${OWNER} + launch_trigger_id: ${TRIGGER_ID} + tags.datadoghq.com/service: ${TRIGGER_ID} + spec: + schedule: "${CRON_SCHEDULE}" + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 0 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: ${BATCH_CURL_JOB_ACTIVE_DEADLINE_SECONDS} + template: + metadata: + labels: + user_id: ${OWNER} + team: ${TEAM} + product: ${PRODUCT} + created_by: ${CREATED_BY} + owner: ${OWNER} + launch_trigger_id: ${TRIGGER_ID} + tags.datadoghq.com/service: ${TRIGGER_ID} + spec: + containers: + - name: ${NAME} + image: curlimages/curl:7.72.0 + imagePullPolicy: IfNotPresent + command: + - curl + - -X + - 'POST' + - '${HOST}/v1/docker-image-batch-jobs' + - -H + - 'accept: application/json' + - -H + - 'Content-Type: application/json' + - -d + - '{ "docker_image_batch_job_bundle_id": "${DOCKER_IMAGE_BATCH_JOB_BUNDLE_ID}", "job_config": ${JOB_CONFIG}, "labels": ${JOB_METADATA} }' + - -u + - '${OWNER}:' + restartPolicy: Never