-
Notifications
You must be signed in to change notification settings - Fork 39
/
caikit-servingruntime.yaml
73 lines (72 loc) · 1.78 KB
/
caikit-servingruntime.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
---
apiVersion: v1
kind: ConfigMap
metadata:
name: caikit-tgis-config
data:
caikit.yml: |
runtime:
library: caikit_nlp
local_models_dir: /mnt/models/
lazy_load_local_models: true
model_management:
finders:
default:
type: MULTI
config:
finder_priority:
- tgis-auto
tgis-auto:
type: TGIS-AUTO
config:
test_connection: true
initializers:
default:
type: LOCAL
config:
backend_priority:
- type: TGIS
config:
connection:
hostname: localhost:8033
---
apiVersion: serving.kserve.io/v1alpha1
kind: ServingRuntime
metadata:
name: caikit-runtime
spec:
multiModel: false
supportedModelFormats:
# Note: this currently *only* supports caikit format models
- autoSelect: true
name: caikit
containers:
- name: kserve-container
image: quay.io/opendatahub/text-generation-inference:stable
command: ["text-generation-launcher"]
args: ["--model-name=/mnt/models/artifacts/"]
env:
- name: TRANSFORMERS_CACHE
value: /tmp/transformers_cache
# resources: # configure as required
# requests:
# cpu: 8
# memory: 16Gi
- name: transformer-container
image: quay.io/opendatahub/caikit-tgis-serving:stable
volumeMounts:
- name: config-volume
mountPath: /caikit/config/
readOnly: true
ports:
- containerPort: 8085
name: h2c
protocol: TCP
# resources: # configure as required
# requests:
# cpu: 8
# memory: 16Gi
volumes:
- name: config-volume
configMap:
name: caikit-tgis-config