lorax/charts/lorax/values.yaml at main · predibase/lorax · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
deployment:
  replicas: 1
  updateStrategy: {}

  image:
    repository: "ghcr.io/predibase/lorax"
    tag: "latest"

  args:
    - name: "--model-id"
      value: "mistralai/Mistral-7B-Instruct-v0.1"
    - name: "--max-input-length"
      value: "512"
    - name: "--max-total-tokens"
      value: "1024"
    - name: "--max-batch-total-tokens"
      value: "4096"
    - name: "--max-batch-prefill-tokens"
      value: "2048"
    - name: "--eager-prefill"
      value: "false"
    - name: "--compile"
      value: "" # --complie does not take a second argument

  env:
    # Your huggingface hub token. Required for some models such as the llama-2 family.
    - name: "HUGGING_FACE_HUB_TOKEN"
      value: ""

  resources:
    limits:
      nvidia.com/gpu: "1"
    requests:
      nvidia.com/gpu: "1"

  livenessProbe:
    {}
    # failureThreshold: 240
    # httpGet:
    #   path: /health
    #   port: http
    #   scheme: HTTP
    # initialDelaySeconds: 5
    # periodSeconds: 5
    # successThreshold: 1
    # timeoutSeconds: 1

  readinessProbe:
    {}
    # failureThreshold: 600
    # httpGet:
    #   path: /health
    #   port: http
    #   scheme: HTTP
    # initialDelaySeconds: 5
    # periodSeconds: 5
    # successThreshold: 1
    # timeoutSeconds: 1

  nodeSelector: {}
  tolerations: []
  additionalLabels: {}
  additionalPodLabels: {}

  additionalAnnotations: {}
  additionalPodAnnotations: {}
  affinity: {}

  priorityClassName: ""

service:
  name: "lorax"
  serviceType: ClusterIP
  port: 80
  additionalLabels: {}