/
Tiltfile
57 lines (44 loc) · 1.92 KB
/
Tiltfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
load('ext://namespace', 'namespace_create', 'namespace_inject')
# --- Variables ---
ns='llm-example'
backend_host='llm-backend'
chatbot_ui_folder='../chatbot-ui'
openai_key=os.environ.get('OPENAI_KEY_BASE64', '')
# Change the following as required
chat_controller='llama'
llama_model_path='/Users/sgerogia/src/github.com/fredi-python/llama.cpp/models/'
llama_model_file='llama-2-13b-chat.ggmlv3.q2_K.bin'
# --- Execution ---
print("📢 Creating namespace: " + ns)
namespace_create(ns)
print("📢 Creating Llama model PVC")
raw_yaml = read_file('./k8s/llama-volume.yaml')
yaml = str(raw_yaml).format(
llama_model_path=llama_model_path,
)
k8s_yaml(namespace_inject(blob(yaml), ns))
print("📢 Applying llm-backend K8s resources")
raw_yaml = read_file('./k8s/llm-backend.yaml')
yaml = str(raw_yaml).format(
openai_key=openai_key,
chat_controller=chat_controller,
llama_model_file=llama_model_file,
)
#k8s_yaml(namespace_inject(blob(yaml), ns))
print("📢 Applying chatbot-ui K8s resources")
raw_yaml = read_file(chatbot_ui_folder + '/k8s/chatbot-ui.yaml')
yaml = str(raw_yaml).format(
#api_host='http://' + backend_host + ':8080',
api_host='https://gf69zjzw9flulx-5000.proxy.runpod.net',
default_model='gpt-3.5-turbo',
system_prompt='You are S.T.R.A.T.O.S, a large language model trained to make the world a better place. Follow the user`s instructions carefully. Respond using markdown.',
temperature='1',
log_level='debug',
)
k8s_yaml(namespace_inject(blob(yaml), ns))
print("📢 Building docker images")
#docker_build('llm-example/llm-backend', '.', dockerfile='Dockerfile.local', ignore=['README.md', '.gitignore'])
docker_build('llm-example/chatbot-ui', chatbot_ui_folder, dockerfile=chatbot_ui_folder + '/Dockerfile', ignore=['README.md', '.gitignore'])
print("📢 Launching K8s resources")
#k8s_resource('llm-backend', port_forwards='5000:5000')
k8s_resource('chatbot-ui', port_forwards='3000:3000')