/
index.html
208 lines (199 loc) · 7.63 KB
/
index.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
<html>
<head>
<script
defer
src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"
></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/marked/5.1.2/marked.min.js"></script>
<link href="https://unpkg.com/@tailwindcss/typography@0.4.1/dist/typography.min.css" rel="stylesheet">
<script src="https://cdn.tailwindcss.com"></script>
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Modal | LLM Engine</title>
</head>
<body>
<section x-data="state()" class="max-w-2xl mx-auto pt-16 px-4">
<div class="text-xs font-semibold tracking-wide uppercase text-center text-black">
<a
href="https://modal.com/docs/examples/text_generation_inference"
class="inline-flex gap-x-1 items-center bg-lime-400 py-0.5 px-3 rounded-full hover:text-lime-400 hover:ring hover:ring-lime-400 hover:bg-white focus:outline-neutral-400"
target="_blank"
>
powered by Modal
<svg
xmlns="http://www.w3.org/2000/svg"
class="w-4 h-4 animate-pulse -mr-1"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
class="lucide lucide-chevrons-right"
>
<path d="m6 17 5-5-5-5" />
<path d="m13 17 5-5-5-5" />
</svg>
</a>
</div>
<div class="text-4xl mt-4 mb-4 font-semibold tracking-tighter text-center">
Modal LLM Engine
</div>
<div x-show="info.loaded && info.model" x-text="info.model" class="text-2xl mb-4 font-medium tracking-tighter text-center">
</div>
<div class="flex flex-wrap justify-center items-center mt-4 mb-6">
<div
x-init="setInterval(() => refreshInfo(), 1000)"
class="inline-flex flex-col sm:flex-row justify-center items-center gap-x-4 text-sm text-white px-3 py-1 bg-neutral-600 rounded-full"
>
<div x-show="!info.loaded" class="flex items-center gap-x-1">
<div class="animate-spin w-4 h-4">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
>
<path d="M21 12a9 9 0 1 1-6.219-8.56" />
</svg>
</div>
<span>loading stats</span>
</div>
<div x-show="info.loaded && info.backlog > 0">
<span x-text="info.backlog"></span>
<span x-text="info.backlog === 1 ? 'input in queue' : 'inputs in queue'"></span>
</div>
<div x-show="info.loaded && (info.backlog === 0)">
<span x-text="info.num_total_runners"></span>
<span x-text="info.num_total_runners === 1 ? 'container online' : 'containers online'"></span>
</div>
<div
class="flex items-center gap-x-1"
x-show="info.loaded && info.backlog > 0"
>
<div class="animate-spin w-4 h-4">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
>
<path d="M21 12a9 9 0 1 1-6.219-8.56" />
</svg>
</div>
<span> cold-starting </span>
</div>
</div>
</div>
<form class="relative">
<input
x-model="nextPrompt"
type="text"
placeholder="Ask something ..."
class="flex grow w-full h-10 pl-4 pr-12 py-2 text-md bg-white border rounded-3xl border-neutral-300 ring-offset-background placeholder:text-neutral-500 focus:border-neutral-300 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-neutral-400 disabled:cursor-not-allowed disabled:opacity-50"
@keydown.window.prevent.ctrl.k="$el.focus()"
@keydown.window.prevent.cmd.k="$el.focus()"
autofocus
/>
<div class="absolute top-0 right-0 flex items-center h-full pr-[0.3125rem]">
<button
@click.prevent="callApi()"
class="rounded-full bg-lime-400 p-2 focus:border-neutral-300 focus:outline-neutral-400"
>
<svg
xmlns="http://www.w3.org/2000/svg"
class="w-4 h-4"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
class="lucide lucide-plus"
>
<path d="M5 12h14" />
<path d="M12 5v14" />
</svg>
</button>
</div>
</form>
<div class="flex flex-col gap-y-4 my-8">
<template x-for="(item, index) in [...items].reverse()" :key="index">
<div class="w-full border px-4 py-2 rounded-3xl">
<div
x-data
class="text-sm mt-2 mb-4 whitespace-pre-line"
x-text="item.prompt"
:class="{'animate-pulse': item.loading}"
></div>
<div
x-show="item.completion.length === 0"
class="h-4 w-2 mt-2 mb-4 bg-neutral-500 animate-pulse"
></div>
<div
class="text-sm mt-2 mb-4 text-neutral-500 w-full prose max-w-none prose-neutral-100 leading-6"
x-show="item.completion.length > 0"
x-html="item.markdownCompletion"
>
</div>
</div>
</template>
</div>
<script>
function state() {
return {
nextPrompt: "",
items: [],
info: { backlog: 0, num_total_runners: 0 },
callApi() {
console.log(this.nextPrompt);
if (!this.nextPrompt) return;
let item = {
id: Math.random(),
prompt: this.nextPrompt,
completion: "",
loading: true,
markdownCompletion: "",
};
this.nextPrompt = "";
this.items.push(item);
const eventSource = new EventSource(
`/completion/${encodeURIComponent(item.prompt)}`,
);
console.log("Created event source ...");
eventSource.onmessage = (event) => {
item.completion += JSON.parse(event.data).text;
item.markdownCompletion = marked.parse(item.completion, {mangle: false, headerIds: false});
// Hacky way to notify element to update
this.items = this.items.map((i) =>
i.id === item.id ? { ...item } : i,
);
};
eventSource.onerror = (event) => {
eventSource.close();
item.loading = false;
this.items = this.items.map((i) =>
i.id === item.id ? { ...item } : i,
);
console.log(item.completion);
};
},
refreshInfo() {
fetch("/stats")
.then((response) => response.json())
.then((data) => {
this.info = { ...data, loaded: true };
})
.catch((error) => console.log(error));
},
};
}
</script>
</section>
</body>
</html>