-
-
Notifications
You must be signed in to change notification settings - Fork 1
/
tiktoken.h
65 lines (44 loc) · 2.06 KB
/
tiktoken.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
/* https://github.com/kojix2/tiktoken-c */
/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. */
#include <stdarg.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
typedef void CoreBPE;
typedef struct CFunctionCall {
const char *name;
const char *arguments;
} CFunctionCall;
typedef struct CChatCompletionRequestMessage {
const char *role;
const char *content;
const char *name;
const struct CFunctionCall *function_call;
} CChatCompletionRequestMessage;
void tiktoken_init_logger(void);
CoreBPE *tiktoken_r50k_base(void);
CoreBPE *tiktoken_p50k_base(void);
CoreBPE *tiktoken_p50k_edit(void);
CoreBPE *tiktoken_cl100k_base(void);
CoreBPE *tiktoken_o200k_base(void);
void tiktoken_destroy_corebpe(CoreBPE *ptr);
CoreBPE *tiktoken_get_bpe_from_model(const char *model);
size_t tiktoken_get_completion_max_tokens(const char *model, const char *prompt);
size_t tiktoken_num_tokens_from_messages(const char *model,
uint32_t num_messages,
const struct CChatCompletionRequestMessage *messages);
size_t tiktoken_get_chat_completion_max_tokens(const char *model,
uint32_t num_messages,
const struct CChatCompletionRequestMessage *messages);
size_t *tiktoken_corebpe_encode_ordinary(CoreBPE *ptr, const char *text, size_t *num_tokens);
size_t *tiktoken_corebpe_encode(CoreBPE *ptr,
const char *text,
const char *const *allowed_special,
size_t allowed_special_len,
size_t *num_tokens);
size_t *tiktoken_corebpe_encode_with_special_tokens(CoreBPE *ptr,
const char *text,
size_t *num_tokens);
char *tiktoken_corebpe_decode(CoreBPE *ptr, const size_t *tokens, size_t num_tokens);
const char *tiktoken_c_version(void);