-
Notifications
You must be signed in to change notification settings - Fork 1
/
textcat.h
159 lines (136 loc) · 5.17 KB
/
textcat.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
/*
+----------------------------------------------------------------------+
| Copyright (c) 2010 The PHP Group |
+----------------------------------------------------------------------+
| This source file is subject to version 3.01 of the PHP license, |
| that is bundled with this package in the file LICENSE, and is |
| available through the world-wide-web at the following url: |
| http://www.php.net/license/3_01.txt |
| If you did not receive a copy of the PHP license and are unable to |
| obtain it through the world-wide-web, please send a note to |
| license@php.net so we can mail you a copy immediately. |
+----------------------------------------------------------------------+
| Author: César Rodas <crodas@member.fsf.org> |
+----------------------------------------------------------------------+
*/
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
/* Definitions {{{ */
#define TC_HASH_SIZE 1000
#define TC_BUFFER_SIZE (16 * 1024)
#define TC_MAX_NGRAMS 400
#define Bool char
#define uchar unsigned char
#define TC_TRUE 1
#define TC_FALSE 0
#define TC_FREE 1
#define TC_BUSY 0
#define MIN_NGRAM_LEN 1
#define MAX_NGRAM_LEN 5
#define TC_THRESHOLD 1.003
#define TC_OK TC_TRUE
#define TC_ERR (-1)
#define TC_ERR_MEM (-2)
#define TC_NO_FILE (-3)
#define TC_ERR_FILE_SIZE (-4)
#define TC_NO_NGRAM (-5)
#define TC_ERR_CALLBACK (-6)
#define TC_ERR_NO_KNOWLEDGE (-7)
#define TEXTCAT_FUNCTION(type, name) textcat_callback_##type##_##name
#define TEXTCAT_LIST(name) Bool TEXTCAT_FUNCTION(list, name)(void * memptr, uchar *** list, int * size, void * param)
#define TEXTCAT_SAVE(name) Bool TEXTCAT_FUNCTION(save, name)(void * memptr, const uchar * id, NGrams * result, void * param)
#define TEXTCAT_LOAD(name) Bool TEXTCAT_FUNCTION(load, name)(void * memptr, const uchar * id, NGrams * result, int max, void * param)
#define TEXTCAT_DISTANCE(name) long TEXTCAT_FUNCTION(dist, name)(NGrams *a, NGrams *b, void * param)
#define TEXTCAT_PARSER(name) long TEXTCAT_FUNCTION(parser, name)(void * memptr, int min, int max, const uchar * text, size_t length, int * (*set_ngram)(TextCat *, const uchar *, size_t), void * param, TextCat * tc)
#define tc_malloc(size) TextCat_malloc(memptr, size)
#define tc_strdup(str) TextCat_strndup(memptr, str, strlen(str));
#define tc_strndup(str, len) TextCat_strndup(memptr, str, len);
/* }}} */
/* Data types {{{ */
typedef struct {
uchar * str;
long freq;
int len;
struct ngram_t * next;
} ngram_t;
typedef struct {
/* linked list */
ngram_t * first;
ngram_t * last;
long total;
} ngram_set;
typedef struct {
ngram_set * table;
long size;
long ngrams;
} ngram_hash;
typedef struct result_stack {
struct NGrams * result;
struct result_stack * next;
} result_stack;
typedef struct TextCat {
/* pools of memory */
void * temp; /* temporary memory */
void * memory; /* "return" structures and internal stack */
/* callback */
void * (*malloc)(size_t);
void (*free)(void *);
Bool (*parse_str)(void *, int, int, uchar *, size_t , int * (*set_ngram)(struct TextCat *, const uchar *, size_t), void *, struct TextCat *);
Bool (*save)(void *, const uchar *, struct NGrams *, void *);
Bool (*list)(void *, uchar ***, int *, void *);
Bool (*load)(void *, const uchar *, struct NGram *, int, void *);
long (*distance)(struct NGrams *, struct NGrams *, void *);
void * param;
/* config issues */
size_t allocate_size;
int hash_size;
int min_ngram_len;
int max_ngram_len;
int max_ngrams;
float threshold;
/* internal stuff */
ngram_hash hash;
result_stack * results;
uchar ** klNames;
struct NGrams * klContent;
int klTotal;
/* status */
int error;
int status;
} TextCat;
typedef struct {
uchar * str;
int size;
long freq;
long position;
} NGram;
typedef struct NGrams {
NGram * ngram;
long size;
} NGrams;
/* }}} */
Bool TextCat_Init_ex(TextCat ** tcc, void * (*xmalloc)(size_t), void (*xfree)(void *));
Bool TextCat_Init(TextCat ** tc);
Bool TextCat_Destroy(TextCat * tc);
Bool TextCat_reset(TextCat * tc);
Bool TextCat_reset_handlers(TextCat * tc);
Bool TextCat_load(TextCat *tc);
Bool TextCat_getCategory(TextCat *tc, const uchar * text, size_t length, uchar *** result, int * n);
Bool TextCat_parse_ex(TextCat * tc, const uchar * text, size_t length, NGrams ** ngrams, Bool store_stack);
Bool TextCat_parse(TextCat * tc, const uchar * text, size_t length, NGrams ** ngram);
Bool TextCat_parse_file(TextCat * tc, const uchar * filename, NGrams ** ngrams);
Bool TextCat_list(TextCat * tc, uchar *** list, int * len);
Bool TextCat_load(TextCat *tc);
void * TextCat_malloc(void * memblock, size_t size);
void * TextCat_strndup(void * memblock, uchar * str, size_t len);
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: sw=4 ts=4 fdm=marker
* vim<600: sw=4 ts=4
*/