-
Notifications
You must be signed in to change notification settings - Fork 5.3k
/
encoding.h
359 lines (286 loc) · 14.1 KB
/
encoding.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
/**********************************************************************
encoding.h -
$Author: matz $
created at: Thu May 24 11:49:41 JST 2007
Copyright (C) 2007 Yukihiro Matsumoto
**********************************************************************/
#ifndef RUBY_ENCODING_H
#define RUBY_ENCODING_H 1
#if defined(__cplusplus)
extern "C" {
#if 0
} /* satisfy cc-mode */
#endif
#endif
#include <stdarg.h>
#include "ruby/oniguruma.h"
RUBY_SYMBOL_EXPORT_BEGIN
#define ENCODING_INLINE_MAX 127
#define ENCODING_SHIFT (FL_USHIFT+10)
#define ENCODING_MASK (((VALUE)ENCODING_INLINE_MAX)<<ENCODING_SHIFT) /* FL_USER10|FL_USER11|FL_USER12|FL_USER13|FL_USER14|FL_USER15|FL_USER16 */
#define ENCODING_SET_INLINED(obj,i) do {\
RBASIC(obj)->flags &= ~ENCODING_MASK;\
RBASIC(obj)->flags |= (VALUE)(i) << ENCODING_SHIFT;\
} while (0)
#define ENCODING_SET(obj,i) rb_enc_set_index((obj), (i))
#define ENCODING_GET_INLINED(obj) (int)((RBASIC(obj)->flags & ENCODING_MASK)>>ENCODING_SHIFT)
#define ENCODING_GET(obj) \
(ENCODING_GET_INLINED(obj) != ENCODING_INLINE_MAX ? \
ENCODING_GET_INLINED(obj) : \
rb_enc_get_index(obj))
#define ENCODING_IS_ASCII8BIT(obj) (ENCODING_GET_INLINED(obj) == 0)
#define ENCODING_MAXNAMELEN 42
#define ENC_CODERANGE_MASK ((int)(FL_USER8|FL_USER9))
#define ENC_CODERANGE_UNKNOWN 0
#define ENC_CODERANGE_7BIT ((int)FL_USER8)
#define ENC_CODERANGE_VALID ((int)FL_USER9)
#define ENC_CODERANGE_BROKEN ((int)(FL_USER8|FL_USER9))
#define ENC_CODERANGE(obj) ((int)RBASIC(obj)->flags & ENC_CODERANGE_MASK)
#define ENC_CODERANGE_ASCIIONLY(obj) (ENC_CODERANGE(obj) == ENC_CODERANGE_7BIT)
#define ENC_CODERANGE_SET(obj,cr) (RBASIC(obj)->flags = \
(RBASIC(obj)->flags & ~ENC_CODERANGE_MASK) | (cr))
#define ENC_CODERANGE_CLEAR(obj) ENC_CODERANGE_SET((obj),0)
/* assumed ASCII compatibility */
#define ENC_CODERANGE_AND(a, b) \
((a) == ENC_CODERANGE_7BIT ? (b) : \
(a) == ENC_CODERANGE_VALID ? ((b) == ENC_CODERANGE_7BIT ? ENC_CODERANGE_VALID : (b)) : \
ENC_CODERANGE_UNKNOWN)
#define ENCODING_CODERANGE_SET(obj, encindex, cr) \
do { \
VALUE rb_encoding_coderange_obj = (obj); \
ENCODING_SET(rb_encoding_coderange_obj, (encindex)); \
ENC_CODERANGE_SET(rb_encoding_coderange_obj, (cr)); \
} while (0)
typedef const OnigEncodingType rb_encoding;
int rb_char_to_option_kcode(int c, int *option, int *kcode);
int rb_enc_replicate(const char *, rb_encoding *);
int rb_define_dummy_encoding(const char *);
int rb_enc_dummy_p(rb_encoding *enc);
int rb_enc_to_index(rb_encoding *enc);
int rb_enc_get_index(VALUE obj);
void rb_enc_set_index(VALUE obj, int encindex);
int rb_enc_find_index(const char *name);
int rb_to_encoding_index(VALUE);
rb_encoding *rb_to_encoding(VALUE);
rb_encoding *rb_find_encoding(VALUE);
rb_encoding *rb_enc_get(VALUE);
rb_encoding *rb_enc_compatible(VALUE,VALUE);
rb_encoding *rb_enc_check(VALUE,VALUE);
VALUE rb_enc_associate_index(VALUE, int);
VALUE rb_enc_associate(VALUE, rb_encoding*);
void rb_enc_copy(VALUE dst, VALUE src);
VALUE rb_enc_str_new(const char*, long, rb_encoding*);
VALUE rb_enc_str_new_cstr(const char*, rb_encoding*);
VALUE rb_enc_str_new_static(const char*, long, rb_encoding*);
VALUE rb_enc_reg_new(const char*, long, rb_encoding*, int);
PRINTF_ARGS(VALUE rb_enc_sprintf(rb_encoding *, const char*, ...), 2, 3);
VALUE rb_enc_vsprintf(rb_encoding *, const char*, va_list);
long rb_enc_strlen(const char*, const char*, rb_encoding*);
char* rb_enc_nth(const char*, const char*, long, rb_encoding*);
VALUE rb_obj_encoding(VALUE);
VALUE rb_enc_str_buf_cat(VALUE str, const char *ptr, long len, rb_encoding *enc);
VALUE rb_enc_uint_chr(unsigned int code, rb_encoding *enc);
VALUE rb_external_str_new_with_enc(const char *ptr, long len, rb_encoding *);
VALUE rb_str_export_to_enc(VALUE, rb_encoding *);
VALUE rb_str_conv_enc(VALUE str, rb_encoding *from, rb_encoding *to);
VALUE rb_str_conv_enc_opts(VALUE str, rb_encoding *from, rb_encoding *to, int ecflags, VALUE ecopts);
#if defined(__GNUC__) && !defined(__PCC__)
#define rb_enc_str_new(str, len, enc) __extension__ ( \
{ \
(__builtin_constant_p(str) && __builtin_constant_p(len)) ? \
rb_enc_str_new_static((str), (len), (enc)) : \
rb_enc_str_new((str), (len), (enc)); \
})
#define rb_enc_str_new_cstr(str, enc) __extension__ ( \
{ \
(__builtin_constant_p(str)) ? \
rb_enc_str_new_static((str), (long)strlen(str), (enc)) : \
rb_enc_str_new_cstr((str), (enc)); \
})
#endif
PRINTF_ARGS(NORETURN(void rb_enc_raise(rb_encoding *, VALUE, const char*, ...)), 3, 4);
/* index -> rb_encoding */
rb_encoding *rb_enc_from_index(int idx);
/* name -> rb_encoding */
rb_encoding *rb_enc_find(const char *name);
/* rb_encoding * -> name */
#define rb_enc_name(enc) (enc)->name
/* rb_encoding * -> minlen/maxlen */
#define rb_enc_mbminlen(enc) (enc)->min_enc_len
#define rb_enc_mbmaxlen(enc) (enc)->max_enc_len
/* -> mbclen (no error notification: 0 < ret <= e-p, no exception) */
int rb_enc_mbclen(const char *p, const char *e, rb_encoding *enc);
/* -> mbclen (only for valid encoding) */
int rb_enc_fast_mbclen(const char *p, const char *e, rb_encoding *enc);
/* -> chlen, invalid or needmore */
int rb_enc_precise_mbclen(const char *p, const char *e, rb_encoding *enc);
#define MBCLEN_CHARFOUND_P(ret) ONIGENC_MBCLEN_CHARFOUND_P(ret)
#define MBCLEN_CHARFOUND_LEN(ret) ONIGENC_MBCLEN_CHARFOUND_LEN(ret)
#define MBCLEN_INVALID_P(ret) ONIGENC_MBCLEN_INVALID_P(ret)
#define MBCLEN_NEEDMORE_P(ret) ONIGENC_MBCLEN_NEEDMORE_P(ret)
#define MBCLEN_NEEDMORE_LEN(ret) ONIGENC_MBCLEN_NEEDMORE_LEN(ret)
/* -> 0x00..0x7f, -1 */
int rb_enc_ascget(const char *p, const char *e, int *len, rb_encoding *enc);
/* -> code (and len) or raise exception */
unsigned int rb_enc_codepoint_len(const char *p, const char *e, int *len, rb_encoding *enc);
/* prototype for obsolete function */
unsigned int rb_enc_codepoint(const char *p, const char *e, rb_encoding *enc);
/* overriding macro */
#define rb_enc_codepoint(p,e,enc) rb_enc_codepoint_len((p),(e),0,(enc))
#define rb_enc_mbc_to_codepoint(p, e, enc) ONIGENC_MBC_TO_CODE((enc),(UChar*)(p),(UChar*)(e))
/* -> codelen>0 or raise exception */
int rb_enc_codelen(int code, rb_encoding *enc);
/* -> 0 for invalid codepoint */
int rb_enc_code_to_mbclen(int code, rb_encoding *enc);
#define rb_enc_code_to_mbclen(c, enc) ONIGENC_CODE_TO_MBCLEN((enc), (c));
/* code,ptr,encoding -> write buf */
#define rb_enc_mbcput(c,buf,enc) ONIGENC_CODE_TO_MBC((enc),(c),(UChar*)(buf))
/* start, ptr, end, encoding -> prev_char */
#define rb_enc_prev_char(s,p,e,enc) ((char *)onigenc_get_prev_char_head((enc),(UChar*)(s),(UChar*)(p),(UChar*)(e)))
/* start, ptr, end, encoding -> next_char */
#define rb_enc_left_char_head(s,p,e,enc) ((char *)onigenc_get_left_adjust_char_head((enc),(UChar*)(s),(UChar*)(p),(UChar*)(e)))
#define rb_enc_right_char_head(s,p,e,enc) ((char *)onigenc_get_right_adjust_char_head((enc),(UChar*)(s),(UChar*)(p),(UChar*)(e)))
#define rb_enc_step_back(s,p,e,n,enc) ((char *)onigenc_step_back((enc),(UChar*)(s),(UChar*)(p),(UChar*)(e),(int)(n)))
/* ptr, ptr, encoding -> newline_or_not */
#define rb_enc_is_newline(p,end,enc) ONIGENC_IS_MBC_NEWLINE((enc),(UChar*)(p),(UChar*)(end))
#define rb_enc_isctype(c,t,enc) ONIGENC_IS_CODE_CTYPE((enc),(c),(t))
#define rb_enc_isascii(c,enc) ONIGENC_IS_CODE_ASCII(c)
#define rb_enc_isalpha(c,enc) ONIGENC_IS_CODE_ALPHA((enc),(c))
#define rb_enc_islower(c,enc) ONIGENC_IS_CODE_LOWER((enc),(c))
#define rb_enc_isupper(c,enc) ONIGENC_IS_CODE_UPPER((enc),(c))
#define rb_enc_ispunct(c,enc) ONIGENC_IS_CODE_PUNCT((enc),(c))
#define rb_enc_isalnum(c,enc) ONIGENC_IS_CODE_ALNUM((enc),(c))
#define rb_enc_isprint(c,enc) ONIGENC_IS_CODE_PRINT((enc),(c))
#define rb_enc_isspace(c,enc) ONIGENC_IS_CODE_SPACE((enc),(c))
#define rb_enc_isdigit(c,enc) ONIGENC_IS_CODE_DIGIT((enc),(c))
#define rb_enc_asciicompat(enc) (rb_enc_mbminlen(enc)==1 && !rb_enc_dummy_p(enc))
int rb_enc_casefold(char *to, const char *p, const char *e, rb_encoding *enc);
int rb_enc_toupper(int c, rb_encoding *enc);
int rb_enc_tolower(int c, rb_encoding *enc);
ID rb_intern3(const char*, long, rb_encoding*);
ID rb_interned_id_p(const char *, long, rb_encoding *);
int rb_enc_symname_p(const char*, rb_encoding*);
int rb_enc_symname2_p(const char*, long, rb_encoding*);
int rb_enc_str_coderange(VALUE);
long rb_str_coderange_scan_restartable(const char*, const char*, rb_encoding*, int*);
int rb_enc_str_asciionly_p(VALUE);
#define rb_enc_str_asciicompat_p(str) rb_enc_asciicompat(rb_enc_get(str))
VALUE rb_enc_from_encoding(rb_encoding *enc);
int rb_enc_unicode_p(rb_encoding *enc);
rb_encoding *rb_ascii8bit_encoding(void);
rb_encoding *rb_utf8_encoding(void);
rb_encoding *rb_usascii_encoding(void);
rb_encoding *rb_locale_encoding(void);
rb_encoding *rb_filesystem_encoding(void);
rb_encoding *rb_default_external_encoding(void);
rb_encoding *rb_default_internal_encoding(void);
#ifndef rb_ascii8bit_encindex
int rb_ascii8bit_encindex(void);
#endif
#ifndef rb_utf8_encindex
int rb_utf8_encindex(void);
#endif
#ifndef rb_usascii_encindex
int rb_usascii_encindex(void);
#endif
int rb_locale_encindex(void);
int rb_filesystem_encindex(void);
VALUE rb_enc_default_external(void);
VALUE rb_enc_default_internal(void);
void rb_enc_set_default_external(VALUE encoding);
void rb_enc_set_default_internal(VALUE encoding);
VALUE rb_locale_charmap(VALUE klass);
long rb_memsearch(const void*,long,const void*,long,rb_encoding*);
char *rb_enc_path_next(const char *,const char *,rb_encoding*);
char *rb_enc_path_skip_prefix(const char *,const char *,rb_encoding*);
char *rb_enc_path_last_separator(const char *,const char *,rb_encoding*);
char *rb_enc_path_end(const char *,const char *,rb_encoding*);
const char *ruby_enc_find_basename(const char *name, long *baselen, long *alllen, rb_encoding *enc);
const char *ruby_enc_find_extname(const char *name, long *len, rb_encoding *enc);
ID rb_check_id_cstr(const char *ptr, long len, rb_encoding *enc);
VALUE rb_check_symbol_cstr(const char *ptr, long len, rb_encoding *enc);
RUBY_EXTERN VALUE rb_cEncoding;
/* econv stuff */
typedef enum {
econv_invalid_byte_sequence,
econv_undefined_conversion,
econv_destination_buffer_full,
econv_source_buffer_empty,
econv_finished,
econv_after_output,
econv_incomplete_input
} rb_econv_result_t;
typedef struct rb_econv_t rb_econv_t;
VALUE rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts);
int rb_econv_has_convpath_p(const char* from_encoding, const char* to_encoding);
int rb_econv_prepare_options(VALUE opthash, VALUE *ecopts, int ecflags);
int rb_econv_prepare_opts(VALUE opthash, VALUE *ecopts);
rb_econv_t *rb_econv_open(const char *source_encoding, const char *destination_encoding, int ecflags);
rb_econv_t *rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE ecopts);
rb_econv_result_t rb_econv_convert(rb_econv_t *ec,
const unsigned char **source_buffer_ptr, const unsigned char *source_buffer_end,
unsigned char **destination_buffer_ptr, unsigned char *destination_buffer_end,
int flags);
void rb_econv_close(rb_econv_t *ec);
/* result: 0:success -1:failure */
int rb_econv_set_replacement(rb_econv_t *ec, const unsigned char *str, size_t len, const char *encname);
/* result: 0:success -1:failure */
int rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name);
int rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name);
VALUE rb_econv_open_exc(const char *senc, const char *denc, int ecflags);
/* result: 0:success -1:failure */
int rb_econv_insert_output(rb_econv_t *ec,
const unsigned char *str, size_t len, const char *str_encoding);
/* encoding that rb_econv_insert_output doesn't need conversion */
const char *rb_econv_encoding_to_insert_output(rb_econv_t *ec);
/* raise an error if the last rb_econv_convert is error */
void rb_econv_check_error(rb_econv_t *ec);
/* returns an exception object or nil */
VALUE rb_econv_make_exception(rb_econv_t *ec);
int rb_econv_putbackable(rb_econv_t *ec);
void rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n);
/* returns the corresponding ASCII compatible encoding for encname,
* or NULL if encname is not ASCII incompatible encoding. */
const char *rb_econv_asciicompat_encoding(const char *encname);
VALUE rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags);
VALUE rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags);
VALUE rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags);
VALUE rb_econv_substr_append(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, VALUE dst, int flags);
VALUE rb_econv_append(rb_econv_t *ec, const char *bytesrc, long bytesize, VALUE dst, int flags);
void rb_econv_binmode(rb_econv_t *ec);
/* flags for rb_econv_open */
#define ECONV_ERROR_HANDLER_MASK 0x000000ff
#define ECONV_INVALID_MASK 0x0000000f
#define ECONV_INVALID_REPLACE 0x00000002
#define ECONV_UNDEF_MASK 0x000000f0
#define ECONV_UNDEF_REPLACE 0x00000020
#define ECONV_UNDEF_HEX_CHARREF 0x00000030
#define ECONV_DECORATOR_MASK 0x0000ff00
#define ECONV_NEWLINE_DECORATOR_MASK 0x00003f00
#define ECONV_NEWLINE_DECORATOR_READ_MASK 0x00000f00
#define ECONV_NEWLINE_DECORATOR_WRITE_MASK 0x00003000
#define ECONV_UNIVERSAL_NEWLINE_DECORATOR 0x00000100
#define ECONV_CRLF_NEWLINE_DECORATOR 0x00001000
#define ECONV_CR_NEWLINE_DECORATOR 0x00002000
#define ECONV_XML_TEXT_DECORATOR 0x00004000
#define ECONV_XML_ATTR_CONTENT_DECORATOR 0x00008000
#define ECONV_STATEFUL_DECORATOR_MASK 0x00f00000
#define ECONV_XML_ATTR_QUOTE_DECORATOR 0x00100000
#if defined(RUBY_TEST_CRLF_ENVIRONMENT) || defined(_WIN32)
#define ECONV_DEFAULT_NEWLINE_DECORATOR ECONV_CRLF_NEWLINE_DECORATOR
#else
#define ECONV_DEFAULT_NEWLINE_DECORATOR 0
#endif
/* end of flags for rb_econv_open */
/* flags for rb_econv_convert */
#define ECONV_PARTIAL_INPUT 0x00010000
#define ECONV_AFTER_OUTPUT 0x00020000
/* end of flags for rb_econv_convert */
RUBY_SYMBOL_EXPORT_END
#if defined(__cplusplus)
#if 0
{ /* satisfy cc-mode */
#endif
} /* extern "C" { */
#endif
#endif /* RUBY_ENCODING_H */