Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 118 lines (95 sloc) 5.297 kb
be86e3d NARUSE, Yui * transocode.c: register_functional_transcoder() added.
nurse authored
1 /**********************************************************************
2
3 transcode_data.h -
4
5 $Author$
6 created at: Mon 10 Dec 2007 14:01:47 JST 2007
7
8 Copyright (C) 2007 Martin Duerst
9
10 **********************************************************************/
11
12 #include "ruby/ruby.h"
13
14 #ifndef RUBY_TRANSCODE_DATA_H
15 #define RUBY_TRANSCODE_DATA_H 1
16
0bd71ff Nobuyoshi Nakada * configure.in (XCFLAGS): use -fvisibility=hidden if possible.
nobu authored
17 #if defined __GNUC__ && __GNUC__ >= 4
18 #pragma GCC visibility push(default)
19 #endif
20
6378201 akr * transcode_data.h (WORDINDEX_SHIFT_BITS): defined.
akr authored
21 #define WORDINDEX_SHIFT_BITS 2
22 #define WORDINDEX2INFO(widx) ((widx) << WORDINDEX_SHIFT_BITS)
23 #define INFO2WORDINDEX(info) ((info) >> WORDINDEX_SHIFT_BITS)
41e1933 akr * transcode_data.h (base_element): removed.
akr authored
24 #define BYTE_LOOKUP_BASE(bl) ((bl)[0])
25 #define BYTE_LOOKUP_INFO(bl) ((bl)[1])
2ae7b03 akr * transcode_data.h (BYTE_LOOKUP_BASE): abstract accessor for
akr authored
26
41e1933 akr * transcode_data.h (base_element): removed.
akr authored
27 #define PType (unsigned int)
7ded13f * transcode.c: new file to provide encoding conversion features.
matz authored
28
3abca79 NARUSE, Yui Fix: DON'T move in_p because before in_p is replaced by buffered data.
nurse authored
29 #define NOMAP (PType 0x01) /* direct map */
793e942 Fri Dec 28 01:55:04 2007 Martin Duerst <duerst@it.aoyama.ac.jp>
duerst authored
30 #define ONEbt (0x02) /* one byte payload */
31 #define TWObt (0x03) /* two bytes payload */
32 #define THREEbt (0x05) /* three bytes payload */
33 #define FOURbt (0x06) /* four bytes payload, UTF-8 only, macros start at getBT0 */
34 #define INVALID (PType 0x07) /* invalid byte sequence */
35 #define UNDEF (PType 0x09) /* legal but undefined */
36 #define ZERObt (PType 0x0A) /* zero bytes of payload, i.e. remove */
37 #define FUNii (PType 0x0B) /* function from info to info */
3d0c7be Sun Jan 20 15:08:08 2008 Martin Duerst <duerst@it.aoyama.ac.jp>
duerst authored
38 #define FUNsi (PType 0x0D) /* function from start to info */
39 #define FUNio (PType 0x0E) /* function from info to output */
40 #define FUNso (PType 0x0F) /* function from start to output */
6eb3843 akr * transcode_data.h (STR1_LENGTH): defined.
akr authored
41 #define STR1 (PType 0x11) /* string 4 <= len <= 259 bytes: 1byte length + content */
82c673d * enc/trans/gb18030.trans, gb18030-tbl.rb:
duerst authored
42 #define GB4bt (PType 0x12) /* GB18030 four bytes payload */
b32ee85 * transcode_data.h, transcode.c, tool/transcode-tblgen.rb: Added
duerst authored
43 #define FUNsio (PType 0x13) /* function from start and info to output */
c0bec2f akr * transcode_data.h (STR1): defined for a string up to 255 bytes.
akr authored
44
d1abc53 Nobuyoshi Nakada * array.c, bignum.c, dln.c, error.c, gc.c, io.c, marshal.c,
nobu authored
45 #define STR1_LENGTH(byte_addr) (unsigned int)(*(byte_addr) + 4)
c0bec2f akr * transcode_data.h (STR1): defined for a string up to 255 bytes.
akr authored
46 #define STR1_BYTEINDEX(w) ((w) >> 6)
47 #define makeSTR1(bi) (((bi) << 6) | STR1)
6eb3843 akr * transcode_data.h (STR1_LENGTH): defined.
akr authored
48 #define makeSTR1LEN(len) ((len)-4)
793e942 Fri Dec 28 01:55:04 2007 Martin Duerst <duerst@it.aoyama.ac.jp>
duerst authored
49
50 #define o1(b1) (PType((((unsigned char)(b1))<<8)|ONEbt))
51 #define o2(b1,b2) (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|TWObt))
e166801 akr * transcode_data.h: suppress warnings of overflow.
akr authored
52 #define o3(b1,b2,b3) (PType(((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|(((unsigned int)(unsigned char)(b3))<<24)|THREEbt)&0xffffffffU))
82c673d * enc/trans/gb18030.trans, gb18030-tbl.rb:
duerst authored
53 #define o4(b0,b1,b2,b3) (PType(((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|(((unsigned char)(b3))<<24)|((((unsigned char)(b0))&0x07)<<5)|FOURbt)&0xffffffffU))
e166801 akr * transcode_data.h: suppress warnings of overflow.
akr authored
54 #define g4(b0,b1,b2,b3) (PType(((((unsigned char)(b0))<<8)|(((unsigned char)(b2))<<16)|((((unsigned char)(b1))&0x0f)<<24)|((((unsigned int)(unsigned char)(b3))&0x0f)<<28)|GB4bt)&0xffffffffU))
b32ee85 * transcode_data.h, transcode.c, tool/transcode-tblgen.rb: Added
duerst authored
55 #define funsio(diff) (PType((((unsigned int)(diff))<<8)|FUNsio))
793e942 Fri Dec 28 01:55:04 2007 Martin Duerst <duerst@it.aoyama.ac.jp>
duerst authored
56
d1abc53 Nobuyoshi Nakada * array.c, bignum.c, dln.c, error.c, gc.c, io.c, marshal.c,
nobu authored
57 #define getBT1(a) ((unsigned char)((a)>> 8))
58 #define getBT2(a) ((unsigned char)((a)>>16))
59 #define getBT3(a) ((unsigned char)((a)>>24))
60 #define getBT0(a) (((unsigned char)((a)>> 5)&0x07)|0xF0) /* for UTF-8 only!!! */
793e942 Fri Dec 28 01:55:04 2007 Martin Duerst <duerst@it.aoyama.ac.jp>
duerst authored
61
d1abc53 Nobuyoshi Nakada * array.c, bignum.c, dln.c, error.c, gc.c, io.c, marshal.c,
nobu authored
62 #define getGB4bt0(a) ((unsigned char)((a)>> 8))
da3cdd2 NARUSE, Yui * gc.c (gc_lazy_sweep): clean a warning.
nurse authored
63 #define getGB4bt1(a) (((unsigned char)((a)>>24)&0x0F)|0x30)
d1abc53 Nobuyoshi Nakada * array.c, bignum.c, dln.c, error.c, gc.c, io.c, marshal.c,
nobu authored
64 #define getGB4bt2(a) ((unsigned char)((a)>>16))
da3cdd2 NARUSE, Yui * gc.c (gc_lazy_sweep): clean a warning.
nurse authored
65 #define getGB4bt3(a) (((unsigned char)((a)>>28)&0x0F)|0x30)
82c673d * enc/trans/gb18030.trans, gb18030-tbl.rb:
duerst authored
66
793e942 Fri Dec 28 01:55:04 2007 Martin Duerst <duerst@it.aoyama.ac.jp>
duerst authored
67 #define o2FUNii(b1,b2) (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|FUNii))
7ded13f * transcode.c: new file to provide encoding conversion features.
matz authored
68
69 /* do we need these??? maybe not, can be done with simple tables */
70 #define ONETRAIL /* legal but undefined if one more trailing UTF-8 */
71 #define TWOTRAIL /* legal but undefined if two more trailing UTF-8 */
72 #define THREETRAIL /* legal but undefined if three more trailing UTF-8 */
73
a55e167 akr * transcode_data.h (rb_transcoder_stateful_type_t): defined.
akr authored
74 typedef enum {
6270ad5 akr * include/ruby/encoding.h (rb_econv_asciicompat_encoding): renamed
akr authored
75 asciicompat_converter, /* ASCII-compatible -> ASCII-compatible */
76 asciicompat_decoder, /* ASCII-incompatible -> ASCII-compatible */
77 asciicompat_encoder /* ASCII-compatible -> ASCII-incompatible */
b9e0294 akr fix typos.
akr authored
78 /* ASCII-incompatible -> ASCII-incompatible is intentionally omitted. */
6270ad5 akr * include/ruby/encoding.h (rb_econv_asciicompat_encoding): renamed
akr authored
79 } rb_transcoder_asciicompat_type_t;
a55e167 akr * transcode_data.h (rb_transcoder_stateful_type_t): defined.
akr authored
80
c136113 akr * transcode_data.h (rb_transcoder): typedef at first.
akr authored
81 typedef struct rb_transcoder rb_transcoder;
82
be86e3d NARUSE, Yui * transocode.c: register_functional_transcoder() added.
nurse authored
83 /* static structure, one per supported encoding pair */
c136113 akr * transcode_data.h (rb_transcoder): typedef at first.
akr authored
84 struct rb_transcoder {
f549f1a akr * transcode_data.h (rb_transcoder): rename fields:
akr authored
85 const char *src_encoding;
86 const char *dst_encoding;
41e1933 akr * transcode_data.h (base_element): removed.
akr authored
87 unsigned int conv_tree_start;
7908180 akr * tool/transcode-tblgen.rb: record offsets array as index of
akr authored
88 const unsigned char *byte_array;
4277473 akr * transcode_data.h (rb_transcoder): new field: byte_array_length and
akr authored
89 unsigned int byte_array_length;
41e1933 akr * transcode_data.h (base_element): removed.
akr authored
90 const unsigned int *word_array;
4277473 akr * transcode_data.h (rb_transcoder): new field: byte_array_length and
akr authored
91 unsigned int word_array_length;
7908180 akr * tool/transcode-tblgen.rb: record offsets array as index of
akr authored
92 int word_size;
139234e akr * transcode_data.h (rb_transcoding): add fields for restartable
akr authored
93 int input_unit_length;
94 int max_input;
be86e3d NARUSE, Yui * transocode.c: register_functional_transcoder() added.
nurse authored
95 int max_output;
6270ad5 akr * include/ruby/encoding.h (rb_econv_asciicompat_encoding): renamed
akr authored
96 rb_transcoder_asciicompat_type_t asciicompat_type;
f6441bf akr * transcode_data.h (rb_transcoding): remove stateful field.
akr authored
97 size_t state_size;
c359b80 akr comment updated.
akr authored
98 int (*state_init_func)(void*); /* ret==0:success ret!=0:failure(errno) */
99 int (*state_fini_func)(void*); /* ret==0:success ret!=0:failure(errno) */
f6441bf akr * transcode_data.h (rb_transcoding): remove stateful field.
akr authored
100 VALUE (*func_ii)(void*, VALUE); /* info -> info */
101 VALUE (*func_si)(void*, const unsigned char*, size_t); /* start -> info */
c767be3 akr * transcode_data.h: return output functions ssize_t.
akr authored
102 ssize_t (*func_io)(void*, VALUE, const unsigned char*, size_t); /* info -> output */
103 ssize_t (*func_so)(void*, const unsigned char*, size_t, unsigned char*, size_t); /* start -> output */
104 ssize_t (*finish_func)(void*, unsigned char*, size_t); /* -> output */
0675246 akr * transcode_data.h (rb_transcoder): resetsize_func and resetstate_func
akr authored
105 ssize_t (*resetsize_func)(void*); /* -> len */
106 ssize_t (*resetstate_func)(void*, unsigned char*, size_t); /* -> output */
b32ee85 * transcode_data.h, transcode.c, tool/transcode-tblgen.rb: Added
duerst authored
107 ssize_t (*func_sio)(void*, const unsigned char*, size_t, VALUE, unsigned char*, size_t); /* start -> output */
c136113 akr * transcode_data.h (rb_transcoder): typedef at first.
akr authored
108 };
b7db903 Nobuyoshi Nakada * common.mk (COMMONOBJS): transcode_data_*.c moved under enc/trans.
nobu authored
109
110 void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib);
111 void rb_register_transcoder(const rb_transcoder *);
be86e3d NARUSE, Yui * transocode.c: register_functional_transcoder() added.
nurse authored
112
0bd71ff Nobuyoshi Nakada * configure.in (XCFLAGS): use -fvisibility=hidden if possible.
nobu authored
113 #if defined __GNUC__ && __GNUC__ >= 4
114 #pragma GCC visibility pop
115 #endif
116
be86e3d NARUSE, Yui * transocode.c: register_functional_transcoder() added.
nurse authored
117 #endif /* RUBY_TRANSCODE_DATA_H */
Something went wrong with that request. Please try again.