Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 127 lines (101 sloc) 4.853 kb
be86e3d @nurse * transocode.c: register_functional_transcoder() added.
nurse authored
1 /**********************************************************************
2
3 transcode_data.h -
4
5 $Author$
6 created at: Mon 10 Dec 2007 14:01:47 JST 2007
7
8 Copyright (C) 2007 Martin Duerst
9
10 **********************************************************************/
11
12 #include "ruby/ruby.h"
13
14 #ifndef RUBY_TRANSCODE_DATA_H
15 #define RUBY_TRANSCODE_DATA_H 1
16
7ded13f * transcode.c: new file to provide encoding conversion features.
matz authored
17 typedef unsigned char base_element;
18
19 typedef struct byte_lookup {
20 const base_element *base;
3a3bda7 @nobu * string.c (rb_str_tmp_new): creates hidden temporary buffer.
nobu authored
21 const struct byte_lookup *const *info;
7ded13f * transcode.c: new file to provide encoding conversion features.
matz authored
22 } BYTE_LOOKUP;
23
b7db903 @nobu * common.mk (COMMONOBJS): transcode_data_*.c moved under enc/trans.
nobu authored
24 #ifndef PType
7ded13f * transcode.c: new file to provide encoding conversion features.
matz authored
25 /* data file needs to treat this as a pointer, to remove warnings */
3a3bda7 @nobu * string.c (rb_str_tmp_new): creates hidden temporary buffer.
nobu authored
26 #define PType (const BYTE_LOOKUP *)
7ded13f * transcode.c: new file to provide encoding conversion features.
matz authored
27 #endif
28
793e942 Fri Dec 28 01:55:04 2007 Martin Duerst <duerst@it.aoyama.ac.jp>
duerst authored
29 #define NOMAP (PType 0x01) /* single byte direct map */
30 #define ONEbt (0x02) /* one byte payload */
31 #define TWObt (0x03) /* two bytes payload */
32 #define THREEbt (0x05) /* three bytes payload */
33 #define FOURbt (0x06) /* four bytes payload, UTF-8 only, macros start at getBT0 */
34 #define INVALID (PType 0x07) /* invalid byte sequence */
35 #define UNDEF (PType 0x09) /* legal but undefined */
36 #define ZERObt (PType 0x0A) /* zero bytes of payload, i.e. remove */
37 #define FUNii (PType 0x0B) /* function from info to info */
3d0c7be Sun Jan 20 15:08:08 2008 Martin Duerst <duerst@it.aoyama.ac.jp>
duerst authored
38 #define FUNsi (PType 0x0D) /* function from start to info */
39 #define FUNio (PType 0x0E) /* function from info to output */
40 #define FUNso (PType 0x0F) /* function from start to output */
793e942 Fri Dec 28 01:55:04 2007 Martin Duerst <duerst@it.aoyama.ac.jp>
duerst authored
41
42 #define o1(b1) (PType((((unsigned char)(b1))<<8)|ONEbt))
43 #define o2(b1,b2) (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|TWObt))
44 #define o3(b1,b2,b3) (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|(((unsigned char)(b3))<<24)|THREEbt))
45 #define o4(b0,b1,b2,b3) (PType((((unsigned char)(b1))<< 8)|(((unsigned char)(b2))<<16)|(((unsigned char)(b3))<<24)|((((unsigned char)(b0))&0x07)<<5)|FOURbt))
46
47 #define getBT1(a) (((a)>> 8)&0xFF)
48 #define getBT2(a) (((a)>>16)&0xFF)
49 #define getBT3(a) (((a)>>24)&0xFF)
50 #define getBT0(a) ((((a)>> 5)&0x07)|0xF0) /* for UTF-8 only!!! */
51
52 #define o2FUNii(b1,b2) (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|FUNii))
7ded13f * transcode.c: new file to provide encoding conversion features.
matz authored
53
54 /* do we need these??? maybe not, can be done with simple tables */
55 #define ONETRAIL /* legal but undefined if one more trailing UTF-8 */
56 #define TWOTRAIL /* legal but undefined if two more trailing UTF-8 */
57 #define THREETRAIL /* legal but undefined if three more trailing UTF-8 */
58
a55e167 @akr * transcode_data.h (rb_transcoder_stateful_type_t): defined.
akr authored
59 typedef enum {
60 stateless_converter, /* stateless -> stateless */
61 stateful_decoder, /* stateful -> stateless */
62 stateful_encoder /* stateless -> stateful */
63 /* stateful -> stateful is intentionally ommitted. */
64 } rb_transcoder_stateful_type_t;
65
c136113 @akr * transcode_data.h (rb_transcoder): typedef at first.
akr authored
66 typedef struct rb_transcoder rb_transcoder;
67
be86e3d @nurse * transocode.c: register_functional_transcoder() added.
nurse authored
68 /* dynamic structure, one per conversion (similar to iconv_t) */
69 /* may carry conversion state (e.g. for iso-2022-jp) */
b7db903 @nobu * common.mk (COMMONOBJS): transcode_data_*.c moved under enc/trans.
nobu authored
70 typedef struct rb_transcoding {
c136113 @akr * transcode_data.h (rb_transcoder): typedef at first.
akr authored
71 const rb_transcoder *transcoder;
1504652 @akr * transcode_data.h (rb_transcoding): new field "stateful".
akr authored
72
436a136 @akr * transcode_data.h (rb_transcoding): new field: flags.
akr authored
73 int flags;
74
139234e @akr * transcode_data.h (rb_transcoding): add fields for restartable
akr authored
75 int resume_position;
76 const BYTE_LOOKUP *next_table;
4bceda1 @akr * transcode_data.h (rb_transcoding): new fields: next_info and next_byte...
akr authored
77 VALUE next_info;
78 unsigned char next_byte;
04f7390 @akr * transcode_data.h (rb_transcoding): add fields: writebuf_off,
akr authored
79
bd5bf37 @akr * transcode_data.h (rb_transcoding): rename fields.
akr authored
80 int recognized_len; /* already interpreted */
81 int readagain_len; /* not yet interpreted */
139234e @akr * transcode_data.h (rb_transcoding): add fields for restartable
akr authored
82 union {
83 unsigned char ary[8]; /* max_input <= sizeof(ary) */
04f7390 @akr * transcode_data.h (rb_transcoding): add fields: writebuf_off,
akr authored
84 unsigned char *ptr; /* length: max_input */
bd5bf37 @akr * transcode_data.h (rb_transcoding): rename fields.
akr authored
85 } readbuf; /* recognized_len + readagain_len used */
139234e @akr * transcode_data.h (rb_transcoding): add fields for restartable
akr authored
86
04f7390 @akr * transcode_data.h (rb_transcoding): add fields: writebuf_off,
akr authored
87 int writebuf_off;
88 int writebuf_len;
89 union {
90 unsigned char ary[8]; /* max_output <= sizeof(ary) */
91 unsigned char *ptr; /* length: max_output */
92 } writebuf;
93
1504652 @akr * transcode_data.h (rb_transcoding): new field "stateful".
akr authored
94 unsigned char stateful[256]; /* opaque data for stateful encoding */
b7db903 @nobu * common.mk (COMMONOBJS): transcode_data_*.c moved under enc/trans.
nobu authored
95 } rb_transcoding;
139234e @akr * transcode_data.h (rb_transcoding): add fields for restartable
akr authored
96 #define TRANSCODING_READBUF(tc) \
97 ((tc)->transcoder->max_input <= sizeof((tc)->readbuf.ary) ? \
98 (tc)->readbuf.ary : \
99 (tc)->readbuf.ptr)
04f7390 @akr * transcode_data.h (rb_transcoding): add fields: writebuf_off,
akr authored
100 #define TRANSCODING_WRITEBUF(tc) \
101 ((tc)->transcoder->max_output <= sizeof((tc)->writebuf.ary) ? \
102 (tc)->writebuf.ary : \
103 (tc)->writebuf.ptr)
be86e3d @nurse * transocode.c: register_functional_transcoder() added.
nurse authored
104
105 /* static structure, one per supported encoding pair */
c136113 @akr * transcode_data.h (rb_transcoder): typedef at first.
akr authored
106 struct rb_transcoder {
be86e3d @nurse * transocode.c: register_functional_transcoder() added.
nurse authored
107 const char *from_encoding;
108 const char *to_encoding;
109 const BYTE_LOOKUP *conv_tree_start;
139234e @akr * transcode_data.h (rb_transcoding): add fields for restartable
akr authored
110 int input_unit_length;
111 int max_input;
be86e3d @nurse * transocode.c: register_functional_transcoder() added.
nurse authored
112 int max_output;
a55e167 @akr * transcode_data.h (rb_transcoder_stateful_type_t): defined.
akr authored
113 rb_transcoder_stateful_type_t stateful_type;
1504652 @akr * transcode_data.h (rb_transcoding): new field "stateful".
akr authored
114 VALUE (*func_ii)(rb_transcoding*, VALUE); /* info -> info */
115 VALUE (*func_si)(rb_transcoding*, const unsigned char*, size_t); /* start -> info */
116 int (*func_io)(rb_transcoding*, VALUE, const unsigned char*); /* info -> output */
117 int (*func_so)(rb_transcoding*, const unsigned char*, size_t, unsigned char*); /* start -> output */
118 int (*finish_func)(rb_transcoding*, unsigned char*); /* -> output */
74a36d5 @akr * include/ruby/encoding.h (rb_econv_output): declared.
akr authored
119 int (*resetsize_func)(rb_transcoding*); /* -> len */
120 int (*resetstate_func)(rb_transcoding*, unsigned char*); /* -> output */
c136113 @akr * transcode_data.h (rb_transcoder): typedef at first.
akr authored
121 };
b7db903 @nobu * common.mk (COMMONOBJS): transcode_data_*.c moved under enc/trans.
nobu authored
122
123 void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib);
124 void rb_register_transcoder(const rb_transcoder *);
be86e3d @nurse * transocode.c: register_functional_transcoder() added.
nurse authored
125
126 #endif /* RUBY_TRANSCODE_DATA_H */
Something went wrong with that request. Please try again.