Skip to content

Commit 55e0a9c

Browse files
author
bar@gw.udmsearch.izhnet.ru
committed
New UTF8 charset
1 parent 1b54e7c commit 55e0a9c

14 files changed

Lines changed: 2088 additions & 54 deletions

File tree

‎configure.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1872,7 +1872,7 @@ CHARSETS_AVAILABLE="armscii8 big5 cp1251 cp1257
18721872
latin1 latin1_de latin2 latin5 sjis swe7 tis620 ujis
18731873
usa7 utf8 win1250 win1251ukr"
18741874
CHARSETS_DEPRECATED="win1251"
1875-
CHARSETS_COMPLEX="big5 czech euc_kr gb2312 gbk latin1_de sjis tis620 ujis"
1875+
CHARSETS_COMPLEX="big5 czech euc_kr gb2312 gbk latin1_de sjis tis620 ujis utf8"
18761876
DEFAULT_CHARSET=latin1
18771877
AC_DIVERT_POP
18781878

‎include/m_ctype.h‎

Lines changed: 67 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,22 @@ extern "C" {
2929

3030
#define CHARSET_DIR "charsets/"
3131

32+
#define my_wc_t ulong
33+
34+
typedef struct unicase_info_st {
35+
uint16 toupper;
36+
uint16 tolower;
37+
uint16 sort;
38+
} MY_UNICASE_INFO;
39+
40+
#define MY_CS_ILSEQ 0
41+
#define MY_CS_ILUNI 0
42+
#define MY_CS_TOOSMALL -1
43+
#define MY_CS_TOOFEW(n) (-1-(n))
44+
45+
46+
47+
3248
typedef struct charset_info_st
3349
{
3450
uint number;
@@ -48,9 +64,9 @@ typedef struct charset_info_st
4864
char *, char *, uint *, uint *);
4965

5066
uint mbmaxlen;
51-
int (*ismbchar)(const char *, const char *);
52-
my_bool (*ismbhead)(uint);
53-
int (*mbcharlen)(uint);
67+
int (*ismbchar)(struct charset_info_st *, const char *, const char *);
68+
my_bool (*ismbhead)(struct charset_info_st *, uint);
69+
int (*mbcharlen)(struct charset_info_st *, uint);
5470

5571
/* Functions for case convertion */
5672
void (*caseup_str)(struct charset_info_st *, char *);
@@ -107,9 +123,9 @@ extern int my_strnncoll_big5(CHARSET_INFO *,const uchar *, uint, const uchar
107123
extern int my_strnxfrm_big5(CHARSET_INFO *,uchar *, uint, const uchar *, uint);
108124
extern my_bool my_like_range_big5(CHARSET_INFO *,const char *, uint, pchar, uint,
109125
char *, char *, uint *, uint *);
110-
extern int ismbchar_big5(const char *, const char *);
111-
extern my_bool ismbhead_big5(uint);
112-
extern int mbcharlen_big5(uint);
126+
extern int ismbchar_big5(CHARSET_INFO *, const char *, const char *);
127+
extern my_bool ismbhead_big5(CHARSET_INFO *, uint);
128+
extern int mbcharlen_big5(CHARSET_INFO *, uint);
113129
#endif
114130

115131
#ifdef HAVE_CHARSET_czech
@@ -125,17 +141,17 @@ extern my_bool my_like_range_czech(CHARSET_INFO *,
125141
#ifdef HAVE_CHARSET_euc_kr
126142
/* declarations for the euc_kr character set */
127143
extern uchar ctype_euc_kr[], to_lower_euc_kr[], to_upper_euc_kr[], sort_order_euc_kr[];
128-
extern int ismbchar_euc_kr(const char *, const char *);
129-
extern my_bool ismbhead_euc_kr(uint);
130-
extern int mbcharlen_euc_kr(uint);
144+
extern int ismbchar_euc_kr(CHARSET_INFO *, const char *, const char *);
145+
extern my_bool ismbhead_euc_kr(CHARSET_INFO *, uint);
146+
extern int mbcharlen_euc_kr(CHARSET_INFO *, uint);
131147
#endif
132148

133149
#ifdef HAVE_CHARSET_gb2312
134150
/* declarations for the gb2312 character set */
135151
extern uchar ctype_gb2312[], to_lower_gb2312[], to_upper_gb2312[], sort_order_gb2312[];
136-
extern int ismbchar_gb2312(const char *, const char *);
137-
extern my_bool ismbhead_gb2312(uint);
138-
extern int mbcharlen_gb2312(uint);
152+
extern int ismbchar_gb2312(CHARSET_INFO *, const char *, const char *);
153+
extern my_bool ismbhead_gb2312(CHARSET_INFO *, uint);
154+
extern int mbcharlen_gb2312(CHARSET_INFO *, uint);
139155
#endif
140156

141157
#ifdef HAVE_CHARSET_gbk
@@ -145,9 +161,9 @@ extern int my_strnncoll_gbk(CHARSET_INFO *, const uchar *, uint, const uchar
145161
extern int my_strnxfrm_gbk(CHARSET_INFO *, uchar *, uint, const uchar *, uint);
146162
extern my_bool my_like_range_gbk(CHARSET_INFO *, const char *, uint, pchar, uint,
147163
char *, char *, uint *, uint *);
148-
extern int ismbchar_gbk(const char *, const char *);
149-
extern my_bool ismbhead_gbk(uint);
150-
extern int mbcharlen_gbk(uint);
164+
extern int ismbchar_gbk(CHARSET_INFO *, const char *, const char *);
165+
extern my_bool ismbhead_gbk(CHARSET_INFO *, uint);
166+
extern int mbcharlen_gbk(CHARSET_INFO *, uint);
151167
#endif
152168

153169
#ifdef HAVE_CHARSET_latin1_de
@@ -166,9 +182,9 @@ extern int my_strnncoll_sjis(CHARSET_INFO *, const uchar *, uint, const ucha
166182
extern int my_strnxfrm_sjis(CHARSET_INFO *, uchar *, uint, const uchar *, uint);
167183
extern my_bool my_like_range_sjis(CHARSET_INFO *, const char *, uint, pchar, uint,
168184
char *, char *, uint *, uint *);
169-
extern int ismbchar_sjis(const char *, const char *);
170-
extern my_bool ismbhead_sjis(uint);
171-
extern int mbcharlen_sjis(uint);
185+
extern int ismbchar_sjis(CHARSET_INFO *, const char *, const char *);
186+
extern my_bool ismbhead_sjis(CHARSET_INFO *, uint);
187+
extern int mbcharlen_sjis(CHARSET_INFO *, uint);
172188
#endif
173189

174190
#ifdef HAVE_CHARSET_tis620
@@ -183,11 +199,38 @@ extern my_bool my_like_range_tis620(CHARSET_INFO *, const char *, uint, pchar, u
183199
#ifdef HAVE_CHARSET_ujis
184200
/* declarations for the ujis character set */
185201
extern uchar ctype_ujis[], to_lower_ujis[], to_upper_ujis[], sort_order_ujis[];
186-
extern int ismbchar_ujis(const char *, const char *);
187-
extern my_bool ismbhead_ujis(uint);
188-
extern int mbcharlen_ujis(uint);
202+
extern int ismbchar_ujis(CHARSET_INFO *, const char *, const char *);
203+
extern my_bool ismbhead_ujis(CHARSET_INFO *, uint);
204+
extern int mbcharlen_ujis(CHARSET_INFO *, uint);
189205
#endif
190206

207+
#ifdef HAVE_CHARSET_utf8
208+
209+
extern uchar ctype_utf8[];
210+
extern uchar to_lower_utf8[];
211+
extern uchar to_upper_utf8[];
212+
213+
int my_strnncoll_utf8(CHARSET_INFO *cs,
214+
const uchar *s, uint s_len, const uchar *t, uint t_len);
215+
216+
int my_strnxfrm_utf8(CHARSET_INFO *cs,
217+
uchar *dest, uint destlen, const uchar *src, uint srclen);
218+
219+
int my_ismbchar_utf8(CHARSET_INFO *cs, const char *b, const char *e);
220+
my_bool my_ismbhead_utf8(CHARSET_INFO * cs, uint ch);
221+
int my_mbcharlen_utf8(CHARSET_INFO *cs, uint c);
222+
223+
void my_caseup_str_utf8(CHARSET_INFO * cs, char * s);
224+
void my_casedn_str_utf8(CHARSET_INFO *cs, char * s);
225+
void my_caseup_utf8(CHARSET_INFO *cs, char *s, uint len);
226+
void my_casedn_utf8(CHARSET_INFO *cs, char *s, uint len);
227+
228+
int my_strcasecmp_utf8(CHARSET_INFO *cs, const char *s, const char *t);
229+
int my_strncasecmp_utf8(CHARSET_INFO *cs, const char *s,const char *t,uint l);
230+
231+
int my_utf8_uni (CHARSET_INFO *cs, my_wc_t *p, const uchar *s, const uchar *e);
232+
int my_uni_utf8 (CHARSET_INFO *cs, my_wc_t pwc , uchar *b, uchar *e);
233+
#endif
191234

192235
#define _U 01 /* Upper case */
193236
#define _L 02 /* Lower case */
@@ -229,9 +272,9 @@ extern int mbcharlen_ujis(uint);
229272
((s)->like_range((s), (a), (b), (c), (d), (e), (f), (g), (h)))
230273

231274
#define use_mb(s) ((s)->ismbchar != NULL)
232-
#define my_ismbchar(s, a, b) ((s)->ismbchar((a), (b)))
233-
#define my_ismbhead(s, a) ((s)->ismbhead((a)))
234-
#define my_mbcharlen(s, a) ((s)->mbcharlen((a)))
275+
#define my_ismbchar(s, a, b) ((s)->ismbchar((s), (a), (b)))
276+
#define my_ismbhead(s, a) ((s)->ismbhead((s), (a)))
277+
#define my_mbcharlen(s, a) ((s)->mbcharlen((s),(a)))
235278

236279
#define my_caseup(s, a, l) ((s)->caseup((s), (a), (l)))
237280
#define my_casedn(s, a, l) ((s)->casedn((s), (a), (l)))

‎libmysql/Makefile.shared‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ mystringsobjects = strmov.lo strxmov.lo strxnmov.lo strnmov.lo \
4141
ctype.lo ctype-simple.lo ctype-mb.lo \
4242
ctype-big5.lo ctype-czech.lo ctype-euc_kr.lo \
4343
ctype-gb2312.lo ctype-gbk.lo ctype-latin1_de.lo \
44-
ctype-sjis.lo ctype-tis620.lo ctype-ujis.lo
44+
ctype-sjis.lo ctype-tis620.lo ctype-ujis.lo ctype-utf8.lo
4545

4646
mystringsextra= strto.c
4747
dbugobjects = dbug.lo # IT IS IN SAFEMALLOC.C sanity.lo

‎sql/init.cc‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,10 @@ void unireg_init(ulong options)
5757

5858
for (cs=compiled_charsets; cs->number; cs++)
5959
{
60-
uchar max_char=cs->sort_order[(uchar) cs->max_sort_char];
60+
uchar max_char;
61+
if (!cs->sort_order)
62+
continue;
63+
cs->sort_order[(uchar) cs->max_sort_char];
6164
for (i = 0; i < 256; i++)
6265
{
6366
if ((uchar) cs->sort_order[i] > max_char)

‎sql/share/charsets/Index‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,4 @@ cp1257 29
3737
latin5 30
3838
latin1_de 31
3939
armscii8 32
40+
utf8 33

‎strings/Makefile.am‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,27 +22,27 @@ pkglib_LIBRARIES = libmystrings.a
2222
# Exact one of ASSEMBLER_X
2323
if ASSEMBLER_x86
2424
ASRCS = strings-x86.s longlong2str-x86.s
25-
CSRCS = bfill.c bmove.c bmove512.c bchange.c strxnmov.c int2str.c str2int.c r_strinstr.c atof.c bcmp.c strtol.c strtoul.c strtoll.c strtoull.c llstr.c strnlen.c ctype.c ctype-simple.c ctype-mb.c ctype-big5.c ctype-czech.c ctype-euc_kr.c ctype-gb2312.c ctype-gbk.c ctype-latin1_de.c ctype-sjis.c ctype-tis620.c ctype-ujis.c
25+
CSRCS = bfill.c bmove.c bmove512.c bchange.c strxnmov.c int2str.c str2int.c r_strinstr.c atof.c bcmp.c strtol.c strtoul.c strtoll.c strtoull.c llstr.c strnlen.c ctype.c ctype-simple.c ctype-mb.c ctype-big5.c ctype-czech.c ctype-euc_kr.c ctype-gb2312.c ctype-gbk.c ctype-latin1_de.c ctype-sjis.c ctype-tis620.c ctype-ujis.c ctype-utf8.c
2626
else
2727
if ASSEMBLER_sparc
2828
# These file MUST all be on the same line!! Otherwise automake
2929
# generats a very broken makefile
3030
ASRCS = bmove_upp-sparc.s strappend-sparc.s strend-sparc.s strinstr-sparc.s strmake-sparc.s strmov-sparc.s strnmov-sparc.s strstr-sparc.s strxmov-sparc.s
31-
CSRCS = strcont.c strfill.c strcend.c is_prefix.c longlong2str.c bfill.c bmove.c bmove512.c bchange.c strxnmov.c int2str.c str2int.c r_strinstr.c atof.c bcmp.c strtol.c strtoul.c strtoll.c strtoull.c llstr.c strnlen.c ctype.c ctype-simple.c ctype-mb.c ctype-big5.c ctype-czech.c ctype-euc_kr.c ctype-gb2312.c ctype-gbk.c ctype-latin1_de.c ctype-sjis.c ctype-tis620.c ctype-ujis.c
31+
CSRCS = strcont.c strfill.c strcend.c is_prefix.c longlong2str.c bfill.c bmove.c bmove512.c bchange.c strxnmov.c int2str.c str2int.c r_strinstr.c atof.c bcmp.c strtol.c strtoul.c strtoll.c strtoull.c llstr.c strnlen.c ctype.c ctype-simple.c ctype-mb.c ctype-big5.c ctype-czech.c ctype-euc_kr.c ctype-gb2312.c ctype-gbk.c ctype-latin1_de.c ctype-sjis.c ctype-tis620.c ctype-ujis.cctype-utf8.c
3232
else
3333
#no assembler
3434
ASRCS =
3535
# These file MUST all be on the same line!! Otherwise automake
3636
# generats a very broken makefile
37-
CSRCS = strxmov.c bmove_upp.c strappend.c strcont.c strend.c strfill.c strcend.c is_prefix.c strstr.c strinstr.c strmake.c strnmov.c strmov.c longlong2str.c bfill.c bmove.c bmove512.c bchange.c strxnmov.c int2str.c str2int.c r_strinstr.c atof.c bcmp.c strtol.c strtoul.c strtoll.c strtoull.c llstr.c strnlen.c ctype.c ctype-simple.c ctype-mb.c ctype-big5.c ctype-czech.c ctype-euc_kr.c ctype-gb2312.c ctype-gbk.c ctype-latin1_de.c ctype-sjis.c ctype-tis620.c ctype-ujis.c
37+
CSRCS = strxmov.c bmove_upp.c strappend.c strcont.c strend.c strfill.c strcend.c is_prefix.c strstr.c strinstr.c strmake.c strnmov.c strmov.c longlong2str.c bfill.c bmove.c bmove512.c bchange.c strxnmov.c int2str.c str2int.c r_strinstr.c atof.c bcmp.c strtol.c strtoul.c strtoll.c strtoull.c llstr.c strnlen.c ctype.c ctype-simple.c ctype-mb.c ctype-big5.c ctype-czech.c ctype-euc_kr.c ctype-gb2312.c ctype-gbk.c ctype-latin1_de.c ctype-sjis.c ctype-tis620.c ctype-ujis.c ctype-utf8.c
3838
endif
3939
endif
4040

4141
libmystrings_a_SOURCES = $(ASRCS) $(CSRCS)
4242
noinst_PROGRAMS = conf_to_src
4343
# Default charset definitions
4444
EXTRA_DIST = ctype-big5.c ctype-czech.c ctype-euc_kr.c \
45-
ctype-gb2312.c ctype-gbk.c ctype-sjis.c \
45+
ctype-gb2312.c ctype-gbk.c ctype-sjis.c ctype-utf8.c \
4646
ctype-tis620.c ctype-ujis.c ctype-latin1_de.c \
4747
strto.c strings-x86.s longlong2str-x86.s \
4848
strxmov.c bmove_upp.c strappend.c strcont.c strend.c \

‎strings/ctype-big5.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -378,17 +378,17 @@ my_bool my_like_range_big5(CHARSET_INFO *cs,
378378
return 0;
379379
}
380380

381-
int ismbchar_big5(const char* p, const char *e)
381+
int ismbchar_big5(CHARSET_INFO *cs,const char* p, const char *e)
382382
{
383383
return (isbig5head(*(p)) && (e)-(p)>1 && isbig5tail(*((p)+1))? 2: 0);
384384
}
385385

386-
my_bool ismbhead_big5(uint c)
386+
my_bool ismbhead_big5(CHARSET_INFO *cs, uint c)
387387
{
388388
return isbig5head(c);
389389
}
390390

391-
int mbcharlen_big5(uint c)
391+
int mbcharlen_big5(CHARSET_INFO *cs, uint c)
392392
{
393393
return (isbig5head(c)? 2: 0);
394394
}

‎strings/ctype-euc_kr.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -183,19 +183,19 @@ uchar NEAR sort_order_euc_kr[]=
183183
#define iseuc_kr(c) ((0xa1<=(uchar)(c) && (uchar)(c)<=0xfe))
184184

185185

186-
int ismbchar_euc_kr(const char* p, const char *e)
186+
int ismbchar_euc_kr(CHARSET_INFO *cs,const char* p, const char *e)
187187
{
188188
return ((*(uchar*)(p)<0x80)? 0:\
189189
iseuc_kr(*(p)) && (e)-(p)>1 && iseuc_kr(*((p)+1))? 2:\
190190
0);
191191
}
192192

193-
my_bool ismbhead_euc_kr(uint c)
193+
my_bool ismbhead_euc_kr(CHARSET_INFO *cs,uint c)
194194
{
195195
return (iseuc_kr(c));
196196
}
197197

198-
int mbcharlen_euc_kr(uint c)
198+
int mbcharlen_euc_kr(CHARSET_INFO *cs,uint c)
199199
{
200200
return (iseuc_kr(c) ? 2 : 0);
201201
}

‎strings/ctype-gb2312.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -166,17 +166,17 @@ uchar NEAR sort_order_gb2312[]=
166166
#define isgb2312tail(c) (0xa1<=(uchar)(c) && (uchar)(c)<=0xfe)
167167

168168

169-
int ismbchar_gb2312(const char* p, const char *e)
169+
int ismbchar_gb2312(CHARSET_INFO *cs,const char* p, const char *e)
170170
{
171171
return (isgb2312head(*(p)) && (e)-(p)>1 && isgb2312tail(*((p)+1))? 2: 0);
172172
}
173173

174-
my_bool ismbhead_gb2312(uint c)
174+
my_bool ismbhead_gb2312(CHARSET_INFO *cs,uint c)
175175
{
176176
return isgb2312head(c);
177177
}
178178

179-
int mbcharlen_gb2312(uint c)
179+
int mbcharlen_gb2312(CHARSET_INFO *cs,uint c)
180180
{
181181
return (isgb2312head(c)? 2:0);
182182
}

‎strings/ctype-gbk.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2704,17 +2704,17 @@ extern my_bool my_like_range_gbk(CHARSET_INFO *cs,
27042704
}
27052705

27062706

2707-
int ismbchar_gbk(const char* p, const char *e)
2707+
int ismbchar_gbk(CHARSET_INFO *cs,const char* p, const char *e)
27082708
{
27092709
return (isgbkhead(*(p)) && (e)-(p)>1 && isgbktail(*((p)+1))? 2: 0);
27102710
}
27112711

2712-
my_bool ismbhead_gbk(uint c)
2712+
my_bool ismbhead_gbk(CHARSET_INFO *cs,uint c)
27132713
{
27142714
return isgbkhead(c);
27152715
}
27162716

2717-
int mbcharlen_gbk(uint c)
2717+
int mbcharlen_gbk(CHARSET_INFO *cs,uint c)
27182718
{
27192719
return (isgbkhead(c)? 2:0);
27202720
}

0 commit comments

Comments
 (0)