Skip to content

Commit

Permalink
Fix conversion of EUC-CN text (and add test suite)
Browse files Browse the repository at this point in the history 
- Flag truncated multi-byte characters as erroneous.
- Don't allow ASCII control characters to appear in the middle of a
  multi-byte character.
- There was a bug whereby some unrecognized Unicode codepoints would be
  passed through unchanged to the output when converting Unicode to
  EUC-CN.
- Stick to the original EUC-CN standard, rather than CP936 (an extended
  version invented by MS).
  • Loading branch information
@alexdowad
alexdowad committed Jun 29, 2021
1 parent 69c979a commit 8b25e38
Show file tree
Hide file tree
Showing 3 changed files with 7,659 additions and 54 deletions.
124 changes: 70 additions & 54 deletions ext/mbstring/libmbfl/filters/mbfilter_euc_cn.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@

#include "unicode_table_cp936.h"

static int mbfl_filt_conv_euccn_wchar_flush(mbfl_convert_filter *filter);

static const unsigned char mblen_table_euccn[] = { /* 0xA1-0xFE */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
Expand Down Expand Up @@ -70,7 +72,7 @@ const struct mbfl_convert_vtbl vtbl_euccn_wchar = {
mbfl_filt_conv_common_ctor,
NULL,
mbfl_filt_conv_euccn_wchar,
mbfl_filt_conv_common_flush,
mbfl_filt_conv_euccn_wchar_flush,
NULL,
};

Expand All @@ -86,51 +88,46 @@ const struct mbfl_convert_vtbl vtbl_wchar_euccn = {

#define CK(statement) do { if ((statement) < 0) return (-1); } while (0)

/*
* EUC-CN => wchar
*/
int
mbfl_filt_conv_euccn_wchar(int c, mbfl_convert_filter *filter)
int mbfl_filt_conv_euccn_wchar(int c, mbfl_convert_filter *filter)
{
int c1, w;

switch (filter->status) {
case 0:
if (c >= 0 && c < 0x80) { /* latin */
if (c >= 0 && c < 0x80) { /* latin */
CK((*filter->output_function)(c, filter->data));
} else if (c > 0xa0 && c < 0xff) { /* dbcs lead byte */
} else if ((c >= 0xA1 && c <= 0xA9) || (c >= 0xB0 && c <= 0xF7)) { /* dbcs lead byte */
filter->status = 1;
filter->cache = c;
} else {
w = c & MBFL_WCSGROUP_MASK;
w |= MBFL_WCSGROUP_THROUGH;
CK((*filter->output_function)(w, filter->data));
CK((*filter->output_function)(c | MBFL_WCSGROUP_THROUGH, filter->data));
}
break;

case 1: /* dbcs second byte */
case 1: /* dbcs second byte */
filter->status = 0;
c1 = filter->cache;
if (c1 > 0xa0 && c1 < 0xff && c > 0xa0 && c < 0xff) {
if (c > 0xA0 && c < 0xFF) {
w = (c1 - 0x81)*192 + (c - 0x40);
if (w >= 0 && w < cp936_ucs_table_size) {
w = cp936_ucs_table[w];
if (w == 0x1864) {
w = 0x30FB;
} else if (w == 0x186A) {
w = 0x2015;
} else if ((w >= 0x1921 && w <= 0x192A) || w == 0x1963 || (w >= 0x1C59 && w <= 0x1C7E) || (w >= 0x1DBB && w <= 0x1DC4)) {
w = 0;
} else {
w = cp936_ucs_table[w];
}
} else {
w = 0;
}
if (w <= 0) {
w = (c1 << 8) | c;
w &= MBFL_WCSPLANE_MASK;
w |= MBFL_WCSPLANE_GB2312;
w = (c1 << 8) | c | MBFL_WCSPLANE_GB2312;
}
CK((*filter->output_function)(w, filter->data));
} else if ((c >= 0 && c < 0x21) || c == 0x7f) { /* CTLs */
CK((*filter->output_function)(c, filter->data));
} else {
w = (c1 << 8) | c;
w &= MBFL_WCSGROUP_MASK;
w |= MBFL_WCSGROUP_THROUGH;
CK((*filter->output_function)(w, filter->data));
CK((*filter->output_function)((c1 << 8) | c | MBFL_WCSGROUP_THROUGH, filter->data));
}
break;

Expand All @@ -142,62 +139,81 @@ mbfl_filt_conv_euccn_wchar(int c, mbfl_convert_filter *filter)
return c;
}

/*
* wchar => EUC-CN
*/
int
mbfl_filt_conv_wchar_euccn(int c, mbfl_convert_filter *filter)
int mbfl_filt_conv_wchar_euccn(int c, mbfl_convert_filter *filter)
{
int c1, c2, s;
int s = 0;

s = 0;
if (c >= ucs_a1_cp936_table_min && c < ucs_a1_cp936_table_max) {
s = ucs_a1_cp936_table[c - ucs_a1_cp936_table_min];
if (c == 0xB7 || c == 0x144 || c == 0x148 || c == 0x251 || c == 0x261) {
s = 0;
} else {
s = ucs_a1_cp936_table[c - ucs_a1_cp936_table_min];
}
} else if (c >= ucs_a2_cp936_table_min && c < ucs_a2_cp936_table_max) {
s = ucs_a2_cp936_table[c - ucs_a2_cp936_table_min];
if (c == 0x2015) {
s = 0xA1AA;
} else if (c == 0x2014 || (c >= 0x2170 && c <= 0x2179)) {
s = 0;
} else {
s = ucs_a2_cp936_table[c - ucs_a2_cp936_table_min];
}
} else if (c >= ucs_a3_cp936_table_min && c < ucs_a3_cp936_table_max) {
s = ucs_a3_cp936_table[c - ucs_a3_cp936_table_min];
if (c == 0x30FB) {
s = 0xA1A4;
} else {
s = ucs_a3_cp936_table[c - ucs_a3_cp936_table_min];
}
} else if (c >= ucs_i_cp936_table_min && c < ucs_i_cp936_table_max) {
s = ucs_i_cp936_table[c - ucs_i_cp936_table_min];
} else if (c >= ucs_hff_cp936_table_min && c < ucs_hff_cp936_table_max) {
if (c == 0xff04) {
s = 0xa1e7;
} else if (c == 0xff5e) {
s = 0xa1ab;
} else if (c >= 0xff01 && c <= 0xff5d) {
s = c - 0xff01 + 0xa3a1;
} else if (c >= 0xffe0 && c <= 0xffe5) {
s = ucs_hff_s_cp936_table[c-0xffe0];
if (c == 0xFF04) {
s = 0xA1E7;
} else if (c == 0xFF5E) {
s = 0xA1AB;
} else if (c >= 0xFF01 && c <= 0xFF5D) {
s = c - 0xFF01 + 0xA3A1;
} else if (c >= 0xFFE0 && c <= 0xFFE5) {
s = ucs_hff_s_cp936_table[c - 0xFFE0];
}
}
c1 = (s >> 8) & 0xff;
c2 = s & 0xff;

if (c1 < 0xa1 || c2 < 0xa1) { /* exclude CP936 extension */
s = c;
/* exclude CP936 extensions */
if (((s >> 8) & 0xFF) < 0xA1 || (s & 0xFF) < 0xA1) {
s = 0;
}

if (s <= 0) {
c1 = c & ~MBFL_WCSPLANE_MASK;
if (c1 == MBFL_WCSPLANE_GB2312) {
s = c & MBFL_WCSPLANE_MASK;
}
if (c == 0) {
s = 0;
if (c < 0x80) {
s = c;
} else if (s <= 0) {
s = -1;
}
}

if (s >= 0) {
if (s < 0x80) { /* latin */
if (s < 0x80) { /* latin */
CK((*filter->output_function)(s, filter->data));
} else {
CK((*filter->output_function)((s >> 8) & 0xff, filter->data));
CK((*filter->output_function)(s & 0xff, filter->data));
CK((*filter->output_function)((s >> 8) & 0xFF, filter->data));
CK((*filter->output_function)(s & 0xFF, filter->data));
}
} else {
CK(mbfl_filt_conv_illegal_output(c, filter));
}

return c;
}

static int mbfl_filt_conv_euccn_wchar_flush(mbfl_convert_filter *filter)
{
if (filter->status == 1) {
/* 2-byte character was truncated */
CK((*filter->output_function)(filter->cache | MBFL_WCSGROUP_THROUGH, filter->data));
}

if (filter->flush_function) {
(*filter->flush_function)(filter->data);
}

return 0;
}

0 comments on commit 8b25e38

Please sign in to comment.