Remove mbstring identify filters

mbstring had an 'identify filter' for almost every supported text encoding which was used when auto-detecting the most likely encoding for a string. It would run over the string and set a 'flag' if it saw anything which did not appear likely to be the encoding in question. One problem with this scheme was that encodings which merely appeared less likely to be the correct one were completely rejected, even if there was no better candidate. Another problem was that the 'identify filters' had a huge amount of code duplication with the 'conversion filters'. Eliminate the identify filters. Instead, when auto-detecting text encoding, use conversion filters to see whether the input string is valid in candidate encodings or not. At the same type, watch the type of codepoints which the string decodes to and mark it as less likely if non-printable characters (ESC, form feed, bell, etc.) or 'private use area' codepoints are seen. Interestingly, one old test case in which JIS text was misidentified as UTF-8 (and this wrong behavior was enshrined in the test) was 'fixed' and the JIS string is now auto-detected as JIS.
php · Nov 9, 2020 · 3e7acf9 · 3e7acf9
1 parent a416f93
commit 3e7acf9
Show file tree

Hide file tree

Showing 121 changed files with 120 additions and 2,819 deletions.
diff --git a/ext/mbstring/config.m4 b/ext/mbstring/config.m4
@@ -161,7 +161,6 @@ AC_DEFUN([PHP_MBSTRING_SETUP_LIBMBFL], [
     libmbfl/mbfl/mbfl_convert.c
     libmbfl/mbfl/mbfl_encoding.c
     libmbfl/mbfl/mbfl_filter_output.c
-    libmbfl/mbfl/mbfl_ident.c
     libmbfl/mbfl/mbfl_language.c
     libmbfl/mbfl/mbfl_memory_device.c
     libmbfl/mbfl/mbfl_string.c
@@ -177,7 +176,7 @@ AC_DEFUN([PHP_MBSTRING_SETUP_LIBMBFL], [
     libmbfl/nls/nls_tr.c
     libmbfl/nls/nls_ua.c
   ])
-  PHP_MBSTRING_ADD_INSTALL_HEADERS([libmbfl/config.h libmbfl/mbfl/eaw_table.h libmbfl/mbfl/mbfilter.h libmbfl/mbfl/mbfilter_8bit.h libmbfl/mbfl/mbfilter_pass.h libmbfl/mbfl/mbfilter_wchar.h libmbfl/mbfl/mbfl_consts.h libmbfl/mbfl/mbfl_convert.h libmbfl/mbfl/mbfl_defs.h libmbfl/mbfl/mbfl_encoding.h libmbfl/mbfl/mbfl_filter_output.h libmbfl/mbfl/mbfl_ident.h libmbfl/mbfl/mbfl_language.h libmbfl/mbfl/mbfl_memory_device.h libmbfl/mbfl/mbfl_string.h])
+  PHP_MBSTRING_ADD_INSTALL_HEADERS([libmbfl/config.h libmbfl/mbfl/eaw_table.h libmbfl/mbfl/mbfilter.h libmbfl/mbfl/mbfilter_8bit.h libmbfl/mbfl/mbfilter_pass.h libmbfl/mbfl/mbfilter_wchar.h libmbfl/mbfl/mbfl_consts.h libmbfl/mbfl/mbfl_convert.h libmbfl/mbfl/mbfl_defs.h libmbfl/mbfl/mbfl_encoding.h libmbfl/mbfl/mbfl_filter_output.h libmbfl/mbfl/mbfl_language.h libmbfl/mbfl/mbfl_memory_device.h libmbfl/mbfl/mbfl_string.h])
 ])
 
 dnl

diff --git a/ext/mbstring/config.w32 b/ext/mbstring/config.w32
@@ -40,14 +40,14 @@ if (PHP_MBSTRING != "no") {
 
 		ADD_SOURCES("ext/mbstring/libmbfl/mbfl", "mbfilter.c mbfilter_8bit.c \
 			mbfilter_pass.c mbfilter_wchar.c mbfl_convert.c mbfl_encoding.c \
-			mbfl_filter_output.c mbfl_ident.c mbfl_language.c mbfl_memory_device.c \
+			mbfl_filter_output.c mbfl_language.c mbfl_memory_device.c \
 			mbfl_string.c", "mbstring");
 
 		ADD_SOURCES("ext/mbstring/libmbfl/nls", "nls_de.c nls_en.c nls_ja.c \
 			nls_kr.c nls_neutral.c nls_ru.c nls_uni.c nls_zh.c nls_hy.c \
 			nls_ua.c nls_tr.c", "mbstring");
 
-		PHP_INSTALL_HEADERS("ext/mbstring", "mbstring.h libmbfl/config.h libmbfl/mbfl/eaw_table.h libmbfl/mbfl/mbfilter.h libmbfl/mbfl/mbfilter_8bit.h libmbfl/mbfl/mbfilter_pass.h libmbfl/mbfl/mbfilter_wchar.h libmbfl/mbfl/mbfl_consts.h libmbfl/mbfl/mbfl_convert.h libmbfl/mbfl/mbfl_defs.h libmbfl/mbfl/mbfl_encoding.h libmbfl/mbfl/mbfl_filter_output.h libmbfl/mbfl/mbfl_ident.h libmbfl/mbfl/mbfl_language.h libmbfl/mbfl/mbfl_memory_device.h libmbfl/mbfl/mbfl_string.h");
+		PHP_INSTALL_HEADERS("ext/mbstring", "mbstring.h libmbfl/config.h libmbfl/mbfl/eaw_table.h libmbfl/mbfl/mbfilter.h libmbfl/mbfl/mbfilter_8bit.h libmbfl/mbfl/mbfilter_pass.h libmbfl/mbfl/mbfilter_wchar.h libmbfl/mbfl/mbfl_consts.h libmbfl/mbfl/mbfl_convert.h libmbfl/mbfl/mbfl_defs.h libmbfl/mbfl/mbfl_encoding.h libmbfl/mbfl/mbfl_filter_output.h libmbfl/mbfl/mbfl_language.h libmbfl/mbfl/mbfl_memory_device.h libmbfl/mbfl/mbfl_string.h");
 
 		AC_DEFINE('HAVE_MBSTRING', 1, 'Have mbstring support');
 

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_7bit.c b/ext/mbstring/libmbfl/filters/mbfilter_7bit.c
@@ -31,8 +31,6 @@
 #include "mbfilter.h"
 #include "mbfilter_7bit.h"
 
-static int mbfl_filt_ident_7bit(int c, mbfl_identify_filter *filter);
-
 const mbfl_encoding mbfl_encoding_7bit = {
 	mbfl_no_encoding_7bit,
 	"7bit",
@@ -44,12 +42,6 @@ const mbfl_encoding mbfl_encoding_7bit = {
 	NULL
 };
 
-const struct mbfl_identify_vtbl vtbl_identify_7bit = {
-	mbfl_no_encoding_7bit,
-	mbfl_filt_ident_common_ctor,
-	mbfl_filt_ident_7bit
-};
-
 const struct mbfl_convert_vtbl vtbl_8bit_7bit = {
 	mbfl_no_encoding_8bit,
 	mbfl_no_encoding_7bit,
@@ -88,11 +80,3 @@ int mbfl_filt_conv_any_7bit(int c, mbfl_convert_filter *filter)
 	}
 	return c;
 }
-
-static int mbfl_filt_ident_7bit(int c, mbfl_identify_filter *filter)
-{
-	if (c >= 0x80) {
-		filter->flag = 1;
-	}
-	return c;
-}
diff --git a/ext/mbstring/libmbfl/filters/mbfilter_7bit.h b/ext/mbstring/libmbfl/filters/mbfilter_7bit.h
@@ -34,7 +34,6 @@
 #include "mbfilter.h"
 
 extern const mbfl_encoding mbfl_encoding_7bit;
-extern const struct mbfl_identify_vtbl vtbl_identify_7bit;
 extern const struct mbfl_convert_vtbl vtbl_8bit_7bit;
 extern const struct mbfl_convert_vtbl vtbl_7bit_8bit;
 

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_armscii8.c b/ext/mbstring/libmbfl/filters/mbfilter_armscii8.c
@@ -30,8 +30,6 @@
 #include "mbfilter_armscii8.h"
 #include "unicode_table_armscii8.h"
 
-static int mbfl_filt_ident_armscii8(int c, mbfl_identify_filter *filter);
-
 static const char *mbfl_encoding_armscii8_aliases[] = {"ArmSCII-8", "ArmSCII8", "ARMSCII-8", "ARMSCII8", NULL};
 
 const mbfl_encoding mbfl_encoding_armscii8 = {
@@ -45,12 +43,6 @@ const mbfl_encoding mbfl_encoding_armscii8 = {
 	&vtbl_wchar_armscii8
 };
 
-const struct mbfl_identify_vtbl vtbl_identify_armscii8 = {
-	mbfl_no_encoding_armscii8,
-	mbfl_filt_ident_common_ctor,
-	mbfl_filt_ident_armscii8
-};
-
 const struct mbfl_convert_vtbl vtbl_wchar_armscii8 = {
 	mbfl_no_encoding_wchar,
 	mbfl_no_encoding_armscii8,
@@ -108,11 +100,3 @@ int mbfl_filt_conv_wchar_armscii8(int c, mbfl_convert_filter *filter)
 
 	return c;
 }
-
-static int mbfl_filt_ident_armscii8(int c, mbfl_identify_filter *filter)
-{
-	if (c >= armscii8_ucs_table_min && !armscii8_ucs_table[c - armscii8_ucs_table_min]) {
-		filter->flag = 1;
-	}
-	return c;
-}
diff --git a/ext/mbstring/libmbfl/filters/mbfilter_armscii8.h b/ext/mbstring/libmbfl/filters/mbfilter_armscii8.h
@@ -28,7 +28,6 @@
 #include "mbfilter.h"
 
 extern const mbfl_encoding mbfl_encoding_armscii8;
-extern const struct mbfl_identify_vtbl vtbl_identify_armscii8;
 extern const struct mbfl_convert_vtbl vtbl_wchar_armscii8;
 extern const struct mbfl_convert_vtbl vtbl_armscii8_wchar;
 

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_ascii.c b/ext/mbstring/libmbfl/filters/mbfilter_ascii.c
@@ -31,8 +31,6 @@
 #include "mbfilter.h"
 #include "mbfilter_ascii.h"
 
-static int mbfl_filt_ident_ascii(int c, mbfl_identify_filter *filter);
-
 static const char *mbfl_encoding_ascii_aliases[] = {"ANSI_X3.4-1968", "iso-ir-6", "ANSI_X3.4-1986", "ISO_646.irv:1991", "US-ASCII", "ISO646-US", "us", "IBM367", "IBM-367", "cp367", "csASCII", NULL};
 
 const mbfl_encoding mbfl_encoding_ascii = {
@@ -46,12 +44,6 @@ const mbfl_encoding mbfl_encoding_ascii = {
 	&vtbl_wchar_ascii
 };
 
-const struct mbfl_identify_vtbl vtbl_identify_ascii = {
-	mbfl_no_encoding_ascii,
-	mbfl_filt_ident_common_ctor,
-	mbfl_filt_ident_ascii
-};
-
 const struct mbfl_convert_vtbl vtbl_ascii_wchar = {
 	mbfl_no_encoding_ascii,
 	mbfl_no_encoding_wchar,
@@ -101,16 +93,3 @@ int mbfl_filt_conv_wchar_ascii(int c, mbfl_convert_filter *filter)
 
 	return c;
 }
-
-static int mbfl_filt_ident_ascii(int c, mbfl_identify_filter *filter)
-{
-	if (c >= 0x20 && c < 0x80) {
-		;
-	} else if (c == 0x0d || c == 0x0a || c == 0x09 || c == 0) {	/* CR or LF or HTAB or null */
-		;
-	} else {
-		filter->flag = 1;
-	}
-
-	return c;
-}
diff --git a/ext/mbstring/libmbfl/filters/mbfilter_ascii.h b/ext/mbstring/libmbfl/filters/mbfilter_ascii.h
@@ -34,7 +34,6 @@
 #include "mbfilter.h"
 
 extern const mbfl_encoding mbfl_encoding_ascii;
-extern const struct mbfl_identify_vtbl vtbl_identify_ascii;
 extern const struct mbfl_convert_vtbl vtbl_ascii_wchar;
 extern const struct mbfl_convert_vtbl vtbl_wchar_ascii;
 

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_big5.c b/ext/mbstring/libmbfl/filters/mbfilter_big5.c
@@ -32,8 +32,6 @@
 
 #include "unicode_table_big5.h"
 
-static int mbfl_filt_ident_big5(int c, mbfl_identify_filter *filter);
-
 static const unsigned char mblen_table_big5[] = { /* 0x81-0xFE */
   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
@@ -77,18 +75,6 @@ const mbfl_encoding mbfl_encoding_cp950 = {
 	&vtbl_wchar_cp950
 };
 
-const struct mbfl_identify_vtbl vtbl_identify_big5 = {
-	mbfl_no_encoding_big5,
-	mbfl_filt_ident_common_ctor,
-	mbfl_filt_ident_big5
-};
-
-const struct mbfl_identify_vtbl vtbl_identify_cp950 = {
-	mbfl_no_encoding_cp950,
-	mbfl_filt_ident_common_ctor,
-	mbfl_filt_ident_big5
-};
-
 const struct mbfl_convert_vtbl vtbl_big5_wchar = {
 	mbfl_no_encoding_big5,
 	mbfl_no_encoding_wchar,
@@ -322,28 +308,3 @@ mbfl_filt_conv_wchar_big5(int c, mbfl_convert_filter *filter)
 
 	return c;
 }
-
-static int mbfl_filt_ident_big5(int c, mbfl_identify_filter *filter)
-{
-	int c1;
-	if (filter->encoding->no_encoding == mbfl_no_encoding_cp950) {
-		c1 = 0x80;
-	} else {
-		c1 = 0xa0;
-	}
-
-	if (filter->status) {		/* kanji second char */
-		if (c < 0x40 || (c > 0x7e && c < 0xa1) ||c > 0xfe) {	/* bad */
-		    filter->flag = 1;
-		}
-		filter->status = 0;
-	} else if (c >= 0 && c < 0x80) {	/* latin  ok */
-		;
-	} else if (c > c1 && c < 0xff) {	/* DBCS lead byte */
-		filter->status = 1;
-	} else {							/* bad */
-		filter->flag = 1;
-	}
-
-	return c;
-}
diff --git a/ext/mbstring/libmbfl/filters/mbfilter_big5.h b/ext/mbstring/libmbfl/filters/mbfilter_big5.h
@@ -33,12 +33,10 @@
 #include "mbfilter.h"
 
 extern const mbfl_encoding mbfl_encoding_big5;
-extern const struct mbfl_identify_vtbl vtbl_identify_big5;
 extern const struct mbfl_convert_vtbl vtbl_big5_wchar;
 extern const struct mbfl_convert_vtbl vtbl_wchar_big5;
 
 extern const mbfl_encoding mbfl_encoding_cp950;
-extern const struct mbfl_identify_vtbl vtbl_identify_cp950;
 extern const struct mbfl_convert_vtbl vtbl_cp950_wchar;
 extern const struct mbfl_convert_vtbl vtbl_wchar_cp950;
 

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_cp1251.c b/ext/mbstring/libmbfl/filters/mbfilter_cp1251.c
@@ -31,8 +31,6 @@
 #include "mbfilter_cp1251.h"
 #include "unicode_table_cp1251.h"
 
-static int mbfl_filt_ident_cp1251(int c, mbfl_identify_filter *filter);
-
 static const char *mbfl_encoding_cp1251_aliases[] = {"CP1251", "CP-1251", "WINDOWS-1251", NULL};
 
 const mbfl_encoding mbfl_encoding_cp1251 = {
@@ -46,12 +44,6 @@ const mbfl_encoding mbfl_encoding_cp1251 = {
 	&vtbl_wchar_cp1251
 };
 
-const struct mbfl_identify_vtbl vtbl_identify_cp1251 = {
-	mbfl_no_encoding_cp1251,
-	mbfl_filt_ident_common_ctor,
-	mbfl_filt_ident_cp1251
-};
-
 const struct mbfl_convert_vtbl vtbl_wchar_cp1251 = {
 	mbfl_no_encoding_wchar,
 	mbfl_no_encoding_cp1251,
@@ -107,12 +99,3 @@ int mbfl_filt_conv_wchar_cp1251(int c, mbfl_convert_filter *filter)
 
 	return c;
 }
-
-static int mbfl_filt_ident_cp1251(int c, mbfl_identify_filter *filter)
-{
-	/* Only one byte in this single-byte encoding is not used */
-	if (c == 0x98) {
-		filter->flag = 1;
-	}
-	return c;
-}
diff --git a/ext/mbstring/libmbfl/filters/mbfilter_cp1251.h b/ext/mbstring/libmbfl/filters/mbfilter_cp1251.h
@@ -33,7 +33,6 @@
 #include "mbfilter.h"
 
 extern const mbfl_encoding mbfl_encoding_cp1251;
-extern const struct mbfl_identify_vtbl vtbl_identify_cp1251;
 extern const struct mbfl_convert_vtbl vtbl_wchar_cp1251;
 extern const struct mbfl_convert_vtbl vtbl_cp1251_wchar;
 

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_cp1252.c b/ext/mbstring/libmbfl/filters/mbfilter_cp1252.c
@@ -31,8 +31,6 @@
 #include "mbfilter_cp1252.h"
 #include "unicode_table_cp1252.h"
 
-static int mbfl_filt_ident_cp1252(int c, mbfl_identify_filter *filter);
-
 static const char *mbfl_encoding_cp1252_aliases[] = {"cp1252", NULL};
 
 const mbfl_encoding mbfl_encoding_cp1252 = {
@@ -46,12 +44,6 @@ const mbfl_encoding mbfl_encoding_cp1252 = {
 	&vtbl_wchar_cp1252
 };
 
-const struct mbfl_identify_vtbl vtbl_identify_cp1252 = {
-	mbfl_no_encoding_cp1252,
-	mbfl_filt_ident_common_ctor,
-	mbfl_filt_ident_cp1252
-};
-
 const struct mbfl_convert_vtbl vtbl_cp1252_wchar = {
 	mbfl_no_encoding_cp1252,
 	mbfl_no_encoding_wchar,
@@ -115,11 +107,3 @@ int mbfl_filt_conv_cp1252_wchar(int c, mbfl_convert_filter *filter)
 
 	return c;
 }
-
-static int mbfl_filt_ident_cp1252(int c, mbfl_identify_filter *filter)
-{
-	if (c >= 0x80 && c < 0xA0 && !cp1252_ucs_table[c - 0x80]) {
-		filter->flag = 1;
-	}
-	return c;
-}
diff --git a/ext/mbstring/libmbfl/filters/mbfilter_cp1252.h b/ext/mbstring/libmbfl/filters/mbfilter_cp1252.h
@@ -33,7 +33,6 @@
 #include "mbfilter.h"
 
 extern const mbfl_encoding mbfl_encoding_cp1252;
-extern const struct mbfl_identify_vtbl vtbl_identify_cp1252;
 extern const struct mbfl_convert_vtbl vtbl_cp1252_wchar;
 extern const struct mbfl_convert_vtbl vtbl_wchar_cp1252;
 

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_cp1254.c b/ext/mbstring/libmbfl/filters/mbfilter_cp1254.c
@@ -31,8 +31,6 @@
 #include "mbfilter_cp1254.h"
 #include "unicode_table_cp1254.h"
 
-static int mbfl_filt_ident_cp1254(int c, mbfl_identify_filter *filter);
-
 static const char *mbfl_encoding_cp1254_aliases[] = {"CP1254", "CP-1254", "WINDOWS-1254", NULL};
 
 const mbfl_encoding mbfl_encoding_cp1254 = {
@@ -46,12 +44,6 @@ const mbfl_encoding mbfl_encoding_cp1254 = {
 	&vtbl_wchar_cp1254
 };
 
-const struct mbfl_identify_vtbl vtbl_identify_cp1254 = {
-	mbfl_no_encoding_cp1254,
-	mbfl_filt_ident_common_ctor,
-	mbfl_filt_ident_cp1254
-};
-
 const struct mbfl_convert_vtbl vtbl_cp1254_wchar = {
 	mbfl_no_encoding_cp1254,
 	mbfl_no_encoding_wchar,
@@ -107,11 +99,3 @@ int mbfl_filt_conv_cp1254_wchar(int c, mbfl_convert_filter *filter)
 	CK((*filter->output_function)(s, filter->data));
 	return c;
 }
-
-static int mbfl_filt_ident_cp1254(int c, mbfl_identify_filter *filter)
-{
-	if (c >= 0x81 && c <= 0x9E && !cp1254_ucs_table[c - cp1254_ucs_table_min]) {
-		filter->flag = 1;
-	}
-	return c;
-}
diff --git a/ext/mbstring/libmbfl/filters/mbfilter_cp1254.h b/ext/mbstring/libmbfl/filters/mbfilter_cp1254.h
@@ -33,7 +33,6 @@
 #include "mbfilter.h"
 
 extern const mbfl_encoding mbfl_encoding_cp1254;
-extern const struct mbfl_identify_vtbl vtbl_identify_cp1254;
 extern const struct mbfl_convert_vtbl vtbl_cp1254_wchar;
 extern const struct mbfl_convert_vtbl vtbl_wchar_cp1254;