Skip to content

Major overhaul of mbstring (part 26) #9592

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Major overhaul of mbstring (part 26) #9592

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e8e746c
Use fast text conversion filters for mb_convert_case, mb_strtoupper, …
alexdowad Sep 21, 2022
b4207d5
Make enum for valid case_mode values (for php_unicode_convert_case)
alexdowad Sep 24, 2022
beae80b
Use uint32_t for 'illegal_substchar' codepoint in mbstring
alexdowad Oct 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ext/mbstring/libmbfl/mbfl/mbfilter.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ void mbfl_buffer_converter_illegal_mode(mbfl_buffer_converter *convd, int mode)
}
}

void mbfl_buffer_converter_illegal_substchar(mbfl_buffer_converter *convd, int substchar)
void mbfl_buffer_converter_illegal_substchar(mbfl_buffer_converter *convd, uint32_t substchar)
{
if (convd->filter2) {
convd->filter2->illegal_substchar = substchar;
Expand Down Expand Up @@ -387,7 +387,7 @@ mbfl_convert_encoding(

if (filter2 != NULL) {
filter2->illegal_mode = MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR;
filter2->illegal_substchar = 0x3f; /* '?' */
filter2->illegal_substchar = '?';
}

mbfl_memory_device_init(&device, string->len, (string->len >> 2) + 8);
Expand Down
2 changes: 1 addition & 1 deletion ext/mbstring/libmbfl/mbfl/mbfilter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ struct _mbfl_buffer_converter {
MBFLAPI extern mbfl_buffer_converter * mbfl_buffer_converter_new(const mbfl_encoding *from, const mbfl_encoding *to, size_t buf_initsz);
MBFLAPI extern void mbfl_buffer_converter_delete(mbfl_buffer_converter *convd);
MBFLAPI extern void mbfl_buffer_converter_illegal_mode(mbfl_buffer_converter *convd, int mode);
MBFLAPI extern void mbfl_buffer_converter_illegal_substchar(mbfl_buffer_converter *convd, int substchar);
MBFLAPI extern void mbfl_buffer_converter_illegal_substchar(mbfl_buffer_converter *convd, uint32_t substchar);
MBFLAPI extern size_t mbfl_buffer_converter_feed(mbfl_buffer_converter *convd, mbfl_string *string);
MBFLAPI extern void mbfl_buffer_converter_flush(mbfl_buffer_converter *convd);
MBFLAPI extern mbfl_string * mbfl_buffer_converter_result(mbfl_buffer_converter *convd, mbfl_string *result);
Expand Down
2 changes: 1 addition & 1 deletion ext/mbstring/libmbfl/mbfl/mbfl_convert.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ int mbfl_filt_conv_illegal_output(int c, mbfl_convert_filter *filter)
unsigned int w = c;
int ret = 0;
int mode_backup = filter->illegal_mode;
int substchar_backup = filter->illegal_substchar;
uint32_t substchar_backup = filter->illegal_substchar;

/* The used substitution character may not be supported by the target character encoding.
* If that happens, first try to use "?" instead and if that also fails, silently drop the
Expand Down
2 changes: 1 addition & 1 deletion ext/mbstring/libmbfl/mbfl/mbfl_convert.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ struct _mbfl_convert_filter {
const mbfl_encoding *from;
const mbfl_encoding *to;
int illegal_mode;
int illegal_substchar;
uint32_t illegal_substchar;
size_t num_illegalchar;
void *opaque;
};
Expand Down
99 changes: 30 additions & 69 deletions ext/mbstring/mbstring.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -883,8 +883,8 @@ static PHP_INI_MH(OnUpdate_mbstring_substitute_character)
} else {
MBSTRG(filter_illegal_mode) = MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR;
MBSTRG(current_filter_illegal_mode) = MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR;
MBSTRG(filter_illegal_substchar) = 0x3f; /* '?' */
MBSTRG(current_filter_illegal_substchar) = 0x3f; /* '?' */
MBSTRG(filter_illegal_substchar) = '?';
MBSTRG(current_filter_illegal_substchar) = '?';
}

return SUCCESS;
Expand Down Expand Up @@ -1015,9 +1015,9 @@ ZEND_TSRMLS_CACHE_UPDATE();
mbstring_globals->default_detect_order_list = (enum mbfl_no_encoding *) php_mb_default_identify_list_neut;
mbstring_globals->default_detect_order_list_size = sizeof(php_mb_default_identify_list_neut) / sizeof(php_mb_default_identify_list_neut[0]);
mbstring_globals->filter_illegal_mode = MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR;
mbstring_globals->filter_illegal_substchar = 0x3f; /* '?' */
mbstring_globals->filter_illegal_substchar = '?';
mbstring_globals->current_filter_illegal_mode = MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR;
mbstring_globals->current_filter_illegal_substchar = 0x3f; /* '?' */
mbstring_globals->current_filter_illegal_substchar = '?';
mbstring_globals->illegalchars = 0;
mbstring_globals->encoding_translation = 0;
mbstring_globals->strict_detection = 0;
Expand Down Expand Up @@ -2753,24 +2753,18 @@ PHP_FUNCTION(mb_convert_encoding)
}
/* }}} */

static char *mbstring_convert_case(
int case_mode, const char *str, size_t str_len, size_t *ret_len,
const mbfl_encoding *enc) {
return php_unicode_convert_case(
case_mode, str, str_len, ret_len, enc,
MBSTRG(current_filter_illegal_mode), MBSTRG(current_filter_illegal_substchar));
static zend_string *mbstring_convert_case(php_case_mode case_mode, const char *str, size_t str_len, const mbfl_encoding *enc)
{
return php_unicode_convert_case(case_mode, str, str_len, enc, MBSTRG(current_filter_illegal_mode), MBSTRG(current_filter_illegal_substchar));
}

/* {{{ Returns a case-folded version of source_string */
PHP_FUNCTION(mb_convert_case)
{
zend_string *from_encoding = NULL;
char *str;
size_t str_len, ret_len;
zend_string *str, *from_encoding = NULL;
zend_long case_mode = 0;

ZEND_PARSE_PARAMETERS_START(2, 3)
Z_PARAM_STRING(str, str_len)
Z_PARAM_STR(str)
Z_PARAM_LONG(case_mode)
Z_PARAM_OPTIONAL
Z_PARAM_STR_OR_NULL(from_encoding)
Expand All @@ -2781,30 +2775,20 @@ PHP_FUNCTION(mb_convert_case)
RETURN_THROWS();
}

if (case_mode < 0 || case_mode > PHP_UNICODE_CASE_MODE_MAX) {
if (case_mode < 0 || case_mode >= PHP_UNICODE_CASE_MODE_MAX) {
zend_argument_value_error(2, "must be one of the MB_CASE_* constants");
RETURN_THROWS();
}

char *newstr = mbstring_convert_case(case_mode, str, str_len, &ret_len, enc);
/* If newstr is NULL something went wrong in mbfl and this is a bug */
ZEND_ASSERT(newstr != NULL);

// TODO: avoid reallocation ???
RETVAL_STRINGL(newstr, ret_len);
efree(newstr);
RETURN_STR(mbstring_convert_case(case_mode, ZSTR_VAL(str), ZSTR_LEN(str), enc));
}
/* }}} */

/* {{{ Returns a upper cased version of source_string */
PHP_FUNCTION(mb_strtoupper)
{
zend_string *from_encoding = NULL;
char *str;
size_t str_len, ret_len;
zend_string *str, *from_encoding = NULL;

ZEND_PARSE_PARAMETERS_START(1, 2)
Z_PARAM_STRING(str, str_len)
Z_PARAM_STR(str)
Z_PARAM_OPTIONAL
Z_PARAM_STR_OR_NULL(from_encoding)
ZEND_PARSE_PARAMETERS_END();
Expand All @@ -2814,46 +2798,26 @@ PHP_FUNCTION(mb_strtoupper)
RETURN_THROWS();
}

char *newstr = mbstring_convert_case(PHP_UNICODE_CASE_UPPER, str, str_len, &ret_len, enc);
/* If newstr is NULL something went wrong in mbfl and this is a bug */
ZEND_ASSERT(newstr != NULL);

// TODO: avoid reallocation ???
RETVAL_STRINGL(newstr, ret_len);
efree(newstr);
RETURN_STR(mbstring_convert_case(PHP_UNICODE_CASE_UPPER, ZSTR_VAL(str), ZSTR_LEN(str), enc));
}
/* }}} */

/* {{{ Returns a lower cased version of source_string */
PHP_FUNCTION(mb_strtolower)
{
zend_string *from_encoding = NULL;
char *str;
size_t str_len;
char *newstr;
size_t ret_len;
const mbfl_encoding *enc;
zend_string *str, *from_encoding = NULL;

ZEND_PARSE_PARAMETERS_START(1, 2)
Z_PARAM_STRING(str, str_len)
Z_PARAM_STR(str)
Z_PARAM_OPTIONAL
Z_PARAM_STR_OR_NULL(from_encoding)
ZEND_PARSE_PARAMETERS_END();

enc = php_mb_get_encoding(from_encoding, 2);
const mbfl_encoding *enc = php_mb_get_encoding(from_encoding, 2);
if (!enc) {
RETURN_THROWS();
}

newstr = mbstring_convert_case(PHP_UNICODE_CASE_LOWER, str, str_len, &ret_len, enc);
/* If newstr is NULL something went wrong in mbfl and this is a bug */
ZEND_ASSERT(newstr != NULL);

// TODO: avoid reallocation ???
RETVAL_STRINGL(newstr, ret_len);
efree(newstr);
RETURN_STR(mbstring_convert_case(PHP_UNICODE_CASE_LOWER, ZSTR_VAL(str), ZSTR_LEN(str), enc));
}
/* }}} */

static const mbfl_encoding **duplicate_elist(const mbfl_encoding **elist, size_t size)
{
Expand Down Expand Up @@ -4774,33 +4738,32 @@ MBSTRING_API char *php_mb_safe_strrchr(const char *s, unsigned int c, size_t nby
return last;
}

/* {{{ MBSTRING_API int php_mb_stripos() */
MBSTRING_API size_t php_mb_stripos(int mode, const char *old_haystack, size_t old_haystack_len, const char *old_needle, size_t old_needle_len, zend_long offset, const mbfl_encoding *enc)
{
size_t n = (size_t) -1;
size_t n = (size_t)-1;
mbfl_string haystack, needle;
zend_string *haystack_converted = NULL, *needle_converted = NULL;

mbfl_string_init_set(&haystack, enc);
mbfl_string_init_set(&needle, enc);

do {
/* We're using simple case-folding here, because we'd have to deal with remapping of
* offsets otherwise. */

size_t len = 0;
haystack.val = (unsigned char *)mbstring_convert_case(PHP_UNICODE_CASE_FOLD_SIMPLE, (char *)old_haystack, old_haystack_len, &len, enc);
haystack.len = len;
haystack_converted = mbstring_convert_case(PHP_UNICODE_CASE_FOLD_SIMPLE, (char*)old_haystack, old_haystack_len, enc);
haystack.val = (unsigned char*)ZSTR_VAL(haystack_converted);
haystack.len = ZSTR_LEN(haystack_converted);

if (!haystack.val) {
break;
}

if (haystack.len == 0) {
break;
}

needle.val = (unsigned char *)mbstring_convert_case(PHP_UNICODE_CASE_FOLD_SIMPLE, (char *)old_needle, old_needle_len, &len, enc);
needle.len = len;
needle_converted = mbstring_convert_case(PHP_UNICODE_CASE_FOLD_SIMPLE, (char*)old_needle, old_needle_len, enc);
needle.val = (unsigned char*)ZSTR_VAL(needle_converted);
needle.len = ZSTR_LEN(needle_converted);

if (!needle.val) {
break;
Expand All @@ -4809,17 +4772,15 @@ MBSTRING_API size_t php_mb_stripos(int mode, const char *old_haystack, size_t ol
n = mbfl_strpos(&haystack, &needle, offset, mode);
} while(0);

if (haystack.val) {
efree(haystack.val);
if (haystack_converted) {
zend_string_free(haystack_converted);
}

if (needle.val) {
efree(needle.val);
if (needle_converted) {
zend_string_free(needle_converted);
}

return n;
}
/* }}} */

static void php_mb_gpc_get_detect_order(const zend_encoding ***list, size_t *list_size) /* {{{ */
{
Expand Down
4 changes: 2 additions & 2 deletions ext/mbstring/mbstring.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,9 @@ ZEND_BEGIN_MODULE_GLOBALS(mbstring)
enum mbfl_no_encoding *default_detect_order_list;
size_t default_detect_order_list_size;
int filter_illegal_mode;
int filter_illegal_substchar;
uint32_t filter_illegal_substchar;
int current_filter_illegal_mode;
int current_filter_illegal_substchar;
uint32_t current_filter_illegal_substchar;
enum mbfl_no_language language;
bool encoding_translation;
bool strict_detection;
Expand Down
Loading