diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 0a77e6fe9edc036..aeb3a01d12e9256 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2100,6 +2100,58 @@ def test_bug_34294(self): {'tag': 'foo', 'text': None}, {'tag': 'foo', 'text': None}]) + def test_bug_35859(self): + # MARK_PUSH() macro didn't protect MARK-0 + # if it was the only available mark. + self.assertEqual(re.match(r'(ab|a)*?b', 'ab').groups(), ('a',)) + self.assertEqual(re.match(r'(ab|a)+?b', 'ab').groups(), ('a',)) + self.assertEqual(re.match(r'(ab|a){0,2}?b', 'ab').groups(), ('a',)) + self.assertEqual(re.match(r'(.b|a)*?b', 'ab').groups(), ('a',)) + + # JUMP_MIN_UNTIL_2 should MARK_PUSH() if in a repeat + s = 'axxzbcz' + p = r'(?:(?:a|bc)*?(xx)??z)*' + self.assertEqual(re.match(p, s).groups(), ('xx',)) + # test-case provided by issue9134 + s = 'xtcxyzxc' + p = r'((x|yz)+?(t)??c)*' + self.assertEqual(re.match(p, s).groups(), ('xyzxc', 'x', 't')) + + self.assertEqual(re.match(r'(ab?)*?b', 'ab').groups(), ('a',)) + s = 'axxzaz' + p = r'(?:a*?(xx)??z)*' + self.assertEqual(re.match(p, s).groups(), ('xx',)) + + # JUMP_REPEAT_ONE_1 should MARK_PUSH() if in a repeat + s = 'aabaab' + p = r'(?:[^b]*a(?=(b)|(a))ab)*' + m = re.match(p, s) + self.assertEqual(m.span(), (0, 6)) + self.assertEqual(m.groups(), (None, 'a')) + + # JUMP_REPEAT_ONE_2 should MARK_PUSH() if in a repeat + s = 'abab' + p = r'(?:[^b]*(?=(b)|(a))ab)*' + m = re.match(p, s) + self.assertEqual(m.span(), (0, 4)) + self.assertEqual(m.groups(), (None, 'a')) + + # JUMP_MIN_REPEAT_ONE should MARK_PUSH() if in a repeat + s = 'abab' + p = r'(?:.*?(?=(a)|(b))b)*' + m = re.match(p, s) + self.assertEqual(m.span(), (0, 4)) + self.assertEqual(m.groups(), (None, 'b')) + + # JUMP_ASSERT_NOT should LASTMARK_SAVE() + # reported in issue725149 + self.assertEqual(re.match(r'(?!(..)c)', 'ab').groups(), (None,)) + + # JUMP_ASSERT_NOT should MARK_PUSH() if in a repeat + m = re.match(r'((?!(ab)c)(.))*', 'abab') + self.assertEqual(m.span(), (0, 4)) + self.assertEqual(m.groups(), ('b', None, 'b')) + class PatternReprTests(unittest.TestCase): def check(self, pattern, expected): diff --git a/Misc/NEWS.d/next/Library/2019-03-11-17-58-19.bpo-35859.8lFdLe.rst b/Misc/NEWS.d/next/Library/2019-03-11-17-58-19.bpo-35859.8lFdLe.rst new file mode 100644 index 000000000000000..1432280cd1ff8c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-03-11-17-58-19.bpo-35859.8lFdLe.rst @@ -0,0 +1,2 @@ +re module, fix a few bugs about capturing group. In rare cases, capture +group may be an incorrect string. Patch by Ma Lin. diff --git a/Modules/_sre.c b/Modules/_sre.c index 5cea7562f2807a2..de47e7b45a01ec6 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -520,13 +520,6 @@ static void pattern_error(Py_ssize_t status) { switch (status) { - case SRE_ERROR_RECURSION_LIMIT: - /* This error code seems to be unused. */ - PyErr_SetString( - PyExc_RecursionError, - "maximum recursion limit exceeded" - ); - break; case SRE_ERROR_MEMORY: PyErr_NoMemory(); break; @@ -2354,6 +2347,14 @@ pattern_new_match(PatternObject* pattern, SRE_STATE* state, Py_ssize_t status) if (j+1 <= state->lastmark && state->mark[j] && state->mark[j+1]) { match->mark[j+2] = ((char*) state->mark[j] - base) / n; match->mark[j+3] = ((char*) state->mark[j+1] - base) / n; + + /* check wrong span */ + if (match->mark[j+2] > match->mark[j+3]) { + PyErr_Format(PyExc_RuntimeError, + "the span of capturing group %d is wrong," + " please report bug.", i+1); + return NULL; + } } else match->mark[j+2] = match->mark[j+3] = -1; /* undefined */ diff --git a/Modules/sre.h b/Modules/sre.h index a7284881457c3b1..d708932c0889017 100644 --- a/Modules/sre.h +++ b/Modules/sre.h @@ -16,12 +16,12 @@ /* size of a code word (must be unsigned short or larger, and large enough to hold a UCS4 character) */ #define SRE_CODE Py_UCS4 +#define SRE_MAXGROUPS INT_MAX / 2 + #if SIZEOF_SIZE_T > 4 # define SRE_MAXREPEAT (~(SRE_CODE)0) -# define SRE_MAXGROUPS ((~(SRE_CODE)0) / 2) #else # define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX) -# define SRE_MAXGROUPS ((SRE_CODE)PY_SSIZE_T_MAX / SIZEOF_SIZE_T / 2) #endif typedef struct { @@ -72,8 +72,8 @@ typedef struct { int isbytes; int charsize; /* character size */ /* registers */ - Py_ssize_t lastindex; - Py_ssize_t lastmark; + int32_t lastindex; + int32_t lastmark; void** mark; int match_all; int must_advance; diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 437ab43f434a62a..181d944c1c30b98 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -478,20 +478,20 @@ do { \ DATA_STACK_LOOKUP_AT(state,t,p,pos) #define MARK_PUSH(lastmark) \ - do if (lastmark > 0) { \ + do if (lastmark >= 0) { \ i = lastmark; /* ctx->lastmark may change if reallocated */ \ DATA_STACK_PUSH(state, state->mark, (i+1)*sizeof(void*)); \ } while (0) #define MARK_POP(lastmark) \ - do if (lastmark > 0) { \ + do if (lastmark >= 0) { \ DATA_STACK_POP(state, state->mark, (lastmark+1)*sizeof(void*), 1); \ } while (0) #define MARK_POP_KEEP(lastmark) \ - do if (lastmark > 0) { \ + do if (lastmark >= 0) { \ DATA_STACK_POP(state, state->mark, (lastmark+1)*sizeof(void*), 0); \ } while (0) #define MARK_POP_DISCARD(lastmark) \ - do if (lastmark > 0) { \ + do if (lastmark >= 0) { \ DATA_STACK_POP_DISCARD(state, (lastmark+1)*sizeof(void*)); \ } while (0) @@ -530,17 +530,18 @@ do { \ typedef struct { Py_ssize_t last_ctx_pos; - Py_ssize_t jump; SRE_CHAR* ptr; SRE_CODE* pattern; Py_ssize_t count; - Py_ssize_t lastmark; - Py_ssize_t lastindex; union { SRE_CODE chr; SRE_REPEAT* rep; } u; - int toplevel; + int32_t lastindex; + int32_t lastmark; + char in_repeat; + char toplevel; + char jump; } SRE(match_context); /* check if string matches the given pattern. returns <0 for @@ -550,8 +551,9 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) { SRE_CHAR* end = (SRE_CHAR *)state->end; Py_ssize_t alloc_pos, ctx_pos = -1; - Py_ssize_t i, ret = 0; + Py_ssize_t ret = 0; Py_ssize_t jump; + int32_t i; unsigned int sigcount=0; SRE(match_context)* ctx; @@ -796,10 +798,9 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) /* alternation */ /* <0=skip> code ... */ TRACE(("|%p|%p|BRANCH\n", ctx->pattern, ctx->ptr)); + LASTMARK_SAVE(); - ctx->u.rep = state->repeat; - if (ctx->u.rep) - MARK_PUSH(ctx->lastmark); + MARK_PUSH(ctx->lastmark); for (; ctx->pattern[0]; ctx->pattern += ctx->pattern[0]) { if (ctx->pattern[1] == SRE_OP_LITERAL && (ctx->ptr >= end || @@ -813,17 +814,14 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) state->ptr = ctx->ptr; DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1); if (ret) { - if (ctx->u.rep) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (ctx->u.rep) - MARK_POP_KEEP(ctx->lastmark); + MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); } - if (ctx->u.rep) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); RETURN_FAILURE; case SRE_OP_REPEAT_ONE: @@ -868,6 +866,7 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) } LASTMARK_SAVE(); + MARK_PUSH(ctx->lastmark); if (ctx->pattern[ctx->pattern[0]] == SRE_OP_LITERAL) { /* tail starts with a literal. skip positions where @@ -885,16 +884,17 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_REPEAT_ONE_1, jump_repeat_one_1, ctx->pattern+ctx->pattern[0]); if (ret) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - + MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); ctx->ptr--; ctx->count--; } - + MARK_POP_DISCARD(ctx->lastmark); } else { /* general case */ while (ctx->count >= (Py_ssize_t) ctx->pattern[1]) { @@ -902,13 +902,17 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_REPEAT_ONE_2, jump_repeat_one_2, ctx->pattern+ctx->pattern[0]); if (ret) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + MARK_POP_KEEP(ctx->lastmark); + LASTMARK_RESTORE(); + ctx->ptr--; ctx->count--; - LASTMARK_RESTORE(); } + MARK_POP_DISCARD(ctx->lastmark); } RETURN_FAILURE; @@ -957,15 +961,21 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) } else { /* general case */ LASTMARK_SAVE(); + MARK_PUSH(ctx->lastmark); + while ((Py_ssize_t)ctx->pattern[2] == SRE_MAXREPEAT || ctx->count <= (Py_ssize_t)ctx->pattern[2]) { state->ptr = ctx->ptr; DO_JUMP(JUMP_MIN_REPEAT_ONE,jump_min_repeat_one, ctx->pattern+ctx->pattern[0]); if (ret) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + MARK_POP_KEEP(ctx->lastmark); + LASTMARK_RESTORE(); + state->ptr = ctx->ptr; ret = SRE(count)(state, ctx->pattern+3, 1); RETURN_ON_ERROR(ret); @@ -975,8 +985,8 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) assert(ret == 1); ctx->ptr++; ctx->count++; - LASTMARK_RESTORE(); } + MARK_POP_DISCARD(ctx->lastmark); } RETURN_FAILURE; @@ -989,10 +999,9 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) /* install new repeat context */ ctx->u.rep = (SRE_REPEAT*) PyObject_MALLOC(sizeof(*ctx->u.rep)); - if (!ctx->u.rep) { - PyErr_NoMemory(); - RETURN_FAILURE; - } + if (!ctx->u.rep) + RETURN_ERROR(SRE_ERROR_MEMORY); + ctx->u.rep->count = -1; ctx->u.rep->pattern = ctx->pattern; ctx->u.rep->prev = state->repeat; @@ -1028,15 +1037,22 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) TRACE(("|%p|%p|MAX_UNTIL %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, ctx->ptr, ctx->count)); + LASTMARK_SAVE(); + MARK_PUSH(ctx->lastmark); + if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1, ctx->u.rep->pattern+3); if (ret) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); + ctx->u.rep->count = ctx->count-1; state->ptr = ctx->ptr; RETURN_FAILURE; @@ -1048,8 +1064,6 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) /* we may have enough matches, but if we can match another item, do so */ ctx->u.rep->count = ctx->count; - LASTMARK_SAVE(); - MARK_PUSH(ctx->lastmark); /* zero-width match protection */ DATA_PUSH(&ctx->u.rep->last_ptr); ctx->u.rep->last_ptr = state->ptr; @@ -1061,7 +1075,7 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - MARK_POP(ctx->lastmark); + MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); ctx->u.rep->count = ctx->count-1; state->ptr = ctx->ptr; @@ -1071,7 +1085,13 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) tail matches */ state->repeat = ctx->u.rep->prev; DO_JUMP(JUMP_MAX_UNTIL_3, jump_max_until_3, ctx->pattern); - RETURN_ON_SUCCESS(ret); + if (ret) { + MARK_POP_DISCARD(ctx->lastmark); + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); state->repeat = ctx->u.rep; state->ptr = ctx->ptr; RETURN_FAILURE; @@ -1091,39 +1111,47 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) TRACE(("|%p|%p|MIN_UNTIL %" PY_FORMAT_SIZE_T "d %p\n", ctx->pattern, ctx->ptr, ctx->count, ctx->u.rep->pattern)); + LASTMARK_SAVE(); + MARK_PUSH(ctx->lastmark); + if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1, ctx->u.rep->pattern+3); if (ret) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); ctx->u.rep->count = ctx->count-1; state->ptr = ctx->ptr; RETURN_FAILURE; } - LASTMARK_SAVE(); - /* see if the tail matches */ state->repeat = ctx->u.rep->prev; + DO_JUMP(JUMP_MIN_UNTIL_2, jump_min_until_2, ctx->pattern); if (ret) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + MARK_POP_KEEP(ctx->lastmark); + LASTMARK_RESTORE(); state->repeat = ctx->u.rep; state->ptr = ctx->ptr; - LASTMARK_RESTORE(); - if ((ctx->count >= (Py_ssize_t) ctx->u.rep->pattern[2] && ctx->u.rep->pattern[2] != SRE_MAXREPEAT) || - state->ptr == ctx->u.rep->last_ptr) + state->ptr == ctx->u.rep->last_ptr) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_FAILURE; + } ctx->u.rep->count = ctx->count; /* zero-width match protection */ @@ -1133,9 +1161,12 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) ctx->u.rep->pattern+3); DATA_POP(&ctx->u.rep->last_ptr); if (ret) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); ctx->u.rep->count = ctx->count-1; state->ptr = ctx->ptr; RETURN_FAILURE; @@ -1273,8 +1304,18 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) if (ctx->ptr - (SRE_CHAR *)state->beginning < (Py_ssize_t)ctx->pattern[1]) RETURN_FAILURE; state->ptr = ctx->ptr - ctx->pattern[1]; + + LASTMARK_SAVE(); + MARK_PUSH(ctx->lastmark); DO_JUMP0(JUMP_ASSERT, jump_assert, ctx->pattern+2); - RETURN_ON_FAILURE(ret); + if (ret <= 0) { + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); + + RETURN_ON_ERROR(ret); + RETURN_FAILURE; + } + MARK_POP_DISCARD(ctx->lastmark); ctx->pattern += ctx->pattern[0]; break; @@ -1285,11 +1326,17 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) ctx->ptr, ctx->pattern[1])); if (ctx->ptr - (SRE_CHAR *)state->beginning >= (Py_ssize_t)ctx->pattern[1]) { state->ptr = ctx->ptr - ctx->pattern[1]; + LASTMARK_SAVE(); + MARK_PUSH(ctx->lastmark); + DO_JUMP0(JUMP_ASSERT_NOT, jump_assert_not, ctx->pattern+2); if (ret) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_FAILURE; } + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); } ctx->pattern += ctx->pattern[0]; break; @@ -1310,8 +1357,15 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) ctx_pos = ctx->last_ctx_pos; jump = ctx->jump; DATA_POP_DISCARD(ctx); - if (ctx_pos == -1) + if (ctx_pos == -1) { +#ifdef Py_DEBUG + if (ctx->toplevel && ret >= 0) { + assert(state->repeat == NULL); + assert(state->data_stack_base == 0); + } +#endif return ret; + } DATA_LOOKUP_AT(SRE(match_context), ctx, ctx_pos); switch (jump) { @@ -1360,7 +1414,8 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) break; } - return ret; /* should never get here */ + /* should never get here */ + Py_UNREACHABLE(); } /* need to reset capturing groups between two SRE(match) callings in loops */