From c556d5a92cca3f62bae08e6de80101a3dfadb447 Mon Sep 17 00:00:00 2001 From: animalize Date: Fri, 8 Mar 2019 20:48:58 +0800 Subject: [PATCH 01/22] 1. add asserts to prevent wreck the stack --- Modules/sre_lib.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 437ab43f434a62a..f5602f7f0ee433b 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1310,8 +1310,15 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) ctx_pos = ctx->last_ctx_pos; jump = ctx->jump; DATA_POP_DISCARD(ctx); - if (ctx_pos == -1) + if (ctx_pos == -1) { +#ifdef Py_DEBUG + if (ctx->toplevel && ret >= 0) { + assert(state->repeat == NULL); + assert(state->data_stack_base == 0); + } +#endif return ret; + } DATA_LOOKUP_AT(SRE(match_context), ctx, ctx_pos); switch (jump) { @@ -1360,7 +1367,8 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) break; } - return ret; /* should never get here */ + /* should never get here */ + Py_UNREACHABLE(); } /* need to reset capturing groups between two SRE(match) callings in loops */ From 3c294e4170cc068c56ebf63c8bb220eb472aaf74 Mon Sep 17 00:00:00 2001 From: animalize Date: Fri, 8 Mar 2019 20:58:41 +0800 Subject: [PATCH 02/22] 2. macro MARK_PUSH() bug MARK_PUSH() didn't protect MARK0 if it was the only available mark. --- Lib/test/test_re.py | 8 ++++++++ Modules/sre_lib.h | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 0a77e6fe9edc036..ce357707ffd02df 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2100,6 +2100,14 @@ def test_bug_34294(self): {'tag': 'foo', 'text': None}, {'tag': 'foo', 'text': None}]) + def test_bug_35859(self): + # MARK_PUSH() macro didn't protect MARK-0 + # if it was the only available mark. + self.assertEqual(re.match(r'(ab|a)*?b', 'ab').groups(), ('a',)) + self.assertEqual(re.match(r'(ab|a)+?b', 'ab').groups(), ('a',)) + self.assertEqual(re.match(r'(ab|a){0,2}?b', 'ab').groups(), ('a',)) + self.assertEqual(re.match(r'(.b|a)*?b', 'ab').groups(), ('a',)) + class PatternReprTests(unittest.TestCase): def check(self, pattern, expected): diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index f5602f7f0ee433b..b23bd31bde8d270 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -478,20 +478,20 @@ do { \ DATA_STACK_LOOKUP_AT(state,t,p,pos) #define MARK_PUSH(lastmark) \ - do if (lastmark > 0) { \ + do if (lastmark >= 0) { \ i = lastmark; /* ctx->lastmark may change if reallocated */ \ DATA_STACK_PUSH(state, state->mark, (i+1)*sizeof(void*)); \ } while (0) #define MARK_POP(lastmark) \ - do if (lastmark > 0) { \ + do if (lastmark >= 0) { \ DATA_STACK_POP(state, state->mark, (lastmark+1)*sizeof(void*), 1); \ } while (0) #define MARK_POP_KEEP(lastmark) \ - do if (lastmark > 0) { \ + do if (lastmark >= 0) { \ DATA_STACK_POP(state, state->mark, (lastmark+1)*sizeof(void*), 0); \ } while (0) #define MARK_POP_DISCARD(lastmark) \ - do if (lastmark > 0) { \ + do if (lastmark >= 0) { \ DATA_STACK_POP_DISCARD(state, (lastmark+1)*sizeof(void*)); \ } while (0) From 529485eac3a5fdf818d67b3b61fd791c56c0a88c Mon Sep 17 00:00:00 2001 From: animalize Date: Fri, 8 Mar 2019 21:05:25 +0800 Subject: [PATCH 03/22] 3. add a new field in_repeat to match_context to indicate that the current operate is in a repeat. --- Modules/sre_lib.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index b23bd31bde8d270..7b5fe4418c2ce28 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -540,7 +540,8 @@ typedef struct { SRE_CODE chr; SRE_REPEAT* rep; } u; - int toplevel; + char toplevel; + char in_repeat; } SRE(match_context); /* check if string matches the given pattern. returns <0 for From ad74049a845c941b7f9a1373d6c385eef2675518 Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 09:33:10 +0800 Subject: [PATCH 04/22] 4. use ctx->in_repeat in SRE_OP_BRANCH no behavior change, just let the code uses an unified style. --- Modules/sre_lib.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 7b5fe4418c2ce28..9fe841f6295d59f 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -798,8 +798,8 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) /* <0=skip> code ... */ TRACE(("|%p|%p|BRANCH\n", ctx->pattern, ctx->ptr)); LASTMARK_SAVE(); - ctx->u.rep = state->repeat; - if (ctx->u.rep) + ctx->in_repeat = (state->repeat != NULL); + if (ctx->in_repeat) MARK_PUSH(ctx->lastmark); for (; ctx->pattern[0]; ctx->pattern += ctx->pattern[0]) { if (ctx->pattern[1] == SRE_OP_LITERAL && @@ -814,16 +814,16 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) state->ptr = ctx->ptr; DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1); if (ret) { - if (ctx->u.rep) + if (ctx->in_repeat) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (ctx->u.rep) + if (ctx->in_repeat) MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); } - if (ctx->u.rep) + if (ctx->in_repeat) MARK_POP_DISCARD(ctx->lastmark); RETURN_FAILURE; From 01d9b8c238ea02328bdbb308f6aa8c3eff6d79e3 Mon Sep 17 00:00:00 2001 From: animalize Date: Fri, 8 Mar 2019 21:10:36 +0800 Subject: [PATCH 05/22] 5. JUMP_MIN_UNTIL_2 should MARK_PUSH() if in a repeat --- Lib/test/test_re.py | 14 ++++++++++++++ Modules/sre_lib.h | 15 +++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index ce357707ffd02df..112b37983f8b678 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2108,6 +2108,20 @@ def test_bug_35859(self): self.assertEqual(re.match(r'(ab|a){0,2}?b', 'ab').groups(), ('a',)) self.assertEqual(re.match(r'(.b|a)*?b', 'ab').groups(), ('a',)) + # JUMP_MIN_UNTIL_2 should MARK_PUSH() if in a repeat + s = 'axxzbcz' + p = r'(?:(?:a|bc)*?(xx)??z)*' + self.assertEqual(re.match(p, s).groups(), ('xx',)) + # test-case provided by issue9134 + s = 'xtcxyzxc' + p = r'((x|yz)+?(t)??c)*' + self.assertEqual(re.match(p, s).groups(), ('xyzxc', 'x', 't')) + + self.assertEqual(re.match(r'(ab?)*?b', 'ab').groups(), ('a',)) + s = 'axxzaz' + p = r'(?:a*?(xx)??z)*' + self.assertEqual(re.match(p, s).groups(), ('xx',)) + class PatternReprTests(unittest.TestCase): def check(self, pattern, expected): diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 9fe841f6295d59f..ea2f59f1eacd592 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1106,21 +1106,28 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) RETURN_FAILURE; } - LASTMARK_SAVE(); - /* see if the tail matches */ state->repeat = ctx->u.rep->prev; + + LASTMARK_SAVE(); + ctx->in_repeat = (state->repeat != NULL); + if (ctx->in_repeat) + MARK_PUSH(ctx->lastmark); + DO_JUMP(JUMP_MIN_UNTIL_2, jump_min_until_2, ctx->pattern); if (ret) { + if (ctx->in_repeat) + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + if (ctx->in_repeat) + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); state->repeat = ctx->u.rep; state->ptr = ctx->ptr; - LASTMARK_RESTORE(); - if ((ctx->count >= (Py_ssize_t) ctx->u.rep->pattern[2] && ctx->u.rep->pattern[2] != SRE_MAXREPEAT) || state->ptr == ctx->u.rep->last_ptr) From 0db231e1ffe47cff60bf3bb8d7ca27fe2fc5f3e1 Mon Sep 17 00:00:00 2001 From: animalize Date: Sat, 9 Mar 2019 16:00:29 +0800 Subject: [PATCH 06/22] 6. JUMP_REPEAT_ONE_[12] should MARK_PUSH() if in a repeat --- Lib/test/test_re.py | 14 ++++++++++++++ Modules/sre_lib.h | 20 +++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 112b37983f8b678..ca2782089692b78 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2122,6 +2122,20 @@ def test_bug_35859(self): p = r'(?:a*?(xx)??z)*' self.assertEqual(re.match(p, s).groups(), ('xx',)) + # JUMP_REPEAT_ONE_1 should MARK_PUSH() if in a repeat + s = 'aabaab' + p = r'(?:[^b]*a(?=(b)|(a))ab)*' + m = re.match(p, s) + self.assertEqual(m.span(), (0, 6)) + self.assertEqual(m.groups(), (None, 'a')) + + # JUMP_REPEAT_ONE_2 should MARK_PUSH() if in a repeat + s = 'abab' + p = r'(?:[^b]*(?=(b)|(a))ab)*' + m = re.match(p, s) + self.assertEqual(m.span(), (0, 4)) + self.assertEqual(m.groups(), (None, 'a')) + class PatternReprTests(unittest.TestCase): def check(self, pattern, expected): diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index ea2f59f1eacd592..01a900b35e65c6a 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -869,6 +869,9 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) } LASTMARK_SAVE(); + ctx->in_repeat = (state->repeat != NULL); + if (ctx->in_repeat) + MARK_PUSH(ctx->lastmark); if (ctx->pattern[ctx->pattern[0]] == SRE_OP_LITERAL) { /* tail starts with a literal. skip positions where @@ -886,16 +889,20 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_REPEAT_ONE_1, jump_repeat_one_1, ctx->pattern+ctx->pattern[0]); if (ret) { + if (ctx->in_repeat) + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - + if (ctx->in_repeat) + MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); ctx->ptr--; ctx->count--; } - + if (ctx->in_repeat) + MARK_POP_DISCARD(ctx->lastmark); } else { /* general case */ while (ctx->count >= (Py_ssize_t) ctx->pattern[1]) { @@ -903,13 +910,20 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_REPEAT_ONE_2, jump_repeat_one_2, ctx->pattern+ctx->pattern[0]); if (ret) { + if (ctx->in_repeat) + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + if (ctx->in_repeat) + MARK_POP_KEEP(ctx->lastmark); + LASTMARK_RESTORE(); + ctx->ptr--; ctx->count--; - LASTMARK_RESTORE(); } + if (ctx->in_repeat) + MARK_POP_DISCARD(ctx->lastmark); } RETURN_FAILURE; From 0e51eb67a6e66f1c7f620aeade1fd395a6d79495 Mon Sep 17 00:00:00 2001 From: animalize Date: Sat, 9 Mar 2019 16:30:57 +0800 Subject: [PATCH 07/22] 7. JUMP_MIN_REPEAT_ONE should MARK_PUSH() if in a repeat --- Lib/test/test_re.py | 7 +++++++ Modules/sre_lib.h | 13 ++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index ca2782089692b78..5c8d5351fda4a1b 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2136,6 +2136,13 @@ def test_bug_35859(self): self.assertEqual(m.span(), (0, 4)) self.assertEqual(m.groups(), (None, 'a')) + # JUMP_MIN_REPEAT_ONE should MARK_PUSH() if in a repeat + s = 'abab' + p = r'(?:.*?(?=(a)|(b))b)*' + m = re.match(p, s) + self.assertEqual(m.span(), (0, 4)) + self.assertEqual(m.groups(), (None, 'b')) + class PatternReprTests(unittest.TestCase): def check(self, pattern, expected): diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 01a900b35e65c6a..6342a42a8eadcfe 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -972,15 +972,25 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) } else { /* general case */ LASTMARK_SAVE(); + ctx->in_repeat = (state->repeat != NULL); + if (ctx->in_repeat) + MARK_PUSH(ctx->lastmark); + while ((Py_ssize_t)ctx->pattern[2] == SRE_MAXREPEAT || ctx->count <= (Py_ssize_t)ctx->pattern[2]) { state->ptr = ctx->ptr; DO_JUMP(JUMP_MIN_REPEAT_ONE,jump_min_repeat_one, ctx->pattern+ctx->pattern[0]); if (ret) { + if (ctx->in_repeat) + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + if (ctx->in_repeat) + MARK_POP_KEEP(ctx->lastmark); + LASTMARK_RESTORE(); + state->ptr = ctx->ptr; ret = SRE(count)(state, ctx->pattern+3, 1); RETURN_ON_ERROR(ret); @@ -990,8 +1000,9 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) assert(ret == 1); ctx->ptr++; ctx->count++; - LASTMARK_RESTORE(); } + if (ctx->in_repeat) + MARK_POP_DISCARD(ctx->lastmark); } RETURN_FAILURE; From 2bd2eb48d52475516ae7ff935598185ad41c8076 Mon Sep 17 00:00:00 2001 From: animalize Date: Sun, 10 Mar 2019 15:52:31 +0800 Subject: [PATCH 08/22] 8. JUMP_ASSERT_NOT should LASTMARK_SAVE() --- Lib/test/test_re.py | 4 ++++ Modules/sre_lib.h | 2 ++ 2 files changed, 6 insertions(+) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 5c8d5351fda4a1b..c788149d76ead7e 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2143,6 +2143,10 @@ def test_bug_35859(self): self.assertEqual(m.span(), (0, 4)) self.assertEqual(m.groups(), (None, 'b')) + # JUMP_ASSERT_NOT should LASTMARK_SAVE() + # reported in issue725149 + self.assertEqual(re.match(r'(?!(..)c)', 'ab').groups(), (None,)) + class PatternReprTests(unittest.TestCase): def check(self, pattern, expected): diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 6342a42a8eadcfe..b933a62ffba30e6 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1318,11 +1318,13 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) ctx->ptr, ctx->pattern[1])); if (ctx->ptr - (SRE_CHAR *)state->beginning >= (Py_ssize_t)ctx->pattern[1]) { state->ptr = ctx->ptr - ctx->pattern[1]; + LASTMARK_SAVE(); DO_JUMP0(JUMP_ASSERT_NOT, jump_assert_not, ctx->pattern+2); if (ret) { RETURN_ON_ERROR(ret); RETURN_FAILURE; } + LASTMARK_RESTORE(); } ctx->pattern += ctx->pattern[0]; break; From 6a02d5bb60615f857cf8c65162878bfd4a13fcda Mon Sep 17 00:00:00 2001 From: animalize Date: Sun, 10 Mar 2019 15:54:10 +0800 Subject: [PATCH 09/22] 9. JUMP_ASSERT_NOT should MARK_PUSH() if in a repeat other JUMPs use this code: ctx->in_repeat = (state->repeat != NULL) if (ctx->in_repeat) here use: if (state->repeat) this is OK, because JUMP_ASSERT_NOT can't cross a repeat body. --- Lib/test/test_re.py | 7 ++++++- Modules/sre_lib.h | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index c788149d76ead7e..aeb3a01d12e9256 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2141,12 +2141,17 @@ def test_bug_35859(self): p = r'(?:.*?(?=(a)|(b))b)*' m = re.match(p, s) self.assertEqual(m.span(), (0, 4)) - self.assertEqual(m.groups(), (None, 'b')) + self.assertEqual(m.groups(), (None, 'b')) # JUMP_ASSERT_NOT should LASTMARK_SAVE() # reported in issue725149 self.assertEqual(re.match(r'(?!(..)c)', 'ab').groups(), (None,)) + # JUMP_ASSERT_NOT should MARK_PUSH() if in a repeat + m = re.match(r'((?!(ab)c)(.))*', 'abab') + self.assertEqual(m.span(), (0, 4)) + self.assertEqual(m.groups(), ('b', None, 'b')) + class PatternReprTests(unittest.TestCase): def check(self, pattern, expected): diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index b933a62ffba30e6..459f0fe82776880 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1319,11 +1319,18 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) if (ctx->ptr - (SRE_CHAR *)state->beginning >= (Py_ssize_t)ctx->pattern[1]) { state->ptr = ctx->ptr - ctx->pattern[1]; LASTMARK_SAVE(); + if (state->repeat) + MARK_PUSH(ctx->lastmark); + DO_JUMP0(JUMP_ASSERT_NOT, jump_assert_not, ctx->pattern+2); if (ret) { + if (state->repeat) + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_FAILURE; } + if (state->repeat) + MARK_POP(ctx->lastmark); LASTMARK_RESTORE(); } ctx->pattern += ctx->pattern[0]; From edb6d74b02dfca9da0a53cf4f28c672b314222ba Mon Sep 17 00:00:00 2001 From: animalize Date: Sat, 9 Mar 2019 17:14:15 +0800 Subject: [PATCH 10/22] 10. polish match_context struct reduce the size of match_context struct: On 32 bit platform, 36 bytes -> 32 bytes. On 64 bit platform, 72 bytes -> 64 bytes. adjust the order of fields, make it cache friendly. --- Modules/sre_lib.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 459f0fe82776880..2130a318e916b8a 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -530,18 +530,18 @@ do { \ typedef struct { Py_ssize_t last_ctx_pos; - Py_ssize_t jump; SRE_CHAR* ptr; SRE_CODE* pattern; Py_ssize_t count; - Py_ssize_t lastmark; - Py_ssize_t lastindex; union { SRE_CODE chr; SRE_REPEAT* rep; } u; - char toplevel; + Py_ssize_t lastindex; + Py_ssize_t lastmark; char in_repeat; + char toplevel; + char jump; } SRE(match_context); /* check if string matches the given pattern. returns <0 for From c703a723115a5023150b66af19e8d54e2cad8f04 Mon Sep 17 00:00:00 2001 From: animalize Date: Sun, 10 Mar 2019 16:39:27 +0800 Subject: [PATCH 11/22] 11. limit max group to 1,073,741,823 1,073,741,823 groups should enough for most users. the size of match_context struct: On 32 bit platform: 32 bytes, no change. On 64 bit platform: 64 bytes -> 56 bytes. --- Modules/sre.h | 8 ++++---- Modules/sre_lib.h | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Modules/sre.h b/Modules/sre.h index a7284881457c3b1..d708932c0889017 100644 --- a/Modules/sre.h +++ b/Modules/sre.h @@ -16,12 +16,12 @@ /* size of a code word (must be unsigned short or larger, and large enough to hold a UCS4 character) */ #define SRE_CODE Py_UCS4 +#define SRE_MAXGROUPS INT_MAX / 2 + #if SIZEOF_SIZE_T > 4 # define SRE_MAXREPEAT (~(SRE_CODE)0) -# define SRE_MAXGROUPS ((~(SRE_CODE)0) / 2) #else # define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX) -# define SRE_MAXGROUPS ((SRE_CODE)PY_SSIZE_T_MAX / SIZEOF_SIZE_T / 2) #endif typedef struct { @@ -72,8 +72,8 @@ typedef struct { int isbytes; int charsize; /* character size */ /* registers */ - Py_ssize_t lastindex; - Py_ssize_t lastmark; + int32_t lastindex; + int32_t lastmark; void** mark; int match_all; int must_advance; diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 2130a318e916b8a..e799697bb13ac86 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -537,8 +537,8 @@ typedef struct { SRE_CODE chr; SRE_REPEAT* rep; } u; - Py_ssize_t lastindex; - Py_ssize_t lastmark; + int32_t lastindex; + int32_t lastmark; char in_repeat; char toplevel; char jump; @@ -551,8 +551,9 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) { SRE_CHAR* end = (SRE_CHAR *)state->end; Py_ssize_t alloc_pos, ctx_pos = -1; - Py_ssize_t i, ret = 0; + Py_ssize_t ret = 0; Py_ssize_t jump; + int32_t i; unsigned int sigcount=0; SRE(match_context)* ctx; From 2d3fdbabae0a36026d95a5e14d732c141a8caab1 Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 10:18:36 +0800 Subject: [PATCH 12/22] 12. raise RuntimeError if the span of capture group is wrong --- Modules/_sre.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Modules/_sre.c b/Modules/_sre.c index 5cea7562f2807a2..ca37c4bbb10f96b 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -2354,6 +2354,14 @@ pattern_new_match(PatternObject* pattern, SRE_STATE* state, Py_ssize_t status) if (j+1 <= state->lastmark && state->mark[j] && state->mark[j+1]) { match->mark[j+2] = ((char*) state->mark[j] - base) / n; match->mark[j+3] = ((char*) state->mark[j+1] - base) / n; + + /* check wrong span */ + if (match->mark[j+2] > match->mark[j+3]) { + PyErr_Format(PyExc_RuntimeError, + "the span of capturing group %d is wrong," + " please report bug.", i+1); + return NULL; + } } else match->mark[j+2] = match->mark[j+3] = -1; /* undefined */ From ed06d528d23ca6edb3aed259963108b6d97a4f24 Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 10:29:36 +0800 Subject: [PATCH 13/22] 13. clean error process code * remove SRE_ERROR_RECURSION_LIMIT, this code has not been used for a long time. * return SRE_ERROR_MEMORY when allocate fail, rather than set a MemoryError and return a match failure. --- Modules/_sre.c | 7 ------- Modules/sre_lib.h | 7 +++---- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/Modules/_sre.c b/Modules/_sre.c index ca37c4bbb10f96b..de47e7b45a01ec6 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -520,13 +520,6 @@ static void pattern_error(Py_ssize_t status) { switch (status) { - case SRE_ERROR_RECURSION_LIMIT: - /* This error code seems to be unused. */ - PyErr_SetString( - PyExc_RecursionError, - "maximum recursion limit exceeded" - ); - break; case SRE_ERROR_MEMORY: PyErr_NoMemory(); break; diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index e799697bb13ac86..7828574073e60fb 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1016,10 +1016,9 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) /* install new repeat context */ ctx->u.rep = (SRE_REPEAT*) PyObject_MALLOC(sizeof(*ctx->u.rep)); - if (!ctx->u.rep) { - PyErr_NoMemory(); - RETURN_FAILURE; - } + if (!ctx->u.rep) + RETURN_ERROR(SRE_ERROR_MEMORY); + ctx->u.rep->count = -1; ctx->u.rep->pattern = ctx->pattern; ctx->u.rep->prev = state->repeat; From 6c41e2ec183ca3765e887f3533108475ddc06bba Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 17:58:31 +0800 Subject: [PATCH 14/22] 14. add blurb --- .../next/Library/2019-03-11-17-58-19.bpo-35859.8lFdLe.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-03-11-17-58-19.bpo-35859.8lFdLe.rst diff --git a/Misc/NEWS.d/next/Library/2019-03-11-17-58-19.bpo-35859.8lFdLe.rst b/Misc/NEWS.d/next/Library/2019-03-11-17-58-19.bpo-35859.8lFdLe.rst new file mode 100644 index 000000000000000..1432280cd1ff8c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-03-11-17-58-19.bpo-35859.8lFdLe.rst @@ -0,0 +1,2 @@ +re module, fix a few bugs about capturing group. In rare cases, capture +group may be an incorrect string. Patch by Ma Lin. From 5156b0ebfd804ea6915420b61dbb9f285992f366 Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 18:05:56 +0800 Subject: [PATCH 15/22] 15. JUMP_MIN_UNTIL_3 uses MARK_PUSH() protect could not find a test-case to prove that this is necessary. --- Modules/sre_lib.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 7828574073e60fb..2cb65e4e93eea6b 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1159,6 +1159,7 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) RETURN_FAILURE; ctx->u.rep->count = ctx->count; + MARK_PUSH(ctx->lastmark); /* already LASTMARK_SAVE() above */ /* zero-width match protection */ DATA_PUSH(&ctx->u.rep->last_ptr); ctx->u.rep->last_ptr = state->ptr; @@ -1166,9 +1167,12 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) ctx->u.rep->pattern+3); DATA_POP(&ctx->u.rep->last_ptr); if (ret) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); ctx->u.rep->count = ctx->count-1; state->ptr = ctx->ptr; RETURN_FAILURE; From 494c00823aa19d2ddd84992cd48057a5682e0b81 Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 18:21:26 +0800 Subject: [PATCH 16/22] SRE_OP_MAX_UNTIL --- Modules/sre_lib.h | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 2cb65e4e93eea6b..3d94764589e3821 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1054,15 +1054,22 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) TRACE(("|%p|%p|MAX_UNTIL %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, ctx->ptr, ctx->count)); + LASTMARK_SAVE(); + MARK_PUSH(ctx->lastmark); + if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1, ctx->u.rep->pattern+3); if (ret) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); + ctx->u.rep->count = ctx->count-1; state->ptr = ctx->ptr; RETURN_FAILURE; @@ -1074,8 +1081,6 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) /* we may have enough matches, but if we can match another item, do so */ ctx->u.rep->count = ctx->count; - LASTMARK_SAVE(); - MARK_PUSH(ctx->lastmark); /* zero-width match protection */ DATA_PUSH(&ctx->u.rep->last_ptr); ctx->u.rep->last_ptr = state->ptr; @@ -1087,7 +1092,7 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - MARK_POP(ctx->lastmark); + MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); ctx->u.rep->count = ctx->count-1; state->ptr = ctx->ptr; @@ -1097,7 +1102,13 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) tail matches */ state->repeat = ctx->u.rep->prev; DO_JUMP(JUMP_MAX_UNTIL_3, jump_max_until_3, ctx->pattern); - RETURN_ON_SUCCESS(ret); + if (ret) { + MARK_POP_DISCARD(ctx->lastmark); + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); state->repeat = ctx->u.rep; state->ptr = ctx->ptr; RETURN_FAILURE; From da4091362e7c1cbb408211cb2a9719d93efef9b0 Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 18:38:07 +0800 Subject: [PATCH 17/22] SRE_OP_MIN_UNTIL --- Modules/sre_lib.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 3d94764589e3821..98f9ec6de3dbd55 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1128,15 +1128,21 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) TRACE(("|%p|%p|MIN_UNTIL %" PY_FORMAT_SIZE_T "d %p\n", ctx->pattern, ctx->ptr, ctx->count, ctx->u.rep->pattern)); + LASTMARK_SAVE(); + MARK_PUSH(ctx->lastmark); + if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1, ctx->u.rep->pattern+3); if (ret) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); ctx->u.rep->count = ctx->count-1; state->ptr = ctx->ptr; RETURN_FAILURE; @@ -1145,20 +1151,13 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) /* see if the tail matches */ state->repeat = ctx->u.rep->prev; - LASTMARK_SAVE(); - ctx->in_repeat = (state->repeat != NULL); - if (ctx->in_repeat) - MARK_PUSH(ctx->lastmark); - DO_JUMP(JUMP_MIN_UNTIL_2, jump_min_until_2, ctx->pattern); if (ret) { - if (ctx->in_repeat) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (ctx->in_repeat) - MARK_POP(ctx->lastmark); + MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); state->repeat = ctx->u.rep; @@ -1166,11 +1165,12 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) if ((ctx->count >= (Py_ssize_t) ctx->u.rep->pattern[2] && ctx->u.rep->pattern[2] != SRE_MAXREPEAT) || - state->ptr == ctx->u.rep->last_ptr) + state->ptr == ctx->u.rep->last_ptr) { + MARK_POP_DISCARD(ctx->lastmark); RETURN_FAILURE; + } ctx->u.rep->count = ctx->count; - MARK_PUSH(ctx->lastmark); /* already LASTMARK_SAVE() above */ /* zero-width match protection */ DATA_PUSH(&ctx->u.rep->last_ptr); ctx->u.rep->last_ptr = state->ptr; From 9a04391046696e7861bce068ef71566bec12ec44 Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 18:42:59 +0800 Subject: [PATCH 18/22] SRE_OP_REPEAT_ONE --- Modules/sre_lib.h | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 98f9ec6de3dbd55..6455db73ec76f2e 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -870,9 +870,7 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) } LASTMARK_SAVE(); - ctx->in_repeat = (state->repeat != NULL); - if (ctx->in_repeat) - MARK_PUSH(ctx->lastmark); + MARK_PUSH(ctx->lastmark); if (ctx->pattern[ctx->pattern[0]] == SRE_OP_LITERAL) { /* tail starts with a literal. skip positions where @@ -890,20 +888,17 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_REPEAT_ONE_1, jump_repeat_one_1, ctx->pattern+ctx->pattern[0]); if (ret) { - if (ctx->in_repeat) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (ctx->in_repeat) - MARK_POP_KEEP(ctx->lastmark); + MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); ctx->ptr--; ctx->count--; } - if (ctx->in_repeat) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); } else { /* general case */ while (ctx->count >= (Py_ssize_t) ctx->pattern[1]) { @@ -911,20 +906,17 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_REPEAT_ONE_2, jump_repeat_one_2, ctx->pattern+ctx->pattern[0]); if (ret) { - if (ctx->in_repeat) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (ctx->in_repeat) - MARK_POP_KEEP(ctx->lastmark); + MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); ctx->ptr--; ctx->count--; } - if (ctx->in_repeat) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); } RETURN_FAILURE; From 54402bcc1a587685e1d8a4252110eeeb14fa31b4 Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 18:44:50 +0800 Subject: [PATCH 19/22] SRE_OP_MIN_REPEAT_ONE --- Modules/sre_lib.h | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 6455db73ec76f2e..020e1e4acd3b066 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -965,9 +965,7 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) } else { /* general case */ LASTMARK_SAVE(); - ctx->in_repeat = (state->repeat != NULL); - if (ctx->in_repeat) - MARK_PUSH(ctx->lastmark); + MARK_PUSH(ctx->lastmark); while ((Py_ssize_t)ctx->pattern[2] == SRE_MAXREPEAT || ctx->count <= (Py_ssize_t)ctx->pattern[2]) { @@ -975,13 +973,11 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_MIN_REPEAT_ONE,jump_min_repeat_one, ctx->pattern+ctx->pattern[0]); if (ret) { - if (ctx->in_repeat) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (ctx->in_repeat) - MARK_POP_KEEP(ctx->lastmark); + MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); state->ptr = ctx->ptr; @@ -994,8 +990,7 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) ctx->ptr++; ctx->count++; } - if (ctx->in_repeat) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); } RETURN_FAILURE; From 8f2fe8a838cfd86a3064e4baddc6a3be7d81dfd9 Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 18:46:09 +0800 Subject: [PATCH 20/22] SRE_OP_BRANCH --- Modules/sre_lib.h | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 020e1e4acd3b066..28c9b706ff04679 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -798,10 +798,9 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) /* alternation */ /* <0=skip> code ... */ TRACE(("|%p|%p|BRANCH\n", ctx->pattern, ctx->ptr)); + LASTMARK_SAVE(); - ctx->in_repeat = (state->repeat != NULL); - if (ctx->in_repeat) - MARK_PUSH(ctx->lastmark); + MARK_PUSH(ctx->lastmark); for (; ctx->pattern[0]; ctx->pattern += ctx->pattern[0]) { if (ctx->pattern[1] == SRE_OP_LITERAL && (ctx->ptr >= end || @@ -815,17 +814,14 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) state->ptr = ctx->ptr; DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1); if (ret) { - if (ctx->in_repeat) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (ctx->in_repeat) - MARK_POP_KEEP(ctx->lastmark); + MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); } - if (ctx->in_repeat) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); RETURN_FAILURE; case SRE_OP_REPEAT_ONE: From 8e184f5f3093a5b93ecf180cf7de125d0760a1b1 Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 18:50:26 +0800 Subject: [PATCH 21/22] SRE_OP_ASSERT --- Modules/sre_lib.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 28c9b706ff04679..0e8442bc7dd95ff 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1304,8 +1304,18 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) if (ctx->ptr - (SRE_CHAR *)state->beginning < (Py_ssize_t)ctx->pattern[1]) RETURN_FAILURE; state->ptr = ctx->ptr - ctx->pattern[1]; + + LASTMARK_SAVE(); + MARK_PUSH(ctx->lastmark); DO_JUMP0(JUMP_ASSERT, jump_assert, ctx->pattern+2); - RETURN_ON_FAILURE(ret); + if (ret <= 0) { + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); + + RETURN_ON_ERROR(ret); + RETURN_FAILURE; + } + MARK_POP_DISCARD(ctx->lastmark); ctx->pattern += ctx->pattern[0]; break; From 3c31ecc95277383160c03d5c671c1a670dc1b2d8 Mon Sep 17 00:00:00 2001 From: animalize Date: Mon, 11 Mar 2019 18:51:58 +0800 Subject: [PATCH 22/22] SRE_OP_ASSERT_NOT --- Modules/sre_lib.h | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 0e8442bc7dd95ff..181d944c1c30b98 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1327,18 +1327,15 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) if (ctx->ptr - (SRE_CHAR *)state->beginning >= (Py_ssize_t)ctx->pattern[1]) { state->ptr = ctx->ptr - ctx->pattern[1]; LASTMARK_SAVE(); - if (state->repeat) - MARK_PUSH(ctx->lastmark); + MARK_PUSH(ctx->lastmark); DO_JUMP0(JUMP_ASSERT_NOT, jump_assert_not, ctx->pattern+2); if (ret) { - if (state->repeat) - MARK_POP_DISCARD(ctx->lastmark); + MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_FAILURE; } - if (state->repeat) - MARK_POP(ctx->lastmark); + MARK_POP(ctx->lastmark); LASTMARK_RESTORE(); } ctx->pattern += ctx->pattern[0];