Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 836 lines (700 sloc) 29.814 kB
a0d0e21 perl 5.000
Larry Wall authored
1 /* regcomp.h
d637624 @jhi Sprinkle some copyrights (use the oldest timestamp to
jhi authored
2 *
4bb101f @jhi Fix up Larry's copyright statements to my best knowledge.
jhi authored
3 * Copyright (C) 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
663f364 Fix up copyright years for files modified in 2007.
Nicholas Clark authored
4 * 2000, 2001, 2002, 2003, 2005, 2006, 2007, by Larry Wall and others
d637624 @jhi Sprinkle some copyrights (use the oldest timestamp to
jhi authored
5 *
6 * You may distribute under the terms of either the GNU General Public
7 * License or the Artistic License, as specified in the README file.
8 *
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
9 */
e1d1eef @demerphq Change meaning of \v, \V, and add \h, \H to match Perl6, add \R to ma…
demerphq authored
10 #include "regcharclass.h"
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
11
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
12 typedef OP OP_4tree; /* Will be redefined later. */
13
07be1b8 @demerphq Re: [PATCH] Better version of the Aho-Corasick patch and lots of benc…
demerphq authored
14
58e23c8 @demerphq \G with /g results in infinite loop in 5.6 and later
demerphq authored
15 /* Convert branch sequences to more efficient trie ops? */
07be1b8 @demerphq Re: [PATCH] Better version of the Aho-Corasick patch and lots of benc…
demerphq authored
16 #define PERL_ENABLE_TRIE_OPTIMISATION 1
58e23c8 @demerphq \G with /g results in infinite loop in 5.6 and later
demerphq authored
17
18 /* Be really agressive about optimising patterns with trie sequences? */
07be1b8 @demerphq Re: [PATCH] Better version of the Aho-Corasick patch and lots of benc…
demerphq authored
19 #define PERL_ENABLE_EXTENDED_TRIE_OPTIMISATION 1
58e23c8 @demerphq \G with /g results in infinite loop in 5.6 and later
demerphq authored
20
da7fcca @demerphq create new unicode props as defined in POSIX spec (optionally use the…
demerphq authored
21 /* Use old style unicode mappings for perl and posix character classes
22 *
23 * NOTE: Enabling this essentially breaks character class matching against unicode
24 * strings, so that POSIX char classes match when they shouldn't, and \d matches
25 * way more than 10 characters, and sometimes a charclass and its complement either
26 * both match or neither match.
27 * NOTE: Disabling this will cause various backwards compatibility issues to rear
28 * their head, and tests to fail. However it will make the charclass behaviour
29 * consistant regardless of internal string type, and make character class inversions
30 * consistant. The tests that fail in the regex engine are basically broken tests.
31 *
32 * Personally I think 5.12 should disable this for sure. Its a bit more debatable for
33 * 5.10, so for now im leaving it enabled.
ba9ac17 @demerphq set PERL_LEGACY_UNICODE_CHARCLASS_MAPPINGS to 0 and enable proper POS…
demerphq authored
34 * XXX: It is now enabled for 5.11/5.12
da7fcca @demerphq create new unicode props as defined in POSIX spec (optionally use the…
demerphq authored
35 *
36 * -demerphq
37 */
2e84be6 @demerphq revert to 5.8.x semantics for \s \w and \d
demerphq authored
38 #define PERL_LEGACY_UNICODE_CHARCLASS_MAPPINGS 1
da7fcca @demerphq create new unicode props as defined in POSIX spec (optionally use the…
demerphq authored
39
58e23c8 @demerphq \G with /g results in infinite loop in 5.6 and later
demerphq authored
40 /* Should the optimiser take positive assertions into account? */
68ba3a3 @demerphq Disable positive lookaround optimisations
demerphq authored
41 #define PERL_ENABLE_POSITIVE_ASSERTION_STUDY 0
58e23c8 @demerphq \G with /g results in infinite loop in 5.6 and later
demerphq authored
42
43 /* Not for production use: */
07be1b8 @demerphq Re: [PATCH] Better version of the Aho-Corasick patch and lots of benc…
demerphq authored
44 #define PERL_ENABLE_EXPERIMENTAL_REGEX_OPTIMISATIONS 0
58e23c8 @demerphq \G with /g results in infinite loop in 5.6 and later
demerphq authored
45
7122b23 @demerphq Make offsets support conditional
demerphq authored
46 /* Activate offsets code - set to if 1 to enable */
47 #ifdef DEBUGGING
48 #define RE_TRACK_PATTERN_OFFSETS
49 #endif
50
b9b4ddd @demerphq Add possessive quantifiers to regex engine.
demerphq authored
51 /* Unless the next line is uncommented it is illegal to combine lazy
52 matching with possessive matching. Frankly it doesn't make much sense
53 to allow it as X*?+ matches nothing, X+?+ matches a single char only,
54 and X{min,max}?+ matches min times only.
55 */
56 /* #define REG_ALLOW_MINMOD_SUSPEND */
07be1b8 @demerphq Re: [PATCH] Better version of the Aho-Corasick patch and lots of benc…
demerphq authored
57
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
58 /*
59 * The "internal use only" fields in regexp.h are present to pass info from
60 * compile to execute that permits the execute phase to run lots faster on
61 * simple cases. They are:
62 *
bd61b36 Re: [PATCH] s/Null(av|ch)/NULL/g
Steven Schubiger authored
63 * regstart sv that must begin a match; NULL if none obvious
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
64 * reganch is the match anchored (at beginning-of-line only)?
65 * regmust string (pointer into program) that match must include, or NULL
7907280 perl 5.0 alpha 2
Larry Wall authored
66 * [regmust changed to SV* for bminstr()--law]
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
67 * regmlen length of regmust string
68 * [regmlen not used currently]
69 *
70 * Regstart and reganch permit very fast decisions on suitable starting points
71 * for a match, cutting down the work a lot. Regmust permits fast rejection
72 * of lines that cannot possibly match. The regmust tests are costly enough
e50aee7 This is my patch patch.1m for perl5.001.
Andy Dougherty authored
73 * that pregcomp() supplies a regmust only if the r.e. contains something
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
74 * potentially expensive (at present, the only such thing detected is * or +
75 * at the start of the r.e., which can involve a lot of backup). Regmlen is
e50aee7 This is my patch patch.1m for perl5.001.
Andy Dougherty authored
76 * supplied because the test in pregexec() needs it and pregcomp() is computing
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
77 * it anyway.
78 * [regmust is now supplied always. The tests that use regmust have a
79 * heuristic that disables the test if it usually matches.]
80 *
81 * [In fact, we now use regmust in many cases to locate where the search
82 * starts in the string, so if regback is >= 0, the regmust search is never
83 * wasted effort. The regback variable says how many characters back from
84 * where regmust matched is the earliest possible start of the match.
85 * For instance, /[a-z].foo/ has a regmust of 'foo' and a regback of 2.]
86 */
87
88 /*
89 * Structure for regexp "program". This is essentially a linear encoding
90 * of a nondeterministic finite-state machine (aka syntax charts or
91 * "railroad normal form" in parsing technology). Each node is an opcode
92 * plus a "next" pointer, possibly plus an operand. "Next" pointers of
93 * all nodes except BRANCH implement concatenation; a "next" pointer with
94 * a BRANCH on both ends of it is connecting two alternatives. (Here we
95 * have one of the subtle syntax dependencies: an individual BRANCH (as
96 * opposed to a collection of them) is never concatenated with anything
97 * because of operator precedence.) The operand of some types of node is
98 * a literal string; for others, it is a node leading into a sub-FSM. In
99 * particular, the operand of a BRANCH node is the first node of the branch.
100 * (NB this is *not* a tree structure: the tail of the branch connects
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
101 * to the thing following the set of BRANCHes.) The opcodes are defined
102 * in regnodes.h which is generated from regcomp.sym by regcomp.pl.
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
103 */
104
105 /*
106 * A node is one char of opcode followed by two chars of "next" pointer.
107 * "Next" pointers are stored as two 8-bit pieces, high order first. The
108 * value is a positive offset from the opcode of the node containing it.
109 * An operand, if any, simply follows the node. (Note that much of the
110 * code generation knows about this implicit relationship.)
111 *
112 * Using two bytes for the "next" pointer is vast overkill for most things,
113 * but allows patterns to get big without disasters.
114 *
e91177e applied patch, with indentation tweaks
Ilya Zakharevich authored
115 * [The "next" pointer is always aligned on an even
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
116 * boundary, and reads the offset directly as a short. Also, there is no
117 * special test to reverse the sign of BACK pointers since the offset is
118 * stored negative.]
119 */
785a26d @demerphq Further tweaks to make it easier to create regexp engine plug ins.
demerphq authored
120
121 /* This is the stuff that used to live in regexp.h that was truly
122 private to the engine itself. It now lives here. */
123
28d8d7f @demerphq Resolve PL_curpm issues with (??{}) and fix corruption of match resul…
demerphq authored
124
785a26d @demerphq Further tweaks to make it easier to create regexp engine plug ins.
demerphq authored
125
126 typedef struct regexp_internal {
1f1031f @demerphq Re: Named-capture regex syntax
demerphq authored
127 int name_list_idx; /* Optional data index of an array of paren names */
7122b23 @demerphq Make offsets support conditional
demerphq authored
128 union {
129 U32 *offsets; /* offset annotations 20001228 MJD
130 data about mapping the program to the
131 string -
132 offsets[0] is proglen when this is used
133 */
134 U32 proglen;
135 } u;
136
6d5c990 @rgs Better version of last patch, by Yves Orton.
rgs authored
137 regnode *regstclass; /* Optional startclass as identified or constructed
138 by the optimiser */
139 struct reg_data *data; /* Additional miscellaneous data used by the program.
140 Used to make it easier to clone and free arbitrary
141 data that the regops need. Often the ARG field of
142 a regop is an index into this structure */
143 regnode program[1]; /* Unwarranted chumminess with compiler. */
144 } regexp_internal;
145
146 #define RXi_SET(x,y) (x)->pprivate = (void*)(y)
147 #define RXi_GET(x) ((regexp_internal *)((x)->pprivate))
148 #define RXi_GET_DECL(r,ri) regexp_internal *ri = RXi_GET(r)
785a26d @demerphq Further tweaks to make it easier to create regexp engine plug ins.
demerphq authored
149 /*
150 * Flags stored in regexp->intflags
151 * These are used only internally to the regexp engine
152 *
153 * See regexp.h for flags used externally to the regexp engine
154 */
155 #define PREGf_SKIP 0x00000001
156 #define PREGf_IMPLICIT 0x00000002 /* Converted .* to ^.* */
157 #define PREGf_NAUGHTY 0x00000004 /* how exponential is this pattern? */
158 #define PREGf_VERBARG_SEEN 0x00000008
159 #define PREGf_CUTGROUP_SEEN 0x00000010
160
161
162 /* this is where the old regcomp.h started */
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
163
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
164 struct regnode_string {
cd439c5 Clarify RE engine code; inline a static function to a macro;
Ilya Zakharevich authored
165 U8 str_len;
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
166 U8 type;
167 U16 next_off;
cd439c5 Clarify RE engine code; inline a static function to a macro;
Ilya Zakharevich authored
168 char string[1];
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
169 };
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
170
6bda09f @demerphq Re: [PATCH] Add recursive regexes similar to PCRE
demerphq authored
171 /* Argument bearing node - workhorse,
172 arg1 is often for the data field */
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
173 struct regnode_1 {
174 U8 flags;
175 U8 type;
176 U16 next_off;
177 U32 arg1;
178 };
179
6bda09f @demerphq Re: [PATCH] Add recursive regexes similar to PCRE
demerphq authored
180 /* Similar to a regnode_1 but with an extra signed argument */
181 struct regnode_2L {
182 U8 flags;
183 U8 type;
184 U16 next_off;
185 U32 arg1;
186 I32 arg2;
187 };
188
189 /* 'Two field' -- Two 16 bit unsigned args */
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
190 struct regnode_2 {
191 U8 flags;
192 U8 type;
193 U16 next_off;
194 U16 arg1;
195 U16 arg2;
196 };
197
6bda09f @demerphq Re: [PATCH] Add recursive regexes similar to PCRE
demerphq authored
198
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
199 #define ANYOF_BITMAP_SIZE 32 /* 256 b/(8 b/B) */
d1eb317 @demerphq somewhat fix failing regex tests. but break lots of other stuff at th…
demerphq authored
200 #define ANYOF_CLASSBITMAP_SIZE 4 /* up to 32 (8*4) named classes */
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
201
786e8c1 @demerphq Re: [PATCH] Trie jumping
demerphq authored
202 /* also used by trie */
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
203 struct regnode_charclass {
204 U8 flags;
205 U8 type;
206 U16 next_off;
ffc61ed @jhi Polymorphic regexps.
jhi authored
207 U32 arg1;
19103b9 @jhi Document ANYOF_CLASS.
jhi authored
208 char bitmap[ANYOF_BITMAP_SIZE]; /* only compile-time */
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
209 };
210
19103b9 @jhi Document ANYOF_CLASS.
jhi authored
211 struct regnode_charclass_class { /* has [[:blah:]] classes */
212 U8 flags; /* should have ANYOF_CLASS here */
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
213 U8 type;
214 U16 next_off;
ffc61ed @jhi Polymorphic regexps.
jhi authored
215 U32 arg1;
19103b9 @jhi Document ANYOF_CLASS.
jhi authored
216 char bitmap[ANYOF_BITMAP_SIZE]; /* both compile-time */
217 char classflags[ANYOF_CLASSBITMAP_SIZE]; /* and run-time */
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
218 };
219
9c7e81e make REG_INFTY default to something saner when sizeof(short) > 2
Andy Dougherty authored
220 /* XXX fix this description.
221 Impose a limit of REG_INFTY on various pattern matching operations
222 to limit stack growth and to avoid "infinite" recursions.
223 */
224 /* The default size for REG_INFTY is I16_MAX, which is the same as
225 SHORT_MAX (see perl.h). Unfortunately I16 isn't necessarily 16 bits
226 (see handy.h). On the Cray C90, sizeof(short)==4 and hence I16_MAX is
227 ((1<<31)-1), while on the Cray T90, sizeof(short)==8 and I16_MAX is
228 ((1<<63)-1). To limit stack growth to reasonable sizes, supply a
229 smaller default.
230 --Andy Dougherty 11 June 1998
231 */
232 #if SHORTSIZE > 2
233 # ifndef REG_INFTY
234 # define REG_INFTY ((1<<15)-1)
235 # endif
236 #endif
237
238 #ifndef REG_INFTY
83cfe48 Re: [PATCH 5.004_66] REG_INFTY patch corrected
Ilya Zakharevich authored
239 # define REG_INFTY I16_MAX
240 #endif
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
241
e91177e applied patch, with indentation tweaks
Ilya Zakharevich authored
242 #define ARG_VALUE(arg) (arg)
243 #define ARG__SET(arg,val) ((arg) = (val))
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
244
e8e3b0f @jhi Add preemptive #undefs for some of the shorter and more prone
jhi authored
245 #undef ARG
246 #undef ARG1
247 #undef ARG2
248
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
249 #define ARG(p) ARG_VALUE(ARG_LOC(p))
250 #define ARG1(p) ARG_VALUE(ARG1_LOC(p))
251 #define ARG2(p) ARG_VALUE(ARG2_LOC(p))
6bda09f @demerphq Re: [PATCH] Add recursive regexes similar to PCRE
demerphq authored
252 #define ARG2L(p) ARG_VALUE(ARG2L_LOC(p))
786e8c1 @demerphq Re: [PATCH] Trie jumping
demerphq authored
253
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
254 #define ARG_SET(p, val) ARG__SET(ARG_LOC(p), (val))
255 #define ARG1_SET(p, val) ARG__SET(ARG1_LOC(p), (val))
256 #define ARG2_SET(p, val) ARG__SET(ARG2_LOC(p), (val))
6bda09f @demerphq Re: [PATCH] Add recursive regexes similar to PCRE
demerphq authored
257 #define ARG2L_SET(p, val) ARG__SET(ARG2L_LOC(p), (val))
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
258
e8e3b0f @jhi Add preemptive #undefs for some of the shorter and more prone
jhi authored
259 #undef NEXT_OFF
260 #undef NODE_ALIGN
261
075abff @petdance Updated lint infrastructure
petdance authored
262 #define NEXT_OFF(p) ((p)->next_off)
263 #define NODE_ALIGN(node)
264 #define NODE_ALIGN_FILL(node) ((node)->flags = 0xde) /* deadbeef */
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
265
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
266 #define SIZE_ALIGN NODE_ALIGN
267
e8e3b0f @jhi Add preemptive #undefs for some of the shorter and more prone
jhi authored
268 #undef OP
269 #undef OPERAND
270 #undef MASK
271 #undef STRING
272
e91177e applied patch, with indentation tweaks
Ilya Zakharevich authored
273 #define OP(p) ((p)->type)
b4eac3b Subject: regcomp.h: Add macro to get regnode flags
Karl Williamson authored
274 #define FLAGS(p) ((p)->flags) /* Caution: Doesn't apply to all
275 regnode types */
e91177e applied patch, with indentation tweaks
Ilya Zakharevich authored
276 #define OPERAND(p) (((struct regnode_string *)p)->string)
cd439c5 Clarify RE engine code; inline a static function to a macro;
Ilya Zakharevich authored
277 #define MASK(p) ((char*)OPERAND(p))
278 #define STR_LEN(p) (((struct regnode_string *)p)->str_len)
279 #define STRING(p) (((struct regnode_string *)p)->string)
280 #define STR_SZ(l) ((l + sizeof(regnode) - 1) / sizeof(regnode))
281 #define NODE_SZ_STR(p) (STR_SZ(STR_LEN(p))+1)
282
e8e3b0f @jhi Add preemptive #undefs for some of the shorter and more prone
jhi authored
283 #undef NODE_ALIGN
284 #undef ARG_LOC
285 #undef NEXTOPER
286 #undef PREVOPER
287
e91177e applied patch, with indentation tweaks
Ilya Zakharevich authored
288 #define NODE_ALIGN(node)
289 #define ARG_LOC(p) (((struct regnode_1 *)p)->arg1)
290 #define ARG1_LOC(p) (((struct regnode_2 *)p)->arg1)
291 #define ARG2_LOC(p) (((struct regnode_2 *)p)->arg2)
6bda09f @demerphq Re: [PATCH] Add recursive regexes similar to PCRE
demerphq authored
292 #define ARG2L_LOC(p) (((struct regnode_2L *)p)->arg2)
786e8c1 @demerphq Re: [PATCH] Trie jumping
demerphq authored
293
e91177e applied patch, with indentation tweaks
Ilya Zakharevich authored
294 #define NODE_STEP_REGNODE 1 /* sizeof(regnode)/sizeof(regnode) */
295 #define EXTRA_STEP_2ARGS EXTRA_SIZE(struct regnode_2)
296
297 #define NODE_STEP_B 4
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
298
299 #define NEXTOPER(p) ((p) + NODE_STEP_REGNODE)
300 #define PREVOPER(p) ((p) - NODE_STEP_REGNODE)
301
e91177e applied patch, with indentation tweaks
Ilya Zakharevich authored
302 #define FILL_ADVANCE_NODE(ptr, op) STMT_START { \
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
303 (ptr)->type = op; (ptr)->next_off = 0; (ptr)++; } STMT_END
e91177e applied patch, with indentation tweaks
Ilya Zakharevich authored
304 #define FILL_ADVANCE_NODE_ARG(ptr, op, arg) STMT_START { \
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
305 ARG_SET(ptr, arg); FILL_ADVANCE_NODE(ptr, op); (ptr) += 1; } STMT_END
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
306
22c35a8 phase 1 of somewhat major rearrangement of PERL_OBJECT stuff
Gurusamy Sarathy authored
307 #define REG_MAGIC 0234
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
308
830247a [PATCH 5.7.0] make regcomp reenterable
Ilya Zakharevich authored
309 #define SIZE_ONLY (RExC_emit == &PL_regdummy)
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
310
a12cf05 Subject: [perl #58182] partial: Add uni \s,\w matching
Karl Williamson authored
311 /* Flags for node->flags of several of the node types */
312 #define USE_UNI 0x01
313
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
314 /* Flags for node->flags of ANYOF */
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
315
19103b9 @jhi Document ANYOF_CLASS.
jhi authored
316 #define ANYOF_CLASS 0x08 /* has [[:blah:]] classes */
ffc61ed @jhi Polymorphic regexps.
jhi authored
317 #define ANYOF_INVERT 0x04
318 #define ANYOF_FOLD 0x02
319 #define ANYOF_LOCALE 0x01
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
320
653099f apply change#4618 again along with Ilya's patch to fix bugs
Gurusamy Sarathy authored
321 /* Used for regstclass only */
ffc61ed @jhi Polymorphic regexps.
jhi authored
322 #define ANYOF_EOS 0x10 /* Can match an empty string too */
323
324 /* There is a character or a range past 0xff */
325 #define ANYOF_UNICODE 0x20
1aa99e6 @Inaba-Hiroto more UTF8 test suites and an UTF8 patch
Inaba-Hiroto authored
326 #define ANYOF_UNICODE_ALL 0x40 /* Can match any char past 0xff */
ffc61ed @jhi Polymorphic regexps.
jhi authored
327
4f66b38 @hvds Re: [PATCH @9846] dumping ANYOF
hvds authored
328 /* size of node is large (includes class pointer) */
329 #define ANYOF_LARGE 0x80
330
ffc61ed @jhi Polymorphic regexps.
jhi authored
331 /* Are there any runtime flags on in this node? */
332 #define ANYOF_RUNTIME(s) (ANYOF_FLAGS(s) & 0x0f)
333
334 #define ANYOF_FLAGS_ALL 0xff
653099f apply change#4618 again along with Ilya's patch to fix bugs
Gurusamy Sarathy authored
335
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
336 /* Character classes for node->classflags of ANYOF */
653099f apply change#4618 again along with Ilya's patch to fix bugs
Gurusamy Sarathy authored
337 /* Should be synchronized with a table in regprop() */
338 /* 2n should pair with 2n+1 */
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
339
289d4f0 Fix for
Marc Lehmann authored
340 #define ANYOF_ALNUM 0 /* \w, PL_utf8_alnum, utf8::IsWord, ALNUM */
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
341 #define ANYOF_NALNUM 1
aaa51d5 Add [[:blank:]] as suggested in
Jeffrey Friedl authored
342 #define ANYOF_SPACE 2 /* \s */
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
343 #define ANYOF_NSPACE 3
24d786f @demerphq Various changes to regex diagnostics and testing
demerphq authored
344 #define ANYOF_DIGIT 4 /* \d */
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
345 #define ANYOF_NDIGIT 5
24d786f @demerphq Various changes to regex diagnostics and testing
demerphq authored
346 #define ANYOF_ALNUMC 6 /* [[:alnum:]] isalnum(3), utf8::IsAlnum, ALNUMC */
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
347 #define ANYOF_NALNUMC 7
348 #define ANYOF_ALPHA 8
349 #define ANYOF_NALPHA 9
350 #define ANYOF_ASCII 10
351 #define ANYOF_NASCII 11
352 #define ANYOF_CNTRL 12
353 #define ANYOF_NCNTRL 13
354 #define ANYOF_GRAPH 14
355 #define ANYOF_NGRAPH 15
356 #define ANYOF_LOWER 16
357 #define ANYOF_NLOWER 17
358 #define ANYOF_PRINT 18
359 #define ANYOF_NPRINT 19
360 #define ANYOF_PUNCT 20
361 #define ANYOF_NPUNCT 21
362 #define ANYOF_UPPER 22
363 #define ANYOF_NUPPER 23
364 #define ANYOF_XDIGIT 24
365 #define ANYOF_NXDIGIT 25
aaa51d5 Add [[:blank:]] as suggested in
Jeffrey Friedl authored
366 #define ANYOF_PSXSPC 26 /* POSIX space: \s plus the vertical tab */
367 #define ANYOF_NPSXSPC 27
ffc61ed @jhi Polymorphic regexps.
jhi authored
368 #define ANYOF_BLANK 28 /* GNU extension: space and tab: non-vertical space */
289d4f0 Fix for
Marc Lehmann authored
369 #define ANYOF_NBLANK 29
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
370
aaa51d5 Add [[:blank:]] as suggested in
Jeffrey Friedl authored
371 #define ANYOF_MAX 32
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
372
e1d1eef @demerphq Change meaning of \v, \V, and add \h, \H to match Perl6, add \R to ma…
demerphq authored
373 /* pseudo classes, not stored in the class bitmap, but used as flags
374 during compilation of char classes */
375
376 #define ANYOF_VERTWS (ANYOF_MAX+1)
377 #define ANYOF_NVERTWS (ANYOF_MAX+2)
378 #define ANYOF_HORIZWS (ANYOF_MAX+3)
379 #define ANYOF_NHORIZWS (ANYOF_MAX+4)
380
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
381 /* Backward source code compatibility. */
382
383 #define ANYOF_ALNUML ANYOF_ALNUM
384 #define ANYOF_NALNUML ANYOF_NALNUM
385 #define ANYOF_SPACEL ANYOF_SPACE
386 #define ANYOF_NSPACEL ANYOF_NSPACE
387
388 /* Utility macros for the bitmap and classes of ANYOF */
389
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
390 #define ANYOF_SIZE (sizeof(struct regnode_charclass))
391 #define ANYOF_CLASS_SIZE (sizeof(struct regnode_charclass_class))
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
392
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
393 #define ANYOF_FLAGS(p) ((p)->flags)
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
394
395 #define ANYOF_BIT(c) (1 << ((c) & 7))
396
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
397 #define ANYOF_CLASS_BYTE(p, c) (((struct regnode_charclass_class*)(p))->classflags[((c) >> 3) & 3])
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
398 #define ANYOF_CLASS_SET(p, c) (ANYOF_CLASS_BYTE(p, c) |= ANYOF_BIT(c))
399 #define ANYOF_CLASS_CLEAR(p, c) (ANYOF_CLASS_BYTE(p, c) &= ~ANYOF_BIT(c))
400 #define ANYOF_CLASS_TEST(p, c) (ANYOF_CLASS_BYTE(p, c) & ANYOF_BIT(c))
401
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
402 #define ANYOF_CLASS_ZERO(ret) Zero(((struct regnode_charclass_class*)(ret))->classflags, ANYOF_CLASSBITMAP_SIZE, char)
403 #define ANYOF_BITMAP_ZERO(ret) Zero(((struct regnode_charclass*)(ret))->bitmap, ANYOF_BITMAP_SIZE, char)
404
405 #define ANYOF_BITMAP(p) (((struct regnode_charclass*)(p))->bitmap)
786e8c1 @demerphq Re: [PATCH] Trie jumping
demerphq authored
406 #define ANYOF_BITMAP_BYTE(p, c) (ANYOF_BITMAP(p)[(((U8)(c)) >> 3) & 31])
b8c5462 @jhi POSIX [[:character class:]] support for standard, locale,
jhi authored
407 #define ANYOF_BITMAP_SET(p, c) (ANYOF_BITMAP_BYTE(p, c) |= ANYOF_BIT(c))
408 #define ANYOF_BITMAP_CLEAR(p,c) (ANYOF_BITMAP_BYTE(p, c) &= ~ANYOF_BIT(c))
409 #define ANYOF_BITMAP_TEST(p, c) (ANYOF_BITMAP_BYTE(p, c) & ANYOF_BIT(c))
410
f8bef55 memset() is cheaper than a loop of 256 bit-a-a-times
Nicholas Clark authored
411 #define ANYOF_BITMAP_SETALL(p) \
412 memset (ANYOF_BITMAP(p), 255, ANYOF_BITMAP_SIZE)
413 #define ANYOF_BITMAP_CLEARALL(p) \
414 Zero (ANYOF_BITMAP(p), ANYOF_BITMAP_SIZE)
415 /* Check that all 256 bits are all set. Used in S_cl_is_anything() */
416 #define ANYOF_BITMAP_TESTALLSET(p) \
417 memEQ (ANYOF_BITMAP(p), "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377", ANYOF_BITMAP_SIZE)
418
936ed89 Remove the last regnode<==>char*
Ilya Zakharevich authored
419 #define ANYOF_SKIP ((ANYOF_SIZE - 1)/sizeof(regnode))
420 #define ANYOF_CLASS_SKIP ((ANYOF_CLASS_SIZE - 1)/sizeof(regnode))
421 #define ANYOF_CLASS_ADD_SKIP (ANYOF_CLASS_SKIP - ANYOF_SKIP)
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
422
786e8c1 @demerphq Re: [PATCH] Trie jumping
demerphq authored
423
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
424 /*
425 * Utility definitions.
426 */
2304df6 perl 5.0 alpha 8
Andy Dougherty authored
427 #ifndef CHARMASK
075abff @petdance Updated lint infrastructure
petdance authored
428 # define UCHARAT(p) ((int)*(const U8*)(p))
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
429 #else
075abff @petdance Updated lint infrastructure
petdance authored
430 # define UCHARAT(p) ((int)*(p)&CHARMASK)
a687059 @TimToady perl 3.0: (no announcement message available)
TimToady authored
431 #endif
432
c277df4 Jumbo regexp patch applied (with minor fix-up tweaks):
Ilya Zakharevich authored
433 #define EXTRA_SIZE(guy) ((sizeof(guy)-1)/sizeof(struct regnode))
434
3dab1da @demerphq Re: [PATCH] More regex optimisations and debug enhancements (includin…
demerphq authored
435 #define REG_SEEN_ZERO_LEN 0x00000001
436 #define REG_SEEN_LOOKBEHIND 0x00000002
437 #define REG_SEEN_GPOS 0x00000004
438 #define REG_SEEN_EVAL 0x00000008
439 #define REG_SEEN_CANY 0x00000010
cce850e @jhi Continuation of #11575: SANY_SEEN completely deprecated,
jhi authored
440 #define REG_SEEN_SANY REG_SEEN_CANY /* src bckwrd cmpt */
6bda09f @demerphq Re: [PATCH] Add recursive regexes similar to PCRE
demerphq authored
441 #define REG_SEEN_RECURSE 0x00000020
eaf3ca9 @demerphq Re: [perl #8835] fairly large regex optimization bug with 5.7.3
demerphq authored
442 #define REG_TOP_LEVEL_BRANCHES 0x00000040
e2e6a0f @demerphq New regex syntax omnibus
demerphq authored
443 #define REG_SEEN_VERBARG 0x00000080
5d458dd @demerphq Re: [PATCH] New regex syntax omnibus
demerphq authored
444 #define REG_SEEN_CUTGROUP 0x00000100
bcdf740 @demerphq Improve regex stringification code
demerphq authored
445 #define REG_SEEN_RUN_ON_COMMENT 0x00000200
d09b2d2 add patch that generates regnodes.h via regcomp.pl
Ilya Zakharevich authored
446
73c4f7a EXTERN_C declarations for global arrays in various
Gurusamy Sarathy authored
447 START_EXTERN_C
448
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
449 #ifdef PLUGGABLE_RE_EXTENSION
450 #include "re_nodes.h"
451 #else
d09b2d2 add patch that generates regnodes.h via regcomp.pl
Ilya Zakharevich authored
452 #include "regnodes.h"
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
453 #endif
d09b2d2 add patch that generates regnodes.h via regcomp.pl
Ilya Zakharevich authored
454
f9f4320 @demerphq Re: [PATCH] Add hook for re_dup() into regex engine as reg_dupe (make…
demerphq authored
455 #ifndef PLUGGABLE_RE_EXTENSION
456 #ifndef DOINIT
457 EXTCONST regexp_engine PL_core_reg_engine;
458 #else /* DOINIT */
459 EXTCONST regexp_engine PL_core_reg_engine = {
fe578d7 Re: Proposed changes and to regular expression interfaces in core
Ævar Arnfjörð Bjarmason authored
460 Perl_re_compile,
2fdbfb4 FETCH/STORE/LENGTH callbacks for numbered capture variables
Ævar Arnfjörð Bjarmason authored
461 Perl_regexec_flags,
f9f4320 @demerphq Re: [PATCH] Add hook for re_dup() into regex engine as reg_dupe (make…
demerphq authored
462 Perl_re_intuit_start,
463 Perl_re_intuit_string,
2fdbfb4 FETCH/STORE/LENGTH callbacks for numbered capture variables
Ævar Arnfjörð Bjarmason authored
464 Perl_regfree_internal,
465 Perl_reg_numbered_buff_fetch,
466 Perl_reg_numbered_buff_store,
467 Perl_reg_numbered_buff_length,
192b9cd Re: [PATCH] Callbacks for named captures (%+ and %-)
Ævar Arnfjörð Bjarmason authored
468 Perl_reg_named_buff,
469 Perl_reg_named_buff_iter,
49d7dfb Re: [PATCH] Cleanup of the regexp API
Ævar Arnfjörð Bjarmason authored
470 Perl_reg_qr_package,
f9f4320 @demerphq Re: [PATCH] Add hook for re_dup() into regex engine as reg_dupe (make…
demerphq authored
471 #if defined(USE_ITHREADS)
f814945 @demerphq Continue split of perl internal regexp structures from ones that are …
demerphq authored
472 Perl_regdupe_internal
f9f4320 @demerphq Re: [PATCH] Add hook for re_dup() into regex engine as reg_dupe (make…
demerphq authored
473 #endif
474 };
475 #endif /* DOINIT */
476 #endif /* PLUGGABLE_RE_EXTENSION */
477
478
73c4f7a EXTERN_C declarations for global arrays in various
Gurusamy Sarathy authored
479 END_EXTERN_C
cad2e5a @jhi Integrate with Sarathy.
jhi authored
480
481
261faec @mjdominus Patch: document reg_data.what member
mjdominus authored
482 /* .what is a character array with one character for each member of .data
483 * The character describes the function of the corresponding .data item:
af534a0 Properly free paren_name_list with its regexp.
Nicholas Clark authored
484 * a - AV for paren_name_list under DEBUGGING
0111c4f @rgs Rename ${^RE_TRIE_MAXBUFF} to ${^RE_TRIE_MAXBUF},
rgs authored
485 * f - start-class data for regstclass optimization
261faec @mjdominus Patch: document reg_data.what member
mjdominus authored
486 * n - Root of op tree for (?{EVAL}) item
487 * o - Start op for (?{EVAL}) item
3dab1da @demerphq Re: [PATCH] More regex optimisations and debug enhancements (includin…
demerphq authored
488 * p - Pad for (?{EVAL}) item
38a44b8 s/\bunicode\b/Unicode/; # For everything not dual life
Nicholas Clark authored
489 * s - swash for Unicode-style character class, and the multicharacter
9e55ce0 @jhi Finish up (ha!) the Unicode case folding;
jhi authored
490 * strings resulting from casefolding the single-character entries
491 * in the character class
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
492 * t - trie struct
55eed65 Move widecharmap out of the shared structure _reg_trie_data into the
Nicholas Clark authored
493 * u - trie struct's widecharmap (a HV, so can't share, must dup)
2b8b478 Move words and revcharmap out of struct _rev_trie_data and duplicate
Nicholas Clark authored
494 * also used for revcharmap and words under DEBUGGING
07be1b8 @demerphq Re: [PATCH] Better version of the Aho-Corasick patch and lots of benc…
demerphq authored
495 * T - aho-trie struct
81714fb @demerphq Re: [PATCH] Initial attempt at named captures for perls regexp engine
demerphq authored
496 * S - sv for named capture lookup
261faec @mjdominus Patch: document reg_data.what member
mjdominus authored
497 * 20010712 mjd@plover.com
c3613ed adds a comment
Abhijit Menon-Sen authored
498 * (Remember to update re_dup() and pregfree() if you add any items.)
261faec @mjdominus Patch: document reg_data.what member
mjdominus authored
499 */
cad2e5a @jhi Integrate with Sarathy.
jhi authored
500 struct reg_data {
501 U32 count;
502 U8 *what;
503 void* data[1];
504 };
505
a1cac82 Remove code duplication in S_to_utf8_substr() and S_to_byte_substr()
Nicholas Clark authored
506 /* Code in S_to_utf8_substr() and S_to_byte_substr() in regexec.c accesses
507 anchored* and float* via array indexes 0 and 1. */
cad2e5a @jhi Integrate with Sarathy.
jhi authored
508 #define anchored_substr substrs->data[0].substr
33b8afd @jhi Fix for "UTF-8 bug with s///" from Hugo.
jhi authored
509 #define anchored_utf8 substrs->data[0].utf8_substr
cad2e5a @jhi Integrate with Sarathy.
jhi authored
510 #define anchored_offset substrs->data[0].min_offset
1de0632 @demerphq Teach regex optimiser how to handle (?=) and (?<=) properly.
demerphq authored
511 #define anchored_end_shift substrs->data[0].end_shift
512
cad2e5a @jhi Integrate with Sarathy.
jhi authored
513 #define float_substr substrs->data[1].substr
33b8afd @jhi Fix for "UTF-8 bug with s///" from Hugo.
jhi authored
514 #define float_utf8 substrs->data[1].utf8_substr
cad2e5a @jhi Integrate with Sarathy.
jhi authored
515 #define float_min_offset substrs->data[1].min_offset
516 #define float_max_offset substrs->data[1].max_offset
1de0632 @demerphq Teach regex optimiser how to handle (?=) and (?<=) properly.
demerphq authored
517 #define float_end_shift substrs->data[1].end_shift
518
cad2e5a @jhi Integrate with Sarathy.
jhi authored
519 #define check_substr substrs->data[2].substr
33b8afd @jhi Fix for "UTF-8 bug with s///" from Hugo.
jhi authored
520 #define check_utf8 substrs->data[2].utf8_substr
cad2e5a @jhi Integrate with Sarathy.
jhi authored
521 #define check_offset_min substrs->data[2].min_offset
522 #define check_offset_max substrs->data[2].max_offset
1de0632 @demerphq Teach regex optimiser how to handle (?=) and (?<=) properly.
demerphq authored
523 #define check_end_shift substrs->data[2].end_shift
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
524
288b8c0 Make struct regexp the body of SVt_REGEXP SVs, REGEXPs become SVs,
Nicholas Clark authored
525 #define RX_ANCHORED_SUBSTR(rx) (((struct regexp *)SvANY(rx))->anchored_substr)
526 #define RX_ANCHORED_UTF8(rx) (((struct regexp *)SvANY(rx))->anchored_utf8)
527 #define RX_FLOAT_SUBSTR(rx) (((struct regexp *)SvANY(rx))->float_substr)
528 #define RX_FLOAT_UTF8(rx) (((struct regexp *)SvANY(rx))->float_utf8)
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
529
530 /* trie related stuff */
5d9a96c use slabs to allocate save state space in regmatch()
Dave Mitchell authored
531
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
532 /* a transition record for the state machine. the
533 check field determines which state "owns" the
534 transition. the char the transition is for is
535 determined by offset from the owning states base
536 field. the next field determines which state
537 is to be transitioned to if any.
538 */
539 struct _reg_trie_trans {
540 U32 next;
541 U32 check;
542 };
543
544 /* a transition list element for the list based representation */
545 struct _reg_trie_trans_list_elem {
546 U16 forid;
547 U32 newstate;
548 };
549 typedef struct _reg_trie_trans_list_elem reg_trie_trans_le;
550
551 /* a state for compressed nodes. base is an offset
552 into an array of reg_trie_trans array. If wordnum is
553 nonzero the state is accepting. if base is zero then
554 the state has no children (and will be accepting)
555 */
556 struct _reg_trie_state {
557 U16 wordnum;
558 union {
559 U32 base;
560 reg_trie_trans_le* list;
561 } trans;
562 };
563
2e64971 @iabyn tries: don't allocate memory at runtime
iabyn authored
564 /* info per word; indexed by wordnum */
565 typedef struct {
566 U16 prev; /* previous word in acceptance chain; eg in
567 * zzz|abc|ab/ after matching the chars abc, the
568 * accepted word is #2, and the previous accepted
569 * word is #3 */
570 U32 len; /* how many chars long is this word? */
571 U32 accept; /* accept state for this word */
572 } reg_trie_wordinfo;
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
573
574
575 typedef struct _reg_trie_state reg_trie_state;
576 typedef struct _reg_trie_trans reg_trie_trans;
577
578
579 /* anything in here that needs to be freed later
23eab42 Moving the reference count to the front of both _reg_trie_data and
Nicholas Clark authored
580 should be dealt with in pregfree.
581 refcount is first in both this and _reg_ac_data to allow a space
582 optimisation in Perl_regdupe. */
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
583 struct _reg_trie_data {
23eab42 Moving the reference count to the front of both _reg_trie_data and
Nicholas Clark authored
584 U32 refcount; /* number of times this trie is referenced */
786e8c1 @demerphq Re: [PATCH] Trie jumping
demerphq authored
585 U32 lasttrans; /* last valid transition element */
586 U16 *charmap; /* byte to charid lookup array */
587 reg_trie_state *states; /* state data */
588 reg_trie_trans *trans; /* array of transition elements */
589 char *bitmap; /* stclass bitmap */
590 U16 *jump; /* optional 1 indexed array of offsets before tail
591 for the node following a given word. */
2e64971 @iabyn tries: don't allocate memory at runtime
iabyn authored
592 reg_trie_wordinfo *wordinfo; /* array of info per word */
1b5c067 Reorder the members of various regexp structs to reduce their size on
Nicholas Clark authored
593 U16 uniquecharcount; /* unique chars in trie (width of trans table) */
594 U32 startstate; /* initial state - used for common prefix optimisation */
595 STRLEN minlen; /* minimum length of words in trie - build/opt only? */
596 STRLEN maxlen; /* maximum length of words in trie - build/opt only? */
2e64971 @iabyn tries: don't allocate memory at runtime
iabyn authored
597 U32 prefixlen; /* #chars in common prefix */
1e2e3d0 @demerphq Re: Off by one in the trie code?
demerphq authored
598 U32 statecount; /* Build only - number of states in the states array
599 (including the unused zero state) */
786e8c1 @demerphq Re: [PATCH] Trie jumping
demerphq authored
600 U32 wordcount; /* Build only */
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
601 #ifdef DEBUGGING
786e8c1 @demerphq Re: [PATCH] Trie jumping
demerphq authored
602 STRLEN charcount; /* Build only */
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
603 #endif
604 };
2b8b478 Move words and revcharmap out of struct _rev_trie_data and duplicate
Nicholas Clark authored
605 /* There is one (3 under DEBUGGING) pointers that logically belong in this
606 structure, but are held outside as they need duplication on thread cloning,
607 whereas the rest of the structure can be read only:
608 HV *widecharmap; code points > 255 to charid
609 #ifdef DEBUGGING
610 AV *words; Array of words contained in trie, for dumping
611 AV *revcharmap; Map of each charid back to its character representation
612 #endif
613 */
614
615 #define TRIE_WORDS_OFFSET 2
616
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
617 typedef struct _reg_trie_data reg_trie_data;
618
23eab42 Moving the reference count to the front of both _reg_trie_data and
Nicholas Clark authored
619 /* refcount is first in both this and _reg_trie_data to allow a space
620 optimisation in Perl_regdupe. */
07be1b8 @demerphq Re: [PATCH] Better version of the Aho-Corasick patch and lots of benc…
demerphq authored
621 struct _reg_ac_data {
23eab42 Moving the reference count to the front of both _reg_trie_data and
Nicholas Clark authored
622 U32 refcount;
1b5c067 Reorder the members of various regexp structs to reduce their size on
Nicholas Clark authored
623 U32 trie;
07be1b8 @demerphq Re: [PATCH] Better version of the Aho-Corasick patch and lots of benc…
demerphq authored
624 U32 *fail;
625 reg_trie_state *states;
626 };
627 typedef struct _reg_ac_data reg_ac_data;
628
3dab1da @demerphq Re: [PATCH] More regex optimisations and debug enhancements (includin…
demerphq authored
629 /* ANY_BIT doesnt use the structure, so we can borrow it here.
630 This is simpler than refactoring all of it as wed end up with
631 three different sets... */
632
633 #define TRIE_BITMAP(p) (((reg_trie_data *)(p))->bitmap)
786e8c1 @demerphq Re: [PATCH] Trie jumping
demerphq authored
634 #define TRIE_BITMAP_BYTE(p, c) (TRIE_BITMAP(p)[(((U8)(c)) >> 3) & 31])
5dfe062 @demerphq Comply with the 0x80th commandment
demerphq authored
635 #define TRIE_BITMAP_SET(p, c) (TRIE_BITMAP_BYTE(p, c) |= ANYOF_BIT((U8)c))
636 #define TRIE_BITMAP_CLEAR(p,c) (TRIE_BITMAP_BYTE(p, c) &= ~ANYOF_BIT((U8)c))
637 #define TRIE_BITMAP_TEST(p, c) (TRIE_BITMAP_BYTE(p, c) & ANYOF_BIT((U8)c))
3dab1da @demerphq Re: [PATCH] More regex optimisations and debug enhancements (includin…
demerphq authored
638
1de0632 @demerphq Teach regex optimiser how to handle (?=) and (?<=) properly.
demerphq authored
639 #define IS_ANYOF_TRIE(op) ((op)==TRIEC || (op)==AHOCORASICKC)
640 #define IS_TRIE_AC(op) ((op)>=AHOCORASICK)
641
642
786e8c1 @demerphq Re: [PATCH] Trie jumping
demerphq authored
643 #define BITMAP_BYTE(p, c) (((U8*)p)[(((U8)(c)) >> 3) & 31])
644 #define BITMAP_TEST(p, c) (BITMAP_BYTE(p, c) & ANYOF_BIT((U8)c))
3dab1da @demerphq Re: [PATCH] More regex optimisations and debug enhancements (includin…
demerphq authored
645
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
646 /* these defines assume uniquecharcount is the correct variable, and state may be evaluated twice */
647 #define TRIE_NODENUM(state) (((state)-1)/(trie->uniquecharcount)+1)
648 #define SAFE_TRIE_NODENUM(state) ((state) ? (((state)-1)/(trie->uniquecharcount)+1) : (state))
649 #define TRIE_NODEIDX(state) ((state) ? (((state)-1)*(trie->uniquecharcount)+1) : (state))
650
3dab1da @demerphq Re: [PATCH] More regex optimisations and debug enhancements (includin…
demerphq authored
651 #ifdef DEBUGGING
652 #define TRIE_CHARCOUNT(trie) ((trie)->charcount)
653 #else
654 #define TRIE_CHARCOUNT(trie) (trie_charcount)
655 #endif
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
656
0111c4f @rgs Rename ${^RE_TRIE_MAXBUFF} to ${^RE_TRIE_MAXBUF},
rgs authored
657 #define RE_TRIE_MAXBUF_INIT 65536
658 #define RE_TRIE_MAXBUF_NAME "\022E_TRIE_MAXBUF"
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
659 #define RE_DEBUG_FLAGS "\022E_DEBUG_FLAGS"
660
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
661 /*
662
663 RE_DEBUG_FLAGS is used to control what debug output is emitted
664 its divided into three groups of options, some of which interact.
665 The three groups are: Compile, Execute, Extra. There is room for a
666 further group, as currently only the low three bytes are used.
667
668 Compile Options:
07be1b8 @demerphq Re: [PATCH] Better version of the Aho-Corasick patch and lots of benc…
demerphq authored
669
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
670 PARSE
671 PEEP
672 TRIE
673 PROGRAM
674 OFFSETS
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
675
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
676 Execute Options:
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
677
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
678 INTUIT
679 MATCH
680 TRIE
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
681
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
682 Extra Options
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
683
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
684 TRIE
685 OFFSETS
686
687 If you modify any of these make sure you make corresponding changes to
688 re.pm, especially to the documentation.
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
689
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
690 */
691
692
693 /* Compile */
694 #define RE_DEBUG_COMPILE_MASK 0x0000FF
695 #define RE_DEBUG_COMPILE_PARSE 0x000001
696 #define RE_DEBUG_COMPILE_OPTIMISE 0x000002
697 #define RE_DEBUG_COMPILE_TRIE 0x000004
698 #define RE_DEBUG_COMPILE_DUMP 0x000008
f7819f8 @Abigail /p vs (?p)
Abigail authored
699 #define RE_DEBUG_COMPILE_FLAGS 0x000010
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
700
701 /* Execute */
702 #define RE_DEBUG_EXECUTE_MASK 0x00FF00
703 #define RE_DEBUG_EXECUTE_INTUIT 0x000100
704 #define RE_DEBUG_EXECUTE_MATCH 0x000200
705 #define RE_DEBUG_EXECUTE_TRIE 0x000400
706
707 /* Extra */
708 #define RE_DEBUG_EXTRA_MASK 0xFF0000
709 #define RE_DEBUG_EXTRA_TRIE 0x010000
710 #define RE_DEBUG_EXTRA_OFFSETS 0x020000
a5ca303 @demerphq Minor re 'Debug' tweaks, also fix a bug in dumping certain patterns.
demerphq authored
711 #define RE_DEBUG_EXTRA_OFFDEBUG 0x040000
712 #define RE_DEBUG_EXTRA_STATE 0x080000
713 #define RE_DEBUG_EXTRA_OPTIMISE 0x100000
e770707 @demerphq Re: New file: t/op/regexp_email.t
demerphq authored
714 #define RE_DEBUG_EXTRA_BUFFERS 0x400000
2c29696 @demerphq add more positive gofs GPOS tests and fix some bugs too
demerphq authored
715 #define RE_DEBUG_EXTRA_GPOS 0x800000
24b23f3 @demerphq Add more backtracking control verbs to regex engine (?CUT), (?ERROR)
demerphq authored
716 /* combined */
717 #define RE_DEBUG_EXTRA_STACK 0x280000
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
718
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
719 #define RE_DEBUG_FLAG(x) (re_debug_flags & x)
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
720 /* Compile */
721 #define DEBUG_COMPILE_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
722 if (re_debug_flags & RE_DEBUG_COMPILE_MASK) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
723 #define DEBUG_PARSE_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
724 if (re_debug_flags & RE_DEBUG_COMPILE_PARSE) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
725 #define DEBUG_OPTIMISE_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
726 if (re_debug_flags & RE_DEBUG_COMPILE_OPTIMISE) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
727 #define DEBUG_PARSE_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
728 if (re_debug_flags & RE_DEBUG_COMPILE_PARSE) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
729 #define DEBUG_DUMP_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
730 if (re_debug_flags & RE_DEBUG_COMPILE_DUMP) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
731 #define DEBUG_TRIE_COMPILE_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
732 if (re_debug_flags & RE_DEBUG_COMPILE_TRIE) x )
f7819f8 @Abigail /p vs (?p)
Abigail authored
733 #define DEBUG_FLAGS_r(x) DEBUG_r( \
734 if (re_debug_flags & RE_DEBUG_COMPILE_FLAGS) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
735 /* Execute */
736 #define DEBUG_EXECUTE_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
737 if (re_debug_flags & RE_DEBUG_EXECUTE_MASK) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
738 #define DEBUG_INTUIT_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
739 if (re_debug_flags & RE_DEBUG_EXECUTE_INTUIT) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
740 #define DEBUG_MATCH_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
741 if (re_debug_flags & RE_DEBUG_EXECUTE_MATCH) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
742 #define DEBUG_TRIE_EXECUTE_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
743 if (re_debug_flags & RE_DEBUG_EXECUTE_TRIE) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
744
745 /* Extra */
746 #define DEBUG_EXTRA_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
747 if (re_debug_flags & RE_DEBUG_EXTRA_MASK) x )
a5ca303 @demerphq Minor re 'Debug' tweaks, also fix a bug in dumping certain patterns.
demerphq authored
748 #define DEBUG_OFFSETS_r(x) DEBUG_r( \
749 if (re_debug_flags & RE_DEBUG_EXTRA_OFFSETS) x )
ab3bbde @demerphq Updated escaping code. utf8 regex debug output improvements
demerphq authored
750 #define DEBUG_STATE_r(x) DEBUG_r( \
751 if (re_debug_flags & RE_DEBUG_EXTRA_STATE) x )
24b23f3 @demerphq Add more backtracking control verbs to regex engine (?CUT), (?ERROR)
demerphq authored
752 #define DEBUG_STACK_r(x) DEBUG_r( \
753 if (re_debug_flags & RE_DEBUG_EXTRA_STACK) x )
e770707 @demerphq Re: New file: t/op/regexp_email.t
demerphq authored
754 #define DEBUG_BUFFERS_r(x) DEBUG_r( \
755 if (re_debug_flags & RE_DEBUG_EXTRA_BUFFERS) x )
756
a5ca303 @demerphq Minor re 'Debug' tweaks, also fix a bug in dumping certain patterns.
demerphq authored
757 #define DEBUG_OPTIMISE_MORE_r(x) DEBUG_r( \
758 if ((RE_DEBUG_EXTRA_OPTIMISE|RE_DEBUG_COMPILE_OPTIMISE) == \
759 (re_debug_flags & (RE_DEBUG_EXTRA_OPTIMISE|RE_DEBUG_COMPILE_OPTIMISE)) ) x )
07be1b8 @demerphq Re: [PATCH] Better version of the Aho-Corasick patch and lots of benc…
demerphq authored
760 #define MJD_OFFSET_DEBUG(x) DEBUG_r( \
a5ca303 @demerphq Minor re 'Debug' tweaks, also fix a bug in dumping certain patterns.
demerphq authored
761 if (re_debug_flags & RE_DEBUG_EXTRA_OFFDEBUG) \
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
762 Perl_warn_nocontext x )
763 #define DEBUG_TRIE_COMPILE_MORE_r(x) DEBUG_TRIE_COMPILE_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
764 if (re_debug_flags & RE_DEBUG_EXTRA_TRIE) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
765 #define DEBUG_TRIE_EXECUTE_MORE_r(x) DEBUG_TRIE_EXECUTE_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
766 if (re_debug_flags & RE_DEBUG_EXTRA_TRIE) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
767
768 #define DEBUG_TRIE_r(x) DEBUG_r( \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
769 if (re_debug_flags & (RE_DEBUG_COMPILE_TRIE \
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
770 | RE_DEBUG_EXECUTE_TRIE )) x )
2c29696 @demerphq add more positive gofs GPOS tests and fix some bugs too
demerphq authored
771 #define DEBUG_GPOS_r(x) DEBUG_r( \
772 if (re_debug_flags & RE_DEBUG_EXTRA_GPOS) x )
be8e71a @demerphq Re: [PATCH] regexec/regcomp.c cleanups
demerphq authored
773
774 /* initialization */
595598e PATCH: regex longjmp flaws
Karl Williamson authored
775 /* get_sv() can return NULL during global destruction. re_debug_flags can get
776 * clobbered by a longjmp, so must be initialized */
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
777 #define GET_RE_DEBUG_FLAGS DEBUG_r({ \
778 SV * re_debug_flags_sv = NULL; \
595598e PATCH: regex longjmp flaws
Karl Williamson authored
779 re_debug_flags = 0; \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
780 re_debug_flags_sv = get_sv(RE_DEBUG_FLAGS, 1); \
781 if (re_debug_flags_sv) { \
8e9a8a4 @demerphq Re: [PATCH] fix re debug segvs in global destruction, and a tweak to …
demerphq authored
782 if (!SvIOK(re_debug_flags_sv)) \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
783 sv_setuv(re_debug_flags_sv, RE_DEBUG_COMPILE_DUMP | RE_DEBUG_EXECUTE_MASK ); \
8e9a8a4 @demerphq Re: [PATCH] fix re debug segvs in global destruction, and a tweak to …
demerphq authored
784 re_debug_flags=SvIV(re_debug_flags_sv); \
e68ec53 @demerphq fix re debug segvs in global destruction, and a tweak to Benchmark to…
demerphq authored
785 }\
786 })
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
787
788 #ifdef DEBUGGING
ab3bbde @demerphq Updated escaping code. utf8 regex debug output improvements
demerphq authored
789
d53d27f regcomp.c: Get rid of compiler warning.
Karl Williamson authored
790 #define GET_RE_DEBUG_FLAGS_DECL VOL IV re_debug_flags = 0; GET_RE_DEBUG_FLAGS;
ab3bbde @demerphq Updated escaping code. utf8 regex debug output improvements
demerphq authored
791
792 #define RE_PV_COLOR_DECL(rpv,rlen,isuni,dsv,pv,l,m,c1,c2) \
793 const char * const rpv = \
ddc5bc0 @demerphq Re: Fix loads of warnings from the last escaping patch...
demerphq authored
794 pv_pretty((dsv), (pv), (l), (m), \
ab3bbde @demerphq Updated escaping code. utf8 regex debug output improvements
demerphq authored
795 PL_colors[(c1)],PL_colors[(c2)], \
0a0e9af @demerphq Re: [PATCH] Change implementation of %+ to use a proper tied hash int…
demerphq authored
796 PERL_PV_ESCAPE_RE |((isuni) ? PERL_PV_ESCAPE_UNI : 0) ); \
0df25f3 @demerphq More escaping in the RE
demerphq authored
797 const int rlen = SvCUR(dsv)
ab3bbde @demerphq Updated escaping code. utf8 regex debug output improvements
demerphq authored
798
799 #define RE_SV_ESCAPE(rpv,isuni,dsv,sv,m) \
800 const char * const rpv = \
ddc5bc0 @demerphq Re: Fix loads of warnings from the last escaping patch...
demerphq authored
801 pv_pretty((dsv), (SvPV_nolen_const(sv)), (SvCUR(sv)), (m), \
ab3bbde @demerphq Updated escaping code. utf8 regex debug output improvements
demerphq authored
802 PL_colors[(c1)],PL_colors[(c2)], \
0a0e9af @demerphq Re: [PATCH] Change implementation of %+ to use a proper tied hash int…
demerphq authored
803 PERL_PV_ESCAPE_RE |((isuni) ? PERL_PV_ESCAPE_UNI : 0) )
ab3bbde @demerphq Updated escaping code. utf8 regex debug output improvements
demerphq authored
804
805 #define RE_PV_QUOTED_DECL(rpv,isuni,dsv,pv,l,m) \
806 const char * const rpv = \
ddc5bc0 @demerphq Re: Fix loads of warnings from the last escaping patch...
demerphq authored
807 pv_pretty((dsv), (pv), (l), (m), \
ab3bbde @demerphq Updated escaping code. utf8 regex debug output improvements
demerphq authored
808 PL_colors[0], PL_colors[1], \
95b611b @rgs API spelling patch, by Jerry D. Hedden
rgs authored
809 ( PERL_PV_PRETTY_QUOTE | PERL_PV_ESCAPE_RE | PERL_PV_PRETTY_ELLIPSES | \
ab3bbde @demerphq Updated escaping code. utf8 regex debug output improvements
demerphq authored
810 ((isuni) ? PERL_PV_ESCAPE_UNI : 0)) \
0a0e9af @demerphq Re: [PATCH] Change implementation of %+ to use a proper tied hash int…
demerphq authored
811 )
ab3bbde @demerphq Updated escaping code. utf8 regex debug output improvements
demerphq authored
812
813 #define RE_SV_DUMPLEN(ItEm) (SvCUR(ItEm) - (SvTAIL(ItEm)!=0))
814 #define RE_SV_TAIL(ItEm) (SvTAIL(ItEm) ? "$" : "")
815
816 #else /* if not DEBUGGING */
817
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
818 #define GET_RE_DEBUG_FLAGS_DECL
ab3bbde @demerphq Updated escaping code. utf8 regex debug output improvements
demerphq authored
819 #define RE_PV_COLOR_DECL(rpv,rlen,isuni,dsv,pv,l,m,c1,c2)
820 #define RE_SV_ESCAPE(rpv,isuni,dsv,sv,m)
821 #define RE_PV_QUOTED_DECL(rpv,isuni,dsv,pv,l,m)
822 #define RE_SV_DUMPLEN(ItEm)
823 #define RE_SV_TAIL(ItEm)
824
825 #endif /* DEBUG RELATED DEFINES */
a3621e7 @demerphq Re: Reworked Trie Patch
demerphq authored
826
e9a8c09 Add editor blocks to some header files.
Marcus Holland-Moritz authored
827 /*
828 * Local variables:
829 * c-indentation-style: bsd
830 * c-basic-offset: 4
831 * indent-tabs-mode: t
832 * End:
833 *
834 * ex: set ts=8 sts=4 sw=4 noet:
835 */
Something went wrong with that request. Please try again.