/
regex.d
7642 lines (7239 loc) · 271 KB
/
regex.d
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
//Written in the D programming language
/++
$(LUCKY Regular expressions) are a commonly used method of pattern matching
on strings, with $(I regex) being a catchy word for a pattern in this domain
specific language. Typical problems usually solved by regular expressions
include validation of user input and the ubiquitous find & replace
in text processing utilities.
Synposis:
---
import std.regex;
import std.stdio;
void main()
{
// Print out all possible dd/mm/yy(yy) dates found in user input.
// g - global: find all matches.
auto r = regex(r"\b[0-9][0-9]?/[0-9][0-9]?/[0-9][0-9](?:[0-9][0-9])?\b", "g");
foreach(line; stdin.byLine)
{
// Match returns a range that can be iterated
// to get all subsequent matches.
foreach(c; match(line, r))
writeln(c.hit);
}
}
...
// Create a static regex at compile-time, which contains fast native code.
enum ctr = ctRegex!(`^.*/([^/]+)/?$`);
// It works just like a normal regex:
auto m2 = match("foo/bar", ctr); // First match found here, if any
assert(m2); // Be sure to check if there is a match before examining contents!
assert(m2.captures[1] == "bar"); // Captures is a range of submatches: 0 = full match.
...
// The result of the match is directly testable with if/assert/while.
// e.g. test if a string consists of letters:
assert(match("Letter", `^\p{L}+$`));
---
The general usage guideline is to keep regex complexity on the side of simplicity,
as its capabilities reside in purely character-level manipulation,
and as such are ill-suited for tasks involving higher level invariants
like matching an integer number $(U bounded) in an [a,b] interval.
Checks of this sort of are better addressed by additional post-processing.
The basic syntax shouldn't surprise experienced users of regular expressions.
Thankfully, nowadays the web is bustling with resources to help newcomers, and a good
$(WEB www.regular-expressions.info, reference with tutorial) on regular expressions
can be found.
This library uses an ECMAScript syntax flavor with the following extensions:
$(UL
$(LI Named subexpressions, with Python syntax. )
$(LI Unicode properties such as Scripts, Blocks and common binary properties e.g Alphabetic, White_Space, Hex_Digit etc.)
$(LI Arbitrary length and complexity lookbehind, including lookahead in lookbehind and vise-versa.)
)
$(REG_START Pattern syntax )
$(I std.regex operates on codepoint level,
'character' in this table denotes a single unicode codepoint.)
$(REG_TABLE
$(REG_TITLE Pattern element, Semantics )
$(REG_TITLE Atoms, Match single characters )
$(REG_ROW any character except [{|*+?()^$, Matches the character itself. )
$(REG_ROW ., In single line mode matches any charcter.
Otherwise it matches any character except '\n' and '\r'. )
$(REG_ROW [class], Matches a single character
that belongs to this character class. )
$(REG_ROW [^class], Matches a single character that
does $(U not) belong to this character class.)
$(REG_ROW \cC, Matches the control character corresponding to letter C)
$(REG_ROW \xXX, Matches a character with hexadecimal value of XX. )
$(REG_ROW \uXXXX, Matches a character with hexadecimal value of XXXX. )
$(REG_ROW \U00YYYYYY, Matches a character with hexadecimal value of YYYYYY. )
$(REG_ROW \f, Matches a formfeed character. )
$(REG_ROW \n, Matches a linefeed character. )
$(REG_ROW \r, Matches a carriage return character. )
$(REG_ROW \t, Matches a tab character. )
$(REG_ROW \v, Matches a vertical tab character. )
$(REG_ROW \d, Matches any unicode digit. )
$(REG_ROW \D, Matches any character except unicode digits. )
$(REG_ROW \w, Matches any word character (note: this includes numbers).)
$(REG_ROW \W, Matches any non-word character.)
$(REG_ROW \s, Matches whitespace, same as \p{White_Space}.)
$(REG_ROW \S, Matches any character except those recognized as $(I \s ). )
$(REG_ROW \\, Matches \ character. )
$(REG_ROW \c where c is one of [|*+?(), Matches the character c itself. )
$(REG_ROW \p{PropertyName}, Matches a character that belongs
to the unicode PropertyName set.
Single letter abbreviations can be used without surrounding {,}. )
$(REG_ROW \P{PropertyName}, Matches a character that does not belong
to the unicode PropertyName set.
Single letter abbreviations can be used without surrounding {,}. )
$(REG_ROW \p{InBasicLatin}, Matches any character that is part of
the BasicLatin unicode $(U block).)
$(REG_ROW \P{InBasicLatin}, Matches any character except ones in
the BasicLatin unicode $(U block).)
$(REG_ROW \p{Cyrilic}, Matches any character that is part of
Cyrilic $(U script).)
$(REG_ROW \P{Cyrilic}, Matches any character except ones in
Cyrilic $(U script).)
$(REG_TITLE Quantifiers, Specify repetition of other elements)
$(REG_ROW *, Matches previous character/subexpression 0 or more times.
Greedy version - tries as many times as possible.)
$(REG_ROW *?, Matches previous character/subexpression 0 or more times.
Lazy version - stops as early as possible.)
$(REG_ROW +, Matches previous character/subexpression 1 or more times.
Greedy version - tries as many times as possible.)
$(REG_ROW +?, Matches previous character/subexpression 1 or more times.
Lazy version - stops as early as possible.)
$(REG_ROW {n}, Matches previous character/subexpression exactly n times. )
$(REG_ROW {n,}, Matches previous character/subexpression n times or more.
Greedy version - tries as many times as possible. )
$(REG_ROW {n,}?, Matches previous character/subexpression n times or more.
Lazy version - stops as early as possible.)
$(REG_ROW {n,m}, Matches previous character/subexpression n to m times.
Greedy version - tries as many times as possible, but no more than m times. )
$(REG_ROW {n,m}?, Matches previous character/subexpression n to m times.
Lazy version - stops as early as possible, but no less then n times.)
$(REG_TITLE Other, Subexpressions & alternations )
$(REG_ROW (regex), Matches subexpression regex,
saving matched portion of text for later retrieval. )
$(REG_ROW (?:regex), Matches subexpression regex,
$(U not) saving matched portion of text. Useful to speed up matching. )
$(REG_ROW A|B, Matches subexpression A, or failing that, matches B. )
$(REG_ROW (?P<name>regex), Matches named subexpression
regex labeling it with name 'name'.
When referring to a matched portion of text,
names work like aliases in addition to direct numbers.
)
$(REG_TITLE Assertions, Match position rather than character )
$(REG_ROW ^, Matches at the begining of input or line (in multiline mode).)
$(REG_ROW $, Matches at the end of input or line (in multiline mode). )
$(REG_ROW \b, Matches at word boundary. )
$(REG_ROW \B, Matches when $(U not) at word boundary. )
$(REG_ROW (?=regex), Zero-width lookahead assertion.
Matches at a point where the subexpression
regex could be matched starting from the current position.
)
$(REG_ROW (?!regex), Zero-width negative lookahead assertion.
Matches at a point where the subexpression
regex could $(U not) be matched starting from the current position.
)
$(REG_ROW (?<=regex), Zero-width lookbehind assertion. Matches at a point
where the subexpression regex could be matched ending
at the current position (matching goes backwards).
)
$(REG_ROW (?<!regex), Zero-width negative lookbehind assertion.
Matches at a point where the subexpression regex could $(U not)
be matched ending at the current position (matching goes backwards).
)
)
$(REG_START Character classes )
$(REG_TABLE
$(REG_TITLE Pattern element, Semantics )
$(REG_ROW Any atom, Has the same meaning as outside of a character class.)
$(REG_ROW a-z, Includes characters a, b, c, ..., z. )
$(REG_ROW [a||b], [a--b], [a~~b], [a&&b], Where a, b are arbitrary classes,
means union, set difference, symmetric set difference, and intersection respectively.
$(I Any sequence of character class elements implicitly forms a union.) )
)
$(REG_START Regex flags )
$(REG_TABLE
$(REG_TITLE Flag, Semantics )
$(REG_ROW g, Global regex, repeat over the whole input. )
$(REG_ROW i, Case insensitive matching. )
$(REG_ROW m, Multi-line mode, match ^, $ on start and end line separators
as well as start and end of input.)
$(REG_ROW s, Single-line mode, makes . match '\n' and '\r' as well. )
$(REG_ROW x, Free-form syntax, ignores whitespace in pattern,
useful for formatting complex regular expressions. )
)
$(B Unicode support)
This library provides full Level 1 support* according to
$(WEB unicode.org/reports/tr18/, UTS 18). Specifically:
$(UL
$(LI 1.1 Hex notation via any of \uxxxx, \U00YYYYYY, \xZZ.)
$(LI 1.2 Unicode properties.)
$(LI 1.3 Character classes with set operations.)
$(LI 1.4 Word boundaries use the full set of "word" characters.)
$(LI 1.5 Using simple casefolding to match case
insensitively across the full range of codepoints.)
$(LI 1.6 Respecting line breaks as any of
\u000A | \u000B | \u000C | \u000D | \u0085 | \u2028 | \u2029 | \u000D\u000A.)
$(LI 1.7 Operating on codepoint level.)
)
*With exception of point 1.1.1, as of yet, normalization of input
is expected to be enforced by user.
$(B Slicing)
All matches returned by pattern matching functionality in this library
are slices of the original input, with the notable exception of the $(D replace)
family of functions which generate a new string from the input.
Copyright: Copyright Dmitry Olshansky, 2011
License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0).
Authors: Dmitry Olshansky,
API and utility constructs are based on original $(D std.regex)
by Walter Bright and Andrei Alexandrescu.
Source: $(PHOBOSSRC std/_regex.d)
Macros:
REG_ROW = $(TR $(TD $(I $1 )) $(TD $+) )
REG_TITLE = $(TR $(TD $(B $1)) $(TD $(B $2)) )
REG_TABLE = <table border="1" cellspacing="0" cellpadding="5" > $0 </table>
REG_START = <h3><div align="center"> $0 </div></h3>
+/
module std.regex;
import std.internal.uni, std.internal.uni_tab;//unicode property tables
import std.array, std.algorithm, std.range,
std.conv, std.exception, std.traits, std.typetuple,
std.uni, std.utf, std.format, std.typecons, std.bitmanip,
std.functional, std.exception;
import core.bitop, core.stdc.string, core.stdc.stdlib;
import ascii = std.ascii;
import std.string : representation;
debug import std.stdio;
private:
@safe:
//uncomment to get a barrage of debug info
//debug = fred_parser;
//debug = fred_matching;
//debug = fred_charset;
// IR bit pattern: 0b1_xxxxx_yy
// where yy indicates class of instruction, xxxxx for actual operation code
// 00: atom, a normal instruction
// 01: open, opening of a group, has length of contained IR in the low bits
// 10: close, closing of a group, has length of contained IR in the low bits
// 11 unused
//
// Loops with Q (non-greedy, with ? mark) must have the same size / other properties as non Q version
// Possible changes:
//* merge group, option, infinite/repeat start (to never copy during parsing of (a|b){1,2})
//* reorganize groups to make n args easier to find, or simplify the check for groups of similar ops
// (like lookaround), or make it easier to identify hotspots.
enum IR:uint {
Char = 0b1_00000_00, //a character
Any = 0b1_00001_00, //any character
CodepointSet = 0b1_00010_00, //a most generic CodepointSet [...]
Trie = 0b1_00011_00, //CodepointSet implemented as Trie
//match with any of a consecutive OrChar's in this sequence
//(used for case insensitive match)
//OrChar holds in upper two bits of data total number of OrChars in this _sequence_
//the drawback of this representation is that it is difficult
// to detect a jump in the middle of it
OrChar = 0b1_00100_00,
Nop = 0b1_00101_00, //no operation (padding)
End = 0b1_00110_00, //end of program
Bol = 0b1_00111_00, //beginning of a string ^
Eol = 0b1_01000_00, //end of a string $
Wordboundary = 0b1_01001_00, //boundary of a word
Notwordboundary = 0b1_01010_00, //not a word boundary
Backref = 0b1_01011_00, //backreference to a group (that has to be pinned, i.e. locally unique) (group index)
GroupStart = 0b1_01100_00, //start of a group (x) (groupIndex+groupPinning(1bit))
GroupEnd = 0b1_01101_00, //end of a group (x) (groupIndex+groupPinning(1bit))
Option = 0b1_01110_00, //start of an option within an alternation x | y (length)
GotoEndOr = 0b1_01111_00, //end of an option (length of the rest)
//... any additional atoms here
OrStart = 0b1_00000_01, //start of alternation group (length)
OrEnd = 0b1_00000_10, //end of the or group (length,mergeIndex)
//with this instruction order
//bit mask 0b1_00001_00 could be used to test/set greediness
InfiniteStart = 0b1_00001_01, //start of an infinite repetition x* (length)
InfiniteEnd = 0b1_00001_10, //end of infinite repetition x* (length,mergeIndex)
InfiniteQStart = 0b1_00010_01, //start of a non eager infinite repetition x*? (length)
InfiniteQEnd = 0b1_00010_10, //end of non eager infinite repetition x*? (length,mergeIndex)
RepeatStart = 0b1_00011_01, //start of a {n,m} repetition (length)
RepeatEnd = 0b1_00011_10, //end of x{n,m} repetition (length,step,minRep,maxRep)
RepeatQStart = 0b1_00100_01, //start of a non eager x{n,m}? repetition (length)
RepeatQEnd = 0b1_00100_10, //end of non eager x{n,m}? repetition (length,step,minRep,maxRep)
//
LookaheadStart = 0b1_00101_01, //begin of the lookahead group (length)
LookaheadEnd = 0b1_00101_10, //end of a lookahead group (length)
NeglookaheadStart = 0b1_00110_01, //start of a negative lookahead (length)
NeglookaheadEnd = 0b1_00110_10, //end of a negative lookahead (length)
LookbehindStart = 0b1_00111_01, //start of a lookbehind (length)
LookbehindEnd = 0b1_00111_10, //end of a lookbehind (length)
NeglookbehindStart= 0b1_01000_01, //start of a negative lookbehind (length)
NeglookbehindEnd = 0b1_01000_10, //end of negative lookbehind (length)
}
//a shorthand for IR length - full length of specific opcode evaluated at compile time
template IRL(IR code)
{
enum uint IRL = lengthOfIR(code);
}
static assert (IRL!(IR.LookaheadStart) == 3);
//how many parameters follow the IR, should be optimized fixing some IR bits
int immediateParamsIR(IR i){
switch (i){
case IR.OrEnd,IR.InfiniteEnd,IR.InfiniteQEnd:
return 1;
case IR.RepeatEnd, IR.RepeatQEnd:
return 4;
case IR.LookaheadStart, IR.NeglookaheadStart, IR.LookbehindStart, IR.NeglookbehindStart:
return 2;
default:
return 0;
}
}
//full length of IR instruction inlcuding all parameters that might follow it
int lengthOfIR(IR i)
{
return 1 + immediateParamsIR(i);
}
//full length of the paired IR instruction inlcuding all parameters that might follow it
int lengthOfPairedIR(IR i)
{
return 1 + immediateParamsIR(pairedIR(i));
}
//if the operation has a merge point (this relies on the order of the ops)
bool hasMerge(IR i)
{
return (i&0b11)==0b10 && i<=IR.RepeatQEnd;
}
//is an IR that opens a "group"
bool isStartIR(IR i)
{
return (i&0b11)==0b01;
}
//is an IR that ends a "group"
bool isEndIR(IR i)
{
return (i&0b11)==0b10;
}
//is a standalone IR
bool isAtomIR(IR i)
{
return (i&0b11)==0b00;
}
//makes respective pair out of IR i, swapping start/end bits of instruction
IR pairedIR(IR i)
{
assert(isStartIR(i) || isEndIR(i));
return cast(IR)(i ^ 0b11);
}
//encoded IR instruction
struct Bytecode
{
uint raw;
//natural constraints
enum maxSequence = 2+4;
enum maxData = 1<<22;
enum maxRaw = 1<<31;
this(IR code, uint data)
{
assert(data < (1<<22) && code < 256);
raw = code<<24 | data;
}
this(IR code, uint data, uint seq)
{
assert(data < (1<<22) && code < 256 );
assert(seq >= 2 && seq < maxSequence);
raw = code<<24 | ((seq-2)<<22) | data;
}
//store raw data
static Bytecode fromRaw(uint data)
{
Bytecode t;
t.raw = data;
return t;
}
//bit twiddling helpers
@property uint data() const { return raw & 0x003f_ffff; }
//ditto
@property uint sequence() const { return 2+((raw >>22) & 0x3); }
//ditto
@property IR code() const { return cast(IR)(raw>>24); }
//ditto
@property bool hotspot() const { return hasMerge(code); }
//test the class of this instruction
@property bool isAtom() const { return isAtomIR(code); }
//ditto
@property bool isStart() const { return isStartIR(code); }
//ditto
@property bool isEnd() const { return isEndIR(code); }
//number of arguments for this instruction
@property int args() const { return immediateParamsIR(code); }
//mark this GroupStart or GroupEnd as referenced in backreference
void setBackrefence()
{
assert(code == IR.GroupStart || code == IR.GroupEnd);
raw = raw | (1<<23);
}
//is referenced
@property bool backreference() const
{
assert(code == IR.GroupStart || code == IR.GroupEnd);
return cast(bool)(raw & (1<<23));
}
//mark as local reference (for backrefs in lookarounds)
void setLocalRef()
{
assert(code == IR.Backref);
raw = raw | (1<<23);
}
//is a local ref
@property bool localRef() const
{
assert(code == IR.Backref);
return cast(bool)(raw & (1<<23));
}
//human readable name of instruction
@trusted @property string mnemonic() const
{//@@@BUG@@@ to is @system
return to!string(code);
}
//full length of instruction
@property uint length() const
{
return lengthOfIR(code);
}
//full length of respective start/end of this instruction
@property uint pairedLength() const
{
return lengthOfPairedIR(code);
}
//returns bytecode of paired instruction (assuming this one is start or end)
@property Bytecode paired() const
{//depends on bit and struct layout order
assert(isStart || isEnd);
return Bytecode.fromRaw(raw ^ (0b11<<24));
}
//gets an index into IR block of the respective pair
uint indexOfPair(uint pc) const
{
assert(isStart || isEnd);
return isStart ? pc + data + length : pc - data - lengthOfPairedIR(code);
}
}
static assert(Bytecode.sizeof == 4);
//debugging tool, prints out instruction along with opcodes
@trusted string disassemble(in Bytecode[] irb, uint pc, in NamedGroup[] dict=[])
{
auto output = appender!string();
formattedWrite(output,"%s", irb[pc].mnemonic);
switch(irb[pc].code)
{
case IR.Char:
formattedWrite(output, " %s (0x%x)",cast(dchar)irb[pc].data, irb[pc].data);
break;
case IR.OrChar:
formattedWrite(output, " %s (0x%x) seq=%d", cast(dchar)irb[pc].data, irb[pc].data, irb[pc].sequence);
break;
case IR.RepeatStart, IR.InfiniteStart, IR.Option, IR.GotoEndOr, IR.OrStart:
//forward-jump instructions
uint len = irb[pc].data;
formattedWrite(output, " pc=>%u", pc+len+IRL!(IR.RepeatStart));
break;
case IR.RepeatEnd, IR.RepeatQEnd: //backward-jump instructions
uint len = irb[pc].data;
formattedWrite(output, " pc=>%u min=%u max=%u step=%u"
, pc-len, irb[pc+3].raw, irb[pc+4].raw, irb[pc+2].raw);
break;
case IR.InfiniteEnd, IR.InfiniteQEnd, IR.OrEnd: //ditto
uint len = irb[pc].data;
formattedWrite(output, " pc=>%u", pc-len);
break;
case IR.LookaheadEnd, IR.NeglookaheadEnd: //ditto
uint len = irb[pc].data;
formattedWrite(output, " pc=>%u", pc-len);
break;
case IR.GroupStart, IR.GroupEnd:
uint n = irb[pc].data;
string name;
foreach(v;dict)
if(v.group == n)
{
name = "'"~v.name~"'";
break;
}
formattedWrite(output, " %s #%u " ~ (irb[pc].backreference ? "referenced" : ""),
name, n);
break;
case IR.LookaheadStart, IR.NeglookaheadStart, IR.LookbehindStart, IR.NeglookbehindStart:
uint len = irb[pc].data;
uint start = irb[pc+1].raw, end = irb[pc+2].raw;
formattedWrite(output, " pc=>%u [%u..%u]", pc + len + IRL!(IR.LookaheadStart), start, end);
break;
case IR.Backref: case IR.CodepointSet: case IR.Trie:
uint n = irb[pc].data;
formattedWrite(output, " %u", n);
if(irb[pc].code == IR.Backref)
formattedWrite(output, " %s", irb[pc].localRef ? "local" : "global");
break;
default://all data-free instructions
}
if(irb[pc].hotspot)
formattedWrite(output, " Hotspot %u", irb[pc+1].raw);
return output.data;
}
//another pretty printer, writes out the bytecode of a regex and where the pc is
@trusted void prettyPrint(Sink,Char=const(char))
(Sink sink, const(Bytecode)[] irb, uint pc=uint.max, int indent=3, size_t index=0)
if (isOutputRange!(Sink,Char))
{//formattedWrite is @system
while(irb.length>0)
{
formattedWrite(sink,"%3d",index);
if(pc==0 && irb[0].code!=IR.Char)
{
for (int i=0;i<indent-2;++i)
put(sink,"=");
put(sink,"> ");
}
else
{
if(isEndIR(irb[0].code))
{
indent-=2;
}
if(indent>0)
{
string spaces=" ";
put(sink,spaces[0..(indent%spaces.length)]);
for (size_t i=indent/spaces.length;i>0;--i)
put(sink,spaces);
}
}
if(irb[0].code==IR.Char)
{
put(sink,`"`);
int i=0;
do{
put(sink,cast(char[])([cast(dchar)irb[i].data]));
++i;
} while(i<irb.length && irb[i].code==IR.Char);
put(sink,"\"");
if (pc<i){
put(sink,"\n");
for (int ii=indent+pc+1;ii>0;++ii)
put(sink,"=");
put(sink,"^");
}
index+=i;
irb=irb[i..$];
}
else
{
put(sink,irb[0].mnemonic);
put(sink,"(");
formattedWrite(sink,"%d",irb[0].data);
int nArgs= irb[0].args;
for(int iarg=0;iarg<nArgs;++iarg)
{
if(iarg+1<irb.length)
formattedWrite(sink,",%d",irb[iarg+1].data);
else
put(sink,"*error* incomplete irb stream");
}
put(sink,")");
if(isStartIR(irb[0].code))
{
indent+=2;
}
index+=lengthOfIR(irb[0].code);
irb=irb[lengthOfIR(irb[0].code)..$];
}
put(sink,"\n");
}
}
//index entry structure for name --> number of submatch
struct NamedGroup
{
string name;
uint group;
}
//holds pair of start-end markers for a submatch
struct Group(DataIndex)
{
DataIndex begin, end;
@trusted string toString() const
{
auto a = appender!string();
formattedWrite(a, "%s..%s", begin, end);
return a.data;
}
}
//Regular expression engine/parser options:
// global - search all nonoverlapping matches in input
// casefold - case insensitive matching, do casefolding on match in unicode mode
// freeform - ignore whitespace in pattern, to match space use [ ] or \s
// multiline - switch ^, $ detect start and end of linesinstead of just start and end of input
enum RegexOption: uint {
global = 0x1,
casefold = 0x2,
freeform = 0x4,
nonunicode = 0x8,
multiline = 0x10,
singleline = 0x20
};
alias TypeTuple!('g', 'i', 'x', 'U', 'm', 's') RegexOptionNames;//do not reorder this list
static assert( RegexOption.max < 0x80);
enum RegexInfo : uint { oneShot = 0x80 };
private enum NEL = '\u0085', LS = '\u2028', PS = '\u2029';
//test if a given string starts with hex number of maxDigit that's a valid codepoint
//returns it's value and skips these maxDigit chars on success, throws on failure
dchar parseUniHex(Char)(ref Char[] str, uint maxDigit)
{
enforce(str.length >= maxDigit,"incomplete escape sequence");
uint val;
for(int k=0;k<maxDigit;k++)
{
auto current = str[k];//accepts ascii only, so it's OK to index directly
if('0' <= current && current <= '9')
val = val * 16 + current - '0';
else if('a' <= current && current <= 'f')
val = val * 16 + current -'a' + 10;
else if('A' <= current && current <= 'Z')
val = val * 16 + current - 'A' + 10;
else
throw new Exception("invalid escape sequence");
}
enforce(val <= 0x10FFFF, "invalid codepoint");
str = str[maxDigit..$];
return val;
}
//heuristic value determines maximum CodepointSet length suitable for linear search
enum maxCharsetUsed = 6;
enum maxCachedTries = 8;
alias CodepointTrie!8 Trie;
Trie[const(CodepointSet)] trieCache;
//accessor with caching
@trusted Trie getTrie(in CodepointSet set)
{// @@@BUG@@@ 6357 almost all properties of AA are not @safe
if(__ctfe || maxCachedTries == 0)
return Trie(set);
else
{
auto p = set in trieCache;
if(p)
return *p;
if(trieCache.length == maxCachedTries)
{
// flush entries in trieCache
trieCache = null;
}
return (trieCache[set] = Trie(set));
}
}
//property for \w character class
@property CodepointSet wordCharacter()
{
return memoizeExpr!("CodepointSet.init.add(unicodeAlphabetic).add(unicodeMn).add(unicodeMc)
.add(unicodeMe).add(unicodeNd).add(unicodePc)")();
}
@property Trie wordTrie()
{
return memoizeExpr!("Trie(wordCharacter)")();
}
auto memoizeExpr(string expr)()
{
if(__ctfe)
return mixin(expr);
alias typeof(mixin(expr)) T;
static T slot;
static bool initialized;
if(!initialized)
{
slot = mixin(expr);
initialized = true;
}
return slot;
}
/+
fetch codepoint set corresponding to a name (InBlock or binary property)
+/
@trusted const(CodepointSet) getUnicodeSet(in char[] name, bool negated, bool casefold)
{
alias comparePropertyName ucmp;
CodepointSet s;
//unicode property
//helper: direct access with a sanity check
if(ucmp(name, "L") == 0 || ucmp(name, "Letter") == 0)
{
s.add(unicodeLu).add(unicodeLl).add(unicodeLt)
.add(unicodeLo).add(unicodeLm);
}
else if(ucmp(name,"LC") == 0 || ucmp(name,"Cased Letter")==0)
{
s.add(unicodeLl).add(unicodeLu).add(unicodeLt);//Title case
}
else if(ucmp(name, "M") == 0 || ucmp(name, "Mark") == 0)
{
s.add(unicodeMn).add(unicodeMc).add(unicodeMe);
}
else if(ucmp(name, "P") == 0 || ucmp(name, "Punctuation") == 0)
{
s.add(unicodePc).add(unicodePd).add(unicodePs).add(unicodePe)
.add(unicodePi).add(unicodePf).add(unicodePo);
}
else if(ucmp(name, "S") == 0 || ucmp(name, "Symbol") == 0)
{
s.add(unicodeSm).add(unicodeSc).add(unicodeSk).add(unicodeSo);
}
else if(ucmp(name, "Z") == 0 || ucmp(name, "Separator") == 0)
{
s.add(unicodeZs).add(unicodeZl).add(unicodeZp);
}
else if(ucmp(name, "C") == 0 || ucmp(name, "Other") == 0)
{
s.add(unicodeCo).add(unicodeLo).add(unicodeNo)
.add(unicodeSo).add(unicodePo);
}
else if(ucmp(name, "any") == 0)
s.add(Interval(0,0x10FFFF));
else if(ucmp(name, "ascii") == 0)
s.add(Interval(0,0x7f));
else
{
auto range = assumeSorted!((x,y) => ucmp(x.name, y.name) < 0)(unicodeProperties);
//creating empty Codepointset is a workaround
auto eq = range.lowerBound(UnicodeProperty(cast(string)name,CodepointSet.init)).length;
enforce(eq!=range.length && ucmp(name,range[eq].name)==0,"invalid property name");
s = range[eq].set.dup;
}
if(casefold)
s = caseEnclose(s);
if(negated)
s.negate();
return cast(const CodepointSet)s;
}
//basic stack, just in case it gets used anywhere else then Parser
@trusted struct Stack(T, bool CTFE=false)
{
static if(!CTFE)
Appender!(T[]) stack;//compiles but bogus at CTFE
else
{
struct Proxy
{
T[] data;
void put(T val)
{
data ~= val;
}
void shrinkTo(size_t sz){ data = data[0..sz]; }
}
Proxy stack;
}
@property bool empty(){ return stack.data.empty; }
void push(T item)
{
stack.put(item);
}
@property ref T top()
{
assert(!empty);
return stack.data[$-1];
}
@property size_t length() { return stack.data.length; }
T pop()
{
assert(!empty);
auto t = stack.data[$-1];
stack.shrinkTo(stack.data.length-1);
return t;
}
}
//safety limits
enum maxGroupNumber = 2^^19;
enum maxLookaroundDepth = 16;
// *Bytecode.sizeof, i.e. 1Mb of bytecode alone
enum maxCompiledLength = 2^^18;
//amounts to up to 4 Mb of auxilary table for matching
enum maxCumulativeRepetitionLength = 2^^20;
template BasicElementOf(Range)
{
alias Unqual!(ElementEncodingType!Range) BasicElementOf;
}
struct Parser(R, bool CTFE=false)
if (isForwardRange!R && is(ElementType!R : dchar))
{
enum infinite = ~0u;
dchar _current;
bool empty;
R pat, origin; //keep full pattern for pretty printing error messages
Bytecode[] ir; //resulting bytecode
uint re_flags = 0; //global flags e.g. multiline + internal ones
Stack!(uint, CTFE) fixupStack; //stack of opened start instructions
NamedGroup[] dict; //maps name -> user group number
//current num of group, group nesting level and repetitions step
Stack!(uint, CTFE) groupStack;
uint nesting = 0;
uint lookaroundNest = 0;
uint counterDepth = 0; //current depth of nested counted repetitions
const(CodepointSet)[] charsets; //
const(Trie)[] tries; //
uint[] backrefed; //bitarray for groups
@trusted this(S)(R pattern, S flags)
if(isSomeString!S)
{
pat = origin = pattern;
//reserve slightly more then avg as sampled from unittests
if(!__ctfe)
ir.reserve((pat.length*5+2)/4);
parseFlags(flags);
_current = ' ';//a safe default for freeform parsing
next();
try
{
parseRegex();
}
catch(Exception e)
{
error(e.msg);//also adds pattern location
}
put(Bytecode(IR.End, 0));
}
//mark referenced groups for latter processing
void markBackref(uint n)
{
if(n/32 >= backrefed.length)
backrefed.length = n/32 + 1;
backrefed[n/32] |= 1<<(n & 31);
}
@property dchar current(){ return _current; }
bool _next()
{
if(pat.empty)
{
empty = true;
return false;
}
_current = pat.front;
pat.popFront();
return true;
}
void skipSpace()
{
while(isWhite(current) && _next()){ }
}
bool next()
{
if(re_flags & RegexOption.freeform)
{
bool r = _next();
skipSpace();
return r;
}
else
return _next();
}
void put(Bytecode code)
{
enforce(ir.length < maxCompiledLength
, "maximum compiled pattern length is exceeded");
if(__ctfe)
{
ir = ir ~ code;
}
else
ir ~= code;
}
void putRaw(uint number)
{
enforce(ir.length < maxCompiledLength
, "maximum compiled pattern length is exceeded");
ir ~= Bytecode.fromRaw(number);
}
//parsing number with basic overflow check
uint parseDecimal()
{
uint r=0;
while(ascii.isDigit(current))
{
if(r >= (uint.max/10))
error("Overflow in decimal number");
r = 10*r + cast(uint)(current-'0');
if(!next())
break;
}
return r;
}
//parse control code of form \cXXX, c assumed to be the current symbol
dchar parseControlCode()
{
enforce(next(), "Unfinished escape sequence");
enforce(('a' <= current && current <= 'z') || ('A' <= current && current <= 'Z'),
"Only letters are allowed after \\c");
return current & 0x1f;
}
//
@trusted void parseFlags(S)(S flags)
{//@@@BUG@@@ text is @system
foreach(ch; flags)//flags are ASCII anyway
{
L_FlagSwitch:
switch(ch)
{
foreach(i, op; __traits(allMembers, RegexOption))
{
case RegexOptionNames[i]:
if(re_flags & mixin("RegexOption."~op))
throw new RegexException(text("redundant flag specified: ",ch));
re_flags |= mixin("RegexOption."~op);
break L_FlagSwitch;
}
default:
if(__ctfe)
assert(text("unknown regex flag '",ch,"'"));
else
new RegexException(text("unknown regex flag '",ch,"'"));
}
}
}
//parse and store IR for regex pattern
@trusted void parseRegex()
{
fixupStack.push(0);
groupStack.push(1);//0 - whole match
auto maxCounterDepth = counterDepth;