From 118d03d44758f678b9c5e749c92edaf067d6673a Mon Sep 17 00:00:00 2001 From: Nicholas Wilson Date: Thu, 13 Mar 2025 10:47:31 -0700 Subject: [PATCH 1/6] New pcre2_next_match() API to simplify pcre2demo, test, and substitute --- BUILD.bazel | 1 + CMakeLists.txt | 1 + Makefile.am | 3 + NON-AUTOTOOLS-BUILD | 1 + README | 1 + build.zig | 1 + doc/html/NON-AUTOTOOLS-BUILD.txt | 1 + doc/html/README.txt | 1 + doc/html/index.html | 2 + doc/index.html.src | 3 + doc/pcre2_next_match.3 | 40 +++++ doc/pcre2api.3 | 110 +++++++++--- doc/pcre2test.1 | 10 +- maint/UpdateAlways | 1 + maint/manifest-cmakeinstall-freebsd | 2 + maint/manifest-cmakeinstall-linux | 2 + maint/manifest-cmakeinstall-macos | 2 + maint/manifest-cmakeinstall-solaris | 2 + maint/manifest-cmakeinstall-windows | 2 + maint/manifest-libpcre2-16.so | 1 + maint/manifest-libpcre2-32.so | 1 + maint/manifest-libpcre2-8.so | 1 + maint/manifest-makeinstall-freebsd | 2 + maint/manifest-makeinstall-linux | 2 + maint/manifest-makeinstall-solaris | 2 + maint/manifest-tarball | 3 + src/libpcre2-16.sym | 1 + src/libpcre2-32.sym | 1 + src/libpcre2-8.sym | 1 + src/pcre2.h.in | 9 +- src/pcre2_dfa_match.c | 3 +- src/pcre2_intmodedep.h | 1 + src/pcre2_jit_match_inc.h | 1 + src/pcre2_match.c | 2 + src/pcre2_match_next.c | 170 +++++++++++++++++++ src/pcre2_substitute.c | 110 ++++-------- src/pcre2demo.c | 121 ++------------ src/pcre2test.c | 250 ++++++++++------------------ testdata/testoutput2 | 1 + testdata/testoutput5 | 1 + vms/configure.com | 3 + 41 files changed, 497 insertions(+), 376 deletions(-) create mode 100644 doc/pcre2_next_match.3 create mode 100644 src/pcre2_match_next.c diff --git a/BUILD.bazel b/BUILD.bazel index d0766f689..cf3bc5d2c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -80,6 +80,7 @@ cc_library( "src/pcre2_maketables.c", "src/pcre2_match.c", "src/pcre2_match_data.c", + "src/pcre2_match_next.c", "src/pcre2_newline.c", "src/pcre2_ord2utf.c", "src/pcre2_pattern_info.c", diff --git a/CMakeLists.txt b/CMakeLists.txt index a6184b828..2d2e854af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -782,6 +782,7 @@ set( src/pcre2_maketables.c src/pcre2_match.c src/pcre2_match_data.c + src/pcre2_match_next.c src/pcre2_newline.c src/pcre2_ord2utf.c src/pcre2_pattern_info.c diff --git a/Makefile.am b/Makefile.am index 8dbc53a87..7cb4fa452 100644 --- a/Makefile.am +++ b/Makefile.am @@ -73,6 +73,7 @@ dist_html_DATA = \ doc/html/pcre2_match_data_create.html \ doc/html/pcre2_match_data_create_from_pattern.html \ doc/html/pcre2_match_data_free.html \ + doc/html/pcre2_next_match.html \ doc/html/pcre2_pattern_convert.html \ doc/html/pcre2_pattern_info.html \ doc/html/pcre2_serialize_decode.html \ @@ -174,6 +175,7 @@ dist_man_MANS = \ doc/pcre2_match_data_create.3 \ doc/pcre2_match_data_create_from_pattern.3 \ doc/pcre2_match_data_free.3 \ + doc/pcre2_next_match.3 \ doc/pcre2_pattern_convert.3 \ doc/pcre2_pattern_info.3 \ doc/pcre2_serialize_decode.3 \ @@ -419,6 +421,7 @@ COMMON_SOURCES = \ src/pcre2_maketables.c \ src/pcre2_match.c \ src/pcre2_match_data.c \ + src/pcre2_match_next.c \ src/pcre2_newline.c \ src/pcre2_ord2utf.c \ src/pcre2_pattern_info.c \ diff --git a/NON-AUTOTOOLS-BUILD b/NON-AUTOTOOLS-BUILD index 54ee7b5d3..89d97aa01 100644 --- a/NON-AUTOTOOLS-BUILD +++ b/NON-AUTOTOOLS-BUILD @@ -120,6 +120,7 @@ example. pcre2_maketables.c pcre2_match.c pcre2_match_data.c + pcre2_match_next.c pcre2_newline.c pcre2_ord2utf.c pcre2_pattern_info.c diff --git a/README b/README index 8854f004d..fc1cc4a2d 100644 --- a/README +++ b/README @@ -866,6 +866,7 @@ The distribution should contain the files listed below. src/pcre2_maketables.c ) sources for the functions in the library, src/pcre2_match.c ) and some internal functions that they use src/pcre2_match_data.c ) + src/pcre2_match_next.c ) src/pcre2_newline.c ) src/pcre2_ord2utf.c ) src/pcre2_pattern_info.c ) diff --git a/build.zig b/build.zig index 9fad3e360..59de742c1 100644 --- a/build.zig +++ b/build.zig @@ -92,6 +92,7 @@ pub fn build(b: *std.Build) !void { "src/pcre2_maketables.c", "src/pcre2_match.c", "src/pcre2_match_data.c", + "src/pcre2_match_next.c", "src/pcre2_newline.c", "src/pcre2_ord2utf.c", "src/pcre2_pattern_info.c", diff --git a/doc/html/NON-AUTOTOOLS-BUILD.txt b/doc/html/NON-AUTOTOOLS-BUILD.txt index 54ee7b5d3..89d97aa01 100644 --- a/doc/html/NON-AUTOTOOLS-BUILD.txt +++ b/doc/html/NON-AUTOTOOLS-BUILD.txt @@ -120,6 +120,7 @@ example. pcre2_maketables.c pcre2_match.c pcre2_match_data.c + pcre2_match_next.c pcre2_newline.c pcre2_ord2utf.c pcre2_pattern_info.c diff --git a/doc/html/README.txt b/doc/html/README.txt index 8854f004d..fc1cc4a2d 100644 --- a/doc/html/README.txt +++ b/doc/html/README.txt @@ -866,6 +866,7 @@ The distribution should contain the files listed below. src/pcre2_maketables.c ) sources for the functions in the library, src/pcre2_match.c ) and some internal functions that they use src/pcre2_match_data.c ) + src/pcre2_match_next.c ) src/pcre2_newline.c ) src/pcre2_ord2utf.c ) src/pcre2_pattern_info.c ) diff --git a/doc/html/index.html b/doc/html/index.html index 43b0d595b..96b0362aa 100644 --- a/doc/html/index.html +++ b/doc/html/index.html @@ -220,6 +220,8 @@

Perl-compatible Regular Expressions (revised API: PCRE2)

pcre2_match_data_free Free a match data block +// XXX + pcre2_pattern_convert Experimental foreign pattern converter diff --git a/doc/index.html.src b/doc/index.html.src index 43b0d595b..f4aa2ea4f 100644 --- a/doc/index.html.src +++ b/doc/index.html.src @@ -220,6 +220,9 @@ in the library. pcre2_match_data_free Free a match data block +pcre2_next_match + Get the match parameters for the next match + pcre2_pattern_convert Experimental foreign pattern converter diff --git a/doc/pcre2_next_match.3 b/doc/pcre2_next_match.3 new file mode 100644 index 000000000..ea655762c --- /dev/null +++ b/doc/pcre2_next_match.3 @@ -0,0 +1,40 @@ +.TH PCRE2_NEXT_MATCH 3 "01 March 2025" "PCRE2 10.46-DEV" +.SH NAME +PCRE2 - Perl-compatible regular expressions (revised API) +.SH SYNOPSIS +.rs +.sp +.B #include +.PP +.nf +.B int pcre2_next_match(pcre2_match_data *\fImatch_data\fP, +.B " PCRE2_SIZE *\fIpstart_offset\fP, uint32_t *\fIpoptions\fP);" +.fi +. +.SH DESCRIPTION +.rs +.sp +This function can be called after calling one of the matching functions +(\fBpcre2_match()\fP, \fBpcre2_dfa_match()\fP, or \fBpcre2_jit_match()\fP), by +providing the same \fImatch_data\fP parameter. It provides the appropriate +parameters for searching for the next match in the same subject string, and is +suitable for applications providing "global" matching behaviour (replacing all +matches in the subject, or splitting the subject on all matches, or simply +counting the number of matches). +.P +It returns 0 ("false") if there is no need to make any further match attempts, +or 1 ("true") if another match should be attempted. +.P +The \fIpstart_offset\fP and \fIpoptions\fP are set if the function returns 1. +The *\fIpoptions\fP should be combined with the application's match options +using OR. +.P +There is a complete description of the PCRE2 native API in the +.\" HREF +\fBpcre2api\fP +.\" +page and a description of the POSIX API in the +.\" HREF +\fBpcre2posix\fP +.\" +page. diff --git a/doc/pcre2api.3 b/doc/pcre2api.3 index d3df267c6..92d92ffae 100644 --- a/doc/pcre2api.3 +++ b/doc/pcre2api.3 @@ -2957,18 +2957,13 @@ is able to look behind the starting point to discover that it is preceded by a letter. .P Finding all the matches in a subject is tricky when the pattern can match an -empty string. It is possible to emulate Perl's /g behaviour by first trying the -match again at the same offset, with the PCRE2_NOTEMPTY_ATSTART and -PCRE2_ANCHORED options, and then if that fails, advancing the starting offset -and trying an ordinary match again. There is some code that demonstrates how to -do this in the -.\" HREF -\fBpcre2demo\fP +empty string. PCRE2 includes a helper API to assist with this; see the +section entitled "Iterating over all matches" +.\" HTML +.\" +below .\" -sample program. In the most general case, you have to check to see if the -newline convention recognizes CRLF as a newline, and if so, and the current -character is CR followed by LF, advance the starting offset by two characters -instead of one. +for details. .P If a non-zero starting offset is passed when the pattern is anchored, a single attempt to match at the given offset is made. This can only succeed if the @@ -3284,10 +3279,10 @@ set. For example, if two substrings have been captured, the returned value is 3. If there are no captured substrings, the return value from a successful match is 1, indicating that just the first pair of offsets has been set. .P -If a pattern uses the \eK escape sequence within a positive assertion, the -reported start of a successful match can be greater than the end of the match. -For example, if the pattern (?=ab\eK) is matched against "ab", the start and -end offset values for the match are 2 and 0. +If a pattern uses the \eK escape sequence within a positive lookahead assertion, +the reported start of a successful match can be greater than the end of the +match. For example, if the pattern (?=ab\eK) is matched against "ab", the start +and end offset values for the match are 2 and 0. .P If a capture group is matched repeatedly within a single match operation, it is the last portion of the subject that it matched that is returned. @@ -3523,7 +3518,77 @@ returns the number of code units used, excluding the trailing zero. If the error number is unknown, the negative error code PCRE2_ERROR_BADDATA is returned. If the buffer is too small, the message is truncated (but still with a trailing zero), and the negative error code PCRE2_ERROR_NOMEMORY is returned. -None of the messages are very long; a buffer size of 120 code units is ample. +None of the messages is very long; a buffer size of 120 code units is ample. +. +. +.\" HTML +.SH "ITERATING OVER ALL MATCHES" +.rs +.sp +.nf +.B int pcre2_next_match(pcre2_match_data *\fImatch_data\fP, +.B " PCRE2_SIZE *\fIpstart_offset\fP, uint32_t *\fIpoptions\fP);" +.fi +.P +A common task for applications is to implement "global" matching behaviour, +for example, replacing all matches in the subject; splitting the subject on all +matches; or simply counting the number of matches. The \fBpcre2_next_match()\fP +function helps with this task by providing the appropriate parameters for the +next match attempt. +.P +First, a match attempt should be made using one of the matching functions +(\fBpcre2_match()\fP, \fBpcre2_dfa_match()\fP, or \fBpcre2_jit_match()\fP). Then, +\fBpcre2_next_match()\fP can be called, providing the same \fImatch_data\fP +parameter. +.P +It returns 0 ("false") if there is no need to make any further match attempts, +or 1 ("true") if another match should be attempted. +.P +The \fIpstart_offset\fP and \fIpoptions\fP are set if the function returns 1. +The *\fIpoptions\fP should be combined with the application's match options +using OR. +.P +There is some code that demonstrates how to do this in the +.\" HREF +\fBpcre2demo\fP +.\" +sample program. The general pattern is: +.nf + uint32_t app_options = ...; + uint32_t global_options = 0; + PCRE2_SIZE start_offset = 0; + while (1) + { + int rc = pcre2_match(re, subject, subject_len, start_offset, + app_options | global_options, match_data, + match_context); + + if (rc == PCRE2_ERROR_NOMATCH) break; + if (rc < 0) { ... exit } + + if (!pcre2_next_match(match_data, &start_offset, &global_options)) + break; + } +.fi +.P +The guarantees provided by \fBpcre2_next_match()\fP are that the start_offset +will advance, so the loop will definitely terminate. The condition for progress +is that either: (a) pcre2_next_match() returns 0 (false); or (b) the returned +start_offset is strictly greater than the previous start_offset, or (c) if the +previous match was a successful match of the empty string then the returned +start_offset is equal to previous start_offset, but poptions will be set to +PCRE2_NOTEMPTY_ATSTART to prevent another empty match from being returned. +.P +In a loop as shown above, it must terminate, unless there is a bug in PCRE2. As +a measure of "defensive programming", applications are encouraged to add an +assertion or check to break their loop if it does not make progress (and report +the issue as a bug). +.P +Note that we do not guarantee that the matches will always advance: only that +the start_offset will. If an application does not use the flag +PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK... XXX +.P +XXX . . .\" HTML @@ -3562,11 +3627,12 @@ substring zero is available. An attempt to extract any other substring gives the error PCRE2_ERROR_PARTIAL. The next section describes similar functions for extracting captured substrings by name. .P -If a pattern uses the \eK escape sequence within a positive assertion, the -reported start of a successful match can be greater than the end of the match. -For example, if the pattern (?=ab\eK) is matched against "ab", the start and -end offset values for the match are 2 and 0. In this situation, calling these -functions with a zero substring number extracts a zero-length empty string. +If a pattern uses the \eK escape sequence within a positive lookahead assertion, +the reported start of a successful match can be greater than the end of the +match. For example, if the pattern (?=ab\eK) is matched against "ab", the start +and end offset values for the match are 2 and 0. In this situation, calling +these functions with a zero substring number extracts a zero-length empty +string. .P You can find the length in code units of a captured substring without extracting it by calling \fBpcre2_substring_length_bynumber()\fP. The first @@ -4017,7 +4083,7 @@ replacement string, with more particular errors being PCRE2_ERROR_BADREPESCAPE not found), PCRE2_ERROR_BADSUBSTITUTION (syntax error in extended group substitution), and PCRE2_ERROR_BADSUBSPATTERN (the pattern match ended before it started or the match started earlier than the current position in the -subject, which can happen if \eK is used in an assertion). +subject, which can happen if \eK is used in a lookaround assertion). .P As for all PCRE2 errors, a text message that describes the error can be obtained by calling the \fBpcre2_get_error_message()\fP function (see diff --git a/doc/pcre2test.1 b/doc/pcre2test.1 index 37290a1ff..ef672429e 100644 --- a/doc/pcre2test.1 +++ b/doc/pcre2test.1 @@ -1466,13 +1466,9 @@ difference to the matching process if the pattern begins with a lookbehind assertion (including \eb or \eB). .P If an empty string is matched, the next match is done with the -PCRE2_NOTEMPTY_ATSTART and PCRE2_ANCHORED flags set, in order to search for -another, non-empty, match at the same point in the subject. If this match -fails, the start offset is advanced, and the normal match is retried. This -imitates the way Perl handles such cases when using the \fB/g\fP modifier or -the \fBsplit()\fP function. Normally, the start offset is advanced by one -character, but if the newline convention recognizes CRLF as a newline, and the -current character is CR followed by LF, an advance of two characters occurs. +PCRE2_NOTEMPTY_ATSTART flag set, in order to search for another, non-empty, +match at the same point in the subject. This imitates the way Perl handles such +cases when using the \fB/g\fP modifier or the \fBsplit()\fP function. . . .SS "Testing substring extraction functions" diff --git a/maint/UpdateAlways b/maint/UpdateAlways index 01fd80592..b1e64e3d7 100755 --- a/maint/UpdateAlways +++ b/maint/UpdateAlways @@ -292,6 +292,7 @@ c_files=( src/pcre2_maketables.c src/pcre2_match.c src/pcre2_match_data.c + src/pcre2_match_next.c src/pcre2_newline.c src/pcre2_ord2utf.c src/pcre2_pattern_info.c diff --git a/maint/manifest-cmakeinstall-freebsd b/maint/manifest-cmakeinstall-freebsd index 3bc4386f0..194127b6e 100644 --- a/maint/manifest-cmakeinstall-freebsd +++ b/maint/manifest-cmakeinstall-freebsd @@ -87,6 +87,7 @@ drwxr-xr-x install-dir/share/doc/pcre2/html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_create.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_create_from_pattern.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_free.html +-rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_next_match.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_pattern_convert.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_pattern_info.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_serialize_decode.html @@ -195,6 +196,7 @@ drwxr-xr-x install-dir/share/man/man3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_create.3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_create_from_pattern.3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_free.3 +-rw-r--r-- install-dir/share/man/man3/pcre2_next_match.3 -rw-r--r-- install-dir/share/man/man3/pcre2_pattern_convert.3 -rw-r--r-- install-dir/share/man/man3/pcre2_pattern_info.3 -rw-r--r-- install-dir/share/man/man3/pcre2_serialize_decode.3 diff --git a/maint/manifest-cmakeinstall-linux b/maint/manifest-cmakeinstall-linux index 137cc6671..04e46451f 100644 --- a/maint/manifest-cmakeinstall-linux +++ b/maint/manifest-cmakeinstall-linux @@ -87,6 +87,7 @@ drwxr-xr-x install-dir/share/doc/pcre2/html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_create.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_create_from_pattern.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_free.html +-rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_next_match.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_pattern_convert.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_pattern_info.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_serialize_decode.html @@ -195,6 +196,7 @@ drwxr-xr-x install-dir/share/man/man3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_create.3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_create_from_pattern.3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_free.3 +-rw-r--r-- install-dir/share/man/man3/pcre2_next_match.3 -rw-r--r-- install-dir/share/man/man3/pcre2_pattern_convert.3 -rw-r--r-- install-dir/share/man/man3/pcre2_pattern_info.3 -rw-r--r-- install-dir/share/man/man3/pcre2_serialize_decode.3 diff --git a/maint/manifest-cmakeinstall-macos b/maint/manifest-cmakeinstall-macos index 7b536eb1b..73738e0b2 100644 --- a/maint/manifest-cmakeinstall-macos +++ b/maint/manifest-cmakeinstall-macos @@ -87,6 +87,7 @@ drwxr-xr-x install-dir/share/doc/pcre2/html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_create.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_create_from_pattern.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_free.html +-rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_next_match.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_pattern_convert.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_pattern_info.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_serialize_decode.html @@ -195,6 +196,7 @@ drwxr-xr-x install-dir/share/man/man3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_create.3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_create_from_pattern.3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_free.3 +-rw-r--r-- install-dir/share/man/man3/pcre2_next_match.3 -rw-r--r-- install-dir/share/man/man3/pcre2_pattern_convert.3 -rw-r--r-- install-dir/share/man/man3/pcre2_pattern_info.3 -rw-r--r-- install-dir/share/man/man3/pcre2_serialize_decode.3 diff --git a/maint/manifest-cmakeinstall-solaris b/maint/manifest-cmakeinstall-solaris index 93b76a711..f56c3e733 100644 --- a/maint/manifest-cmakeinstall-solaris +++ b/maint/manifest-cmakeinstall-solaris @@ -87,6 +87,7 @@ drwxr-xr-x install-dir/share/doc/pcre2/html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_create.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_create_from_pattern.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_match_data_free.html +-rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_next_match.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_pattern_convert.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_pattern_info.html -rw-r--r-- install-dir/share/doc/pcre2/html/pcre2_serialize_decode.html @@ -195,6 +196,7 @@ drwxr-xr-x install-dir/share/man/man3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_create.3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_create_from_pattern.3 -rw-r--r-- install-dir/share/man/man3/pcre2_match_data_free.3 +-rw-r--r-- install-dir/share/man/man3/pcre2_next_match.3 -rw-r--r-- install-dir/share/man/man3/pcre2_pattern_convert.3 -rw-r--r-- install-dir/share/man/man3/pcre2_pattern_info.3 -rw-r--r-- install-dir/share/man/man3/pcre2_serialize_decode.3 diff --git a/maint/manifest-cmakeinstall-windows b/maint/manifest-cmakeinstall-windows index 0eea735d8..4c35e0ad2 100644 --- a/maint/manifest-cmakeinstall-windows +++ b/maint/manifest-cmakeinstall-windows @@ -82,6 +82,7 @@ d---- .\install-dir\share\doc\pcre2\html -a--- .\install-dir\share\doc\pcre2\html\pcre2_match_data_create.html -a--- .\install-dir\share\doc\pcre2\html\pcre2_match_data_create_from_pattern.html -a--- .\install-dir\share\doc\pcre2\html\pcre2_match_data_free.html +-a--- .\install-dir\share\doc\pcre2\html\pcre2_next_match.html -a--- .\install-dir\share\doc\pcre2\html\pcre2_pattern_convert.html -a--- .\install-dir\share\doc\pcre2\html\pcre2_pattern_info.html -a--- .\install-dir\share\doc\pcre2\html\pcre2_serialize_decode.html @@ -190,6 +191,7 @@ d---- .\install-dir\share\man\man3 -a--- .\install-dir\share\man\man3\pcre2_match_data_create.3 -a--- .\install-dir\share\man\man3\pcre2_match_data_create_from_pattern.3 -a--- .\install-dir\share\man\man3\pcre2_match_data_free.3 +-a--- .\install-dir\share\man\man3\pcre2_next_match.3 -a--- .\install-dir\share\man\man3\pcre2_pattern_convert.3 -a--- .\install-dir\share\man\man3\pcre2_pattern_info.3 -a--- .\install-dir\share\man\man3\pcre2_serialize_decode.3 diff --git a/maint/manifest-libpcre2-16.so b/maint/manifest-libpcre2-16.so index 48f456889..d79c17455 100644 --- a/maint/manifest-libpcre2-16.so +++ b/maint/manifest-libpcre2-16.so @@ -37,6 +37,7 @@ T pcre2_match_context_free_16@@PCRE2_10.46 T pcre2_match_data_create_16@@PCRE2_10.46 T pcre2_match_data_create_from_pattern_16@@PCRE2_10.46 T pcre2_match_data_free_16@@PCRE2_10.46 +T pcre2_next_match_16@@PCRE2_10.46 T pcre2_pattern_convert_16@@PCRE2_10.46 T pcre2_pattern_info_16@@PCRE2_10.46 T pcre2_serialize_decode_16@@PCRE2_10.46 diff --git a/maint/manifest-libpcre2-32.so b/maint/manifest-libpcre2-32.so index 9351caa14..4659ad60b 100644 --- a/maint/manifest-libpcre2-32.so +++ b/maint/manifest-libpcre2-32.so @@ -37,6 +37,7 @@ T pcre2_match_context_free_32@@PCRE2_10.46 T pcre2_match_data_create_32@@PCRE2_10.46 T pcre2_match_data_create_from_pattern_32@@PCRE2_10.46 T pcre2_match_data_free_32@@PCRE2_10.46 +T pcre2_next_match_32@@PCRE2_10.46 T pcre2_pattern_convert_32@@PCRE2_10.46 T pcre2_pattern_info_32@@PCRE2_10.46 T pcre2_serialize_decode_32@@PCRE2_10.46 diff --git a/maint/manifest-libpcre2-8.so b/maint/manifest-libpcre2-8.so index c74740d55..dbf615065 100644 --- a/maint/manifest-libpcre2-8.so +++ b/maint/manifest-libpcre2-8.so @@ -37,6 +37,7 @@ T pcre2_match_context_free_8@@PCRE2_10.46 T pcre2_match_data_create_8@@PCRE2_10.46 T pcre2_match_data_create_from_pattern_8@@PCRE2_10.46 T pcre2_match_data_free_8@@PCRE2_10.46 +T pcre2_next_match_8@@PCRE2_10.46 T pcre2_pattern_convert_8@@PCRE2_10.46 T pcre2_pattern_info_8@@PCRE2_10.46 T pcre2_serialize_decode_8@@PCRE2_10.46 diff --git a/maint/manifest-makeinstall-freebsd b/maint/manifest-makeinstall-freebsd index 844e1a82b..d9cfeb2d4 100644 --- a/maint/manifest-makeinstall-freebsd +++ b/maint/manifest-makeinstall-freebsd @@ -89,6 +89,7 @@ drwxr-xr-x install-dir/usr/local/share/doc/pcre2/html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_match_data_create.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_match_data_create_from_pattern.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_match_data_free.html +-rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_next_match.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_pattern_convert.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_pattern_info.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_serialize_decode.html @@ -197,6 +198,7 @@ drwxr-xr-x install-dir/usr/local/share/man/man3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_match_data_create.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_match_data_create_from_pattern.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_match_data_free.3 +-rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_next_match.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_pattern_convert.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_pattern_info.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_serialize_decode.3 diff --git a/maint/manifest-makeinstall-linux b/maint/manifest-makeinstall-linux index 991ebe131..f23fb82ca 100644 --- a/maint/manifest-makeinstall-linux +++ b/maint/manifest-makeinstall-linux @@ -89,6 +89,7 @@ drwxr-xr-x install-dir/usr/local/share/doc/pcre2/html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_match_data_create.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_match_data_create_from_pattern.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_match_data_free.html +-rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_next_match.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_pattern_convert.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_pattern_info.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_serialize_decode.html @@ -197,6 +198,7 @@ drwxr-xr-x install-dir/usr/local/share/man/man3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_match_data_create.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_match_data_create_from_pattern.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_match_data_free.3 +-rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_next_match.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_pattern_convert.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_pattern_info.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_serialize_decode.3 diff --git a/maint/manifest-makeinstall-solaris b/maint/manifest-makeinstall-solaris index 991ebe131..f23fb82ca 100644 --- a/maint/manifest-makeinstall-solaris +++ b/maint/manifest-makeinstall-solaris @@ -89,6 +89,7 @@ drwxr-xr-x install-dir/usr/local/share/doc/pcre2/html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_match_data_create.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_match_data_create_from_pattern.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_match_data_free.html +-rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_next_match.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_pattern_convert.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_pattern_info.html -rw-r--r-- install-dir/usr/local/share/doc/pcre2/html/pcre2_serialize_decode.html @@ -197,6 +198,7 @@ drwxr-xr-x install-dir/usr/local/share/man/man3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_match_data_create.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_match_data_create_from_pattern.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_match_data_free.3 +-rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_next_match.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_pattern_convert.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_pattern_info.3 -rw-r--r-- install-dir/usr/local/share/man/man3/pcre2_serialize_decode.3 diff --git a/maint/manifest-tarball b/maint/manifest-tarball index cdffc65bf..eae3fec5b 100644 --- a/maint/manifest-tarball +++ b/maint/manifest-tarball @@ -120,6 +120,7 @@ drwxr-xr-x tarball-dir/pcre2-SNAPSHOT/doc/html -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/html/pcre2_match_data_create.html -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/html/pcre2_match_data_create_from_pattern.html -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/html/pcre2_match_data_free.html +-rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/html/pcre2_next_match.html -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/html/pcre2_pattern_convert.html -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/html/pcre2_pattern_info.html -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/html/pcre2_serialize_decode.html @@ -221,6 +222,7 @@ drwxr-xr-x tarball-dir/pcre2-SNAPSHOT/doc/html -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/pcre2_match_data_create.3 -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/pcre2_match_data_create_from_pattern.3 -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/pcre2_match_data_free.3 +-rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/pcre2_next_match.3 -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/pcre2_pattern_convert.3 -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/pcre2_pattern_info.3 -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/doc/pcre2_serialize_decode.3 @@ -340,6 +342,7 @@ drwxr-xr-x tarball-dir/pcre2-SNAPSHOT/src -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/src/pcre2_maketables.c -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/src/pcre2_match.c -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/src/pcre2_match_data.c +-rw-r--r-- tarball-dir/pcre2-SNAPSHOT/src/pcre2_match_next.c -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/src/pcre2_newline.c -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/src/pcre2_ord2utf.c -rw-r--r-- tarball-dir/pcre2-SNAPSHOT/src/pcre2_pattern_info.c diff --git a/src/libpcre2-16.sym b/src/libpcre2-16.sym index 4afa5d457..f755c4d9d 100644 --- a/src/libpcre2-16.sym +++ b/src/libpcre2-16.sym @@ -40,6 +40,7 @@ PCRE2_10.46 { pcre2_match_data_create_16; pcre2_match_data_create_from_pattern_16; pcre2_match_data_free_16; + pcre2_next_match_16; pcre2_pattern_convert_16; pcre2_pattern_info_16; pcre2_serialize_decode_16; diff --git a/src/libpcre2-32.sym b/src/libpcre2-32.sym index af2323285..e2c832afa 100644 --- a/src/libpcre2-32.sym +++ b/src/libpcre2-32.sym @@ -40,6 +40,7 @@ PCRE2_10.46 { pcre2_match_data_create_32; pcre2_match_data_create_from_pattern_32; pcre2_match_data_free_32; + pcre2_next_match_32; pcre2_pattern_convert_32; pcre2_pattern_info_32; pcre2_serialize_decode_32; diff --git a/src/libpcre2-8.sym b/src/libpcre2-8.sym index 96c6e69d3..004804aa4 100644 --- a/src/libpcre2-8.sym +++ b/src/libpcre2-8.sym @@ -40,6 +40,7 @@ PCRE2_10.46 { pcre2_match_data_create_8; pcre2_match_data_create_from_pattern_8; pcre2_match_data_free_8; + pcre2_next_match_8; pcre2_pattern_convert_8; pcre2_pattern_info_8; pcre2_serialize_decode_8; diff --git a/src/pcre2.h.in b/src/pcre2.h.in index ca3f0b413..a2b9aedb8 100644 --- a/src/pcre2.h.in +++ b/src/pcre2.h.in @@ -743,14 +743,14 @@ PCRE2_EXP_DECL pcre2_match_data *PCRE2_CALL_CONVENTION \ PCRE2_EXP_DECL pcre2_match_data *PCRE2_CALL_CONVENTION \ pcre2_match_data_create_from_pattern(const pcre2_code *, \ pcre2_general_context *); \ +PCRE2_EXP_DECL void PCRE2_CALL_CONVENTION \ + pcre2_match_data_free(pcre2_match_data *); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_dfa_match(const pcre2_code *, PCRE2_SPTR, PCRE2_SIZE, PCRE2_SIZE, \ uint32_t, pcre2_match_data *, pcre2_match_context *, int *, PCRE2_SIZE); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_match(const pcre2_code *, PCRE2_SPTR, PCRE2_SIZE, PCRE2_SIZE, \ uint32_t, pcre2_match_data *, pcre2_match_context *); \ -PCRE2_EXP_DECL void PCRE2_CALL_CONVENTION \ - pcre2_match_data_free(pcre2_match_data *); \ PCRE2_EXP_DECL PCRE2_SPTR PCRE2_CALL_CONVENTION \ pcre2_get_mark(pcre2_match_data *); \ PCRE2_EXP_DECL PCRE2_SIZE PCRE2_CALL_CONVENTION \ @@ -762,7 +762,9 @@ PCRE2_EXP_DECL uint32_t PCRE2_CALL_CONVENTION \ PCRE2_EXP_DECL PCRE2_SIZE *PCRE2_CALL_CONVENTION \ pcre2_get_ovector_pointer(pcre2_match_data *); \ PCRE2_EXP_DECL PCRE2_SIZE PCRE2_CALL_CONVENTION \ - pcre2_get_startchar(pcre2_match_data *); + pcre2_get_startchar(pcre2_match_data *); \ +PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ + pcre2_next_match(pcre2_match_data *, PCRE2_SIZE *, uint32_t *); /* Convenience functions for handling matched substrings. */ @@ -942,6 +944,7 @@ pcre2_compile are called by application code. */ #define pcre2_match_data_create PCRE2_SUFFIX(pcre2_match_data_create_) #define pcre2_match_data_create_from_pattern PCRE2_SUFFIX(pcre2_match_data_create_from_pattern_) #define pcre2_match_data_free PCRE2_SUFFIX(pcre2_match_data_free_) +#define pcre2_next_match PCRE2_SUFFIX(pcre2_next_match_) #define pcre2_pattern_convert PCRE2_SUFFIX(pcre2_pattern_convert_) #define pcre2_pattern_info PCRE2_SUFFIX(pcre2_pattern_info_) #define pcre2_serialize_decode PCRE2_SUFFIX(pcre2_serialize_decode_) diff --git a/src/pcre2_dfa_match.c b/src/pcre2_dfa_match.c index 8728de5c6..672a724ef 100644 --- a/src/pcre2_dfa_match.c +++ b/src/pcre2_dfa_match.c @@ -4039,12 +4039,13 @@ for (;;) match_data->ovector[1] = (PCRE2_SIZE)(end_subject - subject); } match_data->subject_length = length; + match_data->start_offset = start_offset; match_data->leftchar = (PCRE2_SIZE)(mb->start_used_ptr - subject); match_data->rightchar = (PCRE2_SIZE)(mb->last_used_ptr - subject); match_data->startchar = (PCRE2_SIZE)(start_match - subject); match_data->rc = rc; - if (rc >= 0 &&(options & PCRE2_COPY_MATCHED_SUBJECT) != 0) + if (rc >= 0 && (options & PCRE2_COPY_MATCHED_SUBJECT) != 0) { length = CU2BYTES(length + was_zero_terminated); match_data->subject = match_data->memctl.malloc(length, diff --git a/src/pcre2_intmodedep.h b/src/pcre2_intmodedep.h index a04c0c8a8..bb3dd2c3c 100644 --- a/src/pcre2_intmodedep.h +++ b/src/pcre2_intmodedep.h @@ -680,6 +680,7 @@ typedef struct pcre2_real_match_data { struct heapframe *heapframes; /* Backtracking frames heap memory */ PCRE2_SIZE heapframes_size; /* Malloc-ed size */ PCRE2_SIZE subject_length; /* Subject length */ + PCRE2_SIZE start_offset; /* Offset to start of search */ PCRE2_SIZE leftchar; /* Offset to leftmost code unit */ PCRE2_SIZE rightchar; /* Offset to rightmost code unit */ PCRE2_SIZE startchar; /* Offset to starting code unit */ diff --git a/src/pcre2_jit_match_inc.h b/src/pcre2_jit_match_inc.h index 68fbe95a4..81cd9ccf9 100644 --- a/src/pcre2_jit_match_inc.h +++ b/src/pcre2_jit_match_inc.h @@ -178,6 +178,7 @@ if (rc > (int)oveccount) match_data->code = re; match_data->subject = (rc >= 0 || rc == PCRE2_ERROR_PARTIAL)? subject : NULL; match_data->subject_length = length; +match_data->start_offset = start_offset; match_data->rc = rc; match_data->startchar = arguments.startchar_ptr - subject; match_data->leftchar = 0; diff --git a/src/pcre2_match.c b/src/pcre2_match.c index f246fc4ca..817580f0f 100644 --- a/src/pcre2_match.c +++ b/src/pcre2_match.c @@ -8102,6 +8102,7 @@ if (rc == MATCH_MATCH) match_data->rc = ((int)mb->end_offset_top >= 2 * match_data->oveccount)? 0 : (int)mb->end_offset_top/2 + 1; match_data->subject_length = length; + match_data->start_offset = start_offset; match_data->startchar = start_match - subject; match_data->leftchar = mb->start_used_ptr - subject; match_data->rightchar = ((mb->last_used_ptr > mb->end_match_ptr)? @@ -8139,6 +8140,7 @@ else if (match_partial != NULL) { match_data->subject = subject; match_data->subject_length = length; + match_data->start_offset = start_offset; match_data->ovector[0] = match_partial - subject; match_data->ovector[1] = end_subject - subject; match_data->startchar = match_partial - subject; diff --git a/src/pcre2_match_next.c b/src/pcre2_match_next.c new file mode 100644 index 000000000..096ca7e04 --- /dev/null +++ b/src/pcre2_match_next.c @@ -0,0 +1,170 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Original API code Copyright (c) 1997-2012 University of Cambridge + New API code Copyright (c) 2016-2024 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +#include "pcre2_internal.h" + + + +/* Advance the offset by one code unit, and return the new value. +It is only called when the offset is not at the end of the subject. */ + +static PCRE2_SIZE do_bumpalong(pcre2_match_data *match_data, + PCRE2_SIZE offset) +{ +PCRE2_SPTR subject = match_data->subject; +PCRE2_SIZE subject_length = match_data->subject_length; +#ifdef SUPPORT_UNICODE +BOOL utf = (match_data->code->overall_options & PCRE2_UTF) != 0; +#endif + +/* Skip over CRLF as an atomic sequence, if CRLF is configured as a newline +sequence. */ + +if (subject[offset] == CHAR_CR && offset + 1 < subject_length && + subject[offset + 1] == CHAR_LF) + { + switch(match_data->code->newline_convention) + { + case PCRE2_NEWLINE_CRLF: + case PCRE2_NEWLINE_ANY: + case PCRE2_NEWLINE_ANYCRLF: + return offset + 2; + } + } + +/* Advance by one full character if in UTF mode. */ + +#ifdef SUPPORT_UNICODE +if (utf) + { + PCRE2_SPTR next = subject + offset + 1; + PCRE2_SPTR subject_end = subject + subject_length; + + FORWARDCHARTEST(next, subject_end); + return next - subject; + } +#endif + +return offset + 1; +} + + + +/************************************************* +* Advance the match * +*************************************************/ + +PCRE2_EXP_DEFN int PCRE2_CALL_CONVENTION +pcre2_next_match(pcre2_match_data *match_data, PCRE2_SIZE *pstart_offset, + uint32_t *poptions) +{ +int rc = match_data->rc; +PCRE2_SIZE start_offset = match_data->start_offset; +PCRE2_SIZE *ovector = match_data->ovector; + +/* Match error, or no match: no further iteration possible. In previous versions +of PCRE2, we recommended that clients use a strategy which involved retrying in +certain cases after PCRE2_ERROR_NOMATCH, but this is no longer required. */ + +if (rc < 0) + return FALSE; + +/* Match succeeded: get the start offset for the next match */ + +/* Although \K can affect the position of ovector[0], there are no ways to do +anything surprising with ovector[1], which must always be >= start_offset. */ + +PCRE2_ASSERT(ovector[1] >= start_offset); + +/* Special handling for patterns which contain \K in a lookaround, which enables +the match start to be pushed back to before the starting search offset +(ovector[0] < start_offset) or after the match ends (ovector[0] > ovector[1]). +This is not a problem if ovector[1] > start_offset, because in this case, we can +just attempt the next match at ovector[1]: we are making progress, which is all +that we require. + +However, if we have ovector[1] == start_offset, then we have a very rare case +which must be handled specially, because it's a non-empty match which +nonetheless fails to make progress through the subject. */ + +if (ovector[0] != start_offset && ovector[1] == start_offset) + { + /* If the match end is at the end of the subject, we are done. */ + + if (start_offset >= match_data->subject_length) + return FALSE; + + /* Otherwise, bump along by one code unit, and do a normal search. */ + + *pstart_offset = do_bumpalong(match_data, ovector[1]); + *poptions = 0; + return TRUE; + } + +/* If the previous match was for an empty string, we are finished if we are at +the end of the subject. Otherwise, arrange to run another match at the same +point to see if a non-empty match can be found. */ + +if (ovector[0] == ovector[1]) + { + /* If the match is at the end of the subject, we are done. */ + + if (ovector[0] >= match_data->subject_length) + return FALSE; + + /* Otherwise, continue at this exact same point, but we must set the flag + which ensures that we don't return the exact same empty match again. */ + + *pstart_offset = ovector[1]; + *poptions = PCRE2_NOTEMPTY_ATSTART; + return TRUE; + } + +/* Finally, we must be in the happy state of a non-empty match, where the end of +the match is further on in the subject than start_offset, so we are easily able +to continue and make progress. */ + +*pstart_offset = ovector[1]; +*poptions = 0; +return TRUE; +} + +/* End of pcre2_match_next.c */ diff --git a/src/pcre2_substitute.c b/src/pcre2_substitute.c index 3a95ab4d2..f56609700 100644 --- a/src/pcre2_substitute.c +++ b/src/pcre2_substitute.c @@ -754,7 +754,7 @@ PCRE2_SPTR repend = NULL; PCRE2_SIZE extra_needed = 0; PCRE2_SIZE buff_offset, buff_length, lengthleft, fraglength; PCRE2_SIZE *ovector; -PCRE2_SIZE ovecsave[3]; +PCRE2_SIZE ovecsave[2]; pcre2_substitute_callout_block scb; PCRE2_SIZE sub_start_extra_needed; PCRE2_SIZE (*substitute_case_callout)(PCRE2_SPTR, PCRE2_SIZE, PCRE2_UCHAR *, @@ -766,7 +766,6 @@ void *substitute_case_callout_data = NULL; buff_offset = 0; lengthleft = buff_length = *blength; *blength = PCRE2_UNSET; -ovecsave[0] = ovecsave[1] = ovecsave[2] = PCRE2_UNSET; if (mcontext != NULL) { @@ -903,7 +902,7 @@ if (!replacement_only) CHECKMEMCPY(subject, start_offset); match is taken from the match_data that was passed in. */ subs = 0; -do +for (;;) { PCRE2_SPTR ptrstack[PTR_STACK_SIZE]; uint32_t ptrstackptr = 0; @@ -923,54 +922,12 @@ do if (utf) options |= PCRE2_NO_UTF_CHECK; /* Only need to check once */ #endif - /* Any error other than no match returns the error code. No match when not - doing the special after-empty-match global rematch, or when at the end of the - subject, breaks the global loop. Otherwise, advance the starting point by one - character, copying it to the output, and try again. */ + /* Any error other than no match returns the error code. No match breaks the + global loop. */ - if (rc < 0) - { - PCRE2_SIZE save_start; - - if (rc != PCRE2_ERROR_NOMATCH) goto EXIT; - if (goptions == 0 || start_offset >= length) break; - - /* Advance by one code point. Then, if CRLF is a valid newline sequence and - we have advanced into the middle of it, advance one more code point. In - other words, do not start in the middle of CRLF, even if CR and LF on their - own are valid newlines. */ - - save_start = start_offset++; - if (subject[start_offset-1] == CHAR_CR && - (code->newline_convention == PCRE2_NEWLINE_CRLF || - code->newline_convention == PCRE2_NEWLINE_ANY || - code->newline_convention == PCRE2_NEWLINE_ANYCRLF) && - start_offset < length && - subject[start_offset] == CHAR_LF) - start_offset++; - - /* Otherwise, in UTF mode, advance past any secondary code points. */ - - else if ((code->overall_options & PCRE2_UTF) != 0) - { -#if PCRE2_CODE_UNIT_WIDTH == 8 - while (start_offset < length && (subject[start_offset] & 0xc0) == 0x80) - start_offset++; -#elif PCRE2_CODE_UNIT_WIDTH == 16 - while (start_offset < length && - (subject[start_offset] & 0xfc00) == 0xdc00) - start_offset++; -#endif - } + if (rc == PCRE2_ERROR_NOMATCH) break; - /* Copy what we have advanced past (unless not required), reset the special - global options, and continue to the next match. */ - - fraglength = start_offset - save_start; - if (!replacement_only) CHECKMEMCPY(subject + save_start, fraglength); - goptions = 0; - continue; - } + if (rc < 0) goto EXIT; /* Handle a successful match. Matches that use \K to end before they start or start before the current point in the subject are not supported. */ @@ -981,26 +938,25 @@ do goto EXIT; } - /* Check for the same match as previous. This is legitimate after matching an - empty string that starts after the initial match offset. We have tried again - at the match point in case the pattern is one like /(?<=\G.)/ which can never - match at its starting point, so running the match achieves the bumpalong. If - we do get the same (null) match at the original match point, it isn't such a - pattern, so we now do the empty string magic. In all other cases, a repeat - match should never occur. */ + /* Assert that our replacement loop is making progress, checked even in + release builds. This should be impossible to hit, however, an infinite loop + would be fairly catastrophic. + + "Progress" is measured as ovector[1] strictly advancing, or, an empty match + after a non-empty match. */ - if (ovecsave[0] == ovector[0] && ovecsave[1] == ovector[1]) + if (subs > 0 && + !(ovector[1] > ovecsave[1] || + (ovector[1] == ovector[0] && ovecsave[1] != ovecsave[0] && + ovector[1] == ovecsave[1]))) { - if (ovector[0] == ovector[1] && ovecsave[2] != start_offset) - { - goptions = PCRE2_NOTEMPTY_ATSTART | PCRE2_ANCHORED; - ovecsave[2] = start_offset; - continue; /* Back to the top of the loop */ - } rc = PCRE2_ERROR_INTERNAL_DUPMATCH; goto EXIT; } + ovecsave[0] = ovector[0]; + ovecsave[1] = ovector[1]; + /* Count substitutions with a paranoid check for integer overflow; surely no real call to this function would ever hit this! */ @@ -1625,19 +1581,25 @@ do } } - /* Save the details of this match. See above for how this data is used. If we - matched an empty string, do the magic for global matches. Update the start - offset to point to the rest of the subject string. If we re-used an existing - match for the first match, switch to the internal match data block. */ + /* Exit the global loop if we are not in global mode, or if pcre2_next_match() + indicates we have reached the end of the subject. */ - ovecsave[0] = ovector[0]; - ovecsave[1] = ovector[1]; - ovecsave[2] = start_offset; + if ((suboptions & PCRE2_SUBSTITUTE_GLOBAL) == 0 || + !pcre2_next_match(match_data, &start_offset, &goptions)) + { + start_offset = ovector[1]; + break; + } + + /* Verify that pcre2_next_match() has not done a bumpalong (because we have + already returned PCRE2_ERROR_BADSUBSPATTERN for \K in lookarounds). + + We would otherwise have to memcpy the fragment spanning from ovector[1] to the + new start_offset.*/ + + PCRE2_ASSERT(start_offset == ovector[1]); - goptions = (ovector[0] != ovector[1] || ovector[0] > start_offset)? 0 : - PCRE2_ANCHORED|PCRE2_NOTEMPTY_ATSTART; - start_offset = ovector[1]; - } while ((suboptions & PCRE2_SUBSTITUTE_GLOBAL) != 0); /* Repeat "do" loop */ + } /* End of global loop */ /* Copy the rest of the subject unless not required, and terminate the output with a binary zero. */ diff --git a/src/pcre2demo.c b/src/pcre2demo.c index 9f86e7a05..23fb49fd1 100644 --- a/src/pcre2demo.c +++ b/src/pcre2demo.c @@ -73,17 +73,13 @@ PCRE2_SPTR pattern; /* PCRE2_SPTR is a pointer to unsigned code units of */ PCRE2_SPTR subject; /* the appropriate width (in this case, 8 bits). */ PCRE2_SPTR name_table; -int crlf_is_newline; int errornumber; int find_all, caseless_match; int i; int rc; -int utf8; -uint32_t option_bits; uint32_t namecount; uint32_t name_entry_size; -uint32_t newline; PCRE2_SIZE erroroffset; PCRE2_SIZE *ovector; @@ -266,7 +262,9 @@ we have to extract the count of named parentheses from the pattern. */ PCRE2_INFO_NAMECOUNT, /* get the number of named substrings */ &namecount); /* where to put the answer */ -if (namecount == 0) printf("No named substrings\n"); else +if (namecount == 0) + printf("No named substrings\n"); +else { PCRE2_SPTR tabptr; printf("Named substrings\n"); @@ -304,28 +302,8 @@ if (namecount == 0) printf("No named substrings\n"); else * to search for additional matches in the subject string, in a similar * * way to the /g option in Perl. This turns out to be trickier than you * * might think because of the possibility of matching an empty string. * -* What happens is as follows: * * * -* If the previous match was NOT for an empty string, we can just start * -* the next match at the end of the previous one. * -* * -* If the previous match WAS for an empty string, we can't do that, as it * -* would lead to an infinite loop. Instead, a call of pcre2_match() is * -* made with the PCRE2_NOTEMPTY_ATSTART and PCRE2_ANCHORED flags set. The * -* first of these tells PCRE2 that an empty string at the start of the * -* subject is not a valid match; other possibilities must be tried. The * -* second flag restricts PCRE2 to one match attempt at the initial string * -* position. If this match succeeds, an alternative to the empty string * -* match has been found, and we can print it and proceed round the loop, * -* advancing by the length of whatever was found. If this match does not * -* succeed, we still stay in the loop, advancing by just one character. * -* In UTF-8 mode, which can be set by (*UTF) in the pattern, this may be * -* more than one byte. * -* * -* However, there is a complication concerned with newlines. When the * -* newline convention is such that CRLF is a valid newline, we must * -* advance by two characters rather than one. The newline convention can * -* be set in the regex by (*CR), etc.; if not, we must find the default. * +* To help with this task, PCRE2 provides the pcre2_next_match() helper. * *************************************************************************/ if (!find_all) /* Check for -g */ @@ -335,60 +313,18 @@ if (!find_all) /* Check for -g */ return 0; /* Exit the program. */ } -/* Before running the loop, check for UTF-8 and whether CRLF is a valid newline -sequence. First, find the options with which the regex was compiled and extract -the UTF state. */ - -(void)pcre2_pattern_info(re, PCRE2_INFO_ALLOPTIONS, &option_bits); -utf8 = (option_bits & PCRE2_UTF) != 0; - -/* Now find the newline convention and see whether CRLF is a valid newline -sequence. */ - -(void)pcre2_pattern_info(re, PCRE2_INFO_NEWLINE, &newline); -crlf_is_newline = newline == PCRE2_NEWLINE_ANY || - newline == PCRE2_NEWLINE_CRLF || - newline == PCRE2_NEWLINE_ANYCRLF; - /* Loop for second and subsequent matches */ for (;;) { - uint32_t options = 0; /* Normally no options */ - PCRE2_SIZE start_offset = ovector[1]; /* Start at end of previous match */ + PCRE2_SIZE start_offset; + uint32_t options; - /* If the previous match was for an empty string, we are finished if we are - at the end of the subject. Otherwise, arrange to run another match at the - same point to see if a non-empty match can be found. */ + /* After each successful match, we use pcre2_next_match() to obtain the match + parameters for subsequent match attempts. */ - if (ovector[0] == ovector[1]) - { - if (ovector[0] == subject_length) break; - options = PCRE2_NOTEMPTY_ATSTART | PCRE2_ANCHORED; - } - - /* If the previous match was not an empty string, there is one tricky case to - consider. If a pattern contains \K within a lookbehind assertion at the - start, the end of the matched string can be at the offset where the match - started. Without special action, this leads to a loop that keeps on matching - the same substring. We must detect this case and arrange to move the start on - by one character. The pcre2_get_startchar() function returns the starting - offset that was passed to pcre2_match(). */ - - else - { - PCRE2_SIZE startchar = pcre2_get_startchar(match_data); - if (start_offset <= startchar) - { - if (startchar >= subject_length) break; /* Reached end of subject. */ - start_offset = startchar + 1; /* Advance by one character. */ - if (utf8) /* If UTF-8, it may be more */ - { /* than one code unit. */ - for (; start_offset < subject_length; start_offset++) - if ((subject[start_offset] & 0xc0) != 0x80) break; - } - } - } + if (!pcre2_next_match(match_data, &start_offset, &options)) + break; /* Run the next matching operation */ @@ -401,38 +337,10 @@ for (;;) match_data, /* block for storing the result */ NULL); /* use default match context */ - /* This time, a result of NOMATCH isn't an error. If the value in "options" - is zero, it just means we have found all possible matches, so the loop ends. - Otherwise, it means we have failed to find a non-empty-string match at a - point where there was a previous empty-string match. In this case, we do what - Perl does: advance the matching position by one character, and continue. We - do this by setting the "end of previous match" offset, because that is picked - up at the top of the loop as the point at which to start again. - - There are two complications: (a) When CRLF is a valid newline sequence, and - the current position is just before it, advance by an extra byte. (b) - Otherwise we must ensure that we skip an entire UTF character if we are in - UTF mode. */ + /* If this match attempt fails, exit the loop for subsequent matches. */ if (rc == PCRE2_ERROR_NOMATCH) - { - if (options == 0) break; /* All matches found */ - ovector[1] = start_offset + 1; /* Advance one code unit */ - if (crlf_is_newline && /* If CRLF is a newline & */ - start_offset < subject_length - 1 && /* we are at CRLF, */ - subject[start_offset] == '\r' && - subject[start_offset + 1] == '\n') - ovector[1] += 1; /* Advance by one more. */ - else if (utf8) /* Otherwise, ensure we */ - { /* advance a whole UTF-8 */ - while (ovector[1] < subject_length) /* character. */ - { - if ((subject[ovector[1]] & 0xc0) != 0x80) break; - ovector[1] += 1; - } - } - continue; /* Go round the loop again */ - } + break; /* Other matching errors are not recoverable. */ @@ -479,7 +387,9 @@ for (;;) printf("%2d: %.*s\n", i, (int)substring_length, (char *)substring_start); } - if (namecount == 0) printf("No named substrings\n"); else + if (namecount == 0) + printf("No named substrings\n"); + else { PCRE2_SPTR tabptr = name_table; printf("Named substrings\n"); @@ -494,6 +404,7 @@ for (;;) } /* End of loop to find second and subsequent matches */ printf("\n"); + pcre2_match_data_free(match_data); pcre2_code_free(re); return 0; diff --git a/src/pcre2test.c b/src/pcre2test.c index 00efdc090..45c195f9d 100644 --- a/src/pcre2test.c +++ b/src/pcre2test.c @@ -1302,6 +1302,14 @@ are supported. */ else \ a = pcre2_get_startchar_32(G(b,32)) +#define PCRE2_NEXT_MATCH(a,b,c,d) \ + if (test_mode == PCRE8_MODE) \ + a = pcre2_next_match_8(G(b,8),c,d); \ + else if (test_mode == PCRE16_MODE) \ + a = pcre2_next_match_16(G(b,16),c,d); \ + else \ + a = pcre2_next_match_32(G(b,32),c,d) + #define PCRE2_JIT_COMPILE(r,a,b) \ if (test_mode == PCRE8_MODE) r = pcre2_jit_compile_8(G(a,8),b); \ else if (test_mode == PCRE16_MODE) r = pcre2_jit_compile_16(G(a,16),b); \ @@ -1672,11 +1680,6 @@ are supported. */ else if (test_mode == PCRE16_MODE) G(x,16)->y = z; \ else G(x,32)->y = z -#define SETFLDVEC(x,y,v,z) \ - if (test_mode == PCRE8_MODE) G(x,8)->y[v] = z; \ - else if (test_mode == PCRE16_MODE) G(x,16)->y[v] = z; \ - else G(x,32)->y[v] = z - #define SETOP(x,y,z) \ if (test_mode == PCRE8_MODE) G(x,8) z y; \ else if (test_mode == PCRE16_MODE) G(x,16) z y; \ @@ -1868,6 +1871,12 @@ the three different cases. */ else \ a = G(pcre2_get_startchar_,BITTWO)(G(b,BITTWO)) +#define PCRE2_NEXT_MATCH(a,b,c,d) \ + if (test_mode == G(G(PCRE,BITONE),_MODE)) \ + a = G(pcre2_next_match_,BITONE)(G(b,BITONE),c,d); \ + else \ + a = G(pcre2_next_match_,BITTWO)(G(b,BITTWO),c,d) + #define PCRE2_JIT_COMPILE(r,a,b) \ if (test_mode == G(G(PCRE,BITONE),_MODE)) \ r = G(pcre2_jit_compile_,BITONE)(G(a,BITONE),b); \ @@ -2173,10 +2182,6 @@ the three different cases. */ if (test_mode == G(G(PCRE,BITONE),_MODE)) G(x,BITONE)->y = z; \ else G(x,BITTWO)->y = z -#define SETFLDVEC(x,y,v,z) \ - if (test_mode == G(G(PCRE,BITONE),_MODE)) G(x,BITONE)->y[v] = z; \ - else G(x,BITTWO)->y[v] = z - #define SETOP(x,y,z) \ if (test_mode == G(G(PCRE,BITONE),_MODE)) G(x,BITONE) z y; \ else G(x,BITTWO) z y @@ -2249,6 +2254,7 @@ the three different cases. */ r = pcre2_get_match_data_heapframes_size_8(G(a,8)) #define PCRE2_GET_OVECTOR_COUNT(a,b) a = pcre2_get_ovector_count_8(G(b,8)) #define PCRE2_GET_STARTCHAR(a,b) a = pcre2_get_startchar_8(G(b,8)) +#define PCRE2_NEXT_MATCH(a,b,c,d) a = pcre2_next_match_8(G(b,8),c,d) #define PCRE2_JIT_COMPILE(r,a,b) r = pcre2_jit_compile_8(G(a,8),b) #define PCRE2_JIT_FREE_UNUSED_MEMORY(a) pcre2_jit_free_unused_memory_8(G(a,8)) #define PCRE2_JIT_MATCH(a,b,c,d,e,f,g,h) \ @@ -2322,7 +2328,6 @@ the three different cases. */ a = pcre2_substring_number_from_name_8(G(b,8),G(c,8)); #define PTR(x) (void *)G(x,8) #define SETFLD(x,y,z) G(x,8)->y = z -#define SETFLDVEC(x,y,v,z) G(x,8)->y[v] = z #define SETOP(x,y,z) G(x,8) z y #define SETCASTPTR(x,y) G(x,8) = (uint8_t *)(y) #define STRLEN(p) (int)strlen((char *)p) @@ -2364,6 +2369,7 @@ the three different cases. */ #define PCRE2_GET_MATCH_DATA_HEAPFRAMES_SIZE(r,a) \ r = pcre2_get_match_data_heapframes_size_16(G(a,16)) #define PCRE2_GET_STARTCHAR(a,b) a = pcre2_get_startchar_16(G(b,16)) +#define PCRE2_NEXT_MATCH(a,b,c,d) a = pcre2_next_match_16(G(b,16),c,d) #define PCRE2_JIT_COMPILE(r,a,b) r = pcre2_jit_compile_16(G(a,16),b) #define PCRE2_JIT_FREE_UNUSED_MEMORY(a) pcre2_jit_free_unused_memory_16(G(a,16)) #define PCRE2_JIT_MATCH(a,b,c,d,e,f,g,h) \ @@ -2435,7 +2441,6 @@ the three different cases. */ a = pcre2_substring_number_from_name_16(G(b,16),G(c,16)); #define PTR(x) (void *)G(x,16) #define SETFLD(x,y,z) G(x,16)->y = z -#define SETFLDVEC(x,y,v,z) G(x,16)->y[v] = z #define SETOP(x,y,z) G(x,16) z y #define SETCASTPTR(x,y) G(x,16) = (uint16_t *)(y) #define STRLEN(p) (int)strlen16((PCRE2_SPTR16)p) @@ -2477,6 +2482,7 @@ the three different cases. */ #define PCRE2_GET_MATCH_DATA_HEAPFRAMES_SIZE(r,a) \ r = pcre2_get_match_data_heapframes_size_32(G(a,32)) #define PCRE2_GET_STARTCHAR(a,b) a = pcre2_get_startchar_32(G(b,32)) +#define PCRE2_NEXT_MATCH(a,b,c,d) a = pcre2_next_match_32(G(b,32),c,d) #define PCRE2_JIT_COMPILE(r,a,b) r = pcre2_jit_compile_32(G(a,32),b) #define PCRE2_JIT_FREE_UNUSED_MEMORY(a) pcre2_jit_free_unused_memory_32(G(a,32)) #define PCRE2_JIT_MATCH(a,b,c,d,e,f,g,h) \ @@ -2548,7 +2554,6 @@ the three different cases. */ a = pcre2_substring_number_from_name_32(G(b,32),G(c,32)); #define PTR(x) (void *)G(x,32) #define SETFLD(x,y,z) G(x,32)->y = z -#define SETFLDVEC(x,y,v,z) G(x,32)->y[v] = z #define SETOP(x,y,z) G(x,32) z y #define SETCASTPTR(x,y) G(x,32) = (uint32_t *)(y) #define STRLEN(p) (int)strlen32((PCRE2_SPTR32)p) @@ -7472,7 +7477,7 @@ BOOL utf; BOOL subject_literal; PCRE2_SIZE *ovector; -PCRE2_SIZE ovecsave[3]; +uint8_t *ovecsave[2]; uint32_t oveccount; #ifdef SUPPORT_PCRE2_8 @@ -8482,10 +8487,7 @@ if (dat_datctl.replacement[0] != 0) } /* End of substitution handling */ /* When a replacement string is not provided, run a loop for global matching -with one of the basic matching functions. For altglobal (or first time round -the loop), set an "unset" value for the previous match info. */ - -ovecsave[0] = ovecsave[1] = ovecsave[2] = PCRE2_UNSET; +with one of the basic matching functions. */ for (gmatched = 0;; gmatched++) { @@ -8701,17 +8703,12 @@ for (gmatched = 0;; gmatched++) pp[0] = 0; } - if (capcount > (int)oveccount) /* Check for lunatic return value */ + if ((unsigned)capcount > oveccount) /* Check for lunatic return value */ { fprintf(outfile, "** PCRE2 error: returned count %d is too big for ovector count %d\n", capcount, oveccount); - capcount = oveccount; - if ((dat_datctl.control & CTL_ANYGLOB) != 0) - { - fprintf(outfile, "** Global loop abandoned\n"); - dat_datctl.control &= ~CTL_ANYGLOB; /* Break g/G loop */ - } + return PR_ABEND; } /* If PCRE2_COPY_MATCHED_SUBJECT was set, check that things are as they @@ -8734,29 +8731,52 @@ for (gmatched = 0;; gmatched++) } /* If this is not the first time round a global loop, check that the - returned string has changed. If it has not, check for an empty string match - at different starting offset from the previous match. This is a failed test - retry for null-matching patterns that don't match at their starting offset, - for example /(?<=\G.)/. A repeated match at the same point is not such a - pattern, and must be discarded, and we then proceed to seek a non-null - match at the current point. For any other repeated match, there is a bug - somewhere and we must break the loop because it will go on for ever. We - know that there are always at least two elements in the ovector. */ - - if (gmatched > 0 && ovecsave[0] == ovector[0] && ovecsave[1] == ovector[1]) + returned string has advanced. + + There is one known case where this doesn't happen: when you have a + "badly-behaved" pattern which uses \K in a lookaround, and breaks the core + sanity rule that start_offset <= ovector[0] <= ovector[1]. An example would + be /(?<=\Ka)/g matching "aaa". + * first attempt, start_offset=0: ovector[0]=0, ovector[1]=1 + * second attempt, start_offset=1: ovector[0]=0, ovector[1]=1 + + You can see that even though we *always* ensure that start_offset advances, + this doesn't guarantee to avoid duplicate matches. + + The pcre2test behaviour is to return all the matches found, except in the + case where two adjacent matches are an exact duplicate. */ + + if (gmatched > 0 && + !(dat_datctl.offset <= ovector[0] && ovector[0] <= ovector[1]) && + pp + code_unit_size * ovector[0] == ovecsave[0] && + pp + code_unit_size * ovector[1] == ovecsave[1]) + { + fprintf(outfile, "global repeat returned the same match as previous\n"); + goto NEXT_MATCH; + } + + /* Outside of this exceptional case, we check that either we have a + "badly-behaved" match (note that not all badly-behaved matches are caught + above, only *duplicate* ones); or else in the well-behaved case the match + must make progress. + + "Progress" is measured as ovector[1] strictly advancing, or, an empty match + after a non-empty match. */ + + if (gmatched > 0 && + (dat_datctl.offset <= ovector[0] && ovector[0] <= ovector[1]) && + !(pp + code_unit_size * ovector[1] > ovecsave[1] || + (ovector[1] == ovector[0] && ovecsave[1] != ovecsave[0] && + pp + code_unit_size * ovector[1] == ovecsave[1]))) { - if (ovector[0] == ovector[1] && ovecsave[2] != dat_datctl.offset) - { - g_notempty = PCRE2_NOTEMPTY_ATSTART | PCRE2_ANCHORED; - ovecsave[2] = dat_datctl.offset; - continue; /* Back to the top of the loop */ - } fprintf(outfile, - "** PCRE2 error: global repeat returned the same string as previous\n"); - fprintf(outfile, "** Global loop abandoned\n"); - dat_datctl.control &= ~CTL_ANYGLOB; /* Break g/G loop */ + "** PCRE2 error: global repeat did not make progress\n"); + return PR_ABEND; } + ovecsave[0] = pp + code_unit_size * ovector[0]; + ovecsave[1] = pp + code_unit_size * ovector[1]; + /* "allcaptures" requests showing of all captures in the pattern, to check unset ones at the end. It may be set on the pattern or the data. Implement by setting capcount to the maximum. This is not relevant for DFA matching, @@ -8975,52 +8995,6 @@ for (gmatched = 0;; gmatched++) break; /* Out of the /g loop */ } /* End of handling partial match */ - /* Failed to match. If this is a /g or /G loop, we might previously have - set g_notempty (to PCRE2_NOTEMPTY_ATSTART|PCRE2_ANCHORED) after a null match. - If that is the case, this is not necessarily the end. We want to advance the - start offset, and continue. We won't be at the end of the string - that was - checked before setting g_notempty. We achieve the effect by pretending that a - single character was matched. - - Complication arises in the case when the newline convention is "any", "crlf", - or "anycrlf". If the previous match was at the end of a line terminated by - CRLF, an advance of one character just passes the CR, whereas we should - prefer the longer newline sequence, as does the code in pcre2_match(). - - Otherwise, in the case of UTF-8 or UTF-16 matching, the advance must be one - character, not one byte. */ - - else if (g_notempty != 0) /* There was a previous null match */ - { - uint16_t nl = FLD(compiled_code, newline_convention); - PCRE2_SIZE start_offset = dat_datctl.offset; /* Where the match was */ - PCRE2_SIZE end_offset = start_offset + 1; - - if ((nl == PCRE2_NEWLINE_CRLF || nl == PCRE2_NEWLINE_ANY || - nl == PCRE2_NEWLINE_ANYCRLF) && - start_offset < ulen - 1 && - CODE_UNIT(pp, start_offset) == CHAR_CR && - CODE_UNIT(pp, end_offset) == CHAR_LF) - end_offset++; - - else if (utf && test_mode != PCRE32_MODE) - { - if (test_mode == PCRE8_MODE) - { - for (; end_offset < ulen; end_offset++) - if ((((PCRE2_SPTR8)pp)[end_offset] & 0xc0) != 0x80) break; - } - else /* 16-bit mode */ - { - for (; end_offset < ulen; end_offset++) - if ((((PCRE2_SPTR16)pp)[end_offset] & 0xfc00) != 0xdc00) break; - } - } - - SETFLDVEC(match_data, ovector, 0, start_offset); - SETFLDVEC(match_data, ovector, 1, end_offset); - } /* End of handling null match in a global loop */ - /* A "normal" match failure. There will be a negative error number in capcount. */ @@ -9070,95 +9044,43 @@ for (gmatched = 0;; gmatched++) break; /* Out of the /g loop */ } /* End of failed match handling */ - /* Control reaches here in two circumstances: (a) after a match, and (b) - after a non-match that immediately followed a match on an empty string when - doing a global search. Such a match is done with PCRE2_NOTEMPTY_ATSTART and - PCRE2_ANCHORED set in g_notempty. The code above turns it into a fake match - of one character. So effectively we get here only after a match. If we - are not doing a global search, we are done. */ - - if ((dat_datctl.control & CTL_ANYGLOB) == 0) break; else - { - PCRE2_SIZE match_offset = FLD(match_data, ovector)[0]; - PCRE2_SIZE end_offset = FLD(match_data, ovector)[1]; - - /* We must now set up for the next iteration of a global search. If we have - matched an empty string, first check to see if we are at the end of the - subject. If so, the loop is over. Otherwise, mimic what Perl's /g option - does. Set PCRE2_NOTEMPTY_ATSTART and PCRE2_ANCHORED and try the match again - at the same point. If this fails it will be picked up above, where a fake - match is set up so that at this point we advance to the next character. + /* Control reaches here after a match. If we are not doing a global search, + we are done. Otherwise, we adjust the parameters for the next match and + continue the matching loop. */ - However, in order to cope with patterns that never match at their starting - offset (e.g. /(?<=\G.)/) we don't do this when the match offset is greater - than the starting offset. This means there will be a retry with the - starting offset at the match offset. If this returns the same match again, - it is picked up above and ignored, and the special action is then taken. */ + NEXT_MATCH: - if (match_offset == end_offset) - { - if (end_offset == ulen) break; /* End of subject */ - if (match_offset <= dat_datctl.offset) - g_notempty = PCRE2_NOTEMPTY_ATSTART | PCRE2_ANCHORED; - } + if ((dat_datctl.control & CTL_ANYGLOB) == 0) + break; + else + { + PCRE2_SIZE new_start_offset = (PCRE2_SIZE)-1; + BOOL rc_nextmatch; - /* However, even after matching a non-empty string, there is still one - tricky case. If a pattern contains \K within a lookbehind assertion at the - start, the end of the matched string can be at the offset where the match - started. In the case of a normal /g iteration without special action, this - leads to a loop that keeps on returning the same substring. The loop would - be caught above, but we really want to move on to the next match. */ + /* Use pcre2_next_match() to safely advance. This guarantees that the start + offset will advance, except after an empty match, in which case it sets + the PCRE2_NOTEMPTY_ATSTART flag to ensure the next match does not return a + duplicate. */ - else - { - g_notempty = 0; /* Set for a "normal" repeat */ - if ((dat_datctl.control & CTL_GLOBAL) != 0) - { - PCRE2_SIZE startchar; - PCRE2_GET_STARTCHAR(startchar, match_data); - if (end_offset <= startchar) - { - if (startchar >= ulen) break; /* End of subject */ - end_offset = startchar + 1; - if (utf && test_mode != PCRE32_MODE) - { - if (test_mode == PCRE8_MODE) - { - for (; end_offset < ulen; end_offset++) - if ((((PCRE2_SPTR8)pp)[end_offset] & 0xc0) != 0x80) break; - } - else /* 16-bit mode */ - { - for (; end_offset < ulen; end_offset++) - if ((((PCRE2_SPTR16)pp)[end_offset] & 0xfc00) != 0xdc00) break; - } - } - } - } - } + PCRE2_NEXT_MATCH(rc_nextmatch, match_data, &new_start_offset, &g_notempty); + if (!rc_nextmatch) break; /* Out of the /g loop */ - /* For a normal global (/g) iteration, save the current ovector[0,1] and - the starting offset so that we can check that they do change each time. - Otherwise a matching bug that returns the same string causes an infinite - loop. It has happened! Then update the start offset, leaving other - parameters alone. */ + /* For a normal global (/g) iteration, update the start offset, leaving + other parameters alone. */ if ((dat_datctl.control & CTL_GLOBAL) != 0) { - ovecsave[0] = ovector[0]; - ovecsave[1] = ovector[1]; - ovecsave[2] = dat_datctl.offset; - dat_datctl.offset = end_offset; + dat_datctl.offset = new_start_offset; } /* For altglobal, just update the pointer and length. */ else { - pp += end_offset * code_unit_size; - len -= end_offset * code_unit_size; - ulen -= end_offset; - if (arg_ulen != PCRE2_ZERO_TERMINATED) arg_ulen -= end_offset; + pp += new_start_offset * code_unit_size; + len -= new_start_offset * code_unit_size; + ulen -= new_start_offset; + if (arg_ulen != PCRE2_ZERO_TERMINATED) arg_ulen -= new_start_offset; } } } /* End of global loop */ diff --git a/testdata/testoutput2 b/testdata/testoutput2 index 16169eb0f..7c1bec701 100644 --- a/testdata/testoutput2 +++ b/testdata/testoutput2 @@ -14735,6 +14735,7 @@ Failed: error 125 at offset 1: length of lookbehind assertion is not limited aaaaa 0: a 0+ aaaa +global repeat returned the same match as previous 0: a 0+ aaa 0: a diff --git a/testdata/testoutput5 b/testdata/testoutput5 index 923387ac6..be93726df 100644 --- a/testdata/testoutput5 +++ b/testdata/testoutput5 @@ -4009,6 +4009,7 @@ Subject length lower bound = 1 \x{17f}\x{17f}\x{17f}\x{17f}\x{17f} 0: \x{17f} 0+ \x{17f}\x{17f}\x{17f}\x{17f} +global repeat returned the same match as previous 0: \x{17f} 0+ \x{17f}\x{17f}\x{17f} 0: \x{17f} diff --git a/vms/configure.com b/vms/configure.com index 1f82b5f88..eccb61980 100644 --- a/vms/configure.com +++ b/vms/configure.com @@ -1085,6 +1085,9 @@ PCRE2_MATCH.OBJ : PCRE2_MATCH.C PCRE2_MATCH_DATA.OBJ : PCRE2_MATCH_DATA.C $(CC) $(CFLAGS) $(MMS$SOURCE) /OBJ=$(MMS$TARGET) +PCRE2_MATCH_NEXT.OBJ : PCRE2_MATCH_NEXT.C + $(CC) $(CFLAGS) $(MMS$SOURCE) /OBJ=$(MMS$TARGET) + PCRE2_NEWLINE.OBJ : PCRE2_NEWLINE.C $(CC) $(CFLAGS) $(MMS$SOURCE) /OBJ=$(MMS$TARGET) From 6a33346cf56f152e3f586e487bf900301be35fcb Mon Sep 17 00:00:00 2001 From: Nicholas Wilson Date: Wed, 19 Mar 2025 07:31:31 +0000 Subject: [PATCH 2/6] Finish off the documentation --- doc/pcre2api.3 | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/doc/pcre2api.3 b/doc/pcre2api.3 index 92d92ffae..c7bff5df5 100644 --- a/doc/pcre2api.3 +++ b/doc/pcre2api.3 @@ -3586,9 +3586,19 @@ the issue as a bug). .P Note that we do not guarantee that the matches will always advance: only that the start_offset will. If an application does not use the flag -PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK... XXX +PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK, then matches are "well-behaved" and satisfy: +.sp + start_offset <= ovector[0] <= ovector[1]. +.sp +In this case, the matches found by pcre2_match() with pcre2_next_match() will be +sorted, non-overlapping (possibly touching), and with no duplicates. .P -XXX +Otherwise, if PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK is used, then the guarantees are +considerably weaker. The matches found by pcre2_match() with pcre2_next_match() +will be a finite sequence (as pcre2_next_match() ensures that start_offset +advances, so the search will terminate). The matches can however be overlapping, +not sorted, and contain duplicates. Each match itself can end before it starts +(ovector[1] < ovector[0]). . . .\" HTML From f11775413379cd64c64572c98623c44ba8a60c1d Mon Sep 17 00:00:00 2001 From: Nicholas Wilson Date: Wed, 19 Mar 2025 07:37:54 +0000 Subject: [PATCH 3/6] Update & sync docs --- doc/html/index.html | 3 +- doc/html/pcre2_next_match.html | 54 ++ doc/html/pcre2api.html | 158 ++++-- doc/html/pcre2demo.html | 121 +---- doc/html/pcre2test.html | 10 +- doc/pcre2.txt | 939 ++++++++++++++++++--------------- doc/pcre2api.3 | 4 +- doc/pcre2demo.3 | 123 +---- doc/pcre2test.txt | 464 ++++++++-------- 9 files changed, 944 insertions(+), 932 deletions(-) create mode 100644 doc/html/pcre2_next_match.html diff --git a/doc/html/index.html b/doc/html/index.html index 96b0362aa..f4aa2ea4f 100644 --- a/doc/html/index.html +++ b/doc/html/index.html @@ -220,7 +220,8 @@

Perl-compatible Regular Expressions (revised API: PCRE2)

pcre2_match_data_free Free a match data block -// XXX +pcre2_next_match + Get the match parameters for the next match pcre2_pattern_convert Experimental foreign pattern converter diff --git a/doc/html/pcre2_next_match.html b/doc/html/pcre2_next_match.html new file mode 100644 index 000000000..fddaf9d34 --- /dev/null +++ b/doc/html/pcre2_next_match.html @@ -0,0 +1,54 @@ + + +pcre2_next_match specification + + +

pcre2_next_match man page

+

+Return to the PCRE2 index page. +

+

+This page is part of the PCRE2 HTML documentation. It was generated +automatically from the original man page. If there is any nonsense in it, +please consult the man page, in case the conversion went wrong. +
+

+SYNOPSIS +

+

+#include <pcre2.h> +

+

+int pcre2_next_match(pcre2_match_data *match_data, + PCRE2_SIZE *pstart_offset, uint32_t *poptions); +

+

+DESCRIPTION +

+

+This function can be called after calling one of the matching functions +(pcre2_match(), pcre2_dfa_match(), or pcre2_jit_match()), by +providing the same match_data parameter. It provides the appropriate +parameters for searching for the next match in the same subject string, and is +suitable for applications providing "global" matching behaviour (replacing all +matches in the subject, or splitting the subject on all matches, or simply +counting the number of matches). +

+

+It returns 0 ("false") if there is no need to make any further match attempts, +or 1 ("true") if another match should be attempted. +

+

+The pstart_offset and poptions are set if the function returns 1. +The *poptions should be combined with the application's match options +using OR. +

+

+There is a complete description of the PCRE2 native API in the +pcre2api +page and a description of the POSIX API in the +pcre2posix +page. +

+Return to the PCRE2 index page. +

diff --git a/doc/html/pcre2api.html b/doc/html/pcre2api.html index 6e5f05f7e..faeec8671 100644 --- a/doc/html/pcre2api.html +++ b/doc/html/pcre2api.html @@ -46,16 +46,17 @@

pcre2api man page

  • OTHER INFORMATION ABOUT A MATCH
  • ERROR RETURNS FROM pcre2_match()
  • OBTAINING A TEXTUAL ERROR MESSAGE -
  • EXTRACTING CAPTURED SUBSTRINGS BY NUMBER -
  • EXTRACTING A LIST OF ALL CAPTURED SUBSTRINGS -
  • EXTRACTING CAPTURED SUBSTRINGS BY NAME -
  • CREATING A NEW STRING WITH SUBSTITUTIONS -
  • DUPLICATE CAPTURE GROUP NAMES -
  • FINDING ALL POSSIBLE MATCHES AT ONE POSITION -
  • MATCHING A PATTERN: THE ALTERNATIVE FUNCTION -
  • SEE ALSO -
  • AUTHOR -
  • REVISION +
  • ITERATING OVER ALL MATCHES +
  • EXTRACTING CAPTURED SUBSTRINGS BY NUMBER +
  • EXTRACTING A LIST OF ALL CAPTURED SUBSTRINGS +
  • EXTRACTING CAPTURED SUBSTRINGS BY NAME +
  • CREATING A NEW STRING WITH SUBSTITUTIONS +
  • DUPLICATE CAPTURE GROUP NAMES +
  • FINDING ALL POSSIBLE MATCHES AT ONE POSITION +
  • MATCHING A PATTERN: THE ALTERNATIVE FUNCTION +
  • SEE ALSO +
  • AUTHOR +
  • REVISION

    #include <pcre2.h> @@ -2977,16 +2978,10 @@

    Finding all the matches in a subject is tricky when the pattern can match an -empty string. It is possible to emulate Perl's /g behaviour by first trying the -match again at the same offset, with the PCRE2_NOTEMPTY_ATSTART and -PCRE2_ANCHORED options, and then if that fails, advancing the starting offset -and trying an ordinary match again. There is some code that demonstrates how to -do this in the -pcre2demo -sample program. In the most general case, you have to check to see if the -newline convention recognizes CRLF as a newline, and if so, and the current -character is CR followed by LF, advance the starting offset by two characters -instead of one. +empty string. PCRE2 includes a helper API to assist with this; see the +section entitled "Iterating over all matches" +below +for details.

    If a non-zero starting offset is passed when the pattern is anchored, a single @@ -3282,10 +3277,10 @@

    HOW PCRE2_MATCH() RETURNS A STRING AND CAPTURED match is 1, indicating that just the first pair of offsets has been set.

    -If a pattern uses the \K escape sequence within a positive assertion, the -reported start of a successful match can be greater than the end of the match. -For example, if the pattern (?=ab\K) is matched against "ab", the start and -end offset values for the match are 2 and 0. +If a pattern uses the \K escape sequence within a positive lookahead assertion, +the reported start of a successful match can be greater than the end of the +match. For example, if the pattern (?=ab\K) is matched against "ab", the start +and end offset values for the match are 2 and 0.

    If a capture group is matched repeatedly within a single match operation, it is @@ -3509,9 +3504,89 @@

    OBTAINING A TEXTUAL ERROR MESSAGE

    error number is unknown, the negative error code PCRE2_ERROR_BADDATA is returned. If the buffer is too small, the message is truncated (but still with a trailing zero), and the negative error code PCRE2_ERROR_NOMEMORY is returned. -None of the messages are very long; a buffer size of 120 code units is ample. +None of the messages is very long; a buffer size of 120 code units is ample. +

    +

    ITERATING OVER ALL MATCHES

    +

    +int pcre2_next_match(pcre2_match_data *match_data, + PCRE2_SIZE *pstart_offset, uint32_t *poptions); +

    +

    +A common task for applications is to implement "global" matching behaviour, +for example, replacing all matches in the subject; splitting the subject on all +matches; or simply counting the number of matches. The pcre2_next_match() +function helps with this task by providing the appropriate parameters for the +next match attempt. +

    +

    +First, a match attempt should be made using one of the matching functions +(pcre2_match(), pcre2_dfa_match(), or pcre2_jit_match()). Then, +pcre2_next_match() can be called, providing the same match_data +parameter. +

    +

    +It returns 0 ("false") if there is no need to make any further match attempts, +or 1 ("true") if another match should be attempted. +

    +

    +The pstart_offset and poptions are set if the function returns 1. +The *poptions should be combined with the application's match options +using OR. +

    +

    +There is some code that demonstrates how to do this in the +pcre2demo +sample program. The general pattern is: +

    +  uint32_t app_options = ...;
+  uint32_t global_options = 0;
+  PCRE2_SIZE start_offset = 0;
+  while (1)
+    {
+    int rc = pcre2_match(re, subject, subject_len, start_offset,
+                         app_options | global_options, match_data,
+                         match_context);
+    if (rc == PCRE2_ERROR_NOMATCH) break;
+    if (rc < 0) { ... exit }
+    if (!pcre2_next_match(match_data, &start_offset, &global_options))
+      break;
+    }
+
    +

    +

    +The guarantees provided by pcre2_next_match() are that the start_offset +will advance, so the loop will definitely terminate. The condition for progress +is that either: (a) pcre2_next_match() returns 0 (false); or (b) the returned +start_offset is strictly greater than the previous start_offset, or (c) if the +previous match was a successful match of the empty string then the returned +start_offset is equal to previous start_offset, but poptions will be set to +PCRE2_NOTEMPTY_ATSTART to prevent another empty match from being returned. +

    +

    +In a loop as shown above, it must terminate, unless there is a bug in PCRE2. As +a measure of "defensive programming", applications are encouraged to add an +assertion or check to break their loop if it does not make progress (and report +the issue as a bug). +

    +

    +Note that we do not guarantee that the matches will always advance: only that +the start_offset will. If an application does not use the flag +PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK, then matches are "well-behaved" and satisfy: +

    +  start_offset <= ovector[0] <= ovector[1].
+
    +In this case, the matches found by pcre2_match() with pcre2_next_match() will be +sorted, non-overlapping (possibly touching), and with no duplicates. +

    +

    +Otherwise, if PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK is used, then the guarantees are +considerably weaker. The matches found by pcre2_match() with pcre2_next_match() +will be a finite sequence (as pcre2_next_match() ensures that start_offset +advances, so the search will terminate). The matches can however be overlapping, +not sorted, and contain duplicates. Each match itself can end before it starts +(ovector[1] < ovector[0]).

    -

    EXTRACTING CAPTURED SUBSTRINGS BY NUMBER

    +

    EXTRACTING CAPTURED SUBSTRINGS BY NUMBER

    int pcre2_substring_length_bynumber(pcre2_match_data *match_data, uint32_t number, PCRE2_SIZE *length); @@ -3546,11 +3621,12 @@

    EXTRACTING CAPTURED SUBSTRINGS BY NUMBER

    -If a pattern uses the \K escape sequence within a positive assertion, the -reported start of a successful match can be greater than the end of the match. -For example, if the pattern (?=ab\K) is matched against "ab", the start and -end offset values for the match are 2 and 0. In this situation, calling these -functions with a zero substring number extracts a zero-length empty string. +If a pattern uses the \K escape sequence within a positive lookahead assertion, +the reported start of a successful match can be greater than the end of the +match. For example, if the pattern (?=ab\K) is matched against "ab", the start +and end offset values for the match are 2 and 0. In this situation, calling +these functions with a zero substring number extracts a zero-length empty +string.

    You can find the length in code units of a captured substring without @@ -3608,7 +3684,7 @@

    EXTRACTING CAPTURED SUBSTRINGS BY NUMBER -

    EXTRACTING A LIST OF ALL CAPTURED SUBSTRINGS

    +

    EXTRACTING A LIST OF ALL CAPTURED SUBSTRINGS

    int pcre2_substring_list_get(pcre2_match_data *match_data, " PCRE2_UCHAR ***listptr, PCRE2_SIZE **lengthsptr); @@ -3647,7 +3723,7 @@

    EXTRACTING A LIST OF ALL CAPTURED SUBSTRINGSpcre2_substring_length_bynumber().

    -

    EXTRACTING CAPTURED SUBSTRINGS BY NAME

    +

    EXTRACTING CAPTURED SUBSTRINGS BY NAME

    int pcre2_substring_number_from_name(const pcre2_code *code, PCRE2_SPTR name); @@ -3707,7 +3783,7 @@

    EXTRACTING CAPTURED SUBSTRINGS BY NAME

    numbers. For this reason, the use of different names for groups with the same number causes an error at compile time.

    -

    CREATING A NEW STRING WITH SUBSTITUTIONS

    +

    CREATING A NEW STRING WITH SUBSTITUTIONS

    int pcre2_substitute(const pcre2_code *code, PCRE2_SPTR subject, PCRE2_SIZE length, PCRE2_SIZE startoffset, @@ -4031,7 +4107,7 @@

    not found), PCRE2_ERROR_BADSUBSTITUTION (syntax error in extended group substitution), and PCRE2_ERROR_BADSUBSPATTERN (the pattern match ended before it started or the match started earlier than the current position in the -subject, which can happen if \K is used in an assertion). +subject, which can happen if \K is used in a lookaround assertion).

    As for all PCRE2 errors, a text message that describes the error can be @@ -4212,7 +4288,7 @@

    more buffer space than expected. The caller must make repeated attempts in a loop.

    -

    DUPLICATE CAPTURE GROUP NAMES

    +

    DUPLICATE CAPTURE GROUP NAMES

    int pcre2_substring_nametable_scan(const pcre2_code *code, PCRE2_SPTR name, PCRE2_SPTR *first, PCRE2_SPTR *last); @@ -4258,7 +4334,7 @@

    DUPLICATE CAPTURE GROUP NAMES

    relevant entries for the name, you can extract each of their numbers, and hence the captured data.

    -

    FINDING ALL POSSIBLE MATCHES AT ONE POSITION

    +

    FINDING ALL POSSIBLE MATCHES AT ONE POSITION

    The traditional matching function uses a similar algorithm to Perl, which stops when it finds the first match at a given point in the subject. If you want to @@ -4276,7 +4352,7 @@

    FINDING ALL POSSIBLE MATCHES AT ONE POSITIONpcre2_match() will yield PCRE2_ERROR_NOMATCH.

    -

    MATCHING A PATTERN: THE ALTERNATIVE FUNCTION

    +

    MATCHING A PATTERN: THE ALTERNATIVE FUNCTION

    int pcre2_dfa_match(const pcre2_code *code, PCRE2_SPTR subject, PCRE2_SIZE length, PCRE2_SIZE startoffset, @@ -4469,13 +4545,13 @@

    should contain data about the previous partial match. If any of these checks fail, this error is given.

    -

    SEE ALSO

    +

    SEE ALSO

    pcre2build(3), pcre2callout(3), pcre2demo(3), pcre2matching(3), pcre2partial(3), pcre2posix(3), pcre2sample(3), pcre2unicode(3).

    -

    AUTHOR

    +

    AUTHOR

    Philip Hazel
    @@ -4484,7 +4560,7 @@

    AUTHOR

    Cambridge, England.

    -

    REVISION

    +

    REVISION

    Last updated: 26 December 2024
    diff --git a/doc/html/pcre2demo.html b/doc/html/pcre2demo.html index a2f93315d..0db6df437 100644 --- a/doc/html/pcre2demo.html +++ b/doc/html/pcre2demo.html @@ -91,17 +91,13 @@

    PCRE2_SPTR subject; /* the appropriate width (in this case, 8 bits). */ PCRE2_SPTR name_table; -int crlf_is_newline; int errornumber; int find_all, caseless_match; int i; int rc; -int utf8; -uint32_t option_bits; uint32_t namecount; uint32_t name_entry_size; -uint32_t newline; PCRE2_SIZE erroroffset; PCRE2_SIZE *ovector; @@ -284,7 +280,9 @@

    PCRE2_INFO_NAMECOUNT, /* get the number of named substrings */ &namecount); /* where to put the answer */ -if (namecount == 0) printf("No named substrings\n"); else +if (namecount == 0) + printf("No named substrings\n"); +else { PCRE2_SPTR tabptr; printf("Named substrings\n"); @@ -322,28 +320,8 @@

    * to search for additional matches in the subject string, in a similar * * way to the /g option in Perl. This turns out to be trickier than you * * might think because of the possibility of matching an empty string. * -* What happens is as follows: * * * -* If the previous match was NOT for an empty string, we can just start * -* the next match at the end of the previous one. * -* * -* If the previous match WAS for an empty string, we can't do that, as it * -* would lead to an infinite loop. Instead, a call of pcre2_match() is * -* made with the PCRE2_NOTEMPTY_ATSTART and PCRE2_ANCHORED flags set. The * -* first of these tells PCRE2 that an empty string at the start of the * -* subject is not a valid match; other possibilities must be tried. The * -* second flag restricts PCRE2 to one match attempt at the initial string * -* position. If this match succeeds, an alternative to the empty string * -* match has been found, and we can print it and proceed round the loop, * -* advancing by the length of whatever was found. If this match does not * -* succeed, we still stay in the loop, advancing by just one character. * -* In UTF-8 mode, which can be set by (*UTF) in the pattern, this may be * -* more than one byte. * -* * -* However, there is a complication concerned with newlines. When the * -* newline convention is such that CRLF is a valid newline, we must * -* advance by two characters rather than one. The newline convention can * -* be set in the regex by (*CR), etc.; if not, we must find the default. * +* To help with this task, PCRE2 provides the pcre2_next_match() helper. * *************************************************************************/ if (!find_all) /* Check for -g */ @@ -353,60 +331,18 @@

    return 0; /* Exit the program. */ } -/* Before running the loop, check for UTF-8 and whether CRLF is a valid newline -sequence. First, find the options with which the regex was compiled and extract -the UTF state. */ - -(void)pcre2_pattern_info(re, PCRE2_INFO_ALLOPTIONS, &option_bits); -utf8 = (option_bits & PCRE2_UTF) != 0; - -/* Now find the newline convention and see whether CRLF is a valid newline -sequence. */ - -(void)pcre2_pattern_info(re, PCRE2_INFO_NEWLINE, &newline); -crlf_is_newline = newline == PCRE2_NEWLINE_ANY || - newline == PCRE2_NEWLINE_CRLF || - newline == PCRE2_NEWLINE_ANYCRLF; - /* Loop for second and subsequent matches */ for (;;) { - uint32_t options = 0; /* Normally no options */ - PCRE2_SIZE start_offset = ovector[1]; /* Start at end of previous match */ + PCRE2_SIZE start_offset; + uint32_t options; - /* If the previous match was for an empty string, we are finished if we are - at the end of the subject. Otherwise, arrange to run another match at the - same point to see if a non-empty match can be found. */ + /* After each successful match, we use pcre2_next_match() to obtain the match + parameters for subsequent match attempts. */ - if (ovector[0] == ovector[1]) - { - if (ovector[0] == subject_length) break; - options = PCRE2_NOTEMPTY_ATSTART | PCRE2_ANCHORED; - } - - /* If the previous match was not an empty string, there is one tricky case to - consider. If a pattern contains \K within a lookbehind assertion at the - start, the end of the matched string can be at the offset where the match - started. Without special action, this leads to a loop that keeps on matching - the same substring. We must detect this case and arrange to move the start on - by one character. The pcre2_get_startchar() function returns the starting - offset that was passed to pcre2_match(). */ - - else - { - PCRE2_SIZE startchar = pcre2_get_startchar(match_data); - if (start_offset <= startchar) - { - if (startchar >= subject_length) break; /* Reached end of subject. */ - start_offset = startchar + 1; /* Advance by one character. */ - if (utf8) /* If UTF-8, it may be more */ - { /* than one code unit. */ - for (; start_offset < subject_length; start_offset++) - if ((subject[start_offset] & 0xc0) != 0x80) break; - } - } - } + if (!pcre2_next_match(match_data, &start_offset, &options)) + break; /* Run the next matching operation */ @@ -419,38 +355,10 @@

    match_data, /* block for storing the result */ NULL); /* use default match context */ - /* This time, a result of NOMATCH isn't an error. If the value in "options" - is zero, it just means we have found all possible matches, so the loop ends. - Otherwise, it means we have failed to find a non-empty-string match at a - point where there was a previous empty-string match. In this case, we do what - Perl does: advance the matching position by one character, and continue. We - do this by setting the "end of previous match" offset, because that is picked - up at the top of the loop as the point at which to start again. - - There are two complications: (a) When CRLF is a valid newline sequence, and - the current position is just before it, advance by an extra byte. (b) - Otherwise we must ensure that we skip an entire UTF character if we are in - UTF mode. */ + /* If this match attempt fails, exit the loop for subsequent matches. */ if (rc == PCRE2_ERROR_NOMATCH) - { - if (options == 0) break; /* All matches found */ - ovector[1] = start_offset + 1; /* Advance one code unit */ - if (crlf_is_newline && /* If CRLF is a newline & */ - start_offset < subject_length - 1 && /* we are at CRLF, */ - subject[start_offset] == '\r' && - subject[start_offset + 1] == '\n') - ovector[1] += 1; /* Advance by one more. */ - else if (utf8) /* Otherwise, ensure we */ - { /* advance a whole UTF-8 */ - while (ovector[1] < subject_length) /* character. */ - { - if ((subject[ovector[1]] & 0xc0) != 0x80) break; - ovector[1] += 1; - } - } - continue; /* Go round the loop again */ - } + break; /* Other matching errors are not recoverable. */ @@ -497,7 +405,9 @@

    printf("%2d: %.*s\n", i, (int)substring_length, (char *)substring_start); } - if (namecount == 0) printf("No named substrings\n"); else + if (namecount == 0) + printf("No named substrings\n"); + else { PCRE2_SPTR tabptr = name_table; printf("Named substrings\n"); @@ -512,6 +422,7 @@

    } /* End of loop to find second and subsequent matches */ printf("\n"); + pcre2_match_data_free(match_data); pcre2_code_free(re); return 0; diff --git a/doc/html/pcre2test.html b/doc/html/pcre2test.html index 9fe6bab8c..440c30d07 100644 --- a/doc/html/pcre2test.html +++ b/doc/html/pcre2test.html @@ -1499,13 +1499,9 @@

    If an empty string is matched, the next match is done with the -PCRE2_NOTEMPTY_ATSTART and PCRE2_ANCHORED flags set, in order to search for -another, non-empty, match at the same point in the subject. If this match -fails, the start offset is advanced, and the normal match is retried. This -imitates the way Perl handles such cases when using the /g modifier or -the split() function. Normally, the start offset is advanced by one -character, but if the newline convention recognizes CRLF as a newline, and the -current character is CR followed by LF, an advance of two characters occurs. +PCRE2_NOTEMPTY_ATSTART flag set, in order to search for another, non-empty, +match at the same point in the subject. This imitates the way Perl handles such +cases when using the /g modifier or the split() function.

    Testing substring extraction functions diff --git a/doc/pcre2.txt b/doc/pcre2.txt index 14342f0af..c65cfcf52 100644 --- a/doc/pcre2.txt +++ b/doc/pcre2.txt @@ -2900,15 +2900,9 @@ MATCHING A PATTERN: THE TRADITIONAL FUNCTION starting point to discover that it is preceded by a letter. Finding all the matches in a subject is tricky when the pattern can - match an empty string. It is possible to emulate Perl's /g behaviour by - first trying the match again at the same offset, with the - PCRE2_NOTEMPTY_ATSTART and PCRE2_ANCHORED options, and then if that - fails, advancing the starting offset and trying an ordinary match - again. There is some code that demonstrates how to do this in the - pcre2demo sample program. In the most general case, you have to check - to see if the newline convention recognizes CRLF as a newline, and if - so, and the current character is CR followed by LF, advance the start- - ing offset by two characters instead of one. + match an empty string. PCRE2 includes a helper API to assist with this; + see the section entitled "Iterating over all matches" below for de- + tails. If a non-zero starting offset is passed when the pattern is anchored, a single attempt to match at the given offset is made. This can only suc- @@ -3178,41 +3172,42 @@ HOW PCRE2_MATCH() RETURNS A STRING AND CAPTURED SUBSTRINGS strings, the return value from a successful match is 1, indicating that just the first pair of offsets has been set. - If a pattern uses the \K escape sequence within a positive assertion, - the reported start of a successful match can be greater than the end of - the match. For example, if the pattern (?=ab\K) is matched against - "ab", the start and end offset values for the match are 2 and 0. + If a pattern uses the \K escape sequence within a positive lookahead + assertion, the reported start of a successful match can be greater than + the end of the match. For example, if the pattern (?=ab\K) is matched + against "ab", the start and end offset values for the match are 2 and + 0. - If a capture group is matched repeatedly within a single match opera- + If a capture group is matched repeatedly within a single match opera- tion, it is the last portion of the subject that it matched that is re- turned. If the ovector is too small to hold all the captured substring offsets, - as much as possible is filled in, and the function returns a value of - zero. If captured substrings are not of interest, pcre2_match() may be + as much as possible is filled in, and the function returns a value of + zero. If captured substrings are not of interest, pcre2_match() may be called with a match data block whose ovector is of minimum length (that is, one pair). - It is possible for capture group number n+1 to match some part of the - subject when group n has not been used at all. For example, if the + It is possible for capture group number n+1 to match some part of the + subject when group n has not been used at all. For example, if the string "abc" is matched against the pattern (a|(z))(bc) the return from - the function is 4, and groups 1 and 3 are matched, but 2 is not. When - this happens, both values in the offset pairs corresponding to unused + the function is 4, and groups 1 and 3 are matched, but 2 is not. When + this happens, both values in the offset pairs corresponding to unused groups are set to PCRE2_UNSET. - Offset values that correspond to unused groups at the end of the ex- - pression are also set to PCRE2_UNSET. For example, if the string "abc" - is matched against the pattern (abc)(x(yz)?)? groups 2 and 3 are not - matched. The return from the function is 2, because the highest used + Offset values that correspond to unused groups at the end of the ex- + pression are also set to PCRE2_UNSET. For example, if the string "abc" + is matched against the pattern (abc)(x(yz)?)? groups 2 and 3 are not + matched. The return from the function is 2, because the highest used capture group number is 1. The offsets for the second and third capture - groups (assuming the vector is large enough, of course) are set to + groups (assuming the vector is large enough, of course) are set to PCRE2_UNSET. Elements in the ovector that do not correspond to capturing parentheses in the pattern are never changed. That is, if a pattern contains n cap- turing parentheses, no more than ovector[0] to ovector[2n+1] are set by - pcre2_match(). The other elements retain whatever values they previ- - ously had. After a failed match attempt, the contents of the ovector + pcre2_match(). The other elements retain whatever values they previ- + ously had. After a failed match attempt, the contents of the ovector are unchanged. @@ -3222,69 +3217,69 @@ OTHER INFORMATION ABOUT A MATCH PCRE2_SIZE pcre2_get_startchar(pcre2_match_data *match_data); - As well as the offsets in the ovector, other information about a match - is retained in the match data block and can be retrieved by the above - functions in appropriate circumstances. If they are called at other + As well as the offsets in the ovector, other information about a match + is retained in the match data block and can be retrieved by the above + functions in appropriate circumstances. If they are called at other times, the result is undefined. - After a successful match, a partial match (PCRE2_ERROR_PARTIAL), or a - failure to match (PCRE2_ERROR_NOMATCH), a mark name may be available. - The function pcre2_get_mark() can be called to access this name, which - can be specified in the pattern by any of the backtracking control + After a successful match, a partial match (PCRE2_ERROR_PARTIAL), or a + failure to match (PCRE2_ERROR_NOMATCH), a mark name may be available. + The function pcre2_get_mark() can be called to access this name, which + can be specified in the pattern by any of the backtracking control verbs, not just (*MARK). The same function applies to all the verbs. It returns a pointer to the zero-terminated name, which is within the com- piled pattern. If no name is available, NULL is returned. The length of - the name (excluding the terminating zero) is stored in the code unit - that precedes the name. You should use this length instead of relying + the name (excluding the terminating zero) is stored in the code unit + that precedes the name. You should use this length instead of relying on the terminating zero if the name might contain a binary zero. - After a successful match, the name that is returned is the last mark + After a successful match, the name that is returned is the last mark name encountered on the matching path through the pattern. Instances of - backtracking verbs without names do not count. Thus, for example, if + backtracking verbs without names do not count. Thus, for example, if the matching path contains (*MARK:A)(*PRUNE), the name "A" is returned. After a "no match" or a partial match, the last encountered name is re- turned. For example, consider this pattern: ^(*MARK:A)((*MARK:B)a|b)c - When it matches "bc", the returned name is A. The B mark is "seen" in - the first branch of the group, but it is not on the matching path. On - the other hand, when this pattern fails to match "bx", the returned + When it matches "bc", the returned name is A. The B mark is "seen" in + the first branch of the group, but it is not on the matching path. On + the other hand, when this pattern fails to match "bx", the returned name is B. - Warning: By default, certain start-of-match optimizations are used to - give a fast "no match" result in some situations. For example, if the - anchoring is removed from the pattern above, there is an initial check - for the presence of "c" in the subject before running the matching en- + Warning: By default, certain start-of-match optimizations are used to + give a fast "no match" result in some situations. For example, if the + anchoring is removed from the pattern above, there is an initial check + for the presence of "c" in the subject before running the matching en- gine. This check fails for "bx", causing a match failure without seeing - any marks. You can disable the start-of-match optimizations by setting - the PCRE2_NO_START_OPTIMIZE option for pcre2_compile() or by starting + any marks. You can disable the start-of-match optimizations by setting + the PCRE2_NO_START_OPTIMIZE option for pcre2_compile() or by starting the pattern with (*NO_START_OPT). - After a successful match, a partial match, or one of the invalid UTF - errors (for example, PCRE2_ERROR_UTF8_ERR5), pcre2_get_startchar() can + After a successful match, a partial match, or one of the invalid UTF + errors (for example, PCRE2_ERROR_UTF8_ERR5), pcre2_get_startchar() can be called. After a successful or partial match it returns the code unit - offset of the character at which the match started. For a non-partial - match, this can be different to the value of ovector[0] if the pattern - contains the \K escape sequence. After a partial match, however, this - value is always the same as ovector[0] because \K does not affect the + offset of the character at which the match started. For a non-partial + match, this can be different to the value of ovector[0] if the pattern + contains the \K escape sequence. After a partial match, however, this + value is always the same as ovector[0] because \K does not affect the result of a partial match. - After a UTF check failure, pcre2_get_startchar() can be used to obtain + After a UTF check failure, pcre2_get_startchar() can be used to obtain the code unit offset of the invalid UTF character. Details are given in the pcre2unicode page. ERROR RETURNS FROM pcre2_match() - If pcre2_match() fails, it returns a negative number. This can be con- - verted to a text string by calling the pcre2_get_error_message() func- - tion (see "Obtaining a textual error message" below). Negative error - codes are also returned by other functions, and are documented with - them. The codes are given names in the header file. If UTF checking is + If pcre2_match() fails, it returns a negative number. This can be con- + verted to a text string by calling the pcre2_get_error_message() func- + tion (see "Obtaining a textual error message" below). Negative error + codes are also returned by other functions, and are documented with + them. The codes are given names in the header file. If UTF checking is in force and an invalid UTF subject string is detected, one of a number - of UTF-specific negative error codes is returned. Details are given in - the pcre2unicode page. The following are the other errors that may be + of UTF-specific negative error codes is returned. Details are given in + the pcre2unicode page. The following are the other errors that may be returned by pcre2_match(): PCRE2_ERROR_NOMATCH @@ -3293,20 +3288,20 @@ ERROR RETURNS FROM pcre2_match() PCRE2_ERROR_PARTIAL - The subject string did not match, but it did match partially. See the + The subject string did not match, but it did match partially. See the pcre2partial documentation for details of partial matching. PCRE2_ERROR_BADMAGIC PCRE2 stores a 4-byte "magic number" at the start of the compiled code, - to catch the case when it is passed a junk pointer. This is the error + to catch the case when it is passed a junk pointer. This is the error that is returned when the magic number is not present. PCRE2_ERROR_BADMODE - This error is given when a compiled pattern is passed to a function in - a library of a different code unit width, for example, a pattern com- - piled by the 8-bit library is passed to a 16-bit or 32-bit library + This error is given when a compiled pattern is passed to a function in + a library of a different code unit width, for example, a pattern com- + piled by the 8-bit library is passed to a 16-bit or 32-bit library function. PCRE2_ERROR_BADOFFSET @@ -3320,15 +3315,15 @@ ERROR RETURNS FROM pcre2_match() PCRE2_ERROR_BADUTFOFFSET The UTF code unit sequence that was passed as a subject was checked and - found to be valid (the PCRE2_NO_UTF_CHECK option was not set), but the - value of startoffset did not point to the beginning of a UTF character + found to be valid (the PCRE2_NO_UTF_CHECK option was not set), but the + value of startoffset did not point to the beginning of a UTF character or the end of the subject. PCRE2_ERROR_CALLOUT - This error is never generated by pcre2_match() itself. It is provided - for use by callout functions that want to cause pcre2_match() or - pcre2_callout_enumerate() to return a distinctive error code. See the + This error is never generated by pcre2_match() itself. It is provided + for use by callout functions that want to cause pcre2_match() or + pcre2_callout_enumerate() to return a distinctive error code. See the pcre2callout documentation for details. PCRE2_ERROR_DEPTHLIMIT @@ -3341,14 +3336,14 @@ ERROR RETURNS FROM pcre2_match() PCRE2_ERROR_INTERNAL - An unexpected internal error has occurred. This error could be caused + An unexpected internal error has occurred. This error could be caused by a bug in PCRE2 or by overwriting of the compiled pattern. PCRE2_ERROR_JIT_STACKLIMIT This error is returned when a pattern that was successfully studied us- ing JIT is being matched, but the memory available for the just-in-time - processing stack is not large enough. See the pcre2jit documentation + processing stack is not large enough. See the pcre2jit documentation for more details. PCRE2_ERROR_MATCHLIMIT @@ -3357,11 +3352,11 @@ ERROR RETURNS FROM pcre2_match() PCRE2_ERROR_NOMEMORY - Heap memory is used to remember backtracking points. This error is - given when the memory allocation function (default or custom) fails. - Note that a different error, PCRE2_ERROR_HEAPLIMIT, is given if the + Heap memory is used to remember backtracking points. This error is + given when the memory allocation function (default or custom) fails. + Note that a different error, PCRE2_ERROR_HEAPLIMIT, is given if the amount of memory needed exceeds the heap limit. PCRE2_ERROR_NOMEMORY is - also returned if PCRE2_COPY_MATCHED_SUBJECT is set and memory alloca- + also returned if PCRE2_COPY_MATCHED_SUBJECT is set and memory alloca- tion fails. PCRE2_ERROR_NULL @@ -3370,12 +3365,12 @@ ERROR RETURNS FROM pcre2_match() PCRE2_ERROR_RECURSELOOP - This error is returned when pcre2_match() detects a recursion loop - within the pattern. Specifically, it means that either the whole pat- + This error is returned when pcre2_match() detects a recursion loop + within the pattern. Specifically, it means that either the whole pat- tern or a capture group has been called recursively for the second time - at the same position in the subject string. Some simple patterns that - might do this are detected and faulted at compile time, but more com- - plicated cases, in particular mutual recursions between two different + at the same position in the subject string. Some simple patterns that + might do this are detected and faulted at compile time, but more com- + plicated cases, in particular mutual recursions between two different groups, cannot be detected until matching is attempted. @@ -3384,21 +3379,93 @@ OBTAINING A TEXTUAL ERROR MESSAGE int pcre2_get_error_message(int errorcode, PCRE2_UCHAR *buffer, PCRE2_SIZE bufflen); - A text message for an error code from any PCRE2 function (compile, - match, or auxiliary) can be obtained by calling pcre2_get_error_mes- - sage(). The code is passed as the first argument, with the remaining - two arguments specifying a code unit buffer and its length in code - units, into which the text message is placed. The message is returned - in code units of the appropriate width for the library that is being + A text message for an error code from any PCRE2 function (compile, + match, or auxiliary) can be obtained by calling pcre2_get_error_mes- + sage(). The code is passed as the first argument, with the remaining + two arguments specifying a code unit buffer and its length in code + units, into which the text message is placed. The message is returned + in code units of the appropriate width for the library that is being used. - The returned message is terminated with a trailing zero, and the func- - tion returns the number of code units used, excluding the trailing + The returned message is terminated with a trailing zero, and the func- + tion returns the number of code units used, excluding the trailing zero. If the error number is unknown, the negative error code PCRE2_ER- - ROR_BADDATA is returned. If the buffer is too small, the message is + ROR_BADDATA is returned. If the buffer is too small, the message is truncated (but still with a trailing zero), and the negative error code - PCRE2_ERROR_NOMEMORY is returned. None of the messages are very long; - a buffer size of 120 code units is ample. + PCRE2_ERROR_NOMEMORY is returned. None of the messages is very long; a + buffer size of 120 code units is ample. + + +ITERATING OVER ALL MATCHES + + int pcre2_next_match(pcre2_match_data *match_data, + PCRE2_SIZE *pstart_offset, uint32_t *poptions); + + A common task for applications is to implement "global" matching behav- + iour, for example, replacing all matches in the subject; splitting the + subject on all matches; or simply counting the number of matches. The + pcre2_next_match() function helps with this task by providing the ap- + propriate parameters for the next match attempt. + + First, a match attempt should be made using one of the matching func- + tions (pcre2_match(), pcre2_dfa_match(), or pcre2_jit_match()). Then, + pcre2_next_match() can be called, providing the same match_data parame- + ter. + + It returns 0 ("false") if there is no need to make any further match + attempts, or 1 ("true") if another match should be attempted. + + The pstart_offset and poptions are set if the function returns 1. The + *poptions should be combined with the application's match options using + OR. + + There is some code that demonstrates how to do this in the pcre2demo + sample program. The general pattern is: + uint32_t app_options = ...; + uint32_t global_options = 0; + PCRE2_SIZE start_offset = 0; + while (1) + { + int rc = pcre2_match(re, subject, subject_len, start_offset, + app_options | global_options, match_data, + match_context); + if (rc == PCRE2_ERROR_NOMATCH) break; + if (rc < 0) { ... exit } + if (!pcre2_next_match(match_data, &start_offset, &global_options)) + break; + } + + The guarantees provided by pcre2_next_match() are that the start_offset + will advance, so the loop will definitely terminate. The condition for + progress is that either: (a) pcre2_next_match() returns 0 (false); or + (b) the returned start_offset is strictly greater than the previous + start_offset, or (c) if the previous match was a successful match of + the empty string then the returned start_offset is equal to previous + start_offset, but poptions will be set to PCRE2_NOTEMPTY_ATSTART to + prevent another empty match from being returned. + + In a loop as shown above, it must terminate, unless there is a bug in + PCRE2. As a measure of "defensive programming", applications are en- + couraged to add an assertion or check to break their loop if it does + not make progress (and report the issue as a bug). + + Note that we do not guarantee that the matches will always advance: + only that the start_offset will. If an application does not use the + flag PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK, then matches are "well-behaved" + and satisfy: + + start_offset <= ovector[0] <= ovector[1]. + + In this case, the matches found by pcre2_match() with + pcre2_next_match() will be sorted, non-overlapping (possibly touching), + and with no duplicates. + + Otherwise, if PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK is used, then the guar- + antees are considerably weaker. The matches found by pcre2_match() with + pcre2_next_match() will be a finite sequence (as pcre2_next_match() en- + sures that start_offset advances, so the search will terminate). The + matches can however be overlapping, not sorted, and contain duplicates. + Each match itself can end before it starts (ovector[1] < ovector[0]). EXTRACTING CAPTURED SUBSTRINGS BY NUMBER @@ -3416,39 +3483,39 @@ EXTRACTING CAPTURED SUBSTRINGS BY NUMBER void pcre2_substring_free(PCRE2_UCHAR *buffer); - Captured substrings can be accessed directly by using the ovector as + Captured substrings can be accessed directly by using the ovector as described above. For convenience, auxiliary functions are provided for - extracting captured substrings as new, separate, zero-terminated + extracting captured substrings as new, separate, zero-terminated strings. A substring that contains a binary zero is correctly extracted - and has a further zero added on the end, but the result is not, of + and has a further zero added on the end, but the result is not, of course, a C string. The functions in this section identify substrings by number. The number zero refers to the entire matched substring, with higher numbers refer- - ring to substrings captured by parenthesized groups. After a partial - match, only substring zero is available. An attempt to extract any - other substring gives the error PCRE2_ERROR_PARTIAL. The next section + ring to substrings captured by parenthesized groups. After a partial + match, only substring zero is available. An attempt to extract any + other substring gives the error PCRE2_ERROR_PARTIAL. The next section describes similar functions for extracting captured substrings by name. - If a pattern uses the \K escape sequence within a positive assertion, - the reported start of a successful match can be greater than the end of - the match. For example, if the pattern (?=ab\K) is matched against - "ab", the start and end offset values for the match are 2 and 0. In - this situation, calling these functions with a zero substring number - extracts a zero-length empty string. - - You can find the length in code units of a captured substring without - extracting it by calling pcre2_substring_length_bynumber(). The first - argument is a pointer to the match data block, the second is the group - number, and the third is a pointer to a variable into which the length - is placed. If you just want to know whether or not the substring has + If a pattern uses the \K escape sequence within a positive lookahead + assertion, the reported start of a successful match can be greater than + the end of the match. For example, if the pattern (?=ab\K) is matched + against "ab", the start and end offset values for the match are 2 and + 0. In this situation, calling these functions with a zero substring + number extracts a zero-length empty string. + + You can find the length in code units of a captured substring without + extracting it by calling pcre2_substring_length_bynumber(). The first + argument is a pointer to the match data block, the second is the group + number, and the third is a pointer to a variable into which the length + is placed. If you just want to know whether or not the substring has been captured, you can pass the third argument as NULL. - The pcre2_substring_copy_bynumber() function copies a captured sub- - string into a supplied buffer, whereas pcre2_substring_get_bynumber() - copies it into new memory, obtained using the same memory allocation - function that was used for the match data block. The first two argu- - ments of these functions are a pointer to the match data block and a + The pcre2_substring_copy_bynumber() function copies a captured sub- + string into a supplied buffer, whereas pcre2_substring_get_bynumber() + copies it into new memory, obtained using the same memory allocation + function that was used for the match data block. The first two argu- + ments of these functions are a pointer to the match data block and a capture group number. The final arguments of pcre2_substring_copy_bynumber() are a pointer to @@ -3457,25 +3524,25 @@ EXTRACTING CAPTURED SUBSTRINGS BY NUMBER for the extracted substring, excluding the terminating zero. For pcre2_substring_get_bynumber() the third and fourth arguments point - to variables that are updated with a pointer to the new memory and the - number of code units that comprise the substring, again excluding the - terminating zero. When the substring is no longer needed, the memory + to variables that are updated with a pointer to the new memory and the + number of code units that comprise the substring, again excluding the + terminating zero. When the substring is no longer needed, the memory should be freed by calling pcre2_substring_free(). - The return value from all these functions is zero for success, or a - negative error code. If the pattern match failed, the match failure - code is returned. If a substring number greater than zero is used af- - ter a partial match, PCRE2_ERROR_PARTIAL is returned. Other possible + The return value from all these functions is zero for success, or a + negative error code. If the pattern match failed, the match failure + code is returned. If a substring number greater than zero is used af- + ter a partial match, PCRE2_ERROR_PARTIAL is returned. Other possible error codes are: PCRE2_ERROR_NOMEMORY - The buffer was too small for pcre2_substring_copy_bynumber(), or the + The buffer was too small for pcre2_substring_copy_bynumber(), or the attempt to get memory failed for pcre2_substring_get_bynumber(). PCRE2_ERROR_NOSUBSTRING - There is no substring with that number in the pattern, that is, the + There is no substring with that number in the pattern, that is, the number is greater than the number of capturing parentheses. PCRE2_ERROR_UNAVAILABLE @@ -3486,8 +3553,8 @@ EXTRACTING CAPTURED SUBSTRINGS BY NUMBER PCRE2_ERROR_UNSET - The substring did not participate in the match. For example, if the - pattern is (abc)|(def) and the subject is "def", and the ovector con- + The substring did not participate in the match. For example, if the + pattern is (abc)|(def) and the subject is "def", and the ovector con- tains at least two capturing slots, substring number 1 is unset. @@ -3498,31 +3565,31 @@ EXTRACTING A LIST OF ALL CAPTURED SUBSTRINGS void pcre2_substring_list_free(PCRE2_UCHAR **list); - The pcre2_substring_list_get() function extracts all available sub- - strings and builds a list of pointers to them. It also (optionally) - builds a second list that contains their lengths (in code units), ex- - cluding a terminating zero that is added to each of them. All this is + The pcre2_substring_list_get() function extracts all available sub- + strings and builds a list of pointers to them. It also (optionally) + builds a second list that contains their lengths (in code units), ex- + cluding a terminating zero that is added to each of them. All this is done in a single block of memory that is obtained using the same memory allocation function that was used to get the match data block. - This function must be called only after a successful match. If called + This function must be called only after a successful match. If called after a partial match, the error code PCRE2_ERROR_PARTIAL is returned. - The address of the memory block is returned via listptr, which is also + The address of the memory block is returned via listptr, which is also the start of the list of string pointers. The end of the list is marked - by a NULL pointer. The address of the list of lengths is returned via - lengthsptr. If your strings do not contain binary zeros and you do not + by a NULL pointer. The address of the list of lengths is returned via + lengthsptr. If your strings do not contain binary zeros and you do not therefore need the lengths, you may supply NULL as the lengthsptr argu- - ment to disable the creation of a list of lengths. The yield of the - function is zero if all went well, or PCRE2_ERROR_NOMEMORY if the mem- - ory block could not be obtained. When the list is no longer needed, it + ment to disable the creation of a list of lengths. The yield of the + function is zero if all went well, or PCRE2_ERROR_NOMEMORY if the mem- + ory block could not be obtained. When the list is no longer needed, it should be freed by calling pcre2_substring_list_free(). If this function encounters a substring that is unset, which can happen - when capture group number n+1 matches some part of the subject, but - group n has not been used at all, it returns an empty string. This can + when capture group number n+1 matches some part of the subject, but + group n has not been used at all, it returns an empty string. This can be distinguished from a genuine zero-length substring by inspecting the - appropriate offset in the ovector, which contain PCRE2_UNSET for unset + appropriate offset in the ovector, which contain PCRE2_UNSET for unset substrings, or by calling pcre2_substring_length_bynumber(). @@ -3542,7 +3609,7 @@ EXTRACTING CAPTURED SUBSTRINGS BY NAME void pcre2_substring_free(PCRE2_UCHAR *buffer); - To extract a substring by name, you first have to find associated num- + To extract a substring by name, you first have to find associated num- ber. For example, for this pattern: (a+)b(?\d+)... @@ -3550,32 +3617,32 @@ EXTRACTING CAPTURED SUBSTRINGS BY NAME the number of the capture group called "xxx" is 2. If the name is known to be unique (PCRE2_DUPNAMES was not set), you can find the number from the name by calling pcre2_substring_number_from_name(). The first argu- - ment is the compiled pattern, and the second is the name. The yield of - the function is the group number, PCRE2_ERROR_NOSUBSTRING if there is - no group with that name, or PCRE2_ERROR_NOUNIQUESUBSTRING if there is - more than one group with that name. Given the number, you can extract - the substring directly from the ovector, or use one of the "bynumber" + ment is the compiled pattern, and the second is the name. The yield of + the function is the group number, PCRE2_ERROR_NOSUBSTRING if there is + no group with that name, or PCRE2_ERROR_NOUNIQUESUBSTRING if there is + more than one group with that name. Given the number, you can extract + the substring directly from the ovector, or use one of the "bynumber" functions described above. - For convenience, there are also "byname" functions that correspond to + For convenience, there are also "byname" functions that correspond to the "bynumber" functions, the only difference being that the second ar- - gument is a name instead of a number. If PCRE2_DUPNAMES is set and + gument is a name instead of a number. If PCRE2_DUPNAMES is set and there are duplicate names, these functions scan all the groups with the - given name, and return the captured substring from the first named + given name, and return the captured substring from the first named group that is set. - If there are no groups with the given name, PCRE2_ERROR_NOSUBSTRING is - returned. If all groups with the name have numbers that are greater + If there are no groups with the given name, PCRE2_ERROR_NOSUBSTRING is + returned. If all groups with the name have numbers that are greater than the number of slots in the ovector, PCRE2_ERROR_UNAVAILABLE is re- - turned. If there is at least one group with a slot in the ovector, but + turned. If there is at least one group with a slot in the ovector, but no group is found to be set, PCRE2_ERROR_UNSET is returned. Warning: If the pattern uses the (?| feature to set up multiple capture - groups with the same number, as described in the section on duplicate + groups with the same number, as described in the section on duplicate group numbers in the pcre2pattern page, you cannot use names to distin- - guish the different capture groups, because names are not included in - the compiled code. The matching process uses only numbers. For this - reason, the use of different names for groups with the same number + guish the different capture groups, because names are not included in + the compiled code. The matching process uses only numbers. For this + reason, the use of different names for groups with the same number causes an error at compile time. @@ -3588,113 +3655,113 @@ CREATING A NEW STRING WITH SUBSTITUTIONS PCRE2_SIZE rlength, PCRE2_UCHAR *outputbuffer, PCRE2_SIZE *outlengthptr); - This function optionally calls pcre2_match() and then makes a copy of - the subject string in outputbuffer, replacing parts that were matched + This function optionally calls pcre2_match() and then makes a copy of + the subject string in outputbuffer, replacing parts that were matched with the replacement string, whose length is supplied in rlength, which - can be given as PCRE2_ZERO_TERMINATED for a zero-terminated string. As - a special case, if replacement is NULL and rlength is zero, the re- - placement is assumed to be an empty string. If rlength is non-zero, an + can be given as PCRE2_ZERO_TERMINATED for a zero-terminated string. As + a special case, if replacement is NULL and rlength is zero, the re- + placement is assumed to be an empty string. If rlength is non-zero, an error occurs if replacement is NULL. There is an option (see PCRE2_SUBSTITUTE_REPLACEMENT_ONLY below) to re- - turn just the replacement string(s). The default action is to perform - just one replacement if the pattern matches, but there is an option - that requests multiple replacements (see PCRE2_SUBSTITUTE_GLOBAL be- + turn just the replacement string(s). The default action is to perform + just one replacement if the pattern matches, but there is an option + that requests multiple replacements (see PCRE2_SUBSTITUTE_GLOBAL be- low). - If successful, pcre2_substitute() returns the number of substitutions - that were carried out. This may be zero if no match was found, and is - never greater than one unless PCRE2_SUBSTITUTE_GLOBAL is set. A nega- + If successful, pcre2_substitute() returns the number of substitutions + that were carried out. This may be zero if no match was found, and is + never greater than one unless PCRE2_SUBSTITUTE_GLOBAL is set. A nega- tive value is returned if an error is detected. - Matches in which a \K item in a lookahead in the pattern causes the - match to end before it starts are not supported, and give rise to an + Matches in which a \K item in a lookahead in the pattern causes the + match to end before it starts are not supported, and give rise to an error return. For global replacements, matches in which \K in a lookbe- - hind causes the match to start earlier than the point that was reached + hind causes the match to start earlier than the point that was reached in the previous iteration are also not supported. - The first seven arguments of pcre2_substitute() are the same as for + The first seven arguments of pcre2_substitute() are the same as for pcre2_match(), except that the partial matching options are not permit- - ted, and match_data may be passed as NULL, in which case a match data - block is obtained and freed within this function, using memory manage- - ment functions from the match context, if provided, or else those that + ted, and match_data may be passed as NULL, in which case a match data + block is obtained and freed within this function, using memory manage- + ment functions from the match context, if provided, or else those that were used to allocate memory for the compiled code. - If match_data is not NULL and PCRE2_SUBSTITUTE_MATCHED is not set, the + If match_data is not NULL and PCRE2_SUBSTITUTE_MATCHED is not set, the provided block is used for all calls to pcre2_match(), and its contents - afterwards are the result of the final call. For global changes, this + afterwards are the result of the final call. For global changes, this will always be a no-match error. The contents of the ovector within the match data block may or may not have been changed. - As well as the usual options for pcre2_match(), a number of additional - options can be set in the options argument of pcre2_substitute(). One - such option is PCRE2_SUBSTITUTE_MATCHED. When this is set, an external - match_data block must be provided, and it must have already been used + As well as the usual options for pcre2_match(), a number of additional + options can be set in the options argument of pcre2_substitute(). One + such option is PCRE2_SUBSTITUTE_MATCHED. When this is set, an external + match_data block must be provided, and it must have already been used for an external call to pcre2_match() with the same pattern and subject - arguments. The data in the match_data block (return code, offset vec- - tor) is then used for the first substitution instead of calling - pcre2_match() from within pcre2_substitute(). This allows an applica- + arguments. The data in the match_data block (return code, offset vec- + tor) is then used for the first substitution instead of calling + pcre2_match() from within pcre2_substitute(). This allows an applica- tion to check for a match before choosing to substitute, without having to repeat the match. - The contents of the externally supplied match data block are not - changed when PCRE2_SUBSTITUTE_MATCHED is set. If PCRE2_SUBSTI- - TUTE_GLOBAL is also set, pcre2_match() is called after the first sub- - stitution to check for further matches, but this is done using an in- - ternally obtained match data block, thus always leaving the external + The contents of the externally supplied match data block are not + changed when PCRE2_SUBSTITUTE_MATCHED is set. If PCRE2_SUBSTI- + TUTE_GLOBAL is also set, pcre2_match() is called after the first sub- + stitution to check for further matches, but this is done using an in- + ternally obtained match data block, thus always leaving the external block unchanged. - The code argument is not used for matching before the first substitu- - tion when PCRE2_SUBSTITUTE_MATCHED is set, but it must be provided, - even when PCRE2_SUBSTITUTE_GLOBAL is not set, because it contains in- + The code argument is not used for matching before the first substitu- + tion when PCRE2_SUBSTITUTE_MATCHED is set, but it must be provided, + even when PCRE2_SUBSTITUTE_GLOBAL is not set, because it contains in- formation such as the UTF setting and the number of capturing parenthe- ses in the pattern. - The default action of pcre2_substitute() is to return a copy of the + The default action of pcre2_substitute() is to return a copy of the subject string with matched substrings replaced. However, if PCRE2_SUB- - STITUTE_REPLACEMENT_ONLY is set, only the replacement substrings are + STITUTE_REPLACEMENT_ONLY is set, only the replacement substrings are returned. In the global case, multiple replacements are concatenated in - the output buffer. Substitution callouts (see below) can be used to + the output buffer. Substitution callouts (see below) can be used to separate them if necessary. - The outlengthptr argument of pcre2_substitute() must point to a vari- - able that contains the length, in code units, of the output buffer. If - the function is successful, the value is updated to contain the length - in code units of the new string, excluding the trailing zero that is + The outlengthptr argument of pcre2_substitute() must point to a vari- + able that contains the length, in code units, of the output buffer. If + the function is successful, the value is updated to contain the length + in code units of the new string, excluding the trailing zero that is automatically added. - If the function is not successful, the value set via outlengthptr de- - pends on the type of error. For syntax errors in the replacement + If the function is not successful, the value set via outlengthptr de- + pends on the type of error. For syntax errors in the replacement string, the value is the offset in the replacement string where the er- - ror was detected. For other errors, the value is PCRE2_UNSET by de- + ror was detected. For other errors, the value is PCRE2_UNSET by de- fault. This includes the case of the output buffer being too small, un- less PCRE2_SUBSTITUTE_OVERFLOW_LENGTH is set. - PCRE2_SUBSTITUTE_OVERFLOW_LENGTH changes what happens when the output + PCRE2_SUBSTITUTE_OVERFLOW_LENGTH changes what happens when the output buffer is too small. The default action is to return PCRE2_ERROR_NOMEM- - ORY immediately. If this option is set, however, pcre2_substitute() + ORY immediately. If this option is set, however, pcre2_substitute() continues to go through the motions of matching and substituting (with- - out, of course, writing anything) in order to compute the size of - buffer that is needed, which will include the extra space for the ter- - minating NUL. This value is passed back via the outlengthptr variable, + out, of course, writing anything) in order to compute the size of + buffer that is needed, which will include the extra space for the ter- + minating NUL. This value is passed back via the outlengthptr variable, with the result of the function still being PCRE2_ERROR_NOMEMORY. - Passing a buffer size of zero is a permitted way of finding out how - much memory is needed for given substitution. However, this does mean + Passing a buffer size of zero is a permitted way of finding out how + much memory is needed for given substitution. However, this does mean that the entire operation is carried out twice. Depending on the appli- - cation, it may be more efficient to allocate a large buffer and free - the excess afterwards, instead of using PCRE2_SUBSTITUTE_OVER- + cation, it may be more efficient to allocate a large buffer and free + the excess afterwards, instead of using PCRE2_SUBSTITUTE_OVER- FLOW_LENGTH. - The replacement string, which is interpreted as a UTF string in UTF - mode, is checked for UTF validity unless PCRE2_NO_UTF_CHECK is set. An + The replacement string, which is interpreted as a UTF string in UTF + mode, is checked for UTF validity unless PCRE2_NO_UTF_CHECK is set. An invalid UTF replacement string causes an immediate return with the rel- evant UTF error code. - If PCRE2_SUBSTITUTE_LITERAL is set, the replacement string is not in- + If PCRE2_SUBSTITUTE_LITERAL is set, the replacement string is not in- terpreted in any way. By default, however, a dollar character is an es- - cape character that can specify the insertion of characters from cap- - ture groups and names from (*MARK) or other control verbs in the pat- + cape character that can specify the insertion of characters from cap- + ture groups and names from (*MARK) or other control verbs in the pat- tern. Dollar is the only escape character (backslash is treated as lit- eral). The following forms are recognized: @@ -3706,22 +3773,22 @@ CREATING A NEW STRING WITH SUBSTITUTIONS $_ insert the entire input string $*MARK or ${*MARK} insert a control verb name - Either a group number or a group name can be given for n, for example - $2 or $NAME. Curly brackets are required only if the following charac- - ter would be interpreted as part of the number or name. The number may - be zero to include the entire matched string. For example, if the pat- - tern a(b)c is matched with "=abc=" and the replacement string + Either a group number or a group name can be given for n, for example + $2 or $NAME. Curly brackets are required only if the following charac- + ter would be interpreted as part of the number or name. The number may + be zero to include the entire matched string. For example, if the pat- + tern a(b)c is matched with "=abc=" and the replacement string "+$1$0$1+", the result is "=+babcb+=". - The JavaScript form $, where the angle brackets are part of the - syntax, is also recognized for group names, but not for group numbers + The JavaScript form $, where the angle brackets are part of the + syntax, is also recognized for group names, but not for group numbers or *MARK. - $*MARK inserts the name from the last encountered backtracking control - verb on the matching path that has a name. (*MARK) must always include - a name, but the other verbs need not. For example, in the case of + $*MARK inserts the name from the last encountered backtracking control + verb on the matching path that has a name. (*MARK) must always include + a name, but the other verbs need not. For example, in the case of (*MARK:A)(*PRUNE) the name inserted is "A", but for (*MARK:A)(*PRUNE:B) - the relevant name is "B". This facility can be used to perform simple + the relevant name is "B". This facility can be used to perform simple simultaneous substitutions, as this pcre2test example shows: /(*MARK:pear)apple|(*MARK:orange)lemon/g,replace=${*MARK} @@ -3729,15 +3796,15 @@ CREATING A NEW STRING WITH SUBSTITUTIONS 2: pear orange PCRE2_SUBSTITUTE_GLOBAL causes the function to iterate over the subject - string, replacing every matching substring. If this option is not set, - only the first matching substring is replaced. The search for matches - takes place in the original subject string (that is, previous replace- - ments do not affect it). Iteration is implemented by advancing the - startoffset value for each search, which is always passed the entire + string, replacing every matching substring. If this option is not set, + only the first matching substring is replaced. The search for matches + takes place in the original subject string (that is, previous replace- + ments do not affect it). Iteration is implemented by advancing the + startoffset value for each search, which is always passed the entire subject string. If an offset limit is set in the match context, search- ing stops when that limit is reached. - You can restrict the effect of a global substitution to a portion of + You can restrict the effect of a global substitution to a portion of the subject string by setting either or both of startoffset and an off- set limit. Here is a pcre2test example: @@ -3745,95 +3812,95 @@ CREATING A NEW STRING WITH SUBSTITUTIONS ABC ABC ABC ABC\=offset=3,offset_limit=12 2: ABC A!C A!C ABC - When continuing with global substitutions after matching a substring + When continuing with global substitutions after matching a substring with zero length, an attempt to find a non-empty match at the same off- set is performed. If this is not successful, the offset is advanced by one character except when CRLF is a valid newline sequence and the next - two characters are CR, LF. In this case, the offset is advanced by two + two characters are CR, LF. In this case, the offset is advanced by two characters. PCRE2_SUBSTITUTE_UNKNOWN_UNSET causes references to capture groups that do not appear in the pattern to be treated as unset groups. This option - should be used with care, because it means that a typo in a group name + should be used with care, because it means that a typo in a group name or number no longer causes the PCRE2_ERROR_NOSUBSTRING error. PCRE2_SUBSTITUTE_UNSET_EMPTY causes unset capture groups (including un- - known groups when PCRE2_SUBSTITUTE_UNKNOWN_UNSET is set) to be treated - as empty strings when inserted as described above. If this option is + known groups when PCRE2_SUBSTITUTE_UNKNOWN_UNSET is set) to be treated + as empty strings when inserted as described above. If this option is not set, an attempt to insert an unset group causes the PCRE2_ERROR_UN- - SET error. This option does not influence the extended substitution + SET error. This option does not influence the extended substitution syntax described below. - PCRE2_SUBSTITUTE_EXTENDED causes extra processing to be applied to the - replacement string. Without this option, only the dollar character is - special, and only the group insertion forms listed above are valid. + PCRE2_SUBSTITUTE_EXTENDED causes extra processing to be applied to the + replacement string. Without this option, only the dollar character is + special, and only the group insertion forms listed above are valid. When PCRE2_SUBSTITUTE_EXTENDED is set, several things change: - Firstly, backslash in a replacement string is interpreted as an escape - character. The usual forms such as \x{ddd} can be used to specify par- + Firstly, backslash in a replacement string is interpreted as an escape + character. The usual forms such as \x{ddd} can be used to specify par- ticular character codes, and backslash followed by any non-alphanumeric - character quotes that character. Extended quoting can be coded using - \Q...\E, exactly as in pattern strings. The escapes \b and \v are in- + character quotes that character. Extended quoting can be coded using + \Q...\E, exactly as in pattern strings. The escapes \b and \v are in- terpreted as the characters backspace and vertical tab, respectively. - The interpretation of backslash followed by one or more digits is the - same as in a pattern, which in Perl has some ambiguities. Details are + The interpretation of backslash followed by one or more digits is the + same as in a pattern, which in Perl has some ambiguities. Details are given in the pcre2pattern page. - The Python form \g, where the angle brackets are part of the syntax + The Python form \g, where the angle brackets are part of the syntax and n is either a group name or number, is recognized as an alternative way of inserting the contents of a group, for example \g<3>. - There are also four escape sequences for forcing the case of inserted - letters. Case forcing applies to all inserted characters, including - those from capture groups and letters within \Q...\E quoted sequences. - The insertion mechanism has three states: no case forcing, force upper - case, and force lower case. The escape sequences change the current - state: \U and \L change to upper or lower case forcing, respectively, - and \E (when not terminating a \Q quoted sequence) reverts to no case - forcing. The sequences \u and \l force the next character (if it is a - letter) to upper or lower case, respectively, and then the state auto- + There are also four escape sequences for forcing the case of inserted + letters. Case forcing applies to all inserted characters, including + those from capture groups and letters within \Q...\E quoted sequences. + The insertion mechanism has three states: no case forcing, force upper + case, and force lower case. The escape sequences change the current + state: \U and \L change to upper or lower case forcing, respectively, + and \E (when not terminating a \Q quoted sequence) reverts to no case + forcing. The sequences \u and \l force the next character (if it is a + letter) to upper or lower case, respectively, and then the state auto- matically reverts to no case forcing. - However, if \u is immediately followed by \L or \l is immediately fol- - lowed by \U, the next character's case is forced by the first escape + However, if \u is immediately followed by \L or \l is immediately fol- + lowed by \U, the next character's case is forced by the first escape sequence, and subsequent characters by the second. This provides a "ti- - tle casing" facility that can be applied to group captures. For exam- - ple, if group 1 has captured "heLLo", the replacement string "\u\L$1" + tle casing" facility that can be applied to group captures. For exam- + ple, if group 1 has captured "heLLo", the replacement string "\u\L$1" becomes "Hello". If either PCRE2_UTF or PCRE2_UCP was set when the pattern was compiled, - Unicode properties are used for case forcing characters whose code - points are greater than 127. However, only simple case folding, as de- - termined by the Unicode file CaseFolding.txt is supported. PCRE2 does - not support language-specific special casing rules such as using dif- - ferent lower case Greek sigmas in the middle and ends of words (as de- + Unicode properties are used for case forcing characters whose code + points are greater than 127. However, only simple case folding, as de- + termined by the Unicode file CaseFolding.txt is supported. PCRE2 does + not support language-specific special casing rules such as using dif- + ferent lower case Greek sigmas in the middle and ends of words (as de- fined in the Unicode file SpecialCasing.txt). Note that case forcing sequences such as \U...\E do not nest. For exam- - ple, the result of processing "\Uaa\LBB\Ecc\E" is "AAbbcc"; the final - \E has no effect. Note also that the PCRE2_ALT_BSUX and PCRE2_EX- + ple, the result of processing "\Uaa\LBB\Ecc\E" is "AAbbcc"; the final + \E has no effect. Note also that the PCRE2_ALT_BSUX and PCRE2_EX- TRA_ALT_BSUX options do not apply to replacement strings. - The final effect of setting PCRE2_SUBSTITUTE_EXTENDED is to add more - flexibility to capture group substitution. The syntax is similar to + The final effect of setting PCRE2_SUBSTITUTE_EXTENDED is to add more + flexibility to capture group substitution. The syntax is similar to that used by Bash: ${n:-string} ${n:+string1:string2} - As in the simple case, n may be a group number or a name. The first - form specifies a default value. If group n is set, its value is in- - serted; if not, the string is expanded and the result inserted. The + As in the simple case, n may be a group number or a name. The first + form specifies a default value. If group n is set, its value is in- + serted; if not, the string is expanded and the result inserted. The second form specifies strings that are expanded and inserted when group - n is set or unset, respectively. The first form is just a convenient + n is set or unset, respectively. The first form is just a convenient shorthand for ${n:+${n}:string} - Backslash can be used to escape colons and closing curly brackets in - the replacement strings. A change of the case forcing state within a - replacement string remains in force afterwards, as shown in this + Backslash can be used to escape colons and closing curly brackets in + the replacement strings. A change of the case forcing state within a + replacement string remains in force afterwards, as shown in this pcre2test example: /(some)?(body)/substitute_extended,replace=${1:+\U:\L}HeLLo @@ -3842,8 +3909,8 @@ CREATING A NEW STRING WITH SUBSTITUTIONS somebody 1: HELLO - The PCRE2_SUBSTITUTE_UNSET_EMPTY option does not affect these extended - substitutions. However, PCRE2_SUBSTITUTE_UNKNOWN_UNSET does cause un- + The PCRE2_SUBSTITUTE_UNSET_EMPTY option does not affect these extended + substitutions. However, PCRE2_SUBSTITUTE_UNKNOWN_UNSET does cause un- known groups in the extended syntax forms to be treated as unset. If PCRE2_SUBSTITUTE_LITERAL is set, PCRE2_SUBSTITUTE_UNKNOWN_UNSET, @@ -3852,39 +3919,39 @@ CREATING A NEW STRING WITH SUBSTITUTIONS Substitution errors - In the event of an error, pcre2_substitute() returns a negative error - code. Except for PCRE2_ERROR_NOMATCH (which is never returned), errors + In the event of an error, pcre2_substitute() returns a negative error + code. Except for PCRE2_ERROR_NOMATCH (which is never returned), errors from pcre2_match() are passed straight back. PCRE2_ERROR_NOSUBSTRING is returned for a non-existent substring inser- tion, unless PCRE2_SUBSTITUTE_UNKNOWN_UNSET is set. PCRE2_ERROR_UNSET is returned for an unset substring insertion (includ- - ing an unknown substring when PCRE2_SUBSTITUTE_UNKNOWN_UNSET is set) - when the simple (non-extended) syntax is used and PCRE2_SUBSTITUTE_UN- + ing an unknown substring when PCRE2_SUBSTITUTE_UNKNOWN_UNSET is set) + when the simple (non-extended) syntax is used and PCRE2_SUBSTITUTE_UN- SET_EMPTY is not set. - PCRE2_ERROR_NOMEMORY is returned if the output buffer is not big + PCRE2_ERROR_NOMEMORY is returned if the output buffer is not big enough. If the PCRE2_SUBSTITUTE_OVERFLOW_LENGTH option is set, the size - of buffer that is needed is returned via outlengthptr. Note that this + of buffer that is needed is returned via outlengthptr. Note that this does not happen by default. PCRE2_ERROR_NULL is returned if PCRE2_SUBSTITUTE_MATCHED is set but the - match_data argument is NULL or if the subject or replacement arguments - are NULL. For backward compatibility reasons an exception is made for + match_data argument is NULL or if the subject or replacement arguments + are NULL. For backward compatibility reasons an exception is made for the replacement argument if the rlength argument is also 0. - PCRE2_ERROR_BADREPLACEMENT is used for miscellaneous syntax errors in - the replacement string, with more particular errors being PCRE2_ER- + PCRE2_ERROR_BADREPLACEMENT is used for miscellaneous syntax errors in + the replacement string, with more particular errors being PCRE2_ER- ROR_BADREPESCAPE (invalid escape sequence), PCRE2_ERROR_REPMISSINGBRACE - (closing curly bracket not found), PCRE2_ERROR_BADSUBSTITUTION (syntax - error in extended group substitution), and PCRE2_ERROR_BADSUBSPATTERN + (closing curly bracket not found), PCRE2_ERROR_BADSUBSTITUTION (syntax + error in extended group substitution), and PCRE2_ERROR_BADSUBSPATTERN (the pattern match ended before it started or the match started earlier - than the current position in the subject, which can happen if \K is - used in an assertion). + than the current position in the subject, which can happen if \K is + used in a lookaround assertion). As for all PCRE2 errors, a text message that describes the error can be - obtained by calling the pcre2_get_error_message() function (see "Ob- + obtained by calling the pcre2_get_error_message() function (see "Ob- taining a textual error message" above). Substitution callouts @@ -3893,23 +3960,23 @@ CREATING A NEW STRING WITH SUBSTITUTIONS int (*callout_function)(pcre2_substitute_callout_block *, void *), void *callout_data); - The pcre2_set_substitute_callout() function can be used to specify a - callout function for pcre2_substitute(). This information is passed in + The pcre2_set_substitute_callout() function can be used to specify a + callout function for pcre2_substitute(). This information is passed in a match context. The callout function is called after each substitution has been processed, but it can cause the replacement not to happen. - The callout function is not called for simulated substitutions that - happen as a result of the PCRE2_SUBSTITUTE_OVERFLOW_LENGTH option. In - this mode, when substitution processing exceeds the buffer space pro- - vided by the caller, processing continues by counting code units. The - simulation is unable to populate the callout block, and so the simula- + The callout function is not called for simulated substitutions that + happen as a result of the PCRE2_SUBSTITUTE_OVERFLOW_LENGTH option. In + this mode, when substitution processing exceeds the buffer space pro- + vided by the caller, processing continues by counting code units. The + simulation is unable to populate the callout block, and so the simula- tion is pessimistic about the required buffer size. Whichever is larger - of accepted or rejected substitution is reported as the required size. + of accepted or rejected substitution is reported as the required size. Therefore, the returned buffer length may be an overestimate (without a substitution callout, it is normally an exact measurement). The first argument of the callout function is a pointer to a substitute - callout block structure, which contains the following fields, not nec- + callout block structure, which contains the following fields, not nec- essarily in this order: uint32_t version; @@ -3920,34 +3987,34 @@ CREATING A NEW STRING WITH SUBSTITUTIONS uint32_t oveccount; PCRE2_SIZE output_offsets[2]; - The version field contains the version number of the block format. The - current version is 0. The version number will increase in future if - more fields are added, but the intention is never to remove any of the + The version field contains the version number of the block format. The + current version is 0. The version number will increase in future if + more fields are added, but the intention is never to remove any of the existing fields. The subscount field is the number of the current match. It is 1 for the first callout, 2 for the second, and so on. The input and output point- ers are copies of the values passed to pcre2_substitute(). - The ovector field points to the ovector, which contains the result of + The ovector field points to the ovector, which contains the result of the most recent match. The oveccount field contains the number of pairs that are set in the ovector, and is always greater than zero. - The output_offsets vector contains the offsets of the replacement in - the output string. This has already been processed for dollar and (if + The output_offsets vector contains the offsets of the replacement in + the output string. This has already been processed for dollar and (if requested) backslash substitutions as described above. - The second argument of the callout function is the value passed as - callout_data when the function was registered. The value returned by + The second argument of the callout function is the value passed as + callout_data when the function was registered. The value returned by the callout function is interpreted as follows: - If the value is zero, the replacement is accepted, and, if PCRE2_SUB- - STITUTE_GLOBAL is set, processing continues with a search for the next - match. If the value is not zero, the current replacement is not ac- - cepted. If the value is greater than zero, processing continues when - PCRE2_SUBSTITUTE_GLOBAL is set. Otherwise (the value is less than zero + If the value is zero, the replacement is accepted, and, if PCRE2_SUB- + STITUTE_GLOBAL is set, processing continues with a search for the next + match. If the value is not zero, the current replacement is not ac- + cepted. If the value is greater than zero, processing continues when + PCRE2_SUBSTITUTE_GLOBAL is set. Otherwise (the value is less than zero or PCRE2_SUBSTITUTE_GLOBAL is not set), the rest of the input is copied - to the output and the call to pcre2_substitute() exits, returning the + to the output and the call to pcre2_substitute() exits, returning the number of matches so far. Substitution case callouts @@ -3959,21 +4026,21 @@ CREATING A NEW STRING WITH SUBSTITUTIONS void *callout_data); The pcre2_set_substitute_case_callout() function can be used to specify - a callout function for pcre2_substitute() to use when performing case - transformations. This does not affect any case insensitivity behaviour + a callout function for pcre2_substitute() to use when performing case + transformations. This does not affect any case insensitivity behaviour when performing a match, but only the user-visible transformations per- formed when processing a substitution such as: pcre2_substitute(..., "\\U$1", ...) - The default case transformations applied by PCRE2 are reasonably com- - plete, and, in UTF or UCP mode, perform the simple locale-invariant - case transformations as specified by Unicode. This is suitable for the - internal (invisible) case-equivalence procedures used during pattern + The default case transformations applied by PCRE2 are reasonably com- + plete, and, in UTF or UCP mode, perform the simple locale-invariant + case transformations as specified by Unicode. This is suitable for the + internal (invisible) case-equivalence procedures used during pattern matching, but an application may wish to use more sophisticated locale- aware processing for the user-visible substitution transformations. - One example implementation of the callout_function using the ICU li- + One example implementation of the callout_function using the ICU li- brary would be: PCRE2_SIZE @@ -3993,48 +4060,48 @@ CREATING A NEW STRING WITH SUBSTITUTIONS return r; } - The first and second arguments of the case callout function are the + The first and second arguments of the case callout function are the Unicode string to transform. The third and fourth arguments are the output buffer and its capacity. - The fifth is one of the constants PCRE2_SUBSTITUTE_CASE_LOWER, - PCRE2_SUBSTITUTE_CASE_UPPER, or PCRE2_SUBSTITUTE_CASE_TITLE_FIRST. - PCRE2_SUBSTITUTE_CASE_LOWER and PCRE2_SUBSTITUTE_CASE_UPPER are passed - to the callout to indicate that the case of the entire callout input + The fifth is one of the constants PCRE2_SUBSTITUTE_CASE_LOWER, + PCRE2_SUBSTITUTE_CASE_UPPER, or PCRE2_SUBSTITUTE_CASE_TITLE_FIRST. + PCRE2_SUBSTITUTE_CASE_LOWER and PCRE2_SUBSTITUTE_CASE_UPPER are passed + to the callout to indicate that the case of the entire callout input should be case-transformed. PCRE2_SUBSTITUTE_CASE_TITLE_FIRST is passed - to indicate that only the first character or glyph should be trans- - formed to Unicode titlecase and the rest to Unicode lowercase (note - that titlecasing sometimes uses Unicode properties to titlecase each - word in a string; but PCRE2 is requesting that only the single leading + to indicate that only the first character or glyph should be trans- + formed to Unicode titlecase and the rest to Unicode lowercase (note + that titlecasing sometimes uses Unicode properties to titlecase each + word in a string; but PCRE2 is requesting that only the single leading character is to be titlecased). - The sixth argument is the callout_data supplied to pcre2_set_substi- + The sixth argument is the callout_data supplied to pcre2_set_substi- tute_case_callout(). The resulting string in the destination buffer may be larger or smaller - than the input, if the casing rules merge or split characters. The re- + than the input, if the casing rules merge or split characters. The re- turn value is the length required for the output string. If a buffer of - sufficient size was provided to the callout, then the result must be + sufficient size was provided to the callout, then the result must be written to the buffer and the number of code units returned. If the re- - sult does not fit in the provided buffer, then the required capacity - must be returned and PCRE2 will not make use of the output buffer. - PCRE2 provides input and output buffers which overlap, so the callout + sult does not fit in the provided buffer, then the required capacity + must be returned and PCRE2 will not make use of the output buffer. + PCRE2 provides input and output buffers which overlap, so the callout must support this by suitable internal buffering. - Alternatively, if the callout wishes to indicate an error, then it may - return (~(PCRE2_SIZE)0). In this case pcre2_substitute() will immedi- + Alternatively, if the callout wishes to indicate an error, then it may + return (~(PCRE2_SIZE)0). In this case pcre2_substitute() will immedi- ately fail with error PCRE2_ERROR_REPLACECASE. When a case callout is combined with the PCRE2_SUBSTITUTE_OVER- - FLOW_LENGTH option, there are situations when pcre2_substitute() will - return an underestimate of the required buffer size. If you call - pcre2_substitute() once with PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, and the + FLOW_LENGTH option, there are situations when pcre2_substitute() will + return an underestimate of the required buffer size. If you call + pcre2_substitute() once with PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, and the input buffer is too small for the replacement string to be constructed, - then instead of calling the case callout, pcre2_substitute() will make - an estimate of the required buffer size. The second call should also - pass PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, because that second call is not - guaranteed to succeed either, if the case callout requires more buffer + then instead of calling the case callout, pcre2_substitute() will make + an estimate of the required buffer size. The second call should also + pass PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, because that second call is not + guaranteed to succeed either, if the case callout requires more buffer space than expected. The caller must make repeated attempts in a loop. @@ -4043,56 +4110,56 @@ DUPLICATE CAPTURE GROUP NAMES int pcre2_substring_nametable_scan(const pcre2_code *code, PCRE2_SPTR name, PCRE2_SPTR *first, PCRE2_SPTR *last); - When a pattern is compiled with the PCRE2_DUPNAMES option, names for - capture groups are not required to be unique. Duplicate names are al- - ways allowed for groups with the same number, created by using the (?| + When a pattern is compiled with the PCRE2_DUPNAMES option, names for + capture groups are not required to be unique. Duplicate names are al- + ways allowed for groups with the same number, created by using the (?| feature. Indeed, if such groups are named, they are required to use the same names. - Normally, patterns that use duplicate names are such that in any one - match, only one of each set of identically-named groups participates. + Normally, patterns that use duplicate names are such that in any one + match, only one of each set of identically-named groups participates. An example is shown in the pcre2pattern documentation. - When duplicates are present, pcre2_substring_copy_byname() and - pcre2_substring_get_byname() return the first substring corresponding - to the given name that is set. Only if none are set is PCRE2_ERROR_UN- - SET is returned. The pcre2_substring_number_from_name() function re- - turns the error PCRE2_ERROR_NOUNIQUESUBSTRING when there are duplicate + When duplicates are present, pcre2_substring_copy_byname() and + pcre2_substring_get_byname() return the first substring corresponding + to the given name that is set. Only if none are set is PCRE2_ERROR_UN- + SET is returned. The pcre2_substring_number_from_name() function re- + turns the error PCRE2_ERROR_NOUNIQUESUBSTRING when there are duplicate names. - If you want to get full details of all captured substrings for a given - name, you must use the pcre2_substring_nametable_scan() function. The - first argument is the compiled pattern, and the second is the name. If - the third and fourth arguments are NULL, the function returns a group + If you want to get full details of all captured substrings for a given + name, you must use the pcre2_substring_nametable_scan() function. The + first argument is the compiled pattern, and the second is the name. If + the third and fourth arguments are NULL, the function returns a group number for a unique name, or PCRE2_ERROR_NOUNIQUESUBSTRING otherwise. When the third and fourth arguments are not NULL, they must be pointers - to variables that are updated by the function. After it has run, they + to variables that are updated by the function. After it has run, they point to the first and last entries in the name-to-number table for the - given name, and the function returns the length of each entry in code - units. In both cases, PCRE2_ERROR_NOSUBSTRING is returned if there are + given name, and the function returns the length of each entry in code + units. In both cases, PCRE2_ERROR_NOSUBSTRING is returned if there are no entries for the given name. The format of the name table is described above in the section entitled - Information about a pattern. Given all the relevant entries for the - name, you can extract each of their numbers, and hence the captured + Information about a pattern. Given all the relevant entries for the + name, you can extract each of their numbers, and hence the captured data. FINDING ALL POSSIBLE MATCHES AT ONE POSITION - The traditional matching function uses a similar algorithm to Perl, - which stops when it finds the first match at a given point in the sub- + The traditional matching function uses a similar algorithm to Perl, + which stops when it finds the first match at a given point in the sub- ject. If you want to find all possible matches, or the longest possible - match at a given position, consider using the alternative matching - function (see below) instead. If you cannot use the alternative func- + match at a given position, consider using the alternative matching + function (see below) instead. If you cannot use the alternative func- tion, you can kludge it up by making use of the callout facility, which is described in the pcre2callout documentation. What you have to do is to insert a callout right at the end of the pat- - tern. When your callout function is called, extract and save the cur- - rent matched substring. Then return 1, which forces pcre2_match() to - backtrack and try other alternatives. Ultimately, when it runs out of + tern. When your callout function is called, extract and save the cur- + rent matched substring. Then return 1, which forces pcre2_match() to + backtrack and try other alternatives. Ultimately, when it runs out of matches, pcre2_match() will yield PCRE2_ERROR_NOMATCH. @@ -4104,27 +4171,27 @@ MATCHING A PATTERN: THE ALTERNATIVE FUNCTION pcre2_match_context *mcontext, int *workspace, PCRE2_SIZE wscount); - The function pcre2_dfa_match() is called to match a subject string - against a compiled pattern, using a matching algorithm that scans the + The function pcre2_dfa_match() is called to match a subject string + against a compiled pattern, using a matching algorithm that scans the subject string just once (not counting lookaround assertions), and does - not backtrack (except when processing lookaround assertions). This has - different characteristics to the normal algorithm, and is not compati- - ble with Perl. Some of the features of PCRE2 patterns are not sup- + not backtrack (except when processing lookaround assertions). This has + different characteristics to the normal algorithm, and is not compati- + ble with Perl. Some of the features of PCRE2 patterns are not sup- ported. Nevertheless, there are times when this kind of matching can be - useful. For a discussion of the two matching algorithms, and a list of + useful. For a discussion of the two matching algorithms, and a list of features that pcre2_dfa_match() does not support, see the pcre2matching documentation. - The arguments for the pcre2_dfa_match() function are the same as for + The arguments for the pcre2_dfa_match() function are the same as for pcre2_match(), plus two extras. The ovector within the match data block is used in a different way, and this is described below. The other com- - mon arguments are used in the same way as for pcre2_match(), so their + mon arguments are used in the same way as for pcre2_match(), so their description is not repeated here. - The two additional arguments provide workspace for the function. The - workspace vector should contain at least 20 elements. It is used for - keeping track of multiple paths through the pattern tree. More work- - space is needed for patterns and subjects where there are a lot of po- + The two additional arguments provide workspace for the function. The + workspace vector should contain at least 20 elements. It is used for + keeping track of multiple paths through the pattern tree. More work- + space is needed for patterns and subjects where there are a lot of po- tential matches. Here is an example of a simple call to pcre2_dfa_match(): @@ -4144,45 +4211,45 @@ MATCHING A PATTERN: THE ALTERNATIVE FUNCTION Option bits for pcre2_dfa_match() - The unused bits of the options argument for pcre2_dfa_match() must be - zero. The only bits that may be set are PCRE2_ANCHORED, - PCRE2_COPY_MATCHED_SUBJECT, PCRE2_ENDANCHORED, PCRE2_NOTBOL, PCRE2_NO- + The unused bits of the options argument for pcre2_dfa_match() must be + zero. The only bits that may be set are PCRE2_ANCHORED, + PCRE2_COPY_MATCHED_SUBJECT, PCRE2_ENDANCHORED, PCRE2_NOTBOL, PCRE2_NO- TEOL, PCRE2_NOTEMPTY, PCRE2_NOTEMPTY_ATSTART, PCRE2_NO_UTF_CHECK, - PCRE2_PARTIAL_HARD, PCRE2_PARTIAL_SOFT, PCRE2_DFA_SHORTEST, and - PCRE2_DFA_RESTART. All but the last four of these are exactly the same + PCRE2_PARTIAL_HARD, PCRE2_PARTIAL_SOFT, PCRE2_DFA_SHORTEST, and + PCRE2_DFA_RESTART. All but the last four of these are exactly the same as for pcre2_match(), so their description is not repeated here. PCRE2_PARTIAL_HARD PCRE2_PARTIAL_SOFT - These have the same general effect as they do for pcre2_match(), but - the details are slightly different. When PCRE2_PARTIAL_HARD is set for - pcre2_dfa_match(), it returns PCRE2_ERROR_PARTIAL if the end of the + These have the same general effect as they do for pcre2_match(), but + the details are slightly different. When PCRE2_PARTIAL_HARD is set for + pcre2_dfa_match(), it returns PCRE2_ERROR_PARTIAL if the end of the subject is reached and there is still at least one matching possibility that requires additional characters. This happens even if some complete - matches have already been found. When PCRE2_PARTIAL_SOFT is set, the - return code PCRE2_ERROR_NOMATCH is converted into PCRE2_ERROR_PARTIAL - if the end of the subject is reached, there have been no complete + matches have already been found. When PCRE2_PARTIAL_SOFT is set, the + return code PCRE2_ERROR_NOMATCH is converted into PCRE2_ERROR_PARTIAL + if the end of the subject is reached, there have been no complete matches, but there is still at least one matching possibility. The por- - tion of the string that was inspected when the longest partial match + tion of the string that was inspected when the longest partial match was found is set as the first matching string in both cases. There is a - more detailed discussion of partial and multi-segment matching, with + more detailed discussion of partial and multi-segment matching, with examples, in the pcre2partial documentation. PCRE2_DFA_SHORTEST - Setting the PCRE2_DFA_SHORTEST option causes the matching algorithm to + Setting the PCRE2_DFA_SHORTEST option causes the matching algorithm to stop as soon as it has found one match. Because of the way the alterna- - tive algorithm works, this is necessarily the shortest possible match + tive algorithm works, this is necessarily the shortest possible match at the first possible matching point in the subject string. PCRE2_DFA_RESTART - When pcre2_dfa_match() returns a partial match, it is possible to call + When pcre2_dfa_match() returns a partial match, it is possible to call it again, with additional subject characters, and have it continue with the same match. The PCRE2_DFA_RESTART option requests this action; when - it is set, the workspace and wscount options must reference the same - vector as before because data about the match so far is left in them + it is set, the workspace and wscount options must reference the same + vector as before because data about the match so far is left in them after a partial match. There is more discussion of this facility in the pcre2partial documentation. @@ -4190,8 +4257,8 @@ MATCHING A PATTERN: THE ALTERNATIVE FUNCTION When pcre2_dfa_match() succeeds, it may have matched more than one sub- string in the subject. Note, however, that all the matches from one run - of the function start at the same point in the subject. The shorter - matches are all initial substrings of the longer matches. For example, + of the function start at the same point in the subject. The shorter + matches are all initial substrings of the longer matches. For example, if the pattern <.*> @@ -4206,80 +4273,80 @@ MATCHING A PATTERN: THE ALTERNATIVE FUNCTION - On success, the yield of the function is a number greater than zero, - which is the number of matched substrings. The offsets of the sub- - strings are returned in the ovector, and can be extracted by number in - the same way as for pcre2_match(), but the numbers bear no relation to - any capture groups that may exist in the pattern, because DFA matching + On success, the yield of the function is a number greater than zero, + which is the number of matched substrings. The offsets of the sub- + strings are returned in the ovector, and can be extracted by number in + the same way as for pcre2_match(), but the numbers bear no relation to + any capture groups that may exist in the pattern, because DFA matching does not support capturing. - Calls to the convenience functions that extract substrings by name re- + Calls to the convenience functions that extract substrings by name re- turn the error PCRE2_ERROR_DFA_UFUNC (unsupported function) if used af- - ter a DFA match. The convenience functions that extract substrings by + ter a DFA match. The convenience functions that extract substrings by number never return PCRE2_ERROR_NOSUBSTRING. - The matched strings are stored in the ovector in reverse order of - length; that is, the longest matching string is first. If there were - too many matches to fit into the ovector, the yield of the function is + The matched strings are stored in the ovector in reverse order of + length; that is, the longest matching string is first. If there were + too many matches to fit into the ovector, the yield of the function is zero, and the vector is filled with the longest matches. - NOTE: PCRE2's "auto-possessification" optimization usually applies to - character repeats at the end of a pattern (as well as internally). For - example, the pattern "a\d+" is compiled as if it were "a\d++". For DFA - matching, this means that only one possible match is found. If you re- + NOTE: PCRE2's "auto-possessification" optimization usually applies to + character repeats at the end of a pattern (as well as internally). For + example, the pattern "a\d+" is compiled as if it were "a\d++". For DFA + matching, this means that only one possible match is found. If you re- ally do want multiple matches in such cases, either use an ungreedy re- - peat such as "a\d+?" or set the PCRE2_NO_AUTO_POSSESS option when com- + peat such as "a\d+?" or set the PCRE2_NO_AUTO_POSSESS option when com- piling. Error returns from pcre2_dfa_match() The pcre2_dfa_match() function returns a negative number when it fails. - Many of the errors are the same as for pcre2_match(), as described + Many of the errors are the same as for pcre2_match(), as described above. There are in addition the following errors that are specific to pcre2_dfa_match(): PCRE2_ERROR_DFA_UITEM - This return is given if pcre2_dfa_match() encounters an item in the - pattern that it does not support, for instance, the use of \C in a UTF + This return is given if pcre2_dfa_match() encounters an item in the + pattern that it does not support, for instance, the use of \C in a UTF mode or a backreference. PCRE2_ERROR_DFA_UCOND - This return is given if pcre2_dfa_match() encounters a condition item + This return is given if pcre2_dfa_match() encounters a condition item that uses a backreference for the condition, or a test for recursion in a specific capture group. These are not supported. PCRE2_ERROR_DFA_UINVALID_UTF - This return is given if pcre2_dfa_match() is called for a pattern that - was compiled with PCRE2_MATCH_INVALID_UTF. This is not supported for + This return is given if pcre2_dfa_match() is called for a pattern that + was compiled with PCRE2_MATCH_INVALID_UTF. This is not supported for DFA matching. PCRE2_ERROR_DFA_WSSIZE - This return is given if pcre2_dfa_match() runs out of space in the + This return is given if pcre2_dfa_match() runs out of space in the workspace vector. PCRE2_ERROR_DFA_RECURSE When a recursion or subroutine call is processed, the matching function - calls itself recursively, using private memory for the ovector and - workspace. This error is given if the internal ovector is not large - enough. This should be extremely rare, as a vector of size 1000 is + calls itself recursively, using private memory for the ovector and + workspace. This error is given if the internal ovector is not large + enough. This should be extremely rare, as a vector of size 1000 is used. PCRE2_ERROR_DFA_BADRESTART - When pcre2_dfa_match() is called with the PCRE2_DFA_RESTART option, - some plausibility checks are made on the contents of the workspace, - which should contain data about the previous partial match. If any of + When pcre2_dfa_match() is called with the PCRE2_DFA_RESTART option, + some plausibility checks are made on the contents of the workspace, + which should contain data about the previous partial match. If any of these checks fail, this error is given. SEE ALSO - pcre2build(3), pcre2callout(3), pcre2demo(3), pcre2matching(3), + pcre2build(3), pcre2callout(3), pcre2demo(3), pcre2matching(3), pcre2partial(3), pcre2posix(3), pcre2sample(3), pcre2unicode(3). diff --git a/doc/pcre2api.3 b/doc/pcre2api.3 index c7bff5df5..4dd61553a 100644 --- a/doc/pcre2api.3 +++ b/doc/pcre2api.3 @@ -3562,10 +3562,10 @@ sample program. The general pattern is: int rc = pcre2_match(re, subject, subject_len, start_offset, app_options | global_options, match_data, match_context); - +. if (rc == PCRE2_ERROR_NOMATCH) break; if (rc < 0) { ... exit } - +. if (!pcre2_next_match(match_data, &start_offset, &global_options)) break; } diff --git a/doc/pcre2demo.3 b/doc/pcre2demo.3 index 6e3f7e088..de2357c07 100644 --- a/doc/pcre2demo.3 +++ b/doc/pcre2demo.3 @@ -1,4 +1,4 @@ -.TH PCRE2DEMO 3 "28 February 2025" "PCRE2 10.46-DEV" +.TH PCRE2DEMO 3 "18 March 2025" "PCRE2 10.46-DEV" .\"AUTOMATICALLY GENERATED BY UpdateAlways - do not EDIT! .SH NAME PCRE2DEMO - A demonstration C program for PCRE2 @@ -101,17 +101,13 @@ PCRE2_SPTR pattern; /* PCRE2_SPTR is a pointer to unsigned code units of */ PCRE2_SPTR subject; /* the appropriate width (in this case, 8 bits). */ PCRE2_SPTR name_table; -int crlf_is_newline; int errornumber; int find_all, caseless_match; int i; int rc; -int utf8; -uint32_t option_bits; uint32_t namecount; uint32_t name_entry_size; -uint32_t newline; PCRE2_SIZE erroroffset; PCRE2_SIZE *ovector; @@ -294,7 +290,9 @@ we have to extract the count of named parentheses from the pattern. */ PCRE2_INFO_NAMECOUNT, /* get the number of named substrings */ &namecount); /* where to put the answer */ -if (namecount == 0) printf("No named substrings\en"); else +if (namecount == 0) + printf("No named substrings\en"); +else { PCRE2_SPTR tabptr; printf("Named substrings\en"); @@ -332,28 +330,8 @@ if (namecount == 0) printf("No named substrings\en"); else * to search for additional matches in the subject string, in a similar * * way to the /g option in Perl. This turns out to be trickier than you * * might think because of the possibility of matching an empty string. * -* What happens is as follows: * * * -* If the previous match was NOT for an empty string, we can just start * -* the next match at the end of the previous one. * -* * -* If the previous match WAS for an empty string, we can't do that, as it * -* would lead to an infinite loop. Instead, a call of pcre2_match() is * -* made with the PCRE2_NOTEMPTY_ATSTART and PCRE2_ANCHORED flags set. The * -* first of these tells PCRE2 that an empty string at the start of the * -* subject is not a valid match; other possibilities must be tried. The * -* second flag restricts PCRE2 to one match attempt at the initial string * -* position. If this match succeeds, an alternative to the empty string * -* match has been found, and we can print it and proceed round the loop, * -* advancing by the length of whatever was found. If this match does not * -* succeed, we still stay in the loop, advancing by just one character. * -* In UTF-8 mode, which can be set by (*UTF) in the pattern, this may be * -* more than one byte. * -* * -* However, there is a complication concerned with newlines. When the * -* newline convention is such that CRLF is a valid newline, we must * -* advance by two characters rather than one. The newline convention can * -* be set in the regex by (*CR), etc.; if not, we must find the default. * +* To help with this task, PCRE2 provides the pcre2_next_match() helper. * *************************************************************************/ if (!find_all) /* Check for -g */ @@ -363,60 +341,18 @@ if (!find_all) /* Check for -g */ return 0; /* Exit the program. */ } -/* Before running the loop, check for UTF-8 and whether CRLF is a valid newline -sequence. First, find the options with which the regex was compiled and extract -the UTF state. */ - -(void)pcre2_pattern_info(re, PCRE2_INFO_ALLOPTIONS, &option_bits); -utf8 = (option_bits & PCRE2_UTF) != 0; - -/* Now find the newline convention and see whether CRLF is a valid newline -sequence. */ - -(void)pcre2_pattern_info(re, PCRE2_INFO_NEWLINE, &newline); -crlf_is_newline = newline == PCRE2_NEWLINE_ANY || - newline == PCRE2_NEWLINE_CRLF || - newline == PCRE2_NEWLINE_ANYCRLF; - /* Loop for second and subsequent matches */ for (;;) { - uint32_t options = 0; /* Normally no options */ - PCRE2_SIZE start_offset = ovector[1]; /* Start at end of previous match */ + PCRE2_SIZE start_offset; + uint32_t options; - /* If the previous match was for an empty string, we are finished if we are - at the end of the subject. Otherwise, arrange to run another match at the - same point to see if a non-empty match can be found. */ + /* After each successful match, we use pcre2_next_match() to obtain the match + parameters for subsequent match attempts. */ - if (ovector[0] == ovector[1]) - { - if (ovector[0] == subject_length) break; - options = PCRE2_NOTEMPTY_ATSTART | PCRE2_ANCHORED; - } - - /* If the previous match was not an empty string, there is one tricky case to - consider. If a pattern contains \eK within a lookbehind assertion at the - start, the end of the matched string can be at the offset where the match - started. Without special action, this leads to a loop that keeps on matching - the same substring. We must detect this case and arrange to move the start on - by one character. The pcre2_get_startchar() function returns the starting - offset that was passed to pcre2_match(). */ - - else - { - PCRE2_SIZE startchar = pcre2_get_startchar(match_data); - if (start_offset <= startchar) - { - if (startchar >= subject_length) break; /* Reached end of subject. */ - start_offset = startchar + 1; /* Advance by one character. */ - if (utf8) /* If UTF-8, it may be more */ - { /* than one code unit. */ - for (; start_offset < subject_length; start_offset++) - if ((subject[start_offset] & 0xc0) != 0x80) break; - } - } - } + if (!pcre2_next_match(match_data, &start_offset, &options)) + break; /* Run the next matching operation */ @@ -429,38 +365,10 @@ for (;;) match_data, /* block for storing the result */ NULL); /* use default match context */ - /* This time, a result of NOMATCH isn't an error. If the value in "options" - is zero, it just means we have found all possible matches, so the loop ends. - Otherwise, it means we have failed to find a non-empty-string match at a - point where there was a previous empty-string match. In this case, we do what - Perl does: advance the matching position by one character, and continue. We - do this by setting the "end of previous match" offset, because that is picked - up at the top of the loop as the point at which to start again. - - There are two complications: (a) When CRLF is a valid newline sequence, and - the current position is just before it, advance by an extra byte. (b) - Otherwise we must ensure that we skip an entire UTF character if we are in - UTF mode. */ + /* If this match attempt fails, exit the loop for subsequent matches. */ if (rc == PCRE2_ERROR_NOMATCH) - { - if (options == 0) break; /* All matches found */ - ovector[1] = start_offset + 1; /* Advance one code unit */ - if (crlf_is_newline && /* If CRLF is a newline & */ - start_offset < subject_length - 1 && /* we are at CRLF, */ - subject[start_offset] == '\er' && - subject[start_offset + 1] == '\en') - ovector[1] += 1; /* Advance by one more. */ - else if (utf8) /* Otherwise, ensure we */ - { /* advance a whole UTF-8 */ - while (ovector[1] < subject_length) /* character. */ - { - if ((subject[ovector[1]] & 0xc0) != 0x80) break; - ovector[1] += 1; - } - } - continue; /* Go round the loop again */ - } + break; /* Other matching errors are not recoverable. */ @@ -507,7 +415,9 @@ for (;;) printf("%2d: %.*s\en", i, (int)substring_length, (char *)substring_start); } - if (namecount == 0) printf("No named substrings\en"); else + if (namecount == 0) + printf("No named substrings\en"); + else { PCRE2_SPTR tabptr = name_table; printf("Named substrings\en"); @@ -522,6 +432,7 @@ for (;;) } /* End of loop to find second and subsequent matches */ printf("\en"); + pcre2_match_data_free(match_data); pcre2_code_free(re); return 0; diff --git a/doc/pcre2test.txt b/doc/pcre2test.txt index 0de6b8141..2080033ca 100644 --- a/doc/pcre2test.txt +++ b/doc/pcre2test.txt @@ -1352,60 +1352,56 @@ SUBJECT MODIFIERS hind assertion (including \b or \B). If an empty string is matched, the next match is done with the - PCRE2_NOTEMPTY_ATSTART and PCRE2_ANCHORED flags set, in order to search - for another, non-empty, match at the same point in the subject. If this - match fails, the start offset is advanced, and the normal match is re- - tried. This imitates the way Perl handles such cases when using the /g - modifier or the split() function. Normally, the start offset is ad- - vanced by one character, but if the newline convention recognizes CRLF - as a newline, and the current character is CR followed by LF, an ad- - vance of two characters occurs. + PCRE2_NOTEMPTY_ATSTART flag set, in order to search for another, non- + empty, match at the same point in the subject. This imitates the way + Perl handles such cases when using the /g modifier or the split() func- + tion. Testing substring extraction functions - The copy and get modifiers can be used to test the pcre2_sub- + The copy and get modifiers can be used to test the pcre2_sub- string_copy_xxx() and pcre2_substring_get_xxx() functions. They can be given more than once, and each can specify a capture group name or num- ber, for example: abcd\=copy=1,copy=3,get=G1 - If the #subject command is used to set default copy and/or get lists, - these can be unset by specifying a negative number to cancel all num- + If the #subject command is used to set default copy and/or get lists, + these can be unset by specifying a negative number to cancel all num- bered groups and an empty name to cancel all named groups. - The getall modifier tests pcre2_substring_list_get(), which extracts + The getall modifier tests pcre2_substring_list_get(), which extracts all captured substrings. - If the subject line is successfully matched, the substrings extracted - by the convenience functions are output with C, G, or L after the - string number instead of a colon. This is in addition to the normal - full list. The string length (that is, the return from the extraction + If the subject line is successfully matched, the substrings extracted + by the convenience functions are output with C, G, or L after the + string number instead of a colon. This is in addition to the normal + full list. The string length (that is, the return from the extraction function) is given in parentheses after each substring, followed by the name when the extraction was by name. Testing the substitution function - If the replace modifier is set, the pcre2_substitute() function is - called instead of one of the matching functions (or after one call of - pcre2_match() in the case of PCRE2_SUBSTITUTE_MATCHED). Note that re- - placement strings cannot contain commas, because a comma signifies the - end of a modifier. This is not thought to be an issue in a test pro- + If the replace modifier is set, the pcre2_substitute() function is + called instead of one of the matching functions (or after one call of + pcre2_match() in the case of PCRE2_SUBSTITUTE_MATCHED). Note that re- + placement strings cannot contain commas, because a comma signifies the + end of a modifier. This is not thought to be an issue in a test pro- gram. - Specifying a completely empty replacement string disables this modi- - fier. However, it is possible to specify an empty replacement by pro- - viding a buffer length, as described below, for an otherwise empty re- + Specifying a completely empty replacement string disables this modi- + fier. However, it is possible to specify an empty replacement by pro- + viding a buffer length, as described below, for an otherwise empty re- placement. - Unlike subject strings, pcre2test does not process replacement strings - for escape sequences. In UTF mode, a replacement string is checked to - see if it is a valid UTF-8 string. If so, it is correctly converted to - a UTF string of the appropriate code unit width. If it is not a valid - UTF-8 string, the individual code units are copied directly. This pro- + Unlike subject strings, pcre2test does not process replacement strings + for escape sequences. In UTF mode, a replacement string is checked to + see if it is a valid UTF-8 string. If so, it is correctly converted to + a UTF string of the appropriate code unit width. If it is not a valid + UTF-8 string, the individual code units are copied directly. This pro- vides a means of passing an invalid UTF-8 string for testing purposes. - The following modifiers set options (in additional to the normal match + The following modifiers set options (in additional to the normal match options) for pcre2_substitute(): global PCRE2_SUBSTITUTE_GLOBAL @@ -1419,8 +1415,8 @@ SUBJECT MODIFIERS See the pcre2api documentation for details of these options. - After a successful substitution, the modified string is output, pre- - ceded by the number of replacements. This may be zero if there were no + After a successful substitution, the modified string is output, pre- + ceded by the number of replacements. This may be zero if there were no matches. Here is a simple example of a substitution test: /abc/replace=xxx @@ -1429,12 +1425,12 @@ SUBJECT MODIFIERS =abc=abc=\=global 2: =xxx=xxx= - Subject and replacement strings should be kept relatively short (fewer - than 256 characters) for substitution tests, as fixed-size buffers are - used. To make it easy to test for buffer overflow, if the replacement - string starts with a number in square brackets, that number is passed - to pcre2_substitute() as the size of the output buffer, with the re- - placement string starting at the next character. Here is an example + Subject and replacement strings should be kept relatively short (fewer + than 256 characters) for substitution tests, as fixed-size buffers are + used. To make it easy to test for buffer overflow, if the replacement + string starts with a number in square brackets, that number is passed + to pcre2_substitute() as the size of the output buffer, with the re- + placement string starting at the next character. Here is an example that tests the edge case: /abc/ @@ -1444,12 +1440,12 @@ SUBJECT MODIFIERS Failed: error -48: no more memory The default action of pcre2_substitute() is to return PCRE2_ER- - ROR_NOMEMORY when the output buffer is too small. However, if the - PCRE2_SUBSTITUTE_OVERFLOW_LENGTH option is set (by using the substi- + ROR_NOMEMORY when the output buffer is too small. However, if the + PCRE2_SUBSTITUTE_OVERFLOW_LENGTH option is set (by using the substi- tute_overflow_length modifier), pcre2_substitute() continues to go - through the motions of matching and substituting (but not doing any - callouts), in order to compute the size of buffer that is required. - When this happens, pcre2test shows the required buffer length (which + through the motions of matching and substituting (but not doing any + callouts), in order to compute the size of buffer that is required. + When this happens, pcre2test shows the required buffer length (which includes space for the trailing zero) as part of the error message. For example: @@ -1458,15 +1454,15 @@ SUBJECT MODIFIERS Failed: error -48: no more memory: 10 code units are needed A replacement string is ignored with POSIX and DFA matching. Specifying - partial matching provokes an error return ("bad option value") from + partial matching provokes an error return ("bad option value") from pcre2_substitute(). Testing substitute callouts If the substitute_callout modifier is set, a substitution callout func- - tion is set up. The null_context modifier must not be set, because the - address of the callout function is passed in a match context. When the - callout function is called (after each substitution), details of the + tion is set up. The null_context modifier must not be set, because the + address of the callout function is passed in a match context. When the + callout function is called (after each substitution), details of the input and output strings are output. For example: /abc/g,replace=<$0>,substitute_callout @@ -1475,19 +1471,19 @@ SUBJECT MODIFIERS 2(1) Old 6 9 "abc" New 8 13 "" 2: defpqr - The first number on each callout line is the count of matches. The + The first number on each callout line is the count of matches. The parenthesized number is the number of pairs that are set in the ovector - (that is, one more than the number of capturing groups that were set). + (that is, one more than the number of capturing groups that were set). Then are listed the offsets of the old substring, its contents, and the same for the replacement. - By default, the substitution callout function returns zero, which ac- - cepts the replacement and causes matching to continue if /g was used. - Two further modifiers can be used to test other return values. If sub- - stitute_skip is set to a value greater than zero the callout function - returns +1 for the match of that number, and similarly substitute_stop - returns -1. These cause the replacement to be rejected, and -1 causes - no further matching to take place. If either of them are set, substi- + By default, the substitution callout function returns zero, which ac- + cepts the replacement and causes matching to continue if /g was used. + Two further modifiers can be used to test other return values. If sub- + stitute_skip is set to a value greater than zero the callout function + returns +1 for the match of that number, and similarly substitute_stop + returns -1. These cause the replacement to be rejected, and -1 causes + no further matching to take place. If either of them are set, substi- tute_callout is assumed. For example: /abc/g,replace=<$0>,substitute_skip=1 @@ -1505,193 +1501,193 @@ SUBJECT MODIFIERS Testing substitute case callouts - If the substitute_case_callout modifier is set, a substitution case - callout function is set up. The callout function is called for each + If the substitute_case_callout modifier is set, a substitution case + callout function is set up. The callout function is called for each substituted chunk which is to be case-transformed. The callout function passed is a fixed function with implementation for - certain behaviours: inputs which shrink when case-transformed; inputs + certain behaviours: inputs which shrink when case-transformed; inputs which grow; inputs with distinct upper/lower/titlecase forms. The char- acters which are not special-cased for testing purposes are left unmod- ified, as if they are caseless characters. Setting the JIT stack size - The jitstack modifier provides a way of setting the maximum stack size - that is used by the just-in-time optimization code. It is ignored if - JIT optimization is not being used. The value is a number of kibibytes - (units of 1024 bytes). Setting zero reverts to the default of 32KiB. + The jitstack modifier provides a way of setting the maximum stack size + that is used by the just-in-time optimization code. It is ignored if + JIT optimization is not being used. The value is a number of kibibytes + (units of 1024 bytes). Setting zero reverts to the default of 32KiB. Providing a stack that is larger than the default is necessary only for - very complicated patterns. If jitstack is set non-zero on a subject + very complicated patterns. If jitstack is set non-zero on a subject line it overrides any value that was set on the pattern. Setting heap, match, and depth limits - The heap_limit, match_limit, and depth_limit modifiers set the appro- - priate limits in the match context. These values are ignored when the + The heap_limit, match_limit, and depth_limit modifiers set the appro- + priate limits in the match context. These values are ignored when the find_limits or find_limits_noheap modifier is specified. Finding minimum limits - If the find_limits modifier is present on a subject line, pcre2test - calls the relevant matching function several times, setting different - values in the match context via pcre2_set_heap_limit(), - pcre2_set_match_limit(), or pcre2_set_depth_limit() until it finds the - smallest value for each parameter that allows the match to complete + If the find_limits modifier is present on a subject line, pcre2test + calls the relevant matching function several times, setting different + values in the match context via pcre2_set_heap_limit(), + pcre2_set_match_limit(), or pcre2_set_depth_limit() until it finds the + smallest value for each parameter that allows the match to complete without a "limit exceeded" error. The match itself may succeed or fail. An alternative modifier, find_limits_noheap, omits the heap limit. This - is used in the standard tests, because the minimum heap limit varies - between systems. If JIT is being used, only the match limit is rele- + is used in the standard tests, because the minimum heap limit varies + between systems. If JIT is being used, only the match limit is rele- vant, and the other two are automatically omitted. When using this modifier, the pattern should not contain any limit set- - tings such as (*LIMIT_MATCH=...) within it. If such a setting is + tings such as (*LIMIT_MATCH=...) within it. If such a setting is present and is lower than the minimum matching value, the minimum value - cannot be found because pcre2_set_match_limit() etc. are only able to + cannot be found because pcre2_set_match_limit() etc. are only able to reduce the value of an in-pattern limit; they cannot increase it. - For non-DFA matching, the minimum depth_limit number is a measure of + For non-DFA matching, the minimum depth_limit number is a measure of how much nested backtracking happens (that is, how deeply the pattern's - tree is searched). In the case of DFA matching, depth_limit controls - the depth of recursive calls of the internal function that is used for + tree is searched). In the case of DFA matching, depth_limit controls + the depth of recursive calls of the internal function that is used for handling pattern recursion, lookaround assertions, and atomic groups. For non-DFA matching, the match_limit number is a measure of the amount of backtracking that takes place, and learning the minimum value can be - instructive. For most simple matches, the number is quite small, but - for patterns with very large numbers of matching possibilities, it can - become large very quickly with increasing length of subject string. In - the case of DFA matching, match_limit controls the total number of + instructive. For most simple matches, the number is quite small, but + for patterns with very large numbers of matching possibilities, it can + become large very quickly with increasing length of subject string. In + the case of DFA matching, match_limit controls the total number of calls, both recursive and non-recursive, to the internal matching func- tion, thus controlling the overall amount of computing resource that is used. - For both kinds of matching, the heap_limit number, which is in - kibibytes (units of 1024 bytes), limits the amount of heap memory used + For both kinds of matching, the heap_limit number, which is in + kibibytes (units of 1024 bytes), limits the amount of heap memory used for matching. Showing MARK names The mark modifier causes the names from backtracking control verbs that - are returned from calls to pcre2_match() to be displayed. If a mark is - returned for a match, non-match, or partial match, pcre2test shows it. - For a match, it is on a line by itself, tagged with "MK:". Otherwise, + are returned from calls to pcre2_match() to be displayed. If a mark is + returned for a match, non-match, or partial match, pcre2test shows it. + For a match, it is on a line by itself, tagged with "MK:". Otherwise, it is added to the non-match message. Showing memory usage - The memory modifier causes pcre2test to log the sizes of all heap mem- - ory allocation and freeing calls that occur during a call to - pcre2_match() or pcre2_dfa_match(). In the latter case, heap memory is - used only when a match requires more internal workspace that the de- - fault allocation on the stack, so in many cases there will be no out- - put. No heap memory is allocated during matching with JIT. For this + The memory modifier causes pcre2test to log the sizes of all heap mem- + ory allocation and freeing calls that occur during a call to + pcre2_match() or pcre2_dfa_match(). In the latter case, heap memory is + used only when a match requires more internal workspace that the de- + fault allocation on the stack, so in many cases there will be no out- + put. No heap memory is allocated during matching with JIT. For this modifier to work, the null_context modifier must not be set on both the pattern and the subject, though it can be set on one or the other. Showing the heap frame overall vector size - The heapframes_size modifier is relevant for matches using + The heapframes_size modifier is relevant for matches using pcre2_match() without JIT. After a match has run (whether successful or - not) the size, in bytes, of the allocated heap frames vector that is - left attached to the match data block is shown. If the matching action - involved several calls to pcre2_match() (for example, global matching + not) the size, in bytes, of the allocated heap frames vector that is + left attached to the match data block is shown. If the matching action + involved several calls to pcre2_match() (for example, global matching or for timing) only the final value is shown. - This modifier is ignored, with a warning, for POSIX or DFA matching. + This modifier is ignored, with a warning, for POSIX or DFA matching. JIT matching does not use the heap frames vector, so the size is always - zero, unless there was a previous non-JIT match. Note that specifing a + zero, unless there was a previous non-JIT match. Note that specifing a size of zero for the output vector (see below) causes pcre2test to free its match data block (and associated heap frames vector) and allocate a new one. Setting a starting offset - The offset modifier sets an offset in the subject string at which + The offset modifier sets an offset in the subject string at which matching starts. Its value is a number of code units, not characters. Setting an offset limit - The offset_limit modifier sets a limit for unanchored matches. If a + The offset_limit modifier sets a limit for unanchored matches. If a match cannot be found starting at or before this offset in the subject, a "no match" return is given. The data value is a number of code units, - not characters. When this modifier is used, the use_offset_limit modi- + not characters. When this modifier is used, the use_offset_limit modi- fier must have been set for the pattern; if not, an error is generated. Setting the size of the output vector - The ovector modifier applies only to the subject line in which it ap- + The ovector modifier applies only to the subject line in which it ap- pears, though of course it can also be used to set a default in a #sub- - ject command. It specifies the number of pairs of offsets that are + ject command. It specifies the number of pairs of offsets that are available for storing matching information. The default is 15. - A value of zero is useful when testing the POSIX API because it causes + A value of zero is useful when testing the POSIX API because it causes regexec() to be called with a NULL capture vector. When not testing the - POSIX API, a value of zero is used to cause pcre2_match_data_cre- - ate_from_pattern() to be called, in order to create a new match block - of exactly the right size for the pattern. (It is not possible to cre- - ate a match block with a zero-length ovector; there is always at least + POSIX API, a value of zero is used to cause pcre2_match_data_cre- + ate_from_pattern() to be called, in order to create a new match block + of exactly the right size for the pattern. (It is not possible to cre- + ate a match block with a zero-length ovector; there is always at least one pair of offsets.) The old match data block is freed. Passing the subject as zero-terminated By default, the subject string is passed to a native API matching func- tion with its correct length. In order to test the facility for passing - a zero-terminated string, the zero_terminate modifier is provided. It - causes the length to be passed as PCRE2_ZERO_TERMINATED. When matching + a zero-terminated string, the zero_terminate modifier is provided. It + causes the length to be passed as PCRE2_ZERO_TERMINATED. When matching via the POSIX interface, this modifier is ignored, with a warning. - When testing pcre2_substitute(), this modifier also has the effect of + When testing pcre2_substitute(), this modifier also has the effect of passing the replacement string as zero-terminated. Passing a NULL context, subject, or replacement - Normally, pcre2test passes a context block to pcre2_match(), - pcre2_dfa_match(), pcre2_jit_match() or pcre2_substitute(). If the - null_context modifier is set, however, NULL is passed. This is for - testing that the matching and substitution functions behave correctly - in this case (they use default values). This modifier cannot be used - with the find_limits, find_limits_noheap, or substitute_callout modi- + Normally, pcre2test passes a context block to pcre2_match(), + pcre2_dfa_match(), pcre2_jit_match() or pcre2_substitute(). If the + null_context modifier is set, however, NULL is passed. This is for + testing that the matching and substitution functions behave correctly + in this case (they use default values). This modifier cannot be used + with the find_limits, find_limits_noheap, or substitute_callout modi- fiers. - Similarly, for testing purposes, if the null_subject or null_replace- - ment modifier is set, the subject or replacement string pointers are + Similarly, for testing purposes, if the null_subject or null_replace- + ment modifier is set, the subject or replacement string pointers are passed as NULL, respectively, to the relevant functions. THE ALTERNATIVE MATCHING FUNCTION - By default, pcre2test uses the standard PCRE2 matching function, + By default, pcre2test uses the standard PCRE2 matching function, pcre2_match() to match each subject line. PCRE2 also supports an alter- - native matching function, pcre2_dfa_match(), which operates in a dif- - ferent way, and has some restrictions. The differences between the two + native matching function, pcre2_dfa_match(), which operates in a dif- + ferent way, and has some restrictions. The differences between the two functions are described in the pcre2matching documentation. - If the dfa modifier is set, the alternative matching function is used. - This function finds all possible matches at a given point in the sub- - ject. If, however, the dfa_shortest modifier is set, processing stops - after the first match is found. This is always the shortest possible + If the dfa modifier is set, the alternative matching function is used. + This function finds all possible matches at a given point in the sub- + ject. If, however, the dfa_shortest modifier is set, processing stops + after the first match is found. This is always the shortest possible match. DEFAULT OUTPUT FROM pcre2test - This section describes the output when the normal matching function, + This section describes the output when the normal matching function, pcre2_match(), is being used. - When a match succeeds, pcre2test outputs the list of captured sub- - strings, starting with number 0 for the string that matched the whole + When a match succeeds, pcre2test outputs the list of captured sub- + strings, starting with number 0 for the string that matched the whole pattern. Otherwise, it outputs "No match" when the return is PCRE2_ER- - ROR_NOMATCH, or "Partial match:" followed by the partially matching - substring when the return is PCRE2_ERROR_PARTIAL. (Note that this is - the entire substring that was inspected during the partial match; it - may include characters before the actual match start if a lookbehind + ROR_NOMATCH, or "Partial match:" followed by the partially matching + substring when the return is PCRE2_ERROR_PARTIAL. (Note that this is + the entire substring that was inspected during the partial match; it + may include characters before the actual match start if a lookbehind assertion, \K, \b, or \B was involved.) For any other return, pcre2test outputs the PCRE2 negative error number - and a short descriptive phrase. If the error is a failed UTF string - check, the code unit offset of the start of the failing character is + and a short descriptive phrase. If the error is a failed UTF string + check, the code unit offset of the start of the failing character is also output. Here is an example of an interactive pcre2test run. $ pcre2test @@ -1707,8 +1703,8 @@ DEFAULT OUTPUT FROM pcre2test Unset capturing substrings that are not followed by one that is set are not shown by pcre2test unless the allcaptures modifier is specified. In the following example, there are two capturing substrings, but when the - first data line is matched, the second, unset substring is not shown. - An "internal" unset substring is shown as "", as for the second + first data line is matched, the second, unset substring is not shown. + An "internal" unset substring is shown as "", as for the second data line. re> /(a)|(b)/ @@ -1720,11 +1716,11 @@ DEFAULT OUTPUT FROM pcre2test 1: 2: b - If the strings contain any non-printing characters, they are output as - \xhh escapes if the value is less than 256 and UTF mode is not set. + If the strings contain any non-printing characters, they are output as + \xhh escapes if the value is less than 256 and UTF mode is not set. Otherwise they are output as \x{hh...} escapes. See below for the defi- - nition of non-printing characters. If the aftertext modifier is set, - the output for substring 0 is followed by the rest of the subject + nition of non-printing characters. If the aftertext modifier is set, + the output for substring 0 is followed by the rest of the subject string, identified by "0+" like this: re> /cat/aftertext @@ -1744,8 +1740,8 @@ DEFAULT OUTPUT FROM pcre2test 0: ipp 1: pp - "No match" is output only if the first match attempt fails. Here is an - example of a failure message (the offset 4 that is specified by the + "No match" is output only if the first match attempt fails. Here is an + example of a failure message (the offset 4 that is specified by the offset modifier is past the end of the subject string): re> /xyz/ @@ -1753,7 +1749,7 @@ DEFAULT OUTPUT FROM pcre2test Error -24 (bad offset value) Note that whereas patterns can be continued over several lines (a plain - ">" prompt is used for continuations), subject lines may not. However + ">" prompt is used for continuations), subject lines may not. However newlines can be included in a subject by means of the \n escape (or \r, \r\n, etc., depending on the newline sequence setting). @@ -1761,7 +1757,7 @@ DEFAULT OUTPUT FROM pcre2test OUTPUT FROM THE ALTERNATIVE MATCHING FUNCTION When the alternative matching function, pcre2_dfa_match(), is used, the - output consists of a list of all the matches that start at the first + output consists of a list of all the matches that start at the first point in the subject where there is at least one match. For example: re> /(tang|tangerine|tan)/ @@ -1770,11 +1766,11 @@ OUTPUT FROM THE ALTERNATIVE MATCHING FUNCTION 1: tang 2: tan - Using the normal matching function on this data finds only "tang". The - longest matching string is always given first (and numbered zero). Af- - ter a PCRE2_ERROR_PARTIAL return, the output is "Partial match:", fol- + Using the normal matching function on this data finds only "tang". The + longest matching string is always given first (and numbered zero). Af- + ter a PCRE2_ERROR_PARTIAL return, the output is "Partial match:", fol- lowed by the partially matching substring. Note that this is the entire - substring that was inspected during the partial match; it may include + substring that was inspected during the partial match; it may include characters before the actual match start if a lookbehind assertion, \b, or \B was involved. (\K is not supported for DFA matching.) @@ -1790,16 +1786,16 @@ OUTPUT FROM THE ALTERNATIVE MATCHING FUNCTION 1: tan 0: tan - The alternative matching function does not support substring capture, - so the modifiers that are concerned with captured substrings are not + The alternative matching function does not support substring capture, + so the modifiers that are concerned with captured substrings are not relevant. RESTARTING AFTER A PARTIAL MATCH - When the alternative matching function has given the PCRE2_ERROR_PAR- + When the alternative matching function has given the PCRE2_ERROR_PAR- TIAL return, indicating that the subject partially matched the pattern, - you can restart the match with additional subject data by means of the + you can restart the match with additional subject data by means of the dfa_restart modifier. For example: re> /^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$/ @@ -1808,37 +1804,37 @@ RESTARTING AFTER A PARTIAL MATCH data> n05\=dfa,dfa_restart 0: n05 - For further information about partial matching, see the pcre2partial + For further information about partial matching, see the pcre2partial documentation. CALLOUTS If the pattern contains any callout requests, pcre2test's callout func- - tion is called during matching unless callout_none is specified. This + tion is called during matching unless callout_none is specified. This works with both matching functions, and with JIT, though there are some - differences in behaviour. The output for callouts with numerical argu- + differences in behaviour. The output for callouts with numerical argu- ments and those with string arguments is slightly different. Callouts with numerical arguments By default, the callout function displays the callout number, the start - and current positions in the subject text at the callout time, and the + and current positions in the subject text at the callout time, and the next pattern item to be tested. For example: --->pqrabcdef 0 ^ ^ \d - This output indicates that callout number 0 occurred for a match at- - tempt starting at the fourth character of the subject string, when the - pointer was at the seventh character, and when the next pattern item - was \d. Just one circumflex is output if the start and current posi- + This output indicates that callout number 0 occurred for a match at- + tempt starting at the fourth character of the subject string, when the + pointer was at the seventh character, and when the next pattern item + was \d. Just one circumflex is output if the start and current posi- tions are the same, or if the current position precedes the start posi- tion, which can happen if the callout is in a lookbehind assertion. Callouts numbered 255 are assumed to be automatic callouts, inserted as a result of the auto_callout pattern modifier. In this case, instead of - showing the callout number, the offset in the pattern, preceded by a + showing the callout number, the offset in the pattern, preceded by a plus, is output. For example: re> /\d?[A-E]\*/auto_callout @@ -1865,17 +1861,17 @@ CALLOUTS +12 ^ ^ 0: abc - The mark changes between matching "a" and "b", but stays the same for - the rest of the match, so nothing more is output. If, as a result of - backtracking, the mark reverts to being unset, the text "" is + The mark changes between matching "a" and "b", but stays the same for + the rest of the match, so nothing more is output. If, as a result of + backtracking, the mark reverts to being unset, the text "" is output. Callouts with string arguments The output for a callout with a string argument is similar, except that - instead of outputting a callout number before the position indicators, - the callout string and its offset in the pattern string are output be- - fore the reflection of the subject string, and the subject string is + instead of outputting a callout number before the position indicators, + the callout string and its offset in the pattern string are output be- + fore the reflection of the subject string, and the subject string is reflected for each callout. For example: re> /^ab(?C'first')cd(?C"second")ef/ @@ -1891,26 +1887,26 @@ CALLOUTS Callout modifiers - The callout function in pcre2test returns zero (carry on matching) by - default, but you can use a callout_fail modifier in a subject line to + The callout function in pcre2test returns zero (carry on matching) by + default, but you can use a callout_fail modifier in a subject line to change this and other parameters of the callout (see below). If the callout_capture modifier is set, the current captured groups are output when a callout occurs. This is useful only for non-DFA matching, - as pcre2_dfa_match() does not support capturing, so no captures are + as pcre2_dfa_match() does not support capturing, so no captures are ever shown. The normal callout output, showing the callout number or pattern offset - (as described above) is suppressed if the callout_no_where modifier is + (as described above) is suppressed if the callout_no_where modifier is set. - When using the interpretive matching function pcre2_match() without - JIT, setting the callout_extra modifier causes additional output from - pcre2test's callout function to be generated. For the first callout in - a match attempt at a new starting position in the subject, "New match - attempt" is output. If there has been a backtrack since the last call- + When using the interpretive matching function pcre2_match() without + JIT, setting the callout_extra modifier causes additional output from + pcre2test's callout function to be generated. For the first callout in + a match attempt at a new starting position in the subject, "New match + attempt" is output. If there has been a backtrack since the last call- out (or start of matching if this is the first callout), "Backtrack" is - output, followed by "No other matching paths" if the backtrack ended + output, followed by "No other matching paths" if the backtrack ended the previous match attempt. For example: re> /(a+)b/auto_callout,no_start_optimize,no_auto_possess @@ -1947,86 +1943,86 @@ CALLOUTS +1 ^ a+ No match - Notice that various optimizations must be turned off if you want all - possible matching paths to be scanned. If no_start_optimize is not - used, there is an immediate "no match", without any callouts, because - the starting optimization fails to find "b" in the subject, which it - knows must be present for any match. If no_auto_possess is not used, - the "a+" item is turned into "a++", which reduces the number of back- + Notice that various optimizations must be turned off if you want all + possible matching paths to be scanned. If no_start_optimize is not + used, there is an immediate "no match", without any callouts, because + the starting optimization fails to find "b" in the subject, which it + knows must be present for any match. If no_auto_possess is not used, + the "a+" item is turned into "a++", which reduces the number of back- tracks. - The callout_extra modifier has no effect if used with the DFA matching + The callout_extra modifier has no effect if used with the DFA matching function, or with JIT. Return values from callouts - The default return from the callout function is zero, which allows + The default return from the callout function is zero, which allows matching to continue. The callout_fail modifier can be given one or two numbers. If there is only one number, 1 is returned instead of 0 (caus- ing matching to backtrack) when a callout of that number is reached. If - two numbers (:) are given, 1 is returned when callout is - reached and there have been at least callouts. The callout_error + two numbers (:) are given, 1 is returned when callout is + reached and there have been at least callouts. The callout_error modifier is similar, except that PCRE2_ERROR_CALLOUT is returned, caus- - ing the entire matching process to be aborted. If both these modifiers - are set for the same callout number, callout_error takes precedence. - Note that callouts with string arguments are always given the number + ing the entire matching process to be aborted. If both these modifiers + are set for the same callout number, callout_error takes precedence. + Note that callouts with string arguments are always given the number zero. - The callout_data modifier can be given an unsigned or a negative num- - ber. This is set as the "user data" that is passed to the matching - function, and passed back when the callout function is invoked. Any - value other than zero is used as a return from pcre2test's callout + The callout_data modifier can be given an unsigned or a negative num- + ber. This is set as the "user data" that is passed to the matching + function, and passed back when the callout function is invoked. Any + value other than zero is used as a return from pcre2test's callout function. Inserting callouts can be helpful when using pcre2test to check compli- - cated regular expressions. For further information about callouts, see + cated regular expressions. For further information about callouts, see the pcre2callout documentation. NON-PRINTING CHARACTERS When pcre2test is outputting text in the compiled version of a pattern, - bytes other than 32-126 are always treated as non-printing characters + bytes other than 32-126 are always treated as non-printing characters and are therefore shown as hex escapes. - When pcre2test is outputting text that is a matched part of a subject - string, it behaves in the same way, unless a different locale has been - set for the pattern (using the locale modifier). In this case, the is- + When pcre2test is outputting text that is a matched part of a subject + string, it behaves in the same way, unless a different locale has been + set for the pattern (using the locale modifier). In this case, the is- print() function is used to distinguish printing and non-printing char- acters. SAVING AND RESTORING COMPILED PATTERNS - It is possible to save compiled patterns on disc or elsewhere, and re- - load them later, subject to a number of restrictions. JIT data cannot - be saved. The host on which the patterns are reloaded must be running + It is possible to save compiled patterns on disc or elsewhere, and re- + load them later, subject to a number of restrictions. JIT data cannot + be saved. The host on which the patterns are reloaded must be running the same version of PCRE2, with the same code unit width, and must also - have the same endianness, pointer width and PCRE2_SIZE type. Before - compiled patterns can be saved they must be serialized, that is, con- - verted to a stream of bytes. A single byte stream may contain any num- - ber of compiled patterns, but they must all use the same character ta- - bles. A single copy of the tables is included in the byte stream (its + have the same endianness, pointer width and PCRE2_SIZE type. Before + compiled patterns can be saved they must be serialized, that is, con- + verted to a stream of bytes. A single byte stream may contain any num- + ber of compiled patterns, but they must all use the same character ta- + bles. A single copy of the tables is included in the byte stream (its size is 1088 bytes). - The functions whose names begin with pcre2_serialize_ are used for se- - rializing and de-serializing. They are described in the pcre2serialize - documentation. In this section we describe the features of pcre2test + The functions whose names begin with pcre2_serialize_ are used for se- + rializing and de-serializing. They are described in the pcre2serialize + documentation. In this section we describe the features of pcre2test that can be used to test these functions. - Note that "serialization" in PCRE2 does not convert compiled patterns - to an abstract format like Java or .NET. It just makes a reloadable + Note that "serialization" in PCRE2 does not convert compiled patterns + to an abstract format like Java or .NET. It just makes a reloadable byte code stream. Hence the restrictions on reloading mentioned above. - In pcre2test, when a pattern with push modifier is successfully com- - piled, it is pushed onto a stack of compiled patterns, and pcre2test - expects the next line to contain a new pattern (or command) instead of + In pcre2test, when a pattern with push modifier is successfully com- + piled, it is pushed onto a stack of compiled patterns, and pcre2test + expects the next line to contain a new pattern (or command) instead of a subject line. By contrast, the pushcopy modifier causes a copy of the - compiled pattern to be stacked, leaving the original available for im- - mediate matching. By using push and/or pushcopy, a number of patterns - can be compiled and retained. These modifiers are incompatible with + compiled pattern to be stacked, leaving the original available for im- + mediate matching. By using push and/or pushcopy, a number of patterns + can be compiled and retained. These modifiers are incompatible with posix, and control modifiers that act at match time are ignored (with a - message) for the stacked patterns. The jitverify modifier applies only + message) for the stacked patterns. The jitverify modifier applies only at compile time. The command @@ -2034,21 +2030,21 @@ SAVING AND RESTORING COMPILED PATTERNS #save causes all the stacked patterns to be serialized and the result written - to the named file. Afterwards, all the stacked patterns are freed. The + to the named file. Afterwards, all the stacked patterns are freed. The command #load - reads the data in the file, and then arranges for it to be de-serial- - ized, with the resulting compiled patterns added to the pattern stack. - The pattern on the top of the stack can be retrieved by the #pop com- - mand, which must be followed by lines of subjects that are to be - matched with the pattern, terminated as usual by an empty line or end - of file. This command may be followed by a modifier list containing - only control modifiers that act after a pattern has been compiled. In - particular, hex, posix, posix_nosub, push, and pushcopy are not al- - lowed, nor are any option-setting modifiers. The JIT modifiers are, - however permitted. Here is an example that saves and reloads two pat- + reads the data in the file, and then arranges for it to be de-serial- + ized, with the resulting compiled patterns added to the pattern stack. + The pattern on the top of the stack can be retrieved by the #pop com- + mand, which must be followed by lines of subjects that are to be + matched with the pattern, terminated as usual by an empty line or end + of file. This command may be followed by a modifier list containing + only control modifiers that act after a pattern has been compiled. In + particular, hex, posix, posix_nosub, push, and pushcopy are not al- + lowed, nor are any option-setting modifiers. The JIT modifiers are, + however permitted. Here is an example that saves and reloads two pat- terns. /abc/push @@ -2061,10 +2057,10 @@ SAVING AND RESTORING COMPILED PATTERNS #pop jit,bincode abc - If jitverify is used with #pop, it does not automatically imply jit, + If jitverify is used with #pop, it does not automatically imply jit, which is different behaviour from when it is used on a pattern. - The #popcopy command is analogous to the pushcopy modifier in that it + The #popcopy command is analogous to the pushcopy modifier in that it makes current a copy of the topmost stack pattern, leaving the original still on the stack. From 31c64e7a70c489303eafed4c9556e9ce33807f5e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 19 Mar 2025 07:40:38 +0000 Subject: [PATCH 4/6] Sync autogenerated files #noupdate --- src/pcre2.h.generic | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pcre2.h.generic b/src/pcre2.h.generic index ee90b3d00..ddfbed59c 100644 --- a/src/pcre2.h.generic +++ b/src/pcre2.h.generic @@ -743,14 +743,14 @@ PCRE2_EXP_DECL pcre2_match_data *PCRE2_CALL_CONVENTION \ PCRE2_EXP_DECL pcre2_match_data *PCRE2_CALL_CONVENTION \ pcre2_match_data_create_from_pattern(const pcre2_code *, \ pcre2_general_context *); \ +PCRE2_EXP_DECL void PCRE2_CALL_CONVENTION \ + pcre2_match_data_free(pcre2_match_data *); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_dfa_match(const pcre2_code *, PCRE2_SPTR, PCRE2_SIZE, PCRE2_SIZE, \ uint32_t, pcre2_match_data *, pcre2_match_context *, int *, PCRE2_SIZE); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_match(const pcre2_code *, PCRE2_SPTR, PCRE2_SIZE, PCRE2_SIZE, \ uint32_t, pcre2_match_data *, pcre2_match_context *); \ -PCRE2_EXP_DECL void PCRE2_CALL_CONVENTION \ - pcre2_match_data_free(pcre2_match_data *); \ PCRE2_EXP_DECL PCRE2_SPTR PCRE2_CALL_CONVENTION \ pcre2_get_mark(pcre2_match_data *); \ PCRE2_EXP_DECL PCRE2_SIZE PCRE2_CALL_CONVENTION \ @@ -762,7 +762,9 @@ PCRE2_EXP_DECL uint32_t PCRE2_CALL_CONVENTION \ PCRE2_EXP_DECL PCRE2_SIZE *PCRE2_CALL_CONVENTION \ pcre2_get_ovector_pointer(pcre2_match_data *); \ PCRE2_EXP_DECL PCRE2_SIZE PCRE2_CALL_CONVENTION \ - pcre2_get_startchar(pcre2_match_data *); + pcre2_get_startchar(pcre2_match_data *); \ +PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ + pcre2_next_match(pcre2_match_data *, PCRE2_SIZE *, uint32_t *); /* Convenience functions for handling matched substrings. */ @@ -942,6 +944,7 @@ pcre2_compile are called by application code. */ #define pcre2_match_data_create PCRE2_SUFFIX(pcre2_match_data_create_) #define pcre2_match_data_create_from_pattern PCRE2_SUFFIX(pcre2_match_data_create_from_pattern_) #define pcre2_match_data_free PCRE2_SUFFIX(pcre2_match_data_free_) +#define pcre2_next_match PCRE2_SUFFIX(pcre2_next_match_) #define pcre2_pattern_convert PCRE2_SUFFIX(pcre2_pattern_convert_) #define pcre2_pattern_info PCRE2_SUFFIX(pcre2_pattern_info_) #define pcre2_serialize_decode PCRE2_SUFFIX(pcre2_serialize_decode_) From 0136d7e52d53edcb7a1a567ef271fe8513d65dfe Mon Sep 17 00:00:00 2001 From: Nicholas Wilson Date: Wed, 19 Mar 2025 20:20:07 +0000 Subject: [PATCH 5/6] Add some additional documentation --- doc/html/pcre2_next_match.html | 21 +++++----- doc/html/pcre2api.html | 68 +++++++++++++++++------------- doc/html/pcre2compat.html | 4 +- doc/html/pcre2demo.html | 51 +++++++++++++++++++---- doc/pcre2.txt | 75 +++++++++++++++++++--------------- doc/pcre2_next_match.3 | 21 +++++----- doc/pcre2api.3 | 73 ++++++++++++++++++--------------- doc/pcre2compat.3 | 4 +- doc/pcre2demo.3 | 53 +++++++++++++++++++----- src/pcre2_match_next.c | 1 + src/pcre2_substitute.c | 4 +- src/pcre2demo.c | 51 +++++++++++++++++++---- src/pcre2test.c | 2 +- 13 files changed, 281 insertions(+), 147 deletions(-) diff --git a/doc/html/pcre2_next_match.html b/doc/html/pcre2_next_match.html index fddaf9d34..f5f8c4664 100644 --- a/doc/html/pcre2_next_match.html +++ b/doc/html/pcre2_next_match.html @@ -26,22 +26,23 @@

    DESCRIPTION

    -This function can be called after calling one of the matching functions -(pcre2_match(), pcre2_dfa_match(), or pcre2_jit_match()), by -providing the same match_data parameter. It provides the appropriate -parameters for searching for the next match in the same subject string, and is -suitable for applications providing "global" matching behaviour (replacing all -matches in the subject, or splitting the subject on all matches, or simply -counting the number of matches). +This function can be called after one of the match functions +(pcre2_match(), pcre2_dfa_match(), or pcre2_jit_match()), and +must be provided with the same match_data parameter. It outputs the +appropriate parameters for searching for the next match in the same subject +string, and is suitable for applications providing "global" matching behaviour +(for example, replacing all matches in the subject, or splitting the subject on +all matches, or simply counting the number of matches).

    It returns 0 ("false") if there is no need to make any further match attempts, or 1 ("true") if another match should be attempted.

    -The pstart_offset and poptions are set if the function returns 1. -The *poptions should be combined with the application's match options -using OR. +The *pstart_offset and *poptions are set if the function returns 1. +The *pstart_offset should be passed to the next match attempt directly, +and the *poptions should be passed to the next match attempt by combining +with the application's match options using OR.

    There is a complete description of the PCRE2 native API in the diff --git a/doc/html/pcre2api.html b/doc/html/pcre2api.html index faeec8671..4524fc98e 100644 --- a/doc/html/pcre2api.html +++ b/doc/html/pcre2api.html @@ -3520,23 +3520,28 @@

    ITERATING OVER ALL MATCHES

    First, a match attempt should be made using one of the matching functions -(pcre2_match(), pcre2_dfa_match(), or pcre2_jit_match()). Then, -pcre2_next_match() can be called, providing the same match_data -parameter. +(pcre2_match(), pcre2_dfa_match(), or pcre2_jit_match()). +Then, pcre2_next_match() can be called, providing the same +match_data parameter.

    -It returns 0 ("false") if there is no need to make any further match attempts, -or 1 ("true") if another match should be attempted. +It returns 0 ("false") if there is no need to make a further match attempt, or +1 ("true") if another match should be attempted. Returning 1 does not imply that +there is another match, only that another match should be attempted (which may +return PCRE2_ERROR_NOMATCH).

    -The pstart_offset and poptions are set if the function returns 1. -The *poptions should be combined with the application's match options -using OR. +The *pstart_offset and *poptions are set if the function returns 1. +The *pstart_offset should be passed to the next match attempt directly, +and the *poptions should be passed to the next match attempt by combining +with the application's match options using OR.

    There is some code that demonstrates how to do this in the pcre2demo sample program. The general pattern is: +
    +

       uint32_t app_options = ...;
   uint32_t global_options = 0;
@@ -3546,32 +3551,36 @@ 
    ITERATING OVER ALL MATCHES
    
     int rc = pcre2_match(re, subject, subject_len, start_offset,
                          app_options | global_options, match_data,
                          match_context);
-    if (rc == PCRE2_ERROR_NOMATCH) break;
+
+    if (rc == PCRE2_ERROR_NOMATCH) break; /* no match, and no more attempts */
     if (rc < 0) { ... exit }
+
+    ...handle the match
+
     if (!pcre2_next_match(match_data, &start_offset, &global_options))
-      break;
+      break; /* no more attempts */
     }

    The guarantees provided by pcre2_next_match() are that the start_offset -will advance, so the loop will definitely terminate. The condition for progress -is that either: (a) pcre2_next_match() returns 0 (false); or (b) the returned -start_offset is strictly greater than the previous start_offset, or (c) if the -previous match was a successful match of the empty string then the returned -start_offset is equal to previous start_offset, but poptions will be set to -PCRE2_NOTEMPTY_ATSTART to prevent another empty match from being returned. +will advance, so the loop will definitely terminate. The conditions which +ensure this are that either: (a) pcre2_next_match() returns 0 (false); or +(b) the returned *pstart_offset is strictly greater than the previous +start_offset; or (c) if the previous match was a successful match of the empty +string then the returned *pstart_offset is equal to the previous +ovector[1], and *poptions will be set to PCRE2_NOTEMPTY_ATSTART to prevent +another empty match from being returned.

    -In a loop as shown above, it must terminate, unless there is a bug in PCRE2. As -a measure of "defensive programming", applications are encouraged to add an -assertion or check to break their loop if it does not make progress (and report -the issue as a bug). +A loop implemented as shown above will always terminate, unless there is a bug +in PCRE2. As a measure of "defensive programming", applications are encouraged +to add an assertion or check to break their loop if it does not make progress +(and report the issue as a bug).

    -Note that we do not guarantee that the matches will always advance: only that -the start_offset will. If an application does not use the flag -PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK, then matches are "well-behaved" and satisfy: +If an application does not use the flag PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK, then +each match is "well-behaved" and satisfies: 

       start_offset <= ovector[0] <= ovector[1].
    @@ -3580,11 +3589,14 @@

    ITERATING OVER ALL MATCHES

    Otherwise, if PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK is used, then the guarantees are -considerably weaker. The matches found by pcre2_match() with pcre2_next_match() -will be a finite sequence (as pcre2_next_match() ensures that start_offset -advances, so the search will terminate). The matches can however be overlapping, -not sorted, and contain duplicates. Each match itself can end before it starts -(ovector[1] < ovector[0]). +considerably weaker. We do not guarantee that the matches will always advance: +only that the start_offset will. The matches found by pcre2_match() with +pcre2_next_match() will be a finite sequence (as pcre2_next_match() ensures that +start_offset advances, so the search will terminate). The matches can however be +overlapping, can contain duplicates, and (in truly pathological examples) may +not even be sorted by ovector[0]. Additionally, each match itself can end before +it starts (ovector[1] < ovector[0]). We recommend that applications do not set +PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK.

    EXTRACTING CAPTURED SUBSTRINGS BY NUMBER

    diff --git a/doc/html/pcre2compat.html b/doc/html/pcre2compat.html index 4d35d51d8..2bd027479 100644 --- a/doc/html/pcre2compat.html +++ b/doc/html/pcre2compat.html @@ -218,7 +218,7 @@

    (h) The partial matching facility is PCRE2-specific.

    -(i) The alternative matching function (pcre2_dfa_match() matches in a +(i) The alternative matching function (pcre2_dfa_match()) matches in a different way and is not Perl-compatible.

    @@ -266,7 +266,7 @@

    23. Both PCRE2 and Perl error when \x{ escapes are invalid, but Perl tries to recover and prints a warning if the problem was that an invalid hexadecimal -digit was found, since PCRE2 doesn't have warnings it returns an error instead. +digit was found. Since PCRE2 doesn't have warnings it returns an error instead. Additionally, Perl accepts \x{} and generates NUL unlike PCRE2.

    diff --git a/doc/html/pcre2demo.html b/doc/html/pcre2demo.html index 0db6df437..37525fdb7 100644 --- a/doc/html/pcre2demo.html +++ b/doc/html/pcre2demo.html @@ -101,6 +101,7 @@

    PCRE2_SIZE erroroffset; PCRE2_SIZE *ovector; +PCRE2_SIZE ovector_last[2]; PCRE2_SIZE subject_length; pcre2_match_data *match_data; @@ -237,11 +238,12 @@

    printf("ovector was not big enough for all the captured substrings\n"); /* Since release 10.38 PCRE2 has locked out the use of \K in lookaround -assertions. However, there is an option to re-enable the old behaviour. If that -is set, it is possible to run patterns such as /(?=.\K)/ that use \K in an -assertion to set the start of a match later than its end. In this demonstration -program, we show how to detect this case, but it shouldn't arise because the -option is never set. */ +assertions. This is the recommended behaviour. However, the option +PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK allows applications to re-enable the old +behaviour. If that is set, it is possible to run patterns such as /(?=.\K)/ that +use \K in an assertion to set the start of a match later than its end. In this +demonstration program, we show how to detect this case, although it cannot arise +because the option is never set. */ if (ovector[0] > ovector[1]) { @@ -333,6 +335,9 @@

    /* Loop for second and subsequent matches */ +ovector_last[0] = ovector[0]; +ovector_last[1] = ovector[1]; + for (;;) { PCRE2_SIZE start_offset; @@ -370,7 +375,33 @@

    return 1; } - /* Match succeeded */ + /* This demonstration program depends on pcre2_next_match() to ensure that the + loop for second and subsequent matches does not run forever. However, it would + be robust practice for a production application to verify this. The following + block of code shows how to do this. This error case is not reachable unless + there is a bug in PCRE2. + + Because this program does not set the PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK option, + the logic is simple. We verify that either ovector[1] has advanced, or that we + have an empty match touching the end of a previous non-empty match. See the + API documentation for guidance if your application uses + PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK and searches for multiple matches. */ + + if (!(ovector[1] > ovector_last[1] || + (ovector[1] == ovector[0] && ovector_last[1] > ovector_last[0] && + ovector[1] == ovector_last[1]))) + { + printf("\\K was used in an assertion to yield non-advancing matches.\n"); + printf("Run abandoned\n"); + pcre2_match_data_free(match_data); + pcre2_code_free(re); + return 1; + } + + ovector_last[0] = ovector[0]; + ovector_last[1] = ovector[1]; + + /* Match succeeded. */ printf("\nMatch succeeded again at offset %d\n", (int)ovector[0]); @@ -380,9 +411,11 @@

    if (rc == 0) printf("ovector was not big enough for all the captured substrings\n"); - /* We must guard against patterns such as /(?=.\K)/ that use \K in an - assertion to set the start of a match later than its end. In this - demonstration program, we just detect this case and give up. */ + /* We guard against patterns such as /(?=.\K)/ that use \K in an assertion to + set the start of a match later than its end. As explained above, this case + should not occur because this demonstration program does not set the + PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK option, however, we do include code showing + how to detect it. */ if (ovector[0] > ovector[1]) { diff --git a/doc/pcre2.txt b/doc/pcre2.txt index c65cfcf52..c80b484d9 100644 --- a/doc/pcre2.txt +++ b/doc/pcre2.txt @@ -3408,19 +3408,23 @@ ITERATING OVER ALL MATCHES propriate parameters for the next match attempt. First, a match attempt should be made using one of the matching func- - tions (pcre2_match(), pcre2_dfa_match(), or pcre2_jit_match()). Then, + tions (pcre2_match(), pcre2_dfa_match(), or pcre2_jit_match()). Then, pcre2_next_match() can be called, providing the same match_data parame- ter. - It returns 0 ("false") if there is no need to make any further match - attempts, or 1 ("true") if another match should be attempted. + It returns 0 ("false") if there is no need to make a further match at- + tempt, or 1 ("true") if another match should be attempted. Returning 1 + does not imply that there is another match, only that another match + should be attempted (which may return PCRE2_ERROR_NOMATCH). - The pstart_offset and poptions are set if the function returns 1. The - *poptions should be combined with the application's match options using - OR. + The *pstart_offset and *poptions are set if the function returns 1. + The *pstart_offset should be passed to the next match attempt directly, + and the *poptions should be passed to the next match attempt by combin- + ing with the application's match options using OR. There is some code that demonstrates how to do this in the pcre2demo sample program. The general pattern is: + uint32_t app_options = ...; uint32_t global_options = 0; PCRE2_SIZE start_offset = 0; @@ -3429,30 +3433,33 @@ ITERATING OVER ALL MATCHES int rc = pcre2_match(re, subject, subject_len, start_offset, app_options | global_options, match_data, match_context); - if (rc == PCRE2_ERROR_NOMATCH) break; + + if (rc == PCRE2_ERROR_NOMATCH) break; /* no match, and no more attempts */ if (rc < 0) { ... exit } + + ...handle the match + if (!pcre2_next_match(match_data, &start_offset, &global_options)) - break; + break; /* no more attempts */ } The guarantees provided by pcre2_next_match() are that the start_offset - will advance, so the loop will definitely terminate. The condition for - progress is that either: (a) pcre2_next_match() returns 0 (false); or - (b) the returned start_offset is strictly greater than the previous - start_offset, or (c) if the previous match was a successful match of - the empty string then the returned start_offset is equal to previous - start_offset, but poptions will be set to PCRE2_NOTEMPTY_ATSTART to - prevent another empty match from being returned. - - In a loop as shown above, it must terminate, unless there is a bug in - PCRE2. As a measure of "defensive programming", applications are en- - couraged to add an assertion or check to break their loop if it does - not make progress (and report the issue as a bug). - - Note that we do not guarantee that the matches will always advance: - only that the start_offset will. If an application does not use the - flag PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK, then matches are "well-behaved" - and satisfy: + will advance, so the loop will definitely terminate. The conditions + which ensure this are that either: (a) pcre2_next_match() returns 0 + (false); or (b) the returned *pstart_offset is strictly greater than + the previous start_offset; or (c) if the previous match was a success- + ful match of the empty string then the returned *pstart_offset is equal + to the previous ovector[1], and *poptions will be set to + PCRE2_NOTEMPTY_ATSTART to prevent another empty match from being re- + turned. + + A loop implemented as shown above will always terminate, unless there + is a bug in PCRE2. As a measure of "defensive programming", applica- + tions are encouraged to add an assertion or check to break their loop + if it does not make progress (and report the issue as a bug). + + If an application does not use the flag PCRE2_EXTRA_AL- + LOW_LOOKAROUND_BSK, then each match is "well-behaved" and satisfies: start_offset <= ovector[0] <= ovector[1]. @@ -3461,11 +3468,15 @@ ITERATING OVER ALL MATCHES and with no duplicates. Otherwise, if PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK is used, then the guar- - antees are considerably weaker. The matches found by pcre2_match() with - pcre2_next_match() will be a finite sequence (as pcre2_next_match() en- - sures that start_offset advances, so the search will terminate). The - matches can however be overlapping, not sorted, and contain duplicates. - Each match itself can end before it starts (ovector[1] < ovector[0]). + antees are considerably weaker. We do not guarantee that the matches + will always advance: only that the start_offset will. The matches found + by pcre2_match() with pcre2_next_match() will be a finite sequence (as + pcre2_next_match() ensures that start_offset advances, so the search + will terminate). The matches can however be overlapping, can contain + duplicates, and (in truly pathological examples) may not even be sorted + by ovector[0]. Additionally, each match itself can end before it starts + (ovector[1] < ovector[0]). We recommend that applications do not set + PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK. EXTRACTING CAPTURED SUBSTRINGS BY NUMBER @@ -5633,7 +5644,7 @@ DIFFERENCES BETWEEN PCRE2 AND PERL (h) The partial matching facility is PCRE2-specific. - (i) The alternative matching function (pcre2_dfa_match() matches in a + (i) The alternative matching function (pcre2_dfa_match()) matches in a different way and is not Perl-compatible. (j) PCRE2 recognizes some special sequences such as (*CR) or (*NO_JIT) @@ -5674,7 +5685,7 @@ DIFFERENCES BETWEEN PCRE2 AND PERL 23. Both PCRE2 and Perl error when \x{ escapes are invalid, but Perl tries to recover and prints a warning if the problem was that an in- - valid hexadecimal digit was found, since PCRE2 doesn't have warnings it + valid hexadecimal digit was found. Since PCRE2 doesn't have warnings it returns an error instead. Additionally, Perl accepts \x{} and gener- ates NUL unlike PCRE2. diff --git a/doc/pcre2_next_match.3 b/doc/pcre2_next_match.3 index ea655762c..fa1f6e696 100644 --- a/doc/pcre2_next_match.3 +++ b/doc/pcre2_next_match.3 @@ -14,20 +14,21 @@ PCRE2 - Perl-compatible regular expressions (revised API) .SH DESCRIPTION .rs .sp -This function can be called after calling one of the matching functions -(\fBpcre2_match()\fP, \fBpcre2_dfa_match()\fP, or \fBpcre2_jit_match()\fP), by -providing the same \fImatch_data\fP parameter. It provides the appropriate -parameters for searching for the next match in the same subject string, and is -suitable for applications providing "global" matching behaviour (replacing all -matches in the subject, or splitting the subject on all matches, or simply -counting the number of matches). +This function can be called after one of the match functions +(\fBpcre2_match()\fP, \fBpcre2_dfa_match()\fP, or \fBpcre2_jit_match()\fP), and +must be provided with the same \fImatch_data\fP parameter. It outputs the +appropriate parameters for searching for the next match in the same subject +string, and is suitable for applications providing "global" matching behaviour +(for example, replacing all matches in the subject, or splitting the subject on +all matches, or simply counting the number of matches). .P It returns 0 ("false") if there is no need to make any further match attempts, or 1 ("true") if another match should be attempted. .P -The \fIpstart_offset\fP and \fIpoptions\fP are set if the function returns 1. -The *\fIpoptions\fP should be combined with the application's match options -using OR. +The *\fIpstart_offset\fP and *\fIpoptions\fP are set if the function returns 1. +The *\fIpstart_offset\fP should be passed to the next match attempt directly, +and the *\fIpoptions\fP should be passed to the next match attempt by combining +with the application's match options using OR. .P There is a complete description of the PCRE2 native API in the .\" HREF diff --git a/doc/pcre2api.3 b/doc/pcre2api.3 index 4dd61553a..1a5db85de 100644 --- a/doc/pcre2api.3 +++ b/doc/pcre2api.3 @@ -3537,22 +3537,26 @@ function helps with this task by providing the appropriate parameters for the next match attempt. .P First, a match attempt should be made using one of the matching functions -(\fBpcre2_match()\fP, \fBpcre2_dfa_match()\fP, or \fBpcre2_jit_match()\fP). Then, -\fBpcre2_next_match()\fP can be called, providing the same \fImatch_data\fP -parameter. +(\fBpcre2_match()\fP, \fBpcre2_dfa_match()\fP, or \fBpcre2_jit_match()\fP). +Then, \fBpcre2_next_match()\fP can be called, providing the same +\fImatch_data\fP parameter. .P -It returns 0 ("false") if there is no need to make any further match attempts, -or 1 ("true") if another match should be attempted. +It returns 0 ("false") if there is no need to make a further match attempt, or +1 ("true") if another match should be attempted. Returning 1 does not imply that +there is another match, only that another match should be attempted (which may +return PCRE2_ERROR_NOMATCH). .P -The \fIpstart_offset\fP and \fIpoptions\fP are set if the function returns 1. -The *\fIpoptions\fP should be combined with the application's match options -using OR. +The *\fIpstart_offset\fP and *\fIpoptions\fP are set if the function returns 1. +The *\fIpstart_offset\fP should be passed to the next match attempt directly, +and the *\fIpoptions\fP should be passed to the next match attempt by combining +with the application's match options using OR. .P There is some code that demonstrates how to do this in the .\" HREF \fBpcre2demo\fP .\" sample program. The general pattern is: +.sp .nf uint32_t app_options = ...; uint32_t global_options = 0; @@ -3562,31 +3566,33 @@ sample program. The general pattern is: int rc = pcre2_match(re, subject, subject_len, start_offset, app_options | global_options, match_data, match_context); -. - if (rc == PCRE2_ERROR_NOMATCH) break; +\& + if (rc == PCRE2_ERROR_NOMATCH) break; /* no match, and no more attempts */ if (rc < 0) { ... exit } -. +\& + ...handle the match +\& if (!pcre2_next_match(match_data, &start_offset, &global_options)) - break; + break; /* no more attempts */ } .fi .P The guarantees provided by \fBpcre2_next_match()\fP are that the start_offset -will advance, so the loop will definitely terminate. The condition for progress -is that either: (a) pcre2_next_match() returns 0 (false); or (b) the returned -start_offset is strictly greater than the previous start_offset, or (c) if the -previous match was a successful match of the empty string then the returned -start_offset is equal to previous start_offset, but poptions will be set to -PCRE2_NOTEMPTY_ATSTART to prevent another empty match from being returned. -.P -In a loop as shown above, it must terminate, unless there is a bug in PCRE2. As -a measure of "defensive programming", applications are encouraged to add an -assertion or check to break their loop if it does not make progress (and report -the issue as a bug). -.P -Note that we do not guarantee that the matches will always advance: only that -the start_offset will. If an application does not use the flag -PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK, then matches are "well-behaved" and satisfy: +will advance, so the loop will definitely terminate. The conditions which +ensure this are that either: (a) pcre2_next_match() returns 0 (false); or +(b) the returned *\fIpstart_offset\fP is strictly greater than the previous +start_offset; or (c) if the previous match was a successful match of the empty +string then the returned *\fIpstart_offset\fP is equal to the previous +ovector[1], and *\fIpoptions\fP will be set to PCRE2_NOTEMPTY_ATSTART to prevent +another empty match from being returned. +.P +A loop implemented as shown above will always terminate, unless there is a bug +in PCRE2. As a measure of "defensive programming", applications are encouraged +to add an assertion or check to break their loop if it does not make progress +(and report the issue as a bug). +.P +If an application does not use the flag PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK, then +each match is "well-behaved" and satisfies: .sp start_offset <= ovector[0] <= ovector[1]. .sp @@ -3594,11 +3600,14 @@ In this case, the matches found by pcre2_match() with pcre2_next_match() will be sorted, non-overlapping (possibly touching), and with no duplicates. .P Otherwise, if PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK is used, then the guarantees are -considerably weaker. The matches found by pcre2_match() with pcre2_next_match() -will be a finite sequence (as pcre2_next_match() ensures that start_offset -advances, so the search will terminate). The matches can however be overlapping, -not sorted, and contain duplicates. Each match itself can end before it starts -(ovector[1] < ovector[0]). +considerably weaker. We do not guarantee that the matches will always advance: +only that the start_offset will. The matches found by pcre2_match() with +pcre2_next_match() will be a finite sequence (as pcre2_next_match() ensures that +start_offset advances, so the search will terminate). The matches can however be +overlapping, can contain duplicates, and (in truly pathological examples) may +not even be sorted by ovector[0]. Additionally, each match itself can end before +it starts (ovector[1] < ovector[0]). We recommend that applications do not set +PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK. . . .\" HTML diff --git a/doc/pcre2compat.3 b/doc/pcre2compat.3 index ff0a760c3..b52395f2d 100644 --- a/doc/pcre2compat.3 +++ b/doc/pcre2compat.3 @@ -186,7 +186,7 @@ variable interpolation, but not general hooks on every match. .sp (h) The partial matching facility is PCRE2-specific. .sp -(i) The alternative matching function (\fBpcre2_dfa_match()\fP matches in a +(i) The alternative matching function (\fBpcre2_dfa_match()\fP) matches in a different way and is not Perl-compatible. .sp (j) PCRE2 recognizes some special sequences such as (*CR) or (*NO_JIT) at @@ -228,7 +228,7 @@ handled by PCRE2, either by the interpreter or the JIT. An example is .P 23. Both PCRE2 and Perl error when \ex{ escapes are invalid, but Perl tries to recover and prints a warning if the problem was that an invalid hexadecimal -digit was found, since PCRE2 doesn't have warnings it returns an error instead. +digit was found. Since PCRE2 doesn't have warnings it returns an error instead. Additionally, Perl accepts \ex{} and generates NUL unlike PCRE2. .P 24. From release 10.45, PCRE2 gives an error if \ex is not followed by a diff --git a/doc/pcre2demo.3 b/doc/pcre2demo.3 index de2357c07..ab98714ba 100644 --- a/doc/pcre2demo.3 +++ b/doc/pcre2demo.3 @@ -1,4 +1,4 @@ -.TH PCRE2DEMO 3 "18 March 2025" "PCRE2 10.46-DEV" +.TH PCRE2DEMO 3 "19 March 2025" "PCRE2 10.46-DEV" .\"AUTOMATICALLY GENERATED BY UpdateAlways - do not EDIT! .SH NAME PCRE2DEMO - A demonstration C program for PCRE2 @@ -111,6 +111,7 @@ uint32_t name_entry_size; PCRE2_SIZE erroroffset; PCRE2_SIZE *ovector; +PCRE2_SIZE ovector_last[2]; PCRE2_SIZE subject_length; pcre2_match_data *match_data; @@ -247,11 +248,12 @@ if (rc == 0) printf("ovector was not big enough for all the captured substrings\en"); /* Since release 10.38 PCRE2 has locked out the use of \eK in lookaround -assertions. However, there is an option to re-enable the old behaviour. If that -is set, it is possible to run patterns such as /(?=.\eK)/ that use \eK in an -assertion to set the start of a match later than its end. In this demonstration -program, we show how to detect this case, but it shouldn't arise because the -option is never set. */ +assertions. This is the recommended behaviour. However, the option +PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK allows applications to re-enable the old +behaviour. If that is set, it is possible to run patterns such as /(?=.\eK)/ that +use \eK in an assertion to set the start of a match later than its end. In this +demonstration program, we show how to detect this case, although it cannot arise +because the option is never set. */ if (ovector[0] > ovector[1]) { @@ -343,6 +345,9 @@ if (!find_all) /* Check for -g */ /* Loop for second and subsequent matches */ +ovector_last[0] = ovector[0]; +ovector_last[1] = ovector[1]; + for (;;) { PCRE2_SIZE start_offset; @@ -380,7 +385,33 @@ for (;;) return 1; } - /* Match succeeded */ + /* This demonstration program depends on pcre2_next_match() to ensure that the + loop for second and subsequent matches does not run forever. However, it would + be robust practice for a production application to verify this. The following + block of code shows how to do this. This error case is not reachable unless + there is a bug in PCRE2. + + Because this program does not set the PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK option, + the logic is simple. We verify that either ovector[1] has advanced, or that we + have an empty match touching the end of a previous non-empty match. See the + API documentation for guidance if your application uses + PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK and searches for multiple matches. */ + + if (!(ovector[1] > ovector_last[1] || + (ovector[1] == ovector[0] && ovector_last[1] > ovector_last[0] && + ovector[1] == ovector_last[1]))) + { + printf("\e\eK was used in an assertion to yield non-advancing matches.\en"); + printf("Run abandoned\en"); + pcre2_match_data_free(match_data); + pcre2_code_free(re); + return 1; + } + + ovector_last[0] = ovector[0]; + ovector_last[1] = ovector[1]; + + /* Match succeeded. */ printf("\enMatch succeeded again at offset %d\en", (int)ovector[0]); @@ -390,9 +421,11 @@ for (;;) if (rc == 0) printf("ovector was not big enough for all the captured substrings\en"); - /* We must guard against patterns such as /(?=.\eK)/ that use \eK in an - assertion to set the start of a match later than its end. In this - demonstration program, we just detect this case and give up. */ + /* We guard against patterns such as /(?=.\eK)/ that use \eK in an assertion to + set the start of a match later than its end. As explained above, this case + should not occur because this demonstration program does not set the + PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK option, however, we do include code showing + how to detect it. */ if (ovector[0] > ovector[1]) { diff --git a/src/pcre2_match_next.c b/src/pcre2_match_next.c index 096ca7e04..ecd21d380 100644 --- a/src/pcre2_match_next.c +++ b/src/pcre2_match_next.c @@ -78,6 +78,7 @@ if (utf) PCRE2_SPTR next = subject + offset + 1; PCRE2_SPTR subject_end = subject + subject_length; + (void)subject_end; /* Suppress warning; 32-bit FORWARDCHARTEST ignores this */ FORWARDCHARTEST(next, subject_end); return next - subject; } diff --git a/src/pcre2_substitute.c b/src/pcre2_substitute.c index f56609700..4aea7f90e 100644 --- a/src/pcre2_substitute.c +++ b/src/pcre2_substitute.c @@ -754,7 +754,7 @@ PCRE2_SPTR repend = NULL; PCRE2_SIZE extra_needed = 0; PCRE2_SIZE buff_offset, buff_length, lengthleft, fraglength; PCRE2_SIZE *ovector; -PCRE2_SIZE ovecsave[2]; +PCRE2_SIZE ovecsave[2] = { 0, 0 }; pcre2_substitute_callout_block scb; PCRE2_SIZE sub_start_extra_needed; PCRE2_SIZE (*substitute_case_callout)(PCRE2_SPTR, PCRE2_SIZE, PCRE2_UCHAR *, @@ -947,7 +947,7 @@ for (;;) if (subs > 0 && !(ovector[1] > ovecsave[1] || - (ovector[1] == ovector[0] && ovecsave[1] != ovecsave[0] && + (ovector[1] == ovector[0] && ovecsave[1] > ovecsave[0] && ovector[1] == ovecsave[1]))) { rc = PCRE2_ERROR_INTERNAL_DUPMATCH; diff --git a/src/pcre2demo.c b/src/pcre2demo.c index 23fb49fd1..fb9a7e2ee 100644 --- a/src/pcre2demo.c +++ b/src/pcre2demo.c @@ -83,6 +83,7 @@ uint32_t name_entry_size; PCRE2_SIZE erroroffset; PCRE2_SIZE *ovector; +PCRE2_SIZE ovector_last[2]; PCRE2_SIZE subject_length; pcre2_match_data *match_data; @@ -219,11 +220,12 @@ if (rc == 0) printf("ovector was not big enough for all the captured substrings\n"); /* Since release 10.38 PCRE2 has locked out the use of \K in lookaround -assertions. However, there is an option to re-enable the old behaviour. If that -is set, it is possible to run patterns such as /(?=.\K)/ that use \K in an -assertion to set the start of a match later than its end. In this demonstration -program, we show how to detect this case, but it shouldn't arise because the -option is never set. */ +assertions. This is the recommended behaviour. However, the option +PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK allows applications to re-enable the old +behaviour. If that is set, it is possible to run patterns such as /(?=.\K)/ that +use \K in an assertion to set the start of a match later than its end. In this +demonstration program, we show how to detect this case, although it cannot arise +because the option is never set. */ if (ovector[0] > ovector[1]) { @@ -315,6 +317,9 @@ if (!find_all) /* Check for -g */ /* Loop for second and subsequent matches */ +ovector_last[0] = ovector[0]; +ovector_last[1] = ovector[1]; + for (;;) { PCRE2_SIZE start_offset; @@ -352,7 +357,33 @@ for (;;) return 1; } - /* Match succeeded */ + /* This demonstration program depends on pcre2_next_match() to ensure that the + loop for second and subsequent matches does not run forever. However, it would + be robust practice for a production application to verify this. The following + block of code shows how to do this. This error case is not reachable unless + there is a bug in PCRE2. + + Because this program does not set the PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK option, + the logic is simple. We verify that either ovector[1] has advanced, or that we + have an empty match touching the end of a previous non-empty match. See the + API documentation for guidance if your application uses + PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK and searches for multiple matches. */ + + if (!(ovector[1] > ovector_last[1] || + (ovector[1] == ovector[0] && ovector_last[1] > ovector_last[0] && + ovector[1] == ovector_last[1]))) + { + printf("\\K was used in an assertion to yield non-advancing matches.\n"); + printf("Run abandoned\n"); + pcre2_match_data_free(match_data); + pcre2_code_free(re); + return 1; + } + + ovector_last[0] = ovector[0]; + ovector_last[1] = ovector[1]; + + /* Match succeeded. */ printf("\nMatch succeeded again at offset %d\n", (int)ovector[0]); @@ -362,9 +393,11 @@ for (;;) if (rc == 0) printf("ovector was not big enough for all the captured substrings\n"); - /* We must guard against patterns such as /(?=.\K)/ that use \K in an - assertion to set the start of a match later than its end. In this - demonstration program, we just detect this case and give up. */ + /* We guard against patterns such as /(?=.\K)/ that use \K in an assertion to + set the start of a match later than its end. As explained above, this case + should not occur because this demonstration program does not set the + PCRE2_EXTRA_ALLOW_LOOKAROUND_BSK option, however, we do include code showing + how to detect it. */ if (ovector[0] > ovector[1]) { diff --git a/src/pcre2test.c b/src/pcre2test.c index 45c195f9d..a35ba2e37 100644 --- a/src/pcre2test.c +++ b/src/pcre2test.c @@ -7477,7 +7477,7 @@ BOOL utf; BOOL subject_literal; PCRE2_SIZE *ovector; -uint8_t *ovecsave[2]; +uint8_t *ovecsave[2] = { NULL, NULL }; uint32_t oveccount; #ifdef SUPPORT_PCRE2_8 From 6effdefc302e5a7b7b97283c0f3265a8f78c6fd5 Mon Sep 17 00:00:00 2001 From: Nicholas Wilson Date: Thu, 20 Mar 2025 21:27:38 +0000 Subject: [PATCH 6/6] Boost test coverage a little --- src/pcre2test.c | 15 +++++++ testdata/testinput2 | 23 ++++++++++ testdata/testoutput2 | 101 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+) diff --git a/src/pcre2test.c b/src/pcre2test.c index a35ba2e37..8f7caa29d 100644 --- a/src/pcre2test.c +++ b/src/pcre2test.c @@ -8685,6 +8685,21 @@ for (gmatched = 0;; gmatched++) } } + /* Verify that it's safe to call pcre2_next_match with rc < 0. */ + + if (capcount < 0 && (dat_datctl.control & CTL_ANYGLOB) != 0) + { + BOOL rc_nextmatch; + PCRE2_SIZE tmp_offset = 0xcd; + uint32_t tmp_options = 0xcd; + PCRE2_NEXT_MATCH(rc_nextmatch, match_data, &tmp_offset, &tmp_options); + if (rc_nextmatch || tmp_offset != 0xcd || tmp_options != 0xcd) + { + fprintf(outfile, "** unexpected pcre2_next_match() for rc < 0\n"); + return PR_ABEND; + } + } + /* The result of the match is now in capcount. First handle a successful match. If pp was forced to be NULL (to test NULL handling) it will have been treated as an empty string if the length was zero. So re-create that for diff --git a/testdata/testinput2 b/testdata/testinput2 index b803a4787..7ee574567 100644 --- a/testdata/testinput2 +++ b/testdata/testinput2 @@ -4544,12 +4544,35 @@ # Perl loops on this (PCRE2 used to!) +# duplicate matches /(?<=\Ka)/g,aftertext,allow_lookaround_bsk aaaaa /(?<=\Ka)/altglobal,aftertext,allow_lookaround_bsk aaaaa +# overlapping matches +/(?(?=\Gc)(?<=\Kb)c|(?<=\Kab))/g,aftertext,allow_lookaround_bsk + abc + +# not-sorted matches +/(?(?=\Gc)(?<=\Kab)|(?<=\Kb))/g,aftertext,allow_lookaround_bsk + abc + +# CRLF cases + +/(?<=\Kback)|\Gstart/g,aftertext,allow_lookaround_bsk,newline=LF + back\rstart back\nstart back\r\nstart back\r + +/(?<=\Kback)|\Gstart/g,aftertext,allow_lookaround_bsk,newline=CRLF + back\rstart back\nstart back\r\nstart back\r + +/(?<=\Kback)|\Gstart/g,aftertext,allow_lookaround_bsk,newline=ANYCRLF + back\rstart back\nstart back\r\nstart back\r + +/(?<=\Kback)|\Gstart/g,aftertext,allow_lookaround_bsk,newline=ANY + back\rstart back\nstart back\r\nstart back\r + /((?2){73}(?2))((?1))/info /abc/ diff --git a/testdata/testoutput2 b/testdata/testoutput2 index 7c1bec701..a18f4e125 100644 --- a/testdata/testoutput2 +++ b/testdata/testoutput2 @@ -14731,6 +14731,7 @@ Failed: error 125 at offset 1: length of lookbehind assertion is not limited # Perl loops on this (PCRE2 used to!) +# duplicate matches /(?<=\Ka)/g,aftertext,allow_lookaround_bsk aaaaa 0: a @@ -14758,6 +14759,106 @@ global repeat returned the same match as previous 0: a 0+ +# overlapping matches +/(?(?=\Gc)(?<=\Kb)c|(?<=\Kab))/g,aftertext,allow_lookaround_bsk + abc + 0: ab + 0+ c + 0: bc + 0+ + +# not-sorted matches +/(?(?=\Gc)(?<=\Kab)|(?<=\Kb))/g,aftertext,allow_lookaround_bsk + abc + 0: b + 0+ c + 0: ab + 0+ c + +# CRLF cases + +/(?<=\Kback)|\Gstart/g,aftertext,allow_lookaround_bsk,newline=LF + back\rstart back\nstart back\r\nstart back\r + 0: back + 0+ \x0dstart back\x0astart back\x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: start + 0+ back\x0astart back\x0d\x0astart back\x0d + 0: back + 0+ \x0astart back\x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: start + 0+ back\x0d\x0astart back\x0d + 0: back + 0+ \x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: back + 0+ \x0d +global repeat returned the same match as previous + +/(?<=\Kback)|\Gstart/g,aftertext,allow_lookaround_bsk,newline=CRLF + back\rstart back\nstart back\r\nstart back\r + 0: back + 0+ \x0dstart back\x0astart back\x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: start + 0+ back\x0astart back\x0d\x0astart back\x0d + 0: back + 0+ \x0astart back\x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: start + 0+ back\x0d\x0astart back\x0d + 0: back + 0+ \x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: start + 0+ back\x0d + 0: back + 0+ \x0d +global repeat returned the same match as previous + +/(?<=\Kback)|\Gstart/g,aftertext,allow_lookaround_bsk,newline=ANYCRLF + back\rstart back\nstart back\r\nstart back\r + 0: back + 0+ \x0dstart back\x0astart back\x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: start + 0+ back\x0astart back\x0d\x0astart back\x0d + 0: back + 0+ \x0astart back\x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: start + 0+ back\x0d\x0astart back\x0d + 0: back + 0+ \x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: start + 0+ back\x0d + 0: back + 0+ \x0d +global repeat returned the same match as previous + +/(?<=\Kback)|\Gstart/g,aftertext,allow_lookaround_bsk,newline=ANY + back\rstart back\nstart back\r\nstart back\r + 0: back + 0+ \x0dstart back\x0astart back\x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: start + 0+ back\x0astart back\x0d\x0astart back\x0d + 0: back + 0+ \x0astart back\x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: start + 0+ back\x0d\x0astart back\x0d + 0: back + 0+ \x0d\x0astart back\x0d +global repeat returned the same match as previous + 0: start + 0+ back\x0d + 0: back + 0+ \x0d +global repeat returned the same match as previous + /((?2){73}(?2))((?1))/info Capture group count = 2 May match empty string