From 76a4d17631ee874e66c3207e7608d15c535f06da Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Sun, 18 Feb 2024 13:25:27 +0100 Subject: [PATCH] fix(regex): refactor and fix regex predicate match (#27390) --- docs/usage/string-pattern-matching.md | 19 +++++++- lib/logger/remap.ts | 2 +- lib/util/string-match.spec.ts | 70 ++++++++++++++++++++++++++- lib/util/string-match.ts | 44 ++++++++++++----- 4 files changed, 120 insertions(+), 15 deletions(-) diff --git a/docs/usage/string-pattern-matching.md b/docs/usage/string-pattern-matching.md index 51ae41feb1eb33..6dea652ef5bc0b 100644 --- a/docs/usage/string-pattern-matching.md +++ b/docs/usage/string-pattern-matching.md @@ -4,7 +4,7 @@ Renovate string matching syntax for some configuration options allows the user t ## Regex matching -Users can choose to use regex patterns by starting the pattern string with `/` and ending with `/` or `/i`. +Users can choose to use regex patterns by starting the pattern string with `/` or `!/` and ending with `/` or `/i`. Regex patterns are evaluated with case sensitivity unless the `i` flag is specified. Renovate uses the [`re2`](https://github.com/google/re2) library for regex matching, which is not entirely the same syntax/support as the full regex specification. @@ -13,7 +13,8 @@ For a full list of re2 syntax, see [the re2 syntax wiki page](https://github.com Example regex patterns: - `/^abc/` is a regex pattern matching any string starting with lower-case `abc`. -- `^abc/i` is a regex pattern matching any string starting with `abc` in lower or upper case, or a mix. +- `/^abc/i` is a regex pattern matching any string starting with `abc` in lower or upper case, or a mix. +- `!/^a/` is a regex pattern matching any string no starting with `a` in lower case. If you want to test your patterns interactively online, we recommend [regex101.com](https://regex101.com/?flavor=javascript&flags=ginst). Be aware that backslashes (`\`) of the resulting regex have to still be escaped e.g. `\n\s` --> `\\n\\s`. You can use the Code Generator in the sidebar and copy the regex in the generated "Alternative syntax" comment into JSON. @@ -30,6 +31,20 @@ Examples: - `abc123` matches `abc123` exactly, or `AbC123`. - `abc*` matches `abc`, `abc123`, `ABCabc`, etc. +## Negative matching + +Renovate has a specific approach to negative matching strings. + +"Positive" matches are patterns (in glob or regex) which don't start with `!`. +"Negative" matches are patterns starting with `!` (e.g. `!/^a/` or `!b*`). + +For an array of patterns to match, the following must be true: + +- If any positive matches are included, at least one must match. +- If any negative matches are included, none must match. + +For example, `["/^abc/", "!/^abcd/", "!/abce/"]` would match "abc" and "abcf" but not "foo", "abcd", "abce", or "abcdef". + ## Usage in Renovate configuration options Renovate has matured its approach to string pattern matching over time, but this means that existing configurations may have a mix of approaches and not be entirely consistent with each other. diff --git a/lib/logger/remap.ts b/lib/logger/remap.ts index 57920e2677ea00..241394c516c6e8 100644 --- a/lib/logger/remap.ts +++ b/lib/logger/remap.ts @@ -14,7 +14,7 @@ function match(remap: LogLevelRemap, input: string): boolean { const { matchMessage: pattern } = remap; let matchFn = matcherCache.get(remap); if (!matchFn) { - matchFn = makeRegexOrMinimatchPredicate(pattern) ?? (() => false); + matchFn = makeRegexOrMinimatchPredicate(pattern); matcherCache.set(remap, matchFn); } diff --git a/lib/util/string-match.spec.ts b/lib/util/string-match.spec.ts index 02ab8c4598e470..e658357825f865 100644 --- a/lib/util/string-match.spec.ts +++ b/lib/util/string-match.spec.ts @@ -1,6 +1,60 @@ -import { configRegexPredicate } from './string-match'; +import { + anyMatchRegexOrMinimatch, + configRegexPredicate, + matchRegexOrMinimatch, +} from './string-match'; describe('util/string-match', () => { + describe('anyMatchRegexOrMinimatch()', () => { + it('returns false if empty patterns', () => { + expect(anyMatchRegexOrMinimatch('test', [])).toBeFalse(); + }); + + it('returns false if no match', () => { + expect(anyMatchRegexOrMinimatch('test', ['/test2/'])).toBeFalse(); + }); + + it('returns true if any match', () => { + expect(anyMatchRegexOrMinimatch('test', ['test', '/test2/'])).toBeTrue(); + }); + + it('returns true if one match with negative patterns', () => { + expect(anyMatchRegexOrMinimatch('test', ['!/test2/'])).toBeTrue(); + }); + + it('returns true if every match with negative patterns', () => { + expect( + anyMatchRegexOrMinimatch('test', ['!/test2/', '!/test3/']), + ).toBeTrue(); + }); + + it('returns true if matching positive and negative patterns', () => { + expect(anyMatchRegexOrMinimatch('test', ['test', '!/test3/'])).toBeTrue(); + }); + + it('returns true if matching every negative pattern (regex)', () => { + expect( + anyMatchRegexOrMinimatch('test', ['test', '!/test3/', '!/test4/']), + ).toBeTrue(); + }); + + it('returns false if not matching every negative pattern (regex)', () => { + expect( + anyMatchRegexOrMinimatch('test', ['!/test3/', '!/test/']), + ).toBeFalse(); + }); + + it('returns true if matching every negative pattern (glob)', () => { + expect( + anyMatchRegexOrMinimatch('test', ['test', '!test3', '!test4']), + ).toBeTrue(); + }); + + it('returns false if not matching every negative pattern (glob)', () => { + expect(anyMatchRegexOrMinimatch('test', ['!test3', '!te*'])).toBeFalse(); + }); + }); + describe('configRegexPredicate', () => { it('allows valid regex pattern', () => { expect(configRegexPredicate('/hello/')).not.toBeNull(); @@ -22,4 +76,18 @@ describe('util/string-match', () => { expect(configRegexPredicate('hello')).toBeNull(); }); }); + + describe('matchRegexOrMinimatch()', () => { + it('returns true if positive regex pattern matched', () => { + expect(matchRegexOrMinimatch('test', '/test/')).toBeTrue(); + }); + + it('returns true if negative regex is not matched', () => { + expect(matchRegexOrMinimatch('test', '!/test3/')).toBeTrue(); + }); + + it('returns false if negative pattern is matched', () => { + expect(matchRegexOrMinimatch('test', '!/te/')).toBeFalse(); + }); + }); }); diff --git a/lib/util/string-match.ts b/lib/util/string-match.ts index 5576ba3d179ac2..3f03e06f20b532 100644 --- a/lib/util/string-match.ts +++ b/lib/util/string-match.ts @@ -10,14 +10,10 @@ export function isDockerDigest(input: string): boolean { export function makeRegexOrMinimatchPredicate( pattern: string, -): StringMatchPredicate | null { - if (pattern.length > 2 && pattern.startsWith('/') && pattern.endsWith('/')) { - try { - const regex = regEx(pattern.slice(1, -1)); - return (x: string): boolean => regex.test(x); - } catch (err) { - return null; - } +): StringMatchPredicate { + const regExPredicate = configRegexPredicate(pattern); + if (regExPredicate) { + return regExPredicate; } const mm = minimatch(pattern, { dot: true }); @@ -26,14 +22,40 @@ export function makeRegexOrMinimatchPredicate( export function matchRegexOrMinimatch(input: string, pattern: string): boolean { const predicate = makeRegexOrMinimatchPredicate(pattern); - return predicate ? predicate(input) : false; + return predicate(input); } export function anyMatchRegexOrMinimatch( input: string, patterns: string[], -): boolean | null { - return patterns.some((pattern) => matchRegexOrMinimatch(input, pattern)); +): boolean { + if (!patterns.length) { + return false; + } + + // Return false if there are positive patterns and none match + const positivePatterns = patterns.filter( + (pattern) => !pattern.startsWith('!'), + ); + if ( + positivePatterns.length && + !positivePatterns.some((pattern) => matchRegexOrMinimatch(input, pattern)) + ) { + return false; + } + + // Every negative pattern must be true to return true + const negativePatterns = patterns.filter((pattern) => + pattern.startsWith('!'), + ); + if ( + negativePatterns.length && + !negativePatterns.every((pattern) => matchRegexOrMinimatch(input, pattern)) + ) { + return false; + } + + return true; } export const UUIDRegex = regEx(