feat(manager/regex)!: allow arbitrary regex groups for templates (#12296

)

Allow the usage of arbitrary capture groups inside of regex manager templates and further adds refactor which this allows.

BREAKING_CHANGE
Only regex managers using the combination matchStringStrategy are affected of this change!
Currently capture groups which are empty but still match the regex are ignored.
This is no longer the case!
Subsequent matchGroups will now overwrite previous ones, even if the later one is empty.

lib/manager/regex/__fixtures__/ansible.yml

@@ -1,5 +1,7 @@
 prometheus_image: "prom/prometheus"  // depName gets initially set
+
+prometheus_registry: "docker.io"  // depName gets initially set
+prometheus_repository: "prom/prometheus"  // depName gets initially set
 prometheus_version: "v2.21.0" // currentValue get set
 
-someother_image: "" // will not be set as group value is null/empty string
 someother_version: "0.12.0" // overwrites currentValue as later values take precedence.

lib/manager/regex/__snapshots__/index.spec.ts.snap

@@ -257,6 +257,28 @@ Object {
 }
 `;
 
+exports[`manager/regex/index extracts with combination strategy and non standard capture groups 1`] = `
+Object {
+  "datasourceTemplate": "docker",
+  "depNameTemplate": "{{{ registry }}}/{{{ repository }}}",
+  "deps": Array [
+    Object {
+      "currentValue": "v2.21.0",
+      "datasource": "docker",
+      "depName": "docker.io/prom/prometheus",
+      "replaceString": "prometheus_version: \\"v2.21.0\\" //",
+    },
+  ],
+  "matchStrings": Array [
+    "prometheus_registry:\\\\s*\\"(?<registry>.*)\\"\\\\s*\\\\/\\\\/",
+    "prometheus_repository:\\\\s*\\"(?<repository>.*)\\"\\\\s*\\\\/\\\\/",
+    "prometheus_tag:\\\\s*\\"(?<tag>.*)\\"\\\\s*\\\\/\\\\/",
+    "prometheus_version:\\\\s*\\"(?<currentValue>.*)\\"\\\\s*\\\\/\\\\/",
+  ],
+  "matchStringsStrategy": "combination",
+}
+`;
+
 exports[`manager/regex/index extracts with combination strategy and registry url 1`] = `
 Object {
   "datasourceTemplate": "helm",

lib/manager/regex/index.spec.ts

@@ -206,6 +206,29 @@ describe('manager/regex/index', () => {
     expect(res).toMatchSnapshot();
     expect(res.deps).toHaveLength(1);
   });
+
+  it('extracts with combination strategy and non standard capture groups', async () => {
+    const config: CustomExtractConfig = {
+      matchStrings: [
+        'prometheus_registry:\\s*"(?<registry>.*)"\\s*\\/\\/',
+        'prometheus_repository:\\s*"(?<repository>.*)"\\s*\\/\\/',
+        'prometheus_tag:\\s*"(?<tag>.*)"\\s*\\/\\/',
+        'prometheus_version:\\s*"(?<currentValue>.*)"\\s*\\/\\/',
+      ],
+      matchStringsStrategy: 'combination',
+      datasourceTemplate: 'docker',
+      depNameTemplate: '{{{ registry }}}/{{{ repository }}}',
+    };
+    const res = await extractPackageFile(
+      ansibleYamlContent,
+      'ansible.yml',
+      config
+    );
+    expect(res.deps).toHaveLength(1);
+    expect(res.deps[0].depName).toEqual('docker.io/prom/prometheus');
+    expect(res).toMatchSnapshot();
+  });
+
   it('extracts with combination strategy and multiple matches', async () => {
     const config: CustomExtractConfig = {
       matchStrings: [

lib/manager/regex/index.ts

@@ -8,6 +8,7 @@ import type {
   PackageFile,
   Result,
 } from '../types';
+import type { ExtractionTemplate } from './types';
 
 export const defaultConfig = {
   pinDigests: false,
@@ -25,8 +26,6 @@ const validMatchFields = [
   'depType',
 ];
 
-const mergeFields = ['registryUrls', ...validMatchFields];
-
 function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] {
   const matches: RegExpMatchArray[] = [];
   let matchResult;
@@ -40,13 +39,12 @@ function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] {
 }
 
 function createDependency(
-  matchResult: RegExpMatchArray,
-  combinedGroups: Record<string, string>,
+  extractionTemplate: ExtractionTemplate,
   config: CustomExtractConfig,
   dep?: PackageDependency
 ): PackageDependency {
   const dependency = dep || {};
-  const { groups } = matchResult;
+  const { groups, replaceString } = extractionTemplate;
 
   function updateDependency(field: string, value: string): void {
     switch (field) {
@@ -69,11 +67,7 @@ function createDependency(
     const fieldTemplate = `${field}Template`;
     if (config[fieldTemplate]) {
       try {
-        const compiled = template.compile(
-          config[fieldTemplate],
-          combinedGroups ?? groups,
-          false
-        );
+        const compiled = template.compile(config[fieldTemplate], groups, false);
         updateDependency(field, compiled);
       } catch (err) {
         logger.warn(
@@ -86,26 +80,10 @@ function createDependency(
       updateDependency(field, groups[field]);
     }
   }
-  dependency.replaceString = String(matchResult[0]);
+  dependency.replaceString = replaceString;
   return dependency;
 }
 
-function mergeDependency(deps: PackageDependency[]): PackageDependency {
-  const result: PackageDependency = {};
-  deps.forEach((dep) => {
-    mergeFields.forEach((field) => {
-      if (dep[field]) {
-        result[field] = dep[field];
-        // save the line replaceString of the section which contains the current Value for a speed up lookup during the replace phase
-        if (field === 'currentValue') {
-          result.replaceString = dep.replaceString;
-        }
-      }
-    });
-  });
-  return result;
-}
-
 function handleAny(
   content: string,
   packageFile: string,
@@ -114,25 +92,29 @@ function handleAny(
   return config.matchStrings
     .map((matchString) => regEx(matchString, 'g'))
     .flatMap((regex) => regexMatchAll(regex, content)) // match all regex to content, get all matches, reduce to single array
-    .map((matchResult) => createDependency(matchResult, null, config));
+    .map((matchResult) =>
+      createDependency(
+        { groups: matchResult.groups, replaceString: matchResult[0] },
+        config
+      )
+    );
 }
 
 function mergeGroups(
   mergedGroup: Record<string, string>,
   secondGroup: Record<string, string>
 ): Record<string, string> {
-  const resultGroup = Object.create(null); // prevent prototype pollution
+  return { ...mergedGroup, ...secondGroup };
+}
 
-  Object.keys(mergedGroup).forEach(
-    // eslint-disable-next-line no-return-assign
-    (key) => (resultGroup[key] = mergedGroup[key])
-  );
-  Object.keys(secondGroup).forEach((key) => {
-    if (secondGroup[key] && secondGroup[key] !== '') {
-      resultGroup[key] = secondGroup[key];
-    }
-  });
-  return resultGroup;
+export function mergeExtractionTemplate(
+  base: ExtractionTemplate,
+  addition: ExtractionTemplate
+): ExtractionTemplate {
+  return {
+    groups: mergeGroups(base.groups, addition.groups),
+    replaceString: addition.replaceString ?? base.replaceString,
+  };
 }
 
 function handleCombination(
@@ -148,20 +130,13 @@ function handleCombination(
     return [];
   }
 
-  const combinedGroup = matches
-    .map((match) => match.groups)
-    .reduce((mergedGroup, currentGroup) =>
-      mergeGroups(mergedGroup, currentGroup)
-    );
-
-  // TODO: this seems to be buggy behavior, needs to be checked #11387
-  const dep = matches
-    .map((match) => createDependency(match, combinedGroup, config))
-    .reduce(
-      (mergedDep, currentDep) => mergeDependency([mergedDep, currentDep]),
-      {}
-    ); // merge fields of dependencies
-  return [dep];
+  const extraction = matches
+    .map((match) => ({
+      groups: match.groups,
+      replaceString: match?.groups?.currentValue ? match[0] : undefined,
+    }))
+    .reduce((base, addition) => mergeExtractionTemplate(base, addition));
+  return [createDependency(extraction, config)];
 }
 
 function handleRecursive(
@@ -182,8 +157,10 @@ function handleRecursive(
     // if we have a depName and a currentValue with have the minimal viable definition
     if (match?.groups?.depName && match?.groups?.currentValue) {
       return createDependency(
-        match,
-        mergeGroups(combinedGroups, match.groups),
+        {
+          groups: mergeGroups(combinedGroups, match.groups),
+          replaceString: match[0],
+        },
         config
       );
     }

lib/manager/regex/types.ts

@@ -0,0 +1,4 @@
+export interface ExtractionTemplate {
+  groups: Record<string, string>;
+  replaceString: string;
+}