Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor(regex): cleanup up index.ts and move matchStringStrategy to enum #12525

Merged
merged 8 commits into from
Dec 10, 2021
Merged

refactor(regex): cleanup up index.ts and move matchStringStrategy to enum #12525

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d96a133
refactor(regex): functions to util.ts and strategies.ts
secustor Nov 5, 2021
99af288
Merge branch 'main' into refactor_regex
secustor Nov 5, 2021
b52c967
refactor(regex): change MatchstringStrategy to enum
secustor Nov 5, 2021
43a92fa
Merge branch 'main' into refactor_regex
secustor Nov 6, 2021
8f1e389
Revert "refactor(regex): change MatchstringStrategy to enum"
secustor Nov 6, 2021
055288f
refactor(regex): add type annotation
secustor Nov 6, 2021
14c14a7
Merge branch 'main' into refactor_regex
rarkins Dec 8, 2021
43125f7
Merge branch 'main' into refactor_regex
rarkins Dec 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 5 additions & 1 deletion lib/config/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,11 @@ export type UpdateType =
| 'rollback'
| 'bump';

export type MatchStringsStrategy = 'any' | 'recursive' | 'combination';
export enum MatchStringsStrategy {
rarkins marked this conversation as resolved.
Show resolved Hide resolved
ANY = 'any',
RECURSIVE = 'recursive',
COMBINATION = 'combination',
}

export type MergeStrategy =
| 'auto'
Expand Down
25 changes: 13 additions & 12 deletions lib/manager/regex/index.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { loadFixture } from '../../../test/util';
import { MatchStringsStrategy } from '../../config/types';
import { logger } from '../../logger';
import type { CustomExtractConfig } from '../types';
import { defaultConfig, extractPackageFile } from '.';
Expand Down Expand Up @@ -195,7 +196,7 @@ describe('manager/regex/index', () => {
'prometheus_image:\\s*"(?<depName>.*)"\\s*\\/\\/',
'prometheus_version:\\s*"(?<currentValue>.*)"\\s*\\/\\/',
],
matchStringsStrategy: 'combination',
matchStringsStrategy: MatchStringsStrategy.COMBINATION,
datasourceTemplate: 'docker',
};
const res = await extractPackageFile(
Expand All @@ -215,7 +216,7 @@ describe('manager/regex/index', () => {
'prometheus_tag:\\s*"(?<tag>.*)"\\s*\\/\\/',
'prometheus_version:\\s*"(?<currentValue>.*)"\\s*\\/\\/',
],
matchStringsStrategy: 'combination',
matchStringsStrategy: MatchStringsStrategy.COMBINATION,
datasourceTemplate: 'docker',
depNameTemplate: '{{{ registry }}}/{{{ repository }}}',
};
Expand All @@ -235,7 +236,7 @@ describe('manager/regex/index', () => {
'.*_image:\\s*"(?<depName>.*)"\\s*\\/\\/',
'.*_version:\\s*"(?<currentValue>.*)"\\s*\\/\\/',
],
matchStringsStrategy: 'combination',
matchStringsStrategy: MatchStringsStrategy.COMBINATION,
datasourceTemplate: 'docker',
};
const res = await extractPackageFile(
Expand All @@ -248,7 +249,7 @@ describe('manager/regex/index', () => {
});
it('extracts with combination strategy and registry url', async () => {
const config: CustomExtractConfig = {
matchStringsStrategy: 'combination',
matchStringsStrategy: MatchStringsStrategy.COMBINATION,
matchStrings: [
'CHART_VERSION: (?<currentValue>.*?)\n',
'CHART_REPOSITORY_URL: "(?<registryUrl>.*?)"',
Expand All @@ -267,7 +268,7 @@ describe('manager/regex/index', () => {

it('extracts with combination strategy and templates', async () => {
const config: CustomExtractConfig = {
matchStringsStrategy: 'combination',
matchStringsStrategy: MatchStringsStrategy.COMBINATION,
matchStrings: [
'CHART_REPOSITORY_URL: "(?<registryUrl>.*)\\/(?<depName>[a-z]+)\\/"',
'CHART_VERSION: (?<currentValue>.*?)\n',
Expand All @@ -286,7 +287,7 @@ describe('manager/regex/index', () => {

it('extracts with combination strategy and empty file', async () => {
const config: CustomExtractConfig = {
matchStringsStrategy: 'combination',
matchStringsStrategy: MatchStringsStrategy.COMBINATION,
matchStrings: [
'CHART_REPOSITORY_URL: "(?<registryUrl>.*)\\/(?<depName>[a-z]+)\\/"',
'CHART_VERSION: (?<currentValue>.*?)\n',
Expand All @@ -304,7 +305,7 @@ describe('manager/regex/index', () => {
'"group1":\\s*\\{[^}]*}',
'"name":\\s*"(?<depName>.*)"[^"]*"type":\\s*"(?<datasource>.*)"[^"]*"value":\\s*"(?<currentValue>.*)"',
],
matchStringsStrategy: 'recursive',
matchStringsStrategy: MatchStringsStrategy.RECURSIVE,
};
const res = await extractPackageFile(
exampleJsonContent,
Expand All @@ -320,7 +321,7 @@ describe('manager/regex/index', () => {
'"group.{1}":\\s*\\{[^}]*}',
'"name":\\s*"(?<depName>.*)"[^"]*"type":\\s*"(?<datasource>.*)"[^"]*"value":\\s*"(?<currentValue>.*)"',
],
matchStringsStrategy: 'recursive',
matchStringsStrategy: MatchStringsStrategy.RECURSIVE,
};
const res = await extractPackageFile(
exampleJsonContent,
Expand All @@ -337,7 +338,7 @@ describe('manager/regex/index', () => {
'"test":\\s*\\{[^}]*}',
'"name":\\s*"(?<depName>.*)"[^"]*"type":\\s*"(?<datasource>.*)"[^"]*"value":\\s*"(?<currentValue>.*)"',
],
matchStringsStrategy: 'recursive',
matchStringsStrategy: MatchStringsStrategy.RECURSIVE,
};
const res = await extractPackageFile(
exampleJsonContent,
Expand All @@ -350,7 +351,7 @@ describe('manager/regex/index', () => {
it('extracts with recursive strategy and fail because of not sufficient regexes', async () => {
const config: CustomExtractConfig = {
matchStrings: ['"group.{1}":\\s*\\{[^}]*}'],
matchStringsStrategy: 'recursive',
matchStringsStrategy: MatchStringsStrategy.RECURSIVE,
};
const res = await extractPackageFile(
exampleJsonContent,
Expand All @@ -363,7 +364,7 @@ describe('manager/regex/index', () => {
it('extracts with recursive strategy and fail because there is no match', async () => {
const config: CustomExtractConfig = {
matchStrings: ['"trunk.{1}":\\s*\\{[^}]*}'],
matchStringsStrategy: 'recursive',
matchStringsStrategy: MatchStringsStrategy.RECURSIVE,
};
const res = await extractPackageFile(
exampleJsonContent,
Expand All @@ -380,7 +381,7 @@ describe('manager/regex/index', () => {
'"(?<second>[^"]*)":\\s*\\{[^}]*}',
'"name":\\s*"(?<depName>.*)"[^"]*"type":\\s*"(?<datasource>.*)"[^"]*"value":\\s*"(?<currentValue>.*)"',
],
matchStringsStrategy: 'recursive',
matchStringsStrategy: MatchStringsStrategy.RECURSIVE,
depNameTemplate: '{{{ first }}}/{{{ second }}}/{{{ depName }}}',
};
const res = await extractPackageFile(
Expand Down
175 changes: 6 additions & 169 deletions lib/manager/regex/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,180 +1,17 @@
import { URL } from 'url';
import { logger } from '../../logger';
import { regEx } from '../../util/regex';
import * as template from '../../util/template';
import { MatchStringsStrategy } from '../../config/types';
import type {
CustomExtractConfig,
PackageDependency,
PackageFile,
Result,
} from '../types';
import type { ExtractionTemplate } from './types';
import { handleAny, handleCombination, handleRecursive } from './strategies';
import { validMatchFields } from './utils';

export const defaultConfig = {
pinDigests: false,
};

const validMatchFields = [
'depName',
'lookupName',
'currentValue',
'currentDigest',
'datasource',
'versioning',
'extractVersion',
'registryUrl',
'depType',
];

function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] {
const matches: RegExpMatchArray[] = [];
let matchResult;
do {
matchResult = regex.exec(content);
if (matchResult) {
matches.push(matchResult);
}
} while (matchResult);
return matches;
}

function createDependency(
extractionTemplate: ExtractionTemplate,
config: CustomExtractConfig,
dep?: PackageDependency
): PackageDependency {
const dependency = dep || {};
const { groups, replaceString } = extractionTemplate;

function updateDependency(field: string, value: string): void {
switch (field) {
case 'registryUrl':
// check if URL is valid and pack inside an array
try {
const url = new URL(value).toString();
dependency.registryUrls = [url];
} catch (err) {
logger.warn({ value }, 'Invalid regex manager registryUrl');
}
break;
default:
dependency[field] = value;
break;
}
}

for (const field of validMatchFields) {
const fieldTemplate = `${field}Template`;
if (config[fieldTemplate]) {
try {
const compiled = template.compile(config[fieldTemplate], groups, false);
updateDependency(field, compiled);
} catch (err) {
logger.warn(
{ template: config[fieldTemplate] },
'Error compiling template for custom manager'
);
return null;
}
} else if (groups[field]) {
updateDependency(field, groups[field]);
}
}
dependency.replaceString = replaceString;
return dependency;
}

function handleAny(
content: string,
packageFile: string,
config: CustomExtractConfig
): PackageDependency[] {
return config.matchStrings
.map((matchString) => regEx(matchString, 'g'))
.flatMap((regex) => regexMatchAll(regex, content)) // match all regex to content, get all matches, reduce to single array
.map((matchResult) =>
createDependency(
{ groups: matchResult.groups, replaceString: matchResult[0] },
config
)
);
}

function mergeGroups(
mergedGroup: Record<string, string>,
secondGroup: Record<string, string>
): Record<string, string> {
return { ...mergedGroup, ...secondGroup };
}

export function mergeExtractionTemplate(
base: ExtractionTemplate,
addition: ExtractionTemplate
): ExtractionTemplate {
return {
groups: mergeGroups(base.groups, addition.groups),
replaceString: addition.replaceString ?? base.replaceString,
};
}

function handleCombination(
content: string,
packageFile: string,
config: CustomExtractConfig
): PackageDependency[] {
const matches = config.matchStrings
.map((matchString) => regEx(matchString, 'g'))
.flatMap((regex) => regexMatchAll(regex, content)); // match all regex to content, get all matches, reduce to single array

if (!matches.length) {
return [];
}

const extraction = matches
.map((match) => ({
groups: match.groups,
replaceString: match?.groups?.currentValue ? match[0] : undefined,
}))
.reduce((base, addition) => mergeExtractionTemplate(base, addition));
return [createDependency(extraction, config)];
}

function handleRecursive(
content: string,
packageFile: string,
config: CustomExtractConfig,
index = 0,
combinedGroups: Record<string, string> = {}
): PackageDependency[] {
const regexes = config.matchStrings.map((matchString) =>
regEx(matchString, 'g')
);
// abort if we have no matchString anymore
if (regexes[index] == null) {
return [];
}
return regexMatchAll(regexes[index], content).flatMap((match) => {
// if we have a depName and a currentValue which have the minimal viable definition
if (match?.groups?.depName && match?.groups?.currentValue) {
return createDependency(
{
groups: mergeGroups(combinedGroups, match.groups),
replaceString: match[0],
},
config
);
}

return handleRecursive(
match[0],
packageFile,
config,
index + 1,
mergeGroups(combinedGroups, match.groups || {})
);
});
}

export function extractPackageFile(
content: string,
packageFile: string,
Expand All @@ -183,13 +20,13 @@ export function extractPackageFile(
let deps: PackageDependency[];
switch (config.matchStringsStrategy) {
default:
case 'any':
case MatchStringsStrategy.ANY:
deps = handleAny(content, packageFile, config);
break;
case 'combination':
case MatchStringsStrategy.COMBINATION:
deps = handleCombination(content, packageFile, config);
break;
case 'recursive':
case MatchStringsStrategy.RECURSIVE:
deps = handleRecursive(content, packageFile, config);
break;
}
Expand Down