Skip to content

Commit

Permalink
refactor(manager/poetry): use zod schema validation (#23830)
Browse files Browse the repository at this point in the history
  • Loading branch information
zeshuaro committed Aug 18, 2023
1 parent 0777f54 commit cec6fae
Show file tree
Hide file tree
Showing 7 changed files with 184 additions and 50 deletions.
1 change: 1 addition & 0 deletions lib/modules/manager/poetry/__fixtures__/pyproject.2.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ dep1 = { version = "*" }
dep2 = { version = "^0.6.0" }
dep3 = { path = "/some/path/", version = '^0.33.6' }
dep4 = { path = "/some/path/" }
dep5 = {}

[tool.poetry.extras]
extra_dep1 = "^0.8.3"
Expand Down
10 changes: 10 additions & 0 deletions lib/modules/manager/poetry/__snapshots__/extract.spec.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,16 @@ exports[`modules/manager/poetry/extract extractPackageFile() extracts multiple d
},
"skipReason": "path-dependency",
},
{
"currentValue": "",
"datasource": "pypi",
"depName": "dep5",
"depType": "dependencies",
"managerData": {
"nestedVersion": false,
},
"versioning": "pep440",
},
{
"currentValue": "^0.8.3",
"datasource": "pypi",
Expand Down
2 changes: 1 addition & 1 deletion lib/modules/manager/poetry/extract.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ describe('modules/manager/poetry/extract', () => {
it('extracts multiple dependencies (with dep = {version = "1.2.3"} case)', async () => {
const res = await extractPackageFile(pyproject2toml, filename);
expect(res).toMatchSnapshot();
expect(res?.deps).toHaveLength(7);
expect(res?.deps).toHaveLength(8);
});

it('handles case with no dependencies', async () => {
Expand Down
112 changes: 63 additions & 49 deletions lib/modules/manager/poetry/extract.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { parse } from '@iarna/toml';
import is from '@sindresorhus/is';
import { logger } from '../../../logger';
import type { SkipReason } from '../../../types';
Expand All @@ -15,34 +14,50 @@ import * as pep440Versioning from '../../versioning/pep440';
import * as poetryVersioning from '../../versioning/poetry';
import type { PackageDependency, PackageFileContent } from '../types';
import { extractLockFileEntries } from './locked-version';
import type { PoetryDependency, PoetryFile, PoetrySection } from './types';
import type {
PoetryDependencyRecord,
PoetryGroupRecord,
PoetrySchema,
PoetrySectionSchema,
} from './schema';
import { parsePoetry } from './utils';

function extractFromDependenciesSection(
parsedFile: PoetryFile,
section: keyof Omit<PoetrySection, 'source' | 'group'>,
parsedFile: PoetrySchema,
section: keyof Omit<PoetrySectionSchema, 'source' | 'group'>,
poetryLockfile: Record<string, string>
): PackageDependency[] {
return extractFromSection(
parsedFile.tool?.poetry?.[section],
parsedFile?.tool?.poetry?.[section],
section,
poetryLockfile
);
}

function extractFromDependenciesGroupSection(
parsedFile: PoetryFile,
group: string,
groupSections: PoetryGroupRecord | undefined,
poetryLockfile: Record<string, string>
): PackageDependency[] {
return extractFromSection(
parsedFile.tool?.poetry?.group[group]?.dependencies,
group,
poetryLockfile
);
if (!groupSections) {
return [];
}

const deps = [];
for (const groupName of Object.keys(groupSections)) {
deps.push(
...extractFromSection(
groupSections[groupName]?.dependencies,
groupName,
poetryLockfile
)
);
}

return deps;
}

function extractFromSection(
sectionContent: Record<string, PoetryDependency | string> | undefined,
sectionContent: PoetryDependencyRecord | undefined,
depType: string,
poetryLockfile: Record<string, string>
): PackageDependency[] {
Expand All @@ -68,35 +83,39 @@ function extractFromSection(
lockedVersion = poetryLockfile[packageName];
}
if (!is.string(currentValue)) {
const version = currentValue.version;
const path = currentValue.path;
const git = currentValue.git;
if (version) {
currentValue = version;
nestedVersion = true;
if (!!path || git) {
skipReason = path ? 'path-dependency' : 'git-dependency';
}
} else if (path) {
if (is.array(currentValue)) {
currentValue = '';
skipReason = 'path-dependency';
} else if (git) {
if (currentValue.tag) {
currentValue = currentValue.tag;
datasource = GithubTagsDatasource.id;
const githubPackageName = extractGithubPackageName(git);
if (githubPackageName) {
packageName = githubPackageName;
skipReason = 'multiple-constraint-dep';
} else {
const version = currentValue.version;
const path = currentValue.path;
const git = currentValue.git;
if (version) {
currentValue = version;
nestedVersion = true;
if (!!path || git) {
skipReason = path ? 'path-dependency' : 'git-dependency';
}
} else if (path) {
currentValue = '';
skipReason = 'path-dependency';
} else if (git) {
if (currentValue.tag) {
currentValue = currentValue.tag;
datasource = GithubTagsDatasource.id;
const githubPackageName = extractGithubPackageName(git);
if (githubPackageName) {
packageName = githubPackageName;
} else {
skipReason = 'git-dependency';
}
} else {
currentValue = '';
skipReason = 'git-dependency';
}
} else {
currentValue = '';
skipReason = 'git-dependency';
}
} else {
currentValue = '';
skipReason = 'multiple-constraint-dep';
}
}
const dep: PackageDependency = {
Expand Down Expand Up @@ -126,8 +145,8 @@ function extractFromSection(
return deps;
}

function extractRegistries(pyprojectfile: PoetryFile): string[] | undefined {
const sources = pyprojectfile.tool?.poetry?.source;
function extractRegistries(pyprojectfile: PoetrySchema): string[] | undefined {
const sources = pyprojectfile?.tool?.poetry?.source;

if (!Array.isArray(sources) || sources.length === 0) {
return undefined;
Expand All @@ -149,14 +168,8 @@ export async function extractPackageFile(
packageFile: string
): Promise<PackageFileContent | null> {
logger.trace(`poetry.extractPackageFile(${packageFile})`);
let pyprojectfile: PoetryFile;
try {
pyprojectfile = parse(content);
} catch (err) {
logger.debug({ err, packageFile }, 'Error parsing pyproject.toml file');
return null;
}
if (!pyprojectfile.tool?.poetry) {
const pyprojectfile = parsePoetry(packageFile, content);
if (!pyprojectfile?.tool?.poetry) {
logger.debug({ packageFile }, `contains no poetry section`);
return null;
}
Expand All @@ -180,8 +193,9 @@ export async function extractPackageFile(
lockfileMapping
),
...extractFromDependenciesSection(pyprojectfile, 'extras', lockfileMapping),
...Object.keys(pyprojectfile.tool?.poetry?.group ?? []).flatMap((group) =>
extractFromDependenciesGroupSection(pyprojectfile, group, lockfileMapping)
...extractFromDependenciesGroupSection(
pyprojectfile?.tool?.poetry?.group,
lockfileMapping
),
];

Expand All @@ -191,9 +205,9 @@ export async function extractPackageFile(

const extractedConstraints: Record<string, any> = {};

if (is.nonEmptyString(pyprojectfile.tool?.poetry?.dependencies?.python)) {
if (is.nonEmptyString(pyprojectfile?.tool?.poetry?.dependencies?.python)) {
extractedConstraints.python =
pyprojectfile.tool?.poetry?.dependencies?.python;
pyprojectfile?.tool?.poetry?.dependencies?.python;
}

const res: PackageFileContent = {
Expand Down
55 changes: 55 additions & 0 deletions lib/modules/manager/poetry/schema.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import { z } from 'zod';
import { LooseRecord, Toml } from '../../../util/schema-utils';

const PoetryDependencySchema = z.object({
path: z.string().optional(),
git: z.string().optional(),
tag: z.string().optional(),
version: z.string().optional(),
});

export const PoetryDependencyRecord = LooseRecord(
z.string(),
z.union([PoetryDependencySchema, z.array(PoetryDependencySchema), z.string()])
);

export type PoetryDependencyRecord = z.infer<typeof PoetryDependencyRecord>;

export const PoetryGroupRecord = LooseRecord(
z.string(),
z.object({
dependencies: PoetryDependencyRecord.optional(),
})
);

export type PoetryGroupRecord = z.infer<typeof PoetryGroupRecord>;

export const PoetrySectionSchema = z.object({
dependencies: PoetryDependencyRecord.optional(),
'dev-dependencies': PoetryDependencyRecord.optional(),
extras: PoetryDependencyRecord.optional(),
group: PoetryGroupRecord.optional(),
source: z
.array(z.object({ name: z.string(), url: z.string().optional() }))
.optional(),
});

export type PoetrySectionSchema = z.infer<typeof PoetrySectionSchema>;

export const PoetrySchema = z.object({
tool: z
.object({
poetry: PoetrySectionSchema.optional(),
})
.optional(),
'build-system': z
.object({
requires: z.array(z.string()),
'build-backend': z.string().optional(),
})
.optional(),
});

export type PoetrySchema = z.infer<typeof PoetrySchema>;

export const PoetrySchemaToml = Toml.pipe(PoetrySchema);
40 changes: 40 additions & 0 deletions lib/modules/manager/poetry/utils.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import { codeBlock } from 'common-tags';
import { parsePoetry } from './utils';

describe('modules/manager/poetry/utils', () => {
const fileName = 'fileName';

describe('parsePoetry', () => {
it('load and parse successfully', () => {
const fileContent = codeBlock`
[tool.poetry.dependencies]
dep1 = "1.0.0"
[tool.poetry.group.dev.dependencies]
dep2 = "1.0.1"
`;
const actual = parsePoetry(fileName, fileContent);
expect(actual).toMatchObject({
tool: {
poetry: {
dependencies: { dep1: '1.0.0' },
group: { dev: { dependencies: { dep2: '1.0.1' } } },
},
},
});
});

it('invalid toml', () => {
const actual = parsePoetry(fileName, 'clearly_invalid');
expect(actual).toBeNull();
});

it('invalid schema', () => {
const fileContent = codeBlock`
[tool.poetry.dependencies]:
dep1 = 1
`;
const actual = parsePoetry(fileName, fileContent);
expect(actual).toBeNull();
});
});
});
14 changes: 14 additions & 0 deletions lib/modules/manager/poetry/utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import { logger } from '../../../logger';
import { type PoetrySchema, PoetrySchemaToml } from './schema';

export function parsePoetry(
fileName: string,
fileContent: string
): PoetrySchema | null {
const res = PoetrySchemaToml.safeParse(fileContent);
if (res.success) {
return res.data;
}
logger.debug({ err: res.error, fileName }, 'Error parsing poetry lockfile.');
return null;
}

0 comments on commit cec6fae

Please sign in to comment.