Skip to content

Commit

Permalink
fix(repology): Properly handle of response with multiple package vers…
Browse files Browse the repository at this point in the history 
…ions (#8489)
  • Loading branch information
@zeldigas
zeldigas committed Feb 3, 2021
1 parent b7debbc commit 8032ffd
Show file tree
Hide file tree
Showing 4 changed files with 121 additions and 42 deletions.
1 change: 1 addition & 0 deletions lib/datasource/repology/__fixtures__/openjdk.json
View file

Large diffs are not rendered by default.

56 changes: 53 additions & 3 deletions lib/datasource/repology/__snapshots__/index.spec.ts.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,32 @@ Array [
]
`;

exports[`datasource/repology/index getReleases returns null for ambiguous package results 1`] = `
exports[`datasource/repology/index getReleases returns multiple versions if they are present in repository 1`] = `
Object {
"releases": Array [
Object {
"version": "1:11.0.7.10-1.el8_1",
},
Object {
"version": "1:11.0.8.10-0.el8_2",
},
Object {
"version": "1:11.0.8.10-6.el8",
},
Object {
"version": "1:11.0.9.11-0.el8_2",
},
Object {
"version": "1:11.0.9.11-2.el8_3",
},
Object {
"version": "1:11.0.9.11-3.el8_3",
},
],
}
`;

exports[`datasource/repology/index getReleases returns multiple versions if they are present in repository 2`] = `
Array [
Object {
"headers": Object {
Expand All @@ -155,7 +180,7 @@ Array [
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=dummy&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=example",
"url": "https://repology.org/tools/project-by?repo=centos_8&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=java-11-openjdk",
},
Object {
"headers": Object {
Expand All @@ -165,7 +190,7 @@ Array [
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=dummy&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=example",
"url": "https://repology.org/tools/project-by?repo=centos_8&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=java-11-openjdk",
},
]
`;
Expand Down Expand Up @@ -220,6 +245,31 @@ Array [
]
`;

exports[`datasource/repology/index getReleases returns null for scenario when repo is not in package results 1`] = `
Array [
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"host": "repology.org",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=dummy&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=example",
},
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"host": "repology.org",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=dummy&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=example",
},
]
`;

exports[`datasource/repology/index getReleases throws error on API request timeout 1`] = `
Array [
Object {
Expand Down
31 changes: 28 additions & 3 deletions lib/datasource/repology/index.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ const fixturePulseaudio = fs.readFileSync(
`${__dirname}/__fixtures__/pulseaudio.json`,
'utf8'
);
const fixtureJdk = fs.readFileSync(
`${__dirname}/__fixtures__/openjdk.json`,
'utf8'
);

describe(getName(__filename), () => {
describe('getReleases', () => {
Expand Down Expand Up @@ -297,10 +301,31 @@ describe(getName(__filename), () => {
expect(httpMock.getTrace()).toMatchSnapshot();
});

it('returns null for ambiguous package results', async () => {
it('returns multiple versions if they are present in repository', async () => {
mockResolverCall('centos_8', 'java-11-openjdk', 'binname', {
status: 404,
});
mockResolverCall('centos_8', 'java-11-openjdk', 'srcname', {
status: 200,
body: fixtureJdk,
});

const res = await getPkgReleases({
datasource,
versioning,
depName: 'centos_8/java-11-openjdk',
});
expect(res).toMatchSnapshot();
expect(res.releases).toHaveLength(6);
expect(res.releases[0].version).toEqual('1:11.0.7.10-1.el8_1');
expect(res.releases[5].version).toEqual('1:11.0.9.11-3.el8_3');
expect(httpMock.getTrace()).toMatchSnapshot();
});

it('returns null for scenario when repo is not in package results', async () => {
const pkgs: RepologyPackage[] = [
{ repo: 'dummy', version: '1.0.0', visiblename: 'example' },
{ repo: 'dummy', version: '2.0.0', visiblename: 'example' },
{ repo: 'not-dummy', version: '1.0.0', visiblename: 'example' },
{ repo: 'not-dummy', version: '2.0.0', visiblename: 'example' },
];
const pkgsJSON = JSON.stringify(pkgs);

Expand Down
75 changes: 39 additions & 36 deletions lib/datasource/repology/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { GetReleasesConfig, ReleaseResult } from '../common';
export const id = 'repology';

const http = new Http(id);
const cacheNamespace = `datasource-${id}`;
const cacheNamespace = `datasource-${id}-list`;
const cacheMinutes = 60;

export type RepologyPackageType = 'binname' | 'srcname';
Expand Down Expand Up @@ -79,42 +79,41 @@ function findPackageInResponse(
repoName: string,
pkgName: string,
types: RepologyPackageType[]
): RepologyPackage | undefined {
let pkgs = response.filter((pkg) => pkg.repo === repoName);

// In some cases Repology bundles multiple packages into a single project,
// which would result in ambiguous results. If we have more than one result
// left, we should try to determine the correct package by comparing either
// binname or srcname (depending on `types`) to the given dependency name.
if (pkgs.length > 1) {
for (const pkgType of types) {
pkgs = pkgs.filter((pkg) => !pkg[pkgType] || pkg[pkgType] === pkgName);
if (pkgs.length === 1) {
break;
}
}
): RepologyPackage[] | undefined {
const repoPackages = response.filter((pkg) => pkg.repo === repoName);

if (repoPackages.length === 0) {
// no packages associated with repoName
return null;
}

// Abort if there is still more than one package left, as the result would
// be ambiguous and unreliable. This should usually not happen...
if (pkgs.length > 1) {
logger.warn(
{ repoName, pkgName, packageTypes, pkgs },
'Repology lookup returned ambiguous results, ignoring...'
if (repoPackages.length === 1) {
// repo contains exactly one package, so we can return them safely
return repoPackages;
}

// In some cases Repology bundles multiple packages into a single project, which might result in ambiguous results.
// We need to do additional filtering by matching allowed package types passed as params with package description.
// Remaining packages are the one we are looking for
let packagesWithType;
for (const pkgType of types) {
packagesWithType = repoPackages.filter(
(pkg) => !pkg[pkgType] || pkg[pkgType] === pkgName
);
return null;
if (packagesWithType.length === 1) {
break;
}
}

// pkgs might be an empty array here and in that case we return undefined
return pkgs[0];
return packagesWithType.length > 0 ? packagesWithType : null;
}

async function queryPackage(
repoName: string,
pkgName: string
): Promise<RepologyPackage> {
): Promise<RepologyPackage[]> {
let response: RepologyPackage[];
let pkg: RepologyPackage;
let pkg: RepologyPackage[];
// Try getting the packages from tools/project-by first for type binname and
// afterwards for srcname. This needs to be done first, because some packages
// resolve to repology projects which have a different name than the package
Expand All @@ -125,10 +124,12 @@ async function queryPackage(
for (const pkgType of packageTypes) {
response = await queryPackagesViaResolver(repoName, pkgName, pkgType);

pkg = findPackageInResponse(response, repoName, pkgName, [pkgType]);
if (pkg) {
// exit immediately if package found
return pkg;
if (response) {
pkg = findPackageInResponse(response, repoName, pkgName, [pkgType]);
if (pkg) {
// exit immediately if package found
return pkg;
}
}
}
} catch (err) {
Expand Down Expand Up @@ -163,10 +164,10 @@ async function queryPackage(
async function getCachedPackage(
repoName: string,
pkgName: string
): Promise<RepologyPackage> {
): Promise<RepologyPackage[]> {
// Fetch previous result from cache if available
const cacheKey = `${repoName}/${pkgName}`;
const cachedResult = await packageCache.get<RepologyPackage>(
const cachedResult = await packageCache.get<RepologyPackage[]>(
cacheNamespace,
cacheKey
);
Expand All @@ -175,9 +176,9 @@ async function getCachedPackage(
return cachedResult;
}

// Attempt a package lookup and return if successfully
// Attempt a package lookup and return if found non empty list
const pkg = await queryPackage(repoName, pkgName);
if (pkg) {
if (pkg && pkg.length > 0) {
await packageCache.set(cacheNamespace, cacheKey, pkg, cacheMinutes);
return pkg;
}
Expand Down Expand Up @@ -209,8 +210,10 @@ export async function getReleases({

// Always prefer origversion if available, otherwise default to version
// This is required as source packages usually have no origversion
const version = pkg.origversion ?? pkg.version;
return { releases: [{ version }] };
const releases = pkg.map((item) => ({
version: item.origversion ?? item.version,
}));
return { releases };
} catch (err) {
if (err.message === HOST_DISABLED) {
// istanbul ignore next
Expand Down

0 comments on commit 8032ffd

Please sign in to comment.