Skip to content

Commit

Permalink
refactor(maven): Prepare for hybrid fetching approach (#12617)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zharinov
zharinov committed Nov 13, 2021
1 parent dc2da1a commit 216b11f
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 93 deletions.
178 changes: 88 additions & 90 deletions lib/datasource/maven/index.ts
View file
@@ -1,4 +1,5 @@
import pMap from 'p-map';
import is from '@sindresorhus/is';
import pAll from 'p-all';
import { XmlDocument } from 'xmldoc';
import { logger } from '../../logger';
import * as packageCache from '../../util/cache/package';
Expand All @@ -7,11 +8,7 @@ import * as mavenVersioning from '../../versioning/maven';
import { compare } from '../../versioning/maven/compare';
import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
import { MAVEN_REPO } from './common';
import type {
ArtifactInfoResult,
ArtifactsInfo,
MavenDependency,
} from './types';
import type { MavenDependency, ReleaseMap } from './types';
import {
downloadMavenXml,
getDependencyInfo,
Expand Down Expand Up @@ -49,15 +46,15 @@ function extractVersions(metadata: XmlDocument): string[] {
return elements.map((el) => el.val);
}

async function getVersionsFromMetadata(
async function fetchReleasesFromMetadata(
dependency: MavenDependency,
repoUrl: string
): Promise<string[] | null> {
): Promise<ReleaseMap> {
const metadataUrl = getMavenUrl(dependency, repoUrl, 'maven-metadata.xml');

const cacheNamespace = 'datasource-maven-metadata';
const cacheNamespace = 'datasource-maven:metadata-xml';
const cacheKey = metadataUrl.toString();
const cachedVersions = await packageCache.get<string[]>(
const cachedVersions = await packageCache.get<ReleaseMap>(
cacheNamespace,
cacheKey
);
Expand All @@ -70,25 +67,18 @@ async function getVersionsFromMetadata(
metadataUrl
);
if (!mavenMetadata) {
return null;
return {};
}

const versions = extractVersions(mavenMetadata);
const releaseMap = versions.reduce(
(acc, version) => ({ ...acc, [version]: null }),
{}
);
if (!authorization) {
await packageCache.set(cacheNamespace, cacheKey, versions, 30);
}
return versions;
}

// istanbul ignore next
function isValidArtifactsInfo(
info: ArtifactsInfo | null,
versions: string[]
): boolean {
if (!info) {
return false;
await packageCache.set(cacheNamespace, cacheKey, releaseMap, 30);
}
return versions.every((v) => info[v] !== undefined);
return releaseMap;
}

function isSnapshotVersion(version: string): boolean {
Expand Down Expand Up @@ -166,74 +156,83 @@ async function createUrlForDependencyPom(
return `${version}/${dependency.name}-${version}.pom`;
}

async function getReleasesFromHeadRequests(
async function addReleasesUsingHeadRequests(
inputReleaseMap: ReleaseMap,
dependency: MavenDependency,
repoUrl: string,
versions: string[]
): Promise<Release[] | null> {
repoUrl: string
): Promise<ReleaseMap> {
const releaseMap = { ...inputReleaseMap };

if (process.env.RENOVATE_EXPERIMENTAL_NO_MAVEN_POM_CHECK) {
return null;
return releaseMap;
}

const cacheNamespace = 'datasource-maven-metadata';
const cacheNs = 'datasource-maven:head-requests';
const cacheKey = `${repoUrl}${dependency.dependencyUrl}`;
let artifactsInfo: ArtifactsInfo | null =
await packageCache.get<ArtifactsInfo>(cacheNamespace, cacheKey);

// If the cache contains any artifacts that we were previously unable to determine if they exist,
// retry the existence checks on them.
if (!isValidArtifactsInfo(artifactsInfo, versions)) {
// For each version, determine if there is a POM file available for it
const results: ArtifactInfoResult[] = await pMap(
versions,
async (version): Promise<ArtifactInfoResult | null> => {
// Create the URL that the POM file should be available at
const artifactUrl = getMavenUrl(
dependency,
repoUrl,
await createUrlForDependencyPom(version, dependency, repoUrl)
);

// Return an ArtifactInfoResult that maps the version to the result of the check if the POM file exists in the repo
return [version, await isHttpResourceExists(artifactUrl)];
},
{ concurrency: 5 }
);
let workingReleaseMap: ReleaseMap = await packageCache.get<ReleaseMap>(
cacheNs,
cacheKey
);

artifactsInfo = results.reduce(
(acc, [key, value]) => ({
...acc,
[key]: value,
}),
{}
);
if (!workingReleaseMap) {
workingReleaseMap = {};

// Retry earlier for status other than 404
const cacheTTL = Object.values(artifactsInfo).some((x) => x === null)
? 60
: 24 * 60;
const unknownVersions = Object.entries(releaseMap)
.filter(([version, release]) => {
const isDiscoveredOutside = !!release;
const isDiscoveredInsideAndCached = !is.undefined(
workingReleaseMap[version]
);
const isDiscovered = isDiscoveredOutside || isDiscoveredInsideAndCached;
return !isDiscovered;
})
.map(([k]) => k);

if (unknownVersions.length) {
let retryEarlier = false;
const queue = unknownVersions.map(
(version) => async (): Promise<void> => {
const pomUrl = await createUrlForDependencyPom(
version,
dependency,
repoUrl
);
const artifactUrl = getMavenUrl(dependency, repoUrl, pomUrl);
const res = await isHttpResourceExists(artifactUrl);
const release: Release = { version };

if (is.string(res)) {
release.releaseTimestamp = res;
}

// Retry earlier for error status other than 404
if (res === null) {
retryEarlier = true;
}

workingReleaseMap[version] = res ? release : null;
}
);

await pAll(queue, { concurrency: 5 });
const cacheTTL = retryEarlier ? 60 : 24 * 60;
await packageCache.set(cacheNs, cacheKey, workingReleaseMap, cacheTTL);
}
}

await packageCache.set(cacheNamespace, cacheKey, artifactsInfo, cacheTTL);
for (const version of Object.keys(releaseMap)) {
releaseMap[version] ||= workingReleaseMap[version] ?? null;
}

// Create releases for every version that exists in the repository
return versions
.filter((v) => artifactsInfo[v])
.map((version) => {
const release: Release = { version };
const releaseTimestamp = artifactsInfo[version];
if (releaseTimestamp && typeof releaseTimestamp === 'string') {
release.releaseTimestamp = releaseTimestamp;
}
return release;
});
return releaseMap;
}

function getFallbackReleases(versions: string[]): Release[] | null {
const result = versions?.map((version) => ({
version,
}));
return result ?? null;
function getReleasesFromMap(releaseMap: ReleaseMap): Release[] {
const releases = Object.values(releaseMap).filter(Boolean);
if (releases.length) {
return releases;
}
return Object.keys(releaseMap).map((version) => ({ version }));
}

export async function getReleases({
Expand All @@ -245,14 +244,17 @@ export async function getReleases({

logger.debug(`Looking up ${dependency.display} in repository ${repoUrl}`);

let releases: Release[] = null;
const versions = await getVersionsFromMetadata(dependency, repoUrl);
releases ||= await getReleasesFromHeadRequests(dependency, repoUrl, versions);
releases ||= getFallbackReleases(versions);

let releaseMap = await fetchReleasesFromMetadata(dependency, repoUrl);
releaseMap = await addReleasesUsingHeadRequests(
releaseMap,
dependency,
repoUrl
);
const releases = getReleasesFromMap(releaseMap);
if (!releases?.length) {
return null;
}

logger.debug(
`Found ${releases.length} new releases for ${dependency.display} in repository ${repoUrl}`
);
Expand All @@ -262,9 +264,5 @@ export async function getReleases({
latestStableVersion &&
(await getDependencyInfo(dependency, repoUrl, latestStableVersion));

return {
...dependency,
...dependencyInfo,
releases,
};
return { ...dependency, ...dependencyInfo, releases };
}
5 changes: 2 additions & 3 deletions lib/datasource/maven/types.ts
View file
@@ -1,4 +1,5 @@
import type { XmlDocument } from 'xmldoc';
import type { Release } from '../types';

export interface MavenDependency {
display: string;
Expand All @@ -12,6 +13,4 @@ export interface MavenXml {
xml?: XmlDocument;
}

export type ArtifactsInfo = Record<string, boolean | null>;

export type ArtifactInfoResult = [string, boolean | string | null];
export type ReleaseMap = Record<string, Release | null>;

0 comments on commit 216b11f

Please sign in to comment.