Skip to content

Commit

Permalink
refactor: npm preset lookup direct (#9225)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rarkins
rarkins committed Mar 20, 2021
1 parent 11454b5 commit 9d6e96e
Show file tree
Hide file tree
Showing 5 changed files with 110 additions and 93 deletions.
19 changes: 15 additions & 4 deletions lib/config/presets/npm/index.ts
View file
@@ -1,13 +1,24 @@
import { getDependency } from '../../../datasource/npm/get';
import { resolvePackage } from '../../../datasource/npm/npmrc';
import { NpmResponse } from '../../../datasource/npm/types';
import { logger } from '../../../logger';
import { Http } from '../../../util/http';
import type { Preset, PresetConfig } from '../types';

const id = 'npm';

const http = new Http(id);

export async function getPreset({
packageName: pkgName,
packageName,
presetName = 'default',
}: PresetConfig): Promise<Preset> {
const dep = await getDependency(pkgName);
if (!dep) {
let dep;
try {
const { headers, packageUrl } = resolvePackage(packageName);
const body = (await http.getJson<NpmResponse>(packageUrl, { headers }))
.body;
dep = body.versions[body['dist-tags'].latest];
} catch (err) {
throw new Error('dep not found');
}
if (!dep['renovate-config']) {
Expand Down
105 changes: 18 additions & 87 deletions lib/datasource/npm/get.ts
View file
@@ -1,16 +1,13 @@
import { OutgoingHttpHeaders } from 'http';
import url from 'url';
import is from '@sindresorhus/is';
import registryAuthToken from 'registry-auth-token';
import getRegistryUrl from 'registry-auth-token/registry-url';
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import * as packageCache from '../../util/cache/package';
import { Http, HttpOptions } from '../../util/http';
import { maskToken } from '../../util/mask';
import type { Release, ReleaseResult } from '../types';
import { id } from './common';
import { getNpmrc } from './npmrc';
import { resolvePackage } from './npmrc';
import { NpmResponse } from './types';

const http = new Http(id);

Expand All @@ -36,35 +33,9 @@ export interface NpmDependency extends ReleaseResult {
sourceUrl: string;
versions: Record<string, any>;
'dist-tags': Record<string, string>;
'renovate-config': any;
sourceDirectory?: string;
}

export interface NpmResponse {
_id: string;
name?: string;
versions?: Record<
string,
{
repository?: {
url: string;
directory: string;
};
homepage?: string;
deprecated?: boolean;
gitHead?: string;
dependencies?: Record<string, string>;
devDependencies?: Record<string, string>;
}
>;
repository?: {
url?: string;
directory?: string;
};
homepage?: string;
time?: Record<string, string>;
}

export async function getDependency(
packageName: string
): Promise<NpmDependency | null> {
Expand All @@ -76,55 +47,20 @@ export async function getDependency(
return JSON.parse(memcache[packageName]) as NpmDependency;
}

const scope = packageName.split('/')[0];
let regUrl: string;
const npmrc = getNpmrc();
try {
regUrl = getRegistryUrl(scope, npmrc);
} catch (err) {
regUrl = 'https://registry.npmjs.org';
}
const pkgUrl = url.resolve(
regUrl,
encodeURIComponent(packageName).replace(/^%40/, '@')
);
const { headers, packageUrl, registryUrl } = resolvePackage(packageName);

// Now check the persistent cache
const cacheNamespace = 'datasource-npm';
const cachedResult = await packageCache.get<NpmDependency>(
cacheNamespace,
pkgUrl
packageUrl
);
// istanbul ignore if
if (cachedResult) {
return cachedResult;
}
const headers: OutgoingHttpHeaders = {};
let authInfo = registryAuthToken(regUrl, { npmrc, recursive: true });

if (
!authInfo &&
npmrc &&
npmrc._authToken &&
regUrl.replace(/\/?$/, '/') === npmrc.registry?.replace(/\/?$/, '/')
) {
authInfo = { type: 'Bearer', token: npmrc._authToken };
}

if (authInfo?.type && authInfo.token) {
headers.authorization = `${authInfo.type} ${authInfo.token}`;
logger.trace(
{ token: maskToken(authInfo.token), npmName: packageName },
'Using auth (via npmrc) for npm lookup'
);
} else if (process.env.NPM_TOKEN && process.env.NPM_TOKEN !== 'undefined') {
logger.trace(
{ token: maskToken(process.env.NPM_TOKEN), npmName: packageName },
'Using auth (via process.env.NPM_TOKEN) for npm lookup'
);
headers.authorization = `Bearer ${process.env.NPM_TOKEN}`;
}

const uri = url.parse(pkgUrl);
const uri = url.parse(packageUrl);

if (uri.host === 'registry.npmjs.org' && !uri.pathname.startsWith('/@')) {
// Delete the authorization header for non-scoped public packages to improve http caching
Expand All @@ -137,7 +73,7 @@ export async function getDependency(
const opts: HttpOptions = {
headers,
};
const raw = await http.getJson<NpmResponse>(pkgUrl, opts);
const raw = await http.getJson<NpmResponse>(packageUrl, opts);
const res = raw.body;
if (!res.versions || !Object.keys(res.versions).length) {
// Registry returned a 200 OK but with no versions
Expand Down Expand Up @@ -167,14 +103,13 @@ export async function getDependency(
versions: {},
releases: null,
'dist-tags': res['dist-tags'],
'renovate-config': latestVersion['renovate-config'],
registryUrl: regUrl,
registryUrl,
};
if (res.repository?.directory) {
dep.sourceDirectory = res.repository.directory;
}
if (latestVersion.deprecated) {
dep.deprecationMessage = `On registry \`${regUrl}\`, the "latest" version of dependency \`${packageName}\` has the following deprecation notice:\n\n\`${latestVersion.deprecated}\`\n\nMarking the latest version of an npm package as deprecated results in the entire package being considered deprecated, so contact the package author you think this is a mistake.`;
dep.deprecationMessage = `On registry \`${registryUrl}\`, the "latest" version of dependency \`${packageName}\` has the following deprecation notice:\n\n\`${latestVersion.deprecated}\`\n\nMarking the latest version of an npm package as deprecated results in the entire package being considered deprecated, so contact the package author you think this is a mistake.`;
dep.deprecationSource = id;
}
dep.releases = Object.keys(res.versions).map((version) => {
Expand Down Expand Up @@ -207,18 +142,17 @@ export async function getDependency(
];
if (
!raw.authorization &&
(whitelistedPublicScopes.includes(scope) || !packageName.startsWith('@'))
(whitelistedPublicScopes.includes(packageName.split('/')[0]) ||
!packageName.startsWith('@'))
) {
await packageCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes);
await packageCache.set(cacheNamespace, packageUrl, dep, cacheMinutes);
}
return dep;
} catch (err) {
if (err.statusCode === 401 || err.statusCode === 403) {
logger.debug(
{
pkgUrl,
authInfoType: authInfo ? authInfo.type : undefined,
authInfoToken: authInfo ? maskToken(authInfo.token) : undefined,
packageUrl,
err,
statusCode: err.statusCode,
packageName,
Expand All @@ -230,9 +164,7 @@ export async function getDependency(
if (err.statusCode === 402) {
logger.debug(
{
pkgUrl,
authInfoType: authInfo ? authInfo.type : undefined,
authInfoToken: authInfo ? maskToken(authInfo.token) : undefined,
packageUrl,
err,
statusCode: err.statusCode,
packageName,
Expand All @@ -242,11 +174,10 @@ export async function getDependency(
return null;
}
if (err.statusCode === 404 || err.code === 'ENOTFOUND') {
logger.debug({ packageName }, `Dependency lookup failure: not found`);
logger.debug({
err,
token: authInfo ? maskToken(authInfo.token) : 'none',
});
logger.debug(
{ err, packageName },
`Dependency lookup failure: not found`
);
return null;
}
if (uri.host === 'registry.npmjs.org') {
Expand Down
54 changes: 53 additions & 1 deletion lib/datasource/npm/npmrc.ts
View file
@@ -1,13 +1,20 @@
import { OutgoingHttpHeaders } from 'http';
import url from 'url';
import is from '@sindresorhus/is';
import ini from 'ini';
import registryAuthToken from 'registry-auth-token';
import getRegistryUrl from 'registry-auth-token/registry-url';
import { getAdminConfig } from '../../config/admin';
import { logger } from '../../logger';
import { maskToken } from '../../util/mask';
import { add } from '../../util/sanitize';

let npmrc: Record<string, any> | null = null;
let npmrcRaw: string;

export function getNpmrc(): Record<string, any> | null {
export type Npmrc = Record<string, any>;

export function getNpmrc(): Npmrc | null {
return npmrc;
}

Expand Down Expand Up @@ -86,3 +93,48 @@ export function setNpmrc(input?: string): void {
npmrcRaw = null;
}
}

export interface PackageResolution {
headers: OutgoingHttpHeaders;
packageUrl: string;
registryUrl: string;
}

export function resolvePackage(packageName: string): PackageResolution {
const scope = packageName.split('/')[0];
let registryUrl: string;
try {
registryUrl = getRegistryUrl(scope, getNpmrc());
} catch (err) {
registryUrl = 'https://registry.npmjs.org';
}
const packageUrl = url.resolve(
registryUrl,
encodeURIComponent(packageName).replace(/^%40/, '@')
);
const headers: OutgoingHttpHeaders = {};
let authInfo = registryAuthToken(registryUrl, { npmrc, recursive: true });
if (
!authInfo &&
npmrc &&
npmrc._authToken &&
registryUrl.replace(/\/?$/, '/') === npmrc.registry?.replace(/\/?$/, '/')
) {
authInfo = { type: 'Bearer', token: npmrc._authToken };
}

if (authInfo?.type && authInfo.token) {
headers.authorization = `${authInfo.type} ${authInfo.token}`;
logger.trace(
{ token: maskToken(authInfo.token), npmName: packageName },
'Using auth (via npmrc) for npm lookup'
);
} else if (process.env.NPM_TOKEN && process.env.NPM_TOKEN !== 'undefined') {
logger.trace(
{ token: maskToken(process.env.NPM_TOKEN), npmName: packageName },
'Using auth (via process.env.NPM_TOKEN) for npm lookup'
);
headers.authorization = `Bearer ${process.env.NPM_TOKEN}`;
}
return { headers, packageUrl, registryUrl };
}
1 change: 0 additions & 1 deletion lib/datasource/npm/releases.ts
View file
Expand Up @@ -13,7 +13,6 @@ export async function getReleases({
if (res) {
res.tags = res['dist-tags'];
delete res['dist-tags'];
delete res['renovate-config'];
}
return res;
}
24 changes: 24 additions & 0 deletions lib/datasource/npm/types.ts
View file
@@ -0,0 +1,24 @@
export interface NpmResponse {
_id: string;
name?: string;
versions?: Record<
string,
{
repository?: {
url: string;
directory: string;
};
homepage?: string;
deprecated?: boolean;
gitHead?: string;
dependencies?: Record<string, string>;
devDependencies?: Record<string, string>;
}
>;
repository?: {
url?: string;
directory?: string;
};
homepage?: string;
time?: Record<string, string>;
}

0 comments on commit 9d6e96e

Please sign in to comment.