From ca6faa687a33838ee79233fd20d696bab22eeddf Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti Date: Sat, 6 Aug 2022 21:00:40 +0300 Subject: [PATCH 01/11] refactor(repo/init): use memCache and return raw config from detectRepoFileConfig --- lib/workers/repository/init/merge.spec.ts | 57 +++++++++++++++++------ lib/workers/repository/init/merge.ts | 45 ++++++++++++++---- lib/workers/repository/init/types.ts | 1 + 3 files changed, 82 insertions(+), 21 deletions(-) diff --git a/lib/workers/repository/init/merge.spec.ts b/lib/workers/repository/init/merge.spec.ts index 3086a7eebbbf8c..71fd21c1d0ac95 100644 --- a/lib/workers/repository/init/merge.spec.ts +++ b/lib/workers/repository/init/merge.spec.ts @@ -8,6 +8,8 @@ import { } from '../../../../test/util'; import * as _migrateAndValidate from '../../../config/migrate-validate'; import * as _migrate from '../../../config/migration'; +import { logger } from '../../../logger'; +import * as memCache from '../../../util/cache/memory'; import { initRepoCache } from '../../../util/cache/repository/init'; import { checkForRepoConfigError, @@ -36,6 +38,7 @@ jest.mock('../../../config/migrate-validate'); describe('workers/repository/init/merge', () => { describe('detectRepoFileConfig()', () => { beforeEach(async () => { + memCache.init(); await initRepoCache({}); }); @@ -54,14 +57,24 @@ describe('workers/repository/init/merge', () => { }, }); fs.readLocalFile.mockResolvedValue(pJson); - platform.getJsonFile.mockResolvedValueOnce(pJson); + platform.getRawFile.mockResolvedValueOnce(pJson); expect(await detectRepoFileConfig()).toEqual({ configFileName: 'package.json', configFileParsed: { prHourlyLimit: 10 }, + configFileRaw: undefined, }); + // get from memCache expect(await detectRepoFileConfig()).toEqual({ configFileName: 'package.json', - configFileParsed: undefined, + configFileParsed: { prHourlyLimit: 10 }, + configFileRaw: undefined, + }); + memCache.reset(); + // get from repoCache + expect(await detectRepoFileConfig()).toEqual({ + configFileName: 'package.json', + configFileParsed: { prHourlyLimit: 10 }, + configFileRaw: undefined, }); }); @@ -72,7 +85,7 @@ describe('workers/repository/init/merge', () => { renovate: 'github>renovatebot/renovate', }); fs.readLocalFile.mockResolvedValue(pJson); - platform.getJsonFile.mockResolvedValueOnce(pJson); + platform.getRawFile.mockResolvedValueOnce(pJson); expect(await detectRepoFileConfig()).toEqual({ configFileName: 'package.json', configFileParsed: { extends: ['github>renovatebot/renovate'] }, @@ -107,13 +120,15 @@ describe('workers/repository/init/merge', () => { }); it('finds and parse renovate.json5', async () => { - git.getFileList.mockResolvedValue(['package.json', 'renovate.json5']); - fs.readLocalFile.mockResolvedValue(`{ + const configFileRaw = `{ // this is json5 format - }`); + }`; + git.getFileList.mockResolvedValue(['package.json', 'renovate.json5']); + fs.readLocalFile.mockResolvedValue(configFileRaw); expect(await detectRepoFileConfig()).toEqual({ configFileName: 'renovate.json5', configFileParsed: {}, + configFileRaw, }); }); @@ -126,6 +141,7 @@ describe('workers/repository/init/merge', () => { expect(await detectRepoFileConfig()).toEqual({ configFileName: '.github/renovate.json', configFileParsed: {}, + configFileRaw: '{}', }); }); @@ -138,23 +154,27 @@ describe('workers/repository/init/merge', () => { expect(await detectRepoFileConfig()).toEqual({ configFileName: '.gitlab/renovate.json', configFileParsed: {}, + configFileRaw: '{}', }); }); it('finds .renovaterc.json', async () => { git.getFileList.mockResolvedValue(['package.json', '.renovaterc.json']); fs.readLocalFile.mockResolvedValue('{}'); - platform.getJsonFile.mockResolvedValueOnce('{"something":"new"}'); + platform.getRawFile.mockResolvedValueOnce('{"something":"new"}'); expect(await detectRepoFileConfig()).toEqual({ configFileName: '.renovaterc.json', configFileParsed: {}, + configFileRaw: '{}', + }); + memCache.reset(); + expect(await detectRepoFileConfig()).toEqual({ + configFileName: '.renovaterc.json', + configFileParsed: { + something: 'new', + }, + configFileRaw: '{"something":"new"}', }); - expect(await detectRepoFileConfig()).toMatchInlineSnapshot(` - Object { - "configFileName": ".renovaterc.json", - "configFileParsed": "{\\"something\\":\\"new\\"}", - } - `); }); }); @@ -174,6 +194,8 @@ describe('workers/repository/init/merge', () => { describe('mergeRenovateConfig()', () => { beforeEach(() => { + // memCache.reset(); + platform.getRawFile.mockResolvedValueOnce(null); migrate.migrateConfig.mockReturnValue({ isMigrated: false, migratedConfig: {}, @@ -194,6 +216,9 @@ describe('workers/repository/init/merge', () => { } expect(e).toBeDefined(); expect(e?.toString()).toBe('Error: config-validation'); + expect(logger.debug).toHaveBeenCalledWith( + 'Existing config file no longer exists' + ); }); it('migrates nested config', async () => { @@ -224,6 +249,9 @@ describe('workers/repository/init/merge', () => { }, ], }); + expect(logger.debug).toHaveBeenCalledWith( + 'Existing config file no longer exists' + ); }); it('continues if no errors', async () => { @@ -235,6 +263,9 @@ describe('workers/repository/init/merge', () => { }); config.extends = [':automergeDisabled']; expect(await mergeRenovateConfig(config)).toBeDefined(); + expect(logger.debug).toHaveBeenCalledWith( + 'Existing config file no longer exists' + ); }); }); }); diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index f0528cef20b26b..3e7b6a6b50ea0a 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -17,6 +17,7 @@ import { import { logger } from '../../../logger'; import * as npmApi from '../../../modules/datasource/npm'; import { platform } from '../../../modules/platform'; +import * as memCache from '../../../util/cache/memory'; import { getCache } from '../../../util/cache/repository'; import { readLocalFile } from '../../../util/fs'; import { getFileList } from '../../../util/git'; @@ -24,19 +25,31 @@ import * as hostRules from '../../../util/host-rules'; import type { RepoFileConfig } from './types'; export async function detectRepoFileConfig(): Promise { + const cacheKey = 'detectRepoFileConfig'; + let repoFileConfig = memCache.get(cacheKey); + if (repoFileConfig) { + return repoFileConfig; + } const cache = getCache(); let { configFileName } = cache; if (configFileName) { - let configFileParsed = (await platform.getJsonFile(configFileName))!; + let configFileRaw: string | undefined = (await platform.getRawFile( + configFileName + ))!; + let configFileParsed = JSON5.parse(configFileRaw); if (configFileParsed) { if (configFileName === 'package.json') { configFileParsed = configFileParsed.renovate; + configFileRaw = undefined; } - return { configFileName, configFileParsed }; + repoFileConfig = { configFileName, configFileRaw, configFileParsed }; + memCache.set(cacheKey, repoFileConfig); + return repoFileConfig; } logger.debug('Existing config file no longer exists'); } const fileList = await getFileList(); + async function detectConfigFile(): Promise { for (const fileName of configFileNames) { if (fileName === 'package.json') { @@ -57,6 +70,7 @@ export async function detectRepoFileConfig(): Promise { } return null; } + configFileName = (await detectConfigFile()) ?? undefined; if (!configFileName) { logger.debug('No renovate config file found'); @@ -66,6 +80,7 @@ export async function detectRepoFileConfig(): Promise { logger.debug(`Found ${configFileName} config file`); // TODO #7154 let configFileParsed: any; + let rawFileContents; if (configFileName === 'package.json') { // We already know it parses configFileParsed = JSON.parse( @@ -78,7 +93,7 @@ export async function detectRepoFileConfig(): Promise { } logger.debug({ config: configFileParsed }, 'package.json>renovate config'); } else { - let rawFileContents = await readLocalFile(configFileName, 'utf8'); + rawFileContents = await readLocalFile(configFileName, 'utf8'); // istanbul ignore if if (!is.string(rawFileContents)) { logger.warn({ configFileName }, 'Null contents when reading config file'); @@ -101,10 +116,12 @@ export async function detectRepoFileConfig(): Promise { ); const validationError = 'Invalid JSON5 (parsing failed)'; const validationMessage = `JSON5.parse error: ${String(err.message)}`; - return { + repoFileConfig = { configFileName, configFileParseError: { validationError, validationMessage }, }; + memCache.set(cacheKey, repoFileConfig); + return repoFileConfig; } } else { let allowDuplicateKeys = true; @@ -115,10 +132,12 @@ export async function detectRepoFileConfig(): Promise { if (jsonValidationError) { const validationError = 'Invalid JSON (parsing failed)'; const validationMessage = jsonValidationError; - return { + repoFileConfig = { configFileName, configFileParseError: { validationError, validationMessage }, }; + memCache.set(cacheKey, repoFileConfig); + return repoFileConfig; } allowDuplicateKeys = false; jsonValidationError = jsonValidator.validate( @@ -128,10 +147,12 @@ export async function detectRepoFileConfig(): Promise { if (jsonValidationError) { const validationError = 'Duplicate keys in JSON'; const validationMessage = JSON.stringify(jsonValidationError); - return { + repoFileConfig = { configFileName, configFileParseError: { validationError, validationMessage }, }; + memCache.set(cacheKey, repoFileConfig); + return repoFileConfig; } try { configFileParsed = JSON5.parse(rawFileContents); @@ -142,10 +163,12 @@ export async function detectRepoFileConfig(): Promise { ); const validationError = 'Invalid JSON (parsing failed)'; const validationMessage = `JSON.parse error: ${String(err.message)}`; - return { + repoFileConfig = { configFileName, configFileParseError: { validationError, validationMessage }, }; + memCache.set(cacheKey, repoFileConfig); + return repoFileConfig; } } logger.debug( @@ -153,7 +176,13 @@ export async function detectRepoFileConfig(): Promise { 'Repository config' ); } - return { configFileName, configFileParsed }; + repoFileConfig = { + configFileName, + configFileRaw: rawFileContents, + configFileParsed, + }; + memCache.set(cacheKey, repoFileConfig); + return repoFileConfig; } export function checkForRepoConfigError(repoConfig: RepoFileConfig): void { diff --git a/lib/workers/repository/init/types.ts b/lib/workers/repository/init/types.ts index 96590b420a5df4..1b255f7b7d0771 100644 --- a/lib/workers/repository/init/types.ts +++ b/lib/workers/repository/init/types.ts @@ -5,6 +5,7 @@ export interface RepoConfigError { export interface RepoFileConfig { configFileName?: string; + configFileRaw?: string; configFileParsed?: any; configFileParseError?: RepoConfigError; } From ffc92f23675f8a4e42da9792a8a32561c98647fd Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti Date: Sun, 7 Aug 2022 09:47:17 +0300 Subject: [PATCH 02/11] refactor(repo/init): use memCache and return raw config from detectRepoFileConfig - revert caching --- lib/workers/repository/init/merge.spec.ts | 57 ++++++----------------- lib/workers/repository/init/merge.ts | 45 ++++-------------- 2 files changed, 21 insertions(+), 81 deletions(-) diff --git a/lib/workers/repository/init/merge.spec.ts b/lib/workers/repository/init/merge.spec.ts index 71fd21c1d0ac95..3086a7eebbbf8c 100644 --- a/lib/workers/repository/init/merge.spec.ts +++ b/lib/workers/repository/init/merge.spec.ts @@ -8,8 +8,6 @@ import { } from '../../../../test/util'; import * as _migrateAndValidate from '../../../config/migrate-validate'; import * as _migrate from '../../../config/migration'; -import { logger } from '../../../logger'; -import * as memCache from '../../../util/cache/memory'; import { initRepoCache } from '../../../util/cache/repository/init'; import { checkForRepoConfigError, @@ -38,7 +36,6 @@ jest.mock('../../../config/migrate-validate'); describe('workers/repository/init/merge', () => { describe('detectRepoFileConfig()', () => { beforeEach(async () => { - memCache.init(); await initRepoCache({}); }); @@ -57,24 +54,14 @@ describe('workers/repository/init/merge', () => { }, }); fs.readLocalFile.mockResolvedValue(pJson); - platform.getRawFile.mockResolvedValueOnce(pJson); + platform.getJsonFile.mockResolvedValueOnce(pJson); expect(await detectRepoFileConfig()).toEqual({ configFileName: 'package.json', configFileParsed: { prHourlyLimit: 10 }, - configFileRaw: undefined, }); - // get from memCache expect(await detectRepoFileConfig()).toEqual({ configFileName: 'package.json', - configFileParsed: { prHourlyLimit: 10 }, - configFileRaw: undefined, - }); - memCache.reset(); - // get from repoCache - expect(await detectRepoFileConfig()).toEqual({ - configFileName: 'package.json', - configFileParsed: { prHourlyLimit: 10 }, - configFileRaw: undefined, + configFileParsed: undefined, }); }); @@ -85,7 +72,7 @@ describe('workers/repository/init/merge', () => { renovate: 'github>renovatebot/renovate', }); fs.readLocalFile.mockResolvedValue(pJson); - platform.getRawFile.mockResolvedValueOnce(pJson); + platform.getJsonFile.mockResolvedValueOnce(pJson); expect(await detectRepoFileConfig()).toEqual({ configFileName: 'package.json', configFileParsed: { extends: ['github>renovatebot/renovate'] }, @@ -120,15 +107,13 @@ describe('workers/repository/init/merge', () => { }); it('finds and parse renovate.json5', async () => { - const configFileRaw = `{ - // this is json5 format - }`; git.getFileList.mockResolvedValue(['package.json', 'renovate.json5']); - fs.readLocalFile.mockResolvedValue(configFileRaw); + fs.readLocalFile.mockResolvedValue(`{ + // this is json5 format + }`); expect(await detectRepoFileConfig()).toEqual({ configFileName: 'renovate.json5', configFileParsed: {}, - configFileRaw, }); }); @@ -141,7 +126,6 @@ describe('workers/repository/init/merge', () => { expect(await detectRepoFileConfig()).toEqual({ configFileName: '.github/renovate.json', configFileParsed: {}, - configFileRaw: '{}', }); }); @@ -154,27 +138,23 @@ describe('workers/repository/init/merge', () => { expect(await detectRepoFileConfig()).toEqual({ configFileName: '.gitlab/renovate.json', configFileParsed: {}, - configFileRaw: '{}', }); }); it('finds .renovaterc.json', async () => { git.getFileList.mockResolvedValue(['package.json', '.renovaterc.json']); fs.readLocalFile.mockResolvedValue('{}'); - platform.getRawFile.mockResolvedValueOnce('{"something":"new"}'); + platform.getJsonFile.mockResolvedValueOnce('{"something":"new"}'); expect(await detectRepoFileConfig()).toEqual({ configFileName: '.renovaterc.json', configFileParsed: {}, - configFileRaw: '{}', - }); - memCache.reset(); - expect(await detectRepoFileConfig()).toEqual({ - configFileName: '.renovaterc.json', - configFileParsed: { - something: 'new', - }, - configFileRaw: '{"something":"new"}', }); + expect(await detectRepoFileConfig()).toMatchInlineSnapshot(` + Object { + "configFileName": ".renovaterc.json", + "configFileParsed": "{\\"something\\":\\"new\\"}", + } + `); }); }); @@ -194,8 +174,6 @@ describe('workers/repository/init/merge', () => { describe('mergeRenovateConfig()', () => { beforeEach(() => { - // memCache.reset(); - platform.getRawFile.mockResolvedValueOnce(null); migrate.migrateConfig.mockReturnValue({ isMigrated: false, migratedConfig: {}, @@ -216,9 +194,6 @@ describe('workers/repository/init/merge', () => { } expect(e).toBeDefined(); expect(e?.toString()).toBe('Error: config-validation'); - expect(logger.debug).toHaveBeenCalledWith( - 'Existing config file no longer exists' - ); }); it('migrates nested config', async () => { @@ -249,9 +224,6 @@ describe('workers/repository/init/merge', () => { }, ], }); - expect(logger.debug).toHaveBeenCalledWith( - 'Existing config file no longer exists' - ); }); it('continues if no errors', async () => { @@ -263,9 +235,6 @@ describe('workers/repository/init/merge', () => { }); config.extends = [':automergeDisabled']; expect(await mergeRenovateConfig(config)).toBeDefined(); - expect(logger.debug).toHaveBeenCalledWith( - 'Existing config file no longer exists' - ); }); }); }); diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index 3e7b6a6b50ea0a..f0528cef20b26b 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -17,7 +17,6 @@ import { import { logger } from '../../../logger'; import * as npmApi from '../../../modules/datasource/npm'; import { platform } from '../../../modules/platform'; -import * as memCache from '../../../util/cache/memory'; import { getCache } from '../../../util/cache/repository'; import { readLocalFile } from '../../../util/fs'; import { getFileList } from '../../../util/git'; @@ -25,31 +24,19 @@ import * as hostRules from '../../../util/host-rules'; import type { RepoFileConfig } from './types'; export async function detectRepoFileConfig(): Promise { - const cacheKey = 'detectRepoFileConfig'; - let repoFileConfig = memCache.get(cacheKey); - if (repoFileConfig) { - return repoFileConfig; - } const cache = getCache(); let { configFileName } = cache; if (configFileName) { - let configFileRaw: string | undefined = (await platform.getRawFile( - configFileName - ))!; - let configFileParsed = JSON5.parse(configFileRaw); + let configFileParsed = (await platform.getJsonFile(configFileName))!; if (configFileParsed) { if (configFileName === 'package.json') { configFileParsed = configFileParsed.renovate; - configFileRaw = undefined; } - repoFileConfig = { configFileName, configFileRaw, configFileParsed }; - memCache.set(cacheKey, repoFileConfig); - return repoFileConfig; + return { configFileName, configFileParsed }; } logger.debug('Existing config file no longer exists'); } const fileList = await getFileList(); - async function detectConfigFile(): Promise { for (const fileName of configFileNames) { if (fileName === 'package.json') { @@ -70,7 +57,6 @@ export async function detectRepoFileConfig(): Promise { } return null; } - configFileName = (await detectConfigFile()) ?? undefined; if (!configFileName) { logger.debug('No renovate config file found'); @@ -80,7 +66,6 @@ export async function detectRepoFileConfig(): Promise { logger.debug(`Found ${configFileName} config file`); // TODO #7154 let configFileParsed: any; - let rawFileContents; if (configFileName === 'package.json') { // We already know it parses configFileParsed = JSON.parse( @@ -93,7 +78,7 @@ export async function detectRepoFileConfig(): Promise { } logger.debug({ config: configFileParsed }, 'package.json>renovate config'); } else { - rawFileContents = await readLocalFile(configFileName, 'utf8'); + let rawFileContents = await readLocalFile(configFileName, 'utf8'); // istanbul ignore if if (!is.string(rawFileContents)) { logger.warn({ configFileName }, 'Null contents when reading config file'); @@ -116,12 +101,10 @@ export async function detectRepoFileConfig(): Promise { ); const validationError = 'Invalid JSON5 (parsing failed)'; const validationMessage = `JSON5.parse error: ${String(err.message)}`; - repoFileConfig = { + return { configFileName, configFileParseError: { validationError, validationMessage }, }; - memCache.set(cacheKey, repoFileConfig); - return repoFileConfig; } } else { let allowDuplicateKeys = true; @@ -132,12 +115,10 @@ export async function detectRepoFileConfig(): Promise { if (jsonValidationError) { const validationError = 'Invalid JSON (parsing failed)'; const validationMessage = jsonValidationError; - repoFileConfig = { + return { configFileName, configFileParseError: { validationError, validationMessage }, }; - memCache.set(cacheKey, repoFileConfig); - return repoFileConfig; } allowDuplicateKeys = false; jsonValidationError = jsonValidator.validate( @@ -147,12 +128,10 @@ export async function detectRepoFileConfig(): Promise { if (jsonValidationError) { const validationError = 'Duplicate keys in JSON'; const validationMessage = JSON.stringify(jsonValidationError); - repoFileConfig = { + return { configFileName, configFileParseError: { validationError, validationMessage }, }; - memCache.set(cacheKey, repoFileConfig); - return repoFileConfig; } try { configFileParsed = JSON5.parse(rawFileContents); @@ -163,12 +142,10 @@ export async function detectRepoFileConfig(): Promise { ); const validationError = 'Invalid JSON (parsing failed)'; const validationMessage = `JSON.parse error: ${String(err.message)}`; - repoFileConfig = { + return { configFileName, configFileParseError: { validationError, validationMessage }, }; - memCache.set(cacheKey, repoFileConfig); - return repoFileConfig; } } logger.debug( @@ -176,13 +153,7 @@ export async function detectRepoFileConfig(): Promise { 'Repository config' ); } - repoFileConfig = { - configFileName, - configFileRaw: rawFileContents, - configFileParsed, - }; - memCache.set(cacheKey, repoFileConfig); - return repoFileConfig; + return { configFileName, configFileParsed }; } export function checkForRepoConfigError(repoConfig: RepoFileConfig): void { From c9a470e84821972eab978a7758fcce00b3abf0e5 Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti Date: Sun, 7 Aug 2022 10:35:15 +0300 Subject: [PATCH 03/11] refactor(repo/init): use memCache and return raw config from detectRepoFileConfig --- lib/workers/repository/init/merge.spec.ts | 34 ++++++++++++++--------- lib/workers/repository/init/merge.ts | 15 +++++++--- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/lib/workers/repository/init/merge.spec.ts b/lib/workers/repository/init/merge.spec.ts index 3086a7eebbbf8c..e47c8e443ee67f 100644 --- a/lib/workers/repository/init/merge.spec.ts +++ b/lib/workers/repository/init/merge.spec.ts @@ -54,14 +54,15 @@ describe('workers/repository/init/merge', () => { }, }); fs.readLocalFile.mockResolvedValue(pJson); - platform.getJsonFile.mockResolvedValueOnce(pJson); + platform.getRawFile.mockResolvedValueOnce(pJson); expect(await detectRepoFileConfig()).toEqual({ configFileName: 'package.json', configFileParsed: { prHourlyLimit: 10 }, }); + // get from repoCache expect(await detectRepoFileConfig()).toEqual({ configFileName: 'package.json', - configFileParsed: undefined, + configFileParsed: { prHourlyLimit: 10 }, }); }); @@ -72,7 +73,7 @@ describe('workers/repository/init/merge', () => { renovate: 'github>renovatebot/renovate', }); fs.readLocalFile.mockResolvedValue(pJson); - platform.getJsonFile.mockResolvedValueOnce(pJson); + platform.getRawFile.mockResolvedValueOnce(pJson); expect(await detectRepoFileConfig()).toEqual({ configFileName: 'package.json', configFileParsed: { extends: ['github>renovatebot/renovate'] }, @@ -107,13 +108,15 @@ describe('workers/repository/init/merge', () => { }); it('finds and parse renovate.json5', async () => { - git.getFileList.mockResolvedValue(['package.json', 'renovate.json5']); - fs.readLocalFile.mockResolvedValue(`{ + const configFileRaw = `{ // this is json5 format - }`); + }`; + git.getFileList.mockResolvedValue(['package.json', 'renovate.json5']); + fs.readLocalFile.mockResolvedValue(configFileRaw); expect(await detectRepoFileConfig()).toEqual({ configFileName: 'renovate.json5', configFileParsed: {}, + configFileRaw, }); }); @@ -126,6 +129,7 @@ describe('workers/repository/init/merge', () => { expect(await detectRepoFileConfig()).toEqual({ configFileName: '.github/renovate.json', configFileParsed: {}, + configFileRaw: '{}', }); }); @@ -138,23 +142,26 @@ describe('workers/repository/init/merge', () => { expect(await detectRepoFileConfig()).toEqual({ configFileName: '.gitlab/renovate.json', configFileParsed: {}, + configFileRaw: '{}', }); }); it('finds .renovaterc.json', async () => { git.getFileList.mockResolvedValue(['package.json', '.renovaterc.json']); fs.readLocalFile.mockResolvedValue('{}'); - platform.getJsonFile.mockResolvedValueOnce('{"something":"new"}'); + platform.getRawFile.mockResolvedValueOnce('{"something":"new"}'); expect(await detectRepoFileConfig()).toEqual({ configFileName: '.renovaterc.json', configFileParsed: {}, + configFileRaw: '{}', + }); + expect(await detectRepoFileConfig()).toEqual({ + configFileName: '.renovaterc.json', + configFileParsed: { + something: 'new', + }, + configFileRaw: '{"something":"new"}', }); - expect(await detectRepoFileConfig()).toMatchInlineSnapshot(` - Object { - "configFileName": ".renovaterc.json", - "configFileParsed": "{\\"something\\":\\"new\\"}", - } - `); }); }); @@ -174,6 +181,7 @@ describe('workers/repository/init/merge', () => { describe('mergeRenovateConfig()', () => { beforeEach(() => { + platform.getRawFile.mockResolvedValueOnce(null); migrate.migrateConfig.mockReturnValue({ isMigrated: false, migratedConfig: {}, diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index f0528cef20b26b..ffa61cfdcfe62a 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -27,16 +27,21 @@ export async function detectRepoFileConfig(): Promise { const cache = getCache(); let { configFileName } = cache; if (configFileName) { - let configFileParsed = (await platform.getJsonFile(configFileName))!; + let configFileRaw: string | undefined = (await platform.getRawFile( + configFileName + ))!; + let configFileParsed = JSON5.parse(configFileRaw); if (configFileParsed) { if (configFileName === 'package.json') { configFileParsed = configFileParsed.renovate; + configFileRaw = undefined; } - return { configFileName, configFileParsed }; + return { configFileName, configFileRaw, configFileParsed }; } logger.debug('Existing config file no longer exists'); } const fileList = await getFileList(); + async function detectConfigFile(): Promise { for (const fileName of configFileNames) { if (fileName === 'package.json') { @@ -57,6 +62,7 @@ export async function detectRepoFileConfig(): Promise { } return null; } + configFileName = (await detectConfigFile()) ?? undefined; if (!configFileName) { logger.debug('No renovate config file found'); @@ -66,6 +72,7 @@ export async function detectRepoFileConfig(): Promise { logger.debug(`Found ${configFileName} config file`); // TODO #7154 let configFileParsed: any; + let rawFileContents; if (configFileName === 'package.json') { // We already know it parses configFileParsed = JSON.parse( @@ -78,7 +85,7 @@ export async function detectRepoFileConfig(): Promise { } logger.debug({ config: configFileParsed }, 'package.json>renovate config'); } else { - let rawFileContents = await readLocalFile(configFileName, 'utf8'); + rawFileContents = await readLocalFile(configFileName, 'utf8'); // istanbul ignore if if (!is.string(rawFileContents)) { logger.warn({ configFileName }, 'Null contents when reading config file'); @@ -153,7 +160,7 @@ export async function detectRepoFileConfig(): Promise { 'Repository config' ); } - return { configFileName, configFileParsed }; + return { configFileName, configFileRaw: rawFileContents, configFileParsed }; } export function checkForRepoConfigError(repoConfig: RepoFileConfig): void { From 2102ccd41fca5152f371752ff5c6e182b28a8a2a Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti <97394622+Gabriel-Ladzaretti@users.noreply.github.com> Date: Thu, 11 Aug 2022 00:23:34 +0300 Subject: [PATCH 04/11] Apply suggestions from code review Co-authored-by: Michael Kriese --- lib/workers/repository/init/merge.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index ffa61cfdcfe62a..d2c8e7313d92a6 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -27,9 +27,9 @@ export async function detectRepoFileConfig(): Promise { const cache = getCache(); let { configFileName } = cache; if (configFileName) { - let configFileRaw: string | undefined = (await platform.getRawFile( + let configFileRaw = (await platform.getRawFile( configFileName - ))!; + )); let configFileParsed = JSON5.parse(configFileRaw); if (configFileParsed) { if (configFileName === 'package.json') { From df5c20826a20aad41230903072d94bec4e4d2ce4 Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti Date: Thu, 11 Aug 2022 00:24:55 +0300 Subject: [PATCH 05/11] refactor(repo/init): use memCache and return raw config from detectRepoFileConfig --- lib/workers/repository/init/merge.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index d2c8e7313d92a6..1af4f2e88e43da 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -27,18 +27,17 @@ export async function detectRepoFileConfig(): Promise { const cache = getCache(); let { configFileName } = cache; if (configFileName) { - let configFileRaw = (await platform.getRawFile( - configFileName - )); - let configFileParsed = JSON5.parse(configFileRaw); - if (configFileParsed) { - if (configFileName === 'package.json') { - configFileParsed = configFileParsed.renovate; - configFileRaw = undefined; + const configFileRaw = await platform.getRawFile(configFileName); + if (configFileRaw) { + let configFileParsed = JSON5.parse(configFileRaw); + if (configFileName !== 'package.json') { + return { configFileName, configFileRaw, configFileParsed }; } - return { configFileName, configFileRaw, configFileParsed }; + configFileParsed = configFileParsed.renovate; + return { configFileName, configFileParsed }; // don't return raw 'package.json' + } else { + logger.debug('Existing config file no longer exists'); } - logger.debug('Existing config file no longer exists'); } const fileList = await getFileList(); From 4275c39cb0bdfec0eda6ccac587f61ebe1f53e25 Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti <97394622+Gabriel-Ladzaretti@users.noreply.github.com> Date: Tue, 16 Aug 2022 10:15:32 +0300 Subject: [PATCH 06/11] Apply suggestions from code review Co-authored-by: Michael Kriese --- lib/workers/repository/init/merge.ts | 6 +++--- lib/workers/repository/init/types.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index eaa521b12349e8..b4eafa160658a3 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -70,7 +70,7 @@ export async function detectRepoFileConfig(): Promise { logger.debug(`Found ${configFileName} config file`); // TODO #7154 let configFileParsed: any; - let rawFileContents; + let configFileRaw: string | null; if (configFileName === 'package.json') { // We already know it parses configFileParsed = JSON.parse( @@ -83,7 +83,7 @@ export async function detectRepoFileConfig(): Promise { } logger.debug({ config: configFileParsed }, 'package.json>renovate config'); } else { - rawFileContents = await readLocalFile(configFileName, 'utf8'); + configFileRaw = await readLocalFile(configFileName, 'utf8'); // istanbul ignore if if (!is.string(rawFileContents)) { logger.warn({ configFileName }, 'Null contents when reading config file'); @@ -158,7 +158,7 @@ export async function detectRepoFileConfig(): Promise { 'Repository config' ); } - return { configFileName, configFileRaw: rawFileContents, configFileParsed }; + return { configFileName, configFileRaw, configFileParsed }; } export function checkForRepoConfigError(repoConfig: RepoFileConfig): void { diff --git a/lib/workers/repository/init/types.ts b/lib/workers/repository/init/types.ts index 1b255f7b7d0771..4dcb6472a803ba 100644 --- a/lib/workers/repository/init/types.ts +++ b/lib/workers/repository/init/types.ts @@ -5,7 +5,7 @@ export interface RepoConfigError { export interface RepoFileConfig { configFileName?: string; - configFileRaw?: string; + configFileRaw: string | null; configFileParsed?: any; configFileParseError?: RepoConfigError; } From df0061b42cb27571c6129f22ca38c93183136879 Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti Date: Tue, 16 Aug 2022 10:24:59 +0300 Subject: [PATCH 07/11] Revert "Apply suggestions from code review" This reverts commit 4275c39cb0bdfec0eda6ccac587f61ebe1f53e25. --- lib/workers/repository/init/merge.ts | 6 +++--- lib/workers/repository/init/types.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index b4eafa160658a3..eaa521b12349e8 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -70,7 +70,7 @@ export async function detectRepoFileConfig(): Promise { logger.debug(`Found ${configFileName} config file`); // TODO #7154 let configFileParsed: any; - let configFileRaw: string | null; + let rawFileContents; if (configFileName === 'package.json') { // We already know it parses configFileParsed = JSON.parse( @@ -83,7 +83,7 @@ export async function detectRepoFileConfig(): Promise { } logger.debug({ config: configFileParsed }, 'package.json>renovate config'); } else { - configFileRaw = await readLocalFile(configFileName, 'utf8'); + rawFileContents = await readLocalFile(configFileName, 'utf8'); // istanbul ignore if if (!is.string(rawFileContents)) { logger.warn({ configFileName }, 'Null contents when reading config file'); @@ -158,7 +158,7 @@ export async function detectRepoFileConfig(): Promise { 'Repository config' ); } - return { configFileName, configFileRaw, configFileParsed }; + return { configFileName, configFileRaw: rawFileContents, configFileParsed }; } export function checkForRepoConfigError(repoConfig: RepoFileConfig): void { diff --git a/lib/workers/repository/init/types.ts b/lib/workers/repository/init/types.ts index 4dcb6472a803ba..1b255f7b7d0771 100644 --- a/lib/workers/repository/init/types.ts +++ b/lib/workers/repository/init/types.ts @@ -5,7 +5,7 @@ export interface RepoConfigError { export interface RepoFileConfig { configFileName?: string; - configFileRaw: string | null; + configFileRaw?: string; configFileParsed?: any; configFileParseError?: RepoConfigError; } From c5eb47daa6bece636116490619e194751c4d874a Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti Date: Tue, 16 Aug 2022 10:38:31 +0300 Subject: [PATCH 08/11] refactor(repo/init): use memCache and return raw config from detectRepoFileConfig --- lib/workers/repository/init/merge.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index eaa521b12349e8..21839eac3b5da7 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -70,7 +70,7 @@ export async function detectRepoFileConfig(): Promise { logger.debug(`Found ${configFileName} config file`); // TODO #7154 let configFileParsed: any; - let rawFileContents; + let configFileRaw: string | undefined; if (configFileName === 'package.json') { // We already know it parses configFileParsed = JSON.parse( @@ -83,25 +83,25 @@ export async function detectRepoFileConfig(): Promise { } logger.debug({ config: configFileParsed }, 'package.json>renovate config'); } else { - rawFileContents = await readLocalFile(configFileName, 'utf8'); + configFileRaw = (await readLocalFile(configFileName, 'utf8')) ?? undefined; // istanbul ignore if - if (!is.string(rawFileContents)) { + if (!is.string(configFileRaw)) { logger.warn({ configFileName }, 'Null contents when reading config file'); throw new Error(REPOSITORY_CHANGED); } // istanbul ignore if - if (!rawFileContents.length) { - rawFileContents = '{}'; + if (!configFileRaw.length) { + configFileRaw = '{}'; } const fileType = upath.extname(configFileName); if (fileType === '.json5') { try { - configFileParsed = JSON5.parse(rawFileContents); + configFileParsed = JSON5.parse(configFileRaw); } catch (err) /* istanbul ignore next */ { logger.debug( - { renovateConfig: rawFileContents }, + { renovateConfig: configFileRaw }, 'Error parsing renovate config renovate.json5' ); const validationError = 'Invalid JSON5 (parsing failed)'; @@ -114,7 +114,7 @@ export async function detectRepoFileConfig(): Promise { } else { let allowDuplicateKeys = true; let jsonValidationError = jsonValidator.validate( - rawFileContents, + configFileRaw, allowDuplicateKeys ); if (jsonValidationError) { @@ -127,7 +127,7 @@ export async function detectRepoFileConfig(): Promise { } allowDuplicateKeys = false; jsonValidationError = jsonValidator.validate( - rawFileContents, + configFileRaw, allowDuplicateKeys ); if (jsonValidationError) { @@ -139,10 +139,10 @@ export async function detectRepoFileConfig(): Promise { }; } try { - configFileParsed = JSON5.parse(rawFileContents); + configFileParsed = JSON5.parse(configFileRaw); } catch (err) /* istanbul ignore next */ { logger.debug( - { renovateConfig: rawFileContents }, + { renovateConfig: configFileRaw }, 'Error parsing renovate config' ); const validationError = 'Invalid JSON (parsing failed)'; @@ -158,7 +158,7 @@ export async function detectRepoFileConfig(): Promise { 'Repository config' ); } - return { configFileName, configFileRaw: rawFileContents, configFileParsed }; + return { configFileName, configFileRaw, configFileParsed }; } export function checkForRepoConfigError(repoConfig: RepoFileConfig): void { From f17dc8080794d33308ae6c515d1d15faa7b22e20 Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti <97394622+Gabriel-Ladzaretti@users.noreply.github.com> Date: Tue, 16 Aug 2022 14:50:18 +0300 Subject: [PATCH 09/11] Apply suggestions from code review Co-authored-by: Michael Kriese --- lib/workers/repository/init/merge.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index 21839eac3b5da7..a87b7763879bdc 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -70,7 +70,7 @@ export async function detectRepoFileConfig(): Promise { logger.debug(`Found ${configFileName} config file`); // TODO #7154 let configFileParsed: any; - let configFileRaw: string | undefined; + let configFileRaw: string | undefined | null; if (configFileName === 'package.json') { // We already know it parses configFileParsed = JSON.parse( @@ -83,7 +83,7 @@ export async function detectRepoFileConfig(): Promise { } logger.debug({ config: configFileParsed }, 'package.json>renovate config'); } else { - configFileRaw = (await readLocalFile(configFileName, 'utf8')) ?? undefined; + configFileRaw = (await readLocalFile(configFileName, 'utf8')); // istanbul ignore if if (!is.string(configFileRaw)) { logger.warn({ configFileName }, 'Null contents when reading config file'); From 073406f5df74fb865f8b0dcf0f41628a8f8a5836 Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti Date: Tue, 16 Aug 2022 14:50:38 +0300 Subject: [PATCH 10/11] refactor(repo/init): use memCache and return raw config from detectRepoFileConfig --- lib/workers/repository/init/merge.ts | 2 +- lib/workers/repository/init/types.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index a87b7763879bdc..d7b3a52ab0a6ba 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -83,7 +83,7 @@ export async function detectRepoFileConfig(): Promise { } logger.debug({ config: configFileParsed }, 'package.json>renovate config'); } else { - configFileRaw = (await readLocalFile(configFileName, 'utf8')); + configFileRaw = await readLocalFile(configFileName, 'utf8'); // istanbul ignore if if (!is.string(configFileRaw)) { logger.warn({ configFileName }, 'Null contents when reading config file'); diff --git a/lib/workers/repository/init/types.ts b/lib/workers/repository/init/types.ts index 1b255f7b7d0771..ebd6a8731378d9 100644 --- a/lib/workers/repository/init/types.ts +++ b/lib/workers/repository/init/types.ts @@ -5,7 +5,7 @@ export interface RepoConfigError { export interface RepoFileConfig { configFileName?: string; - configFileRaw?: string; + configFileRaw?: string | null; configFileParsed?: any; configFileParseError?: RepoConfigError; } From c67d9425bd56a8958a395449e940b485bfa2e493 Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti <97394622+Gabriel-Ladzaretti@users.noreply.github.com> Date: Thu, 18 Aug 2022 16:45:37 +0300 Subject: [PATCH 11/11] Update lib/workers/repository/init/merge.spec.ts Co-authored-by: Michael Kriese --- lib/workers/repository/init/merge.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/workers/repository/init/merge.spec.ts b/lib/workers/repository/init/merge.spec.ts index e47c8e443ee67f..25c01f66505aaf 100644 --- a/lib/workers/repository/init/merge.spec.ts +++ b/lib/workers/repository/init/merge.spec.ts @@ -181,7 +181,6 @@ describe('workers/repository/init/merge', () => { describe('mergeRenovateConfig()', () => { beforeEach(() => { - platform.getRawFile.mockResolvedValueOnce(null); migrate.migrateConfig.mockReturnValue({ isMigrated: false, migratedConfig: {},