diff --git a/lib/config/presets/gitea/index.ts b/lib/config/presets/gitea/index.ts index 66eef53360a265..adb71c599337d4 100644 --- a/lib/config/presets/gitea/index.ts +++ b/lib/config/presets/gitea/index.ts @@ -29,7 +29,7 @@ export async function fetchJSONFile( } // TODO: null check #22198 - return parsePreset(fromBase64(res.content!)); + return parsePreset(fromBase64(res.content!), fileName); } export function getPresetFromEndpoint( diff --git a/lib/config/presets/github/index.ts b/lib/config/presets/github/index.ts index da087ae1c4582e..4d1584b0d8e6e9 100644 --- a/lib/config/presets/github/index.ts +++ b/lib/config/presets/github/index.ts @@ -34,7 +34,7 @@ export async function fetchJSONFile( throw new Error(PRESET_DEP_NOT_FOUND); } - return parsePreset(fromBase64(res.body.content)); + return parsePreset(fromBase64(res.body.content), fileName); } export function getPresetFromEndpoint( diff --git a/lib/config/presets/gitlab/index.ts b/lib/config/presets/gitlab/index.ts index b1f4071ccd9923..6c3c336bb3a01e 100644 --- a/lib/config/presets/gitlab/index.ts +++ b/lib/config/presets/gitlab/index.ts @@ -52,7 +52,7 @@ export async function fetchJSONFile( throw new Error(PRESET_DEP_NOT_FOUND); } - return parsePreset(res.body); + return parsePreset(res.body, fileName); } export function getPresetFromEndpoint( diff --git a/lib/config/presets/local/common.ts b/lib/config/presets/local/common.ts index bc955972483577..00e11f8517f553 100644 --- a/lib/config/presets/local/common.ts +++ b/lib/config/presets/local/common.ts @@ -29,7 +29,7 @@ export async function fetchJSONFile( throw new Error(PRESET_DEP_NOT_FOUND); } - return parsePreset(raw); + return parsePreset(raw, fileName); } export function getPresetFromEndpoint( diff --git a/lib/config/presets/util.ts b/lib/config/presets/util.ts index f7a22505a98a0c..7b2f166886ff5a 100644 --- a/lib/config/presets/util.ts +++ b/lib/config/presets/util.ts @@ -1,5 +1,5 @@ -import JSON5 from 'json5'; import { logger } from '../../logger'; +import { parseJson } from '../../util/common'; import { regEx } from '../../util/regex'; import { ensureTrailingSlash } from '../../util/url'; import type { FetchPresetConfig, Preset } from './types'; @@ -87,9 +87,9 @@ export async function fetchPreset({ return jsonContent; } -export function parsePreset(content: string): Preset { +export function parsePreset(content: string, fileName: string): Preset { try { - return JSON5.parse(content); + return parseJson(content, fileName) as Preset; } catch (err) { throw new Error(PRESET_INVALID_JSON); } diff --git a/lib/modules/platform/azure/index.ts b/lib/modules/platform/azure/index.ts index f3c869b7b1cec8..e367d9c1423b7d 100644 --- a/lib/modules/platform/azure/index.ts +++ b/lib/modules/platform/azure/index.ts @@ -9,7 +9,6 @@ import { GitVersionDescriptor, PullRequestStatus, } from 'azure-devops-node-api/interfaces/GitInterfaces.js'; -import JSON5 from 'json5'; import { REPOSITORY_ARCHIVED, REPOSITORY_EMPTY, @@ -18,6 +17,7 @@ import { import { logger } from '../../../logger'; import type { BranchStatus } from '../../../types'; import { ExternalHostError } from '../../../types/errors/external-host-error'; +import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import * as hostRules from '../../../util/host-rules'; import { regEx } from '../../../util/regex'; @@ -182,7 +182,7 @@ export async function getJsonFile( branchOrTag?: string ): Promise { const raw = await getRawFile(fileName, repoName, branchOrTag); - return raw ? JSON5.parse(raw) : null; + return parseJson(raw, fileName); } export async function initRepo({ diff --git a/lib/modules/platform/bitbucket-server/index.ts b/lib/modules/platform/bitbucket-server/index.ts index eea3911b79d835..eaea2c62f02e7e 100644 --- a/lib/modules/platform/bitbucket-server/index.ts +++ b/lib/modules/platform/bitbucket-server/index.ts @@ -1,5 +1,4 @@ import { setTimeout } from 'timers/promises'; -import JSON5 from 'json5'; import type { PartialDeep } from 'type-fest'; import { REPOSITORY_CHANGED, @@ -9,6 +8,7 @@ import { import { logger } from '../../../logger'; import type { BranchStatus } from '../../../types'; import type { FileData } from '../../../types/platform/bitbucket-server'; +import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import { deleteBranch } from '../../../util/git'; import * as hostRules from '../../../util/host-rules'; @@ -146,8 +146,8 @@ export async function getJsonFile( branchOrTag?: string ): Promise { // TODO #22198 - const raw = (await getRawFile(fileName, repoName, branchOrTag)) as string; - return JSON5.parse(raw); + const raw = await getRawFile(fileName, repoName, branchOrTag); + return parseJson(raw, fileName); } // Initialize Bitbucket Server by getting base branch diff --git a/lib/modules/platform/bitbucket/index.ts b/lib/modules/platform/bitbucket/index.ts index 96a23aa49c3484..b1b918ee20ed12 100644 --- a/lib/modules/platform/bitbucket/index.ts +++ b/lib/modules/platform/bitbucket/index.ts @@ -1,9 +1,9 @@ import URL from 'node:url'; import is from '@sindresorhus/is'; -import JSON5 from 'json5'; import { REPOSITORY_NOT_FOUND } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import type { BranchStatus } from '../../../types'; +import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import * as hostRules from '../../../util/host-rules'; import { BitbucketHttp, setBaseUrl } from '../../../util/http/bitbucket'; @@ -158,8 +158,8 @@ export async function getJsonFile( branchOrTag?: string ): Promise { // TODO #22198 - const raw = (await getRawFile(fileName, repoName, branchOrTag)) as string; - return JSON5.parse(raw); + const raw = await getRawFile(fileName, repoName, branchOrTag); + return parseJson(raw, fileName); } // Initialize bitbucket by getting base branch and SHA diff --git a/lib/modules/platform/codecommit/index.ts b/lib/modules/platform/codecommit/index.ts index cc6b69d9d7ac20..7b7ddcc449396e 100644 --- a/lib/modules/platform/codecommit/index.ts +++ b/lib/modules/platform/codecommit/index.ts @@ -4,8 +4,6 @@ import { ListRepositoriesOutput, PullRequestStatusEnum, } from '@aws-sdk/client-codecommit'; -import JSON5 from 'json5'; - import { PLATFORM_BAD_CREDENTIALS, REPOSITORY_EMPTY, @@ -14,6 +12,7 @@ import { import { logger } from '../../../logger'; import type { BranchStatus, PrState } from '../../../types'; import { coerceArray } from '../../../util/array'; +import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import { regEx } from '../../../util/regex'; import { sanitize } from '../../../util/sanitize'; @@ -329,7 +328,7 @@ export async function getJsonFile( branchOrTag?: string ): Promise { const raw = await getRawFile(fileName, repoName, branchOrTag); - return raw ? JSON5.parse(raw) : null; + return parseJson(raw, fileName); } export async function getRawFile( diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 79e80177f13166..89729341863b10 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -1,5 +1,4 @@ import is from '@sindresorhus/is'; -import JSON5 from 'json5'; import semver from 'semver'; import { REPOSITORY_ACCESS_FORBIDDEN, @@ -11,6 +10,7 @@ import { } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import type { BranchStatus } from '../../../types'; +import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import { setBaseUrl } from '../../../util/http/gitea'; import { sanitize } from '../../../util/sanitize'; @@ -256,8 +256,8 @@ const platform: Platform = { branchOrTag?: string ): Promise { // TODO #22198 - const raw = (await platform.getRawFile(fileName, repoName, branchOrTag))!; - return JSON5.parse(raw); + const raw = await platform.getRawFile(fileName, repoName, branchOrTag); + return parseJson(raw, fileName); }, async initRepo({ diff --git a/lib/modules/platform/github/index.spec.ts b/lib/modules/platform/github/index.spec.ts index 1ad9921a51d3a6..196168cbc234bc 100644 --- a/lib/modules/platform/github/index.spec.ts +++ b/lib/modules/platform/github/index.spec.ts @@ -3430,6 +3430,17 @@ describe('modules/platform/github/index', () => { }); describe('getJsonFile()', () => { + it('returns null', async () => { + const scope = httpMock.scope(githubApiHost); + initRepoMock(scope, 'some/repo'); + await github.initRepo({ repository: 'some/repo' }); + scope.get('/repos/some/repo/contents/file.json').reply(200, { + content: '', + }); + const res = await github.getJsonFile('file.json'); + expect(res).toBeNull(); + }); + it('returns file content', async () => { const data = { foo: 'bar' }; const scope = httpMock.scope(githubApiHost); diff --git a/lib/modules/platform/github/index.ts b/lib/modules/platform/github/index.ts index bdc39b5457ca98..0df999996ff009 100644 --- a/lib/modules/platform/github/index.ts +++ b/lib/modules/platform/github/index.ts @@ -1,7 +1,6 @@ import URL from 'node:url'; import { setTimeout } from 'timers/promises'; import is from '@sindresorhus/is'; -import JSON5 from 'json5'; import { DateTime } from 'luxon'; import semver from 'semver'; import { GlobalConfig } from '../../../config/global'; @@ -25,6 +24,7 @@ import type { BranchStatus, VulnerabilityAlert } from '../../../types'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import { isGithubFineGrainedPersonalAccessToken } from '../../../util/check-token'; import { coerceToNull } from '../../../util/coerce'; +import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import { listCommitTree, pushCommitToRenovateRef } from '../../../util/git'; import type { @@ -331,9 +331,8 @@ export async function getJsonFile( repoName?: string, branchOrTag?: string ): Promise { - // TODO #22198 - const raw = (await getRawFile(fileName, repoName, branchOrTag)) as string; - return JSON5.parse(raw); + const raw = await getRawFile(fileName, repoName, branchOrTag); + return parseJson(raw, fileName); } export async function listForks( diff --git a/lib/modules/platform/gitlab/index.spec.ts b/lib/modules/platform/gitlab/index.spec.ts index 5f3afefd291ff2..fadc5178b08aa3 100644 --- a/lib/modules/platform/gitlab/index.spec.ts +++ b/lib/modules/platform/gitlab/index.spec.ts @@ -2641,6 +2641,19 @@ These updates have all been created already. Click a checkbox below to force a r }); describe('getJsonFile()', () => { + it('returns null', async () => { + const scope = await initRepo(); + scope + .get( + '/api/v4/projects/some%2Frepo/repository/files/dir%2Ffile.json?ref=HEAD' + ) + .reply(200, { + content: '', + }); + const res = await gitlab.getJsonFile('dir/file.json'); + expect(res).toBeNull(); + }); + it('returns file content', async () => { const data = { foo: 'bar' }; const scope = await initRepo(); diff --git a/lib/modules/platform/gitlab/index.ts b/lib/modules/platform/gitlab/index.ts index 27567a0f00ca24..58e8c6bf9ffd07 100644 --- a/lib/modules/platform/gitlab/index.ts +++ b/lib/modules/platform/gitlab/index.ts @@ -1,7 +1,6 @@ import URL from 'node:url'; import { setTimeout } from 'timers/promises'; import is from '@sindresorhus/is'; -import JSON5 from 'json5'; import pMap from 'p-map'; import semver from 'semver'; import { @@ -19,6 +18,7 @@ import { import { logger } from '../../../logger'; import type { BranchStatus } from '../../../types'; import { coerceArray } from '../../../util/array'; +import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import * as hostRules from '../../../util/host-rules'; import { setBaseUrl } from '../../../util/http/gitlab'; @@ -231,9 +231,8 @@ export async function getJsonFile( repoName?: string, branchOrTag?: string ): Promise { - // TODO #22198 - const raw = (await getRawFile(fileName, repoName, branchOrTag)) as string; - return JSON5.parse(raw); + const raw = await getRawFile(fileName, repoName, branchOrTag); + return parseJson(raw, fileName); } function getRepoUrl( diff --git a/lib/util/common.spec.ts b/lib/util/common.spec.ts index 8505ad27447467..0b053aff055444 100644 --- a/lib/util/common.spec.ts +++ b/lib/util/common.spec.ts @@ -1,6 +1,33 @@ -import { detectPlatform } from './common'; +import { logger } from '../../test/util'; +import { detectPlatform, parseJson } from './common'; import * as hostRules from './host-rules'; +const validJsonString = ` +{ + "name": "John Doe", + "age": 30, + "city": "New York" +} +`; +const invalidJsonString = ` +{ + "name": "Alice", + "age": 25, + "city": "Los Angeles", + "hobbies": ["Reading", "Running", "Cooking"] + "isStudent": true +} +`; +const onlyJson5parsableString = ` +{ + name: "Bob", + age: 35, + city: 'San Francisco', + // This is a comment + "isMarried": false, +} +`; + describe('util/common', () => { beforeEach(() => hostRules.clear()); @@ -60,4 +87,35 @@ describe('util/common', () => { expect(detectPlatform('https://f.example.com/chalk/chalk')).toBeNull(); }); }); + + describe('parseJson', () => { + it('returns null', () => { + expect(parseJson(null, 'renovate.json')).toBeNull(); + }); + + it('returns parsed json', () => { + expect(parseJson(validJsonString, 'renovate.json')).toEqual({ + name: 'John Doe', + age: 30, + city: 'New York', + }); + }); + + it('throws error for invalid json', () => { + expect(() => parseJson(invalidJsonString, 'renovate.json')).toThrow(); + }); + + it('catches and warns if content parsing faield with JSON.parse but not with JSON5.parse', () => { + expect(parseJson(onlyJson5parsableString, 'renovate.json')).toEqual({ + name: 'Bob', + age: 35, + city: 'San Francisco', + isMarried: false, + }); + expect(logger.logger.warn).toHaveBeenCalledWith( + { context: 'renovate.json' }, + 'File contents are invalid JSON but parse using JSON5. Support for this will be removed in a future release so please change to a support .json5 file name or ensure correct JSON syntax.' + ); + }); + }); }); diff --git a/lib/util/common.ts b/lib/util/common.ts index df7db5ab4ab45f..d7348a763c2673 100644 --- a/lib/util/common.ts +++ b/lib/util/common.ts @@ -1,9 +1,11 @@ +import JSON5 from 'json5'; import { BITBUCKET_API_USING_HOST_TYPES, GITEA_API_USING_HOST_TYPES, GITHUB_API_USING_HOST_TYPES, GITLAB_API_USING_HOST_TYPES, } from '../constants'; +import { logger } from '../logger'; import * as hostRules from './host-rules'; import { parseUrl } from './url'; @@ -59,3 +61,32 @@ export function detectPlatform( return null; } + +export function parseJson(content: string | null, filename: string): unknown { + if (!content) { + return null; + } + + return filename.endsWith('.json5') + ? JSON5.parse(content) + : parseJsonWithFallback(content, filename); +} + +export function parseJsonWithFallback( + content: string, + context: string +): unknown { + let parsedJson: unknown; + + try { + parsedJson = JSON.parse(content); + } catch (err) { + parsedJson = JSON5.parse(content); + logger.warn( + { context }, + 'File contents are invalid JSON but parse using JSON5. Support for this will be removed in a future release so please change to a support .json5 file name or ensure correct JSON syntax.' + ); + } + + return parsedJson; +} diff --git a/lib/workers/global/config/parse/file.ts b/lib/workers/global/config/parse/file.ts index 4ccdca5f392bb6..679fdbb0c22f1a 100644 --- a/lib/workers/global/config/parse/file.ts +++ b/lib/workers/global/config/parse/file.ts @@ -6,6 +6,7 @@ import upath from 'upath'; import { migrateConfig } from '../../../../config/migration'; import type { AllConfig, RenovateConfig } from '../../../../config/types'; import { logger } from '../../../../logger'; +import { parseJson } from '../../../../util/common'; import { readSystemFile } from '../../../../util/fs'; export async function getParsedContent(file: string): Promise { @@ -20,7 +21,10 @@ export async function getParsedContent(file: string): Promise { }) as RenovateConfig; case '.json5': case '.json': - return JSON5.parse(await readSystemFile(file, 'utf8')); + return parseJson( + await readSystemFile(file, 'utf8'), + file + ) as RenovateConfig; case '.js': { const tmpConfig = await import(file); let config = tmpConfig.default diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index 7c4b2b0b81ac27..c8a33f23946f40 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -20,6 +20,7 @@ import { platform } from '../../../modules/platform'; import { scm } from '../../../modules/platform/scm'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import { getCache } from '../../../util/cache/repository'; +import { parseJson } from '../../../util/common'; import { readLocalFile } from '../../../util/fs'; import * as hostRules from '../../../util/host-rules'; import * as queue from '../../../util/http/queue'; @@ -69,7 +70,7 @@ export async function detectRepoFileConfig(): Promise { configFileRaw = null; } if (configFileRaw) { - let configFileParsed = JSON5.parse(configFileRaw); + let configFileParsed = parseJson(configFileRaw, configFileName) as any; if (configFileName !== 'package.json') { return { configFileName, configFileRaw, configFileParsed }; } @@ -177,7 +178,7 @@ export async function detectRepoFileConfig(): Promise { }; } try { - configFileParsed = JSON5.parse(configFileRaw); + configFileParsed = parseJson(configFileRaw, configFileName); } catch (err) /* istanbul ignore next */ { logger.debug( { renovateConfig: configFileRaw },