From eac6a9ce67dbae5ec0818bf133c69e9ea2adc37c Mon Sep 17 00:00:00 2001 From: Blaine Bublitz Date: Thu, 26 Jan 2023 13:51:39 -0700 Subject: [PATCH 1/2] feat: Add generic TOML updater --- __snapshots__/generic-toml.js | 69 +++++++++++++++++++++ src/manifest.ts | 9 ++- src/strategies/base.ts | 8 +++ src/updaters/generic-toml.ts | 65 ++++++++++++++++++++ src/util/toml-edit.ts | 7 ++- test/strategies/base.ts | 18 ++++++ test/updaters/fixtures/toml/invalid.txt | 1 + test/updaters/generic-toml.ts | 80 +++++++++++++++++++++++++ 8 files changed, 254 insertions(+), 3 deletions(-) create mode 100644 __snapshots__/generic-toml.js create mode 100644 src/updaters/generic-toml.ts create mode 100644 test/updaters/fixtures/toml/invalid.txt create mode 100644 test/updaters/generic-toml.ts diff --git a/__snapshots__/generic-toml.js b/__snapshots__/generic-toml.js new file mode 100644 index 000000000..85d9d09bc --- /dev/null +++ b/__snapshots__/generic-toml.js @@ -0,0 +1,69 @@ +exports['GenericToml updateContent updates deep entry in toml 1'] = ` +[package] +name = "rust-test-repo" +version = "12.0.0" + +# To learn about other keys, check out Cargo's documentation + +[dependencies] +normal-dep = "1.2.3" + +[dev-dependencies] +dev-dep = { version = "2.3.4" } +dev-dep-2 = { path = "../dev-dep-2" } + +[build-dependencies] +# this is using a private registry +build-dep = { version = "1.2.3", registry = "private", path = ".." } # trailing comment + +[target.'cfg(windows)'.dev-dependencies] +windows-dep = { version = "1.2.3", registry = "private", path = ".." } + +[target.'cfg(unix)'.dependencies] +unix-dep = { version = "1.2.3", registry = "private", path = ".." } + +[target.'cfg(target_arch = "x86")'.dependencies] +x86-dep = { version = "1.2.3", registry = "private", path = ".." } + +[target.'cfg(target_arch = "x86_64")'.dependencies] +x86-64-dep = { version = "1.2.3", registry = "private", path = ".." } + +[target.'cfg(foobar)'.dependencies] +foobar-dep = "1.2.3" + +` + +exports['GenericToml updateContent updates matching entry 1'] = ` +[package] +name = "rust-test-repo" +version = "2.3.4" + +# To learn about other keys, check out Cargo's documentation + +[dependencies] +normal-dep = "1.2.3" + +[dev-dependencies] +dev-dep = { version = "1.2.3" } +dev-dep-2 = { path = "../dev-dep-2" } + +[build-dependencies] +# this is using a private registry +build-dep = { version = "1.2.3", registry = "private", path = ".." } # trailing comment + +[target.'cfg(windows)'.dev-dependencies] +windows-dep = { version = "1.2.3", registry = "private", path = ".." } + +[target.'cfg(unix)'.dependencies] +unix-dep = { version = "1.2.3", registry = "private", path = ".." } + +[target.'cfg(target_arch = "x86")'.dependencies] +x86-dep = { version = "1.2.3", registry = "private", path = ".." } + +[target.'cfg(target_arch = "x86_64")'.dependencies] +x86-64-dep = { version = "1.2.3", registry = "private", path = ".." } + +[target.'cfg(foobar)'.dependencies] +foobar-dep = "1.2.3" + +` diff --git a/src/manifest.ts b/src/manifest.ts index 9c53976ed..4d551a3a8 100644 --- a/src/manifest.ts +++ b/src/manifest.ts @@ -70,12 +70,19 @@ type ExtraPomFile = { path: string; glob?: boolean; }; +type ExtraTomlFile = { + type: 'toml'; + path: string; + jsonpath: string; + glob?: boolean; +}; export type ExtraFile = | string | ExtraJsonFile | ExtraYamlFile | ExtraXmlFile - | ExtraPomFile; + | ExtraPomFile + | ExtraTomlFile; /** * These are configurations provided to each strategy per-path. */ diff --git a/src/strategies/base.ts b/src/strategies/base.ts index 1fa3bc2fc..970ac14a4 100644 --- a/src/strategies/base.ts +++ b/src/strategies/base.ts @@ -41,6 +41,7 @@ import {GenericJson} from '../updaters/generic-json'; import {GenericXml} from '../updaters/generic-xml'; import {PomXml} from '../updaters/java/pom-xml'; import {GenericYaml} from '../updaters/generic-yaml'; +import {GenericToml} from '../updaters/generic-toml'; const DEFAULT_CHANGELOG_PATH = 'CHANGELOG.md'; @@ -398,6 +399,13 @@ export abstract class BaseStrategy implements Strategy { updater: new GenericYaml(extraFile.jsonpath, version), }); break; + case 'toml': + extraFileUpdates.push({ + path: this.addPath(path), + createIfMissing: false, + updater: new GenericToml(extraFile.jsonpath, version), + }); + break; case 'xml': extraFileUpdates.push({ path: this.addPath(path), diff --git a/src/updaters/generic-toml.ts b/src/updaters/generic-toml.ts new file mode 100644 index 000000000..eb94a9af6 --- /dev/null +++ b/src/updaters/generic-toml.ts @@ -0,0 +1,65 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {Updater} from '../update'; +import {Version} from '../version'; +import * as jp from 'jsonpath'; +import {parseWith, replaceTomlValue} from '../util/toml-edit'; +import * as toml from '@iarna/toml'; +import {logger as defaultLogger, Logger} from '../util/logger'; + +/** + * Updates TOML document according to given JSONPath. + * + * Note that used parser does reformat the document and removes all comments, + * and converts everything to pure TOML. + * If you want to retain formatting, use generic updater with comment hints. + */ +export class GenericToml implements Updater { + readonly jsonpath: string; + readonly version: Version; + + constructor(jsonpath: string, version: Version) { + this.jsonpath = jsonpath; + this.version = version; + } + /** + * Given initial file contents, return updated contents. + * @param {string} content The initial content + * @returns {string} The updated content + */ + updateContent(content: string, logger: Logger = defaultLogger): string { + let data: toml.JsonMap; + try { + data = parseWith(content); + } catch (e) { + logger.warn('Invalid toml, cannot be parsed', e); + return content; + } + + const paths = jp.paths(data, this.jsonpath); + if (!paths || paths.length === 0) { + logger.warn(`No entries modified in ${this.jsonpath}`); + return content; + } + + let processed = content; + paths.forEach(path => { + if (path[0] === '$') path = path.slice(1); + processed = replaceTomlValue(processed, path, this.version.toString()); + }); + + return processed; + } +} diff --git a/src/util/toml-edit.ts b/src/util/toml-edit.ts index 4637d97df..5975c885e 100644 --- a/src/util/toml-edit.ts +++ b/src/util/toml-edit.ts @@ -86,7 +86,10 @@ class TaggedTOMLParser extends TOMLParser { * @param input A string * @param parserType The TOML parser to use (might be custom) */ -function parseWith(input: string, parserType: typeof TOMLParser): JsonMap { +export function parseWith( + input: string, + parserType: typeof TOMLParser = TaggedTOMLParser +): JsonMap { const parser = new parserType(); parser.parse(input); return parser.finish(); @@ -114,7 +117,7 @@ function isTaggedValue(x: unknown): x is TaggedValue { */ export function replaceTomlValue( input: string, - path: string[], + path: (string | number)[], newValue: string ) { // our pointer into the object "tree", initially points to the root. diff --git a/test/strategies/base.ts b/test/strategies/base.ts index 16a1980b5..54a1b39e6 100644 --- a/test/strategies/base.ts +++ b/test/strategies/base.ts @@ -30,6 +30,7 @@ import {Generic} from '../../src/updaters/generic'; import {GenericXml} from '../../src/updaters/generic-xml'; import {PomXml} from '../../src/updaters/java/pom-xml'; import {GenericYaml} from '../../src/updaters/generic-yaml'; +import {GenericToml} from '../../src/updaters/generic-toml'; const sandbox = sinon.createSandbox(); @@ -145,6 +146,23 @@ describe('Strategy', () => { assertHasUpdate(updates!, '0', Generic); assertHasUpdate(updates!, '3.yaml', GenericYaml); }); + it('updates extra TOML files', async () => { + const strategy = new TestStrategy({ + targetBranch: 'main', + github, + component: 'google-cloud-automl', + extraFiles: ['0', {type: 'toml', path: '/3.toml', jsonpath: '$.foo'}], + }); + const pullRequest = await strategy.buildReleasePullRequest( + buildMockConventionalCommit('fix: a bugfix'), + undefined + ); + expect(pullRequest).to.exist; + const updates = pullRequest?.updates; + expect(updates).to.be.an('array'); + assertHasUpdate(updates!, '0', Generic); + assertHasUpdate(updates!, '3.toml', GenericToml); + }); it('updates extra Xml files', async () => { const strategy = new TestStrategy({ targetBranch: 'main', diff --git a/test/updaters/fixtures/toml/invalid.txt b/test/updaters/fixtures/toml/invalid.txt new file mode 100644 index 000000000..1acd2d2c2 --- /dev/null +++ b/test/updaters/fixtures/toml/invalid.txt @@ -0,0 +1 @@ +invalid = diff --git a/test/updaters/generic-toml.ts b/test/updaters/generic-toml.ts new file mode 100644 index 000000000..389bd0b83 --- /dev/null +++ b/test/updaters/generic-toml.ts @@ -0,0 +1,80 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {readFileSync} from 'fs'; +import {resolve} from 'path'; +import * as snapshot from 'snap-shot-it'; +import {describe, it} from 'mocha'; +import {Version} from '../../src/version'; +import {expect, assert} from 'chai'; +import {GenericToml} from '../../src/updaters/generic-toml'; + +const fixturesPath = './test/updaters/fixtures'; + +describe('GenericToml', () => { + describe('updateContent', () => { + it('updates matching entry', async () => { + const oldContent = readFileSync( + resolve(fixturesPath, './Cargo.toml'), + 'utf8' + ).replace(/\r\n/g, '\n'); + const updater = new GenericToml( + '$.package.version', + Version.parse('v2.3.4') + ); + const newContent = updater.updateContent(oldContent); + snapshot(newContent); + }); + it('updates deep entry in toml', async () => { + const oldContent = readFileSync( + resolve(fixturesPath, './Cargo.toml'), + 'utf8' + ).replace(/\r\n/g, '\n'); + const updater = new GenericToml( + "$['dev-dependencies']..version", + Version.parse('v2.3.4') + ); + const newContent = updater.updateContent(oldContent); + snapshot(newContent); + }); + it('ignores non-matching entry', async () => { + const oldContent = readFileSync( + resolve(fixturesPath, './Cargo.toml'), + 'utf8' + ).replace(/\r\n/g, '\n'); + const updater = new GenericToml('$.nonExistent', Version.parse('v2.3.4')); + const newContent = updater.updateContent(oldContent); + expect(newContent).to.eql(oldContent); + }); + it('warns on invalid jsonpath', async () => { + const oldContent = readFileSync( + resolve(fixturesPath, './Cargo.toml'), + 'utf8' + ).replace(/\r\n/g, '\n'); + const updater = new GenericToml('bad jsonpath', Version.parse('v2.3.4')); + assert.throws(() => { + updater.updateContent(oldContent); + }); + }); + it('ignores invalid file', async () => { + const oldContent = readFileSync( + resolve(fixturesPath, './toml/invalid.txt'), + 'utf8' + ).replace(/\r\n/g, '\n'); + const updater = new GenericToml('$.boo', Version.parse('v2.3.4')); + const newContent = updater.updateContent(oldContent); + expect(newContent).to.eql(oldContent); + }); + }); +}); From f43d68c79e191674883380d1b96f55aa3b17a930 Mon Sep 17 00:00:00 2001 From: Blaine Bublitz Date: Thu, 26 Jan 2023 14:12:28 -0700 Subject: [PATCH 2/2] chore(docs): Add generic YAML & TOML updaters --- docs/customizing.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/customizing.md b/docs/customizing.md index 7cbcacef4..8c25791d6 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -201,3 +201,41 @@ configuration. ] } ``` + +## Updating arbitrary YAML files + +For most release strategies, you can provide additional files to update +using the [GenericYaml](/src/updaters/generic-yaml.ts) updater. You can +specify a configuration object in the `extra-files` option in the manifest +configuration. + +```json +{ + "extra-files": [ + { + "type": "yaml", + "path": "path/to/file.yaml", + "jsonpath": "$.json.path.to.field" + } + ] +} +``` + +## Updating arbitrary TOML files + +For most release strategies, you can provide additional files to update +using the [GenericToml](/src/updaters/generic-toml.ts) updater. You can +specify a configuration object in the `extra-files` option in the manifest +configuration. + +```json +{ + "extra-files": [ + { + "type": "toml", + "path": "path/to/file.toml", + "jsonpath": "$.json.path.to.field" + } + ] +} +```