From 6c12537eff1c2265d0267481dbf1d7302fcb004a Mon Sep 17 00:00:00 2001 From: mshanemc Date: Sun, 23 Oct 2022 18:15:38 -0500 Subject: [PATCH 01/41] refactor: pre-wi-cleanup --- src/client/metadataApiDeploy.ts | 2 +- src/convert/convertContext.ts | 13 ++----- src/convert/streams.ts | 11 +++--- .../defaultMetadataTransformer.ts | 38 +++++++++---------- .../staticResourceMetadataTransformer.ts | 18 ++++----- src/resolve/sourceComponent.ts | 6 +-- 6 files changed, 36 insertions(+), 52 deletions(-) diff --git a/src/client/metadataApiDeploy.ts b/src/client/metadataApiDeploy.ts index 4dbf5c5930..e5d50d89f3 100644 --- a/src/client/metadataApiDeploy.ts +++ b/src/client/metadataApiDeploy.ts @@ -387,7 +387,7 @@ export class MetadataApiDeploy extends MetadataTransfer { - const writeInfos: WriteInfo[] = []; - - if (component.content) { - for (const source of component.walkContent()) { - writeInfos.push({ - source: component.tree.stream(source), - output: getContentSourceDestination(source, targetFormat, component, mergeWith), - }); - } - } - - if (component.xml) { - writeInfos.push({ - source: component.tree.stream(component.xml), - output: getXmlDestination(targetFormat, component, mergeWith), - }); - } - - return writeInfos; -}; +): WriteInfo[] => + component + .walkContent() + .map((path) => ({ + source: component.tree.stream(path), + output: getContentSourceDestination(path, targetFormat, component, mergeWith), + })) + .concat( + component.xml + ? [ + { + source: component.tree.stream(component.xml), + output: getXmlDestination(targetFormat, component, mergeWith), + }, + ] + : [] + ); // assumes component has content const getContentSourceDestination = ( diff --git a/src/convert/transformers/staticResourceMetadataTransformer.ts b/src/convert/transformers/staticResourceMetadataTransformer.ts index bf62da992c..49a8e9c121 100644 --- a/src/convert/transformers/staticResourceMetadataTransformer.ts +++ b/src/convert/transformers/staticResourceMetadataTransformer.ts @@ -6,7 +6,7 @@ */ import { basename, dirname, isAbsolute, join } from 'path'; import { Readable } from 'stream'; -import { create as createArchive } from 'archiver'; +import { create as createArchive, Archiver } from 'archiver'; import { getExtension } from 'mime'; import { Open } from 'unzipper'; import { JsonMap } from '@salesforce/ts-types'; @@ -38,22 +38,20 @@ export class StaticResourceMetadataTransformer extends BaseMetadataTransformer { public async toMetadataFormat(component: SourceComponent): Promise { const { content, type, xml } = component; - let contentSource: Readable; - - if (await componentIsExpandedArchive(component)) { + const zipIt = async (): Promise => { // toolbelt was using level 9 for static resources, so we'll do the same. // Otherwise, you'll see errors like https://github.com/forcedotcom/cli/issues/1098 const zip = createArchive('zip', { zlib: { level: 9 } }); zip.directory(content, false); - void zip.finalize(); - contentSource = zip; - } else { - contentSource = component.tree.stream(content); - } + await zip.finalize(); + return zip; + }; + + // const contentSource = (; return [ { - source: contentSource, + source: (await componentIsExpandedArchive(component)) ? await zipIt() : component.tree.stream(content), output: join(type.directoryName, `${baseName(content)}.${type.suffix}`), }, { diff --git a/src/resolve/sourceComponent.ts b/src/resolve/sourceComponent.ts index 90f100b754..ff7ec3cc7f 100644 --- a/src/resolve/sourceComponent.ts +++ b/src/resolve/sourceComponent.ts @@ -179,11 +179,7 @@ export class SourceComponent implements MetadataComponent { * @return ForceIgnore */ public getForceIgnore(): ForceIgnore { - if (this.forceIgnore) { - return this.forceIgnore; - } else { - return ForceIgnore.findAndCreate(this.content); - } + return this.forceIgnore ?? ForceIgnore.findAndCreate(this.content); } /** From 94669e3eeb4427cfc527b3e5ccbfe505a7929db0 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Sun, 23 Oct 2022 18:34:42 -0500 Subject: [PATCH 02/41] refactor: constructor shorthand --- src/convert/streams.ts | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/convert/streams.ts b/src/convert/streams.ts index 1de8736477..bfae713bcb 100644 --- a/src/convert/streams.ts +++ b/src/convert/streams.ts @@ -65,22 +65,16 @@ export class ComponentReader extends Readable { export class ComponentConverter extends Transform { public readonly context = new ConvertContext(); - private targetFormat: SfdxFileFormat; - private mergeSet: ComponentSet; private transformerFactory: MetadataTransformerFactory; - private defaultDirectory: string; public constructor( - targetFormat: SfdxFileFormat, + private targetFormat: SfdxFileFormat, registry: RegistryAccess, - mergeSet?: ComponentSet, - defaultDirectory?: string + private mergeSet?: ComponentSet, + private defaultDirectory?: string ) { super({ objectMode: true }); - this.targetFormat = targetFormat; - this.mergeSet = mergeSet; this.transformerFactory = new MetadataTransformerFactory(registry, this.context); - this.defaultDirectory = defaultDirectory; } public async _transform( @@ -154,12 +148,10 @@ export abstract class ComponentWriter extends Writable { export class StandardWriter extends ComponentWriter { public converted: SourceComponent[] = []; - private resolver: MetadataResolver; private logger: Logger; - public constructor(rootDestination: SourcePath, resolver = new MetadataResolver()) { + public constructor(rootDestination: SourcePath, private resolver = new MetadataResolver()) { super(rootDestination); - this.resolver = resolver; this.logger = Logger.childFromRoot(this.constructor.name); } @@ -288,11 +280,8 @@ export class ZipWriter extends ComponentWriter { * even though it's not beneficial in the typical way a stream is. */ export class JsToXml extends Readable { - private xmlObject: JsonMap; - - public constructor(xmlObject: JsonMap) { + public constructor(private xmlObject: JsonMap) { super(); - this.xmlObject = xmlObject; } public _read(): void { From 1ae12ec130c7619f9d4feb948f60c8576b09ee21 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Sun, 23 Oct 2022 18:48:26 -0500 Subject: [PATCH 03/41] test: record perf [ci skip] --- .../eda.json | 18 ++++++++++++++++++ .../lotsOfClasses.json | 18 ++++++++++++++++++ .../lotsOfClassesOneDir.json | 18 ++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json create mode 100644 test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json create mode 100644 test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json new file mode 100644 index 0000000000..78e1834713 --- /dev/null +++ b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json @@ -0,0 +1,18 @@ +[ + { + "name": "componentSetCreate", + "duration": 518.6464899927378 + }, + { + "name": "sourceToMdapi", + "duration": 5463.311020001769 + }, + { + "name": "sourceToZip", + "duration": 4546.940956994891 + }, + { + "name": "mdapiToSource", + "duration": 4583.583284005523 + } +] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json new file mode 100644 index 0000000000..20c4d3d204 --- /dev/null +++ b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json @@ -0,0 +1,18 @@ +[ + { + "name": "componentSetCreate", + "duration": 456.29641300439835 + }, + { + "name": "sourceToMdapi", + "duration": 9177.259455993772 + }, + { + "name": "sourceToZip", + "duration": 7334.71290999651 + }, + { + "name": "mdapiToSource", + "duration": 6288.891626000404 + } +] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json new file mode 100644 index 0000000000..f0de3b6aee --- /dev/null +++ b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json @@ -0,0 +1,18 @@ +[ + { + "name": "componentSetCreate", + "duration": 736.6738519966602 + }, + { + "name": "sourceToMdapi", + "duration": 13345.731360003352 + }, + { + "name": "sourceToZip", + "duration": 14143.896373003721 + }, + { + "name": "mdapiToSource", + "duration": 10690.781048998237 + } +] \ No newline at end of file From 109ce54d5d63dbbdffee2f8dc6f75d527a7c4781 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Sun, 23 Oct 2022 18:53:55 -0500 Subject: [PATCH 04/41] test: set perf baseline for mac --- .../eda.json | 18 ------------------ .../lotsOfClasses.json | 18 ------------------ .../lotsOfClassesOneDir.json | 18 ------------------ .../eda.json | 10 +++++----- .../lotsOfClasses.json | 10 +++++----- .../lotsOfClassesOneDir.json | 10 +++++----- 6 files changed, 15 insertions(+), 69 deletions(-) delete mode 100644 test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/eda.json delete mode 100644 test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/lotsOfClasses.json delete mode 100644 test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/lotsOfClassesOneDir.json diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/eda.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/eda.json deleted file mode 100644 index 2527931b59..0000000000 --- a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/eda.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "name": "componentSetCreate", - "duration": 492.98764300346375 - }, - { - "name": "sourceToMdapi", - "duration": 5277.533275008202 - }, - { - "name": "sourceToZip", - "duration": 3976.5643639862537 - }, - { - "name": "mdapiToSource", - "duration": 7064.533327996731 - } -] diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/lotsOfClasses.json deleted file mode 100644 index 5a2e458616..0000000000 --- a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/lotsOfClasses.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "name": "componentSetCreate", - "duration": 388.5986630022526 - }, - { - "name": "sourceToMdapi", - "duration": 8307.766464978456 - }, - { - "name": "sourceToZip", - "duration": 6753.930882006884 - }, - { - "name": "mdapiToSource", - "duration": 7472.047949999571 - } -] diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/lotsOfClassesOneDir.json deleted file mode 100644 index 7da90fd399..0000000000 --- a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/lotsOfClassesOneDir.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "name": "componentSetCreate", - "duration": 648.0908780097961 - }, - { - "name": "sourceToMdapi", - "duration": 12539.2236790061 - }, - { - "name": "sourceToZip", - "duration": 8442.249673008919 - }, - { - "name": "mdapiToSource", - "duration": 12450.891524016857 - } -] diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json index 78e1834713..2527931b59 100644 --- a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json +++ b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 518.6464899927378 + "duration": 492.98764300346375 }, { "name": "sourceToMdapi", - "duration": 5463.311020001769 + "duration": 5277.533275008202 }, { "name": "sourceToZip", - "duration": 4546.940956994891 + "duration": 3976.5643639862537 }, { "name": "mdapiToSource", - "duration": 4583.583284005523 + "duration": 7064.533327996731 } -] \ No newline at end of file +] diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json index 20c4d3d204..5a2e458616 100644 --- a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 456.29641300439835 + "duration": 388.5986630022526 }, { "name": "sourceToMdapi", - "duration": 9177.259455993772 + "duration": 8307.766464978456 }, { "name": "sourceToZip", - "duration": 7334.71290999651 + "duration": 6753.930882006884 }, { "name": "mdapiToSource", - "duration": 6288.891626000404 + "duration": 7472.047949999571 } -] \ No newline at end of file +] diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json index f0de3b6aee..7da90fd399 100644 --- a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 736.6738519966602 + "duration": 648.0908780097961 }, { "name": "sourceToMdapi", - "duration": 13345.731360003352 + "duration": 12539.2236790061 }, { "name": "sourceToZip", - "duration": 14143.896373003721 + "duration": 8442.249673008919 }, { "name": "mdapiToSource", - "duration": 10690.781048998237 + "duration": 12450.891524016857 } -] \ No newline at end of file +] From 3f78f989b597050e0a2e25c1a3a8c1335a8a8adf Mon Sep 17 00:00:00 2001 From: mshanemc Date: Sun, 23 Oct 2022 18:56:34 -0500 Subject: [PATCH 05/41] test: record perf [ci skip] --- HANDBOOK.md | 9 +------ src/convert/metadataConverter.ts | 5 ++-- src/convert/streams.ts | 26 ------------------- test/convert/metadataConverter.test.ts | 6 ----- test/convert/streams.test.ts | 12 --------- .../eda.json | 10 +++---- .../lotsOfClasses.json | 10 +++---- .../lotsOfClassesOneDir.json | 10 +++---- 8 files changed, 19 insertions(+), 69 deletions(-) diff --git a/HANDBOOK.md b/HANDBOOK.md index 75039d18e0..d65377d0d6 100644 --- a/HANDBOOK.md +++ b/HANDBOOK.md @@ -20,7 +20,6 @@ - [Overview](#overview-2) - [Converting metadata](#converting-metadata) - [The conversion pipeline](#the-conversion-pipeline) - - [ComponentReader](#componentreader) - [ComponentConverter](#componentconverter) - [ComponentWriter](#componentwriter) - [ConvertContext](#convertcontext) @@ -214,7 +213,7 @@ A `TreeContainer` is an encapsulation of a file system that enables I/O against Clients can implement new tree containers by extending the `TreeContainer` base class and expanding functionality. Not all methods of a tree container have to be implemented, but an error will be thrown if the container is being used in a context that requires particular methods. -πŸ’‘*The author, Brian, demonstrated the extensibility of tree containers for a side project by creating a* `GitTreeContainer`_. This enabled resolving components against a git object tree, allowing us to perform component diffs between git refs and analyze GitHub projects. See the [SFDX Badge Generator](https://sfdx-badge.herokuapp.com/). This could be expanded into a plugin of some sort._ +πŸ’‘_The author, Brian, demonstrated the extensibility of tree containers for a side project by creating a_ `GitTreeContainer`_. This enabled resolving components against a git object tree, allowing us to perform component diffs between git refs and analyze GitHub projects. See the [SFDX Badge Generator](https://sfdx-badge.herokuapp.com/). This could be expanded into a plugin of some sort._ #### Creating mock components with the VirtualTreeContainer @@ -315,12 +314,6 @@ const converter = new MetadataConverter(); When `convert` is called, the method prepares the inputs for setting up the conversion pipeline. The pipeline consists of chaining three custom NodeJS stream, one for each stage of the copy operation. To more deeply understand what is happening in the conversion process, it’s recommended to familiarize yourself with streaming concepts and the NodeJS API. See [Stream NodeJS documentation](https://nodejs.org/api/stream.html) and [Understanding Streams in NodeJS](https://nodesource.com/blog/understanding-streams-in-nodejs/). -#### ComponentReader - -The reader is fairly simple, it takes a collection of source components and implements the stream API to push them out one-by-one. - -🧽 _When this aspect of the library was first written,_ `Readable.from(iterable)` _was not yet available. This simple API could probably replace the_ `ComponentReader`_._ - #### ComponentConverter Here is where file transformation is done, but without being written to the destination yet. Similar to how source resolution uses adapters to determine how to construct components for a type (see [The resolver constructs components based…](#resolving-from-metadata-files)), conversion uses `MetadataTransformer` implementations to describe the transformations. As you might guess, types are assigned a transformer, if they need one, in their metadata registry definition, otherwise the default one is used. Each transformer implements a `toSourceFormat` and a `toMetadataFormat` method, which are called by the `ComponentConverter` based on what the target format is. The methods will return a collection of `WriteInfo` objects, which as we’ve been touching on are β€œdescriptions” of how to write a given file. diff --git a/src/convert/metadataConverter.ts b/src/convert/metadataConverter.ts index 80e11dbd9a..f8678bc676 100644 --- a/src/convert/metadataConverter.ts +++ b/src/convert/metadataConverter.ts @@ -4,6 +4,7 @@ * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ +import { Readable } from 'stream'; import { dirname, join, normalize } from 'path'; import { Messages, SfError } from '@salesforce/core'; import { promises } from 'graceful-fs'; @@ -12,7 +13,7 @@ import { ensureDirectoryExists } from '../utils/fileSystemHandler'; import { SourcePath } from '../common'; import { ComponentSet, DestructiveChangesType } from '../collections'; import { RegistryAccess } from '../registry'; -import { ComponentConverter, ComponentReader, pipeline, StandardWriter, ZipWriter } from './streams'; +import { ComponentConverter, pipeline, StandardWriter, ZipWriter } from './streams'; import { ConvertOutputConfig, ConvertResult, DirectoryConfig, SfdxFileFormat, ZipConfig } from './types'; Messages.importMessagesDirectory(__dirname); @@ -111,7 +112,7 @@ export class MetadataConverter { } const conversionPipeline = pipeline( - new ComponentReader(components), + Readable.from(components), new ComponentConverter(targetFormat, this.registry, mergeSet, defaultDirectory), writer ); diff --git a/src/convert/streams.ts b/src/convert/streams.ts index bfae713bcb..9358093699 100644 --- a/src/convert/streams.ts +++ b/src/convert/streams.ts @@ -36,32 +36,6 @@ export const stream2buffer = async (stream: Stream): Promise => // eslint-disable-next-line @typescript-eslint/restrict-template-expressions stream.on('error', (err) => reject(`error converting stream - ${err}`)); }); -export class ComponentReader extends Readable { - private iter: Iterator; - - public constructor(components: Iterable) { - super({ objectMode: true }); - this.iter = this.createIterator(components); - } - - public _read(): void { - let next = this.iter.next(); - while (!next.done) { - this.push(next.value); - next = this.iter.next(); - } - this.push(null); - } - - // preserved to isolate from other classes in this file - // componentReader should go away (see note in handbook) - // eslint-disable-next-line class-methods-use-this - private *createIterator(components: Iterable): Iterator { - for (const component of components) { - yield component; - } - } -} export class ComponentConverter extends Transform { public readonly context = new ConvertContext(); diff --git a/test/convert/metadataConverter.test.ts b/test/convert/metadataConverter.test.ts index 66bdece395..28fe06c866 100644 --- a/test/convert/metadataConverter.test.ts +++ b/test/convert/metadataConverter.test.ts @@ -13,7 +13,6 @@ import { assert, expect } from 'chai'; import { TestContext } from '@salesforce/core/lib/testSetup'; import { xmlInFolder } from '../mock'; import * as streams from '../../src/convert/streams'; -import { ComponentReader } from '../../src/convert/streams'; import * as fsUtil from '../../src/utils/fileSystemHandler'; import { COMPONENTS } from '../mock/type-constants/documentFolderConstant'; import { ComponentSet, DestructiveChangesType, MetadataConverter, registry, SourceComponent } from '../../src'; @@ -45,7 +44,6 @@ describe('MetadataConverter', () => { /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ function validatePipelineArgs(pipelineArgs: any[], targetFormat = 'metadata'): void { - expect(pipelineArgs[0] instanceof streams.ComponentReader).to.be.true; expect(pipelineArgs[1] instanceof streams.ComponentConverter).to.be.true; expect(pipelineArgs[1].targetFormat).to.equal(targetFormat); expect(pipelineArgs[2] instanceof streams.ComponentWriter).to.be.true; @@ -462,8 +460,6 @@ describe('MetadataConverter', () => { }); it('should create conversion pipeline with addressable components', async () => { - // @ts-ignore private - const componentReaderSpy = $$.SANDBOX.spy(ComponentReader.prototype, 'createIterator'); components.push({ type: registry.types.customobjecttranslation.children.types.customfieldtranslation, name: 'myFieldTranslation', @@ -479,10 +475,8 @@ describe('MetadataConverter', () => { const pipelineArgs = pipelineStub.firstCall.args; validatePipelineArgs(pipelineArgs, 'source'); - expect(componentReaderSpy.firstCall.args[0].length).to.equal(3); // pop off the CFT that should be filtered off for the assertion components.pop(); - expect(componentReaderSpy.firstCall.args[0]).to.deep.equal(components); expect(pipelineArgs[2].rootDestination).to.equal(defaultDirectory); }); diff --git a/test/convert/streams.test.ts b/test/convert/streams.test.ts index 613abc129c..082d038ec4 100644 --- a/test/convert/streams.test.ts +++ b/test/convert/streams.test.ts @@ -44,18 +44,6 @@ class TestTransformer extends BaseMetadataTransformer { describe('Streams', () => { afterEach(() => env.restore()); - describe('ComponentReader', () => { - it('should read metadata components one at a time', async () => { - const reader = new streams.ComponentReader(COMPONENTS); - let currentIndex = 0; - for await (const component of reader) { - expect(component).to.deep.equal(COMPONENTS[currentIndex]); - currentIndex += 1; - } - expect(currentIndex).to.equal(COMPONENTS.length); - }); - }); - /** * NOTE: tests that call _transform methods must utilize Mocha.done to signal * when a test has finished and to pass on any assertion failures to the test diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json index 2527931b59..1658a7e458 100644 --- a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json +++ b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 492.98764300346375 + "duration": 493.49703399837017 }, { "name": "sourceToMdapi", - "duration": 5277.533275008202 + "duration": 5436.327586993575 }, { "name": "sourceToZip", - "duration": 3976.5643639862537 + "duration": 4556.773661002517 }, { "name": "mdapiToSource", - "duration": 7064.533327996731 + "duration": 4874.957931995392 } -] +] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json index 5a2e458616..a06d81e129 100644 --- a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 388.5986630022526 + "duration": 433.693730995059 }, { "name": "sourceToMdapi", - "duration": 8307.766464978456 + "duration": 8342.872605994344 }, { "name": "sourceToZip", - "duration": 6753.930882006884 + "duration": 7030.860814988613 }, { "name": "mdapiToSource", - "duration": 7472.047949999571 + "duration": 5282.1073610037565 } -] +] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json index 7da90fd399..e535353bc8 100644 --- a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU-2-40GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 648.0908780097961 + "duration": 760.8551979959011 }, { "name": "sourceToMdapi", - "duration": 12539.2236790061 + "duration": 15508.725342988968 }, { "name": "sourceToZip", - "duration": 8442.249673008919 + "duration": 11001.649205997586 }, { "name": "mdapiToSource", - "duration": 12450.891524016857 + "duration": 10145.04680301249 } -] +] \ No newline at end of file From a8eb9d7afe288b541e47501a93fb07dd4cd1c718 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Sun, 23 Oct 2022 19:00:00 -0500 Subject: [PATCH 06/41] chore: auto-update metadata coverage in METADATA_SUPPORT.md --- METADATA_SUPPORT.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/METADATA_SUPPORT.md b/METADATA_SUPPORT.md index 779186fbed..7172c7051e 100644 --- a/METADATA_SUPPORT.md +++ b/METADATA_SUPPORT.md @@ -4,7 +4,7 @@ This list compares metadata types found in Salesforce v56 with the [metadata reg This repository is used by both the Salesforce CLIs and Salesforce's VSCode Extensions. -Currently, there are 480/512 supported metadata types. +Currently, there are 481/512 supported metadata types. For status on any existing gaps, please search or file an issue in the [Salesforce CLI issues only repo](https://github.com/forcedotcom/cli/issues). To contribute a new metadata type, please see the [Contributing Metadata Types to the Registry](./contributing/metadata.md) @@ -73,7 +73,7 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t |BldgEnrgyIntensityCnfg|βœ…|| |BlockchainSettings|βœ…|| |Bot|βœ…|| -|BotBlock|❌|Not supported, but support could be added| +|BotBlock|βœ…|| |BotBlockVersion|❌|Not supported, but support could be added| |BotSettings|βœ…|| |BotTemplate|βœ…|| @@ -532,11 +532,12 @@ v57 introduces the following new types. Here's their current level of support |:---|:---|:---| |ActionableListDefinition|❌|Not supported, but support could be added| |AffinityScoreDefinition|❌|Not supported, but support could be added| -|CampaignTemplateDefinition|❌|Not supported, but support could be added| |ClauseCatgConfiguration|❌|Not supported, but support could be added| |DisclosureDefinition|❌|Not supported, but support could be added| |DisclosureDefinitionVersion|❌|Not supported, but support could be added| |DisclosureType|❌|Not supported, but support could be added| +|EngagementMessagingSettings|βœ…|| +|ExternalClientAppSettings|βœ…|| |ExternalClientApplication|βœ…|| |ExternalDocStorageConfig|❌|Not supported, but support could be added| |ExtlClntAppMobileSet|❌|Not supported, but support could be added| @@ -545,7 +546,9 @@ v57 introduces the following new types. Here's their current level of support |IntegrationProviderDef|❌|Not supported, but support could be added| |LocationUse|❌|Not supported, but support could be added| |OmniSupervisorConfig|❌|Not supported, but support could be added (but not for tracking)| +|PipelineInspMetricConfig|❌|Not supported, but support could be added| |ProductSpecificationTypeDefinition|❌|Not supported, but support could be added| +|ServiceProcess|❌|Not supported, but support could be added| |WaveAnalyticAssetCollection|❌|Not supported, but support could be added| ## Additional Types From 536a6a461fe0a315b9d24f034b49e47cfb8619cb Mon Sep 17 00:00:00 2001 From: mshanemc Date: Mon, 24 Oct 2022 13:22:01 -0500 Subject: [PATCH 07/41] feat: mark sourceComponents with replacements --- package.json | 3 +- replacementsTODO.md | 11 ++ src/convert/metadataConverter.ts | 18 +++- src/convert/replacements.ts | 111 +++++++++++++++++++ src/resolve/sourceComponent.ts | 3 +- src/resolve/types.ts | 29 +++++ test/convert/metadataConverter.test.ts | 34 +++--- test/convert/replacements.test.ts | 141 +++++++++++++++++++++++++ 8 files changed, 326 insertions(+), 24 deletions(-) create mode 100644 replacementsTODO.md create mode 100644 src/convert/replacements.ts create mode 100644 test/convert/replacements.test.ts diff --git a/package.json b/package.json index 07fe0a409d..38e33d1182 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "graceful-fs": "^4.2.10", "ignore": "^5.2.0", "mime": "2.6.0", + "minimatch": "^5.1.0", "proxy-agent": "^5.0.0", "proxy-from-env": "^1.1.0", "unzipper": "0.10.11" @@ -114,4 +115,4 @@ "yarn": "1.22.4" }, "config": {} -} \ No newline at end of file +} diff --git a/replacementsTODO.md b/replacementsTODO.md new file mode 100644 index 0000000000..9ba2e4ea3c --- /dev/null +++ b/replacementsTODO.md @@ -0,0 +1,11 @@ +pipeline stage in metadataConverter for adding replacement info to source components + +- stream in streams for markReplacements + +props/methods on SourceComponent for storing marked replacements + +4 transformers need to work with replacements on SourceComponent +default +staticResource +decomposed +nondecomposed diff --git a/src/convert/metadataConverter.ts b/src/convert/metadataConverter.ts index f8678bc676..3ac52af98b 100644 --- a/src/convert/metadataConverter.ts +++ b/src/convert/metadataConverter.ts @@ -4,7 +4,7 @@ * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -import { Readable } from 'stream'; +import { Readable, PassThrough } from 'stream'; import { dirname, join, normalize } from 'path'; import { Messages, SfError } from '@salesforce/core'; import { promises } from 'graceful-fs'; @@ -15,6 +15,7 @@ import { ComponentSet, DestructiveChangesType } from '../collections'; import { RegistryAccess } from '../registry'; import { ComponentConverter, pipeline, StandardWriter, ZipWriter } from './streams'; import { ConvertOutputConfig, ConvertResult, DirectoryConfig, SfdxFileFormat, ZipConfig } from './types'; +import { getReplacementStream } from './replacements'; Messages.importMessagesDirectory(__dirname); const messages = Messages.load('@salesforce/source-deploy-retrieve', 'sdr', [ @@ -33,6 +34,7 @@ export class MetadataConverter { public constructor(registry = new RegistryAccess()) { this.registry = registry; } + // eslint-disable-next-line complexity public async convert( comps: ComponentSet | Iterable, targetFormat: SfdxFileFormat, @@ -44,7 +46,7 @@ export class MetadataConverter { (comps instanceof ComponentSet ? Array.from(comps.getSourceComponents()) : comps) as SourceComponent[] ).filter((comp) => comp.type.isAddressable !== false); - const isSource = targetFormat === 'source'; + const targetFormatIsSource = targetFormat === 'source'; const tasks: Array> = []; let writer: StandardWriter | ZipWriter; @@ -60,7 +62,7 @@ export class MetadataConverter { packagePath = getPackagePath(output); defaultDirectory = packagePath; writer = new StandardWriter(packagePath); - if (!isSource) { + if (!targetFormatIsSource) { const manifestPath = join(packagePath, MetadataConverter.PACKAGE_XML_FILE); tasks.push( promises.writeFile(manifestPath, await cs.getPackageXml()), @@ -79,13 +81,16 @@ export class MetadataConverter { if (output.packageName) { cs.fullName = output.packageName; } + packagePath = getPackagePath(output); defaultDirectory = packagePath; writer = new ZipWriter(packagePath); - if (!isSource) { + if (!targetFormatIsSource) { writer.addToZip(await cs.getPackageXml(), MetadataConverter.PACKAGE_XML_FILE); + // for each of the destructive changes in the component set, convert and write the correct metadata // to each manifest + for (const destructiveChangeType of cs.getTypesOfDestructiveChanges()) { writer.addToZip( // TODO: can this be safely parallelized? @@ -97,7 +102,7 @@ export class MetadataConverter { } break; case 'merge': - if (!isSource) { + if (!targetFormatIsSource) { throw new SfError(messages.getMessage('error_merge_metadata_target_unsupported')); } defaultDirectory = output.defaultDirectory; @@ -113,6 +118,9 @@ export class MetadataConverter { const conversionPipeline = pipeline( Readable.from(components), + output.type === 'zip' && !targetFormatIsSource + ? (await getReplacementStream()) ?? new PassThrough() + : new PassThrough(), new ComponentConverter(targetFormat, this.registry, mergeSet, defaultDirectory), writer ); diff --git a/src/convert/replacements.ts b/src/convert/replacements.ts new file mode 100644 index 0000000000..55c4c117b7 --- /dev/null +++ b/src/convert/replacements.ts @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2020, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { readFile } from 'fs/promises'; +import { Transform } from 'stream'; +import { SfProject } from '@salesforce/core'; +import * as minimatch from 'minimatch'; +import { MarkedReplacement, ReplacementConfig } from '../resolve/types'; +import { SourceComponent } from '../resolve/sourceComponent'; + +const fileContentsCache = new Map(); + +export const readReplacementsFromProject = async (): Promise => { + const proj = await SfProject.resolve(); + const projJson = (await proj.resolveProjectConfig()) as { replacements?: ReplacementConfig[] }; + return projJson.replacements; +}; + +export const getReplacementStream = async (): Promise => { + // remove any that don't agree with current env + const filteredReplacements = envFilter(await readReplacementsFromProject()); + if (filteredReplacements.length) { + return new ReplacementMarkingStream(filteredReplacements); + } +}; +/** + * Stream for marking replacements on a component. + * Returns a mutated component with a `replacements` property if replacements are found. + */ +export class ReplacementMarkingStream extends Transform { + public constructor(private readonly replacementConfigs: ReplacementConfig[]) { + super({ objectMode: true }); + } + + public async _transform( + chunk: SourceComponent, + encoding: string, + callback: (err: Error, data: SourceComponent) => void + ): Promise { + let err: Error; + // if deleting, or no configs, just pass through + if (!chunk.isMarkedForDelete() && this.replacementConfigs?.length) { + try { + chunk.replacements = await getReplacements(chunk, this.replacementConfigs); + } catch (e) { + if (!(e instanceof Error)) { + throw e; + } + err = e; + } + } + callback(err, chunk); + } +} + +export const getContents = async (path: string): Promise => { + if (!fileContentsCache.has(path)) { + fileContentsCache.set(path, await readFile(path, 'utf8')); + } + return fileContentsCache.get(path); +}; + +/** + * Regardless of any components, return the ReplacementConfig that are valid with the current env. + * These can be checked globally and don't need to be checked per component. + */ +export const envFilter = (replacementConfigs: ReplacementConfig[] = []): ReplacementConfig[] => + replacementConfigs.filter( + (replacement) => + !replacement.replaceWhenEnv || + replacement.replaceWhenEnv.every((envConditional) => process.env[envConditional.env] === envConditional.value) + ); + +/** + * Build the replacements property for a sourceComponent + */ +export const getReplacements = async ( + cmp: SourceComponent, + replacementConfigs: ReplacementConfig[] = [] +): Promise => { + // all possible filenames for this component + const filenames = [cmp.xml, ...cmp.walkContent()].filter(Boolean); + // eslint-disable-next-line no-console + const replacementsForComponent = ( + await Promise.all( + filenames.map( + async (f): Promise<[string, MarkedReplacement[]]> => [ + f, + await Promise.all( + replacementConfigs + .filter((r) => matchesFile(f, r)) + .map(async (r) => ({ + toReplace: r.stringToReplace ?? new RegExp(r.regexToReplace), + replaceWith: r.replaceWithEnv ? process.env[r.replaceWithEnv] : await getContents(r.replaceWithFile), + })) + ), + ] + ) + ) + ).filter(([, replacements]) => replacements.length > 0); + + if (replacementsForComponent.length) { + return Object.fromEntries(replacementsForComponent); + } +}; + +export const matchesFile = (f: string, r: ReplacementConfig): boolean => + r.filename === f || (r.glob && minimatch(f, r.glob)); diff --git a/src/resolve/sourceComponent.ts b/src/resolve/sourceComponent.ts index ff7ec3cc7f..2d6c7d19af 100644 --- a/src/resolve/sourceComponent.ts +++ b/src/resolve/sourceComponent.ts @@ -15,7 +15,7 @@ import { SfdxFileFormat } from '../convert'; import { MetadataType } from '../registry'; import { DestructiveChangesType } from '../collections'; import { filePathsFromMetadataComponent } from '../utils/filePathGenerator'; -import { MetadataComponent, VirtualDirectory } from './types'; +import { MarkedReplacement, MetadataComponent, VirtualDirectory } from './types'; import { NodeFSTreeContainer, TreeContainer, VirtualTreeContainer } from './treeContainers'; import { ForceIgnore } from './forceIgnore'; @@ -44,6 +44,7 @@ export class SourceComponent implements MetadataComponent { public readonly parent?: SourceComponent; public parentType?: MetadataType; public content?: string; + public replacements: Record; private treeContainer: TreeContainer; private forceIgnore: ForceIgnore; private markedForDelete = false; diff --git a/src/resolve/types.ts b/src/resolve/types.ts index 2e00245b65..926e5ed478 100644 --- a/src/resolve/types.ts +++ b/src/resolve/types.ts @@ -58,3 +58,32 @@ export interface SourceAdapter { */ allowMetadataWithContent(): boolean; } + +// TODO: what's the right way to get this into core/sfdxProjectJson +export type ReplacementConfig = + // requires a filename or a glob + { + filename?: string; + + glob?: string; + + stringToReplace?: string; + + regexToReplace?: string; + + replaceWithEnv?: string; + + replaceWithFile?: string; + + replaceWhenEnv?: [ + { + env: string; + value: string | number | boolean; + } + ]; + }; + +export interface MarkedReplacement { + toReplace: string | RegExp; + replaceWith: string; +} diff --git a/test/convert/metadataConverter.test.ts b/test/convert/metadataConverter.test.ts index 28fe06c866..60a59562e6 100644 --- a/test/convert/metadataConverter.test.ts +++ b/test/convert/metadataConverter.test.ts @@ -44,9 +44,9 @@ describe('MetadataConverter', () => { /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ function validatePipelineArgs(pipelineArgs: any[], targetFormat = 'metadata'): void { - expect(pipelineArgs[1] instanceof streams.ComponentConverter).to.be.true; - expect(pipelineArgs[1].targetFormat).to.equal(targetFormat); - expect(pipelineArgs[2] instanceof streams.ComponentWriter).to.be.true; + expect(pipelineArgs[2] instanceof streams.ComponentConverter).to.be.true; + expect(pipelineArgs[2].targetFormat).to.equal(targetFormat); + expect(pipelineArgs[3] instanceof streams.ComponentWriter).to.be.true; } beforeEach(() => { @@ -67,7 +67,7 @@ describe('MetadataConverter', () => { outputDirectory, }); - expect(pipelineStub.firstCall.args[2].rootDestination).to.equal(packagePath); + expect(pipelineStub.firstCall.args[3].rootDestination).to.equal(packagePath); }); it('should convert to specified output dir', async () => { @@ -77,7 +77,7 @@ describe('MetadataConverter', () => { genUniqueDir: false, }); - expect(pipelineStub.firstCall.args[2].rootDestination).to.equal(outputDirectory); + expect(pipelineStub.firstCall.args[3].rootDestination).to.equal(outputDirectory); }); it('should throw ConversionError when an error occurs', async () => { @@ -123,8 +123,8 @@ describe('MetadataConverter', () => { const pipelineArgs = pipelineStub.firstCall.args; validatePipelineArgs(pipelineArgs); - expect(pipelineArgs[2] instanceof streams.StandardWriter).to.be.true; - expect(pipelineArgs[2].rootDestination).to.equal(packageOutput); + expect(pipelineArgs[3] instanceof streams.StandardWriter).to.be.true; + expect(pipelineArgs[3].rootDestination).to.equal(packageOutput); }); it('should create conversion pipeline with normalized output directory', async () => { @@ -136,8 +136,8 @@ describe('MetadataConverter', () => { const pipelineArgs = pipelineStub.firstCall.args; validatePipelineArgs(pipelineArgs); - expect(pipelineArgs[2] instanceof streams.StandardWriter).to.be.true; - expect(pipelineArgs[2].rootDestination).to.equal(packageName); + expect(pipelineArgs[3] instanceof streams.StandardWriter).to.be.true; + expect(pipelineArgs[3].rootDestination).to.equal(packageName); }); it('should return packagePath in result', async () => { @@ -312,8 +312,8 @@ describe('MetadataConverter', () => { // secondCall is used because ZipWriter uses pipeline upon construction const pipelineArgs = pipelineStub.secondCall.args; validatePipelineArgs(pipelineArgs); - expect(pipelineArgs[2] instanceof streams.ZipWriter).to.be.true; - expect(pipelineArgs[2].rootDestination).to.equal(`${packageOutput}.zip`); + expect(pipelineArgs[3] instanceof streams.ZipWriter).to.be.true; + expect(pipelineArgs[3].rootDestination).to.equal(`${packageOutput}.zip`); }); it('should create conversion pipeline with in-memory configuration', async () => { @@ -321,8 +321,8 @@ describe('MetadataConverter', () => { const pipelineArgs = pipelineStub.secondCall.args; validatePipelineArgs(pipelineArgs); - expect(pipelineArgs[2] instanceof streams.ZipWriter).to.be.true; - expect(pipelineArgs[2].rootDestination).to.be.undefined; + expect(pipelineArgs[3] instanceof streams.ZipWriter).to.be.true; + expect(pipelineArgs[3].rootDestination).to.be.undefined; }); it('should return zipBuffer result for in-memory configuration', async () => { @@ -455,8 +455,8 @@ describe('MetadataConverter', () => { const pipelineArgs = pipelineStub.firstCall.args; validatePipelineArgs(pipelineArgs, 'source'); - expect(pipelineArgs[1].mergeSet).to.deep.equal(new ComponentSet(COMPONENTS)); - expect(pipelineArgs[2].rootDestination).to.equal(defaultDirectory); + expect(pipelineArgs[2].mergeSet).to.deep.equal(new ComponentSet(COMPONENTS)); + expect(pipelineArgs[3].rootDestination).to.equal(defaultDirectory); }); it('should create conversion pipeline with addressable components', async () => { @@ -477,7 +477,7 @@ describe('MetadataConverter', () => { // pop off the CFT that should be filtered off for the assertion components.pop(); - expect(pipelineArgs[2].rootDestination).to.equal(defaultDirectory); + expect(pipelineArgs[3].rootDestination).to.equal(defaultDirectory); }); it('should ensure merge set contains parents of child components instead of the children themselves', async () => { @@ -489,7 +489,7 @@ describe('MetadataConverter', () => { const pipelineArgs = pipelineStub.firstCall.args; validatePipelineArgs(pipelineArgs, 'source'); - expect(pipelineArgs[1].mergeSet).to.deep.equal(new ComponentSet([DECOMPOSED_CHILD_COMPONENT_1.parent])); + expect(pipelineArgs[2].mergeSet).to.deep.equal(new ComponentSet([DECOMPOSED_CHILD_COMPONENT_1.parent])); }); }); }); diff --git a/test/convert/replacements.test.ts b/test/convert/replacements.test.ts new file mode 100644 index 0000000000..1c3f12c9c5 --- /dev/null +++ b/test/convert/replacements.test.ts @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2020, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { expect } from 'chai'; +import Sinon = require('sinon'); +import { getReplacements, matchesFile } from '../../src/convert/replacements'; +import { matchingContentFile } from '../mock'; +import * as replacementsForMock from '../../src/convert/replacements'; + +describe('file matching', () => { + it('file matches string', () => { + expect(matchesFile('foo', { filename: 'foo' })).to.be.true; + expect(matchesFile('bar', { filename: 'foo' })).to.not.be.true; + }); + it('file matches glob', () => { + expect(matchesFile('foo/bar', { glob: 'foo/**' })).to.be.true; + expect(matchesFile('foo/bar', { glob: 'foo/*' })).to.be.true; + expect(matchesFile('foo/bar', { glob: 'foo' })).to.be.false; + expect(matchesFile('foo/bar', { glob: '**/*' })).to.be.true; + }); + it('test absolute vs. relative paths'); +}); + +describe('env filters', () => {}); + +describe('marking replacements on a component', () => { + before(() => { + // replaceFromFile uses the contents of a file. This prevents the test from hitting real FS for that. + Sinon.stub(replacementsForMock, 'getContents').resolves('bar'); + }); + + process.env.FOO_REPLACEMENT = 'bar'; + const cmp = matchingContentFile.COMPONENT; + + beforeEach(() => { + delete cmp.replacements; + }); + + it('marks no replacements when passed no configs', async () => { + expect(await getReplacements(cmp)).to.be.undefined; + expect(await getReplacements(cmp, [])).to.be.undefined; + }); + it('marks a string replacement from env', async () => { + const result = await getReplacements(cmp, [ + { filename: cmp.xml, stringToReplace: 'foo', replaceWithEnv: 'FOO_REPLACEMENT' }, + ]); + expect(result).to.deep.equal({ + [cmp.xml]: [ + { + toReplace: 'foo', + replaceWith: 'bar', + }, + ], + }); + }); + it('marks string replacements from file', async () => { + const result = await getReplacements(cmp, [{ filename: cmp.xml, stringToReplace: 'foo', replaceWithFile: 'bar' }]); + expect(result).to.deep.equal({ + [cmp.xml]: [ + { + toReplace: 'foo', + replaceWith: 'bar', + }, + ], + }); + }); + + it('marks regex replacements on a matching file', async () => { + const result = await getReplacements(cmp, [ + { filename: cmp.xml, regexToReplace: '.*foo.*', replaceWithEnv: 'FOO_REPLACEMENT' }, + ]); + expect(result).to.deep.equal({ + [cmp.xml]: [ + { + toReplace: /.*foo.*/, + replaceWith: 'bar', + }, + ], + }); + }); + it('marks 2 replacements on one file', async () => { + const result = await getReplacements(cmp, [ + { filename: cmp.xml, stringToReplace: 'foo', replaceWithEnv: 'FOO_REPLACEMENT' }, + { filename: cmp.xml, stringToReplace: 'baz', replaceWithEnv: 'FOO_REPLACEMENT' }, + ]); + expect(result).to.deep.equal({ + [cmp.xml]: [ + { + toReplace: 'foo', + replaceWith: 'bar', + }, + { + toReplace: 'baz', + replaceWith: 'bar', + }, + ], + }); + }); + it('marks two files with 1 replacement each for greedy glob', async () => { + const result = await getReplacements(cmp, [ + { glob: '**/*', stringToReplace: 'foo', replaceWithEnv: 'FOO_REPLACEMENT' }, + ]); + expect(result).to.deep.equal({ + [cmp.xml]: [ + { + toReplace: 'foo', + replaceWith: 'bar', + }, + ], + [cmp.content]: [ + { + toReplace: 'foo', + replaceWith: 'bar', + }, + ], + }); + }); + it('marks replacement on multiple files from multiple configs', async () => { + const result = await getReplacements(cmp, [ + { filename: cmp.xml, stringToReplace: 'foo', replaceWithEnv: 'FOO_REPLACEMENT' }, + { filename: cmp.content, stringToReplace: 'foo', replaceWithEnv: 'FOO_REPLACEMENT' }, + ]); + expect(result).to.deep.equal({ + [cmp.xml]: [ + { + toReplace: 'foo', + replaceWith: 'bar', + }, + ], + [cmp.content]: [ + { + toReplace: 'foo', + replaceWith: 'bar', + }, + ], + }); + }); +}); From a68664b629146ccc357de0b96cba668901762d1f Mon Sep 17 00:00:00 2001 From: mshanemc Date: Mon, 24 Oct 2022 21:09:28 -0500 Subject: [PATCH 08/41] feat: defaultTransformer handles replacements --- package.json | 1 + src/convert/metadataConverter.ts | 8 +- src/convert/replacements.ts | 96 +++++++++++++++++-- .../defaultMetadataTransformer.ts | 5 +- .../staticResourceMetadataTransformer.ts | 20 +++- src/convert/types.ts | 32 +++++++ src/resolve/sourceComponent.ts | 4 +- src/resolve/types.ts | 29 ------ yarn.lock | 5 + 9 files changed, 150 insertions(+), 50 deletions(-) diff --git a/package.json b/package.json index 38e33d1182..3fae4c461b 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "@types/archiver": "^5.3.1", "@types/deep-equal-in-any-order": "^1.0.1", "@types/mime": "2.0.3", + "@types/minimatch": "^5.1.2", "@types/proxy-from-env": "^1.0.1", "@types/shelljs": "^0.8.11", "@types/unzipper": "^0.10.5", diff --git a/src/convert/metadataConverter.ts b/src/convert/metadataConverter.ts index 3ac52af98b..c4cfbe182e 100644 --- a/src/convert/metadataConverter.ts +++ b/src/convert/metadataConverter.ts @@ -15,7 +15,7 @@ import { ComponentSet, DestructiveChangesType } from '../collections'; import { RegistryAccess } from '../registry'; import { ComponentConverter, pipeline, StandardWriter, ZipWriter } from './streams'; import { ConvertOutputConfig, ConvertResult, DirectoryConfig, SfdxFileFormat, ZipConfig } from './types'; -import { getReplacementStream } from './replacements'; +import { getReplacementMarkingStream } from './replacements'; Messages.importMessagesDirectory(__dirname); const messages = Messages.load('@salesforce/source-deploy-retrieve', 'sdr', [ @@ -118,9 +118,9 @@ export class MetadataConverter { const conversionPipeline = pipeline( Readable.from(components), - output.type === 'zip' && !targetFormatIsSource - ? (await getReplacementStream()) ?? new PassThrough() - : new PassThrough(), + !targetFormatIsSource && (process.env.DEBUG_REPLACEMENTS_VIA_CONVERT === 'true' || output.type === 'zip') + ? (await getReplacementMarkingStream()) ?? new PassThrough({ objectMode: true }) + : new PassThrough({ objectMode: true }), new ComponentConverter(targetFormat, this.registry, mergeSet, defaultDirectory), writer ); diff --git a/src/convert/replacements.ts b/src/convert/replacements.ts index 55c4c117b7..9dc8fd382a 100644 --- a/src/convert/replacements.ts +++ b/src/convert/replacements.ts @@ -4,31 +4,92 @@ * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ +import { join } from 'path'; import { readFile } from 'fs/promises'; -import { Transform } from 'stream'; -import { SfProject } from '@salesforce/core'; +import { Transform, Readable } from 'stream'; +import { Lifecycle, SfError, SfProject } from '@salesforce/core'; import * as minimatch from 'minimatch'; -import { MarkedReplacement, ReplacementConfig } from '../resolve/types'; +import { SourcePath } from '../common'; import { SourceComponent } from '../resolve/sourceComponent'; +import { MarkedReplacement, ReplacementConfig } from './types'; const fileContentsCache = new Map(); +/** If a component has replacements, you get it piped through the replacementStream + * Otherwise, you'll get the original readable stream + */ +export const getReplacementStreamForReadable = ( + component: SourceComponent, + path: SourcePath +): Readable | ReplacementStream => + component.replacements?.[path] + ? component.tree.stream(path).pipe(new ReplacementStream(component.replacements?.[path])) + : component.tree.stream(path); + +/** + * A stream for replacing the contents of a single SourceComponent. + * + */ +export class ReplacementStream extends Transform { + public constructor(private readonly replacements: MarkedReplacement[]) { + super({ objectMode: true }); + } + + public async _transform( + chunk: Buffer, + encoding: string, + callback: (error?: Error, data?: Buffer) => void + ): Promise { + let error: Error; + // read and do the various replacements + callback(error, Buffer.from(await replacementIterations(chunk.toString(), this.replacements))); + } +} + +export const replacementIterations = async (input: string, replacements: MarkedReplacement[]): Promise => { + let output = input; + for (const replacement of replacements) { + // TODO: node 16+ has String.replaceAll for non-regex scenarios + const regex = + typeof replacement.toReplace === 'string' ? new RegExp(replacement.toReplace, 'g') : replacement.toReplace; + // TODO: warn when the replacement is not found + const replaced = output.replace(regex, replacement.replaceWith); + if (replaced === output) { + // replacements need to be done sequentially + // eslint-disable-next-line no-await-in-loop + await Lifecycle.getInstance().emitWarning( + `Your sfdx-project.json specifies that ${replacement.toReplace.toString()} should be replaced, but it was not found.` + ); + } + output = replaced; + } + return output; +}; + +/** + * Read the `replacement` property from sfdx-project.json + */ export const readReplacementsFromProject = async (): Promise => { const proj = await SfProject.resolve(); const projJson = (await proj.resolveProjectConfig()) as { replacements?: ReplacementConfig[] }; return projJson.replacements; }; -export const getReplacementStream = async (): Promise => { +/** + * Reads the project, gets replacements, removes an that aren't applicable due ot environment conditionals, and returns an instance of the ReplacementMarkingStream + */ +export const getReplacementMarkingStream = async (): Promise => { // remove any that don't agree with current env const filteredReplacements = envFilter(await readReplacementsFromProject()); if (filteredReplacements.length) { return new ReplacementMarkingStream(filteredReplacements); } }; + /** * Stream for marking replacements on a component. - * Returns a mutated component with a `replacements` property if replacements are found. + * Returns a mutated component with a `replacements` property if any replacements are found. + * Throws if any replacements reference a file or env that does not exist */ export class ReplacementMarkingStream extends Transform { public constructor(private readonly replacementConfigs: ReplacementConfig[]) { @@ -58,7 +119,13 @@ export class ReplacementMarkingStream extends Transform { export const getContents = async (path: string): Promise => { if (!fileContentsCache.has(path)) { - fileContentsCache.set(path, await readFile(path, 'utf8')); + try { + fileContentsCache.set(path, await readFile(path, 'utf8')); + } catch (e) { + throw new SfError( + `The file "${path}" specified in the "replacements" property of sfdx-project.json could not be read.` + ); + } } return fileContentsCache.get(path); }; @@ -93,8 +160,8 @@ export const getReplacements = async ( replacementConfigs .filter((r) => matchesFile(f, r)) .map(async (r) => ({ - toReplace: r.stringToReplace ?? new RegExp(r.regexToReplace), - replaceWith: r.replaceWithEnv ? process.env[r.replaceWithEnv] : await getContents(r.replaceWithFile), + toReplace: r.stringToReplace ?? new RegExp(r.regexToReplace, 'g'), + replaceWith: r.replaceWithEnv ? getEnvValue(r.replaceWithEnv) : await getContents(r.replaceWithFile), })) ), ] @@ -108,4 +175,15 @@ export const getReplacements = async ( }; export const matchesFile = (f: string, r: ReplacementConfig): boolean => - r.filename === f || (r.glob && minimatch(f, r.glob)); + // filenames will be absolute. We don't have convenient access to the pkgDirs, + // so we need to be more open than an exact match + f.endsWith(r.filename) || (r.glob && minimatch(f, join('**', r.glob))); + +const getEnvValue = (env: string): string => { + if (process.env[env]) { + return process.env[env]; + } + throw new SfError( + `"${env}" is in sfdx-project.json as a value for "replaceWithEnv" property, but it's not set in your environment.` + ); +}; diff --git a/src/convert/transformers/defaultMetadataTransformer.ts b/src/convert/transformers/defaultMetadataTransformer.ts index a1d4d451ab..b9f0bd7cd0 100644 --- a/src/convert/transformers/defaultMetadataTransformer.ts +++ b/src/convert/transformers/defaultMetadataTransformer.ts @@ -9,6 +9,7 @@ import { META_XML_SUFFIX, SourcePath } from '../../common'; import { SfdxFileFormat, WriteInfo } from '../types'; import { SourceComponent } from '../../resolve'; import { extName, trimUntil } from '../../utils'; +import { getReplacementStreamForReadable } from '../replacements'; import { BaseMetadataTransformer } from './baseMetadataTransformer'; const ORIGINAL_SUFFIX_REGEX = new RegExp('(.)([a-zA-Z]+)(' + META_XML_SUFFIX + ')$'); @@ -40,14 +41,14 @@ const getWriteInfos = ( component .walkContent() .map((path) => ({ - source: component.tree.stream(path), + source: getReplacementStreamForReadable(component, path), output: getContentSourceDestination(path, targetFormat, component, mergeWith), })) .concat( component.xml ? [ { - source: component.tree.stream(component.xml), + source: getReplacementStreamForReadable(component, component.xml), output: getXmlDestination(targetFormat, component, mergeWith), }, ] diff --git a/src/convert/transformers/staticResourceMetadataTransformer.ts b/src/convert/transformers/staticResourceMetadataTransformer.ts index 49a8e9c121..32768a64e6 100644 --- a/src/convert/transformers/staticResourceMetadataTransformer.ts +++ b/src/convert/transformers/staticResourceMetadataTransformer.ts @@ -18,6 +18,7 @@ import { SourceComponent } from '../../resolve'; import { SourcePath } from '../../common'; import { ensureFileExists } from '../../utils/fileSystemHandler'; import { pipeline } from '../streams'; +import { getReplacementStreamForReadable } from '../replacements'; import { BaseMetadataTransformer } from './baseMetadataTransformer'; Messages.importMessagesDirectory(__dirname); @@ -42,20 +43,29 @@ export class StaticResourceMetadataTransformer extends BaseMetadataTransformer { // toolbelt was using level 9 for static resources, so we'll do the same. // Otherwise, you'll see errors like https://github.com/forcedotcom/cli/issues/1098 const zip = createArchive('zip', { zlib: { level: 9 } }); - zip.directory(content, false); + if (!component.replacements) { + // the easy way...no replacements required + zip.directory(content, false); + } else { + // the hard way--we have to walk the content and do replacements on each of the files. + for (const path of component.walkContent()) { + const replacementStream = getReplacementStreamForReadable(component, path); + zip.append(replacementStream, { name: path }); + } + } await zip.finalize(); return zip; }; - // const contentSource = (; - return [ { - source: (await componentIsExpandedArchive(component)) ? await zipIt() : component.tree.stream(content), + source: (await componentIsExpandedArchive(component)) + ? await zipIt() + : getReplacementStreamForReadable(component, content), output: join(type.directoryName, `${baseName(content)}.${type.suffix}`), }, { - source: component.tree.stream(xml), + source: getReplacementStreamForReadable(component, xml), output: join(type.directoryName, basename(xml)), }, ]; diff --git a/src/convert/types.ts b/src/convert/types.ts index d63e6d28f8..6ae27c5034 100644 --- a/src/convert/types.ts +++ b/src/convert/types.ts @@ -105,3 +105,35 @@ export type ConvertResult = { */ converted?: SourceComponent[]; }; + +// TODO: what's the right way to get this into core/sfdxProjectJson +export type ReplacementConfig = Location & + ReplacementSource & + ReplacementTarget & { + /** Only do the replacement if ALL of the environment values in this array match */ + replaceWhenEnv?: [ + { + env: string; + value: string | number | boolean; + } + ]; + }; + +/** Stored by file on SourceComponent for stream processing */ +export interface MarkedReplacement { + toReplace: string | RegExp; + replaceWith: string; +} + +type Location = { filename: string; glob: never } | { filename: never; glob: string }; +type ReplacementSource = + | { replaceWithEnv: string; replaceWithFile: never } + | { replaceWithEnv: never; replaceWithFile: string }; + +type ReplacementTarget = + | { stringToReplace: string; regexToReplace: never } + | { + stringToReplace: never; + /** When putting regex into json, you have to use an extra backslash to escape your regex backslashes because JSON also treats backslash as an escape character */ + regexToReplace: string; + }; diff --git a/src/resolve/sourceComponent.ts b/src/resolve/sourceComponent.ts index 2d6c7d19af..c8a890493f 100644 --- a/src/resolve/sourceComponent.ts +++ b/src/resolve/sourceComponent.ts @@ -15,7 +15,9 @@ import { SfdxFileFormat } from '../convert'; import { MetadataType } from '../registry'; import { DestructiveChangesType } from '../collections'; import { filePathsFromMetadataComponent } from '../utils/filePathGenerator'; -import { MarkedReplacement, MetadataComponent, VirtualDirectory } from './types'; +import { MarkedReplacement } from '../convert/types'; +import { MetadataComponent, VirtualDirectory } from './types'; + import { NodeFSTreeContainer, TreeContainer, VirtualTreeContainer } from './treeContainers'; import { ForceIgnore } from './forceIgnore'; diff --git a/src/resolve/types.ts b/src/resolve/types.ts index 926e5ed478..2e00245b65 100644 --- a/src/resolve/types.ts +++ b/src/resolve/types.ts @@ -58,32 +58,3 @@ export interface SourceAdapter { */ allowMetadataWithContent(): boolean; } - -// TODO: what's the right way to get this into core/sfdxProjectJson -export type ReplacementConfig = - // requires a filename or a glob - { - filename?: string; - - glob?: string; - - stringToReplace?: string; - - regexToReplace?: string; - - replaceWithEnv?: string; - - replaceWithFile?: string; - - replaceWhenEnv?: [ - { - env: string; - value: string | number | boolean; - } - ]; - }; - -export interface MarkedReplacement { - toReplace: string | RegExp; - replaceWith: string; -} diff --git a/yarn.lock b/yarn.lock index 3d5d7f9208..e6f81c1c4b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -948,6 +948,11 @@ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ== +"@types/minimatch@^5.1.2": + version "5.1.2" + resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-5.1.2.tgz#07508b45797cb81ec3f273011b054cd0755eddca" + integrity sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA== + "@types/minimist@^1.2.0": version "1.2.2" resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c" From 55abc368d95473291409c05eb45ce660a8132e36 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Mon, 24 Oct 2022 22:04:15 -0500 Subject: [PATCH 09/41] feat: works correctly for customObject --- src/resolve/sourceComponent.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/resolve/sourceComponent.ts b/src/resolve/sourceComponent.ts index c8a890493f..66498e1ae4 100644 --- a/src/resolve/sourceComponent.ts +++ b/src/resolve/sourceComponent.ts @@ -9,6 +9,7 @@ import { Messages, SfError } from '@salesforce/core'; import { parse, validate } from 'fast-xml-parser'; import { get, getString, JsonMap } from '@salesforce/ts-types'; import { ensureArray } from '@salesforce/kit'; +import { replacementIterations } from '../../src/convert/replacements'; import { baseName, parseMetadataXml, trimUntil } from '../utils'; import { DEFAULT_PACKAGE_ROOT_SFDX } from '../common'; import { SfdxFileFormat } from '../convert'; @@ -162,7 +163,11 @@ export class SourceComponent implements MetadataComponent { const xml = xmlFilePath ?? this.xml; if (xml) { const contents = (await this.tree.readFile(xml)).toString(); - return this.parseAndValidateXML(contents, xml); + const replacements = this.replacements?.[xml] ?? this.parent?.replacements?.[xml]; + return this.parseAndValidateXML( + replacements ? await replacementIterations(contents, replacements) : contents, + xml + ); } return {} as T; } From 65dd37f0642d0dedb11187a676fe5da37cdfba28 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Mon, 24 Oct 2022 22:14:59 -0500 Subject: [PATCH 10/41] feat: works for zipped static resources --- src/convert/transformers/staticResourceMetadataTransformer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/convert/transformers/staticResourceMetadataTransformer.ts b/src/convert/transformers/staticResourceMetadataTransformer.ts index 32768a64e6..90a1afdc1e 100644 --- a/src/convert/transformers/staticResourceMetadataTransformer.ts +++ b/src/convert/transformers/staticResourceMetadataTransformer.ts @@ -4,7 +4,7 @@ * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -import { basename, dirname, isAbsolute, join } from 'path'; +import { basename, dirname, isAbsolute, join, relative } from 'path'; import { Readable } from 'stream'; import { create as createArchive, Archiver } from 'archiver'; import { getExtension } from 'mime'; @@ -50,7 +50,7 @@ export class StaticResourceMetadataTransformer extends BaseMetadataTransformer { // the hard way--we have to walk the content and do replacements on each of the files. for (const path of component.walkContent()) { const replacementStream = getReplacementStreamForReadable(component, path); - zip.append(replacementStream, { name: path }); + zip.append(replacementStream, { name: relative(content, path) }); } } await zip.finalize(); From f0cadafc433cd3356f4c01d0f1ec8913c44459cd Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 09:43:31 -0500 Subject: [PATCH 11/41] chore: cleanup unnecessary exports and TODO --- replacementsTODO.md | 11 ----------- src/convert/replacements.ts | 12 ++++++++---- 2 files changed, 8 insertions(+), 15 deletions(-) delete mode 100644 replacementsTODO.md diff --git a/replacementsTODO.md b/replacementsTODO.md deleted file mode 100644 index 9ba2e4ea3c..0000000000 --- a/replacementsTODO.md +++ /dev/null @@ -1,11 +0,0 @@ -pipeline stage in metadataConverter for adding replacement info to source components - -- stream in streams for markReplacements - -props/methods on SourceComponent for storing marked replacements - -4 transformers need to work with replacements on SourceComponent -default -staticResource -decomposed -nondecomposed diff --git a/src/convert/replacements.ts b/src/convert/replacements.ts index 9dc8fd382a..c14589f9e3 100644 --- a/src/convert/replacements.ts +++ b/src/convert/replacements.ts @@ -30,7 +30,7 @@ export const getReplacementStreamForReadable = ( * A stream for replacing the contents of a single SourceComponent. * */ -export class ReplacementStream extends Transform { +class ReplacementStream extends Transform { public constructor(private readonly replacements: MarkedReplacement[]) { super({ objectMode: true }); } @@ -46,6 +46,10 @@ export class ReplacementStream extends Transform { } } +/** + * perform an array of replacements on a string + * emits warnings when an expected replacement target isn't found + */ export const replacementIterations = async (input: string, replacements: MarkedReplacement[]): Promise => { let output = input; for (const replacement of replacements) { @@ -69,7 +73,7 @@ export const replacementIterations = async (input: string, replacements: MarkedR /** * Read the `replacement` property from sfdx-project.json */ -export const readReplacementsFromProject = async (): Promise => { +const readReplacementsFromProject = async (): Promise => { const proj = await SfProject.resolve(); const projJson = (await proj.resolveProjectConfig()) as { replacements?: ReplacementConfig[] }; return projJson.replacements; @@ -91,7 +95,7 @@ export const getReplacementMarkingStream = async (): Promise => { * Regardless of any components, return the ReplacementConfig that are valid with the current env. * These can be checked globally and don't need to be checked per component. */ -export const envFilter = (replacementConfigs: ReplacementConfig[] = []): ReplacementConfig[] => +const envFilter = (replacementConfigs: ReplacementConfig[] = []): ReplacementConfig[] => replacementConfigs.filter( (replacement) => !replacement.replaceWhenEnv || From 82f7533d35819553201ce83724ba3c52311c1cd7 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 09:44:44 -0500 Subject: [PATCH 12/41] chore: todo comment cleanup --- src/convert/replacements.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/convert/replacements.ts b/src/convert/replacements.ts index c14589f9e3..07e4b96c8c 100644 --- a/src/convert/replacements.ts +++ b/src/convert/replacements.ts @@ -56,7 +56,6 @@ export const replacementIterations = async (input: string, replacements: MarkedR // TODO: node 16+ has String.replaceAll for non-regex scenarios const regex = typeof replacement.toReplace === 'string' ? new RegExp(replacement.toReplace, 'g') : replacement.toReplace; - // TODO: warn when the replacement is not found const replaced = output.replace(regex, replacement.replaceWith); if (replaced === output) { // replacements need to be done sequentially From 1946edd035ee89fa84c4d17dec25d5e3c49209bd Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 09:55:39 -0500 Subject: [PATCH 13/41] fix: replacemenConfig typing --- src/convert/types.ts | 26 +++++++++++++------------- test/convert/replacements.test.ts | 13 +++++++------ 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/convert/types.ts b/src/convert/types.ts index 6ae27c5034..f1c338f145 100644 --- a/src/convert/types.ts +++ b/src/convert/types.ts @@ -85,7 +85,7 @@ export interface MetadataTransformer { * * `metadata` - Structure for use with the metadata api. * - * `source` - Friendly for local editing and comitting files to source control. + * `source` - Friendly for local editing and committing files to source control. */ export type SfdxFileFormat = 'metadata' | 'source'; @@ -101,11 +101,17 @@ export type ConvertResult = { */ zipBuffer?: Buffer; /** - * Converted source components. Not set if archving the package. + * Converted source components. Not set if archiving the package. */ converted?: SourceComponent[]; }; +/** Stored by file on SourceComponent for stream processing */ +export interface MarkedReplacement { + toReplace: string | RegExp; + replaceWith: string; +} + // TODO: what's the right way to get this into core/sfdxProjectJson export type ReplacementConfig = Location & ReplacementSource & @@ -119,21 +125,15 @@ export type ReplacementConfig = Location & ]; }; -/** Stored by file on SourceComponent for stream processing */ -export interface MarkedReplacement { - toReplace: string | RegExp; - replaceWith: string; -} - -type Location = { filename: string; glob: never } | { filename: never; glob: string }; +type Location = { filename: string; glob?: never } | { filename?: never; glob: string }; type ReplacementSource = - | { replaceWithEnv: string; replaceWithFile: never } - | { replaceWithEnv: never; replaceWithFile: string }; + | { replaceWithEnv: string; replaceWithFile?: never } + | { replaceWithEnv?: never; replaceWithFile: string }; type ReplacementTarget = - | { stringToReplace: string; regexToReplace: never } + | { stringToReplace: string; regexToReplace?: never } | { - stringToReplace: never; + stringToReplace?: never; /** When putting regex into json, you have to use an extra backslash to escape your regex backslashes because JSON also treats backslash as an escape character */ regexToReplace: string; }; diff --git a/test/convert/replacements.test.ts b/test/convert/replacements.test.ts index 1c3f12c9c5..e2c4122aaf 100644 --- a/test/convert/replacements.test.ts +++ b/test/convert/replacements.test.ts @@ -11,15 +11,16 @@ import { matchingContentFile } from '../mock'; import * as replacementsForMock from '../../src/convert/replacements'; describe('file matching', () => { + const base = { replaceWithEnv: 'foo', stringToReplace: 'foo' }; it('file matches string', () => { - expect(matchesFile('foo', { filename: 'foo' })).to.be.true; - expect(matchesFile('bar', { filename: 'foo' })).to.not.be.true; + expect(matchesFile('foo', { filename: 'foo', ...base })).to.be.true; + expect(matchesFile('bar', { filename: 'foo', ...base })).to.not.be.true; }); it('file matches glob', () => { - expect(matchesFile('foo/bar', { glob: 'foo/**' })).to.be.true; - expect(matchesFile('foo/bar', { glob: 'foo/*' })).to.be.true; - expect(matchesFile('foo/bar', { glob: 'foo' })).to.be.false; - expect(matchesFile('foo/bar', { glob: '**/*' })).to.be.true; + expect(matchesFile('foo/bar', { glob: 'foo/**', ...base })).to.be.true; + expect(matchesFile('foo/bar', { glob: 'foo/*', ...base })).to.be.true; + expect(matchesFile('foo/bar', { glob: 'foo', ...base })).to.be.false; + expect(matchesFile('foo/bar', { glob: '**/*', ...base })).to.be.true; }); it('test absolute vs. relative paths'); }); From c968eb1fbedbdf6a8d886d89486b8c7b1777bb95 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 09:58:18 -0500 Subject: [PATCH 14/41] test: expect global replace --- test/convert/replacements.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/convert/replacements.test.ts b/test/convert/replacements.test.ts index e2c4122aaf..3cdaca3f28 100644 --- a/test/convert/replacements.test.ts +++ b/test/convert/replacements.test.ts @@ -76,7 +76,7 @@ describe('marking replacements on a component', () => { expect(result).to.deep.equal({ [cmp.xml]: [ { - toReplace: /.*foo.*/, + toReplace: /.*foo.*/g, replaceWith: 'bar', }, ], From 94bfee976d48c04c44ba13554249923dde47a076 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 11:23:46 -0500 Subject: [PATCH 15/41] fix: check windows path for glob --- src/convert/replacements.ts | 44 ++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/convert/replacements.ts b/src/convert/replacements.ts index 07e4b96c8c..b4e50662dc 100644 --- a/src/convert/replacements.ts +++ b/src/convert/replacements.ts @@ -4,7 +4,6 @@ * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -import { join } from 'path'; import { readFile } from 'fs/promises'; import { Transform, Readable } from 'stream'; import { Lifecycle, SfError, SfProject } from '@salesforce/core'; @@ -69,15 +68,6 @@ export const replacementIterations = async (input: string, replacements: MarkedR return output; }; -/** - * Read the `replacement` property from sfdx-project.json - */ -const readReplacementsFromProject = async (): Promise => { - const proj = await SfProject.resolve(); - const projJson = (await proj.resolveProjectConfig()) as { replacements?: ReplacementConfig[] }; - return projJson.replacements; -}; - /** * Reads the project, gets replacements, removes an that aren't applicable due ot environment conditionals, and returns an instance of the ReplacementMarkingStream */ @@ -133,17 +123,6 @@ export const getContents = async (path: string): Promise => { return fileContentsCache.get(path); }; -/** - * Regardless of any components, return the ReplacementConfig that are valid with the current env. - * These can be checked globally and don't need to be checked per component. - */ -const envFilter = (replacementConfigs: ReplacementConfig[] = []): ReplacementConfig[] => - replacementConfigs.filter( - (replacement) => - !replacement.replaceWhenEnv || - replacement.replaceWhenEnv.every((envConditional) => process.env[envConditional.env] === envConditional.value) - ); - /** * Build the replacements property for a sourceComponent */ @@ -180,8 +159,20 @@ export const getReplacements = async ( export const matchesFile = (f: string, r: ReplacementConfig): boolean => // filenames will be absolute. We don't have convenient access to the pkgDirs, // so we need to be more open than an exact match - f.endsWith(r.filename) || (r.glob && minimatch(f, join('**', r.glob))); + f.endsWith(r.filename) || (r.glob && minimatch(f, `**/${r.glob}`)); +/** + * Regardless of any components, return the ReplacementConfig that are valid with the current env. + * These can be checked globally and don't need to be checked per component. + */ +const envFilter = (replacementConfigs: ReplacementConfig[] = []): ReplacementConfig[] => + replacementConfigs.filter( + (replacement) => + !replacement.replaceWhenEnv || + replacement.replaceWhenEnv.every((envConditional) => process.env[envConditional.env] === envConditional.value) + ); + +/** A "getter" for envs to implement the warning when an expected env is not present */ const getEnvValue = (env: string): string => { if (process.env[env]) { return process.env[env]; @@ -190,3 +181,12 @@ const getEnvValue = (env: string): string => { `"${env}" is in sfdx-project.json as a value for "replaceWithEnv" property, but it's not set in your environment.` ); }; + +/** + * Read the `replacement` property from sfdx-project.json + */ +const readReplacementsFromProject = async (): Promise => { + const proj = await SfProject.resolve(); + const projJson = (await proj.resolveProjectConfig()) as { replacements?: ReplacementConfig[] }; + return projJson.replacements; +}; From 9aa6ff7cd296e248621facd5be868b06c87a0002 Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Tue, 25 Oct 2022 16:37:23 +0000 Subject: [PATCH 16/41] test: record perf --- .../eda.json | 10 +++++----- .../lotsOfClasses.json | 10 +++++----- .../lotsOfClassesOneDir.json | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/eda.json index d9c9c02db7..22766497d6 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 329.62463900004514 + "duration": 346.8111230000213 }, { "name": "sourceToMdapi", - "duration": 7505.23844700004 + "duration": 5533.629581999994 }, { "name": "sourceToZip", - "duration": 6470.010548999999 + "duration": 5044.335148000013 }, { "name": "mdapiToSource", - "duration": 5877.771837000037 + "duration": 5773.252553999977 } -] +] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClasses.json index 89695e54d3..8ac8ce28cb 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 644.1402209999505 + "duration": 702.3725049999775 }, { "name": "sourceToMdapi", - "duration": 11856.592849999957 + "duration": 12299.356131000037 }, { "name": "sourceToZip", - "duration": 10338.099250999978 + "duration": 9477.310148000019 }, { "name": "mdapiToSource", - "duration": 8291.250971000001 + "duration": 8208.13733399997 } -] +] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClassesOneDir.json index da21a2c969..b27c3da4bb 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 1090.1220250000479 + "duration": 1136.5274949999875 }, { "name": "sourceToMdapi", - "duration": 17970.10952900001 + "duration": 19908.283182999992 }, { "name": "sourceToZip", - "duration": 15943.596995999978 + "duration": 14181.441193000006 }, { "name": "mdapiToSource", - "duration": 14545.061661000014 + "duration": 13951.946159999992 } -] +] \ No newline at end of file From 20cc61375852a8c02fe2e032f68a444950ef3941 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 15:37:45 -0500 Subject: [PATCH 17/41] test: more UT for replacements --- test/convert/replacements.test.ts | 62 ++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/test/convert/replacements.test.ts b/test/convert/replacements.test.ts index 3cdaca3f28..312b952c91 100644 --- a/test/convert/replacements.test.ts +++ b/test/convert/replacements.test.ts @@ -6,7 +6,8 @@ */ import { expect } from 'chai'; import Sinon = require('sinon'); -import { getReplacements, matchesFile } from '../../src/convert/replacements'; +import { Lifecycle } from '@salesforce/core'; +import { getReplacements, matchesFile, replacementIterations } from '../../src/convert/replacements'; import { matchingContentFile } from '../mock'; import * as replacementsForMock from '../../src/convert/replacements'; @@ -33,6 +34,10 @@ describe('marking replacements on a component', () => { Sinon.stub(replacementsForMock, 'getContents').resolves('bar'); }); + after(() => { + Sinon.restore(); + }); + process.env.FOO_REPLACEMENT = 'bar'; const cmp = matchingContentFile.COMPONENT; @@ -139,4 +144,59 @@ describe('marking replacements on a component', () => { ], }); }); + it('throws when env is missing'); +}); + +describe('executes replacements on a string', () => { + describe('string', () => { + it('basic replacement', async () => { + expect(await replacementIterations('ThisIsATest', [{ toReplace: 'This', replaceWith: 'That' }])).to.equal( + 'ThatIsATest' + ); + }); + it('same replacement occuring multiple times', async () => { + expect( + await replacementIterations('ThisIsATestWithThisAndThis', [{ toReplace: 'This', replaceWith: 'That' }]) + ).to.equal('ThatIsATestWithThatAndThat'); + }); + it('multiple replacements', async () => { + expect( + await replacementIterations('ThisIsATestWithThisAndThis', [ + { toReplace: 'This', replaceWith: 'That' }, + { toReplace: 'ATest', replaceWith: 'AnAwesomeTest' }, + ]) + ).to.equal('ThatIsAnAwesomeTestWithThatAndThat'); + }); + }); + describe('regex', () => { + it('basic replacement', async () => { + expect(await replacementIterations('ThisIsATest', [{ toReplace: /Is/g, replaceWith: 'IsNot' }])).to.equal( + 'ThisIsNotATest' + ); + }); + it('same replacement occuring multiple times', async () => { + expect( + await replacementIterations('ThisIsATestWithThisAndThis', [{ toReplace: /s/g, replaceWith: 'S' }]) + ).to.equal('ThiSISATeStWithThiSAndThiS'); + }); + it('multiple replacements', async () => { + expect( + await replacementIterations('This Is A Test With This And This', [ + { toReplace: /^T.{2}s/, replaceWith: 'That' }, + { toReplace: /T.{2}s$/, replaceWith: 'Stuff' }, + ]) + ).to.equal('That Is A Test With This And Stuff'); + }); + }); + + describe('warning when no replacement happened', () => { + it('emits warning only when no change', async () => { + const warnSpy = Sinon.spy(Lifecycle.getInstance(), 'emitWarning'); + await replacementIterations('ThisIsATest', [{ toReplace: 'Nope', replaceWith: 'Nah' }]); + expect(warnSpy.calledOnce).to.be.true; + await replacementIterations('ThisIsATest', [{ toReplace: 'Test', replaceWith: 'SpyTest' }]); + expect(warnSpy.calledOnce).to.be.true; + warnSpy.restore(); + }); + }); }); From d06b90d9fb5b7152f997dd746141a9c77d3e38ef Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 15:42:45 -0500 Subject: [PATCH 18/41] test: e2e nuts for conversions with replacements --- .github/workflows/test.yml | 11 +++ package.json | 1 + .../local/replacements/replacements.nut.ts | 90 +++++++++++++++++++ .../testProj/config/project-scratch-def.json | 13 +++ .../main/default/classes/replaceStuff.cls | 11 +++ .../default/classes/replaceStuff.cls-meta.xml | 5 ++ .../TestObj__c/TestObj__c.object-meta.xml | 21 +++++ .../fields/FieldA__c.field-meta.xml | 12 +++ .../staticresources/Test.resource-meta.xml | 7 ++ .../staticresources/Test/folder/test2.css | 1 + .../default/staticresources/Test/test.css | 1 + .../local/replacements/testProj/label.txt | 1 + .../replacements/testProj/replacements.txt | 1 + .../replacements/testProj/sfdx-project.json | 44 +++++++++ 14 files changed, 219 insertions(+) create mode 100644 test/nuts/local/replacements/replacements.nut.ts create mode 100644 test/nuts/local/replacements/testProj/config/project-scratch-def.json create mode 100644 test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls create mode 100644 test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls-meta.xml create mode 100644 test/nuts/local/replacements/testProj/force-app/main/default/objects/TestObj__c/TestObj__c.object-meta.xml create mode 100644 test/nuts/local/replacements/testProj/force-app/main/default/objects/TestObj__c/fields/FieldA__c.field-meta.xml create mode 100644 test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test.resource-meta.xml create mode 100644 test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test/folder/test2.css create mode 100644 test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test/test.css create mode 100644 test/nuts/local/replacements/testProj/label.txt create mode 100644 test/nuts/local/replacements/testProj/replacements.txt create mode 100644 test/nuts/local/replacements/testProj/sfdx-project.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba8dea1767..588c3551ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,6 +11,17 @@ on: jobs: unit-tests: uses: salesforcecli/github-workflows/.github/workflows/unitTest.yml@main + nuts: + needs: unit-tests + uses: salesforcecli/github-workflows/.github/workflows/nut.yml@main + secrets: inherit + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + fail-fast: false + with: + os: ${{ matrix.os }} + perf-scale-nuts-linux: uses: ./.github/workflows/perfScaleNut.yml needs: unit-tests diff --git a/package.json b/package.json index 3fae4c461b..dde7bc497d 100644 --- a/package.json +++ b/package.json @@ -100,6 +100,7 @@ "pretest": "sf-compile-test", "repl": "node --inspect ./scripts/repl.js", "test": "sf-test", + "test:nuts": "mocha \"test/nuts/local/**/*.nut.ts\" --timeout 500000", "test:nuts:scale": "mocha \"test/nuts/scale/eda.nut.ts\" --timeout 500000; mocha \"test/nuts/scale/lotsOfClasses.nut.ts\" --timeout 500000; mocha \"test/nuts/scale/lotsOfClassesOneDir.nut.ts\" --timeout 500000", "test:nuts:scale:record": "yarn test:nuts:scale && git add . && git commit -m \"test: record perf [ci skip]\" --no-verify && git push --no-verify", "test:registry": "mocha ./test/registry/registryCompleteness.test.ts --timeout 50000", diff --git a/test/nuts/local/replacements/replacements.nut.ts b/test/nuts/local/replacements/replacements.nut.ts new file mode 100644 index 0000000000..07855698f3 --- /dev/null +++ b/test/nuts/local/replacements/replacements.nut.ts @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2020, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import * as path from 'path'; +import * as fs from 'fs'; +import { Open } from 'unzipper'; +import { TestSession } from '@salesforce/cli-plugins-testkit'; +import { expect } from 'chai'; +import { ComponentSetBuilder, MetadataConverter } from '../../../../src'; + +describe('e2e replacements test', () => { + let session: TestSession; + + before(async () => { + session = await TestSession.create({ + project: { + sourceDir: path.join('test', 'nuts', 'local', 'replacements', 'testProj'), + }, + authStrategy: 'NONE', + }); + // Hack: rewrite the file replacement locations relative to the project + const projectJsonPath = path.join(session.project.dir, 'sfdx-project.json'); + const original = await fs.promises.readFile(projectJsonPath, 'utf8'); + await fs.promises.writeFile( + projectJsonPath, + original + .replace('replacements.txt', path.join(session.project.dir, 'replacements.txt')) + .replace('label.txt', path.join(session.project.dir, 'label.txt')) + ); + }); + + after(async () => { + // await session?.clean(); + }); + + describe('various types of replacements', () => { + it('converts a componentSet built from the testProj to a zip', async () => { + process.env.THE_REPLACEMENT = 'foo'; + const converter = new MetadataConverter(); + const cs = await ComponentSetBuilder.build({ sourcepath: [path.join(session.project.dir, 'force-app')] }); + const { zipBuffer } = await converter.convert(cs, 'metadata', { + type: 'zip', + }); + expect(zipBuffer).to.not.be.undefined; + await (await Open.buffer(zipBuffer)).extract({ path: path.join(session.project.dir, 'unzipped') }); + }); + + it('class replacements as expected', async () => { + const classContents = await fs.promises.readFile( + path.join(session.project.dir, 'unzipped', 'classes', 'replaceStuff.cls'), + 'utf8' + ); + expect(classContents).to.not.include('replaceMeWithEnv'); + expect(classContents).to.not.include('replaceMeWithFile'); + expect(classContents).to.not.include('replaceEachOfTheseValuesWithAValueFromTheEnvUsingRegex'); + expect(classContents).to.include('foo'); + expect(classContents).to.include( + await fs.promises.readFile(path.join(session.project.dir, 'replacements.txt'), 'utf8') + ); + expect(classContents).to.include('foo'); + }); + it('decomposed object replacements as expected', async () => { + const objectContents = await fs.promises.readFile( + path.join(session.project.dir, 'unzipped', 'objects', 'TestObj__c.object'), + 'utf8' + ); + expect(objectContents).to.not.include('placeholder'); + expect(objectContents).to.include('foo'); + expect(objectContents).to.include( + await fs.promises.readFile(path.join(session.project.dir, 'label.txt'), 'utf8') + ); + }); + it('static resource object replacements as expected', async () => { + const files = ( + await Open.file(path.join(session.project.dir, 'unzipped', 'staticresources', 'Test.resource')) + ).files.filter((f) => f.type === 'File'); + + const buffers = await Promise.all(files.map(async (f) => f.buffer())); + buffers + .map((b) => b.toString()) + .map((contents) => { + expect(contents).to.not.include('placeholder'); + expect(contents).to.include('foo'); + }); + }); + }); +}); diff --git a/test/nuts/local/replacements/testProj/config/project-scratch-def.json b/test/nuts/local/replacements/testProj/config/project-scratch-def.json new file mode 100644 index 0000000000..760f654ba0 --- /dev/null +++ b/test/nuts/local/replacements/testProj/config/project-scratch-def.json @@ -0,0 +1,13 @@ +{ + "orgName": "shane.mclaughlin company", + "edition": "Developer", + "features": ["EnableSetPasswordInApi"], + "settings": { + "lightningExperienceSettings": { + "enableS1DesktopEnabled": true + }, + "mobileSettings": { + "enableS1EncryptedStoragePref2": false + } + } +} diff --git a/test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls b/test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls new file mode 100644 index 0000000000..cc95e3e650 --- /dev/null +++ b/test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls @@ -0,0 +1,11 @@ +public with sharing class replaceStuff { + // replaceMeWithEnv + // replaceMeWithFile + + // replaceEachOfTheseValuesWithAValueFromTheEnvUsingRegex + // replaceEachOfTheseValuesWithAValueFromTheEnvUsingRegex + // replaceEachOfTheseValuesWithAValueFromTheEnvUsingRegex + public replaceStuff() { + + } +} diff --git a/test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls-meta.xml b/test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls-meta.xml new file mode 100644 index 0000000000..4b0bc9f387 --- /dev/null +++ b/test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls-meta.xml @@ -0,0 +1,5 @@ + + + 55.0 + Active + diff --git a/test/nuts/local/replacements/testProj/force-app/main/default/objects/TestObj__c/TestObj__c.object-meta.xml b/test/nuts/local/replacements/testProj/force-app/main/default/objects/TestObj__c/TestObj__c.object-meta.xml new file mode 100644 index 0000000000..5d5498adb5 --- /dev/null +++ b/test/nuts/local/replacements/testProj/force-app/main/default/objects/TestObj__c/TestObj__c.object-meta.xml @@ -0,0 +1,21 @@ + + + true + true + true + true + true + true + true + true + TestObjs + TestObj__c + placeholder + ReadWrite + + Text + + + Deployed + + diff --git a/test/nuts/local/replacements/testProj/force-app/main/default/objects/TestObj__c/fields/FieldA__c.field-meta.xml b/test/nuts/local/replacements/testProj/force-app/main/default/objects/TestObj__c/fields/FieldA__c.field-meta.xml new file mode 100644 index 0000000000..6f0dc262d5 --- /dev/null +++ b/test/nuts/local/replacements/testProj/force-app/main/default/objects/TestObj__c/fields/FieldA__c.field-meta.xml @@ -0,0 +1,12 @@ + + + FieldA__c + Text + 255 + placeholder + + false + false + Public + + diff --git a/test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test.resource-meta.xml b/test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test.resource-meta.xml new file mode 100644 index 0000000000..2d18c2446f --- /dev/null +++ b/test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test.resource-meta.xml @@ -0,0 +1,7 @@ + + + Private + application/zip + added from sfdx plugin + Test + \ No newline at end of file diff --git a/test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test/folder/test2.css b/test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test/folder/test2.css new file mode 100644 index 0000000000..d053e6fda2 --- /dev/null +++ b/test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test/folder/test2.css @@ -0,0 +1 @@ +/* This is a placeholder */ diff --git a/test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test/test.css b/test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test/test.css new file mode 100644 index 0000000000..d053e6fda2 --- /dev/null +++ b/test/nuts/local/replacements/testProj/force-app/main/default/staticresources/Test/test.css @@ -0,0 +1 @@ +/* This is a placeholder */ diff --git a/test/nuts/local/replacements/testProj/label.txt b/test/nuts/local/replacements/testProj/label.txt new file mode 100644 index 0000000000..d82154b7ef --- /dev/null +++ b/test/nuts/local/replacements/testProj/label.txt @@ -0,0 +1 @@ + diff --git a/test/nuts/local/replacements/testProj/replacements.txt b/test/nuts/local/replacements/testProj/replacements.txt new file mode 100644 index 0000000000..91f2add8fc --- /dev/null +++ b/test/nuts/local/replacements/testProj/replacements.txt @@ -0,0 +1 @@ +this is a longer comment that's going to replace something in a metadata file diff --git a/test/nuts/local/replacements/testProj/sfdx-project.json b/test/nuts/local/replacements/testProj/sfdx-project.json new file mode 100644 index 0000000000..3806966302 --- /dev/null +++ b/test/nuts/local/replacements/testProj/sfdx-project.json @@ -0,0 +1,44 @@ +{ + "packageDirectories": [ + { + "path": "force-app", + "default": true + } + ], + "name": "replacementTest", + "namespace": "", + "sfdcLoginUrl": "https://login.salesforce.com", + "sourceApiVersion": "55.0", + "replacements": [ + { + "glob": "force-app/**/*.cls", + "stringToReplace": "replaceMeWithEnv", + "replaceWithEnv": "THE_REPLACEMENT" + }, + { + "glob": "force-app/**/*.cls", + "stringToReplace": "replaceMeWithFile", + "replaceWithFile": "replacements.txt" + }, + { + "glob": "force-app/**/*.cls", + "regexToReplace": "\\b.*Regex", + "replaceWithEnv": "THE_REPLACEMENT" + }, + { + "glob": "force-app/main/default/objects/**/*", + "stringToReplace": "placeholder", + "replaceWithEnv": "THE_REPLACEMENT" + }, + { + "glob": "**/*.field-meta.xml", + "stringToReplace": "", + "replaceWithFile": "label.txt" + }, + { + "glob": "**/*.css", + "stringToReplace": "placeholder", + "replaceWithEnv": "THE_REPLACEMENT" + } + ] +} From 9c5b95cd388f6307c35656d3ea1c329d3c555233 Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Tue, 25 Oct 2022 20:52:16 +0000 Subject: [PATCH 19/41] test: record perf --- .../eda.json | 8 ++++---- .../lotsOfClasses.json | 8 ++++---- .../lotsOfClassesOneDir.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json index 31e8ffefed..8520dafb01 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 214.46215100004338 + "duration": 217.24560199998086 }, { "name": "sourceToMdapi", - "duration": 6232.359846999985 + "duration": 6034.749643999967 }, { "name": "sourceToZip", - "duration": 4866.491668000002 + "duration": 4773.798967999988 }, { "name": "mdapiToSource", - "duration": 4188.821088000026 + "duration": 3753.1327789999777 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json index e0d411e22f..d9271c466f 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 428.76088600000367 + "duration": 445.2519559999928 }, { "name": "sourceToMdapi", - "duration": 8602.956126999983 + "duration": 8093.97652299999 }, { "name": "sourceToZip", - "duration": 6805.6596850000205 + "duration": 6444.346208999981 }, { "name": "mdapiToSource", - "duration": 5127.636436000001 + "duration": 4924.401616999996 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json index b74a7235de..efa553c905 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 776.4044480000157 + "duration": 751.4006550000049 }, { "name": "sourceToMdapi", - "duration": 12145.154593000014 + "duration": 11578.667857999972 }, { "name": "sourceToZip", - "duration": 11030.154324000003 + "duration": 9852.379058999999 }, { "name": "mdapiToSource", - "duration": 12548.084606999997 + "duration": 8332.220241000003 } ] \ No newline at end of file From 83de18bd361e87cb20a3bc8fe1bb8f4833a3c21a Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Tue, 25 Oct 2022 21:03:23 +0000 Subject: [PATCH 20/41] test: record perf --- .../eda.json | 8 ++++---- .../lotsOfClasses.json | 8 ++++---- .../lotsOfClassesOneDir.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json index 02f05fb8c1..c3c7699eeb 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 256.72601899999427 + "duration": 236.53314699999464 }, { "name": "sourceToMdapi", - "duration": 6796.90185200001 + "duration": 7124.251390000005 }, { "name": "sourceToZip", - "duration": 5362.048718999984 + "duration": 4977.3159419999865 }, { "name": "mdapiToSource", - "duration": 5302.37870999999 + "duration": 4778.834580999974 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json index 3130730968..677d6b2f78 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 552.7289729999902 + "duration": 467.00805299999774 }, { "name": "sourceToMdapi", - "duration": 11819.851255999994 + "duration": 9261.286629999988 }, { "name": "sourceToZip", - "duration": 8429.234218000027 + "duration": 7447.448482000007 }, { "name": "mdapiToSource", - "duration": 6296.675203000021 + "duration": 5548.334333000006 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json index 0d2a950086..50c96e3b71 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 946.1013220000314 + "duration": 820.5652899999986 }, { "name": "sourceToMdapi", - "duration": 15312.147095999971 + "duration": 12836.822408999986 }, { "name": "sourceToZip", - "duration": 13901.202661000018 + "duration": 11242.880424000003 }, { "name": "mdapiToSource", - "duration": 12138.095666000037 + "duration": 13836.042866999982 } ] \ No newline at end of file From c9aba8ddb1967d856ef6414b106382fee782a8e1 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 17:22:44 -0500 Subject: [PATCH 21/41] test: positive and negative coverage for whenEnv --- .../local/replacements/replacements.nut.ts | 3 +++ .../main/default/classes/replaceStuff.cls | 3 +++ .../replacements/testProj/sfdx-project.json | 22 +++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/test/nuts/local/replacements/replacements.nut.ts b/test/nuts/local/replacements/replacements.nut.ts index 07855698f3..c2b3125ca7 100644 --- a/test/nuts/local/replacements/replacements.nut.ts +++ b/test/nuts/local/replacements/replacements.nut.ts @@ -61,6 +61,9 @@ describe('e2e replacements test', () => { await fs.promises.readFile(path.join(session.project.dir, 'replacements.txt'), 'utf8') ); expect(classContents).to.include('foo'); + + expect(classContents).to.include('doNotReplaceThis'); + expect(classContents).to.not.include('conditionallyReplaceThis'); }); it('decomposed object replacements as expected', async () => { const objectContents = await fs.promises.readFile( diff --git a/test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls b/test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls index cc95e3e650..b45f737881 100644 --- a/test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls +++ b/test/nuts/local/replacements/testProj/force-app/main/default/classes/replaceStuff.cls @@ -5,6 +5,9 @@ public with sharing class replaceStuff { // replaceEachOfTheseValuesWithAValueFromTheEnvUsingRegex // replaceEachOfTheseValuesWithAValueFromTheEnvUsingRegex // replaceEachOfTheseValuesWithAValueFromTheEnvUsingRegex + + // doNotReplaceThis + // conditionallyReplaceThis public replaceStuff() { } diff --git a/test/nuts/local/replacements/testProj/sfdx-project.json b/test/nuts/local/replacements/testProj/sfdx-project.json index 3806966302..30a894d891 100644 --- a/test/nuts/local/replacements/testProj/sfdx-project.json +++ b/test/nuts/local/replacements/testProj/sfdx-project.json @@ -15,6 +15,28 @@ "stringToReplace": "replaceMeWithEnv", "replaceWithEnv": "THE_REPLACEMENT" }, + { + "glob": "force-app/**/*.cls", + "stringToReplace": "doNotReplaceThis", + "replaceWithEnv": "THE_REPLACEMENT", + "replaceWhenEnv": [ + { + "env": "THE_REPLACEMENT", + "value": "bar" + } + ] + }, + { + "glob": "force-app/**/*.cls", + "stringToReplace": "conditionallyReplaceThis", + "replaceWithEnv": "THE_REPLACEMENT", + "replaceWhenEnv": [ + { + "env": "THE_REPLACEMENT", + "value": "foo" + } + ] + }, { "glob": "force-app/**/*.cls", "stringToReplace": "replaceMeWithFile", From 9d65449ebb705c990edb465872235993dc0dea5c Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Tue, 25 Oct 2022 22:43:47 +0000 Subject: [PATCH 22/41] test: record perf --- .../eda.json | 8 ++++---- .../lotsOfClasses.json | 8 ++++---- .../lotsOfClassesOneDir.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json index c3c7699eeb..3f0dc36b69 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 236.53314699999464 + "duration": 257.30675800004974 }, { "name": "sourceToMdapi", - "duration": 7124.251390000005 + "duration": 6828.252761000011 }, { "name": "sourceToZip", - "duration": 4977.3159419999865 + "duration": 5002.58094200003 }, { "name": "mdapiToSource", - "duration": 4778.834580999974 + "duration": 4595.628456000006 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json index 677d6b2f78..e093264791 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 467.00805299999774 + "duration": 533.6999279999873 }, { "name": "sourceToMdapi", - "duration": 9261.286629999988 + "duration": 9686.690252 }, { "name": "sourceToZip", - "duration": 7447.448482000007 + "duration": 8595.064142999996 }, { "name": "mdapiToSource", - "duration": 5548.334333000006 + "duration": 6881.827874999959 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json index 50c96e3b71..b3f6bb327b 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 820.5652899999986 + "duration": 856.0232480000122 }, { "name": "sourceToMdapi", - "duration": 12836.822408999986 + "duration": 13682.971561999992 }, { "name": "sourceToZip", - "duration": 11242.880424000003 + "duration": 11930.237198000017 }, { "name": "mdapiToSource", - "duration": 13836.042866999982 + "duration": 15782.31124900002 } ] \ No newline at end of file From c308f4911385258e8fb3891fb3591e6dff9aa17d Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 17:52:49 -0500 Subject: [PATCH 23/41] refactor: pr feedback --- src/convert/replacements.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/convert/replacements.ts b/src/convert/replacements.ts index b4e50662dc..73bc2cd430 100644 --- a/src/convert/replacements.ts +++ b/src/convert/replacements.ts @@ -129,29 +129,36 @@ export const getContents = async (path: string): Promise => { export const getReplacements = async ( cmp: SourceComponent, replacementConfigs: ReplacementConfig[] = [] -): Promise => { +): Promise => { // all possible filenames for this component const filenames = [cmp.xml, ...cmp.walkContent()].filter(Boolean); - // eslint-disable-next-line no-console const replacementsForComponent = ( await Promise.all( + // build a nested array that can be run through Object.fromEntries + // one MarkedReplacement[] for each file in the component filenames.map( async (f): Promise<[string, MarkedReplacement[]]> => [ f, await Promise.all( replacementConfigs + // filter out any that don't match the current file .filter((r) => matchesFile(f, r)) .map(async (r) => ({ + // Config is json which might use the regex. If so, turn it into an actual regex toReplace: r.stringToReplace ?? new RegExp(r.regexToReplace, 'g'), + // get the literal replacement (either from env or file contents) replaceWith: r.replaceWithEnv ? getEnvValue(r.replaceWithEnv) : await getContents(r.replaceWithFile), })) ), ] ) ) - ).filter(([, replacements]) => replacements.length > 0); + ) + // filter out any that don't have any replacements + .filter(([, replacements]) => replacements.length > 0); if (replacementsForComponent.length) { + // turn into a Dictionary-style object so it's easier to lookup by filename return Object.fromEntries(replacementsForComponent); } }; From 97613442e49fc0ec051dece4546cfaabb9c329fb Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Tue, 25 Oct 2022 23:03:26 +0000 Subject: [PATCH 24/41] test: record perf --- .../eda.json | 10 +++++----- .../lotsOfClasses.json | 10 +++++----- .../lotsOfClassesOneDir.json | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/eda.json index e4a6d2f73d..fc9f6e2537 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 197.51509400000214 + "duration": 208.11863400001312 }, { "name": "sourceToMdapi", - "duration": 5877.151366999999 + "duration": 5460.954419000016 }, { "name": "sourceToZip", - "duration": 5290.5017290000105 + "duration": 4563.612835999986 }, { "name": "mdapiToSource", - "duration": 6807.484392000013 + "duration": 3369.516849000007 } -] +] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClasses.json index cad20d624b..6cbd7391db 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 395.9298290000006 + "duration": 386.58763799996814 }, { "name": "sourceToMdapi", - "duration": 7620.608355000004 + "duration": 7342.486907000013 }, { "name": "sourceToZip", - "duration": 7224.629343000008 + "duration": 6253.100625000021 }, { "name": "mdapiToSource", - "duration": 6781.243828000006 + "duration": 3802.610989000008 } -] +] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClassesOneDir.json index bbcf0c93c7..0e84ef797d 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 712.6349019999907 + "duration": 662.6731090000249 }, { "name": "sourceToMdapi", - "duration": 10835.973329 + "duration": 9887.500337000005 }, { "name": "sourceToZip", - "duration": 9996.543145999982 + "duration": 10186.69299499999 }, { "name": "mdapiToSource", - "duration": 13979.638694000023 + "duration": 12108.392731999978 } -] +] \ No newline at end of file From 869ec0cd453c0d09b7d554af66be962e77abe3ab Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 18:14:21 -0500 Subject: [PATCH 25/41] chore: restore clean [no ci] --- test/nuts/local/replacements/replacements.nut.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nuts/local/replacements/replacements.nut.ts b/test/nuts/local/replacements/replacements.nut.ts index c2b3125ca7..343e296d13 100644 --- a/test/nuts/local/replacements/replacements.nut.ts +++ b/test/nuts/local/replacements/replacements.nut.ts @@ -33,7 +33,7 @@ describe('e2e replacements test', () => { }); after(async () => { - // await session?.clean(); + await session?.clean(); }); describe('various types of replacements', () => { From f2c456c987d2a7d8eb4ee9021dc1847d02006326 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 18:32:19 -0500 Subject: [PATCH 26/41] test: handle windows paths better --- test/nuts/local/replacements/replacements.nut.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/nuts/local/replacements/replacements.nut.ts b/test/nuts/local/replacements/replacements.nut.ts index 343e296d13..fdd6591794 100644 --- a/test/nuts/local/replacements/replacements.nut.ts +++ b/test/nuts/local/replacements/replacements.nut.ts @@ -27,8 +27,12 @@ describe('e2e replacements test', () => { await fs.promises.writeFile( projectJsonPath, original - .replace('replacements.txt', path.join(session.project.dir, 'replacements.txt')) - .replace('label.txt', path.join(session.project.dir, 'label.txt')) + // we're putting this in a json file which doesnt like windows backslashes. The file will require posix paths + .replace( + 'replacements.txt', + path.join(session.project.dir, 'replacements.txt').split(path.sep).join(path.posix.sep) + ) + .replace('label.txt', path.join(session.project.dir, 'label.txt').split(path.sep).join(path.posix.sep)) ); }); From 9419aeeb11ac292284dfdb0b48bd6d4c3266d859 Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Tue, 25 Oct 2022 23:41:38 +0000 Subject: [PATCH 27/41] test: record perf --- .../eda.json | 8 ++++---- .../lotsOfClasses.json | 8 ++++---- .../lotsOfClassesOneDir.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/eda.json index 1b43fc0144..7929dcb85f 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 328.08031600000686 + "duration": 324.7964989999891 }, { "name": "sourceToMdapi", - "duration": 7152.719090000028 + "duration": 7310.667111000017 }, { "name": "sourceToZip", - "duration": 6235.375627000001 + "duration": 6250.426998999988 }, { "name": "mdapiToSource", - "duration": 5907.137515000009 + "duration": 5462.673781999998 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClasses.json index d9990b0787..1d70dc2dc5 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 666.5169450000394 + "duration": 664.4817089999851 }, { "name": "sourceToMdapi", - "duration": 12308.666303000005 + "duration": 11463.153294000018 }, { "name": "sourceToZip", - "duration": 9254.614367999951 + "duration": 10041.71440299999 }, { "name": "mdapiToSource", - "duration": 10135.46397099999 + "duration": 9396.318423999997 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClassesOneDir.json index aa6f4e3bca..c1cc84ac97 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 1145.7118510000291 + "duration": 1068.983653000003 }, { "name": "sourceToMdapi", - "duration": 17694.542747 + "duration": 15904.999413000012 }, { "name": "sourceToZip", - "duration": 15521.322923999978 + "duration": 15449.051027999958 }, { "name": "mdapiToSource", - "duration": 13688.250294000027 + "duration": 14017.64948100003 } ] \ No newline at end of file From 08157d2bd1c46ba7e02fbabd5659a8f0aaa13009 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Tue, 25 Oct 2022 19:05:06 -0500 Subject: [PATCH 28/41] test(temp): windows ut and nuts --- .github/workflows/test.yml | 209 +++++++++++++++--------------- src/convert/replacements.ts | 2 +- test/convert/replacements.test.ts | 9 +- 3 files changed, 114 insertions(+), 106 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 588c3551ff..f1beb702b9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,119 +12,120 @@ jobs: unit-tests: uses: salesforcecli/github-workflows/.github/workflows/unitTest.yml@main nuts: - needs: unit-tests + # needs: unit-tests uses: salesforcecli/github-workflows/.github/workflows/nut.yml@main secrets: inherit strategy: matrix: - os: [ubuntu-latest, windows-latest] + # os: [ubuntu-latest, windows-latest] + os: [windows-latest] fail-fast: false with: os: ${{ matrix.os }} - perf-scale-nuts-linux: - uses: ./.github/workflows/perfScaleNut.yml - needs: unit-tests - perf-scale-nuts-windows: - uses: ./.github/workflows/perfScaleNut.yml - needs: unit-tests - with: - os: 'windows-latest' + # perf-scale-nuts-linux: + # uses: ./.github/workflows/perfScaleNut.yml + # needs: unit-tests + # perf-scale-nuts-windows: + # uses: ./.github/workflows/perfScaleNut.yml + # needs: unit-tests + # with: + # os: 'windows-latest' - # run a quick nut on each OS to populate the cache - # the following is highly duplicative to allow linux to start all the nuts without waiting for windows primer - extNuts-primer-linux: - name: extNUTs-linux-prime - needs: unit-tests - uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main - with: - packageName: '@salesforce/source-deploy-retrieve' - externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' - command: 'yarn test:nuts:manifest:create' - preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' - postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' - preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' - os: 'ubuntu-latest' - secrets: inherit + # # run a quick nut on each OS to populate the cache + # # the following is highly duplicative to allow linux to start all the nuts without waiting for windows primer + # extNuts-primer-linux: + # name: extNUTs-linux-prime + # needs: unit-tests + # uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main + # with: + # packageName: '@salesforce/source-deploy-retrieve' + # externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' + # command: 'yarn test:nuts:manifest:create' + # preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' + # postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' + # preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' + # os: 'ubuntu-latest' + # secrets: inherit - extNuts-primer-windows: - name: extNUTs-windows-prime - needs: unit-tests - uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main - with: - packageName: '@salesforce/source-deploy-retrieve' - externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' - command: 'yarn test:nuts:manifest:create' - preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' - postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' - preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' - os: 'windows-latest' - secrets: inherit + # extNuts-primer-windows: + # name: extNUTs-windows-prime + # needs: unit-tests + # uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main + # with: + # packageName: '@salesforce/source-deploy-retrieve' + # externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' + # command: 'yarn test:nuts:manifest:create' + # preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' + # postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' + # preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' + # os: 'windows-latest' + # secrets: inherit - # now run the rest of the nuts for each os via cached version of al the setup steps - external-nuts-full-linux: - needs: [perf-scale-nuts-linux, extNuts-primer-linux] - name: extNUTs-linux - strategy: - fail-fast: false - matrix: - command: - - 'yarn test:nuts:convert' - - 'yarn test:nuts:delete' - - 'yarn test:nuts:deploy:async' - - 'yarn test:nuts:deploy:destructive' - - 'yarn test:nuts:deploy:manifest' - - 'yarn test:nuts:deploy:metadata' - - 'yarn test:nuts:deploy:quick' - - 'yarn test:nuts:deploy:rest' - - 'yarn test:nuts:deploy:sourcepath' - - 'yarn test:nuts:deploy:testlevel' - - 'yarn test:nuts:mdapi' - - 'yarn test:nuts:retrieve' - - 'yarn test:nuts:specialTypes' - - 'yarn test:nuts:tracking' - uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main - with: - packageName: '@salesforce/source-deploy-retrieve' - externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' - command: ${{matrix.command}} - # the next 3 lines are likely unnecessary due to cache use - preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' - postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' - preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' - os: ubuntu-latest - secrets: inherit + # # now run the rest of the nuts for each os via cached version of al the setup steps + # external-nuts-full-linux: + # needs: [perf-scale-nuts-linux, extNuts-primer-linux] + # name: extNUTs-linux + # strategy: + # fail-fast: false + # matrix: + # command: + # - 'yarn test:nuts:convert' + # - 'yarn test:nuts:delete' + # - 'yarn test:nuts:deploy:async' + # - 'yarn test:nuts:deploy:destructive' + # - 'yarn test:nuts:deploy:manifest' + # - 'yarn test:nuts:deploy:metadata' + # - 'yarn test:nuts:deploy:quick' + # - 'yarn test:nuts:deploy:rest' + # - 'yarn test:nuts:deploy:sourcepath' + # - 'yarn test:nuts:deploy:testlevel' + # - 'yarn test:nuts:mdapi' + # - 'yarn test:nuts:retrieve' + # - 'yarn test:nuts:specialTypes' + # - 'yarn test:nuts:tracking' + # uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main + # with: + # packageName: '@salesforce/source-deploy-retrieve' + # externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' + # command: ${{matrix.command}} + # # the next 3 lines are likely unnecessary due to cache use + # preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' + # postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' + # preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' + # os: ubuntu-latest + # secrets: inherit - # now run the rest of the nuts for each os via cached version of all the setup steps - external-nuts-full-windows: - needs: [perf-scale-nuts-windows, extNuts-primer-windows] - name: extNUTs-windows - strategy: - fail-fast: false - matrix: - command: - - 'yarn test:nuts:convert' - - 'yarn test:nuts:delete' - - 'yarn test:nuts:deploy:async' - - 'yarn test:nuts:deploy:destructive' - - 'yarn test:nuts:deploy:manifest' - - 'yarn test:nuts:deploy:metadata' - - 'yarn test:nuts:deploy:quick' - - 'yarn test:nuts:deploy:rest' - - 'yarn test:nuts:deploy:sourcepath' - - 'yarn test:nuts:deploy:testlevel' - - 'yarn test:nuts:mdapi' - - 'yarn test:nuts:retrieve' - - 'yarn test:nuts:specialTypes' - - 'yarn test:nuts:tracking' - uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main - with: - packageName: '@salesforce/source-deploy-retrieve' - externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' - command: ${{matrix.command}} - # the next 3 lines are likely unnecessary due to cache use - preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' - postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' - preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' - os: windows-latest - secrets: inherit + # # now run the rest of the nuts for each os via cached version of all the setup steps + # external-nuts-full-windows: + # needs: [perf-scale-nuts-windows, extNuts-primer-windows] + # name: extNUTs-windows + # strategy: + # fail-fast: false + # matrix: + # command: + # - 'yarn test:nuts:convert' + # - 'yarn test:nuts:delete' + # - 'yarn test:nuts:deploy:async' + # - 'yarn test:nuts:deploy:destructive' + # - 'yarn test:nuts:deploy:manifest' + # - 'yarn test:nuts:deploy:metadata' + # - 'yarn test:nuts:deploy:quick' + # - 'yarn test:nuts:deploy:rest' + # - 'yarn test:nuts:deploy:sourcepath' + # - 'yarn test:nuts:deploy:testlevel' + # - 'yarn test:nuts:mdapi' + # - 'yarn test:nuts:retrieve' + # - 'yarn test:nuts:specialTypes' + # - 'yarn test:nuts:tracking' + # uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main + # with: + # packageName: '@salesforce/source-deploy-retrieve' + # externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' + # command: ${{matrix.command}} + # # the next 3 lines are likely unnecessary due to cache use + # preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' + # postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' + # preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' + # os: windows-latest + # secrets: inherit diff --git a/src/convert/replacements.ts b/src/convert/replacements.ts index 73bc2cd430..db13a45c55 100644 --- a/src/convert/replacements.ts +++ b/src/convert/replacements.ts @@ -179,7 +179,7 @@ const envFilter = (replacementConfigs: ReplacementConfig[] = []): ReplacementCon replacement.replaceWhenEnv.every((envConditional) => process.env[envConditional.env] === envConditional.value) ); -/** A "getter" for envs to implement the warning when an expected env is not present */ +/** A "getter" for envs to throw an error when an expected env is not present */ const getEnvValue = (env: string): string => { if (process.env[env]) { return process.env[env]; diff --git a/test/convert/replacements.test.ts b/test/convert/replacements.test.ts index 312b952c91..740141d53f 100644 --- a/test/convert/replacements.test.ts +++ b/test/convert/replacements.test.ts @@ -4,6 +4,7 @@ * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ +import * as path from 'path'; import { expect } from 'chai'; import Sinon = require('sinon'); import { Lifecycle } from '@salesforce/core'; @@ -17,12 +18,18 @@ describe('file matching', () => { expect(matchesFile('foo', { filename: 'foo', ...base })).to.be.true; expect(matchesFile('bar', { filename: 'foo', ...base })).to.not.be.true; }); - it('file matches glob', () => { + it('file matches glob (posix paths)', () => { expect(matchesFile('foo/bar', { glob: 'foo/**', ...base })).to.be.true; expect(matchesFile('foo/bar', { glob: 'foo/*', ...base })).to.be.true; expect(matchesFile('foo/bar', { glob: 'foo', ...base })).to.be.false; expect(matchesFile('foo/bar', { glob: '**/*', ...base })).to.be.true; }); + it('file matches glob (os-dependet paths)', () => { + expect(matchesFile(path.join('foo', 'bar'), { glob: 'foo/**', ...base })).to.be.true; + expect(matchesFile(path.join('foo', 'bar'), { glob: 'foo/*', ...base })).to.be.true; + expect(matchesFile(path.join('foo', 'bar'), { glob: 'foo', ...base })).to.be.false; + expect(matchesFile(path.join('foo', 'bar'), { glob: '**/*', ...base })).to.be.true; + }); it('test absolute vs. relative paths'); }); From 8c33b68926928dd8518c97753efcdaf31e4acc0a Mon Sep 17 00:00:00 2001 From: mshanemc Date: Wed, 26 Oct 2022 09:05:00 -0500 Subject: [PATCH 29/41] test: file trimming to avoid newline issues --- src/convert/replacements.ts | 8 +++++--- test/convert/replacements.test.ts | 2 +- test/nuts/local/replacements/replacements.nut.ts | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/convert/replacements.ts b/src/convert/replacements.ts index db13a45c55..fbbae082f9 100644 --- a/src/convert/replacements.ts +++ b/src/convert/replacements.ts @@ -110,10 +110,10 @@ class ReplacementMarkingStream extends Transform { } } -export const getContents = async (path: string): Promise => { +export const getContentsOfReplacementFile = async (path: string): Promise => { if (!fileContentsCache.has(path)) { try { - fileContentsCache.set(path, await readFile(path, 'utf8')); + fileContentsCache.set(path, (await readFile(path, 'utf8')).trim()); } catch (e) { throw new SfError( `The file "${path}" specified in the "replacements" property of sfdx-project.json could not be read.` @@ -147,7 +147,9 @@ export const getReplacements = async ( // Config is json which might use the regex. If so, turn it into an actual regex toReplace: r.stringToReplace ?? new RegExp(r.regexToReplace, 'g'), // get the literal replacement (either from env or file contents) - replaceWith: r.replaceWithEnv ? getEnvValue(r.replaceWithEnv) : await getContents(r.replaceWithFile), + replaceWith: r.replaceWithEnv + ? getEnvValue(r.replaceWithEnv) + : await getContentsOfReplacementFile(r.replaceWithFile), })) ), ] diff --git a/test/convert/replacements.test.ts b/test/convert/replacements.test.ts index 740141d53f..19d7dd6a65 100644 --- a/test/convert/replacements.test.ts +++ b/test/convert/replacements.test.ts @@ -24,7 +24,7 @@ describe('file matching', () => { expect(matchesFile('foo/bar', { glob: 'foo', ...base })).to.be.false; expect(matchesFile('foo/bar', { glob: '**/*', ...base })).to.be.true; }); - it('file matches glob (os-dependet paths)', () => { + it('file matches glob (os-dependent paths)', () => { expect(matchesFile(path.join('foo', 'bar'), { glob: 'foo/**', ...base })).to.be.true; expect(matchesFile(path.join('foo', 'bar'), { glob: 'foo/*', ...base })).to.be.true; expect(matchesFile(path.join('foo', 'bar'), { glob: 'foo', ...base })).to.be.false; diff --git a/test/nuts/local/replacements/replacements.nut.ts b/test/nuts/local/replacements/replacements.nut.ts index fdd6591794..d109a21a79 100644 --- a/test/nuts/local/replacements/replacements.nut.ts +++ b/test/nuts/local/replacements/replacements.nut.ts @@ -62,7 +62,7 @@ describe('e2e replacements test', () => { expect(classContents).to.not.include('replaceEachOfTheseValuesWithAValueFromTheEnvUsingRegex'); expect(classContents).to.include('foo'); expect(classContents).to.include( - await fs.promises.readFile(path.join(session.project.dir, 'replacements.txt'), 'utf8') + (await fs.promises.readFile(path.join(session.project.dir, 'replacements.txt'), 'utf8')).trim() ); expect(classContents).to.include('foo'); @@ -77,7 +77,7 @@ describe('e2e replacements test', () => { expect(objectContents).to.not.include('placeholder'); expect(objectContents).to.include('foo'); expect(objectContents).to.include( - await fs.promises.readFile(path.join(session.project.dir, 'label.txt'), 'utf8') + (await fs.promises.readFile(path.join(session.project.dir, 'label.txt'), 'utf8')).trim() ); }); it('static resource object replacements as expected', async () => { From 5e63ae9f77963e12020c46d51209354d4a6c7124 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Wed, 26 Oct 2022 09:08:20 -0500 Subject: [PATCH 30/41] test: rename getContents --- test/convert/replacements.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/convert/replacements.test.ts b/test/convert/replacements.test.ts index 19d7dd6a65..e499dd8c25 100644 --- a/test/convert/replacements.test.ts +++ b/test/convert/replacements.test.ts @@ -38,7 +38,7 @@ describe('env filters', () => {}); describe('marking replacements on a component', () => { before(() => { // replaceFromFile uses the contents of a file. This prevents the test from hitting real FS for that. - Sinon.stub(replacementsForMock, 'getContents').resolves('bar'); + Sinon.stub(replacementsForMock, 'getContentsOfReplacementFile').resolves('bar'); }); after(() => { From e13f08fe38f3f722f06ef836e4838eb2fc00cfcb Mon Sep 17 00:00:00 2001 From: mshanemc Date: Wed, 26 Oct 2022 09:29:05 -0500 Subject: [PATCH 31/41] test: restore full test suite --- .github/workflows/test.yml | 208 ++++++++++++++++++------------------- 1 file changed, 103 insertions(+), 105 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f1beb702b9..3f4d2b5e7e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,120 +12,118 @@ jobs: unit-tests: uses: salesforcecli/github-workflows/.github/workflows/unitTest.yml@main nuts: - # needs: unit-tests uses: salesforcecli/github-workflows/.github/workflows/nut.yml@main secrets: inherit strategy: matrix: - # os: [ubuntu-latest, windows-latest] - os: [windows-latest] + os: [ubuntu-latest, windows-latest] fail-fast: false with: os: ${{ matrix.os }} - # perf-scale-nuts-linux: - # uses: ./.github/workflows/perfScaleNut.yml - # needs: unit-tests - # perf-scale-nuts-windows: - # uses: ./.github/workflows/perfScaleNut.yml - # needs: unit-tests - # with: - # os: 'windows-latest' + perf-scale-nuts-linux: + uses: ./.github/workflows/perfScaleNut.yml + needs: [unit-tests, nuts] + perf-scale-nuts-windows: + uses: ./.github/workflows/perfScaleNut.yml + needs: [unit-tests, nuts] + with: + os: 'windows-latest' - # # run a quick nut on each OS to populate the cache - # # the following is highly duplicative to allow linux to start all the nuts without waiting for windows primer - # extNuts-primer-linux: - # name: extNUTs-linux-prime - # needs: unit-tests - # uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main - # with: - # packageName: '@salesforce/source-deploy-retrieve' - # externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' - # command: 'yarn test:nuts:manifest:create' - # preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' - # postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' - # preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' - # os: 'ubuntu-latest' - # secrets: inherit + # run a quick nut on each OS to populate the cache + # the following is highly duplicative to allow linux to start all the nuts without waiting for windows primer + extNuts-primer-linux: + name: extNUTs-linux-prime + needs: [unit-tests, nuts] + uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main + with: + packageName: '@salesforce/source-deploy-retrieve' + externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' + command: 'yarn test:nuts:manifest:create' + preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' + postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' + preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' + os: 'ubuntu-latest' + secrets: inherit - # extNuts-primer-windows: - # name: extNUTs-windows-prime - # needs: unit-tests - # uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main - # with: - # packageName: '@salesforce/source-deploy-retrieve' - # externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' - # command: 'yarn test:nuts:manifest:create' - # preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' - # postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' - # preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' - # os: 'windows-latest' - # secrets: inherit + extNuts-primer-windows: + name: extNUTs-windows-prime + needs: [unit-tests, nuts] + uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main + with: + packageName: '@salesforce/source-deploy-retrieve' + externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' + command: 'yarn test:nuts:manifest:create' + preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' + postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' + preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' + os: 'windows-latest' + secrets: inherit - # # now run the rest of the nuts for each os via cached version of al the setup steps - # external-nuts-full-linux: - # needs: [perf-scale-nuts-linux, extNuts-primer-linux] - # name: extNUTs-linux - # strategy: - # fail-fast: false - # matrix: - # command: - # - 'yarn test:nuts:convert' - # - 'yarn test:nuts:delete' - # - 'yarn test:nuts:deploy:async' - # - 'yarn test:nuts:deploy:destructive' - # - 'yarn test:nuts:deploy:manifest' - # - 'yarn test:nuts:deploy:metadata' - # - 'yarn test:nuts:deploy:quick' - # - 'yarn test:nuts:deploy:rest' - # - 'yarn test:nuts:deploy:sourcepath' - # - 'yarn test:nuts:deploy:testlevel' - # - 'yarn test:nuts:mdapi' - # - 'yarn test:nuts:retrieve' - # - 'yarn test:nuts:specialTypes' - # - 'yarn test:nuts:tracking' - # uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main - # with: - # packageName: '@salesforce/source-deploy-retrieve' - # externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' - # command: ${{matrix.command}} - # # the next 3 lines are likely unnecessary due to cache use - # preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' - # postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' - # preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' - # os: ubuntu-latest - # secrets: inherit + # now run the rest of the nuts for each os via cached version of al the setup steps + external-nuts-full-linux: + needs: [perf-scale-nuts-linux, extNuts-primer-linux] + name: extNUTs-linux + strategy: + fail-fast: false + matrix: + command: + - 'yarn test:nuts:convert' + - 'yarn test:nuts:delete' + - 'yarn test:nuts:deploy:async' + - 'yarn test:nuts:deploy:destructive' + - 'yarn test:nuts:deploy:manifest' + - 'yarn test:nuts:deploy:metadata' + - 'yarn test:nuts:deploy:quick' + - 'yarn test:nuts:deploy:rest' + - 'yarn test:nuts:deploy:sourcepath' + - 'yarn test:nuts:deploy:testlevel' + - 'yarn test:nuts:mdapi' + - 'yarn test:nuts:retrieve' + - 'yarn test:nuts:specialTypes' + - 'yarn test:nuts:tracking' + uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main + with: + packageName: '@salesforce/source-deploy-retrieve' + externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' + command: ${{matrix.command}} + # the next 3 lines are likely unnecessary due to cache use + preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' + postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' + preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' + os: ubuntu-latest + secrets: inherit - # # now run the rest of the nuts for each os via cached version of all the setup steps - # external-nuts-full-windows: - # needs: [perf-scale-nuts-windows, extNuts-primer-windows] - # name: extNUTs-windows - # strategy: - # fail-fast: false - # matrix: - # command: - # - 'yarn test:nuts:convert' - # - 'yarn test:nuts:delete' - # - 'yarn test:nuts:deploy:async' - # - 'yarn test:nuts:deploy:destructive' - # - 'yarn test:nuts:deploy:manifest' - # - 'yarn test:nuts:deploy:metadata' - # - 'yarn test:nuts:deploy:quick' - # - 'yarn test:nuts:deploy:rest' - # - 'yarn test:nuts:deploy:sourcepath' - # - 'yarn test:nuts:deploy:testlevel' - # - 'yarn test:nuts:mdapi' - # - 'yarn test:nuts:retrieve' - # - 'yarn test:nuts:specialTypes' - # - 'yarn test:nuts:tracking' - # uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main - # with: - # packageName: '@salesforce/source-deploy-retrieve' - # externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' - # command: ${{matrix.command}} - # # the next 3 lines are likely unnecessary due to cache use - # preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' - # postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' - # preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' - # os: windows-latest - # secrets: inherit + # now run the rest of the nuts for each os via cached version of all the setup steps + external-nuts-full-windows: + needs: [perf-scale-nuts-windows, extNuts-primer-windows] + name: extNUTs-windows + strategy: + fail-fast: false + matrix: + command: + - 'yarn test:nuts:convert' + - 'yarn test:nuts:delete' + - 'yarn test:nuts:deploy:async' + - 'yarn test:nuts:deploy:destructive' + - 'yarn test:nuts:deploy:manifest' + - 'yarn test:nuts:deploy:metadata' + - 'yarn test:nuts:deploy:quick' + - 'yarn test:nuts:deploy:rest' + - 'yarn test:nuts:deploy:sourcepath' + - 'yarn test:nuts:deploy:testlevel' + - 'yarn test:nuts:mdapi' + - 'yarn test:nuts:retrieve' + - 'yarn test:nuts:specialTypes' + - 'yarn test:nuts:tracking' + uses: salesforcecli/github-workflows/.github/workflows/externalNut.yml@main + with: + packageName: '@salesforce/source-deploy-retrieve' + externalProjectGitUrl: 'https://github.com/salesforcecli/plugin-source' + command: ${{matrix.command}} + # the next 3 lines are likely unnecessary due to cache use + preBuildCommands: 'shx rm -rf node_modules/@salesforce/kit; shx rm -rf node_modules/@typescript-eslint; shx rm -rf node_modules/eslint-plugin-header; shx rm -rf node_modules/eslint-plugin-import; shx rm -rf node_modules/eslint-plugin-jsdoc; shx rm -rf node_modules/eslint-plugin-prettier' + postbuildCommands: 'cp src/registry/metadataRegistry.json lib/src/registry' + preExternalBuildCommands: 'shx rm -rf node_modules/@salesforce/source-tracking/node_modules/@salesforce/source-deploy-retrieve' + os: windows-latest + secrets: inherit From bf001bf5572e162dfa1b3fd18a2a9a7119211ec7 Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Wed, 26 Oct 2022 14:43:40 +0000 Subject: [PATCH 32/41] test: record perf --- .../eda.json | 8 ++++---- .../lotsOfClasses.json | 8 ++++---- .../lotsOfClassesOneDir.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json index 3f0dc36b69..5cb5d8eac3 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 257.30675800004974 + "duration": 258.7272329999978 }, { "name": "sourceToMdapi", - "duration": 6828.252761000011 + "duration": 7064.5000979999895 }, { "name": "sourceToZip", - "duration": 5002.58094200003 + "duration": 5043.188347000003 }, { "name": "mdapiToSource", - "duration": 4595.628456000006 + "duration": 4514.134340000019 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json index e093264791..3bb8d75dad 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 533.6999279999873 + "duration": 479.82419299997855 }, { "name": "sourceToMdapi", - "duration": 9686.690252 + "duration": 8941.918268000009 }, { "name": "sourceToZip", - "duration": 8595.064142999996 + "duration": 10533.41592499998 }, { "name": "mdapiToSource", - "duration": 6881.827874999959 + "duration": 5452.256045999995 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json index b3f6bb327b..44ddb1dd6e 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8171M-CPU-2-60GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 856.0232480000122 + "duration": 830.0290470000182 }, { "name": "sourceToMdapi", - "duration": 13682.971561999992 + "duration": 12821.065571999992 }, { "name": "sourceToZip", - "duration": 11930.237198000017 + "duration": 11678.985925000015 }, { "name": "mdapiToSource", - "duration": 15782.31124900002 + "duration": 13113.897208999988 } ] \ No newline at end of file From 5a9bd94c11b4376402d8f6b52d8c33e595ff4624 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Wed, 26 Oct 2022 10:59:27 -0500 Subject: [PATCH 33/41] feat: replacement-not-found warnings don't happen for globs --- src/convert/replacements.ts | 4 +++- src/convert/types.ts | 9 +++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/convert/replacements.ts b/src/convert/replacements.ts index fbbae082f9..546d33f277 100644 --- a/src/convert/replacements.ts +++ b/src/convert/replacements.ts @@ -56,7 +56,7 @@ export const replacementIterations = async (input: string, replacements: MarkedR const regex = typeof replacement.toReplace === 'string' ? new RegExp(replacement.toReplace, 'g') : replacement.toReplace; const replaced = output.replace(regex, replacement.replaceWith); - if (replaced === output) { + if (replacement.singleFile && replaced === output) { // replacements need to be done sequentially // eslint-disable-next-line no-await-in-loop await Lifecycle.getInstance().emitWarning( @@ -144,6 +144,8 @@ export const getReplacements = async ( // filter out any that don't match the current file .filter((r) => matchesFile(f, r)) .map(async (r) => ({ + // used during replacement stream to limit warnings to explicit filenames, not globs + singleFile: Boolean(r.filename), // Config is json which might use the regex. If so, turn it into an actual regex toReplace: r.stringToReplace ?? new RegExp(r.regexToReplace, 'g'), // get the literal replacement (either from env or file contents) diff --git a/src/convert/types.ts b/src/convert/types.ts index f1c338f145..026959dc92 100644 --- a/src/convert/types.ts +++ b/src/convert/types.ts @@ -107,13 +107,14 @@ export type ConvertResult = { }; /** Stored by file on SourceComponent for stream processing */ -export interface MarkedReplacement { +export type MarkedReplacement = { toReplace: string | RegExp; replaceWith: string; -} + singleFile?: boolean; +}; // TODO: what's the right way to get this into core/sfdxProjectJson -export type ReplacementConfig = Location & +export type ReplacementConfig = ReplacementLocation & ReplacementSource & ReplacementTarget & { /** Only do the replacement if ALL of the environment values in this array match */ @@ -125,7 +126,7 @@ export type ReplacementConfig = Location & ]; }; -type Location = { filename: string; glob?: never } | { filename?: never; glob: string }; +type ReplacementLocation = { filename: string; glob?: never } | { filename?: never; glob: string }; type ReplacementSource = | { replaceWithEnv: string; replaceWithFile?: never } | { replaceWithEnv?: never; replaceWithFile: string }; From 87b3003ce1199b91e93c524c7b7b256b4f3b7a13 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Wed, 26 Oct 2022 11:04:38 -0500 Subject: [PATCH 34/41] refactor: use current env var styling --- src/convert/metadataConverter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/convert/metadataConverter.ts b/src/convert/metadataConverter.ts index c4cfbe182e..1fe0f9da7e 100644 --- a/src/convert/metadataConverter.ts +++ b/src/convert/metadataConverter.ts @@ -118,7 +118,7 @@ export class MetadataConverter { const conversionPipeline = pipeline( Readable.from(components), - !targetFormatIsSource && (process.env.DEBUG_REPLACEMENTS_VIA_CONVERT === 'true' || output.type === 'zip') + !targetFormatIsSource && (process.env.SF_APPLY_REPLACEMENTS_ON_CONVERT === 'true' || output.type === 'zip') ? (await getReplacementMarkingStream()) ?? new PassThrough({ objectMode: true }) : new PassThrough({ objectMode: true }), new ComponentConverter(targetFormat, this.registry, mergeSet, defaultDirectory), From 3b2de0454d9dcc6061f2702017a0ac8470083f82 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Wed, 26 Oct 2022 11:50:44 -0500 Subject: [PATCH 35/41] test: singleFile boolean and warning test for glob vs filename --- test/convert/replacements.test.ts | 57 ++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/test/convert/replacements.test.ts b/test/convert/replacements.test.ts index e499dd8c25..ea992f9ed4 100644 --- a/test/convert/replacements.test.ts +++ b/test/convert/replacements.test.ts @@ -65,6 +65,7 @@ describe('marking replacements on a component', () => { { toReplace: 'foo', replaceWith: 'bar', + singleFile: true, }, ], }); @@ -76,6 +77,7 @@ describe('marking replacements on a component', () => { { toReplace: 'foo', replaceWith: 'bar', + singleFile: true, }, ], }); @@ -90,6 +92,7 @@ describe('marking replacements on a component', () => { { toReplace: /.*foo.*/g, replaceWith: 'bar', + singleFile: true, }, ], }); @@ -104,10 +107,12 @@ describe('marking replacements on a component', () => { { toReplace: 'foo', replaceWith: 'bar', + singleFile: true, }, { toReplace: 'baz', replaceWith: 'bar', + singleFile: true, }, ], }); @@ -121,12 +126,14 @@ describe('marking replacements on a component', () => { { toReplace: 'foo', replaceWith: 'bar', + singleFile: false, }, ], [cmp.content]: [ { toReplace: 'foo', replaceWith: 'bar', + singleFile: false, }, ], }); @@ -141,12 +148,14 @@ describe('marking replacements on a component', () => { { toReplace: 'foo', replaceWith: 'bar', + singleFile: true, }, ], [cmp.content]: [ { toReplace: 'foo', replaceWith: 'bar', + singleFile: true, }, ], }); @@ -157,13 +166,15 @@ describe('marking replacements on a component', () => { describe('executes replacements on a string', () => { describe('string', () => { it('basic replacement', async () => { - expect(await replacementIterations('ThisIsATest', [{ toReplace: 'This', replaceWith: 'That' }])).to.equal( - 'ThatIsATest' - ); + expect( + await replacementIterations('ThisIsATest', [{ toReplace: 'This', replaceWith: 'That', singleFile: true }]) + ).to.equal('ThatIsATest'); }); it('same replacement occuring multiple times', async () => { expect( - await replacementIterations('ThisIsATestWithThisAndThis', [{ toReplace: 'This', replaceWith: 'That' }]) + await replacementIterations('ThisIsATestWithThisAndThis', [ + { toReplace: 'This', replaceWith: 'That', singleFile: true }, + ]) ).to.equal('ThatIsATestWithThatAndThat'); }); it('multiple replacements', async () => { @@ -177,33 +188,47 @@ describe('executes replacements on a string', () => { }); describe('regex', () => { it('basic replacement', async () => { - expect(await replacementIterations('ThisIsATest', [{ toReplace: /Is/g, replaceWith: 'IsNot' }])).to.equal( - 'ThisIsNotATest' - ); + expect( + await replacementIterations('ThisIsATest', [{ toReplace: /Is/g, replaceWith: 'IsNot', singleFile: true }]) + ).to.equal('ThisIsNotATest'); }); it('same replacement occuring multiple times', async () => { expect( - await replacementIterations('ThisIsATestWithThisAndThis', [{ toReplace: /s/g, replaceWith: 'S' }]) + await replacementIterations('ThisIsATestWithThisAndThis', [ + { toReplace: /s/g, replaceWith: 'S', singleFile: true }, + ]) ).to.equal('ThiSISATeStWithThiSAndThiS'); }); it('multiple replacements', async () => { expect( await replacementIterations('This Is A Test With This And This', [ - { toReplace: /^T.{2}s/, replaceWith: 'That' }, - { toReplace: /T.{2}s$/, replaceWith: 'Stuff' }, + { toReplace: /^T.{2}s/, replaceWith: 'That', singleFile: false }, + { toReplace: /T.{2}s$/, replaceWith: 'Stuff', singleFile: false }, ]) ).to.equal('That Is A Test With This And Stuff'); }); }); describe('warning when no replacement happened', () => { - it('emits warning only when no change', async () => { - const warnSpy = Sinon.spy(Lifecycle.getInstance(), 'emitWarning'); - await replacementIterations('ThisIsATest', [{ toReplace: 'Nope', replaceWith: 'Nah' }]); - expect(warnSpy.calledOnce).to.be.true; - await replacementIterations('ThisIsATest', [{ toReplace: 'Test', replaceWith: 'SpyTest' }]); - expect(warnSpy.calledOnce).to.be.true; + let warnSpy: Sinon.SinonSpy; + + beforeEach(() => { + warnSpy = Sinon.spy(Lifecycle.getInstance(), 'emitWarning'); + }); + afterEach(() => { warnSpy.restore(); }); + it('emits warning only when no change', async () => { + await replacementIterations('ThisIsATest', [{ toReplace: 'Nope', replaceWith: 'Nah', singleFile: true }]); + expect(warnSpy.callCount).to.equal(1); + }); + it('no warning when string is replaced', async () => { + await replacementIterations('ThisIsATest', [{ toReplace: 'Test', replaceWith: 'SpyTest', singleFile: true }]); + expect(warnSpy.callCount).to.equal(0); + }); + it('no warning when no replacement but not a single file (ex: glob)', async () => { + await replacementIterations('ThisIsATest', [{ toReplace: 'Nope', replaceWith: 'Nah', singleFile: false }]); + expect(warnSpy.callCount).to.equal(0); + }); }); }); From 6a0c464aa15125a5433e17d9dc4b6b7cf8a3284a Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Wed, 26 Oct 2022 17:06:09 +0000 Subject: [PATCH 36/41] test: record perf --- .../eda.json | 8 ++++---- .../lotsOfClasses.json | 8 ++++---- .../lotsOfClassesOneDir.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/eda.json index fc9f6e2537..384e83d52f 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 208.11863400001312 + "duration": 199.40888699999778 }, { "name": "sourceToMdapi", - "duration": 5460.954419000016 + "duration": 5154.8034650000045 }, { "name": "sourceToZip", - "duration": 4563.612835999986 + "duration": 4552.947791000013 }, { "name": "mdapiToSource", - "duration": 3369.516849000007 + "duration": 3261.958102000004 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClasses.json index 6cbd7391db..a379e75f44 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 386.58763799996814 + "duration": 389.6657969999942 }, { "name": "sourceToMdapi", - "duration": 7342.486907000013 + "duration": 7406.009941000026 }, { "name": "sourceToZip", - "duration": 6253.100625000021 + "duration": 9112.750708999985 }, { "name": "mdapiToSource", - "duration": 3802.610989000008 + "duration": 3721.0864500000025 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClassesOneDir.json index 0e84ef797d..50502c68d2 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8370C-CPU-2-80GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 662.6731090000249 + "duration": 676.9866699999548 }, { "name": "sourceToMdapi", - "duration": 9887.500337000005 + "duration": 10331.149006000021 }, { "name": "sourceToZip", - "duration": 10186.69299499999 + "duration": 8735.021562999988 }, { "name": "mdapiToSource", - "duration": 12108.392731999978 + "duration": 7454.696846999985 } ] \ No newline at end of file From a3f92cb5c30c27154074266f4819b1593e213afe Mon Sep 17 00:00:00 2001 From: Shane McLaughlin Date: Fri, 28 Oct 2022 14:11:04 -0500 Subject: [PATCH 37/41] feat: replacements in output (#752) * chore: auto-update metadata coverage in METADATA_SUPPORT.md * test: use filenames from mocks * test: record perf Co-authored-by: svc-cli-bot --- METADATA_SUPPORT.md | 4 +- src/client/metadataApiDeploy.ts | 33 +++++++++--- src/client/types.ts | 1 - src/convert/replacements.ts | 21 ++++++-- src/convert/types.ts | 6 +++ test/convert/replacements.test.ts | 50 +++++++++++++++---- .../eda.json | 10 ++-- .../lotsOfClasses.json | 10 ++-- .../lotsOfClassesOneDir.json | 10 ++-- 9 files changed, 104 insertions(+), 41 deletions(-) diff --git a/METADATA_SUPPORT.md b/METADATA_SUPPORT.md index 7172c7051e..53a953104f 100644 --- a/METADATA_SUPPORT.md +++ b/METADATA_SUPPORT.md @@ -533,6 +533,7 @@ v57 introduces the following new types. Here's their current level of support |ActionableListDefinition|❌|Not supported, but support could be added| |AffinityScoreDefinition|❌|Not supported, but support could be added| |ClauseCatgConfiguration|❌|Not supported, but support could be added| +|CommerceRuleSettings|βœ…|| |DisclosureDefinition|❌|Not supported, but support could be added| |DisclosureDefinitionVersion|❌|Not supported, but support could be added| |DisclosureType|❌|Not supported, but support could be added| @@ -540,8 +541,9 @@ v57 introduces the following new types. Here's their current level of support |ExternalClientAppSettings|βœ…|| |ExternalClientApplication|βœ…|| |ExternalDocStorageConfig|❌|Not supported, but support could be added| -|ExtlClntAppMobileSet|❌|Not supported, but support could be added| +|ExtlClntAppMobileSettings|βœ…|| |ExtlClntAppOauthPlcyCnfg|❌|Not supported, but support could be added| +|ExtlClntAppOauthSettings|βœ…|| |IdentityProviderSettings|βœ…|| |IntegrationProviderDef|❌|Not supported, but support could be added| |LocationUse|❌|Not supported, but support could be added| diff --git a/src/client/metadataApiDeploy.ts b/src/client/metadataApiDeploy.ts index e5d50d89f3..f1fadbfa5d 100644 --- a/src/client/metadataApiDeploy.ts +++ b/src/client/metadataApiDeploy.ts @@ -10,6 +10,7 @@ import { create as createArchive } from 'archiver'; import * as fs from 'graceful-fs'; import { Lifecycle, Messages, SfError } from '@salesforce/core'; import { ensureArray } from '@salesforce/kit'; +import { ReplacementEvent } from '../convert/types'; import { MetadataConverter } from '../convert'; import { ComponentLike, SourceComponent } from '../resolve'; import { ComponentSet } from '../collections'; @@ -31,16 +32,15 @@ Messages.importMessagesDirectory(__dirname); const messages = Messages.load('@salesforce/source-deploy-retrieve', 'sdr', ['error_no_job_id']); export class DeployResult implements MetadataTransferResult { - public readonly response: MetadataApiDeployStatus; - public readonly components: ComponentSet; private readonly diagnosticUtil = new DiagnosticUtil('metadata'); private fileResponses: FileResponse[]; private readonly shouldConvertPaths = sep !== posix.sep; - public constructor(response: MetadataApiDeployStatus, components: ComponentSet) { - this.response = response; - this.components = components; - } + public constructor( + public readonly response: MetadataApiDeployStatus, + public readonly components: ComponentSet, + public readonly replacements: Map = new Map() + ) {} public getFileResponses(): FileResponse[] { // this involves FS operations, so only perform once! @@ -236,6 +236,7 @@ export class MetadataApiDeploy extends MetadataTransfer = new Map(); private orgId: string; // Keep track of rest deploys separately since Connection.deploy() removes it // from the apiOptions and we need it for telemetry. @@ -310,6 +311,7 @@ export class MetadataApiDeploy extends MetadataTransfer { + const LifecycleInstance = Lifecycle.getInstance(); const connection = await this.getConnection(); // store for use in the scopedPostDeploy event this.orgId = connection.getAuthInfoFields().orgId; @@ -320,11 +322,26 @@ export class MetadataApiDeploy extends MetadataTransfer + // lifecycle have to be async, so wrapped in a promise + new Promise((resolve) => { + if (!this.replacements.has(replacement.filename)) { + this.replacements.set(replacement.filename, [replacement.replaced]); + } else { + this.replacements.get(replacement.filename).push(replacement.replaced); + } + resolve(); + }) + ); + const [zipBuffer] = await Promise.all([this.getZipBuffer(), this.maybeSaveTempDirectory('metadata')]); // SDR modifies what the mdapi expects by adding a rest param const { rest, ...optionsWithoutRest } = this.options.apiOptions; @@ -370,7 +387,7 @@ export class MetadataApiDeploy extends MetadataTransfer(); @@ -50,20 +50,30 @@ class ReplacementStream extends Transform { * emits warnings when an expected replacement target isn't found */ export const replacementIterations = async (input: string, replacements: MarkedReplacement[]): Promise => { + const lifecycleInstance = Lifecycle.getInstance(); let output = input; for (const replacement of replacements) { // TODO: node 16+ has String.replaceAll for non-regex scenarios const regex = typeof replacement.toReplace === 'string' ? new RegExp(replacement.toReplace, 'g') : replacement.toReplace; const replaced = output.replace(regex, replacement.replaceWith); - if (replacement.singleFile && replaced === output) { + + if (replaced !== output) { + output = replaced; + // eslint-disable-next-line no-await-in-loop + await lifecycleInstance.emit('replacement', { + filename: replacement.matchedFilename, + replaced: replacement.toReplace.toString(), + } as ReplacementEvent); + } else if (replacement.singleFile) { // replacements need to be done sequentially // eslint-disable-next-line no-await-in-loop - await Lifecycle.getInstance().emitWarning( - `Your sfdx-project.json specifies that ${replacement.toReplace.toString()} should be replaced, but it was not found.` + await lifecycleInstance.emitWarning( + `Your sfdx-project.json specifies that ${replacement.toReplace.toString()} should be replaced in ${ + replacement.matchedFilename + }, but it was not found.` ); } - output = replaced; } return output; }; @@ -144,6 +154,7 @@ export const getReplacements = async ( // filter out any that don't match the current file .filter((r) => matchesFile(f, r)) .map(async (r) => ({ + matchedFilename: f, // used during replacement stream to limit warnings to explicit filenames, not globs singleFile: Boolean(r.filename), // Config is json which might use the regex. If so, turn it into an actual regex diff --git a/src/convert/types.ts b/src/convert/types.ts index 026959dc92..2be5c60053 100644 --- a/src/convert/types.ts +++ b/src/convert/types.ts @@ -110,6 +110,7 @@ export type ConvertResult = { export type MarkedReplacement = { toReplace: string | RegExp; replaceWith: string; + matchedFilename: string; singleFile?: boolean; }; @@ -138,3 +139,8 @@ type ReplacementTarget = /** When putting regex into json, you have to use an extra backslash to escape your regex backslashes because JSON also treats backslash as an escape character */ regexToReplace: string; }; + +export type ReplacementEvent = { + filename: string; + replaced: string; +}; diff --git a/test/convert/replacements.test.ts b/test/convert/replacements.test.ts index ea992f9ed4..f7d575fb2f 100644 --- a/test/convert/replacements.test.ts +++ b/test/convert/replacements.test.ts @@ -63,6 +63,7 @@ describe('marking replacements on a component', () => { expect(result).to.deep.equal({ [cmp.xml]: [ { + matchedFilename: cmp.xml, toReplace: 'foo', replaceWith: 'bar', singleFile: true, @@ -75,6 +76,7 @@ describe('marking replacements on a component', () => { expect(result).to.deep.equal({ [cmp.xml]: [ { + matchedFilename: cmp.xml, toReplace: 'foo', replaceWith: 'bar', singleFile: true, @@ -90,6 +92,7 @@ describe('marking replacements on a component', () => { expect(result).to.deep.equal({ [cmp.xml]: [ { + matchedFilename: cmp.xml, toReplace: /.*foo.*/g, replaceWith: 'bar', singleFile: true, @@ -105,11 +108,13 @@ describe('marking replacements on a component', () => { expect(result).to.deep.equal({ [cmp.xml]: [ { + matchedFilename: cmp.xml, toReplace: 'foo', replaceWith: 'bar', singleFile: true, }, { + matchedFilename: cmp.xml, toReplace: 'baz', replaceWith: 'bar', singleFile: true, @@ -124,6 +129,7 @@ describe('marking replacements on a component', () => { expect(result).to.deep.equal({ [cmp.xml]: [ { + matchedFilename: cmp.xml, toReplace: 'foo', replaceWith: 'bar', singleFile: false, @@ -131,6 +137,7 @@ describe('marking replacements on a component', () => { ], [cmp.content]: [ { + matchedFilename: cmp.content, toReplace: 'foo', replaceWith: 'bar', singleFile: false, @@ -146,6 +153,7 @@ describe('marking replacements on a component', () => { expect(result).to.deep.equal({ [cmp.xml]: [ { + matchedFilename: cmp.xml, toReplace: 'foo', replaceWith: 'bar', singleFile: true, @@ -153,6 +161,7 @@ describe('marking replacements on a component', () => { ], [cmp.content]: [ { + matchedFilename: cmp.content, toReplace: 'foo', replaceWith: 'bar', singleFile: true, @@ -164,24 +173,27 @@ describe('marking replacements on a component', () => { }); describe('executes replacements on a string', () => { + const matchedFilename = 'foo'; describe('string', () => { it('basic replacement', async () => { expect( - await replacementIterations('ThisIsATest', [{ toReplace: 'This', replaceWith: 'That', singleFile: true }]) + await replacementIterations('ThisIsATest', [ + { matchedFilename, toReplace: 'This', replaceWith: 'That', singleFile: true }, + ]) ).to.equal('ThatIsATest'); }); it('same replacement occuring multiple times', async () => { expect( await replacementIterations('ThisIsATestWithThisAndThis', [ - { toReplace: 'This', replaceWith: 'That', singleFile: true }, + { matchedFilename, toReplace: 'This', replaceWith: 'That', singleFile: true }, ]) ).to.equal('ThatIsATestWithThatAndThat'); }); it('multiple replacements', async () => { expect( await replacementIterations('ThisIsATestWithThisAndThis', [ - { toReplace: 'This', replaceWith: 'That' }, - { toReplace: 'ATest', replaceWith: 'AnAwesomeTest' }, + { matchedFilename, toReplace: 'This', replaceWith: 'That' }, + { matchedFilename, toReplace: 'ATest', replaceWith: 'AnAwesomeTest' }, ]) ).to.equal('ThatIsAnAwesomeTestWithThatAndThat'); }); @@ -189,21 +201,23 @@ describe('executes replacements on a string', () => { describe('regex', () => { it('basic replacement', async () => { expect( - await replacementIterations('ThisIsATest', [{ toReplace: /Is/g, replaceWith: 'IsNot', singleFile: true }]) + await replacementIterations('ThisIsATest', [ + { toReplace: /Is/g, replaceWith: 'IsNot', singleFile: true, matchedFilename }, + ]) ).to.equal('ThisIsNotATest'); }); it('same replacement occuring multiple times', async () => { expect( await replacementIterations('ThisIsATestWithThisAndThis', [ - { toReplace: /s/g, replaceWith: 'S', singleFile: true }, + { toReplace: /s/g, replaceWith: 'S', singleFile: true, matchedFilename }, ]) ).to.equal('ThiSISATeStWithThiSAndThiS'); }); it('multiple replacements', async () => { expect( await replacementIterations('This Is A Test With This And This', [ - { toReplace: /^T.{2}s/, replaceWith: 'That', singleFile: false }, - { toReplace: /T.{2}s$/, replaceWith: 'Stuff', singleFile: false }, + { toReplace: /^T.{2}s/, replaceWith: 'That', singleFile: false, matchedFilename }, + { toReplace: /T.{2}s$/, replaceWith: 'Stuff', singleFile: false, matchedFilename }, ]) ).to.equal('That Is A Test With This And Stuff'); }); @@ -211,24 +225,38 @@ describe('executes replacements on a string', () => { describe('warning when no replacement happened', () => { let warnSpy: Sinon.SinonSpy; + let emitSpy: Sinon.SinonSpy; beforeEach(() => { + // everything is an emit. Warn calls emit, too. warnSpy = Sinon.spy(Lifecycle.getInstance(), 'emitWarning'); + emitSpy = Sinon.spy(Lifecycle.getInstance(), 'emit'); }); afterEach(() => { warnSpy.restore(); + emitSpy.restore(); }); it('emits warning only when no change', async () => { - await replacementIterations('ThisIsATest', [{ toReplace: 'Nope', replaceWith: 'Nah', singleFile: true }]); + await replacementIterations('ThisIsATest', [ + { toReplace: 'Nope', replaceWith: 'Nah', singleFile: true, matchedFilename }, + ]); expect(warnSpy.callCount).to.equal(1); + expect(emitSpy.callCount).to.equal(1); }); it('no warning when string is replaced', async () => { - await replacementIterations('ThisIsATest', [{ toReplace: 'Test', replaceWith: 'SpyTest', singleFile: true }]); + await replacementIterations('ThisIsATest', [ + { toReplace: 'Test', replaceWith: 'SpyTest', singleFile: true, matchedFilename }, + ]); expect(warnSpy.callCount).to.equal(0); + // because it emits the replacement event + expect(emitSpy.callCount).to.equal(1); }); it('no warning when no replacement but not a single file (ex: glob)', async () => { - await replacementIterations('ThisIsATest', [{ toReplace: 'Nope', replaceWith: 'Nah', singleFile: false }]); + await replacementIterations('ThisIsATest', [ + { toReplace: 'Nope', replaceWith: 'Nah', singleFile: false, matchedFilename }, + ]); expect(warnSpy.callCount).to.equal(0); + expect(emitSpy.callCount).to.equal(0); }); }); }); diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json index 0f72f60780..bca290887b 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 217.24560199998086 + "duration": 217.22331100000883 }, { "name": "sourceToMdapi", - "duration": 6034.749643999967 + "duration": 5528.8957320000045 }, { "name": "sourceToZip", - "duration": 4773.798967999988 + "duration": 4973.436105999979 }, { "name": "mdapiToSource", - "duration": 3753.1327789999777 + "duration": 3836.91319500003 } -] +] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json index 3b8eae7955..57c2e7e73b 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 445.2519559999928 + "duration": 412.3087709999527 }, { "name": "sourceToMdapi", - "duration": 8093.97652299999 + "duration": 7418.153514000005 }, { "name": "sourceToZip", - "duration": 6444.346208999981 + "duration": 6142.985255000007 }, { "name": "mdapiToSource", - "duration": 4924.401616999996 + "duration": 4409.731972999987 } -] +] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json index 2862ad2241..64679058f6 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 751.4006550000049 + "duration": 702.3643789999769 }, { "name": "sourceToMdapi", - "duration": 11578.667857999972 + "duration": 11670.428872999968 }, { "name": "sourceToZip", - "duration": 9852.379058999999 + "duration": 10227.489082999993 }, { "name": "mdapiToSource", - "duration": 8332.220241000003 + "duration": 10798.030672999972 } -] +] \ No newline at end of file From 10345119fb68936ec550d30dc31610fb90e478a3 Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Fri, 28 Oct 2022 19:21:20 +0000 Subject: [PATCH 38/41] test: record perf --- .../eda.json | 8 ++++---- .../lotsOfClasses.json | 8 ++++---- .../lotsOfClassesOneDir.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/eda.json index 22766497d6..8bf5d980a7 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 346.8111230000213 + "duration": 324.71387500001583 }, { "name": "sourceToMdapi", - "duration": 5533.629581999994 + "duration": 7122.961764000007 }, { "name": "sourceToZip", - "duration": 5044.335148000013 + "duration": 6274.939941999997 }, { "name": "mdapiToSource", - "duration": 5773.252553999977 + "duration": 5758.478280999989 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClasses.json index 8ac8ce28cb..1a280577eb 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 702.3725049999775 + "duration": 651.9775850000151 }, { "name": "sourceToMdapi", - "duration": 12299.356131000037 + "duration": 10889.311952999997 }, { "name": "sourceToZip", - "duration": 9477.310148000019 + "duration": 9392.743359000015 }, { "name": "mdapiToSource", - "duration": 8208.13733399997 + "duration": 7095.85508899999 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClassesOneDir.json index b27c3da4bb..a41ec885e7 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v4-2-30GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 1136.5274949999875 + "duration": 1033.0352329999732 }, { "name": "sourceToMdapi", - "duration": 19908.283182999992 + "duration": 16734.752318999992 }, { "name": "sourceToZip", - "duration": 14181.441193000006 + "duration": 15027.624977999978 }, { "name": "mdapiToSource", - "duration": 13951.946159999992 + "duration": 12297.485982999991 } ] \ No newline at end of file From 4086069490cf9fe7764d600618013e646d62ad79 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Fri, 28 Oct 2022 14:46:53 -0500 Subject: [PATCH 39/41] fix: escaping on string to regex conversion --- src/convert/replacements.ts | 10 ++++++++-- src/convert/types.ts | 2 +- test/convert/replacements.test.ts | 32 +++++++++++++++---------------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/convert/replacements.ts b/src/convert/replacements.ts index 070811e105..70e5a55734 100644 --- a/src/convert/replacements.ts +++ b/src/convert/replacements.ts @@ -79,7 +79,7 @@ export const replacementIterations = async (input: string, replacements: MarkedR }; /** - * Reads the project, gets replacements, removes an that aren't applicable due ot environment conditionals, and returns an instance of the ReplacementMarkingStream + * Reads the project, gets replacements, removes any that aren't applicable due to environment conditionals, and returns an instance of the ReplacementMarkingStream */ export const getReplacementMarkingStream = async (): Promise => { // remove any that don't agree with current env @@ -158,7 +158,7 @@ export const getReplacements = async ( // used during replacement stream to limit warnings to explicit filenames, not globs singleFile: Boolean(r.filename), // Config is json which might use the regex. If so, turn it into an actual regex - toReplace: r.stringToReplace ?? new RegExp(r.regexToReplace, 'g'), + toReplace: r.stringToReplace ? stringToRegex(r.stringToReplace) : new RegExp(r.regexToReplace, 'g'), // get the literal replacement (either from env or file contents) replaceWith: r.replaceWithEnv ? getEnvValue(r.replaceWithEnv) @@ -212,3 +212,9 @@ const readReplacementsFromProject = async (): Promise => { const projJson = (await proj.resolveProjectConfig()) as { replacements?: ReplacementConfig[] }; return projJson.replacements; }; + +/** escape any special characters used in the string so it can be used as a regex */ +export const stringToRegex = (input: string): RegExp => + // being overly conservative + // eslint-disable-next-line no-useless-escape + new RegExp(input.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'), 'g'); diff --git a/src/convert/types.ts b/src/convert/types.ts index 2be5c60053..cb5df08830 100644 --- a/src/convert/types.ts +++ b/src/convert/types.ts @@ -108,7 +108,7 @@ export type ConvertResult = { /** Stored by file on SourceComponent for stream processing */ export type MarkedReplacement = { - toReplace: string | RegExp; + toReplace: RegExp; replaceWith: string; matchedFilename: string; singleFile?: boolean; diff --git a/test/convert/replacements.test.ts b/test/convert/replacements.test.ts index f7d575fb2f..dbd892b26c 100644 --- a/test/convert/replacements.test.ts +++ b/test/convert/replacements.test.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import { expect } from 'chai'; import Sinon = require('sinon'); import { Lifecycle } from '@salesforce/core'; -import { getReplacements, matchesFile, replacementIterations } from '../../src/convert/replacements'; +import { getReplacements, matchesFile, replacementIterations, stringToRegex } from '../../src/convert/replacements'; import { matchingContentFile } from '../mock'; import * as replacementsForMock from '../../src/convert/replacements'; @@ -64,7 +64,7 @@ describe('marking replacements on a component', () => { [cmp.xml]: [ { matchedFilename: cmp.xml, - toReplace: 'foo', + toReplace: stringToRegex('foo'), replaceWith: 'bar', singleFile: true, }, @@ -77,7 +77,7 @@ describe('marking replacements on a component', () => { [cmp.xml]: [ { matchedFilename: cmp.xml, - toReplace: 'foo', + toReplace: stringToRegex('foo'), replaceWith: 'bar', singleFile: true, }, @@ -109,13 +109,13 @@ describe('marking replacements on a component', () => { [cmp.xml]: [ { matchedFilename: cmp.xml, - toReplace: 'foo', + toReplace: stringToRegex('foo'), replaceWith: 'bar', singleFile: true, }, { matchedFilename: cmp.xml, - toReplace: 'baz', + toReplace: stringToRegex('baz'), replaceWith: 'bar', singleFile: true, }, @@ -130,7 +130,7 @@ describe('marking replacements on a component', () => { [cmp.xml]: [ { matchedFilename: cmp.xml, - toReplace: 'foo', + toReplace: stringToRegex('foo'), replaceWith: 'bar', singleFile: false, }, @@ -138,7 +138,7 @@ describe('marking replacements on a component', () => { [cmp.content]: [ { matchedFilename: cmp.content, - toReplace: 'foo', + toReplace: stringToRegex('foo'), replaceWith: 'bar', singleFile: false, }, @@ -154,7 +154,7 @@ describe('marking replacements on a component', () => { [cmp.xml]: [ { matchedFilename: cmp.xml, - toReplace: 'foo', + toReplace: stringToRegex('foo'), replaceWith: 'bar', singleFile: true, }, @@ -162,7 +162,7 @@ describe('marking replacements on a component', () => { [cmp.content]: [ { matchedFilename: cmp.content, - toReplace: 'foo', + toReplace: stringToRegex('foo'), replaceWith: 'bar', singleFile: true, }, @@ -178,22 +178,22 @@ describe('executes replacements on a string', () => { it('basic replacement', async () => { expect( await replacementIterations('ThisIsATest', [ - { matchedFilename, toReplace: 'This', replaceWith: 'That', singleFile: true }, + { matchedFilename, toReplace: stringToRegex('This'), replaceWith: 'That', singleFile: true }, ]) ).to.equal('ThatIsATest'); }); it('same replacement occuring multiple times', async () => { expect( await replacementIterations('ThisIsATestWithThisAndThis', [ - { matchedFilename, toReplace: 'This', replaceWith: 'That', singleFile: true }, + { matchedFilename, toReplace: stringToRegex('This'), replaceWith: 'That', singleFile: true }, ]) ).to.equal('ThatIsATestWithThatAndThat'); }); it('multiple replacements', async () => { expect( await replacementIterations('ThisIsATestWithThisAndThis', [ - { matchedFilename, toReplace: 'This', replaceWith: 'That' }, - { matchedFilename, toReplace: 'ATest', replaceWith: 'AnAwesomeTest' }, + { matchedFilename, toReplace: stringToRegex('This'), replaceWith: 'That' }, + { matchedFilename, toReplace: stringToRegex('ATest'), replaceWith: 'AnAwesomeTest' }, ]) ).to.equal('ThatIsAnAwesomeTestWithThatAndThat'); }); @@ -238,14 +238,14 @@ describe('executes replacements on a string', () => { }); it('emits warning only when no change', async () => { await replacementIterations('ThisIsATest', [ - { toReplace: 'Nope', replaceWith: 'Nah', singleFile: true, matchedFilename }, + { toReplace: stringToRegex('Nope'), replaceWith: 'Nah', singleFile: true, matchedFilename }, ]); expect(warnSpy.callCount).to.equal(1); expect(emitSpy.callCount).to.equal(1); }); it('no warning when string is replaced', async () => { await replacementIterations('ThisIsATest', [ - { toReplace: 'Test', replaceWith: 'SpyTest', singleFile: true, matchedFilename }, + { toReplace: stringToRegex('Test'), replaceWith: 'SpyTest', singleFile: true, matchedFilename }, ]); expect(warnSpy.callCount).to.equal(0); // because it emits the replacement event @@ -253,7 +253,7 @@ describe('executes replacements on a string', () => { }); it('no warning when no replacement but not a single file (ex: glob)', async () => { await replacementIterations('ThisIsATest', [ - { toReplace: 'Nope', replaceWith: 'Nah', singleFile: false, matchedFilename }, + { toReplace: stringToRegex('Nope'), replaceWith: 'Nah', singleFile: false, matchedFilename }, ]); expect(warnSpy.callCount).to.equal(0); expect(emitSpy.callCount).to.equal(0); From 5b4ac19e380a8e0a746478a36a79a18ff5164a52 Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Fri, 28 Oct 2022 20:02:35 +0000 Subject: [PATCH 40/41] test: record perf --- .../eda.json | 8 ++++---- .../lotsOfClasses.json | 8 ++++---- .../lotsOfClassesOneDir.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json index bca290887b..b9756d8043 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 217.22331100000883 + "duration": 217.77746399999887 }, { "name": "sourceToMdapi", - "duration": 5528.8957320000045 + "duration": 5706.744642000005 }, { "name": "sourceToZip", - "duration": 4973.436105999979 + "duration": 4938.870052000013 }, { "name": "mdapiToSource", - "duration": 3836.91319500003 + "duration": 4062.5320680000004 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json index 57c2e7e73b..e36fe8fc68 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 412.3087709999527 + "duration": 470.6312699999835 }, { "name": "sourceToMdapi", - "duration": 7418.153514000005 + "duration": 7751.746233000013 }, { "name": "sourceToZip", - "duration": 6142.985255000007 + "duration": 6782.484058000002 }, { "name": "mdapiToSource", - "duration": 4409.731972999987 + "duration": 4625.043036999996 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json index 64679058f6..b0a92e7c9d 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-Platinum-8272CL-CPU-2-60GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 702.3643789999769 + "duration": 729.9692550000036 }, { "name": "sourceToMdapi", - "duration": 11670.428872999968 + "duration": 9952.340448999981 }, { "name": "sourceToZip", - "duration": 10227.489082999993 + "duration": 10424.467409000004 }, { "name": "mdapiToSource", - "duration": 10798.030672999972 + "duration": 7663.856468000013 } ] \ No newline at end of file From 0685814e747ed233305a357bab376d0add38ac62 Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Fri, 28 Oct 2022 22:08:57 +0000 Subject: [PATCH 41/41] test: record perf --- .../eda.json | 8 ++++---- .../lotsOfClasses.json | 8 ++++---- .../lotsOfClassesOneDir.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/eda.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/eda.json index 7929dcb85f..10323c33c2 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/eda.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/eda.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 324.7964989999891 + "duration": 356.4171700000006 }, { "name": "sourceToMdapi", - "duration": 7310.667111000017 + "duration": 7073.228318999987 }, { "name": "sourceToZip", - "duration": 6250.426998999988 + "duration": 6332.181806000008 }, { "name": "mdapiToSource", - "duration": 5462.673781999998 + "duration": 6008.502924 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClasses.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClasses.json index 1d70dc2dc5..d1d4cdf8c7 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClasses.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClasses.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 664.4817089999851 + "duration": 679.544412999996 }, { "name": "sourceToMdapi", - "duration": 11463.153294000018 + "duration": 13732.503479000006 }, { "name": "sourceToZip", - "duration": 10041.71440299999 + "duration": 10400.92113599999 }, { "name": "mdapiToSource", - "duration": 9396.318423999997 + "duration": 8115.602960999997 } ] \ No newline at end of file diff --git a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClassesOneDir.json b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClassesOneDir.json index c1cc84ac97..19c86c028c 100644 --- a/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClassesOneDir.json +++ b/test/nuts/perfResults/x64-linux-2xIntel-Xeon-CPU-E5-2673-v3-2-40GHz/lotsOfClassesOneDir.json @@ -1,18 +1,18 @@ [ { "name": "componentSetCreate", - "duration": 1068.983653000003 + "duration": 1184.3568459999806 }, { "name": "sourceToMdapi", - "duration": 15904.999413000012 + "duration": 17854.442958 }, { "name": "sourceToZip", - "duration": 15449.051027999958 + "duration": 17075.238639000017 }, { "name": "mdapiToSource", - "duration": 14017.64948100003 + "duration": 14847.72293400002 } ] \ No newline at end of file