From 04014bcf849fcf7b6b15622ada7da50cee485e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20V=C3=A1lek?= Date: Fri, 28 Apr 2023 14:30:48 +0200 Subject: [PATCH 1/6] feat: more efficient parsing --- .../utilities/src/webhook_payload_template.ts | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/utilities/src/webhook_payload_template.ts b/packages/utilities/src/webhook_payload_template.ts index 2e79ff2bb..9d4f4652f 100644 --- a/packages/utilities/src/webhook_payload_template.ts +++ b/packages/utilities/src/webhook_payload_template.ts @@ -23,6 +23,12 @@ export class InvalidVariableError extends Error { } } +interface ParsePosition { + isInsideString: boolean; + openBraceIndex: number; + closeBraceIndex: number; +} + /** * WebhookPayloadTemplate enables creation and parsing of webhook payload template strings. * Template strings are JSON that may include template variables enclosed in double @@ -125,31 +131,32 @@ export class WebhookPayloadTemplate { } private _parse() { + let currentIndex = 0; while (true) { // eslint-disable-line no-constant-condition try { return JSON.parse(this.payload); } catch (err) { - const position = this._findPositionOfNextVariable(); - if (position) { - this._replaceVariable(position); - } else { - // When we catch an error from JSON.parse, but there's - // no variable, we must have an invalid JSON. + const position = this._findPositionOfNextVariable(currentIndex); + // When we catch an error from JSON.parse, but there's no remaining variable, we must have an invalid JSON. + if (!position) { throw new InvalidJsonError(err as Error); } + if (!position.isInsideString) { + this._replaceVariable(position); + } + currentIndex = position.openBraceIndex + 1; } } } - private _findPositionOfNextVariable(startIndex = 0): { openBraceIndex: number, closeBraceIndex: number } | null { + private _findPositionOfNextVariable(startIndex = 0): ParsePosition | null { const openBraceIndex = this.payload.indexOf('{{', startIndex); const closeBraceIndex = this.payload.indexOf('}}', openBraceIndex) + 1; const someVariableMaybeExists = (openBraceIndex > -1) && (closeBraceIndex > -1); if (!someVariableMaybeExists) return null; const isInsideString = this._isVariableInsideString(openBraceIndex); - if (!isInsideString) return { openBraceIndex, closeBraceIndex }; - return this._findPositionOfNextVariable(openBraceIndex + 1); + return { isInsideString, openBraceIndex, closeBraceIndex }; } private _isVariableInsideString(openBraceIndex: number): boolean { @@ -170,7 +177,7 @@ export class WebhookPayloadTemplate { return unescapedQuoteCount; } - private _replaceVariable({ openBraceIndex, closeBraceIndex }: { openBraceIndex: number, closeBraceIndex: number }): void { + private _replaceVariable({ openBraceIndex, closeBraceIndex }: ParsePosition): void { const variableName = this.payload.substring(openBraceIndex + 2, closeBraceIndex - 1); this._validateVariableName(variableName); const replacement = this._getVariableReplacement(variableName)!; From eb857da7d0770a776c05d821b10ef81678aa1ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20V=C3=A1lek?= Date: Fri, 28 Apr 2023 15:20:49 +0200 Subject: [PATCH 2/6] feat: in-string parsing --- .../utilities/src/webhook_payload_template.ts | 67 +++++++++++++++++-- test/webhook_payload_template.test.ts | 17 +++++ 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/packages/utilities/src/webhook_payload_template.ts b/packages/utilities/src/webhook_payload_template.ts index 9d4f4652f..96f0e2af0 100644 --- a/packages/utilities/src/webhook_payload_template.ts +++ b/packages/utilities/src/webhook_payload_template.ts @@ -87,7 +87,8 @@ export class WebhookPayloadTemplate { const type = typeof payloadTemplate; if (type !== 'string') throw new Error(`Cannot parse a ${type} payload template.`); const template = new WebhookPayloadTemplate(payloadTemplate, allowedVariables, context); - return template._parse(); // eslint-disable-line no-underscore-dangle + const data = template._parse(); // eslint-disable-line no-underscore-dangle + return template._interpolate(data); // eslint-disable-line no-underscore-dangle } /** @@ -150,6 +151,59 @@ export class WebhookPayloadTemplate { } } + /** + * Process variables that are inside strings. + * + * @param data + * @returns + */ + private _interpolate(data: Record): Record { + return this._interpolateWhatever(data); + } + + private _interpolateWhatever(value: any): any { + if (typeof value === 'string') { + return this._interpolateString(value); + } + // Array needs to go before object! + if (Array.isArray(value)) { + return this._interpolateArray(value); + } + if (typeof value === 'object' && value !== null) { + return this._interpolateObject(value); + } + // We can't interpolate anything else + return value; + } + + // TODO: Just replace the variables in this case + private _interpolateString(value: string): string { + // If the string matches exactly, we return the variable value including the type + if (value.match(/^\{\{var:([a-zA-Z0-9.]*)\}\}$/)) { + const variableName = value.substring(6, value.length - 2); + this._validateVariableName(variableName); + return this._getVariableValue(variableName); + } + // If it's just a part of substring, we replace the respective variables with their string variants + return value.replace(/\{\{var:([a-zA-Z0-9.]*)\}\}/g, (match, variableName) => { + this._validateVariableName(variableName); + const variableValue = this._getVariableValue(variableName); + return `${variableValue}`; + }); + } + + private _interpolateObject(value: Record): Record { + const result = {}; + Object.entries(value).forEach(([key, v]) => { + result[key] = this._interpolateWhatever(v); + }); + return result; + } + + private _interpolateArray(value: Array): Array { + return value.map(this._interpolateWhatever.bind(this)); + } + private _findPositionOfNextVariable(startIndex = 0): ParsePosition | null { const openBraceIndex = this.payload.indexOf('{{', startIndex); const closeBraceIndex = this.payload.indexOf('}}', openBraceIndex) + 1; @@ -197,13 +251,18 @@ export class WebhookPayloadTemplate { if (!isVariableValid) throw new InvalidVariableError(variableName); } - private _getVariableReplacement(variableName: string): string | null { + private _getVariableValue(variableName: string): any { const [variable, ...properties] = variableName.split('.'); const context = this.context[variable]; - const replacement = properties.reduce((ctx, prop) => { + const value = properties.reduce((ctx, prop) => { if (!ctx || typeof ctx !== 'object') return null; return ctx[prop]; }, context); - return replacement ? JSON.stringify(replacement) : null; + return value; + } + + private _getVariableReplacement(variableName: string): string | null { + const value = this._getVariableValue(variableName); + return value ? JSON.stringify(value) : null; } } diff --git a/test/webhook_payload_template.test.ts b/test/webhook_payload_template.test.ts index 2f2c4ced6..d6517fb56 100644 --- a/test/webhook_payload_template.test.ts +++ b/test/webhook_payload_template.test.ts @@ -114,6 +114,23 @@ describe('WebhookPayloadTemplate', () => { } }); + it('replaces variables in strings', () => { + const payload = WebhookPayloadTemplate.parse(` + { + "justVariable": "{{var:foo}}", + "someOtherContent": "bar{{var:foo}}", + "twoVariables": "bar{{var:foo}}baz{{var:foo}}bar{{var:lol}}", + "bar": {{xyz}} + }`, null, { + foo: 'foo', + lol: 'lol', + }); + expect(payload.justVariable).toBe('foo'); + expect(payload.someOtherContent).toBe('barfoo'); + expect(payload.twoVariables).toBe('barfoobazfoobarlol'); + expect(payload.bar).toBe(null); + }); + it('should throw InvalidJsonError on invalid json', () => { try { WebhookPayloadTemplate.parse(invalidJson); From 818e141b618caf79de036bfc998ab0f7e4dca0c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20V=C3=A1lek?= Date: Fri, 28 Apr 2023 15:25:30 +0200 Subject: [PATCH 3/6] chore: added todo --- packages/utilities/src/webhook_payload_template.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/utilities/src/webhook_payload_template.ts b/packages/utilities/src/webhook_payload_template.ts index 96f0e2af0..50a0d9392 100644 --- a/packages/utilities/src/webhook_payload_template.ts +++ b/packages/utilities/src/webhook_payload_template.ts @@ -88,6 +88,7 @@ export class WebhookPayloadTemplate { if (type !== 'string') throw new Error(`Cannot parse a ${type} payload template.`); const template = new WebhookPayloadTemplate(payloadTemplate, allowedVariables, context); const data = template._parse(); // eslint-disable-line no-underscore-dangle + // TODO: Maybe make this configurable via some options, to make "sure" about backwards compatibility? return template._interpolate(data); // eslint-disable-line no-underscore-dangle } @@ -176,7 +177,6 @@ export class WebhookPayloadTemplate { return value; } - // TODO: Just replace the variables in this case private _interpolateString(value: string): string { // If the string matches exactly, we return the variable value including the type if (value.match(/^\{\{var:([a-zA-Z0-9.]*)\}\}$/)) { From 33bb3424fb224ab7e6eccdd85def3f15c8422759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20V=C3=A1lek?= Date: Fri, 28 Apr 2023 15:27:11 +0200 Subject: [PATCH 4/6] chore: renames --- .../utilities/src/webhook_payload_template.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/packages/utilities/src/webhook_payload_template.ts b/packages/utilities/src/webhook_payload_template.ts index 50a0d9392..c7e823203 100644 --- a/packages/utilities/src/webhook_payload_template.ts +++ b/packages/utilities/src/webhook_payload_template.ts @@ -152,17 +152,7 @@ export class WebhookPayloadTemplate { } } - /** - * Process variables that are inside strings. - * - * @param data - * @returns - */ - private _interpolate(data: Record): Record { - return this._interpolateWhatever(data); - } - - private _interpolateWhatever(value: any): any { + private _interpolate(value: any): any { if (typeof value === 'string') { return this._interpolateString(value); } @@ -195,13 +185,13 @@ export class WebhookPayloadTemplate { private _interpolateObject(value: Record): Record { const result = {}; Object.entries(value).forEach(([key, v]) => { - result[key] = this._interpolateWhatever(v); + result[key] = this._interpolate(v); }); return result; } private _interpolateArray(value: Array): Array { - return value.map(this._interpolateWhatever.bind(this)); + return value.map(this._interpolate.bind(this)); } private _findPositionOfNextVariable(startIndex = 0): ParsePosition | null { From aaad608e00de9748b3c3f4579863da8159d6bf72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20V=C3=A1lek?= Date: Wed, 3 May 2023 13:52:49 +0200 Subject: [PATCH 5/6] fix: better tests, drop var prefix, and fix bug --- .../utilities/src/webhook_payload_template.ts | 22 ++++++--- test/webhook_payload_template.test.ts | 46 +++++++++++++------ 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/packages/utilities/src/webhook_payload_template.ts b/packages/utilities/src/webhook_payload_template.ts index c7e823203..27a1fc56f 100644 --- a/packages/utilities/src/webhook_payload_template.ts +++ b/packages/utilities/src/webhook_payload_template.ts @@ -83,13 +83,20 @@ export class WebhookPayloadTemplate { * Parse also validates the template structure, so it can be used * to check validity of the template JSON and usage of allowedVariables. */ - static parse(payloadTemplate: string, allowedVariables: Set | null = null, context: Record = {}): Record { + static parse( + payloadTemplate: string, + allowedVariables: Set | null = null, + context: Record = {}, + options: { interpolateStrings?: boolean } = {}, + ): Record { const type = typeof payloadTemplate; if (type !== 'string') throw new Error(`Cannot parse a ${type} payload template.`); const template = new WebhookPayloadTemplate(payloadTemplate, allowedVariables, context); const data = template._parse(); // eslint-disable-line no-underscore-dangle - // TODO: Maybe make this configurable via some options, to make "sure" about backwards compatibility? - return template._interpolate(data); // eslint-disable-line no-underscore-dangle + if (options.interpolateStrings) { + return template._interpolate(data); // eslint-disable-line no-underscore-dangle + } + return data; } /** @@ -169,13 +176,14 @@ export class WebhookPayloadTemplate { private _interpolateString(value: string): string { // If the string matches exactly, we return the variable value including the type - if (value.match(/^\{\{var:([a-zA-Z0-9.]*)\}\}$/)) { - const variableName = value.substring(6, value.length - 2); + if (value.match(/^\{\{([a-zA-Z0-9.]+)\}\}$/)) { + // This just strpis the {{ and }} + const variableName = value.substring(2, value.length - 2); this._validateVariableName(variableName); return this._getVariableValue(variableName); } // If it's just a part of substring, we replace the respective variables with their string variants - return value.replace(/\{\{var:([a-zA-Z0-9.]*)\}\}/g, (match, variableName) => { + return value.replace(/\{\{([a-zA-Z0-9.]+)\}\}/g, (match, variableName) => { this._validateVariableName(variableName); const variableValue = this._getVariableValue(variableName); return `${variableValue}`; @@ -226,7 +234,7 @@ export class WebhookPayloadTemplate { this._validateVariableName(variableName); const replacement = this._getVariableReplacement(variableName)!; this.replacedVariables.push({ variableName, replacement }); - this.payload = this.payload.replace(`{{${variableName}}}`, replacement); + this.payload = this.payload.substring(0, openBraceIndex) + replacement + this.payload.substring(closeBraceIndex + 1); } private _validateVariableName(variableName: string): void { diff --git a/test/webhook_payload_template.test.ts b/test/webhook_payload_template.test.ts index d6517fb56..f9d083d28 100644 --- a/test/webhook_payload_template.test.ts +++ b/test/webhook_payload_template.test.ts @@ -22,7 +22,8 @@ const validTemplate = ` const validTemplateWithVariableInString = ` { "foo": "bar\\"{{foo}}\\"", - "bar": {{xyz}} + "bar": {{xyz}}, + "baz": "{{foo}}" } `; @@ -102,7 +103,7 @@ describe('WebhookPayloadTemplate', () => { expect(payload).toEqual(context); }); - it('does not replace variables in strings', () => { + it('does not replace variables in strings by default', () => { const payload = WebhookPayloadTemplate.parse(validTemplateWithVariableInString); expect(payload.foo).toBe('bar"{{foo}}"'); @@ -114,21 +115,38 @@ describe('WebhookPayloadTemplate', () => { } }); - it('replaces variables in strings', () => { + it('replaces variables in strings if interpolateStrings=true', () => { + const context = { + foo: 'hello', + lol: 'world', + resource: { + defaultDatasetId: 'dataset-123', + }, + arrayField: [1, 2, 3], + }; const payload = WebhookPayloadTemplate.parse(` { - "justVariable": "{{var:foo}}", - "someOtherContent": "bar{{var:foo}}", - "twoVariables": "bar{{var:foo}}baz{{var:foo}}bar{{var:lol}}", - "bar": {{xyz}} - }`, null, { - foo: 'foo', - lol: 'lol', - }); - expect(payload.justVariable).toBe('foo'); - expect(payload.someOtherContent).toBe('barfoo'); - expect(payload.twoVariables).toBe('barfoobazfoobarlol'); + "justVariable": "{{foo}}", + "someOtherContent": "bar{{foo}}", + "twoVariables": "bar{{foo}}baz{{foo}}bar{{lol}}", + "bar": {{xyz}}, + "datasetId": "{{resource.defaultDatasetId}}", + "arrayField": "{{arrayField}}", + "arrayFieldInString": "This is my array {{arrayField}}", + "resourceWrapped": "{{resource}}", + "resourceDirect": {{resource}}, + "resourceInString": "This is my object {{resource}}" + }`, null, context, { interpolateStrings: true }); + expect(payload.justVariable).toBe('hello'); + expect(payload.someOtherContent).toBe('barhello'); + expect(payload.twoVariables).toBe('barhellobazhellobarworld'); expect(payload.bar).toBe(null); + expect(payload.datasetId).toBe('dataset-123'); + expect(payload.arrayField).toStrictEqual(context.arrayField); + expect(payload.arrayFieldInString).toBe('This is my array 1,2,3'); + expect(payload.resourceWrapped).toStrictEqual(context.resource); + expect(payload.resourceDirect).toStrictEqual(context.resource); + expect(payload.resourceInString).toBe('This is my object [object Object]'); }); it('should throw InvalidJsonError on invalid json', () => { From d11174b8908047905924ce098241daa058f855f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20V=C3=A1lek?= Date: Tue, 9 May 2023 13:13:24 +0200 Subject: [PATCH 6/6] Update packages/utilities/src/webhook_payload_template.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: FrantiĊĦek Nesveda --- packages/utilities/src/webhook_payload_template.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/utilities/src/webhook_payload_template.ts b/packages/utilities/src/webhook_payload_template.ts index 27a1fc56f..c799d681a 100644 --- a/packages/utilities/src/webhook_payload_template.ts +++ b/packages/utilities/src/webhook_payload_template.ts @@ -177,7 +177,7 @@ export class WebhookPayloadTemplate { private _interpolateString(value: string): string { // If the string matches exactly, we return the variable value including the type if (value.match(/^\{\{([a-zA-Z0-9.]+)\}\}$/)) { - // This just strpis the {{ and }} + // This just strips the {{ and }} const variableName = value.substring(2, value.length - 2); this._validateVariableName(variableName); return this._getVariableValue(variableName);