From c549d6ac62946256f822977f3e7f2ef0b4992ab7 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:10:55 -0400 Subject: [PATCH 01/17] Collect client errors into common, order alphabetically --- libraries/common/src/errors.ts | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/libraries/common/src/errors.ts b/libraries/common/src/errors.ts index b191dad6..4c4c8ad1 100644 --- a/libraries/common/src/errors.ts +++ b/libraries/common/src/errors.ts @@ -17,24 +17,24 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -export class ConflictError extends Error { +export class BadRequestError extends Error { constructor(message: string) { super(message); - this.name = 'Conflict'; + this.name = 'BadRequest'; } } -export class BadRequestError extends Error { +export class ConflictError extends Error { constructor(message: string) { super(message); - this.name = 'BadRequest'; + this.name = 'Conflict'; } } -export class MalformedVersionError extends Error { +export class ForbiddenError extends Error { constructor(message: string) { super(message); - this.name = 'MalformedVersion'; + this.name = 'Forbidden'; } } @@ -45,30 +45,36 @@ export class InternalServerError extends Error { } } -export class NotFoundError extends Error { +export class InvalidArgument extends Error { + constructor(argumentName: string) { + super(`Invalid argument : ${argumentName}`); + } +} + +export class InvalidReferenceError extends Error { constructor(message: string) { super(message); - this.name = 'NotFound'; + this.name = 'InvalidReference'; } } -export class UnauthorizedError extends Error { +export class MalformedVersionError extends Error { constructor(message: string) { super(message); - this.name = 'Unauthorized'; + this.name = 'MalformedVersion'; } } -export class ForbiddenError extends Error { +export class NotFoundError extends Error { constructor(message: string) { super(message); - this.name = 'Forbidden'; + this.name = 'NotFound'; } } -export class InvalidReferenceError extends Error { +export class UnauthorizedError extends Error { constructor(message: string) { super(message); - this.name = 'InvalidReference'; + this.name = 'Unauthorized'; } } From ec2a4156279d1f5c330cae43d9182c38e58d3338 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:12:42 -0400 Subject: [PATCH 02/17] Create Singular utility type to extract element types from array - helps simply type notation when processing DataRecords in client --- libraries/common/src/types/index.ts | 1 + libraries/common/src/types/singular.ts | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 libraries/common/src/types/singular.ts diff --git a/libraries/common/src/types/index.ts b/libraries/common/src/types/index.ts index 9fdaabd5..68cee3d1 100644 --- a/libraries/common/src/types/index.ts +++ b/libraries/common/src/types/index.ts @@ -1,2 +1,3 @@ export * from './generics'; export * from './result'; +export * from './singular'; diff --git a/libraries/common/src/types/singular.ts b/libraries/common/src/types/singular.ts new file mode 100644 index 00000000..06c3f1c1 --- /dev/null +++ b/libraries/common/src/types/singular.ts @@ -0,0 +1,16 @@ +/** + * Utility type that finds the type or types from inside an Array or nested Arrays. + * + * @example + * type String = Singular; // `String` is `string` + * type JustAString = Singular; // `JustAString` is `string` + * type StillJustAString = Singular<(string[][][][][]>; // `StillJustAString` is `string` + * type StringOrNumberFromUnion = Singular<(string[] | number)[] | string>; // `StringOrNumber` is `string | number` + * type TupleContents = Singular<[number, string, boolean]> // `TupleContents` is `number | string | boolean` + * type ArrayOfTuplesContents = Singular<[number, string, boolean]> // `ArrayOfTuplesContents` is `number | string | boolean` + */ +export type Singular = T extends Array + ? Singular + : T extends ReadonlyArray + ? Singular + : T; From 2759626acdcfc0872e61faa34ffbf10cc03df5d7 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:19:31 -0400 Subject: [PATCH 03/17] Make a default RangeRestriction schema validator available --- libraries/dictionary/src/types/dictionaryTypes.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/dictionary/src/types/dictionaryTypes.ts b/libraries/dictionary/src/types/dictionaryTypes.ts index 5d81c001..e7bcd5e4 100644 --- a/libraries/dictionary/src/types/dictionaryTypes.ts +++ b/libraries/dictionary/src/types/dictionaryTypes.ts @@ -52,6 +52,7 @@ export type SchemaFieldValueType = zod.infer; * ************ */ export const RestrictionScript = zod.array(zod.string().or(ReferenceTag)).min(1); //TODO: script formatting validation export type RestrictionScript = zod.infer; + export const RestrictionNumberRange = zod .object({ exclusiveMax: zod.number().optional(), @@ -99,6 +100,8 @@ export const RestrictionIntegerRange = zod (data) => !(data.exclusiveMax !== undefined && data.max !== undefined), 'Range restriction cannot have both `exclusiveMax` and `max`.', ); + +export const RestrictionRange = RestrictionNumberRange; export type RestrictionRange = zod.infer; export const RestrictionRegex = zod.string().superRefine((data, context) => { From b1c4bf454ab5a29af9554e65f58d495148506fde Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:20:39 -0400 Subject: [PATCH 04/17] Update diff type FieldChange to potentially be undefined --- libraries/dictionary/src/types/diffTypes.ts | 27 +++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/libraries/dictionary/src/types/diffTypes.ts b/libraries/dictionary/src/types/diffTypes.ts index 44ed8289..976e3a44 100644 --- a/libraries/dictionary/src/types/diffTypes.ts +++ b/libraries/dictionary/src/types/diffTypes.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import { z as zod } from 'zod'; import { SchemaField } from './dictionaryTypes'; @@ -24,8 +43,12 @@ export type ValueChange = zod.infer; // in case of created/delete field we get Change // in case of simple field change we get {"fieldName": {"data":.., "type": ..}} // in case of nested fields: {"fieldName1": {"fieldName2": {"data":.., "type": ..}}} -export type FieldChanges = { [field: string]: FieldChanges } | ValueChange; -export const FieldChanges: zod.ZodType = zod.union([zod.lazy(() => FieldChanges), ValueChange]); +export type FieldChanges = { [field: string]: FieldChanges } | ValueChange | undefined; +export const FieldChanges: zod.ZodType = zod.union([ + zod.lazy(() => FieldChanges), + ValueChange, + zod.undefined(), +]); export const FieldDiff = zod.object({ left: SchemaField.optional(), From 05036f21d17712d6211332832ca40d0aa921e765 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:23:59 -0400 Subject: [PATCH 05/17] Remove namespaces, error types moved to common --- packages/client/src/utils.ts | 38 ++++-------------------------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/packages/client/src/utils.ts b/packages/client/src/utils.ts index 391b2662..bb4d2aa4 100644 --- a/packages/client/src/utils.ts +++ b/packages/client/src/utils.ts @@ -17,40 +17,8 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import fs from 'fs'; import deepFreeze from 'deep-freeze'; import _ from 'lodash'; -import { isArray } from 'util'; - -const fsPromises = fs.promises; - -export namespace Checks { - export const checkNotNull = (argName: string, arg: any) => { - if (!arg) { - throw new Errors.InvalidArgument(argName); - } - }; -} - -export namespace Errors { - export class InvalidArgument extends Error { - constructor(argumentName: string) { - super(`Invalid argument : ${argumentName}`); - } - } - - export class NotFound extends Error { - constructor(msg: string) { - super(msg); - } - } - - export class StateConflict extends Error { - constructor(msg: string) { - super(msg); - } - } -} // type gaurd to filter out undefined and null // https://stackoverflow.com/questions/43118692/typescript-filter-out-nulls-from-an-array @@ -108,7 +76,9 @@ export const isAbsent = (value: string | number | boolean | undefined): value is return !isNotAbsent(value); }; -export const isNotAbsent = (value: string | number | boolean | undefined): value is string | number | boolean => { +export const isNotAbsent = ( + value: string | number | boolean | undefined | null, +): value is string | number | boolean => { return value !== null && value !== undefined; }; @@ -131,7 +101,7 @@ export function toString(obj: any) { } export function isValueEqual(value: any, other: any) { - if (isArray(value) && isArray(other)) { + if (Array.isArray(value) && Array.isArray(other)) { return _.difference(value, other).length === 0; // check equal, ignore order } From dcc0a3032878edc97a069269ebf32b053b19fa72 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:25:52 -0400 Subject: [PATCH 06/17] Refactor changeAnalysis with lectern shared types --- packages/client/src/change-analyzer.ts | 254 ---------------- .../src/changeAnalysis/changeAnalysisTypes.ts | 85 ++++++ .../src/changeAnalysis/changeAnalyzer.ts | 274 ++++++++++++++++++ packages/client/src/changeAnalysis/index.ts | 2 + 4 files changed, 361 insertions(+), 254 deletions(-) delete mode 100644 packages/client/src/change-analyzer.ts create mode 100644 packages/client/src/changeAnalysis/changeAnalysisTypes.ts create mode 100644 packages/client/src/changeAnalysis/changeAnalyzer.ts create mode 100644 packages/client/src/changeAnalysis/index.ts diff --git a/packages/client/src/change-analyzer.ts b/packages/client/src/change-analyzer.ts deleted file mode 100644 index 02e2c080..00000000 --- a/packages/client/src/change-analyzer.ts +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { restClient } from './schema-rest-client'; -import { - SchemasDictionaryDiffs, - FieldChanges, - FieldDiff, - Change, - ChangeAnalysis, - ChangeTypeName, - RestrictionChanges, - FieldDefinition, -} from './schema-entities'; - -const isFieldChange = (obj: any): obj is Change => { - return obj.type !== undefined; -}; - -const isNestedChange = (obj: any): obj is { [field: string]: FieldChanges } => { - return obj.type === undefined; -}; - -const isRestrictionChange = (obj: any): obj is { [field: string]: FieldChanges } => { - return obj.type === undefined; -}; - -export const fetchDiffAndAnalyze = async (serviceUrl: string, name: string, fromVersion: string, toVersion: string) => { - const changes = await restClient.fetchDiff(serviceUrl, name, fromVersion, toVersion); - return analyzeChanges(changes); -}; - -export const analyzeChanges = (schemasDiff: SchemasDictionaryDiffs): ChangeAnalysis => { - const analysis: ChangeAnalysis = { - fields: { - addedFields: [], - renamedFields: [], - deletedFields: [], - }, - isArrayDesignationChanges: [], - restrictionsChanges: { - codeList: { - created: [], - deleted: [], - updated: [], - }, - regex: { - updated: [], - created: [], - deleted: [], - }, - required: { - updated: [], - created: [], - deleted: [], - }, - script: { - updated: [], - created: [], - deleted: [], - }, - range: { - updated: [], - created: [], - deleted: [], - }, - }, - metaChanges: { - core: { - changedToCore: [], - changedFromCore: [], - }, - }, - valueTypeChanges: [], - }; - - for (const field of Object.keys(schemasDiff)) { - const fieldChange: FieldDiff = schemasDiff[field]; - if (fieldChange) { - const fieldDiff = fieldChange.diff; - // if we have type at first level then it's a field add/delete - if (isFieldChange(fieldDiff)) { - categorizeFieldChanges(analysis, field, fieldDiff); - } - - if (isNestedChange(fieldDiff)) { - if (fieldDiff.meta) { - categorizeMetaChagnes(analysis, field, fieldDiff.meta); - } - - if (fieldDiff.restrictions) { - categorizeRestrictionChanges(analysis, field, fieldDiff.restrictions, fieldChange.after); - } - - if (fieldDiff.isArray) { - categorizeFieldArrayDesignationChange(analysis, field, fieldDiff.isArray); - } - - if (fieldDiff.valueType) { - categorizerValueTypeChange(analysis, field, fieldDiff.valueType); - } - } - } - } - - return analysis; -}; - -const categorizeFieldArrayDesignationChange = ( - analysis: ChangeAnalysis, - field: string, - changes: { [field: string]: FieldChanges } | Change, -) => { - // changing isArray designation is a relevant change for all cases except if it is created and set to false - if (!(changes.type === 'created' && changes.data === false)) { - analysis.isArrayDesignationChanges.push(field); - } -}; - -const categorizerValueTypeChange = ( - analysis: ChangeAnalysis, - field: string, - changes: { [field: string]: FieldChanges } | Change, -) => { - analysis.valueTypeChanges.push(field); -}; - -const categorizeRestrictionChanges = ( - analysis: ChangeAnalysis, - field: string, - restrictionsChange: { [field: string]: FieldChanges } | Change, - fieldDefinitionAfter?: FieldDefinition, -) => { - const restrictionsToCheck = ['regex', 'script', 'required', 'codeList', 'range']; - - // additions or deletions of a restriction object as whole (i.e. contains 1 or many restrictions within the 'data') - if (restrictionsChange.type) { - const createOrAddChange = restrictionsChange as Change; - const restrictionsData = createOrAddChange.data as any; - - for (const k of restrictionsToCheck) { - if (restrictionsData[k]) { - analysis.restrictionsChanges[k as keyof RestrictionChanges][restrictionsChange.type as ChangeTypeName].push({ - field: field, - definition: restrictionsData[k], - } as any); - } - } - return; - } - - // in case 'restrictions' key was already there but we modified its contents - const restrictionUpdate = restrictionsChange as { [field: string]: FieldChanges }; - for (const k of restrictionsToCheck) { - if (restrictionUpdate[k]) { - const change = restrictionUpdate[k] as Change; - // we need the '|| change' in case of nested attributes like ranges - /* - "diff": { - "restrictions": { - "range": { - "exclusiveMin": { - "type": "deleted", - "data": 0 - }, - "max": { - "type": "updated", - "data": 200000 - }, - "min": { - "type": "created", - "data": 0 - } - } - } - } - */ - if (k == 'range' && !change.type) { - // if the change is nested (type is at min max level) then the boundries were updated only : ex: - /* - change = { - "max" : { - type: "updated" - data: "..." - }, - "exclusiveMin": { - type: "deleted" - data .. - } - } - */ - const def: any = {}; - if (Object.keys(change).some((k) => k == 'max' || k == 'min' || k == 'exclusiveMin' || k == 'exclusiveMax')) { - analysis.restrictionsChanges[k]['updated'].push({ - field: field, - // we push the whole range definition since it doesnt make sense to just - // push one boundary. - definition: fieldDefinitionAfter?.restrictions?.range, - }); - } - return; - } - const definition = change.data || change; - analysis.restrictionsChanges[k as keyof RestrictionChanges][change.type as ChangeTypeName].push({ - field: field, - definition, - } as any); - } - } -}; - -const categorizeFieldChanges = (analysis: ChangeAnalysis, field: string, changes: Change) => { - const changeType = changes.type; - if (changeType == 'created') { - analysis.fields.addedFields.push({ - name: field, - definition: changes.data, - }); - } else if (changeType == 'deleted') { - analysis.fields.deletedFields.push(field); - } -}; - -const categorizeMetaChagnes = ( - analysis: ChangeAnalysis, - field: string, - metaChanges: { [field: string]: FieldChanges } | Change, -) => { - // **** meta changes - core *** - if (metaChanges?.data?.core === true) { - const changeType = metaChanges.type; - if (changeType === 'created' || changeType === 'updated') { - analysis.metaChanges?.core.changedToCore.push(field); - } else if (changeType === 'deleted') { - analysis.metaChanges?.core.changedFromCore.push(field); - } - } -}; diff --git a/packages/client/src/changeAnalysis/changeAnalysisTypes.ts b/packages/client/src/changeAnalysis/changeAnalysisTypes.ts new file mode 100644 index 00000000..0b1b2733 --- /dev/null +++ b/packages/client/src/changeAnalysis/changeAnalysisTypes.ts @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { SchemaField, ValueChangeTypeName } from 'dictionary'; + +type ChangeOnlyTypeNames = Exclude; + +/* ===== Change Analysis Types (Duplicate?) ===== */ +export interface ChangeAnalysis { + fields: { + addedFields: AddedFieldChange[]; + renamedFields: string[]; + deletedFields: string[]; + }; + isArrayDesignationChanges: string[]; + restrictionsChanges: RestrictionChanges; + metaChanges?: MetaChanges; + valueTypeChanges: string[]; +} + +export type RestrictionChanges = { + range: { + [key in ChangeOnlyTypeNames]: ObjectChange[]; + }; + codeList: { + [key in ChangeOnlyTypeNames]: ObjectChange[]; + }; + regex: { + [key in ChangeOnlyTypeNames]: StringAttributeChange[]; + }; + required: { + [key in ChangeOnlyTypeNames]: BooleanAttributeChange[]; + }; + script: { + [key in ChangeOnlyTypeNames]: StringAttributeChange[]; + }; +}; +export interface AddedFieldChange { + name: string; + definition: SchemaField; +} + +export interface ObjectChange { + field: string; + definition: any; +} + +export interface CodeListChange { + field: string; + definition: any; +} + +export interface StringAttributeChange { + field: string; + definition: string; +} + +export interface BooleanAttributeChange { + field: string; + definition: boolean; +} + +// TODO: This references a specific project's meta properties that should be removed from the client +export type MetaChanges = { + core: { + changedToCore: string[]; // fields that are core now + changedFromCore: string[]; // fields that are not core now + }; +}; diff --git a/packages/client/src/changeAnalysis/changeAnalyzer.ts b/packages/client/src/changeAnalysis/changeAnalyzer.ts new file mode 100644 index 00000000..be430a31 --- /dev/null +++ b/packages/client/src/changeAnalysis/changeAnalyzer.ts @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { DictionaryDiff, FieldChanges, RestrictionRange, SchemaField, ValueChange } from 'dictionary'; +import { restClient } from '../rest'; +import { ChangeAnalysis, RestrictionChanges } from './changeAnalysisTypes'; + +const isValueChange = (input: FieldChanges): input is ValueChange => ValueChange.safeParse(input).success; + +type NestedChanges = { [field: string]: FieldChanges }; +const isNestedChange = (input: FieldChanges): input is NestedChanges => { + // Ensure that the inptu is not undefined and that it doesn't match to a ValueChange + return input !== undefined && !isValueChange(input); +}; + +export const fetchDiffAndAnalyze = async (serviceUrl: string, name: string, fromVersion: string, toVersion: string) => { + const changes = await restClient.fetchDiff(serviceUrl, name, fromVersion, toVersion); + return analyzeChanges(changes); +}; + +export const analyzeChanges = (schemasDiff: DictionaryDiff): ChangeAnalysis => { + const analysis: ChangeAnalysis = { + fields: { + addedFields: [], + renamedFields: [], + deletedFields: [], + }, + isArrayDesignationChanges: [], + restrictionsChanges: { + codeList: { + created: [], + deleted: [], + updated: [], + }, + regex: { + updated: [], + created: [], + deleted: [], + }, + required: { + updated: [], + created: [], + deleted: [], + }, + script: { + updated: [], + created: [], + deleted: [], + }, + range: { + updated: [], + created: [], + deleted: [], + }, + }, + valueTypeChanges: [], + }; + + schemasDiff.forEach((fieldChange, fieldName) => { + if (fieldChange) { + const fieldDiff = fieldChange.diff; + + // if we have type at first level then it's a field add/delete + if (isValueChange(fieldDiff)) { + categorizeFieldChanges(analysis, fieldName, fieldDiff); + } + + if (isNestedChange(fieldDiff)) { + if (fieldDiff.meta) { + categorizeMetaChanges(analysis, fieldName, fieldDiff.meta); + } + + if (fieldDiff.restrictions) { + categorizeRestrictionChanges(analysis, fieldName, fieldDiff.restrictions, fieldChange.right); + } + + if (fieldDiff.isArray) { + categorizeFieldArrayDesignationChange(analysis, fieldName, fieldDiff.isArray); + } + + if (fieldDiff.valueType) { + categorizerValueTypeChange(analysis, fieldName, fieldDiff.valueType); + } + } + } + }); + + return analysis; +}; + +const categorizeFieldArrayDesignationChange = (analysis: ChangeAnalysis, field: string, changes: FieldChanges) => { + // changing isArray designation is a relevant change for all cases except if it is created and set to false + if (!(changes?.type === 'created' && changes.data === false)) { + analysis.isArrayDesignationChanges.push(field); + } +}; + +const categorizerValueTypeChange = (analysis: ChangeAnalysis, field: string, changes: FieldChanges) => { + analysis.valueTypeChanges.push(field); +}; + +const categorizeRestrictionChanges = ( + analysis: ChangeAnalysis, + field: string, + restrictionsChange: FieldChanges, + fieldDefinitionAfter?: SchemaField, +) => { + const restrictionsToCheck: (keyof RestrictionChanges)[] = ['regex', 'script', 'required', 'codeList', 'range']; + + // additions or deletions of a restriction object as whole (i.e. contains 1 or many restrictions within the 'data') + if (isValueChange(restrictionsChange) && restrictionsChange.type !== 'unchanged') { + const createOrAddChange = restrictionsChange; + const restrictionsData = createOrAddChange.data; + + for (const k of restrictionsToCheck) { + if (restrictionsData[k]) { + switch (k) { + case 'codeList': { + analysis.restrictionsChanges[k][restrictionsChange.type].push({ + field: field, + definition: restrictionsData[k], + }); + break; + } + case 'range': { + analysis.restrictionsChanges[k][restrictionsChange.type].push({ + field: field, + definition: restrictionsData[k], + }); + break; + } + case 'regex': { + analysis.restrictionsChanges[k][restrictionsChange.type].push({ + field: field, + definition: restrictionsData[k], + }); + break; + } + case 'required': { + analysis.restrictionsChanges[k][restrictionsChange.type].push({ + field: field, + definition: restrictionsData[k], + }); + break; + } + case 'script': { + analysis.restrictionsChanges[k][restrictionsChange.type].push({ + field: field, + definition: restrictionsData[k], + }); + break; + } + } + } + } + return; + } + + // in case 'restrictions' key was already there but we modified its contents + for (const k of restrictionsToCheck) { + if (restrictionsChange && isNestedChange(restrictionsChange) && k in restrictionsChange) { + const change = restrictionsChange[k]; + if (k === 'range' && change !== undefined && !change.type) { + // if the change is nested (type is at max level) then the boundries were updated only : ex: + /* + change = { + "max" : { + type: "updated" + data: "..." + }, + "exclusiveMin": { + type: "deleted" + data .. + } + } + */ + // TODO: This section breaks from the expected format as defined by the diff types. needs to be evaluated if it is working correctly. + if ( + Object.keys(change).some((k) => k == 'max' || k == 'min' || k == 'exclusiveMin' || k == 'exclusiveMax') && + fieldDefinitionAfter?.restrictions && + 'range' in fieldDefinitionAfter.restrictions + ) { + analysis.restrictionsChanges[k]['updated'].push({ + field: field, + // we push the whole range definition since it doesnt make sense to just + // push one boundary. + definition: fieldDefinitionAfter?.restrictions?.range, + }); + } + return; + } + if (isValueChange(change) && change.type !== 'unchanged') { + const definition = change.data; + switch (k) { + case 'codeList': { + analysis.restrictionsChanges[k][change.type].push({ + field: field, + definition, + }); + break; + } + case 'range': { + analysis.restrictionsChanges[k][change.type].push({ + field: field, + definition, + }); + break; + } + case 'regex': { + analysis.restrictionsChanges[k][change.type].push({ + field: field, + definition, + }); + break; + } + case 'required': { + analysis.restrictionsChanges[k][change.type].push({ + field: field, + definition, + }); + break; + } + case 'script': { + analysis.restrictionsChanges[k][change.type].push({ + field: field, + definition, + }); + break; + } + } + } + } + } +}; + +const categorizeFieldChanges = (analysis: ChangeAnalysis, field: string, changes: ValueChange) => { + const changeType = changes.type; + if (changeType === 'created') { + analysis.fields.addedFields.push({ + name: field, + definition: changes.data, + }); + } else if (changeType === 'deleted') { + analysis.fields.deletedFields.push(field); + } +}; + +const categorizeMetaChanges = (analysis: ChangeAnalysis, field: string, metaChanges: FieldChanges) => { + // **** meta changes - core *** + if (metaChanges?.data?.core === true) { + const changeType = metaChanges.type; + if (changeType === 'created' || changeType === 'updated') { + analysis.metaChanges?.core.changedToCore.push(field); + } else if (changeType === 'deleted') { + analysis.metaChanges?.core.changedFromCore.push(field); + } + } +}; diff --git a/packages/client/src/changeAnalysis/index.ts b/packages/client/src/changeAnalysis/index.ts new file mode 100644 index 00000000..46a0feba --- /dev/null +++ b/packages/client/src/changeAnalysis/index.ts @@ -0,0 +1,2 @@ +export * from './changeAnalysisTypes'; +export * from './changeAnalyzer'; From f04f024b803e335d92ece1826a1f05bc34c993f8 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:28:50 -0400 Subject: [PATCH 07/17] Remove worker-threads code from client The client provides the logic for processing and validating data, any parallelization that is desired should be added to a server implementation and not in the reusable client. --- packages/client/src/parallel.ts | 49 -------------------------- packages/client/src/schema-worker.js | 51 ---------------------------- 2 files changed, 100 deletions(-) delete mode 100644 packages/client/src/parallel.ts delete mode 100644 packages/client/src/schema-worker.js diff --git a/packages/client/src/parallel.ts b/packages/client/src/parallel.ts deleted file mode 100644 index 603d57f8..00000000 --- a/packages/client/src/parallel.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { DataRecord, SchemasDictionary, SchemaProcessingResult } from './schema-entities'; -import { StaticPool } from 'node-worker-threads-pool'; -import * as os from 'os'; -import { loggerFor } from './logger'; -const L = loggerFor(__filename); - -// check allowed cpus or use available -const cpuCount = os.cpus().length; -L.info(`available cpus: ${cpuCount}`); -const availableCpus = Number(process.env.ALLOWED_CPUS) || cpuCount; -L.info(`using ${availableCpus} cpus`); - -const pool = new StaticPool({ - size: availableCpus, - task: __dirname + '/schema-worker.js', -}); - -export const processRecord = async ( - dictionary: SchemasDictionary, - schemaName: string, - record: Readonly, - index: number, -): Promise => { - return (await pool.exec({ - dictionary, - schemaName, - record, - index, - })) as Promise; -}; diff --git a/packages/client/src/schema-worker.js b/packages/client/src/schema-worker.js deleted file mode 100644 index cdb44c0a..00000000 --- a/packages/client/src/schema-worker.js +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2020 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -const workerThreads = require('worker_threads'); -const parentPort = workerThreads.parentPort; -// ts node is required to import ts modules like "schema-functions.ts" in this case since -// worker threads run in their own V8 instance -const tsNode = require('ts-node'); - -/** - * when we run the app with node directly like this: - * node -r ts-node/register server.ts - * we will have a registered ts node instance, registering another one will result in wierd behaviour - * however when we run with mocha: - * mocha -r ts-node/register .ts - * (same applies if we run with ts node directly: ts-node server.ts) - * the worker thread won't have an instance of ts node transpiler - * unlike node for some reason which seem to attach the isntance to the worker thread process. - * - * so we had to add this work around to avoid double registry in different run modes. - * root cause of why mocha acts different than node is not found yet. - */ -if (!process[tsNode.REGISTER_INSTANCE]) { - tsNode.register(); -} -const service = require('./schema-functions'); - -function processProxy(args) { - return service.process(args.dictionary, args.schemaName, args.record, args.index); -} - -parentPort.on('message', (args) => { - const result = processProxy(args); - parentPort.postMessage(result); -}); From b63651a7c10aca1e491b518a97d0f0f3e729d50c Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:32:16 -0400 Subject: [PATCH 08/17] Remove types that are duplicate of Lectern dictionary library --- packages/client/src/schema-entities.ts | 221 ----------------------- packages/client/src/types/dataRecords.ts | 39 ++++ packages/client/src/types/index.ts | 1 + 3 files changed, 40 insertions(+), 221 deletions(-) delete mode 100644 packages/client/src/schema-entities.ts create mode 100644 packages/client/src/types/dataRecords.ts create mode 100644 packages/client/src/types/index.ts diff --git a/packages/client/src/schema-entities.ts b/packages/client/src/schema-entities.ts deleted file mode 100644 index 810c9c80..00000000 --- a/packages/client/src/schema-entities.ts +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { loggerFor } from './logger'; -import { DeepReadonly } from 'deep-freeze'; -const L = loggerFor(__filename); - -export class DataRecord { - readonly [k: string]: string | string[]; -} - -export class TypedDataRecord { - readonly [k: string]: SchemaTypes; -} - -export type SchemaTypes = string | string[] | boolean | boolean[] | number | number[] | undefined; - -export interface SchemasDictionary { - version: string; - name: string; - schemas: Array; -} - -export interface SchemaDefinition { - readonly name: string; - readonly description: string; - readonly restrictions: SchemaRestriction; - readonly fields: ReadonlyArray; -} - -export interface SchemasDictionaryDiffs { - [fieldName: string]: FieldDiff; -} - -export interface FieldDiff { - before?: FieldDefinition; - after?: FieldDefinition; - diff: FieldChanges; -} - -export type SchemaData = ReadonlyArray; - -// changes can be nested -// in case of created/delete field we get Change -// in case of simple field change we get {"fieldName": {"data":.., "type": ..}} -// in case of nested fields: {"fieldName1": {"fieldName2": {"data":.., "type": ..}}} -export type FieldChanges = { [field: string]: FieldChanges } | Change; - -export enum ChangeTypeName { - CREATED = 'created', - DELETED = 'deleted', - UPDATED = 'updated', -} - -export interface Change { - type: ChangeTypeName; - data: any; -} - -export interface SchemaRestriction { - foreignKey?: { - schema: string; - mappings: { - local: string; - foreign: string; - }[]; - }[]; - uniqueKey?: string[]; -} - -export interface FieldDefinition { - name: string; - valueType: ValueType; - description: string; - meta?: { key?: boolean; default?: SchemaTypes; core?: boolean; examples?: string }; - restrictions?: { - codeList?: CodeListRestriction; - regex?: string; - script?: Array | string; - required?: boolean; - unique?: boolean; - range?: RangeRestriction; - }; - isArray?: boolean; -} - -export type CodeListRestriction = Array; - -export type RangeRestriction = { - min?: number; - max?: number; - exclusiveMin?: number; - exclusiveMax?: number; -}; - -export enum ValueType { - STRING = 'string', - INTEGER = 'integer', - NUMBER = 'number', - BOOLEAN = 'boolean', -} - -export type SchemaProcessingResult = DeepReadonly<{ - validationErrors: SchemaValidationError[]; - processedRecord: TypedDataRecord; -}>; - -export type BatchProcessingResult = DeepReadonly<{ - validationErrors: SchemaValidationError[]; - processedRecords: TypedDataRecord[]; -}>; - -export enum SchemaValidationErrorTypes { - MISSING_REQUIRED_FIELD = 'MISSING_REQUIRED_FIELD', - INVALID_FIELD_VALUE_TYPE = 'INVALID_FIELD_VALUE_TYPE', - INVALID_BY_REGEX = 'INVALID_BY_REGEX', - INVALID_BY_RANGE = 'INVALID_BY_RANGE', - INVALID_BY_SCRIPT = 'INVALID_BY_SCRIPT', - INVALID_ENUM_VALUE = 'INVALID_ENUM_VALUE', - UNRECOGNIZED_FIELD = 'UNRECOGNIZED_FIELD', - INVALID_BY_UNIQUE = 'INVALID_BY_UNIQUE', - INVALID_BY_FOREIGN_KEY = 'INVALID_BY_FOREIGN_KEY', - INVALID_BY_UNIQUE_KEY = 'INVALID_BY_UNIQUE_KEY', -} - -export interface SchemaValidationError { - readonly errorType: SchemaValidationErrorTypes; - readonly index: number; - readonly fieldName: string; - readonly info: Record; - readonly message: string; -} - -export interface FieldNamesByPriorityMap { - required: string[]; - optional: string[]; -} - -export interface ChangeAnalysis { - fields: { - addedFields: AddedFieldChange[]; - renamedFields: string[]; - deletedFields: string[]; - }; - isArrayDesignationChanges: string[]; - restrictionsChanges: RestrictionChanges; - metaChanges?: MetaChanges; - valueTypeChanges: string[]; -} - -export type RestrictionChanges = { - range: { - [key in ChangeTypeName]: ObjectChange[]; - }; - codeList: { - [key in ChangeTypeName]: ObjectChange[]; - }; - regex: RegexChanges; - required: RequiredChanges; - script: ScriptChanges; -}; - -export type MetaChanges = { - core: { - changedToCore: string[]; // fields that are core now - changedFromCore: string[]; // fields that are not core now - }; -}; - -export type RegexChanges = { - [key in ChangeTypeName]: StringAttributeChange[]; -}; - -export type RequiredChanges = { - [key in ChangeTypeName]: BooleanAttributeChange[]; -}; - -export type ScriptChanges = { - [key in ChangeTypeName]: StringAttributeChange[]; -}; - -export interface AddedFieldChange { - name: string; - definition: FieldDefinition; -} - -export interface ObjectChange { - field: string; - definition: any; -} - -export interface CodeListChange { - field: string; - definition: any; -} - -export interface StringAttributeChange { - field: string; - definition: string; -} - -export interface BooleanAttributeChange { - field: string; - definition: boolean; -} diff --git a/packages/client/src/types/dataRecords.ts b/packages/client/src/types/dataRecords.ts new file mode 100644 index 00000000..43adfdab --- /dev/null +++ b/packages/client/src/types/dataRecords.ts @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * Represents a data record as taken from an input file. All values are the original strings and have not been validated into + * numbers/bools or split into arrays. + */ +export type UnprocessedDataRecord = { + [k: string]: string | string[]; +}; + +/** + * The available data types for a field in a Lectern Schema. + */ +export type DataRecordValue = string | string[] | number | number[] | boolean | boolean[] | undefined; + +/** + * Represents a data record after processing, with the data checked to be a valid type for a Lectern schema. + * The type of data should match the expected type for the given field. + */ +export type DataRecord = { + [key: string]: DataRecordValue; +}; diff --git a/packages/client/src/types/index.ts b/packages/client/src/types/index.ts new file mode 100644 index 00000000..7eaa8a6b --- /dev/null +++ b/packages/client/src/types/index.ts @@ -0,0 +1 @@ +export * from './dataRecords'; From 5865a60a32f2d87e739f7020a182405c2dbe0396 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:38:05 -0400 Subject: [PATCH 09/17] Move data processing functions out of schema-functions into their own directory --- .../src/processing/convertDataValueTypes.ts | 116 +++++++++ packages/client/src/processing/index.ts | 241 ++++++++++++++++++ .../src/processing/processingResultTypes.ts | 39 +++ 3 files changed, 396 insertions(+) create mode 100644 packages/client/src/processing/convertDataValueTypes.ts create mode 100644 packages/client/src/processing/index.ts create mode 100644 packages/client/src/processing/processingResultTypes.ts diff --git a/packages/client/src/processing/convertDataValueTypes.ts b/packages/client/src/processing/convertDataValueTypes.ts new file mode 100644 index 00000000..986b7283 --- /dev/null +++ b/packages/client/src/processing/convertDataValueTypes.ts @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import _ from 'lodash'; + +import { Singular } from 'common'; +import { Schema, SchemaField, SchemaFieldValueType } from 'dictionary'; + +import { DataRecord, DataRecordValue, UnprocessedDataRecord } from '../types'; +import { convertToArray, isEmpty } from '../utils'; +import { SchemaValidationError, SchemaValidationErrorTypes } from '../validation'; + +/** + * Warning: + * This needs to be provided records that have already had their types validated, there is little to no checking that the values + * this converts will be correct, and no errors are thrown by failed conversions. + * + * @param schemaDef + * @param record + * @param index + * @param recordErrors + * @returns + */ +export const convertFromRawStrings = ( + schemaDef: Schema, + record: UnprocessedDataRecord, + index: number, + recordErrors: ReadonlyArray, +): DataRecord => { + const mutableRecord: DataRecord = { ...record }; + schemaDef.fields.forEach((field) => { + // if there was an error for this field don't convert it. this means a string was passed instead of number or boolean + // this allows us to continue other validations without hiding possible errors downstream. + + if ( + recordErrors.find( + (er) => er.errorType === SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE && er.fieldName === field.name, + ) + ) { + return undefined; + } + + /* + * if the field is missing from the records don't set it to undefined + */ + if (!_.has(record, field.name)) { + return; + } + + // need to check how it behaves for record[field.name] === "" + if (isEmpty(record[field.name])) { + mutableRecord[field.name] = undefined; + return; + } + + const rawValue = record[field.name]; + + if (field.isArray) { + const rawValueAsArray = convertToArray(rawValue); + // TODO: Keeping this type assertion for during the type refactoring process. We need to refactor how values are validated as matching their corresponding types + // refactoring type checking/conversion will result in combining the conversion and type checking code into a single place. Right now its possible to run the converter + // on values that have not been properly validated. + // This type assertion is needed because the code as is results in teh type `(string | number | boolean | undefined)[]` instead of `string[] | number[] | boolean[] | undefined` + mutableRecord[field.name] = rawValueAsArray.map((value) => getTypedValue(field, value)) as DataRecordValue; + } else { + const rawValueAsString = Array.isArray(rawValue) ? rawValue.join('') : rawValue; + mutableRecord[field.name] = getTypedValue(field, rawValueAsString); + } + }); + return mutableRecord; +}; + +const getTypedValue = (field: SchemaField, rawValue: string): Singular => { + switch (field.valueType) { + case SchemaFieldValueType.Values.boolean: { + return Boolean(rawValue.toLowerCase()); + } + case SchemaFieldValueType.Values.integer: { + return Number(rawValue); + } + case SchemaFieldValueType.Values.number: { + return Number(rawValue); + } + case SchemaFieldValueType.Values.string: { + // For string fields with a codeList restriction: + // we want to format the value with the same letter cases as is defined in the codeList + if (field.restrictions?.codeList && Array.isArray(field.restrictions.codeList)) { + const formattedField = field.restrictions.codeList.find( + (codeListOption) => codeListOption.toString().toLowerCase() === rawValue.toString().toLowerCase(), + ); + if (formattedField) { + return formattedField; + } + } + + // Return original string + return rawValue; + } + } +}; diff --git a/packages/client/src/processing/index.ts b/packages/client/src/processing/index.ts new file mode 100644 index 00000000..dfafb5cf --- /dev/null +++ b/packages/client/src/processing/index.ts @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { NotFoundError } from 'common'; +import { Dictionary, Schema } from 'dictionary'; +import _ from 'lodash'; + +import { loggerFor } from '../logger'; +import { DataRecord, UnprocessedDataRecord } from '../types/dataRecords'; +import { convertToArray, isEmpty, isNotAbsent, isString, isStringArray, notEmpty } from '../utils'; +import type { SchemaValidationError } from '../validation'; +import * as validation from '../validation'; +import { convertFromRawStrings } from './convertDataValueTypes'; +import { BatchProcessingResult, FieldNamesByPriorityMap, SchemaProcessingResult } from './processingResultTypes'; + +const L = loggerFor(__filename); + +export const processSchemas = ( + dictionary: Dictionary, + schemasData: Record, +): Record => { + const results: Record = {}; + + Object.keys(schemasData).forEach((schemaName) => { + // Run validations at the record level + const recordLevelValidationResults = processRecords(dictionary, schemaName, schemasData[schemaName]); + + // Run cross-schema validations + const schemaDef = getNotNullSchemaDefinitionFromDictionary(dictionary, schemaName); + const crossSchemaLevelValidationResults = validation + .runCrossSchemaValidationPipeline(schemaDef, schemasData, [validation.validateForeignKeys]) + .filter(notEmpty); + + const allErrorsBySchema: validation.SchemaValidationError[] = [ + ...recordLevelValidationResults.validationErrors, + ...crossSchemaLevelValidationResults, + ]; + + results[schemaName] = { + validationErrors: allErrorsBySchema, + processedRecords: recordLevelValidationResults.processedRecords, + }; + }); + + return results; +}; + +export const processRecords = ( + dictionary: Dictionary, + definition: string, + records: UnprocessedDataRecord[], +): BatchProcessingResult => { + const schemaDef = getNotNullSchemaDefinitionFromDictionary(dictionary, definition); + + let validationErrors: SchemaValidationError[] = []; + const processedRecords: DataRecord[] = []; + + records.forEach((dataRecord, index) => { + const result = process(dictionary, definition, dataRecord, index); + validationErrors = validationErrors.concat(result.validationErrors); + processedRecords.push(_.cloneDeep(result.processedRecord)); + }); + // Record set level validations + const newErrors = validateRecordsSet(schemaDef, processedRecords); + validationErrors.push(...newErrors); + L.debug( + `done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${processedRecords.length}`, + ); + + return { + validationErrors, + processedRecords, + }; +}; + +export const process = ( + dictionary: Dictionary, + schemaName: string, + data: Readonly, + index: number, +): SchemaProcessingResult => { + const schemaDef = dictionary.schemas.find((e) => e.name === schemaName); + + if (!schemaDef) { + throw new Error(`no schema found for : ${schemaName}`); + } + + let validationErrors: SchemaValidationError[] = []; + + const defaultedRecord = populateDefaults(schemaDef, data, index); + L.debug(`done populating defaults for record #${index}`); + const result = validateUnprocessedRecord(schemaDef, defaultedRecord, index); + L.debug(`done validation for record #${index}`); + if (result && result.length > 0) { + L.debug(`${result.length} validation errors for record #${index}`); + validationErrors = validationErrors.concat(result); + } + const convertedRecord = convertFromRawStrings(schemaDef, defaultedRecord, index, result); + L.debug(`converted row #${index} from raw strings`); + const postTypeConversionValidationResult = validateAfterTypeConversion( + schemaDef, + _.cloneDeep(convertedRecord) as DataRecord, + index, + ); + + if (postTypeConversionValidationResult && postTypeConversionValidationResult.length > 0) { + validationErrors = validationErrors.concat(postTypeConversionValidationResult); + } + + L.debug(`done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${convertedRecord}`); + + return { + validationErrors, + processedRecord: convertedRecord, + }; +}; + +const getNotNullSchemaDefinitionFromDictionary = (dictionary: Dictionary, schemaName: string): Schema => { + const schemaDef = dictionary.schemas.find((e) => e.name === schemaName); + if (!schemaDef) { + throw new Error(`no schema found for : ${schemaName}`); + } + return schemaDef; +}; + +export const getSchemaFieldNamesWithPriority = (schema: Dictionary, definition: string): FieldNamesByPriorityMap => { + const schemaDef = schema.schemas.find((schema) => schema.name === definition); + if (!schemaDef) { + throw new NotFoundError(`no schema found for : ${definition}`); + } + const fieldNamesMapped: FieldNamesByPriorityMap = { required: [], optional: [] }; + schemaDef.fields.forEach((field) => { + if (field.restrictions?.required) { + fieldNamesMapped.required.push(field.name); + } else { + fieldNamesMapped.optional.push(field.name); + } + }); + return fieldNamesMapped; +}; + +/** + * Populate the passed records with the default value based on the field name if the field is + * missing from the records it will NOT be added. + * @param definition the name of the schema definition to use for these records + * @param records the list of records to populate with the default values. + */ +const populateDefaults = (schemaDef: Schema, record: UnprocessedDataRecord, index: number): UnprocessedDataRecord => { + const clonedRecord = _.cloneDeep(record); + schemaDef.fields.forEach((field) => { + const defaultValue = field.meta && field.meta.default; + if (isEmpty(defaultValue)) return undefined; + + const value = record[field.name]; + + // data record value is (or is expected to be) just one string + if (isString(value) && !field.isArray) { + if (isNotAbsent(value) && value.trim() === '') { + L.debug(`populating Default: "${defaultValue}" for "${field.name}" of record at index ${index}`); + clonedRecord[field.name] = `${defaultValue}`; + } + return undefined; + } + + // data record value is (or is expected to be) array of string + if (isStringArray(value) && field.isArray) { + if (notEmpty(value) && value.every((v) => v.trim() === '')) { + L.debug(`populating Default: "${defaultValue}" for ${field.name} of record at index ${index}`); + const arrayDefaultValue = convertToArray(defaultValue); + clonedRecord[field.name] = arrayDefaultValue.map((v) => `${v}`); + } + return undefined; + } + }); + + return _.cloneDeep(clonedRecord); +}; + +/** + * Run schema validation pipeline for a schema defintion on the list of records provided. + * @param definition the schema definition name. + * @param record the records to validate. + */ +const validateUnprocessedRecord = ( + schemaDef: Schema, + record: UnprocessedDataRecord, + index: number, +): ReadonlyArray => { + const majorErrors = validation + .runUnprocessedRecordValidationPipeline(record, index, schemaDef.fields, [ + validation.validateFieldNames, + validation.validateNonArrayFields, + validation.validateRequiredFields, + validation.validateValueTypes, + ]) + .filter(notEmpty); + return [...majorErrors]; +}; + +const validateAfterTypeConversion = ( + schemaDef: Schema, + record: DataRecord, + index: number, +): ReadonlyArray => { + const validationErrors = validation + .runRecordValidationPipeline(record, index, schemaDef.fields, [ + validation.validateRegex, + validation.validateRange, + validation.validateCodeList, + validation.validateScript, + ]) + .filter(notEmpty); + + return [...validationErrors]; +}; + +function validateRecordsSet(schemaDef: Schema, processedRecords: DataRecord[]) { + const validationErrors = validation + .runDatasetValidationPipeline(processedRecords, schemaDef, [ + validation.validateUnique, + validation.validateUniqueKey, + ]) + .filter(notEmpty); + return validationErrors; +} diff --git a/packages/client/src/processing/processingResultTypes.ts b/packages/client/src/processing/processingResultTypes.ts new file mode 100644 index 00000000..0a66a4b5 --- /dev/null +++ b/packages/client/src/processing/processingResultTypes.ts @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { Schema } from 'dictionary'; +import { DataRecord } from '../types/dataRecords'; +import { SchemaValidationError } from '../validation/types/validationErrorTypes'; + +export type ProcessingFunction = (schema: Schema, rec: Readonly, index: number) => any; + +export type SchemaProcessingResult = { + validationErrors: SchemaValidationError[]; + processedRecord: DataRecord; +}; + +export type BatchProcessingResult = { + validationErrors: SchemaValidationError[]; + processedRecords: DataRecord[]; +}; + +export interface FieldNamesByPriorityMap { + required: string[]; + optional: string[]; +} From 054ce0ebbc6e691d5b91baa21df99c9dd80ae9c2 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:39:28 -0400 Subject: [PATCH 10/17] Reorganize validation and error message code --- packages/client/src/schema-error-messages.ts | 109 --- packages/client/src/schema-functions.ts | 855 ------------------ .../fieldNamesValidation.ts | 48 + .../valueTypeValidation.ts | 118 +++ .../fieldRestrictions/codeListValidation.ts | 81 ++ .../fieldRestrictions/rangeValidation.ts | 84 ++ .../fieldRestrictions/regexValidation.ts | 84 ++ .../fieldRestrictions/requiredValidation.ts | 72 ++ .../fieldRestrictions/scriptValidation.ts | 134 +++ packages/client/src/validation/index.ts | 31 + .../foreignKeysValidation.ts | 127 +++ .../schemaRestrictions/uniqueKeyValidation.ts | 69 ++ .../schemaRestrictions/uniqueValidation.ts | 63 ++ packages/client/src/validation/types/index.ts | 2 + .../validation/types/validationErrorTypes.ts | 110 +++ .../types/validationFunctionTypes.ts | 45 + .../utils/datasetUtils.ts} | 68 +- .../src/validation/utils/fieldTypeUtils.ts | 43 + .../src/validation/utils/rangeToSymbol.ts | 47 + .../src/validation/validationPipelines.ts | 79 ++ 20 files changed, 1260 insertions(+), 1009 deletions(-) delete mode 100644 packages/client/src/schema-error-messages.ts delete mode 100644 packages/client/src/schema-functions.ts create mode 100644 packages/client/src/validation/dataRecordValidation/fieldNamesValidation.ts create mode 100644 packages/client/src/validation/dataRecordValidation/valueTypeValidation.ts create mode 100644 packages/client/src/validation/fieldRestrictions/codeListValidation.ts create mode 100644 packages/client/src/validation/fieldRestrictions/rangeValidation.ts create mode 100644 packages/client/src/validation/fieldRestrictions/regexValidation.ts create mode 100644 packages/client/src/validation/fieldRestrictions/requiredValidation.ts create mode 100644 packages/client/src/validation/fieldRestrictions/scriptValidation.ts create mode 100644 packages/client/src/validation/index.ts create mode 100644 packages/client/src/validation/schemaRestrictions/foreignKeysValidation.ts create mode 100644 packages/client/src/validation/schemaRestrictions/uniqueKeyValidation.ts create mode 100644 packages/client/src/validation/schemaRestrictions/uniqueValidation.ts create mode 100644 packages/client/src/validation/types/index.ts create mode 100644 packages/client/src/validation/types/validationErrorTypes.ts create mode 100644 packages/client/src/validation/types/validationFunctionTypes.ts rename packages/client/src/{records-operations.ts => validation/utils/datasetUtils.ts} (53%) create mode 100644 packages/client/src/validation/utils/fieldTypeUtils.ts create mode 100644 packages/client/src/validation/utils/rangeToSymbol.ts create mode 100644 packages/client/src/validation/validationPipelines.ts diff --git a/packages/client/src/schema-error-messages.ts b/packages/client/src/schema-error-messages.ts deleted file mode 100644 index 3ed08ab0..00000000 --- a/packages/client/src/schema-error-messages.ts +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { isArray } from 'lodash'; -import { RangeRestriction } from './schema-entities'; - -function getForeignKeyErrorMsg(errorData: any) { - const valueEntries = Object.entries(errorData.info.value); - const formattedKeyValues: string[] = valueEntries.map(([key, value]) => { - if (isArray(value)) { - return `${key}: [${value.join(', ')}]`; - } else { - return `${key}: ${value}`; - } - }); - const valuesAsString = formattedKeyValues.join(', '); - const detail = `Key ${valuesAsString} is not present in schema ${errorData.info.foreignSchema}`; - const msg = `Record violates foreign key restriction defined for field(s) ${errorData.fieldName}. ${detail}.`; - return msg; -} - -function getUniqueKeyErrorMsg(errorData: any) { - const uniqueKeyFields: string[] = errorData.info.uniqueKeyFields; - const formattedKeyValues: string[] = uniqueKeyFields.map((fieldName) => { - const value = errorData.info.value[fieldName]; - if (isArray(value)) { - return `${fieldName}: [${value.join(', ')}]`; - } else { - return `${fieldName}: ${value === '' ? 'null' : value}`; - } - }); - const valuesAsString = formattedKeyValues.join(', '); - const msg = `Key ${valuesAsString} must be unique.`; - return msg; -} - -const INVALID_VALUE_ERROR_MESSAGE = 'The value is not permissible for this field.'; -const ERROR_MESSAGES: { [key: string]: (errorData: any) => string } = { - INVALID_FIELD_VALUE_TYPE: () => INVALID_VALUE_ERROR_MESSAGE, - INVALID_BY_REGEX: (errData) => getRegexErrorMsg(errData.info), - INVALID_BY_RANGE: (errorData) => `Value is out of permissible range, value must be ${rangeToSymbol(errorData.info)}.`, - INVALID_BY_SCRIPT: (error) => error.info.message, - INVALID_ENUM_VALUE: () => INVALID_VALUE_ERROR_MESSAGE, - MISSING_REQUIRED_FIELD: (errorData) => `${errorData.fieldName} is a required field.`, - INVALID_BY_UNIQUE: (errorData) => `Value for ${errorData.fieldName} must be unique.`, - INVALID_BY_FOREIGN_KEY: (errorData) => getForeignKeyErrorMsg(errorData), - INVALID_BY_UNIQUE_KEY: (errorData) => getUniqueKeyErrorMsg(errorData), -}; - -// Returns the formatted message for the given error key, taking any required properties from the info object -// Default value is the errorType itself (so we can identify errorTypes that we are missing messages for and the user could look up the error meaning in our docs) -const schemaErrorMessage = (errorType: string, errorData: any = {}): string => { - return errorType && Object.keys(ERROR_MESSAGES).includes(errorType) - ? ERROR_MESSAGES[errorType](errorData) - : errorType; -}; - -const rangeToSymbol = (range: RangeRestriction): string => { - let minString = ''; - let maxString = ''; - - const hasBothRange = - (range.min !== undefined || range.exclusiveMin !== undefined) && - (range.max != undefined || range.exclusiveMax !== undefined); - - if (range.min !== undefined) { - minString = `>= ${range.min}`; - } - - if (range.exclusiveMin !== undefined) { - minString = `> ${range.exclusiveMin}`; - } - - if (range.max !== undefined) { - maxString = `<= ${range.max}`; - } - - if (range.exclusiveMax !== undefined) { - maxString = `< ${range.exclusiveMax}`; - } - - return hasBothRange ? `${minString} and ${maxString}` : `${minString}${maxString}`; -}; - -function getRegexErrorMsg(info: any) { - let msg = `The value is not a permissible for this field, it must meet the regular expression: "${info.regex}".`; - if (info.examples) { - msg = msg + ` Examples: ${info.examples}`; - } - return msg; -} - -export default schemaErrorMessage; diff --git a/packages/client/src/schema-functions.ts b/packages/client/src/schema-functions.ts deleted file mode 100644 index 52c0d7fa..00000000 --- a/packages/client/src/schema-functions.ts +++ /dev/null @@ -1,855 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { - SchemaValidationError, - TypedDataRecord, - SchemaTypes, - SchemaProcessingResult, - FieldNamesByPriorityMap, - BatchProcessingResult, - CodeListRestriction, - RangeRestriction, - SchemaData, -} from './schema-entities'; -import vm from 'vm'; -import { - SchemasDictionary, - SchemaDefinition, - FieldDefinition, - ValueType, - DataRecord, - SchemaValidationErrorTypes, -} from './schema-entities'; - -import { - Checks, - notEmpty, - isEmptyString, - isAbsent, - F, - isNotAbsent, - isStringArray, - isString, - isEmpty, - convertToArray, - isNumberArray, -} from './utils'; -import schemaErrorMessage from './schema-error-messages'; -import { loggerFor } from './logger'; -import { DeepReadonly } from 'deep-freeze'; -import _, { isArray } from 'lodash'; -import { findDuplicateKeys, findMissingForeignKeys } from './records-operations'; -const L = loggerFor(__filename); - -export const getSchemaFieldNamesWithPriority = ( - schema: SchemasDictionary, - definition: string, -): FieldNamesByPriorityMap => { - const schemaDef: SchemaDefinition | undefined = schema.schemas.find((schema) => schema.name === definition); - if (!schemaDef) { - throw new Error(`no schema found for : ${definition}`); - } - const fieldNamesMapped: FieldNamesByPriorityMap = { required: [], optional: [] }; - schemaDef.fields.forEach((field) => { - if (field.restrictions && field.restrictions.required) { - fieldNamesMapped.required.push(field.name); - } else { - fieldNamesMapped.optional.push(field.name); - } - }); - return fieldNamesMapped; -}; - -const getNotNullSchemaDefinitionFromDictionary = ( - dictionary: SchemasDictionary, - schemaName: string, -): SchemaDefinition => { - const schemaDef: SchemaDefinition | undefined = dictionary.schemas.find((e) => e.name === schemaName); - if (!schemaDef) { - throw new Error(`no schema found for : ${schemaName}`); - } - return schemaDef; -}; - -export const processSchemas = ( - dictionary: SchemasDictionary, - schemasData: Record, -): Record => { - Checks.checkNotNull('dictionary', dictionary); - Checks.checkNotNull('schemasData', schemasData); - - const results: Record = {}; - - Object.keys(schemasData).forEach((schemaName) => { - // Run validations at the record level - const recordLevelValidationResults = processRecords(dictionary, schemaName, schemasData[schemaName]); - - // Run cross-schema validations - const schemaDef: SchemaDefinition = getNotNullSchemaDefinitionFromDictionary(dictionary, schemaName); - const crossSchemaLevelValidationResults = validation - .runCrossSchemaValidationPipeline(schemaDef, schemasData, [validation.validateForeignKey]) - .filter(notEmpty); - - const recordLevelErrors = recordLevelValidationResults.validationErrors.map((x) => { - return { - errorType: x.errorType, - index: x.index, - fieldName: x.fieldName, - info: x.info, - message: x.message, - }; - }); - - const crossSchemaLevelErrors = crossSchemaLevelValidationResults.map((x) => { - return { - errorType: x.errorType, - index: x.index, - fieldName: x.fieldName, - info: x.info, - message: x.message, - }; - }); - - const allErrorsBySchema = [...recordLevelErrors, ...crossSchemaLevelErrors]; - - results[schemaName] = F({ - validationErrors: allErrorsBySchema, - processedRecords: recordLevelValidationResults.processedRecords, - }); - }); - - return results; -}; - -export const processRecords = ( - dataSchema: SchemasDictionary, - definition: string, - records: ReadonlyArray, -): BatchProcessingResult => { - Checks.checkNotNull('records', records); - Checks.checkNotNull('dataSchema', dataSchema); - Checks.checkNotNull('definition', definition); - - const schemaDef: SchemaDefinition = getNotNullSchemaDefinitionFromDictionary(dataSchema, definition); - - let validationErrors: SchemaValidationError[] = []; - const processedRecords: TypedDataRecord[] = []; - - records.forEach((r, i) => { - const result = process(dataSchema, definition, r, i); - validationErrors = validationErrors.concat(result.validationErrors); - processedRecords.push(_.cloneDeep(result.processedRecord) as TypedDataRecord); - }); - // Record set level validations - const newErrors = validateRecordsSet(schemaDef, processedRecords); - validationErrors.push(...newErrors); - L.debug( - `done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${processedRecords.length}`, - ); - - return F({ - validationErrors, - processedRecords, - }); -}; - -export const process = ( - dataSchema: SchemasDictionary, - definition: string, - rec: Readonly, - index: number, -): SchemaProcessingResult => { - Checks.checkNotNull('records', rec); - Checks.checkNotNull('dataSchema', dataSchema); - Checks.checkNotNull('definition', definition); - - const schemaDef: SchemaDefinition | undefined = dataSchema.schemas.find((e) => e.name === definition); - - if (!schemaDef) { - throw new Error(`no schema found for : ${definition}`); - } - - let validationErrors: SchemaValidationError[] = []; - - const defaultedRecord: DataRecord = populateDefaults(schemaDef, F(rec), index); - L.debug(`done populating defaults for record #${index}`); - const result = validate(schemaDef, defaultedRecord, index); - L.debug(`done validation for record #${index}`); - if (result && result.length > 0) { - L.debug(`${result.length} validation errors for record #${index}`); - validationErrors = validationErrors.concat(result); - } - const convertedRecord = convertFromRawStrings(schemaDef, defaultedRecord, index, result); - L.debug(`converted row #${index} from raw strings`); - const postTypeConversionValidationResult = validateAfterTypeConversion( - schemaDef, - _.cloneDeep(convertedRecord) as DataRecord, - index, - ); - - if (postTypeConversionValidationResult && postTypeConversionValidationResult.length > 0) { - validationErrors = validationErrors.concat(postTypeConversionValidationResult); - } - - L.debug(`done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${convertedRecord}`); - - return F({ - validationErrors, - processedRecord: convertedRecord, - }); -}; - -/** - * Populate the passed records with the default value based on the field name if the field is - * missing from the records it will NOT be added. - * @param definition the name of the schema definition to use for these records - * @param records the list of records to populate with the default values. - */ -const populateDefaults = ( - schemaDef: Readonly, - record: DeepReadonly, - index: number, -): DataRecord => { - Checks.checkNotNull('records', record); - L.debug(`in populateDefaults ${schemaDef.name}, ${record}`); - const mutableRecord: RawMutableRecord = _.cloneDeep(record) as RawMutableRecord; - const x: SchemaDefinition = schemaDef; - schemaDef.fields.forEach((field) => { - const defaultValue = field.meta && field.meta.default; - if (isEmpty(defaultValue)) return undefined; - - const value = record[field.name]; - - // data record value is (or is expected to be) just one string - if (isString(value) && !field.isArray) { - if (isNotAbsent(value) && value.trim() === '') { - L.debug(`populating Default: ${defaultValue} for ${field.name} in record : ${record}`); - mutableRecord[field.name] = `${defaultValue}`; - } - return undefined; - } - - // data record value is (or is expected to be) array of string - if (isStringArray(value) && field.isArray) { - if (notEmpty(value) && value.every((v) => v.trim() === '')) { - L.debug(`populating Default: ${defaultValue} for ${field.name} in record : ${record}`); - const arrayDefaultValue = convertToArray(defaultValue); - mutableRecord[field.name] = arrayDefaultValue.map((v) => `${v}`); - } - return undefined; - } - }); - - return _.cloneDeep(mutableRecord); -}; - -const convertFromRawStrings = ( - schemaDef: SchemaDefinition, - record: DataRecord, - index: number, - recordErrors: ReadonlyArray, -): DeepReadonly => { - const mutableRecord: MutableRecord = { ...record }; - schemaDef.fields.forEach((field) => { - // if there was an error for this field don't convert it. this means a string was passed instead of number or boolean - // this allows us to continue other validations without hiding possible errors down. - if ( - recordErrors.find( - (er) => er.errorType == SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE && er.fieldName == field.name, - ) - ) { - return undefined; - } - - /* - * if the field is missing from the records don't set it to undefined - */ - if (!_.has(record, field.name)) { - return; - } - - // need to check how it behaves for record[field.name] == "" - if (isEmpty(record[field.name])) { - mutableRecord[field.name] = undefined; - return; - } - - const valueType = field.valueType; - const rawValue = record[field.name]; - - if (field.isArray) { - const rawValues = convertToArray(rawValue); - mutableRecord[field.name] = rawValues.map( - (rv) => getTypedValue(field, valueType, rv) as any, // fix type here - ); - } else { - mutableRecord[field.name] = getTypedValue(field, valueType, rawValue as string); - } - }); - return F(mutableRecord); -}; - -const getTypedValue = (field: FieldDefinition, valueType: ValueType, rawValue: string) => { - let formattedFieldValue = rawValue; - // convert field to match corresponding enum from codelist, if possible - if (field.restrictions && field.restrictions.codeList && valueType === ValueType.STRING) { - const formattedField = field.restrictions.codeList.find( - (e) => e.toString().toLowerCase() === rawValue.toString().toLowerCase(), - ); - if (formattedField) { - formattedFieldValue = formattedField as string; - } - } - - let typedValue: SchemaTypes = rawValue; - switch (valueType) { - case ValueType.STRING: - typedValue = formattedFieldValue; - break; - case ValueType.INTEGER: - typedValue = Number(rawValue); - break; - case ValueType.NUMBER: - typedValue = Number(rawValue); - break; - case ValueType.BOOLEAN: - // we have to lower case in case of inconsistent letters (boolean requires all small letters). - typedValue = Boolean(rawValue.toLowerCase()); - break; - } - - return typedValue; -}; - -/** - * A "select" function that retrieves specific fields from the dataset as a record, as well as the numeric position of each row in the dataset. - * @param dataset Dataset to select fields from. - * @param fields Array with names of the fields to select. - * @returns A tuple array. In each tuple, the first element is the index of the row in the dataset, and the second value is the record with the - * selected values. - */ -const selectFieldsFromDataset = ( - dataset: SchemaData, - fields: string[], -): [number, Record][] => { - const records: [number, Record][] = []; - dataset.forEach((row, index) => { - const values: Record = {}; - fields.forEach((field) => { - values[field] = row[field] || ''; - }); - records.push([index, values]); - }); - return records; -}; - -/** - * Run schema validation pipeline for a schema defintion on the list of records provided. - * @param definition the schema definition name. - * @param record the records to validate. - */ -const validate = ( - schemaDef: SchemaDefinition, - record: DataRecord, - index: number, -): ReadonlyArray => { - const majorErrors = validation - .runValidationPipeline(record, index, schemaDef.fields, [ - validation.validateFieldNames, - validation.validateNonArrayFields, - validation.validateRequiredFields, - validation.validateValueTypes, - ]) - .filter(notEmpty); - return [...majorErrors]; -}; - -const validateAfterTypeConversion = ( - schemaDef: SchemaDefinition, - record: TypedDataRecord, - index: number, -): ReadonlyArray => { - const validationErrors = validation - .runValidationPipeline(record, index, schemaDef.fields, [ - validation.validateRegex, - validation.validateRange, - validation.validateEnum, - validation.validateScript, - ]) - .filter(notEmpty); - - return [...validationErrors]; -}; -export type ProcessingFunction = (schema: SchemaDefinition, rec: Readonly, index: number) => any; - -type MutableRecord = { [key: string]: SchemaTypes }; -type RawMutableRecord = { [key: string]: string | string[] }; - -namespace validation { - // these validation functions run AFTER the record has been converted to the correct types from raw strings - export type TypedValidationFunction = ( - rec: TypedDataRecord, - index: number, - fields: Array, - ) => Array; - - // these validation functions run BEFORE the record has been converted to the correct types from raw strings - export type ValidationFunction = ( - rec: DataRecord, - index: number, - fields: Array, - ) => Array; - - // these validation functions run AFTER the records has been converted to the correct types from raw strings, and apply to a dataset instead of - // individual records - export type TypedDatasetValidationFunction = ( - dataset: Array, - schemaDef: SchemaDefinition, - ) => Array; - - export type CrossSchemaValidationFunction = ( - schemaDef: SchemaDefinition, - schemasData: Record, - ) => Array; - - export const runValidationPipeline = ( - rec: DataRecord | TypedDataRecord, - index: number, - fields: ReadonlyArray, - funs: Array, - ) => { - let result: Array = []; - for (const fun of funs) { - if (rec instanceof DataRecord) { - const typedFunc = fun as ValidationFunction; - result = result.concat(typedFunc(rec as DataRecord, index, getValidFields(result, fields))); - } else { - const typedFunc = fun as TypedValidationFunction; - result = result.concat(typedFunc(rec as TypedDataRecord, index, getValidFields(result, fields))); - } - } - return result; - }; - - export const runDatasetValidationPipeline = ( - dataset: Array, - schemaDef: SchemaDefinition, - funs: Array, - ) => { - let result: Array = []; - for (const fun of funs) { - const typedFunc = fun as TypedDatasetValidationFunction; - result = result.concat(typedFunc(dataset, schemaDef)); - } - return result; - }; - - export const runCrossSchemaValidationPipeline = ( - schemaDef: SchemaDefinition, - schemasData: Record, - funs: Array, - ) => { - let result: Array = []; - for (const fun of funs) { - const typedFunc = fun as CrossSchemaValidationFunction; - result = result.concat(typedFunc(schemaDef, schemasData)); - } - return result; - }; - - export const validateRegex: TypedValidationFunction = ( - rec: TypedDataRecord, - index: number, - fields: ReadonlyArray, - ) => { - return fields - .map((field) => { - const recordFieldValues = convertToArray(rec[field.name]); - if (!isStringArray(recordFieldValues)) return undefined; - - const regex = field.restrictions?.regex; - if (isEmpty(regex)) return undefined; - - const invalidValues = recordFieldValues.filter((v) => isInvalidRegexValue(regex, v)); - if (invalidValues.length !== 0) { - const examples = field.meta?.examples; - const info = { value: invalidValues, regex, examples }; - return buildError(SchemaValidationErrorTypes.INVALID_BY_REGEX, field.name, index, info); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateRange: TypedValidationFunction = ( - rec: TypedDataRecord, - index: number, - fields: ReadonlyArray, - ) => { - return fields - .map((field) => { - const recordFieldValues = convertToArray(rec[field.name]); - if (!isNumberArray(recordFieldValues)) return undefined; - - const range = field.restrictions?.range; - if (isEmpty(range)) return undefined; - - const invalidValues = recordFieldValues.filter((v) => isOutOfRange(range, v)); - if (invalidValues.length !== 0) { - const info = { value: invalidValues, ...range }; - return buildError(SchemaValidationErrorTypes.INVALID_BY_RANGE, field.name, index, info); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateScript: TypedValidationFunction = ( - rec: TypedDataRecord, - index: number, - fields: Array, - ) => { - return fields - .map((field) => { - if (field.restrictions && field.restrictions.script) { - const scriptResult = validateWithScript(field, rec); - if (!scriptResult.valid) { - return buildError(SchemaValidationErrorTypes.INVALID_BY_SCRIPT, field.name, index, { - message: scriptResult.message, - value: rec[field.name], - }); - } - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateEnum: TypedValidationFunction = ( - rec: TypedDataRecord, - index: number, - fields: Array, - ) => { - return fields - .map((field) => { - const codeList = field.restrictions?.codeList || undefined; - if (isEmpty(codeList)) return undefined; - - const recordFieldValues = convertToArray(rec[field.name]); // put all values into array for easier validation - const invalidValues = recordFieldValues.filter((val) => isInvalidEnumValue(codeList, val)); - - if (invalidValues.length !== 0) { - const info = { value: invalidValues }; - return buildError(SchemaValidationErrorTypes.INVALID_ENUM_VALUE, field.name, index, info); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateUnique: TypedDatasetValidationFunction = ( - dataset: Array, - schemaDef: SchemaDefinition, - ) => { - const errors: Array = []; - schemaDef.fields.forEach((field) => { - const unique = field.restrictions?.unique || undefined; - if (!unique) return undefined; - const keysToValidate = selectFieldsFromDataset(dataset as DataRecord[], [field.name]); - const duplicateKeys = findDuplicateKeys(keysToValidate); - - duplicateKeys.forEach(([index, record]) => { - const info = { value: record[field.name] }; - errors.push(buildError(SchemaValidationErrorTypes.INVALID_BY_UNIQUE, field.name, index, info)); - }); - }); - return errors; - }; - - export const validateUniqueKey: TypedDatasetValidationFunction = ( - dataset: Array, - schemaDef: SchemaDefinition, - ) => { - const errors: Array = []; - const uniqueKeyRestriction = schemaDef?.restrictions?.uniqueKey; - if (uniqueKeyRestriction) { - const uniqueKeyFields: string[] = uniqueKeyRestriction; - const keysToValidate = selectFieldsFromDataset(dataset as SchemaData, uniqueKeyFields); - const duplicateKeys = findDuplicateKeys(keysToValidate); - - duplicateKeys.forEach(([index, record]) => { - const info = { value: record, uniqueKeyFields: uniqueKeyFields }; - errors.push( - buildError(SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, uniqueKeyFields.join(', '), index, info), - ); - }); - } - return errors; - }; - - export const validateValueTypes: ValidationFunction = ( - rec: DataRecord, - index: number, - fields: Array, - ) => { - return fields - .map((field) => { - if (isEmpty(rec[field.name])) return undefined; - - const recordFieldValues = convertToArray(rec[field.name]); // put all values into array - const invalidValues = recordFieldValues.filter((v) => isInvalidFieldType(field.valueType, v)); - const info = { value: invalidValues }; - - if (invalidValues.length !== 0) { - return buildError(SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, field.name, index, info); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateRequiredFields = (rec: DataRecord, index: number, fields: Array) => { - return fields - .map((field) => { - if (isRequiredMissing(field, rec)) { - return buildError(SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, field.name, index); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateFieldNames: ValidationFunction = ( - record: Readonly, - index: number, - fields: Array, - ) => { - const expectedFields = new Set(fields.map((field) => field.name)); - return Object.keys(record) - .map((recFieldName) => { - if (!expectedFields.has(recFieldName)) { - return buildError(SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, recFieldName, index); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateNonArrayFields: ValidationFunction = ( - record: Readonly, - index: number, - fields: Array, - ) => { - return fields - .map((field) => { - if (!field.isArray && isStringArray(record[field.name])) { - return buildError(SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, field.name, index); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateForeignKey: CrossSchemaValidationFunction = ( - schemaDef: SchemaDefinition, - schemasData: Record, - ) => { - const errors: Array = []; - const foreignKeyDefinitions = schemaDef?.restrictions?.foreignKey; - if (foreignKeyDefinitions) { - foreignKeyDefinitions.forEach((foreignKeyDefinition) => { - const localSchemaData = schemasData[schemaDef.name] || []; - const foreignSchemaData = schemasData[foreignKeyDefinition.schema] || []; - - // A foreign key can have more than one field, in which case is a composite foreign key. - const localFields = foreignKeyDefinition.mappings.map((x) => x.local); - const foreignFields = foreignKeyDefinition.mappings.map((x) => x.foreign); - - const fieldsMappings = new Map(foreignKeyDefinition.mappings.map((x) => [x.foreign, x.local])); - - // Select the keys of the datasets to compare. The keys are records to support the scenario where the fk is composite. - const localValues: [number, Record][] = selectFieldsFromDataset( - localSchemaData, - localFields, - ); - const foreignValues: [number, Record][] = selectFieldsFromDataset( - foreignSchemaData, - foreignFields, - ); - - // This artificial record in foreignValues allows null references in localValues to be valid. - const emptyRow: Record = {}; - foreignFields.forEach((field) => (emptyRow[field] = '')); - foreignValues.push([-1, emptyRow]); - - const missingForeignKeys = findMissingForeignKeys(localValues, foreignValues, fieldsMappings); - - missingForeignKeys.forEach((record) => { - const index = record[0]; - const info = { - value: record[1], - foreignSchema: foreignKeyDefinition.schema, - }; - - errors.push( - buildError(SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, localFields.join(', '), index, info), - ); - }); - }); - } - return errors; - }; - - export const getValidFields = ( - errs: ReadonlyArray, - fields: ReadonlyArray, - ) => { - return fields.filter((field) => { - return !errs.find((e) => e.fieldName == field.name); - }); - }; - - // return false if the record value is a valid type - export const isInvalidFieldType = (valueType: ValueType, value: string) => { - // optional field if the value is absent at this point - if (isAbsent(value) || isEmptyString(value)) return false; - switch (valueType) { - case ValueType.STRING: - return false; - case ValueType.INTEGER: - return isNaN(Number(value)) || !Number.isInteger(Number(value)); - case ValueType.NUMBER: - return isNaN(Number(value)); - case ValueType.BOOLEAN: - return !(value.toLowerCase() === 'true' || value.toLowerCase() === 'false'); - } - }; - - export const isRequiredMissing = (field: FieldDefinition, record: DataRecord) => { - const isRequired = field.restrictions && field.restrictions.required; - if (!isRequired) return false; - - const recordFieldValues = convertToArray(record[field.name]); - return recordFieldValues.every(isEmptyString); - }; - - const isOutOfRange = (range: RangeRestriction, value: number | undefined) => { - if (value == undefined) return false; - const invalidRange = - // less than the min if defined ? - (range.min !== undefined && value < range.min) || - (range.exclusiveMin !== undefined && value <= range.exclusiveMin) || - // bigger than max if defined ? - (range.max !== undefined && value > range.max) || - (range.exclusiveMax !== undefined && value >= range.exclusiveMax); - return invalidRange; - }; - - const isInvalidEnumValue = (codeList: CodeListRestriction, value: string | boolean | number | undefined) => { - // optional field if the value is absent at this point - if (isAbsent(value) || isEmptyString(value as string)) return false; - return !codeList.find((e) => e === value); - }; - - const isInvalidRegexValue = (regex: string, value: string) => { - // optional field if the value is absent at this point - if (isAbsent(value) || isEmptyString(value)) return false; - const regexPattern = new RegExp(regex); - return !regexPattern.test(value); - }; - - const ctx = vm.createContext(); - - const validateWithScript = ( - field: FieldDefinition, - record: TypedDataRecord, - ): { - valid: boolean; - message: string; - } => { - try { - const args = { - $row: record, - $field: record[field.name], - $name: field.name, - }; - - if (!field.restrictions || !field.restrictions.script) { - throw new Error('called validation by script without script provided'); - } - - // scripts should already be strings inside arrays, but ensure that they are to help transition between lectern versions - // checking for this can be removed in future versions of lectern (feb 2020) - const scripts = - typeof field.restrictions.script === 'string' ? [field.restrictions.script] : field.restrictions.script; - - let result: { - valid: boolean; - message: string; - } = { - valid: false, - message: '', - }; - - for (const scriptString of scripts) { - const script = getScript(scriptString); - const valFunc = script.runInContext(ctx); - if (!valFunc) throw new Error('Invalid script'); - result = valFunc(args); - /* Return the first script that's invalid. Otherwise result will be valid with message: 'ok'*/ - if (!result.valid) break; - } - - return result; - } catch (err) { - console.error( - `failed running validation script ${field.name} for record: ${JSON.stringify(record)}. Error message: ${err}`, - ); - return { - valid: false, - message: 'failed to run script validation, check script and the input', - }; - } - }; - - const getScript = (scriptString: string) => { - const script = new vm.Script(scriptString); - return script; - }; - - const buildError = ( - errorType: SchemaValidationErrorTypes, - fieldName: string, - index: number, - info: object = {}, - ): SchemaValidationError => { - const errorData = { errorType, fieldName, index, info }; - return { ...errorData, message: schemaErrorMessage(errorType, errorData) }; - }; -} -function validateRecordsSet(schemaDef: SchemaDefinition, processedRecords: TypedDataRecord[]) { - const validationErrors = validation - .runDatasetValidationPipeline(processedRecords, schemaDef, [ - validation.validateUnique, - validation.validateUniqueKey, - ]) - .filter(notEmpty); - return validationErrors; -} diff --git a/packages/client/src/validation/dataRecordValidation/fieldNamesValidation.ts b/packages/client/src/validation/dataRecordValidation/fieldNamesValidation.ts new file mode 100644 index 00000000..2df6bbbc --- /dev/null +++ b/packages/client/src/validation/dataRecordValidation/fieldNamesValidation.ts @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + BaseSchemaValidationError, + SchemaValidationErrorTypes, + UnrecognizedFieldValidationError, +} from '../types/validationErrorTypes'; +import { UnprocessedRecordValidationFunction } from '../types/validationFunctionTypes'; + +export const validateFieldNames: UnprocessedRecordValidationFunction = ( + record, + index, + fields, +): UnrecognizedFieldValidationError[] => { + const expectedFields = new Set(fields.map((field) => field.name)); + return Object.keys(record) + .filter((fieldName) => !expectedFields.has(fieldName)) + .map((fieldName) => buildUnrecognizedFieldError({ fieldName, index })); +}; + +export const buildUnrecognizedFieldError = (errorData: BaseSchemaValidationError): UnrecognizedFieldValidationError => { + const message = `${errorData.fieldName} is not an allowed field for this schema.`; + const info = {}; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, + info, + message, + }; +}; diff --git a/packages/client/src/validation/dataRecordValidation/valueTypeValidation.ts b/packages/client/src/validation/dataRecordValidation/valueTypeValidation.ts new file mode 100644 index 00000000..b42304d5 --- /dev/null +++ b/packages/client/src/validation/dataRecordValidation/valueTypeValidation.ts @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { SchemaFieldValueType } from 'dictionary'; + +import { convertToArray, isEmpty, isEmptyString, isStringArray, notEmpty } from '../../utils'; +import { + INVALID_VALUE_ERROR_MESSAGE, + SchemaValidationErrorTypes, + type BaseSchemaValidationError, + type ValueTypeValidationError, +} from '../types/validationErrorTypes'; +import type { UnprocessedRecordValidationFunction, ValidationFunction } from '../types/validationFunctionTypes'; + +/** + * Test the values provided to every field in a DataRecord to find any non-array fields that + * have array values. + * @param record Data Record with original string values provided in data file + * @param index + * @param schemaFields + * @returns + */ +export const validateNonArrayFields: UnprocessedRecordValidationFunction = ( + record, + index, + schemaFields, +): ValueTypeValidationError[] => { + return schemaFields + .map((field) => { + const value = record[field.name]; + if (!field.isArray && isStringArray(value)) { + return buildFieldValueTypeError({ fieldName: field.name, index }, { value }); + } + return undefined; + }) + .filter(notEmpty); +}; + +/** + * Test the values provided to every field in a DataRecord to find any values that cannot be + * converted to the required type defined in the field schema + * @param record Data Record with original string values provided in data file + * @param index + * @param schemaFields + * @returns + */ +export const validateValueTypes: UnprocessedRecordValidationFunction = ( + record, + index, + schemaFields, +): ValueTypeValidationError[] => { + return schemaFields + .map((field) => { + if (isEmpty(record[field.name])) { + return undefined; + } + + const recordFieldValues = convertToArray(record[field.name]); // put all values into array + const invalidValues = recordFieldValues.filter((v) => v !== undefined && isInvalidFieldType(field.valueType, v)); + const info = { value: invalidValues }; + + if (invalidValues.length !== 0) { + return buildFieldValueTypeError({ fieldName: field.name, index }, info); + } + return undefined; + }) + .filter(notEmpty); +}; + +/** + * Check a value is valid for a given schema value type. + * @param valueType + * @param value + * @returns + */ +const isInvalidFieldType = (valueType: SchemaFieldValueType, value: string) => { + // optional field if the value is absent at this point + if (isEmptyString(value)) return false; + switch (valueType) { + case SchemaFieldValueType.Values.string: + return false; + case SchemaFieldValueType.Values.integer: + return !Number.isSafeInteger(Number(value)); + case SchemaFieldValueType.Values.number: + return isNaN(Number(value)); + case SchemaFieldValueType.Values.boolean: + return !(value.toLowerCase() === 'true' || value.toLowerCase() === 'false'); + } +}; + +const buildFieldValueTypeError = ( + errorData: BaseSchemaValidationError, + info: ValueTypeValidationError['info'], +): ValueTypeValidationError => { + const message = INVALID_VALUE_ERROR_MESSAGE; + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + info, + message, + }; +}; diff --git a/packages/client/src/validation/fieldRestrictions/codeListValidation.ts b/packages/client/src/validation/fieldRestrictions/codeListValidation.ts new file mode 100644 index 00000000..50d9a56c --- /dev/null +++ b/packages/client/src/validation/fieldRestrictions/codeListValidation.ts @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { convertToArray, isAbsent, isEmptyString, notEmpty } from '../../utils'; +import { + BaseSchemaValidationError, + EnumValueValidationError, + INVALID_VALUE_ERROR_MESSAGE, + SchemaValidationErrorTypes, +} from '../types/validationErrorTypes'; +import { ValidationFunction } from '../types/validationFunctionTypes'; + +/** + * Check all values of a DataRecord pass codeList restrictions in their schema. + * @param rec + * @param index + * @param fields + * @returns + */ +export const validateCodeList: ValidationFunction = (rec, index, fields): EnumValueValidationError[] => { + return fields + .map((field) => { + if (field.restrictions && 'codeList' in field.restrictions && field.restrictions.codeList !== undefined) { + const codeList = field.restrictions.codeList; + if (!Array.isArray(codeList)) { + // codeList restriction is a string, not array. This happens when the references have not been replaced. + // We cannot proceed without the final array so we will return undefined. + return undefined; + } + + // put all values into array to standardize validation for array and non array fields + const recordFieldValues = convertToArray(rec[field.name]); + const invalidValues = recordFieldValues.filter((val) => isInvalidEnumValue(codeList, val)); + + if (invalidValues.length !== 0) { + return buildCodeListError({ fieldName: field.name, index }, { value: invalidValues }); + } + } + return undefined; + }) + .filter(notEmpty); +}; + +const buildCodeListError = ( + errorData: BaseSchemaValidationError, + info: EnumValueValidationError['info'], +): EnumValueValidationError => { + const message = INVALID_VALUE_ERROR_MESSAGE; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + info, + message, + }; +}; + +const isInvalidEnumValue = (codeList: string[] | number[], value: string | boolean | number | undefined) => { + // only validate existing values + if (isAbsent(value) || (typeof value === 'string' && isEmptyString(value))) { + return false; + } + + return !codeList.some((allowedValue) => allowedValue === value); +}; diff --git a/packages/client/src/validation/fieldRestrictions/rangeValidation.ts b/packages/client/src/validation/fieldRestrictions/rangeValidation.ts new file mode 100644 index 00000000..83d44080 --- /dev/null +++ b/packages/client/src/validation/fieldRestrictions/rangeValidation.ts @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { RestrictionRange } from 'dictionary'; +import { convertToArray, isEmpty, isNumberArray, notEmpty } from '../../utils'; +import { rangeToSymbol } from '../utils/rangeToSymbol'; +import { + BaseSchemaValidationError, + RangeValidationError, + SchemaValidationErrorTypes, +} from '../types/validationErrorTypes'; +import { ValidationFunction } from '../types/validationFunctionTypes'; + +/** + * Check all values of a DataRecord pass range restrictions in their schema. + * @param record + * @param index + * @param schemaFields + * @returns + */ +export const validateRange: ValidationFunction = (record, index, schemaFields): RangeValidationError[] => { + return schemaFields + .map((field) => { + const recordFieldValues = convertToArray(record[field.name]); + if (!isNumberArray(recordFieldValues)) { + return undefined; + } + + const range = field.restrictions && 'range' in field.restrictions ? field.restrictions.range : undefined; + if (isEmpty(range)) { + return undefined; + } + + const invalidValues = recordFieldValues.filter((value) => isOutOfRange(range, value)); + if (invalidValues.length !== 0) { + const info = { value: invalidValues, ...range }; + return buildRangeError({ fieldName: field.name, index }, info); + } + return undefined; + }) + .filter(notEmpty); +}; + +const buildRangeError = ( + errorData: BaseSchemaValidationError, + info: RangeValidationError['info'], +): RangeValidationError => { + const message = `Value is out of permissible range, it must be ${rangeToSymbol(info)}.`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + info, + message, + }; +}; + +const isOutOfRange = (range: RestrictionRange, value: number | undefined) => { + if (value === undefined) return false; + const invalidRange = + // less than the min if defined ? + (range.min !== undefined && value < range.min) || + (range.exclusiveMin !== undefined && value <= range.exclusiveMin) || + // bigger than max if defined ? + (range.max !== undefined && value > range.max) || + (range.exclusiveMax !== undefined && value >= range.exclusiveMax); + return invalidRange; +}; diff --git a/packages/client/src/validation/fieldRestrictions/regexValidation.ts b/packages/client/src/validation/fieldRestrictions/regexValidation.ts new file mode 100644 index 00000000..a501bdd8 --- /dev/null +++ b/packages/client/src/validation/fieldRestrictions/regexValidation.ts @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +import { SchemaFieldValueType } from 'dictionary'; +import { convertToArray, isEmpty, isEmptyString, isStringArray, notEmpty } from '../../utils'; +import { + BaseSchemaValidationError, + RegexValidationError, + SchemaValidationErrorTypes, +} from '../types/validationErrorTypes'; +import { ValidationFunction } from '../types/validationFunctionTypes'; + +/** + * Check all values of a DataRecord pass regex restrictions in their schema. + * @param record + * @param index + * @param fields + * @returns + */ +export const validateRegex: ValidationFunction = (record, index, fields): RegexValidationError[] => { + return fields + .map((field) => { + if ( + field.valueType === SchemaFieldValueType.Values.string && + field.restrictions && + !isEmpty(field.restrictions.regex) + ) { + const regex = field.restrictions.regex; + const recordFieldValues = convertToArray(record[field.name]); + if (!isStringArray(recordFieldValues)) { + // This field value should be string or string array, we will skip validation if the type is wrong. + return undefined; + } + + const invalidValues = recordFieldValues.filter((v) => isInvalidRegexValue(regex, v)); + if (invalidValues.length !== 0) { + const examples = typeof field.meta?.examples === 'string' ? field.meta.examples : undefined; + + return buildRegexError({ fieldName: field.name, index }, { value: invalidValues, regex, examples }); + } + } + + // Field does not have regex validation + return undefined; + }) + .filter(notEmpty); +}; + +const isInvalidRegexValue = (regex: string, value: string) => { + // optional field if the value is absent at this point + if (isEmptyString(value)) return false; + const regexPattern = new RegExp(regex); + return !regexPattern.test(value); +}; + +const buildRegexError = ( + errorData: BaseSchemaValidationError, + info: RegexValidationError['info'], +): RegexValidationError => { + const examplesMessage = info.examples ? ` Examples: ${info.examples}` : ''; + const message = `The value is not a permissible for this field, it must meet the regular expression: "${info.regex}".${examplesMessage}`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + info, + message, + }; +}; diff --git a/packages/client/src/validation/fieldRestrictions/requiredValidation.ts b/packages/client/src/validation/fieldRestrictions/requiredValidation.ts new file mode 100644 index 00000000..82bbc488 --- /dev/null +++ b/packages/client/src/validation/fieldRestrictions/requiredValidation.ts @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { SchemaField } from 'dictionary'; +import { convertToArray, isEmpty, notEmpty } from '../../utils'; +import { + BaseSchemaValidationError, + MissingRequiredFieldValidationError, + SchemaValidationErrorTypes, +} from '../types/validationErrorTypes'; +import { ValidationFunction } from '../types/validationFunctionTypes'; +import { DataRecord } from '../../types/dataRecords'; + +/** + * Check all values of a DataRecord pass required restrictions in their schema. + * @param record + * @param index + * @param fields + * @returns + */ +export const validateRequiredFields: ValidationFunction = ( + record, + index, + fields, +): MissingRequiredFieldValidationError[] => { + return fields + .map((field) => { + if (isRequiredMissing(field, record)) { + return buildRequiredError({ fieldName: field.name, index }, {}); + } + return undefined; + }) + .filter(notEmpty); +}; + +const buildRequiredError = ( + errorData: BaseSchemaValidationError, + info: MissingRequiredFieldValidationError['info'], +): MissingRequiredFieldValidationError => { + const message = `${errorData.fieldName} is a required field.`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + info, + message, + }; +}; + +const isRequiredMissing = (field: SchemaField, record: DataRecord) => { + const isRequired = field.restrictions && field.restrictions.required; + if (!isRequired) return false; + + const recordFieldValues = convertToArray(record[field.name]); + return recordFieldValues.every(isEmpty); +}; diff --git a/packages/client/src/validation/fieldRestrictions/scriptValidation.ts b/packages/client/src/validation/fieldRestrictions/scriptValidation.ts new file mode 100644 index 00000000..5f6ba0c0 --- /dev/null +++ b/packages/client/src/validation/fieldRestrictions/scriptValidation.ts @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { SchemaField } from 'dictionary'; +import vm from 'vm'; +import { DataRecord } from '../../types/dataRecords'; +import { notEmpty } from '../../utils'; +import { + BaseSchemaValidationError, + SchemaValidationErrorTypes, + ScriptValidationError, +} from '../types/validationErrorTypes'; +import { ValidationFunction } from '../types/validationFunctionTypes'; + +const ctx = vm.createContext(); + +/** + * Check all values of a DataRecord pass all script restrictions in their schema. + * This will run all script restrictions from the provided inside a Node VM context. + * + * Running code in teh VM context will protect the global Node data context from interactions + * with the schema script, either being read or written. + * + * @param record + * @param index + * @param fields + * @returns + */ +export const validateScript: ValidationFunction = (record, index, fields) => { + return fields + .map((field) => { + if (field.restrictions && field.restrictions.script) { + const scriptResult = validateWithScript(field, record); + if (!scriptResult.valid) { + return buildScriptError( + { fieldName: field.name, index }, + { + message: scriptResult.message, + value: record[field.name], + }, + ); + } + } + return undefined; + }) + .filter(notEmpty); +}; + +const buildScriptError = ( + errorData: BaseSchemaValidationError, + info: ScriptValidationError['info'], +): ScriptValidationError => { + const message = info.message || `${errorData.fieldName} was invalid based on a script restriction.`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + info, + message, + }; +}; + +const getScript = (scriptString: string) => { + const script = new vm.Script(scriptString); + return script; +}; + +const validateWithScript = ( + field: SchemaField, + record: DataRecord, +): { + valid: boolean; + message: string; +} => { + try { + const args = { + $row: record, + $field: record[field.name], + $name: field.name, + }; + + if (!field.restrictions || !field.restrictions.script) { + throw new Error('called validation by script without script provided'); + } + + // scripts should already be strings inside arrays, but ensure that they are to help transition between lectern versions + // checking for this can be removed in future versions of lectern (feb 2020) + const scripts = + typeof field.restrictions.script === 'string' ? [field.restrictions.script] : field.restrictions.script; + + let result: { + valid: boolean; + message: string; + } = { + valid: false, + message: '', + }; + + for (const scriptString of scripts) { + const script = getScript(scriptString); + const valFunc = script.runInContext(ctx); + if (!valFunc) throw new Error('Invalid script'); + result = valFunc(args); + /* Return the first script that's invalid. Otherwise result will be valid with message: 'ok'*/ + if (!result.valid) break; + } + + return result; + } catch (err) { + console.error( + `failed running validation script ${field.name} for record: ${JSON.stringify(record)}. Error message: ${err}`, + ); + return { + valid: false, + message: 'failed to run script validation, check script and the input', + }; + } +}; diff --git a/packages/client/src/validation/index.ts b/packages/client/src/validation/index.ts new file mode 100644 index 00000000..28f58202 --- /dev/null +++ b/packages/client/src/validation/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './dataRecordValidation/fieldNamesValidation'; +export * from './dataRecordValidation/valueTypeValidation'; +export * from './fieldRestrictions/codeListValidation'; +export * from './fieldRestrictions/rangeValidation'; +export * from './fieldRestrictions/regexValidation'; +export * from './fieldRestrictions/requiredValidation'; +export * from './fieldRestrictions/scriptValidation'; +export * from './schemaRestrictions/foreignKeysValidation'; +export * from './schemaRestrictions/uniqueKeyValidation'; +export * from './schemaRestrictions/uniqueValidation'; +export * from './types'; +export * from './validationPipelines'; diff --git a/packages/client/src/validation/schemaRestrictions/foreignKeysValidation.ts b/packages/client/src/validation/schemaRestrictions/foreignKeysValidation.ts new file mode 100644 index 00000000..9687e99c --- /dev/null +++ b/packages/client/src/validation/schemaRestrictions/foreignKeysValidation.ts @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { differenceWith, isEqual } from 'lodash'; +import { DataRecord } from '../../types/dataRecords'; +import { ForeignKeyValidationError, SchemaValidationErrorTypes } from '../types/validationErrorTypes'; +import { CrossSchemaValidationFunction } from '../types/validationFunctionTypes'; +import { selectFieldsFromDataset } from '../utils/datasetUtils'; + +/** + * Validate all foreign key restrictions in a Schema + * @param schema + * @param data + * @returns + */ +export const validateForeignKeys: CrossSchemaValidationFunction = (schema, data): ForeignKeyValidationError[] => { + const errors: Array = []; + const foreignKeyDefinitions = schema?.restrictions?.foreignKey; + if (foreignKeyDefinitions) { + foreignKeyDefinitions.forEach((foreignKeyDefinition) => { + const localSchemaData = data[schema.name] || []; + const foreignSchemaData = data[foreignKeyDefinition.schema] || []; + + // A foreign key can have more than one field, in which case is a composite foreign key. + const localFields = foreignKeyDefinition.mappings.map((x) => x.local); + const foreignFields = foreignKeyDefinition.mappings.map((x) => x.foreign); + + const fieldsMappings = new Map(foreignKeyDefinition.mappings.map((x) => [x.foreign, x.local])); + + // Select the keys of the datasets to compare. The keys are records to support the scenario where the fk is composite. + const localValues: [number, DataRecord][] = selectFieldsFromDataset(localSchemaData, localFields); + const foreignValues: [number, DataRecord][] = selectFieldsFromDataset(foreignSchemaData, foreignFields); + + // This artificial record in foreignValues allows null references in localValues to be valid. + const emptyRow: Record = {}; + foreignFields.forEach((field) => (emptyRow[field] = '')); + foreignValues.push([-1, emptyRow]); + + const missingForeignKeys = findMissingForeignKeys(localValues, foreignValues, fieldsMappings); + + missingForeignKeys.forEach((record) => { + const index = record[0]; + const info: ForeignKeyValidationError['info'] = { + value: record[1], + foreignSchema: foreignKeyDefinition.schema, + }; + const errorFieldName = localFields.join(', '); + errors.push({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + fieldName: errorFieldName, + index, + info, + message: getForeignKeyErrorMessage({ + fieldName: errorFieldName, + foreignSchema: foreignKeyDefinition.schema, + value: record[1], + }), + }); + }); + }); + } + return errors; +}; + +function getForeignKeyErrorMessage(errorData: { value: DataRecord; foreignSchema: string; fieldName: string }) { + const valueEntries = Object.entries(errorData.value); + const formattedKeyValues: string[] = valueEntries.map(([key, value]) => { + if (Array.isArray(value)) { + return `${key}: [${value.join(', ')}]`; + } else { + return `${key}: ${value}`; + } + }); + const valuesAsString = formattedKeyValues.join(', '); + const detail = `Key ${valuesAsString} is not present in schema ${errorData.foreignSchema}`; + const msg = `Record violates foreign key restriction defined for field(s) ${errorData.fieldName}. ${detail}.`; + return msg; +} + +/** + * Find missing foreign keys by calculating the difference between 2 dataset keys (similar to a set difference). + * Returns rows in `dataKeysA` which are not present in `dataKeysB`. + * @param datasetKeysA Keys of the dataset A. The returned value of this function is a subset of this array. + * @param datasetKeysB Keys of the dataset B. Elements to be substracted from `datasetKeysA`. + * @param fieldsMapping Mapping of the field names so the keys can be compared correctly. + */ +const findMissingForeignKeys = ( + datasetKeysA: [number, DataRecord][], + datasetKeysB: [number, DataRecord][], + fieldsMapping: Map, +): [number, DataRecord][] => { + const diff = differenceWith(datasetKeysA, datasetKeysB, (a, b) => + isEqual(a[1], renameProperties(b[1], fieldsMapping)), + ); + return diff; +}; + +/** + * Renames properties in a record using a mapping between current and new names. + * @param record The record whose properties should be renamed. + * @param fieldsMapping A mapping of current property names to new property names. + * @returns A new record with the properties' names changed according to the mapping. + */ +const renameProperties = (record: DataRecord, fieldsMapping: Map): DataRecord => { + const renamed: DataRecord = {}; + Object.entries(record).forEach(([propertyName, propertyValue]) => { + const newName = fieldsMapping.get(propertyName) ?? propertyName; + renamed[newName] = propertyValue; + }); + return renamed; +}; diff --git a/packages/client/src/validation/schemaRestrictions/uniqueKeyValidation.ts b/packages/client/src/validation/schemaRestrictions/uniqueKeyValidation.ts new file mode 100644 index 00000000..e8414252 --- /dev/null +++ b/packages/client/src/validation/schemaRestrictions/uniqueKeyValidation.ts @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + BaseSchemaValidationError, + SchemaValidationErrorTypes, + UniqueKeyValidationError, +} from '../types/validationErrorTypes'; +import { DatasetValidationFunction } from '../types/validationFunctionTypes'; +import { findDuplicateKeys, selectFieldsFromDataset } from '../utils/datasetUtils'; + +export const validateUniqueKey: DatasetValidationFunction = (dataset, schema): UniqueKeyValidationError[] => { + const errors: Array = []; + const uniqueKeyRestriction = schema?.restrictions?.uniqueKey; + if (uniqueKeyRestriction) { + const uniqueKeyFields: string[] = uniqueKeyRestriction; + const keysToValidate = selectFieldsFromDataset(dataset, uniqueKeyFields); + const duplicateKeys = findDuplicateKeys(keysToValidate); + + duplicateKeys.forEach(([index, record]) => { + const info = { uniqueKeyFields: uniqueKeyFields, value: record }; + errors.push(buildUniqueKeyError({ fieldName: uniqueKeyFields.join(', '), index }, info)); + }); + } + return errors; +}; + +const buildUniqueKeyError = ( + errorData: BaseSchemaValidationError, + info: UniqueKeyValidationError['info'], +): UniqueKeyValidationError => { + const uniqueKeyFields = info.uniqueKeyFields; + const record = info.value; + const formattedKeyValues = uniqueKeyFields.map((fieldName) => { + if (fieldName in record) { + const value = record[fieldName]; + if (Array.isArray(value)) { + return `${fieldName}: [${value.join(', ')}]`; + } else { + return `${fieldName}: ${value === '' ? 'null' : value}`; + } + } + }); + const valuesAsString = formattedKeyValues.join(', '); + const message = `UniqueKey field values "${valuesAsString}" must be unique.`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + info, + message, + }; +}; diff --git a/packages/client/src/validation/schemaRestrictions/uniqueValidation.ts b/packages/client/src/validation/schemaRestrictions/uniqueValidation.ts new file mode 100644 index 00000000..5443d663 --- /dev/null +++ b/packages/client/src/validation/schemaRestrictions/uniqueValidation.ts @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + BaseSchemaValidationError, + SchemaValidationErrorTypes, + UniqueValidationError, +} from '../types/validationErrorTypes'; +import { DatasetValidationFunction } from '../types/validationFunctionTypes'; +import { findDuplicateKeys, selectFieldsFromDataset } from '../utils/datasetUtils'; + +/** + * Validate all unique field restrictions in a schema. This will find all records that have duplicate + * values for fields that are restricted to being unique. + * @param data + * @param schema + * @returns + */ +export const validateUnique: DatasetValidationFunction = (data, schema): UniqueValidationError[] => { + const errors: Array = []; + schema.fields.forEach((field) => { + const unique = field.restrictions?.unique || undefined; + if (!unique) return undefined; + const keysToValidate = selectFieldsFromDataset(data, [field.name]); + const duplicateKeys = findDuplicateKeys(keysToValidate); + + duplicateKeys.forEach(([index, record]) => { + const info = { value: record[field.name] }; + errors.push(buildUniqueError({ fieldName: field.name, index }, info)); + }); + }); + return errors; +}; + +const buildUniqueError = ( + errorData: BaseSchemaValidationError, + info: UniqueValidationError['info'], +): UniqueValidationError => { + const message = `Values for column "${errorData.fieldName}" must be unique.`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + info, + message, + }; +}; diff --git a/packages/client/src/validation/types/index.ts b/packages/client/src/validation/types/index.ts new file mode 100644 index 00000000..7b536617 --- /dev/null +++ b/packages/client/src/validation/types/index.ts @@ -0,0 +1,2 @@ +export * from './validationErrorTypes'; +export * from './validationFunctionTypes'; diff --git a/packages/client/src/validation/types/validationErrorTypes.ts b/packages/client/src/validation/types/validationErrorTypes.ts new file mode 100644 index 00000000..33abed1e --- /dev/null +++ b/packages/client/src/validation/types/validationErrorTypes.ts @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { Values, Singular } from 'common'; +import type { DataRecord, DataRecordValue } from '../../types/dataRecords'; +import type { RestrictionRange } from 'dictionary'; + +/** + * Represents the common structure of a validation error without the custom content provided by specific error types. + * The `message` property is not included here as this allows the rest of the error content to be passed to a message building function + * that will produce the final typed SchemaValidationError + */ +export type BaseSchemaValidationError = { + index: number; + fieldName: string; +}; + +type GenericSchemaValidationError< + ErrorType extends SchemaValidationErrorType, + Info extends object, +> = BaseSchemaValidationError & { + errorType: ErrorType; + info: Info; + message: string; +}; + +// Common string for invalid value errors +export const INVALID_VALUE_ERROR_MESSAGE = 'The value is not permissible for this field.'; + +export const SchemaValidationErrorTypes = { + INVALID_FIELD_VALUE_TYPE: 'INVALID_FIELD_VALUE_TYPE', + INVALID_BY_REGEX: 'INVALID_BY_REGEX', + INVALID_BY_RANGE: 'INVALID_BY_RANGE', + INVALID_BY_SCRIPT: 'INVALID_BY_SCRIPT', + INVALID_ENUM_VALUE: 'INVALID_ENUM_VALUE', + INVALID_BY_UNIQUE: 'INVALID_BY_UNIQUE', + INVALID_BY_FOREIGN_KEY: 'INVALID_BY_FOREIGN_KEY', + INVALID_BY_UNIQUE_KEY: 'INVALID_BY_UNIQUE_KEY', + MISSING_REQUIRED_FIELD: 'MISSING_REQUIRED_FIELD', + UNRECOGNIZED_FIELD: 'UNRECOGNIZED_FIELD', +} as const; +export type SchemaValidationErrorType = Values; + +export type EnumValueValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + { value: DataRecordValue[] } +>; +export type ForeignKeyValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + { value: DataRecord; foreignSchema: string } +>; +export type MissingRequiredFieldValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + {} +>; +export type RangeValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_RANGE, + { value: number[] } & RestrictionRange +>; +export type RegexValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_REGEX, + { value: string[]; regex: string; examples?: string } +>; +export type ScriptValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + { message: string; value: DataRecordValue } +>; +export type UniqueValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + { value: DataRecordValue } +>; +export type UniqueKeyValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + { uniqueKeyFields: string[]; value: DataRecord } +>; +export type UnrecognizedFieldValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, + {} +>; +export type ValueTypeValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + { value: Singular[] } +>; +export type SchemaValidationError = + | EnumValueValidationError + | ValueTypeValidationError + | ForeignKeyValidationError + | MissingRequiredFieldValidationError + | RangeValidationError + | RegexValidationError + | ScriptValidationError + | UniqueValidationError + | UniqueKeyValidationError + | UnrecognizedFieldValidationError; diff --git a/packages/client/src/validation/types/validationFunctionTypes.ts b/packages/client/src/validation/types/validationFunctionTypes.ts new file mode 100644 index 00000000..2709c9cd --- /dev/null +++ b/packages/client/src/validation/types/validationFunctionTypes.ts @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { Schema, SchemaField } from 'dictionary'; +import { DataRecord, UnprocessedDataRecord } from '../../types/dataRecords'; +import { SchemaValidationError } from './validationErrorTypes'; + +// these validation functions run AFTER the record has been converted to the correct types from raw strings +export type UnprocessedRecordValidationFunction = ( + record: UnprocessedDataRecord, + index: number, + schemaFields: Schema['fields'], +) => Array; + +// these validation functions run BEFORE the record has been converted to the correct types from raw strings +export type ValidationFunction = ( + record: DataRecord, + index: number, + schemaFields: Schema['fields'], +) => Array; + +// these validation functions run AFTER the records has been converted to the correct types from raw strings, and apply to a dataset instead of +// individual records +export type DatasetValidationFunction = (data: Array, schema: Schema) => Array; + +export type CrossSchemaValidationFunction = ( + schema: Schema, + data: Record, +) => Array; diff --git a/packages/client/src/records-operations.ts b/packages/client/src/validation/utils/datasetUtils.ts similarity index 53% rename from packages/client/src/records-operations.ts rename to packages/client/src/validation/utils/datasetUtils.ts index 7b964f3f..e07f4c36 100644 --- a/packages/client/src/records-operations.ts +++ b/packages/client/src/validation/utils/datasetUtils.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved * * This program and the accompanying materials are made available under the terms of * the GNU Affero General Public License v3.0. You should have received a copy of the @@ -17,25 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { differenceWith, isEqual } from 'lodash'; - -/** - * Renames properties in a record using a mapping between current and new names. - * @param record The record whose properties should be renamed. - * @param fieldsMapping A mapping of current property names to new property names. - * @returns A new record with the properties' names changed according to the mapping. - */ -const renameProperties = ( - record: Record, - fieldsMapping: Map, -): Record => { - const renamed: Record = {}; - Object.entries(record).forEach(([propertyName, propertyValue]) => { - const newName = fieldsMapping.get(propertyName) ?? propertyName; - renamed[newName] = propertyValue; - }); - return renamed; -}; +import { DataRecord } from '../../types/dataRecords'; /** * Returns a string representation of a record. The record is sorted by its properties so @@ -44,43 +26,23 @@ const renameProperties = ( * @param record Record to be processed. * @returns String representation of the record sorted by its properties. */ -const getSortedRecordKey = (record: Record): string => { +const getSortedRecordKey = (record: DataRecord): string => { const sortedKeys = Object.keys(record).sort(); - const sortedRecord: Record = {}; + const sortedRecord: DataRecord = {}; for (const key of sortedKeys) { sortedRecord[key] = record[key]; } return JSON.stringify(sortedRecord); }; -/** - * Find missing foreign keys by calculating the difference between 2 dataset keys (similar to a set difference). - * Returns rows in `dataKeysA` which are not present in `dataKeysB`. - * @param datasetKeysA Keys of the dataset A. The returned value of this function is a subset of this array. - * @param datasetKeysB Keys of the dataset B. Elements to be substracted from `datasetKeysA`. - * @param fieldsMapping Mapping of the field names so the keys can be compared correctly. - */ -export const findMissingForeignKeys = ( - datasetKeysA: [number, Record][], - datasetKeysB: [number, Record][], - fieldsMapping: Map, -): [number, Record][] => { - const diff = differenceWith(datasetKeysA, datasetKeysB, (a, b) => - isEqual(a[1], renameProperties(b[1], fieldsMapping)), - ); - return diff; -}; - /** * Find duplicate keys in a dataset. * @param datasetKeys Array with the keys to evaluate. * @returns An Array with all the values that appear more than once in the dataset. */ -export const findDuplicateKeys = ( - datasetKeys: [number, Record][], -): [number, Record][] => { - const duplicateKeys: [number, Record][] = []; - const recordKeysMap: Map<[number, Record], string> = new Map(); +export const findDuplicateKeys = (datasetKeys: [number, DataRecord][]): [number, DataRecord][] => { + const duplicateKeys: [number, DataRecord][] = []; + const recordKeysMap: Map<[number, DataRecord], string> = new Map(); const keyCount: Map = new Map(); // Calculate a key per record, which is a string representation that allows to compare records even if their properties @@ -101,3 +63,19 @@ export const findDuplicateKeys = ( }); return duplicateKeys; }; + +/** + * A "select" function that retrieves specific fields from the dataset as a record, as well as the numeric position of each row in the dataset. + * @param dataset Dataset to select fields from. + * @param fields Array with names of the fields to select. + * @returns An array of tuples tuple where the first element is the index of the row in the dataset, and the second value is the record with the + * selected values. + */ +export const selectFieldsFromDataset = (dataset: DataRecord[], fields: string[]): [number, DataRecord][] => + dataset.map((row, index) => { + const filteredRecord = fields.reduce((acc, field) => { + acc[field] = row[field] || ''; + return acc; + }, {}); + return [index, filteredRecord]; + }); diff --git a/packages/client/src/validation/utils/fieldTypeUtils.ts b/packages/client/src/validation/utils/fieldTypeUtils.ts new file mode 100644 index 00000000..405d3d84 --- /dev/null +++ b/packages/client/src/validation/utils/fieldTypeUtils.ts @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { SchemaFieldValueType } from 'dictionary'; + +import { isEmptyString } from '../../utils'; + +/** + * Check a value is valid for a given schema value type. + * @param valueType + * @param value + * @returns + */ +export const isInvalidFieldType = (valueType: SchemaFieldValueType, value: string) => { + // optional field if the value is absent at this point + if (isEmptyString(value)) return false; + switch (valueType) { + case SchemaFieldValueType.Values.string: + return false; + case SchemaFieldValueType.Values.integer: + return !Number.isSafeInteger(Number(value)); + case SchemaFieldValueType.Values.number: + return isNaN(Number(value)); + case SchemaFieldValueType.Values.boolean: + return !(value.toLowerCase() === 'true' || value.toLowerCase() === 'false'); + } +}; diff --git a/packages/client/src/validation/utils/rangeToSymbol.ts b/packages/client/src/validation/utils/rangeToSymbol.ts new file mode 100644 index 00000000..1616ccd1 --- /dev/null +++ b/packages/client/src/validation/utils/rangeToSymbol.ts @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { RestrictionRange } from 'dictionary'; + +export const rangeToSymbol = (range: RestrictionRange): string => { + let minString = ''; + let maxString = ''; + + const hasBothRange = + (range.min !== undefined || range.exclusiveMin !== undefined) && + (range.max !== undefined || range.exclusiveMax !== undefined); + + if (range.min !== undefined) { + minString = `>= ${range.min}`; + } + + if (range.exclusiveMin !== undefined) { + minString = `> ${range.exclusiveMin}`; + } + + if (range.max !== undefined) { + maxString = `<= ${range.max}`; + } + + if (range.exclusiveMax !== undefined) { + maxString = `< ${range.exclusiveMax}`; + } + + return hasBothRange ? `${minString} and ${maxString}` : `${minString}${maxString}`; +}; diff --git a/packages/client/src/validation/validationPipelines.ts b/packages/client/src/validation/validationPipelines.ts new file mode 100644 index 00000000..f11a5fc3 --- /dev/null +++ b/packages/client/src/validation/validationPipelines.ts @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { Schema, SchemaField } from 'dictionary'; + +import { DataRecord, UnprocessedDataRecord } from '../types/dataRecords'; +import { SchemaValidationError } from './types/validationErrorTypes'; +import { + CrossSchemaValidationFunction, + DatasetValidationFunction, + UnprocessedRecordValidationFunction, + ValidationFunction, +} from './types/validationFunctionTypes'; + +export const runUnprocessedRecordValidationPipeline = ( + record: UnprocessedDataRecord, + index: number, + fields: ReadonlyArray, + validationFunctions: Array, +) => { + let result: Array = []; + for (const validationFunction of validationFunctions) { + result = result.concat(validationFunction(record, index, getValidFields(result, fields))); + } + return result; +}; + +export const runRecordValidationPipeline = ( + record: DataRecord, + index: number, + fields: ReadonlyArray, + validationFunctions: Array, +) => { + let result: Array = []; + for (const validationFunction of validationFunctions) { + result = result.concat(validationFunction(record, index, getValidFields(result, fields))); + } + return result; +}; + +export const runDatasetValidationPipeline = ( + data: DataRecord[], + schema: Schema, + validationFunctions: Array, +) => validationFunctions.flatMap((validationFunction) => validationFunction(data, schema)); + +export const runCrossSchemaValidationPipeline = ( + schema: Schema, + data: Record, + validationFunctions: Array, +) => { + let result: Array = []; + for (const validationFunction of validationFunctions) { + result = result.concat(validationFunction(schema, data)); + } + return result; +}; + +const getValidFields = (errs: ReadonlyArray, fields: ReadonlyArray) => { + return fields.filter((field) => { + return !errs.find((e) => e.fieldName === field.name); + }); +}; From 02ec63ad9879e6916fd6f187f5ff649d7b3e6824 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:40:22 -0400 Subject: [PATCH 11/17] Move rest client code and update to Lectern shared types --- .../{schema-rest-client.ts => rest/index.ts} | 65 ++++++++----------- 1 file changed, 27 insertions(+), 38 deletions(-) rename packages/client/src/{schema-rest-client.ts => rest/index.ts} (72%) diff --git a/packages/client/src/schema-rest-client.ts b/packages/client/src/rest/index.ts similarity index 72% rename from packages/client/src/schema-rest-client.ts rename to packages/client/src/rest/index.ts index a9a944c6..1d390733 100644 --- a/packages/client/src/schema-rest-client.ts +++ b/packages/client/src/rest/index.ts @@ -17,25 +17,20 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { loggerFor } from './logger'; +import { unknownToString } from 'common'; +import { Dictionary, DictionaryDiff, DictionaryDiffArray, FieldDiff } from 'dictionary'; import fetch from 'node-fetch'; -import { SchemasDictionary, SchemasDictionaryDiffs, FieldChanges, FieldDiff } from './schema-entities'; import promiseTools from 'promise-tools'; -import { unknownToString } from 'common'; +import { loggerFor } from '../logger'; const L = loggerFor(__filename); export interface SchemaServiceRestClient { - fetchSchema(schemaSvcUrl: string, name: string, version: string): Promise; - fetchDiff( - schemaSvcUrl: string, - name: string, - fromVersion: string, - toVersion: string, - ): Promise; + fetchSchema(schemaSvcUrl: string, name: string, version: string): Promise; + fetchDiff(schemaSvcUrl: string, name: string, fromVersion: string, toVersion: string): Promise; } export const restClient: SchemaServiceRestClient = { - fetchSchema: async (schemaSvcUrl: string, name: string, version: string): Promise => { + fetchSchema: async (schemaSvcUrl: string, name: string, version: string): Promise => { // for testing where we need to work against stub schema if (schemaSvcUrl.startsWith('file://')) { return await loadSchemaFromFile(version, schemaSvcUrl, name); @@ -49,7 +44,7 @@ export const restClient: SchemaServiceRestClient = { L.debug(`in fetch live schema ${version}`); const schemaDictionary = await doRequest(url); // todo validate response and map it to a schema - return schemaDictionary[0] as SchemasDictionary; + return schemaDictionary[0] as Dictionary; } catch (error: unknown) { L.error(`failed to fetch schema at url: ${url} - ${unknownToString(error)}`); throw error; @@ -60,25 +55,19 @@ export const restClient: SchemaServiceRestClient = { name: string, fromVersion: string, toVersion: string, - ): Promise => { - // for testing where we need to work against stub schema - let diffResponse: any; - if (schemaSvcBaseUrl.startsWith('file://')) { - diffResponse = await loadDiffFromFile(schemaSvcBaseUrl, name, fromVersion, toVersion); - } else { - const url = `${schemaSvcBaseUrl}/diff?name=${name}&left=${fromVersion}&right=${toVersion}`; - diffResponse = (await doRequest(url)) as any[]; - } - const result: SchemasDictionaryDiffs = {}; - for (const entry of diffResponse) { - const fieldName = entry[0] as string; + ): Promise => { + // TODO: Error handling (return result?) + const url = `${schemaSvcBaseUrl}/diff?name=${name}&left=${fromVersion}&right=${toVersion}`; + const diffResponse = await doRequest(url); + + const diffArray = DictionaryDiffArray.parse(diffResponse); + + const result: DictionaryDiff = new Map(); + for (const entry of diffArray) { + const fieldName = entry[0]; if (entry[1]) { - const fieldDiff: FieldDiff = { - before: entry[1].left, - after: entry[1].right, - diff: entry[1].diff, - }; - result[fieldName] = fieldDiff; + const fieldDiff: FieldDiff = entry[1]; + result.set(fieldName, fieldDiff); } } return result; @@ -96,26 +85,26 @@ const doRequest = async (url: string) => { return await response.json(); } catch (error: unknown) { L.error(`failed to fetch schema at url: ${url} - ${unknownToString(error)}`); - throw response.status == 404 ? new Error('Not Found') : new Error('Request Failed'); + throw response.status === 404 ? new Error('Not Found') : new Error('Request Failed'); } }; async function loadSchemaFromFile(version: string, schemaSvcUrl: string, name: string) { L.debug(`in fetch stub schema ${version}`); - const result = delay(1000); + const result = delay(1000); const dictionary = await result(() => { - const dictionaries: SchemasDictionary[] = require(schemaSvcUrl.substring(7, schemaSvcUrl.length)) - .dictionaries as SchemasDictionary[]; + const dictionaries: Dictionary[] = require(schemaSvcUrl.substring(7, schemaSvcUrl.length)) + .dictionaries as Dictionary[]; if (!dictionaries) { throw new Error('your mock json is not structured correctly, see sampleFiles/sample-schema.json'); } - const dic = dictionaries.find((d: any) => d.version == version && d.name == name); + const dic = dictionaries.find((d: any) => d.version === version && d.name === name); if (!dic) { return undefined; } return dic; }); - if (dictionary == undefined) { + if (dictionary === undefined) { throw new Error("couldn't load stub dictionary with the criteria specified"); } L.debug(`schema found ${dictionary.version}`); @@ -132,14 +121,14 @@ async function loadDiffFromFile(schemaSvcBaseUrl: string, name: string, fromVers } const diff = diffResponse.find( - (d: any) => d.fromVersion == fromVersion && d.toVersion == toVersion && d.name == name, + (d) => d.fromVersion === fromVersion && d.toVersion === toVersion && d.name === name, ); if (!diff) { return undefined; } return diff; }); - if (diff == undefined) { + if (diff === undefined) { throw new Error("couldn't load stub diff with the criteria specified, check your stub file"); } return diff.data; From 076ba653cb84461cfdb557944a6b4fb350adf5f1 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:43:13 -0400 Subject: [PATCH 12/17] Add Lectern dictionary library, remove worker-threads --- packages/client/package.json | 2 +- pnpm-lock.yaml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/client/package.json b/packages/client/package.json index dbdbef76..f1306e64 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -42,9 +42,9 @@ "cd": "^0.3.3", "common": "workspace:^", "deep-freeze": "^0.0.1", + "dictionary": "workspace:^", "lodash": "^4.17.21", "node-fetch": "^2.6.1", - "node-worker-threads-pool": "^1.4.3", "promise-tools": "^2.1.0", "winston": "^3.3.3" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 48ca3071..b450b8ff 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3384,10 +3384,6 @@ packages: - supports-color dev: false - /node-worker-threads-pool@1.5.1: - resolution: {integrity: sha512-7TXAhpMm+jO4MfESxYLtMGSnJWv+itdNHMdaFmeZuPXxwFGU90mtEB42BciUULXOUAxYBfXILAuvrSG3rQZ7mw==} - dev: false - /nodemon@2.0.22: resolution: {integrity: sha512-B8YqaKMmyuCO7BowF1Z1/mkPqLk6cs/l63Ojtd6otKjMx47Dq1utxfRxcavH1I7VSaL8n5BUaoutadnsX3AAVQ==} engines: {node: '>=8.10.0'} From 4eca6988d62ecae4521fb64745b65649cc1eb33f Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:44:46 -0400 Subject: [PATCH 13/17] Update client entry file for new code directories --- packages/client/src/index.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index e7cf786c..4b8bd69f 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -17,10 +17,9 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import * as entities from './schema-entities'; -import * as analyzer from './change-analyzer'; -import * as functions from './schema-functions'; -import * as parallel from './parallel'; +export * as DictionaryTypes from 'dictionary'; +export * as analyzer from './changeAnalysis'; +export * as functions from './processing'; +export { restClient } from './rest'; -import { restClient } from './schema-rest-client'; -export { entities, analyzer, functions, parallel, restClient }; +export type { DataRecord, DataRecordValue, UnprocessedDataRecord } from './types'; From b656e5364e51c999e8085f7df8f81fe703671265 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 10 Jun 2024 21:45:27 -0400 Subject: [PATCH 14/17] Organize tests and fixtures with clean names Also updated messages in tests that have changed during code cleanup --- ...nalyzer.spec.ts => changeAnalyzer.spec.ts} | 47 +- packages/client/test/fixtures/diffResponse.ts | 317 + .../test/fixtures/registrationSchema.ts | 519 + packages/client/test/processing.spec.ts | 91 + packages/client/test/schema-diff.json | 312 - packages/client/test/schema-functions.spec.ts | 9295 ----------------- packages/client/test/schema.json | 522 - packages/client/test/validation.spec.ts | 828 ++ 8 files changed, 1768 insertions(+), 10163 deletions(-) rename packages/client/test/{change-analyzer.spec.ts => changeAnalyzer.spec.ts} (77%) create mode 100644 packages/client/test/fixtures/diffResponse.ts create mode 100644 packages/client/test/fixtures/registrationSchema.ts create mode 100644 packages/client/test/processing.spec.ts delete mode 100644 packages/client/test/schema-diff.json delete mode 100644 packages/client/test/schema-functions.spec.ts delete mode 100644 packages/client/test/schema.json create mode 100644 packages/client/test/validation.spec.ts diff --git a/packages/client/test/change-analyzer.spec.ts b/packages/client/test/changeAnalyzer.spec.ts similarity index 77% rename from packages/client/test/change-analyzer.spec.ts rename to packages/client/test/changeAnalyzer.spec.ts index b0fb3197..c2bbff03 100644 --- a/packages/client/test/change-analyzer.spec.ts +++ b/packages/client/test/changeAnalyzer.spec.ts @@ -18,23 +18,13 @@ */ import chai from 'chai'; -import * as analyzer from '../src/change-analyzer'; -import { SchemasDictionaryDiffs, FieldDiff, ChangeAnalysis } from '../src/schema-entities'; -import _ from 'lodash'; +import { DiffUtils } from 'dictionary'; +import { analyzer } from '../src'; +import { ChangeAnalysis } from '../src/changeAnalysis'; +import diffResponse from './fixtures/diffResponse'; chai.should(); -const diffResponse: any = require('./schema-diff.json'); -const schemaDiff: SchemasDictionaryDiffs = {}; -for (const entry of diffResponse) { - const fieldName = entry[0] as string; - if (entry[1]) { - const fieldDiff: FieldDiff = { - before: entry[1].left, - after: entry[1].right, - diff: entry[1].diff, - }; - schemaDiff[fieldName] = fieldDiff; - } -} + +const diffFixture = DiffUtils.diffArrayToMap(diffResponse); const expectedResult: ChangeAnalysis = { fields: { @@ -42,12 +32,6 @@ const expectedResult: ChangeAnalysis = { renamedFields: [], deletedFields: ['primary_diagnosis.menopause_status'], }, - metaChanges: { - core: { - changedToCore: [], - changedFromCore: [], - }, - }, restrictionsChanges: { codeList: { created: [], @@ -91,16 +75,6 @@ const expectedResult: ChangeAnalysis = { created: [], deleted: [], }, - script: { - updated: [], - created: [ - { - field: 'donor.survival_time', - definition: ' $field / 2 == 0 ', - }, - ], - deleted: [], - }, range: { updated: [ { @@ -121,14 +95,19 @@ const expectedResult: ChangeAnalysis = { ], deleted: [], }, + script: { + created: [], + deleted: [], + updated: [], + }, }, isArrayDesignationChanges: ['primary_diagnosis.presenting_symptoms'], valueTypeChanges: ['sample_registration.program_id'], }; -describe('change-analyzer', () => { +describe('changeAnalyzer', () => { it('categorize changes correctly', () => { - const result = analyzer.analyzeChanges(schemaDiff); + const result = analyzer.analyzeChanges(diffFixture); result.should.deep.eq(expectedResult); }); }); diff --git a/packages/client/test/fixtures/diffResponse.ts b/packages/client/test/fixtures/diffResponse.ts new file mode 100644 index 00000000..cf164909 --- /dev/null +++ b/packages/client/test/fixtures/diffResponse.ts @@ -0,0 +1,317 @@ +import { DictionaryDiffArray } from 'dictionary'; + +const diffResponse = [ + [ + 'donor.submitter_donor_id', + { + left: { + description: 'Unique identifier of the donor, assigned by the data provider.', + name: 'submitter_donor_id', + restrictions: { + required: true, + regex: '[A-Za-z0-9\\-\\._]{1,64}', + }, + valueType: 'string', + }, + right: { + description: 'Unique identifier of the donor, assigned by the data provider.', + name: 'submitter_donor_id', + restrictions: { + required: true, + regex: '[A-Za-z0-9\\-\\._]{3,64}', + }, + valueType: 'string', + }, + diff: { + restrictions: { + regex: { + type: 'updated', + data: '[A-Za-z0-9\\-\\._]{3,64}', + }, + }, + }, + }, + ], + [ + 'donor.vital_status', + { + left: { + description: 'Donors last known state of living or deceased.', + name: 'vital_status', + restrictions: { + codeList: ['Alive', 'Deceased', 'Not reported', 'Unknown'], + required: true, + }, + valueType: 'string', + }, + right: { + description: 'Donors last known state of living or deceased.', + name: 'vital_status', + restrictions: { + regex: '[A-Z]{3,100}', + required: true, + }, + valueType: 'string', + }, + diff: { + restrictions: { + codeList: { + type: 'deleted', + data: ['Alive', 'Deceased', 'Not reported', 'Unknown'], + }, + regex: { + type: 'created', + data: '[A-Z]{3,100}', + }, + }, + }, + }, + ], + [ + 'donor.cause_of_death', + { + left: { + description: "Description of the cause of a donor's death.", + name: 'cause_of_death', + restrictions: { + codeList: ['Died of cancer', 'Died of other reasons', 'Not reported', 'Unknown'], + }, + valueType: 'string', + }, + right: { + description: "Description of the cause of a donor's death.", + name: 'cause_of_death', + restrictions: { + codeList: ['Died of other reasons', 'Not reported', 'N/A'], + }, + valueType: 'string', + }, + diff: { + restrictions: { + codeList: { + type: 'updated', + data: { + added: ['N/A'], + deleted: ['Died of cancer', 'Unknown'], + }, + }, + }, + }, + }, + ], + [ + 'donor.survival_time', + { + left: { + description: 'Interval of how long the donor has survived since primary diagnosis, in days.', + meta: { + units: 'days', + }, + name: 'survival_time', + valueType: 'integer', + }, + right: { + description: 'Interval of how long the donor has survived since primary diagnosis, in days.', + meta: { + units: 'days', + }, + name: 'survival_time', + valueType: 'integer', + restrictions: { + range: { + min: 0, + max: 200000, + }, + }, + }, + diff: { + restrictions: { + type: 'created', + data: { + range: { + min: 0, + max: 200000, + }, + }, + }, + }, + }, + ], + [ + 'primary_diagnosis.cancer_type_code', + { + left: { + name: 'cancer_type_code', + valueType: 'string', + description: + 'The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.', + restrictions: { + required: true, + regex: '[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{0,1}$', + }, + }, + right: { + name: 'cancer_type_code', + valueType: 'string', + description: + 'The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.', + restrictions: { + required: true, + regex: '[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$', + }, + }, + diff: { + restrictions: { + regex: { + type: 'updated', + data: '[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$', + }, + }, + }, + }, + ], + [ + 'primary_diagnosis.menopause_status', + { + left: { + name: 'menopause_status', + description: 'Indicate the menopause status of the patient at the time of primary diagnosis.', + valueType: 'string', + restrictions: { + codeList: ['Perimenopausal', 'Postmenopausal', 'Premenopausal', 'Unknown'], + }, + }, + diff: { + type: 'deleted', + data: { + name: 'menopause_status', + description: 'Indicate the menopause status of the patient at the time of primary diagnosis.', + valueType: 'string', + restrictions: { + codeList: ['Perimenopausal', 'Postmenopausal', 'Premenopausal', 'Unknown'], + }, + }, + }, + }, + ], + [ + 'primary_diagnosis.presenting_symptoms', + { + left: { + name: 'presenting_symptoms', + description: 'Indicate presenting symptoms at time of primary diagnosis.', + valueType: 'string', + restrictions: { + codeList: ['Abdominal Pain', 'Anemia', 'Diabetes', 'Diarrhea', 'Nausea', 'None'], + }, + }, + right: { + name: 'presenting_symptoms', + description: 'Indicate presenting symptoms at time of primary diagnosis.', + valueType: 'string', + isArray: true, + restrictions: { + codeList: ['Abdominal Pain', 'Anemia', 'Diabetes', 'Diarrhea', 'Nausea', 'None'], + }, + }, + diff: { + isArray: { + type: 'created', + data: true, + }, + }, + }, + ], + [ + 'sample_registration.program_id', + { + left: { + name: 'program_id', + valueType: 'string', + description: 'Unique identifier of the ARGO program.', + meta: { + validationDependency: true, + primaryId: true, + examples: 'PACA-AU,BR-CA', + displayName: 'Program ID', + }, + restrictions: { + required: true, + }, + }, + right: { + name: 'program_id', + valueType: 'integer', + description: 'Unique identifier of the ARGO program.', + meta: { + validationDependency: true, + primaryId: true, + examples: 'PACA-AU,BR-CA', + displayName: 'Program ID', + }, + restrictions: { + required: true, + }, + }, + diff: { + valueType: { + type: 'updated', + data: 'integer', + }, + }, + }, + ], + [ + 'specimen.percent_stromal_cells', + { + left: { + name: 'percent_stromal_cells', + description: + 'Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.', + valueType: 'number', + meta: { + dependsOn: 'sample_registration.tumour_normal_designation', + notes: '', + displayName: 'Percent Stromal Cells', + }, + restrictions: { + range: { + min: 0, + max: 0.1, + }, + }, + }, + right: { + name: 'percent_stromal_cells', + description: + 'Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.', + valueType: 'number', + meta: { + dependsOn: 'sample_registration.tumour_normal_designation', + notes: '', + displayName: 'Percent Stromal Cells', + }, + restrictions: { + range: { + max: 1, + }, + }, + }, + diff: { + restrictions: { + range: { + min: { + type: 'deleted', + data: 0, + }, + max: { + type: 'updated', + data: 1, + }, + }, + }, + }, + }, + ], +] satisfies DictionaryDiffArray; +export default diffResponse; diff --git a/packages/client/test/fixtures/registrationSchema.ts b/packages/client/test/fixtures/registrationSchema.ts new file mode 100644 index 00000000..4d57053a --- /dev/null +++ b/packages/client/test/fixtures/registrationSchema.ts @@ -0,0 +1,519 @@ +import { Dictionary } from 'dictionary'; + +const dictionary: Dictionary = { + schemas: [ + { + name: 'registration', + description: 'TSV for Registration of Donor-Specimen-Sample', + fields: [ + { + name: 'program_id', + valueType: 'string', + description: 'Unique identifier for program', + meta: { + key: true, + examples: 'PACA-CA, BASHAR-LA', + }, + restrictions: { + required: true, + regex: '^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$', + }, + }, + { + name: 'submitter_donor_id', + valueType: 'string', + description: 'Unique identifier for donor, assigned by the data provider.', + meta: { + key: true, + }, + restrictions: { + required: true, + regex: '^(?!(DO|do)).+', + }, + }, + { + name: 'gender', + valueType: 'string', + description: 'The gender of the patient', + meta: { + default: 'Other', + }, + restrictions: { + required: true, + codeList: ['Male', 'Female', 'Other'], + }, + }, + { + name: 'submitter_specimen_id', + valueType: 'string', + description: 'Submitter assigned specimen id', + meta: { + key: true, + }, + restrictions: { + required: true, + regex: '^(?!(SP|sp)).+', + }, + }, + { + name: 'specimen_type', + valueType: 'string', + description: 'Indicate the tissue source of the biospecimen', + meta: { + default: 'Other', + }, + restrictions: { + required: true, + codeList: [ + 'Blood derived', + 'Blood derived - bone marrow', + 'Blood derived - peripheral blood', + 'Bone marrow', + 'Buccal cell', + 'Lymph node', + 'Solid tissue', + 'Plasma', + 'Serum', + 'Urine', + 'Cerebrospinal fluid', + 'Sputum', + 'NOS (Not otherwise specified)', + 'Other', + 'FFPE', + 'Pleural effusion', + 'Mononuclear cells from bone marrow', + 'Saliva', + 'Skin', + ], + }, + }, + { + name: 'tumour_normal_designation', + valueType: 'string', + description: 'Indicate whether specimen is tumour or normal type', + restrictions: { + required: true, + codeList: [ + 'Normal', + 'Normal - tissue adjacent to primary tumour', + 'Primary tumour', + 'Primary tumour - adjacent to normal', + 'Primary tumour - additional new primary', + 'Recurrent tumour', + 'Metastatic tumour', + 'Metastatic tumour - metastasis local to lymph node', + 'Metastatic tumour - metastasis to distant location', + 'Metastatic tumour - additional metastatic', + 'Xenograft - derived from primary tumour', + 'Xenograft - derived from tumour cell line', + 'Cell line - derived from xenograft tissue', + 'Cell line - derived from tumour', + 'Cell line - derived from normal', + ], + }, + }, + { + name: 'submitter_sample_id', + valueType: 'string', + description: 'Submitter assigned sample id', + restrictions: { + required: true, + regex: '^(?!(SA|sa)).+', + }, + }, + { + name: 'sample_type', + valueType: 'string', + description: 'Specimen Type', + restrictions: { + required: true, + codeList: [ + 'Total DNA', + 'Amplified DNA', + 'ctDNA', + 'other DNA enrichments', + 'Total RNA', + 'Ribo-Zero RNA', + 'polyA+ RNA', + 'other RNA fractions', + ], + }, + }, + ], + }, + { + name: 'address', + description: 'adderss schema', + fields: [ + { + name: 'postal_code', + valueType: 'string', + description: 'postal code', + restrictions: { + required: true, + script: + '/** important to return the result object here here */\r\n(function validate(inputs) {\r\n const {$row, $field, $name} = inputs; var person = $row;\r\nvar postalCode = $field; var result = { valid: true, message: "ok"};\r\n\r\n /* custom logic start */\r\n if (person.country === "US") {\r\n var valid = /^[0-9]{5}(?:-[0-9]{4})?$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = "invalid postal code for US";\r\n }\r\n } else if (person.country === "CANADA") {\r\n var valid = /^[A-Za-z]\\d[A-Za-z][ -]?\\d[A-Za-z]\\d$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = "invalid postal code for CANADA";\r\n }\r\n }\r\n /* custom logic end */\r\n\r\n return result;\r\n})\r\n\r\n', + }, + }, + { + name: 'unit_number', + valueType: 'integer', + description: 'unit number', + restrictions: { + range: { + min: 0, + exclusiveMax: 999, + }, + }, + }, + { + name: 'country', + valueType: 'string', + description: 'Country', + restrictions: { + required: true, + codeList: ['US', 'CANADA'], + }, + }, + ], + }, + { + name: 'donor', + description: 'TSV for donor', + fields: [ + { + name: 'program_id', + valueType: 'string', + description: 'Unique identifier for program', + meta: { + key: true, + }, + restrictions: { + required: true, + regex: '^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$', + }, + }, + { + name: 'submitter_donor_id', + valueType: 'string', + description: 'Unique identifier for donor, assigned by the data provider.', + meta: { + key: true, + }, + restrictions: { + required: true, + regex: '^(?!(DO|do)).+', + }, + }, + { + name: 'gender', + valueType: 'string', + description: 'The gender of the patient', + meta: { + default: 'Other', + }, + restrictions: { + required: true, + codeList: ['Male', 'Female', 'Other'], + }, + }, + { + name: 'ethnicity', + valueType: 'string', + description: 'The ethnicity of the patient', + restrictions: { + required: true, + codeList: ['asian', 'black or african american', 'caucasian', 'not reported'], + }, + }, + { + name: 'vital_status', + valueType: 'string', + description: 'Indicate the vital status of the patient', + restrictions: { + required: true, + codeList: ['alive', 'deceased'], + }, + }, + { + name: 'cause_of_death', + valueType: 'string', + description: 'Indicate the cause of death of patient', + restrictions: { + required: false, + codeList: ['died of cancer', 'died of other reasons', 'N/A'], + }, + }, + { + name: 'survival_time', + valueType: 'integer', + description: 'Survival time', + restrictions: { + required: false, + }, + }, + ], + }, + { + name: 'favorite_things', + description: 'favorite things listed', + fields: [ + { + name: 'id', + valueType: 'string', + description: 'Favourite id values', + restrictions: { + required: true, + regex: '^[A-Z1-9][-_A-Z1-9]{2,7}$', + }, + }, + { + name: 'qWords', + valueType: 'string', + description: 'Words starting with q', + restrictions: { + required: false, + regex: '^q.*$', + }, + isArray: true, + }, + { + name: 'qWord', + valueType: 'string', + description: 'Word starting with q', + restrictions: { + required: false, + regex: '^q.*$', + }, + isArray: false, + }, + { + name: 'fruit', + valueType: 'string', + description: 'fruit', + restrictions: { + required: false, + codeList: ['Mango', 'Orange', 'None'], + }, + isArray: true, + }, + { + name: 'fruit_single_value', + valueType: 'string', + description: 'fruit', + restrictions: { + required: false, + codeList: ['Mango', 'Orange', 'None'], + }, + isArray: false, + }, + { + name: 'animal', + valueType: 'string', + description: 'animal', + restrictions: { + required: false, + codeList: ['Dog', 'Cat', 'None'], + }, + isArray: true, + }, + { + name: 'fraction', + valueType: 'number', + description: 'numbers between 0 and 1 exclusive', + restrictions: { required: false, range: { max: 1, exclusiveMin: 0 } }, + isArray: true, + }, + { + name: 'integers', + valueType: 'integer', + description: 'integers between -10 and 10', + restrictions: { required: false, range: { max: 10, min: -10 } }, + isArray: true, + }, + { + name: 'unique_value', + valueType: 'string', + description: 'unique value', + restrictions: { required: false, unique: true }, + isArray: true, + }, + ], + }, + { + name: 'parent_schema_1', + description: 'Parent schema 1. Used to test relational validations', + fields: [ + { + name: 'id', + valueType: 'string', + description: 'Id', + }, + { + name: 'external_id', + valueType: 'string', + description: 'External Id', + }, + { + name: 'name', + valueType: 'string', + description: 'Name', + }, + ], + }, + { + name: 'parent_schema_2', + description: 'Parent schema 1. Used to test relational validations', + fields: [ + { + name: 'id1', + valueType: 'string', + description: 'Id 1', + isArray: true, + }, + { + name: 'id2', + valueType: 'string', + description: 'Id 2', + isArray: true, + }, + ], + }, + { + name: 'child_schema_simple_fk', + description: 'Child schema referencing a field in a foreign schema', + restrictions: { + foreignKey: [ + { + schema: 'parent_schema_1', + mappings: [ + { + local: 'parent_schema_1_id', + foreign: 'id', + }, + ], + }, + ], + }, + fields: [ + { + name: 'id', + valueType: 'number', + description: 'Id', + }, + { + name: 'parent_schema_1_id', + valueType: 'string', + description: 'Reference to id in schema parent_schema_1', + }, + ], + }, + { + name: 'child_schema_composite_fk', + description: 'Child schema referencing several fields in a foreign schema', + restrictions: { + foreignKey: [ + { + schema: 'parent_schema_1', + mappings: [ + { + local: 'parent_schema_1_id', + foreign: 'id', + }, + { + local: 'parent_schema_1_external_id', + foreign: 'external_id', + }, + ], + }, + ], + }, + fields: [ + { + name: 'id', + valueType: 'number', + description: 'Id', + }, + { + name: 'parent_schema_1_id', + valueType: 'string', + description: 'Reference to id in schema parent_schema_1', + }, + { + name: 'parent_schema_1_external_id', + valueType: 'string', + description: 'Reference to external id in schema parent_schema_1', + }, + ], + }, + { + name: 'child_schema_composite_array_values_fk', + description: 'Child schema referencing several fields in a foreign schema', + restrictions: { + foreignKey: [ + { + schema: 'parent_schema_2', + mappings: [ + { + local: 'parent_schema_2_id1', + foreign: 'id1', + }, + { + local: 'parent_schema_2_id12', + foreign: 'id2', + }, + ], + }, + ], + }, + fields: [ + { + name: 'id', + valueType: 'number', + description: 'Id', + }, + { + name: 'parent_schema_2_id1', + valueType: 'string', + description: 'Reference to id1 in schema parent_schema_2', + isArray: true, + }, + { + name: 'parent_schema_2_id12', + valueType: 'string', + description: 'Reference to external id2 in schema parent_schema_2', + isArray: true, + }, + ], + }, + { + name: 'unique_key_schema', + description: 'Schema to test uniqueKey restriction', + restrictions: { + uniqueKey: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + }, + fields: [ + { + name: 'numeric_id_1', + valueType: 'number', + description: 'Id 1. Numeric value as part of a composite unique key', + }, + { + name: 'string_id_2', + valueType: 'string', + description: 'Id 2. String value as part of a composite unique key', + }, + { + name: 'array_string_id_3', + valueType: 'string', + description: 'Id 3. String array as part of a composite unique key', + isArray: true, + }, + ], + }, + ], + name: 'ARGO Clinical Submission', + version: '1.0', +}; + +export default dictionary; diff --git a/packages/client/test/processing.spec.ts b/packages/client/test/processing.spec.ts new file mode 100644 index 00000000..503239ba --- /dev/null +++ b/packages/client/test/processing.spec.ts @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import chai from 'chai'; +import { functions as schemaService } from '../src'; +import { loggerFor } from '../src/logger'; +import { SchemaValidationErrorTypes } from '../src/validation'; +import dictionary from './fixtures/registrationSchema'; +const L = loggerFor(__filename); + +chai.should(); + +const VALUE_NOT_ALLOWED = 'The value is not permissible for this field.'; +const PROGRAM_ID_REQ = 'program_id is a required field.'; + +describe('processing', () => { + it('should populate records based on default value ', () => { + const result = schemaService.processRecords(dictionary, 'registration', [ + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87812', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS1234', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.processedRecords[0].gender).to.eq('Other'); + chai.expect(result.processedRecords[1].gender).to.eq('Other'); + }); + + it('should NOT populate missing columns based on default value ', () => { + const result = schemaService.processRecords(dictionary, 'registration', [ + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gendr: '', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87812', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS1234', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + fieldName: 'gender', + index: 0, + info: {}, + message: 'gender is a required field.', + }); + }); +}); diff --git a/packages/client/test/schema-diff.json b/packages/client/test/schema-diff.json deleted file mode 100644 index e4f7a4dc..00000000 --- a/packages/client/test/schema-diff.json +++ /dev/null @@ -1,312 +0,0 @@ -[ - [ - "donor.submitter_donor_id", - { - "left": { - "description": "Unique identifier of the donor, assigned by the data provider.", - "name": "submitter_donor_id", - "restrictions": { - "required": true, - "regex": "[A-Za-z0-9\\-\\._]{1,64}" - }, - "valueType": "string" - }, - "right": { - "description": "Unique identifier of the donor, assigned by the data provider.", - "name": "submitter_donor_id", - "restrictions": { - "required": true, - "regex": "[A-Za-z0-9\\-\\._]{3,64}" - }, - "valueType": "string" - }, - "diff": { - "restrictions": { - "regex": { - "type": "updated", - "data": "[A-Za-z0-9\\-\\._]{3,64}" - } - } - } - } - ], - [ - "donor.vital_status", - { - "left": { - "description": "Donors last known state of living or deceased.", - "name": "vital_status", - "restrictions": { - "codeList": ["Alive", "Deceased", "Not reported", "Unknown"], - "required": true - }, - "valueType": "string" - }, - "right": { - "description": "Donors last known state of living or deceased.", - "name": "vital_status", - "restrictions": { - "regex": "[A-Z]{3,100}", - "required": true - }, - "valueType": "string" - }, - "diff": { - "restrictions": { - "codeList": { - "type": "deleted", - "data": ["Alive", "Deceased", "Not reported", "Unknown"] - }, - "regex": { - "type": "created", - "data": "[A-Z]{3,100}" - } - } - } - } - ], - [ - "donor.cause_of_death", - { - "left": { - "description": "Description of the cause of a donor's death.", - "name": "cause_of_death", - "restrictions": { - "codeList": ["Died of cancer", "Died of other reasons", "Not reported", "Unknown"] - }, - "valueType": "string" - }, - "right": { - "description": "Description of the cause of a donor's death.", - "name": "cause_of_death", - "restrictions": { - "codeList": ["Died of other reasons", "Not reported", "N/A"] - }, - "valueType": "string" - }, - "diff": { - "restrictions": { - "codeList": { - "type": "updated", - "data": { - "added": ["N/A"], - "deleted": ["Died of cancer", "Unknown"] - } - } - } - } - } - ], - [ - "donor.survival_time", - { - "left": { - "description": "Interval of how long the donor has survived since primary diagnosis, in days.", - "meta": { - "units": "days" - }, - "name": "survival_time", - "valueType": "integer" - }, - "right": { - "description": "Interval of how long the donor has survived since primary diagnosis, in days.", - "meta": { - "units": "days" - }, - "name": "survival_time", - "valueType": "integer", - "restrictions": { - "script": " $field / 2 == 0 ", - "range": { - "min": 0, - "max": 200000 - } - } - }, - "diff": { - "restrictions": { - "type": "created", - "data": { - "range": { - "min": 0, - "max": 200000 - }, - "script": " $field / 2 == 0 " - } - } - } - } - ], - [ - "primary_diagnosis.cancer_type_code", - { - "left": { - "name": "cancer_type_code", - "valueType": "string", - "description": "The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.", - "restrictions": { - "required": true, - "regex": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{0,1}$" - } - }, - "right": { - "name": "cancer_type_code", - "valueType": "string", - "description": "The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.", - "restrictions": { - "required": true, - "regex": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$" - } - }, - "diff": { - "restrictions": { - "regex": { - "type": "updated", - "data": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$" - } - } - } - } - ], - [ - "primary_diagnosis.menopause_status", - { - "left": { - "name": "menopause_status", - "description": "Indicate the menopause status of the patient at the time of primary diagnosis.", - "valueType": "string", - "restrictions": { - "codeList": ["Perimenopausal", "Postmenopausal", "Premenopausal", "Unknown"] - } - }, - "diff": { - "type": "deleted", - "data": { - "name": "menopause_status", - "description": "Indicate the menopause status of the patient at the time of primary diagnosis.", - "valueType": "string", - "restrictions": { - "codeList": ["Perimenopausal", "Postmenopausal", "Premenopausal", "Unknown"] - } - } - } - } - ], - [ - "primary_diagnosis.presenting_symptoms", - { - "left": { - "name": "presenting_symptoms", - "description": "Indicate presenting symptoms at time of primary diagnosis.", - "valueType": "string", - "restrictions": { - "codeList": ["Abdominal Pain", "Anemia", "Diabetes", "Diarrhea", "Nausea", "None"] - } - }, - "right": { - "name": "presenting_symptoms", - "description": "Indicate presenting symptoms at time of primary diagnosis.", - "valueType": "string", - "isArray": true, - "restrictions": { - "codeList": ["Abdominal Pain", "Anemia", "Diabetes", "Diarrhea", "Nausea", "None"] - } - }, - "diff": { - "isArray": { - "type": "created", - "data": true - } - } - } - ], - [ - "sample_registration.program_id", - { - "left": { - "name": "program_id", - "valueType": "string", - "description": "Unique identifier of the ARGO program.", - "meta": { - "validationDependency": true, - "primaryId": true, - "examples": "PACA-AU,BR-CA", - "displayName": "Program ID" - }, - "restrictions": { - "required": true - } - }, - "right": { - "name": "program_id", - "valueType": "integer", - "description": "Unique identifier of the ARGO program.", - "meta": { - "validationDependency": true, - "primaryId": true, - "examples": "PACA-AU,BR-CA", - "displayName": "Program ID" - }, - "restrictions": { - "required": true - } - }, - "diff": { - "valueType": { - "type": "updated", - "data": "integer" - } - } - } - ], - [ - "specimen.percent_stromal_cells", - { - "left": { - "name": "percent_stromal_cells", - "description": "Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.", - "valueType": "number", - "meta": { - "dependsOn": "sample_registration.tumour_normal_designation", - "notes": "", - "displayName": "Percent Stromal Cells" - }, - "restrictions": { - "range": { - "min": 0, - "max": 0.1 - } - } - }, - "right": { - "name": "percent_stromal_cells", - "description": "Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.", - "valueType": "number", - "meta": { - "dependsOn": "sample_registration.tumour_normal_designation", - "notes": "", - "displayName": "Percent Stromal Cells" - }, - "restrictions": { - "range": { - "max": 1 - } - } - }, - "diff": { - "restrictions": { - "range": { - "min": { - "type": "deleted", - "data": 0 - }, - "max": { - "type": "updated", - "data": 1 - } - } - } - } - } - ] -] diff --git a/packages/client/test/schema-functions.spec.ts b/packages/client/test/schema-functions.spec.ts deleted file mode 100644 index 315fa814..00000000 --- a/packages/client/test/schema-functions.spec.ts +++ /dev/null @@ -1,9295 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import chai from 'chai'; -import * as schemaService from '../src/schema-functions'; -import { SchemasDictionary, SchemaValidationErrorTypes } from '../src/schema-entities'; -import schemaErrorMessage from '../src/schema-error-messages'; -import { loggerFor } from '../src/logger'; -const L = loggerFor(__filename); - -chai.should(); -const schema: SchemasDictionary = require('./schema.json')[0]; - -const VALUE_NOT_ALLOWED = 'The value is not permissible for this field.'; -const PROGRAM_ID_REQ = 'program_id is a required field.'; - -describe('schema-functions', () => { - it('should populate records based on default value ', () => { - const result = schemaService.processRecords(schema, 'registration', [ - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gender: '', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gender: '', - submitter_specimen_id: '87812', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS1234', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.processedRecords[0].gender).to.eq('Other'); - chai.expect(result.processedRecords[1].gender).to.eq('Other'); - }); - - it('should NOT populate missing columns based on default value ', () => { - const result = schemaService.processRecords(schema, 'registration', [ - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gendr: '', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gender: '', - submitter_specimen_id: '87812', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS1234', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, - fieldName: 'gender', - index: 0, - info: {}, - message: 'gender is a required field.', - }); - }); - - it('should validate required', () => { - const result = schemaService.processRecords(schema, 'registration', [ - { - submitter_donor_id: 'OD1234', - gender: 'Female', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, - fieldName: 'program_id', - index: 0, - info: {}, - message: PROGRAM_ID_REQ, - }); - }); - - it('should validate value types', () => { - const result = schemaService.processRecords(schema, 'address', [ - { - country: 'US', - unit_number: 'abc', - postal_code: '12345', - }, - ]); - - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, - fieldName: 'unit_number', - index: 0, - info: { value: ['abc'] }, - message: VALUE_NOT_ALLOWED, - }); - }); - - it('should convert string to integer after processing', () => { - const result = schemaService.processRecords(schema, 'address', [ - { - country: 'US', - unit_number: '123', - postal_code: '12345', - }, - ]); - chai.expect(result.processedRecords).to.deep.include({ - country: 'US', - unit_number: 123, - postal_code: '12345', - }); - }); - - it('should validate regex', () => { - const result = schemaService.processRecords(schema, 'registration', [ - { - program_id: 'PEME-CAA', - submitter_donor_id: 'OD1234', - gender: 'Female', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, - fieldName: 'program_id', - index: 0, - info: { - examples: 'PACA-CA, BASHAR-LA', - regex: '^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$', - value: ['PEME-CAA'], - }, - message: - 'The value is not a permissible for this field, it must meet the regular expression: "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$". Examples: PACA-CA, BASHAR-LA', - }); - }); - - it('should validate range', () => { - const result = schemaService.processRecords(schema, 'address', [ - { - country: 'US', - postal_code: '12345', - unit_number: '-1', - }, - { - country: 'US', - postal_code: '12345', - unit_number: '223', - }, - { - country: 'US', - postal_code: '12345', - unit_number: '500000', - }, - ]); - - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - fieldName: 'unit_number', - index: 0, - info: { - exclusiveMax: 999, - min: 0, - value: [-1], - }, - message: schemaErrorMessage(SchemaValidationErrorTypes.INVALID_BY_RANGE, { - info: { - exclusiveMax: 999, - min: 0, - }, - }), - }); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - fieldName: 'unit_number', - index: 2, - info: { - exclusiveMax: 999, - min: 0, - value: [500000], - }, - message: schemaErrorMessage(SchemaValidationErrorTypes.INVALID_BY_RANGE, { - info: { - exclusiveMax: 999, - min: 0, - }, - }), - }); - }); - - it('should validate script', () => { - const result = schemaService.processRecords(schema, 'address', [ - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, - fieldName: 'postal_code', - index: 0, - info: { message: 'invalid postal code for US', value: '12' }, - message: 'invalid postal code for US', - }); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, - fieldName: 'postal_code', - index: 1, - info: { message: 'invalid postal code for CANADA', value: 'ABC' }, - message: 'invalid postal code for CANADA', - }); - }); - - it('should validate if non-required feilds are not provided', () => { - const result = schemaService.processRecords(schema, 'donor', [ - // optional enum field not provided - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0004', - gender: 'Female', - ethnicity: 'black or african american', - vital_status: 'alive', - }, - // optional enum field provided with proper value - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Male', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: 'died of cancer', - survival_time: '124', - }, - // optional enum field provided with no value - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Male', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: '', - survival_time: '124', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should error if integer fields are not valid', () => { - const result = schemaService.processRecords(schema, 'donor', [ - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Other', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: 'died of cancer', - survival_time: '0.5', - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, - fieldName: 'survival_time', - index: 0, - info: { value: ['0.5'] }, - message: VALUE_NOT_ALLOWED, - }); - }); - - it('should validate case insensitive enums, return proper format', () => { - const result = schemaService.processRecords(schema, 'registration', [ - { - program_id: 'PACA-AU', - submitter_donor_id: 'OD1234', - gender: 'feMale', - submitter_specimen_id: '87813', - specimen_type: 'sKiN', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'CTdna', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - chai.expect(result.processedRecords[0]).to.deep.eq({ - program_id: 'PACA-AU', - submitter_donor_id: 'OD1234', - gender: 'Female', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }); - }); - - it('should not validate if unrecognized fields are provided', () => { - const result = schemaService.processRecords(schema, 'donor', [ - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Other', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: 'died of cancer', - survival_time: '5', - hackField: 'muchHack', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, - message: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, - fieldName: 'hackField', - index: 0, - info: {}, - }); - }); - - it('should validate number/integer array with field defined ranges', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - fraction: ['0.2', '2', '3'], - integers: ['-100', '-2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - message: 'Value is out of permissible range, value must be > 0 and <= 1.', - index: 0, - fieldName: 'fraction', - info: { value: [2, 3], max: 1, exclusiveMin: 0 }, - }); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - message: 'Value is out of permissible range, value must be >= -10 and <= 10.', - index: 0, - fieldName: 'integers', - info: { value: [-100], max: 10, min: -10 }, - }); - }); - - it('should validate string array with field defined codelist', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - fruit: ['Mango', '2'], - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, - message: 'The value is not permissible for this field.', - fieldName: 'fruit', - index: 0, - info: { value: ['2'] }, - }); - }); - - it('should validate string with field defined codelist', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - fruit_single_value: 'Banana', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, - message: 'The value is not permissible for this field.', - fieldName: 'fruit_single_value', - index: 0, - info: { value: ['Banana'] }, - }); - }); - - it('should validate string array with field defined regex', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - qWords: ['que', 'not_q'], - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, - message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', - fieldName: 'qWords', - index: 0, - info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, - }); - }); - - it('should validate string with field defined regex', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - qWord: 'not_q', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, - message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', - fieldName: 'qWord', - index: 0, - info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, - }); - }); - - it('should pass unique restriction validation when only null values exists', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - unique_value: '', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should pass unique restriction validation when only one record exists', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - unique_value: 'unique_value_1', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should fail unique restriction validation when duplicate values exist (scalar)', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'ID-1', - unique_value: 'unique_value_1', - }, - { - id: 'ID-2', - unique_value: 'unique_value_1', - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Value for unique_value must be unique.', - fieldName: 'unique_value', - index: 0, - info: { value: ['unique_value_1'] }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Value for unique_value must be unique.', - fieldName: 'unique_value', - index: 1, - info: { value: ['unique_value_1'] }, - }); - }); - it('should fail unique restriction validation when duplicate values exist (array)', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'ID-1', - unique_value: ['unique_value_1', 'unique_value_2'], - }, - { - id: 'ID-2', - unique_value: ['unique_value_1', 'unique_value_2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Value for unique_value must be unique.', - fieldName: 'unique_value', - index: 0, - info: { value: ['unique_value_1', 'unique_value_2'] }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Value for unique_value must be unique.', - fieldName: 'unique_value', - index: 1, - info: { value: ['unique_value_1', 'unique_value_2'] }, - }); - }); - - it('should pass foreignKey restriction validation when values exist in foreign schema', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_simple_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - }, - { - id: '2', - parent_schema_1_id: 'parent_schema_1_id_2', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_simple_fk: child_schema_simple_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - - chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); - chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); - }); - - it('should pass foreignKey restriction validation when local schema has null values', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_simple_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - }, - { - id: '2', - parent_schema_1_id: '', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_simple_fk: child_schema_simple_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - - chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); - chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); - }); - - it('should pass foreignKey restriction validation when values exist in foreign schema (composite fk)', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - external_id: 'parent_schema_1_external_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - external_id: 'parent_schema_1_external_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_composite_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - parent_schema_1_external_id: 'parent_schema_1_external_id_1', - }, - { - id: '2', - parent_schema_1_id: 'parent_schema_1_id_2', - parent_schema_1_external_id: 'parent_schema_1_external_id_2', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_composite_fk: child_schema_composite_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - - chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); - chai.expect(result['child_schema_composite_fk'].validationErrors.length).to.eq(0); - }); - - it('should fail foreignKey restriction validation when value does not exist in foreign schema', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_simple_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - }, - { - id: '2', - parent_schema_1_id: 'non_existing_value_in_foreign_schema', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_simple_fk: child_schema_simple_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - const childSchemaErrors = result['child_schema_simple_fk'].validationErrors; - - chai.expect(childSchemaErrors.length).to.eq(1); - chai.expect(childSchemaErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, - message: - 'Record violates foreign key restriction defined for field(s) parent_schema_1_id. Key parent_schema_1_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', - fieldName: 'parent_schema_1_id', - index: 1, - info: { foreignSchema: 'parent_schema_1', value: { parent_schema_1_id: 'non_existing_value_in_foreign_schema' } }, - }); - }); - - it('should fail foreignKey restriction validation when values do not exist in foreign schema (composite fk)', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - external_id: 'parent_schema_1_external_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - external_id: 'parent_schema_1_external_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_composite_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - parent_schema_1_external_id: 'parent_schema_1_external_id_1', - }, - { - id: '2', - parent_schema_1_id: 'parent_schema_1_id_2', - parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_composite_fk: child_schema_composite_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - const childSchemaErrors = result['child_schema_composite_fk'].validationErrors; - - chai.expect(childSchemaErrors.length).to.eq(1); - chai.expect(childSchemaErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, - message: - 'Record violates foreign key restriction defined for field(s) parent_schema_1_id, parent_schema_1_external_id. Key parent_schema_1_id: parent_schema_1_id_2, parent_schema_1_external_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', - fieldName: 'parent_schema_1_id, parent_schema_1_external_id', - index: 1, - info: { - foreignSchema: 'parent_schema_1', - value: { - parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', - parent_schema_1_id: 'parent_schema_1_id_2', - }, - }, - }); - }); - - it('should fail foreignKey restriction validation when values (array) do not match in foreign schema (composite fk)', () => { - const parent_schema_2_data = [ - { - id1: ['id1_1', 'id1_2'], - id2: ['id2_1'], - }, - ]; - - const child_schema_composite_array_values_fk_data = [ - { - id: '1', - parent_schema_2_id1: ['id1_1'], - parent_schema_2_id12: ['id1_2', 'id2_1'], - }, - ]; - const schemaData = { - parent_schema_2: parent_schema_2_data, - child_schema_composite_array_values_fk: child_schema_composite_array_values_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - const childSchemaErrors = result['child_schema_composite_array_values_fk'].validationErrors; - - chai.expect(childSchemaErrors.length).to.eq(1); - chai.expect(childSchemaErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, - message: - 'Record violates foreign key restriction defined for field(s) parent_schema_2_id1, parent_schema_2_id12. Key parent_schema_2_id1: [id1_1], parent_schema_2_id12: [id1_2, id2_1] is not present in schema parent_schema_2.', - fieldName: 'parent_schema_2_id1, parent_schema_2_id12', - index: 0, - info: { - foreignSchema: 'parent_schema_2', - value: { - parent_schema_2_id1: ['id1_1'], - parent_schema_2_id12: ['id1_2', 'id2_1'], - }, - }, - }); - }); - - it('should pass uniqueKey restriction validation when only a record exists', () => { - const result = schemaService.processRecords(schema, 'unique_key_schema', [ - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should pass uniqueKey restriction validation when values are unique', () => { - const result = schemaService.processRecords(schema, 'unique_key_schema', [ - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_x'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should fail uniqueKey restriction validation when missing values are part of the key and they are not unique', () => { - const result = schemaService.processRecords(schema, 'unique_key_schema', [ - { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: [], - }, - { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: [], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: 'Key numeric_id_1: null, string_id_2: null, array_string_id_3: null must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 0, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: '', - }, - }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: 'Key numeric_id_1: null, string_id_2: null, array_string_id_3: null must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 1, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: '', - }, - }, - }); - }); - - it('should fail uniqueKey restriction validation when values are not unique', () => { - const result = schemaService.processRecords(schema, 'unique_key_schema', [ - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: - 'Key numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2] must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 0, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: 1, - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: - 'Key numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2] must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 1, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: 1, - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - }, - }); - }); -}); - -const records = [ - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, -]; diff --git a/packages/client/test/schema.json b/packages/client/test/schema.json deleted file mode 100644 index 19411a64..00000000 --- a/packages/client/test/schema.json +++ /dev/null @@ -1,522 +0,0 @@ -[ - { - "schemas": [ - { - "name": "registration", - "description": "TSV for Registration of Donor-Specimen-Sample", - "key": "submitter_donor_id", - "fields": [ - { - "name": "program_id", - "valueType": "string", - "description": "Unique identifier for program", - "meta": { - "key": true, - "examples": "PACA-CA, BASHAR-LA" - }, - "restrictions": { - "required": true, - "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$" - } - }, - { - "name": "submitter_donor_id", - "valueType": "string", - "description": "Unique identifier for donor, assigned by the data provider.", - "meta": { - "key": true - }, - "restrictions": { - "required": true, - "regex": "^(?!(DO|do)).+" - } - }, - { - "name": "gender", - "valueType": "string", - "description": "The gender of the patient", - "meta": { - "default": "Other" - }, - "restrictions": { - "required": true, - "codeList": ["Male", "Female", "Other"] - } - }, - { - "name": "submitter_specimen_id", - "valueType": "string", - "description": "Submitter assigned specimen id", - "meta": { - "key": true - }, - "restrictions": { - "required": true, - "regex": "^(?!(SP|sp)).+" - } - }, - { - "name": "specimen_type", - "valueType": "string", - "description": "Indicate the tissue source of the biospecimen", - "meta": { - "default": "Other" - }, - "restrictions": { - "required": true, - "codeList": [ - "Blood derived", - "Blood derived - bone marrow", - "Blood derived - peripheral blood", - "Bone marrow", - "Buccal cell", - "Lymph node", - "Solid tissue", - "Plasma", - "Serum", - "Urine", - "Cerebrospinal fluid", - "Sputum", - "NOS (Not otherwise specified)", - "Other", - "FFPE", - "Pleural effusion", - "Mononuclear cells from bone marrow", - "Saliva", - "Skin" - ] - } - }, - { - "name": "tumour_normal_designation", - "valueType": "string", - "description": "Indicate whether specimen is tumour or normal type", - "restrictions": { - "required": true, - "codeList": [ - "Normal", - "Normal - tissue adjacent to primary tumour", - "Primary tumour", - "Primary tumour - adjacent to normal", - "Primary tumour - additional new primary", - "Recurrent tumour", - "Metastatic tumour", - "Metastatic tumour - metastasis local to lymph node", - "Metastatic tumour - metastasis to distant location", - "Metastatic tumour - additional metastatic", - "Xenograft - derived from primary tumour", - "Xenograft - derived from tumour cell line", - "Cell line - derived from xenograft tissue", - "Cell line - derived from tumour", - "Cell line - derived from normal" - ] - } - }, - { - "name": "submitter_sample_id", - "valueType": "string", - "description": "Submitter assigned sample id", - "restrictions": { - "required": true, - "regex": "^(?!(SA|sa)).+" - } - }, - { - "name": "sample_type", - "valueType": "string", - "description": "Specimen Type", - "restrictions": { - "required": true, - "codeList": [ - "Total DNA", - "Amplified DNA", - "ctDNA", - "other DNA enrichments", - "Total RNA", - "Ribo-Zero RNA", - "polyA+ RNA", - "other RNA fractions" - ] - } - } - ] - }, - { - "name": "address", - "description": "adderss schema", - "fields": [ - { - "name": "postal_code", - "valueType": "string", - "description": "postal code", - "restrictions": { - "required": true, - "script": "/** important to return the result object here here */\r\n(function validate(inputs) {\r\n const {$row, $field, $name} = inputs; var person = $row;\r\nvar postalCode = $field; var result = { valid: true, message: \"ok\"};\r\n\r\n /* custom logic start */\r\n if (person.country === \"US\") {\r\n var valid = /^[0-9]{5}(?:-[0-9]{4})?$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = \"invalid postal code for US\";\r\n }\r\n } else if (person.country === \"CANADA\") {\r\n var valid = /^[A-Za-z]\\d[A-Za-z][ -]?\\d[A-Za-z]\\d$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = \"invalid postal code for CANADA\";\r\n }\r\n }\r\n /* custom logic end */\r\n\r\n return result;\r\n})\r\n\r\n" - } - }, - { - "name": "unit_number", - "valueType": "integer", - "description": "unit number", - "restrictions": { - "range": { - "min": 0, - "exclusiveMax": 999 - } - } - }, - { - "name": "country", - "valueType": "string", - "description": "Country", - "restrictions": { - "required": true, - "codeList": ["US", "CANADA"] - } - } - ] - }, - { - "name": "donor", - "description": "TSV for donor", - "key": "submitter_donor_id", - "fields": [ - { - "name": "program_id", - "valueType": "string", - "description": "Unique identifier for program", - "meta": { - "key": true - }, - "restrictions": { - "required": true, - "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$" - } - }, - { - "name": "submitter_donor_id", - "valueType": "string", - "description": "Unique identifier for donor, assigned by the data provider.", - "meta": { - "key": true - }, - "restrictions": { - "required": true, - "regex": "^(?!(DO|do)).+" - } - }, - { - "name": "gender", - "valueType": "string", - "description": "The gender of the patient", - "meta": { - "default": "Other" - }, - "restrictions": { - "required": true, - "codeList": ["Male", "Female", "Other"] - } - }, - { - "name": "ethnicity", - "valueType": "string", - "description": "The ethnicity of the patient", - "restrictions": { - "required": true, - "codeList": ["asian", "black or african american", "caucasian", "not reported"] - } - }, - { - "name": "vital_status", - "valueType": "string", - "description": "Indicate the vital status of the patient", - "restrictions": { - "required": true, - "codeList": ["alive", "deceased"] - } - }, - { - "name": "cause_of_death", - "valueType": "string", - "description": "Indicate the cause of death of patient", - "restrictions": { - "required": false, - "codeList": ["died of cancer", "died of other reasons", "N/A"] - } - }, - { - "name": "survival_time", - "valueType": "integer", - "description": "Survival time", - "restrictions": { - "required": false - } - } - ] - }, - { - "name": "favorite_things", - "description": "favorite things listed", - "fields": [ - { - "name": "id", - "valueType": "string", - "description": "Favourite id values", - "restrictions": { - "required": true, - "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}$" - } - }, - { - "name": "qWords", - "valueType": "string", - "description": "Words starting with q", - "restrictions": { - "required": false, - "regex": "^q.*$" - }, - "isArray": true - }, - { - "name": "qWord", - "valueType": "string", - "description": "Word starting with q", - "restrictions": { - "required": false, - "regex": "^q.*$" - }, - "isArray": false - }, - { - "name": "fruit", - "valueType": "string", - "description": "fruit", - "restrictions": { - "required": false, - "codeList": ["Mango", "Orange", "None"] - }, - "isArray": true - }, - { - "name": "fruit_single_value", - "valueType": "string", - "description": "fruit", - "restrictions": { - "required": false, - "codeList": ["Mango", "Orange", "None"] - }, - "isArray": false - }, - { - "name": "animal", - "valueType": "string", - "description": "animal", - "restrictions": { - "required": false, - "codeList": ["Dog", "Cat", "None"] - }, - "isArray": true - }, - { - "name": "fraction", - "valueType": "number", - "description": "numbers between 0 and 1 exclusive", - "restrictions": { "required": false, "range": { "max": 1, "exclusiveMin": 0 } }, - "isArray": true - }, - { - "name": "integers", - "valueType": "integer", - "description": "integers between -10 and 10", - "restrictions": { "required": false, "range": { "max": 10, "min": -10 } }, - "isArray": true - }, - { - "name": "unique_value", - "valueType": "string", - "description": "unique value", - "restrictions": { "required": false, "unique": true }, - "isArray": true - } - ] - }, - { - "name": "parent_schema_1", - "description": "Parent schema 1. Used to test relational validations", - "fields": [ - { - "name": "id", - "valueType": "string", - "description": "Id" - }, - { - "name": "external_id", - "valueType": "string", - "description": "External Id" - }, - { - "name": "name", - "valueType": "string", - "description": "Name" - } - ] - }, - { - "name": "parent_schema_2", - "description": "Parent schema 1. Used to test relational validations", - "fields": [ - { - "name": "id1", - "valueType": "string", - "description": "Id 1", - "isArray": true - }, - { - "name": "id2", - "valueType": "string", - "description": "Id 2", - "isArray": true - } - ] - }, - { - "name": "child_schema_simple_fk", - "description": "Child schema referencing a field in a foreign schema", - "restrictions": { - "foreignKey": [ - { - "schema": "parent_schema_1", - "mappings": [ - { - "local": "parent_schema_1_id", - "foreign": "id" - } - ] - } - ] - }, - "fields": [ - { - "name": "id", - "valueType": "number", - "description": "Id" - }, - { - "name": "parent_schema_1_id", - "valueType": "string", - "description": "Reference to id in schema parent_schema_1" - } - ] - }, - { - "name": "child_schema_composite_fk", - "description": "Child schema referencing several fields in a foreign schema", - "restrictions": { - "foreignKey": [ - { - "schema": "parent_schema_1", - "mappings": [ - { - "local": "parent_schema_1_id", - "foreign": "id" - }, - { - "local": "parent_schema_1_external_id", - "foreign": "external_id" - } - ] - } - ] - }, - "fields": [ - { - "name": "id", - "valueType": "number", - "description": "Id" - }, - { - "name": "parent_schema_1_id", - "valueType": "string", - "description": "Reference to id in schema parent_schema_1" - }, - { - "name": "parent_schema_1_external_id", - "valueType": "string", - "description": "Reference to external id in schema parent_schema_1" - } - ] - }, - { - "name": "child_schema_composite_array_values_fk", - "description": "Child schema referencing several fields in a foreign schema", - "restrictions": { - "foreignKey": [ - { - "schema": "parent_schema_2", - "mappings": [ - { - "local": "parent_schema_2_id1", - "foreign": "id1" - }, - { - "local": "parent_schema_2_id12", - "foreign": "id2" - } - ] - } - ] - }, - "fields": [ - { - "name": "id", - "valueType": "number", - "description": "Id" - }, - { - "name": "parent_schema_2_id1", - "valueType": "string", - "description": "Reference to id1 in schema parent_schema_2", - "isArray": true - }, - { - "name": "parent_schema_2_id12", - "valueType": "string", - "description": "Reference to external id2 in schema parent_schema_2", - "isArray": true - } - ] - }, - { - "name": "unique_key_schema", - "description": "Schema to test uniqueKey restriction", - "restrictions": { - "uniqueKey": ["numeric_id_1", "string_id_2", "array_string_id_3"] - }, - "fields": [ - { - "name": "numeric_id_1", - "valueType": "number", - "description": "Id 1. Numeric value as part of a composite unique key" - }, - { - "name": "string_id_2", - "valueType": "string", - "description": "Id 2. String value as part of a composite unique key" - }, - { - "name": "array_string_id_3", - "valueType": "string", - "description": "Id 3. String array as part of a composite unique key", - "isArray": true - } - ] - } - ], - "_id": "5d250369f38d1f0d9376fd38", - "name": "ARGO Clinical Submission", - "version": "1.0", - "createdAt": "2019-07-09T21:13:13.683Z", - "updatedAt": "2019-07-09T21:13:13.683Z", - "__v": 0 - } -] diff --git a/packages/client/test/validation.spec.ts b/packages/client/test/validation.spec.ts new file mode 100644 index 00000000..74e6b294 --- /dev/null +++ b/packages/client/test/validation.spec.ts @@ -0,0 +1,828 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import chai from 'chai'; +import { functions as schemaService } from '../src'; +import { loggerFor } from '../src/logger'; +import { SchemaValidationErrorTypes } from '../src/validation'; +import { rangeToSymbol } from '../src/validation/utils/rangeToSymbol'; +import dictionary from './fixtures/registrationSchema'; +const L = loggerFor(__filename); + +chai.should(); + +const VALUE_NOT_ALLOWED = 'The value is not permissible for this field.'; +const PROGRAM_ID_REQ = 'program_id is a required field.'; + +describe('validation', () => { + it('should validate required', () => { + const result = schemaService.processRecords(dictionary, 'registration', [ + { + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + fieldName: 'program_id', + index: 0, + info: {}, + message: PROGRAM_ID_REQ, + }); + }); + + it('should validate value types', () => { + const result = schemaService.processRecords(dictionary, 'address', [ + { + country: 'US', + unit_number: 'abc', + postal_code: '12345', + }, + ]); + + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + fieldName: 'unit_number', + index: 0, + info: { value: ['abc'] }, + message: VALUE_NOT_ALLOWED, + }); + }); + + it('should convert string to integer after processing', () => { + const result = schemaService.processRecords(dictionary, 'address', [ + { + country: 'US', + unit_number: '123', + postal_code: '12345', + }, + ]); + chai.expect(result.processedRecords).to.deep.include({ + country: 'US', + unit_number: 123, + postal_code: '12345', + }); + }); + + it('should validate regex', () => { + const result = schemaService.processRecords(dictionary, 'registration', [ + { + program_id: 'PEME-CAA', + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + fieldName: 'program_id', + index: 0, + info: { + examples: 'PACA-CA, BASHAR-LA', + regex: '^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$', + value: ['PEME-CAA'], + }, + message: + 'The value is not a permissible for this field, it must meet the regular expression: "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$". Examples: PACA-CA, BASHAR-LA', + }); + }); + + it('should validate range', () => { + const result = schemaService.processRecords(dictionary, 'address', [ + { + country: 'US', + postal_code: '12345', + unit_number: '-1', + }, + { + country: 'US', + postal_code: '12345', + unit_number: '223', + }, + { + country: 'US', + postal_code: '12345', + unit_number: '500000', + }, + ]); + + const info1 = { + exclusiveMax: 999, + min: 0, + value: [-1], + }; + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + fieldName: 'unit_number', + index: 0, + info: info1, + message: `Value is out of permissible range, it must be ${rangeToSymbol(info1)}.`, + }); + + const info2 = { + exclusiveMax: 999, + min: 0, + value: [500000], + }; + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + fieldName: 'unit_number', + index: 2, + info: info2, + message: `Value is out of permissible range, it must be ${rangeToSymbol(info2)}.`, + }); + }); + + it('should validate script', () => { + const result = schemaService.processRecords(dictionary, 'address', [ + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + fieldName: 'postal_code', + index: 0, + info: { message: 'invalid postal code for US', value: '12' }, + message: 'invalid postal code for US', + }); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + fieldName: 'postal_code', + index: 1, + info: { message: 'invalid postal code for CANADA', value: 'ABC' }, + message: 'invalid postal code for CANADA', + }); + }); + + it('should validate if non-required feilds are not provided', () => { + const result = schemaService.processRecords(dictionary, 'donor', [ + // optional enum field not provided + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0004', + gender: 'Female', + ethnicity: 'black or african american', + vital_status: 'alive', + }, + // optional enum field provided with proper value + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Male', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '124', + }, + // optional enum field provided with no value + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Male', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: '', + survival_time: '124', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should error if integer fields are not valid', () => { + const result = schemaService.processRecords(dictionary, 'donor', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Other', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '0.5', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + fieldName: 'survival_time', + index: 0, + info: { value: ['0.5'] }, + message: VALUE_NOT_ALLOWED, + }); + }); + + it('should validate case insensitive enums, return proper format', () => { + const result = schemaService.processRecords(dictionary, 'registration', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'OD1234', + gender: 'feMale', + submitter_specimen_id: '87813', + specimen_type: 'sKiN', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'CTdna', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + chai.expect(result.processedRecords[0]).to.deep.eq({ + program_id: 'PACA-AU', + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }); + }); + + it('should not validate if unrecognized fields are provided', () => { + const result = schemaService.processRecords(dictionary, 'donor', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Other', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '5', + hackField: 'muchHack', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, + message: `hackField is not an allowed field for this schema.`, + fieldName: 'hackField', + index: 0, + info: {}, + }); + }); + + it('should validate number/integer array with field defined ranges', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + fraction: ['0.2', '2', '3'], + integers: ['-100', '-2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + message: 'Value is out of permissible range, it must be > 0 and <= 1.', + index: 0, + fieldName: 'fraction', + info: { value: [2, 3], max: 1, exclusiveMin: 0 }, + }); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + message: 'Value is out of permissible range, it must be >= -10 and <= 10.', + index: 0, + fieldName: 'integers', + info: { value: [-100], max: 10, min: -10 }, + }); + }); + + it('should validate string array with field defined codelist', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + fruit: ['Mango', '2'], + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + message: 'The value is not permissible for this field.', + fieldName: 'fruit', + index: 0, + info: { value: ['2'] }, + }); + }); + + it('should validate string with field defined codelist', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + fruit_single_value: 'Banana', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + message: 'The value is not permissible for this field.', + fieldName: 'fruit_single_value', + index: 0, + info: { value: ['Banana'] }, + }); + }); + + it('should validate string array with field defined regex', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + qWords: ['que', 'not_q'], + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', + fieldName: 'qWords', + index: 0, + info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, + }); + }); + + it('should validate string with field defined regex', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + qWord: 'not_q', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', + fieldName: 'qWord', + index: 0, + info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, + }); + }); + + it('should pass unique restriction validation when only null values exists', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + unique_value: '', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should pass unique restriction validation when only one record exists', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + unique_value: 'unique_value_1', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should fail unique restriction validation when duplicate values exist (scalar)', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'ID-1', + unique_value: 'unique_value_1', + }, + { + id: 'ID-2', + unique_value: 'unique_value_1', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Values for column "unique_value" must be unique.', + fieldName: 'unique_value', + index: 0, + info: { value: ['unique_value_1'] }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Values for column "unique_value" must be unique.', + fieldName: 'unique_value', + index: 1, + info: { value: ['unique_value_1'] }, + }); + }); + it('should fail unique restriction validation when duplicate values exist (array)', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'ID-1', + unique_value: ['unique_value_1', 'unique_value_2'], + }, + { + id: 'ID-2', + unique_value: ['unique_value_1', 'unique_value_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Values for column "unique_value" must be unique.', + fieldName: 'unique_value', + index: 0, + info: { value: ['unique_value_1', 'unique_value_2'] }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Values for column "unique_value" must be unique.', + fieldName: 'unique_value', + index: 1, + info: { value: ['unique_value_1', 'unique_value_2'] }, + }); + }); + + it('should pass foreignKey restriction validation when values exist in foreign schema', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); + }); + + it('should pass foreignKey restriction validation when local schema has null values', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: '', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); + }); + + it('should pass foreignKey restriction validation when values exist in foreign schema (composite fk)', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + external_id: 'parent_schema_1_external_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + external_id: 'parent_schema_1_external_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_composite_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + parent_schema_1_external_id: 'parent_schema_1_external_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + parent_schema_1_external_id: 'parent_schema_1_external_id_2', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_composite_fk: child_schema_composite_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_composite_fk'].validationErrors.length).to.eq(0); + }); + + it('should fail foreignKey restriction validation when value does not exist in foreign schema', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: 'non_existing_value_in_foreign_schema', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + const childSchemaErrors = result['child_schema_simple_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_1_id. Key parent_schema_1_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', + fieldName: 'parent_schema_1_id', + index: 1, + info: { foreignSchema: 'parent_schema_1', value: { parent_schema_1_id: 'non_existing_value_in_foreign_schema' } }, + }); + }); + + it('should fail foreignKey restriction validation when values do not exist in foreign schema (composite fk)', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + external_id: 'parent_schema_1_external_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + external_id: 'parent_schema_1_external_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_composite_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + parent_schema_1_external_id: 'parent_schema_1_external_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_composite_fk: child_schema_composite_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + const childSchemaErrors = result['child_schema_composite_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_1_id, parent_schema_1_external_id. Key parent_schema_1_id: parent_schema_1_id_2, parent_schema_1_external_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', + fieldName: 'parent_schema_1_id, parent_schema_1_external_id', + index: 1, + info: { + foreignSchema: 'parent_schema_1', + value: { + parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', + parent_schema_1_id: 'parent_schema_1_id_2', + }, + }, + }); + }); + + it('should fail foreignKey restriction validation when values (array) do not match in foreign schema (composite fk)', () => { + const parent_schema_2_data = [ + { + id1: ['id1_1', 'id1_2'], + id2: ['id2_1'], + }, + ]; + + const child_schema_composite_array_values_fk_data = [ + { + id: '1', + parent_schema_2_id1: ['id1_1'], + parent_schema_2_id12: ['id1_2', 'id2_1'], + }, + ]; + const schemaData = { + parent_schema_2: parent_schema_2_data, + child_schema_composite_array_values_fk: child_schema_composite_array_values_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + const childSchemaErrors = result['child_schema_composite_array_values_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_2_id1, parent_schema_2_id12. Key parent_schema_2_id1: [id1_1], parent_schema_2_id12: [id1_2, id2_1] is not present in schema parent_schema_2.', + fieldName: 'parent_schema_2_id1, parent_schema_2_id12', + index: 0, + info: { + foreignSchema: 'parent_schema_2', + value: { + parent_schema_2_id1: ['id1_1'], + parent_schema_2_id12: ['id1_2', 'id2_1'], + }, + }, + }); + }); + + it('should pass uniqueKey restriction validation when only a record exists', () => { + const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should pass uniqueKey restriction validation when values are unique', () => { + const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_x'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should fail uniqueKey restriction validation when missing values are part of the key and they are not unique', () => { + const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ + { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: [], + }, + { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: [], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'UniqueKey field values "numeric_id_1: null, string_id_2: null, array_string_id_3: null" must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 0, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: '', + }, + }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'UniqueKey field values "numeric_id_1: null, string_id_2: null, array_string_id_3: null" must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 1, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: '', + }, + }, + }); + }); + + it('should fail uniqueKey restriction validation when values are not unique', () => { + const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'UniqueKey field values "numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2]" must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 0, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: 1, + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'UniqueKey field values "numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2]" must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 1, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: 1, + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + }, + }); + }); +}); From faa4db11dc4600fc214a27a8884ce6ec9900509a Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Tue, 11 Jun 2024 11:50:38 -0400 Subject: [PATCH 15/17] updated lockfile for client package.json changes --- pnpm-lock.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b450b8ff..279f9a5e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -197,15 +197,15 @@ importers: deep-freeze: specifier: ^0.0.1 version: 0.0.1 + dictionary: + specifier: workspace:^ + version: link:../../libraries/dictionary lodash: specifier: ^4.17.21 version: 4.17.21 node-fetch: specifier: ^2.6.1 version: 2.7.0 - node-worker-threads-pool: - specifier: ^1.4.3 - version: 1.5.1 promise-tools: specifier: ^2.1.0 version: 2.1.0 From 69b17b3f943493134855db99a8ef63dfc61880e4 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 14 Jun 2024 18:34:13 -0400 Subject: [PATCH 16/17] Add standard copyright notice --- packages/client/src/changeAnalysis/index.ts | 19 +++++++++++++++++++ packages/client/src/types/index.ts | 19 +++++++++++++++++++ packages/client/src/validation/types/index.ts | 19 +++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/packages/client/src/changeAnalysis/index.ts b/packages/client/src/changeAnalysis/index.ts index 46a0feba..303c3678 100644 --- a/packages/client/src/changeAnalysis/index.ts +++ b/packages/client/src/changeAnalysis/index.ts @@ -1,2 +1,21 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + export * from './changeAnalysisTypes'; export * from './changeAnalyzer'; diff --git a/packages/client/src/types/index.ts b/packages/client/src/types/index.ts index 7eaa8a6b..f35338d5 100644 --- a/packages/client/src/types/index.ts +++ b/packages/client/src/types/index.ts @@ -1 +1,20 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + export * from './dataRecords'; diff --git a/packages/client/src/validation/types/index.ts b/packages/client/src/validation/types/index.ts index 7b536617..ba34998e 100644 --- a/packages/client/src/validation/types/index.ts +++ b/packages/client/src/validation/types/index.ts @@ -1,2 +1,21 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + export * from './validationErrorTypes'; export * from './validationFunctionTypes'; From 77e4885d4699301dc1043ed23922407af44a7635 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 14 Jun 2024 18:36:19 -0400 Subject: [PATCH 17/17] Remove uneccessary comments left from refactor --- packages/client/src/changeAnalysis/changeAnalysisTypes.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/client/src/changeAnalysis/changeAnalysisTypes.ts b/packages/client/src/changeAnalysis/changeAnalysisTypes.ts index 0b1b2733..853a918a 100644 --- a/packages/client/src/changeAnalysis/changeAnalysisTypes.ts +++ b/packages/client/src/changeAnalysis/changeAnalysisTypes.ts @@ -21,7 +21,6 @@ import { SchemaField, ValueChangeTypeName } from 'dictionary'; type ChangeOnlyTypeNames = Exclude; -/* ===== Change Analysis Types (Duplicate?) ===== */ export interface ChangeAnalysis { fields: { addedFields: AddedFieldChange[];