From a3e1448112e7beb709807228bf6b02e774ce5b62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 14 Jul 2023 14:43:05 +0200 Subject: [PATCH 1/7] feat(api-headless-cms-ddb-es): add structuredClone package --- packages/api-aco/package.json | 2 +- packages/api-headless-cms-ddb-es/package.json | 1 + yarn.lock | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/api-aco/package.json b/packages/api-aco/package.json index 5fededfa435..fc1d99e5697 100644 --- a/packages/api-aco/package.json +++ b/packages/api-aco/package.json @@ -22,7 +22,7 @@ "directory": "dist" }, "dependencies": { - "@ungap/structured-clone": "1.2.0", + "@ungap/structured-clone": "^1.2.0", "@webiny/api": "0.0.0", "@webiny/api-headless-cms": "0.0.0", "@webiny/api-i18n": "0.0.0", diff --git a/packages/api-headless-cms-ddb-es/package.json b/packages/api-headless-cms-ddb-es/package.json index cf5f405f5a9..2bc13f6bbda 100644 --- a/packages/api-headless-cms-ddb-es/package.json +++ b/packages/api-headless-cms-ddb-es/package.json @@ -43,6 +43,7 @@ "@babel/preset-env": "^7.22.7", "@elastic/elasticsearch": "7.12.0", "@types/jsonpack": "^1.1.0", + "@ungap/structured-clone": "^1.2.0", "@webiny/api-dynamodb-to-elasticsearch": "0.0.0", "@webiny/api-i18n": "0.0.0", "@webiny/api-security": "0.0.0", diff --git a/yarn.lock b/yarn.lock index 0620f361cd0..a65d17e340a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12365,7 +12365,7 @@ __metadata: languageName: node linkType: hard -"@ungap/structured-clone@npm:1.2.0": +"@ungap/structured-clone@npm:^1.2.0": version: 1.2.0 resolution: "@ungap/structured-clone@npm:1.2.0" checksum: 4f656b7b4672f2ce6e272f2427d8b0824ed11546a601d8d5412b9d7704e83db38a8d9f402ecdf2b9063fc164af842ad0ec4a55819f621ed7e7ea4d1efcc74524 @@ -12847,7 +12847,7 @@ __metadata: "@babel/preset-typescript": ^7.22.5 "@babel/runtime": ^7.22.6 "@types/ungap__structured-clone": ^0.3.0 - "@ungap/structured-clone": 1.2.0 + "@ungap/structured-clone": ^1.2.0 "@webiny/api": 0.0.0 "@webiny/api-headless-cms": 0.0.0 "@webiny/api-i18n": 0.0.0 @@ -13362,6 +13362,7 @@ __metadata: "@babel/runtime": ^7.22.6 "@elastic/elasticsearch": 7.12.0 "@types/jsonpack": ^1.1.0 + "@ungap/structured-clone": ^1.2.0 "@webiny/api": 0.0.0 "@webiny/api-dynamodb-to-elasticsearch": 0.0.0 "@webiny/api-elasticsearch": 0.0.0 From 6240e49bcb55734417db71139f4a1be1d4ba309e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 14 Jul 2023 14:45:17 +0200 Subject: [PATCH 2/7] feat(api-headless-cms-ddb-es): add entry values modifier plugin --- .../src/operations/entry/index.ts | 120 ++++++++++++++---- .../CmsEntryElasticsearchValuesModifier.ts | 55 ++++++++ .../src/plugins/index.ts | 1 + 3 files changed, 153 insertions(+), 23 deletions(-) create mode 100644 packages/api-headless-cms-ddb-es/src/plugins/CmsEntryElasticsearchValuesModifier.ts diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts index ed6a1f744fd..a5789132149 100644 --- a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts @@ -46,31 +46,68 @@ import { createLatestRecordType, createPublishedRecordType, createRecordType } f import { StorageOperationsCmsModelPlugin } from "@webiny/api-headless-cms"; import { DocumentClient } from "aws-sdk/clients/dynamodb"; import { batchReadAll, BatchReadItem } from "@webiny/db-dynamodb"; +import { CmsEntryElasticsearchValuesModifier } from "~/plugins"; +import structuredClone from "@ungap/structured-clone"; -const getEntryData = (input: CmsEntry): CmsEntry => { +interface GetElasticsearchEntryDataParams { + container: PluginsContainer; + entry: CmsEntry; + model: CmsModel; +} + +const modifyEntryValues = async (params: GetElasticsearchEntryDataParams) => { + const { entry, model, container } = params; + const plugins = container.byType( + CmsEntryElasticsearchValuesModifier.type + ); + + let values = structuredClone(entry.values); + + for (const plugin of plugins) { + values = plugin.exec({ + model, + entry, + values + }); + } + + entry.values = values; + + return entry; +}; + +const getEntryData = async (params: GetElasticsearchEntryDataParams): Promise => { const output: any = { - ...input + ...params.entry }; delete output["PK"]; delete output["SK"]; delete output["published"]; delete output["latest"]; - return output; + return modifyEntryValues({ + ...params, + entry: output + }); }; -const getESLatestEntryData = async (plugins: PluginsContainer, entry: CmsEntry) => { - return compress(plugins, { - ...getEntryData(entry), +const getESLatestEntryData = async (params: GetElasticsearchEntryDataParams) => { + const { container } = params; + + const output = await getEntryData(params); + return compress(container, { + ...output, latest: true, TYPE: createLatestRecordType(), __type: createLatestRecordType() }); }; -const getESPublishedEntryData = async (plugins: PluginsContainer, entry: CmsEntry) => { - return compress(plugins, { - ...getEntryData(entry), +const getESPublishedEntryData = async (params: GetElasticsearchEntryDataParams) => { + const { container } = params; + const output = await getEntryData(params); + return compress(container, { + ...output, published: true, TYPE: createPublishedRecordType(), __type: createPublishedRecordType() @@ -171,8 +208,16 @@ export const createEntriesStorageOperations = ( model }); - const esLatestData = await getESLatestEntryData(plugins, esEntry); - const esPublishedData = await getESPublishedEntryData(plugins, esEntry); + const esLatestData = await getESLatestEntryData({ + container: plugins, + model, + entry: esEntry + }); + const esPublishedData = await getESPublishedEntryData({ + container: plugins, + model, + entry: esEntry + }); const revisionKeys = { PK: createPartitionKey({ @@ -324,7 +369,11 @@ export const createEntriesStorageOperations = ( storageEntry: lodashCloneDeep(storageEntry) }); - const esLatestData = await getESLatestEntryData(plugins, esEntry); + const esLatestData = await getESLatestEntryData({ + container: plugins, + model, + entry: esEntry + }); const items = [ entity.putBatch({ @@ -500,7 +549,11 @@ export const createEntriesStorageOperations = ( }) }); - elasticsearchLatestData = await getESLatestEntryData(plugins, esEntry); + elasticsearchLatestData = await getESLatestEntryData({ + container: plugins, + model, + entry: esEntry + }); esItems.push( esEntity.putBatch({ @@ -530,7 +583,11 @@ export const createEntriesStorageOperations = ( }) }); } - elasticsearchPublishedData = await getESPublishedEntryData(plugins, esEntry); + elasticsearchPublishedData = await getESPublishedEntryData({ + container: plugins, + model, + entry: esEntry + }); } else { elasticsearchPublishedData = { ...elasticsearchLatestData, @@ -867,7 +924,11 @@ export const createEntriesStorageOperations = ( storageEntry: lodashCloneDeep(latestStorageEntry) }); - const esLatestData = await getESLatestEntryData(plugins, esEntry); + const esLatestData = await getESLatestEntryData({ + container: plugins, + model, + entry: esEntry + }); /** * In the end we need to set the new latest entry */ @@ -1272,17 +1333,22 @@ export const createEntriesStorageOperations = ( latestEsEntry.data )) as any; + const entryData = { + ...latestEsEntryDataDecompressed, + status: CONTENT_ENTRY_STATUS.PUBLISHED, + locked: true, + savedOn: entry.savedOn, + publishedOn: entry.publishedOn + }; esItems.push( esEntity.putBatch({ index, PK: createPartitionKey(latestEsEntryDataDecompressed), SK: createLatestSortKey(), - data: await getESLatestEntryData(plugins, { - ...latestEsEntryDataDecompressed, - status: CONTENT_ENTRY_STATUS.PUBLISHED, - locked: true, - savedOn: entry.savedOn, - publishedOn: entry.publishedOn + data: await getESLatestEntryData({ + container: plugins, + model, + entry: entryData }) }) ); @@ -1297,7 +1363,11 @@ export const createEntriesStorageOperations = ( /** * Update the published revision entry in ES. */ - const esPublishedData = await getESPublishedEntryData(plugins, preparedEntryData); + const esPublishedData = await getESPublishedEntryData({ + container: plugins, + model, + entry: preparedEntryData + }); esItems.push( esEntity.putBatch({ @@ -1415,7 +1485,11 @@ export const createEntriesStorageOperations = ( storageEntry: lodashCloneDeep(storageEntry) }); - const esLatestData = await getESLatestEntryData(plugins, preparedEntryData); + const esLatestData = await getESLatestEntryData({ + container: plugins, + model, + entry: preparedEntryData + }); esItems.push( esEntity.putBatch({ PK: partitionKey, diff --git a/packages/api-headless-cms-ddb-es/src/plugins/CmsEntryElasticsearchValuesModifier.ts b/packages/api-headless-cms-ddb-es/src/plugins/CmsEntryElasticsearchValuesModifier.ts new file mode 100644 index 00000000000..bc8acc04f63 --- /dev/null +++ b/packages/api-headless-cms-ddb-es/src/plugins/CmsEntryElasticsearchValuesModifier.ts @@ -0,0 +1,55 @@ +import { Plugin } from "@webiny/plugins"; +import { CmsEntry, CmsEntryValues, CmsModel } from "@webiny/api-headless-cms/types"; + +interface CmsEntryElasticsearchValuesModifierCbParamsSetValuesCb { + (prev: Partial): Partial; +} + +interface CmsEntryElasticsearchValuesModifierCbParams { + model: CmsModel; + entry: CmsEntry; + values: T; + setValues: (cb: CmsEntryElasticsearchValuesModifierCbParamsSetValuesCb) => void; +} + +export interface CmsEntryElasticsearchValuesModifierCb { + (params: CmsEntryElasticsearchValuesModifierCbParams): Promise; +} + +export interface CmsEntryElasticsearchValuesModifierExecParams { + model: CmsModel; + entry: CmsEntry; + values: Partial; +} + +export class CmsEntryElasticsearchValuesModifier extends Plugin { + public static override readonly type: string = "cms.entry.elasticsearch.values.modifier"; + + private readonly cb: CmsEntryElasticsearchValuesModifierCb; + + public constructor(cb: CmsEntryElasticsearchValuesModifierCb) { + super(); + this.cb = cb; + } + + public async exec( + params: CmsEntryElasticsearchValuesModifierExecParams + ): Promise> { + let values: any = params.values; + await this.cb({ + model: params.model, + entry: params.entry, + values, + setValues: (cb: CmsEntryElasticsearchValuesModifierCbParamsSetValuesCb) => { + values = cb(values); + } + }); + return values; + } +} + +export const createCmsEntryElasticsearchValuesModifier = ( + cb: CmsEntryElasticsearchValuesModifierCb +) => { + return new CmsEntryElasticsearchValuesModifier(cb); +}; diff --git a/packages/api-headless-cms-ddb-es/src/plugins/index.ts b/packages/api-headless-cms-ddb-es/src/plugins/index.ts index 43e394ef8e7..489b272a233 100644 --- a/packages/api-headless-cms-ddb-es/src/plugins/index.ts +++ b/packages/api-headless-cms-ddb-es/src/plugins/index.ts @@ -5,3 +5,4 @@ export * from "./CmsEntryElasticsearchQueryModifierPlugin"; export * from "./CmsEntryElasticsearchSortModifierPlugin"; export * from "./CmsEntryElasticsearchFullTextSearchPlugin"; export * from "./CmsElasticsearchModelFieldPlugin"; +export * from "./CmsEntryElasticsearchValuesModifier"; From 9835987e36807be190047dc203f136300eeeadc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 14 Jul 2023 14:45:49 +0200 Subject: [PATCH 3/7] test(api-headless-cms-ddb-es): entry values modifier plugin --- .../__tests__/plugins/valuesModifier.test.ts | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 packages/api-headless-cms-ddb-es/__tests__/plugins/valuesModifier.test.ts diff --git a/packages/api-headless-cms-ddb-es/__tests__/plugins/valuesModifier.test.ts b/packages/api-headless-cms-ddb-es/__tests__/plugins/valuesModifier.test.ts new file mode 100644 index 00000000000..4e761793b3b --- /dev/null +++ b/packages/api-headless-cms-ddb-es/__tests__/plugins/valuesModifier.test.ts @@ -0,0 +1,137 @@ +import structuredClone from "@ungap/structured-clone"; +import { createCmsEntryElasticsearchValuesModifier } from "~/plugins"; +import { CmsEntry, CmsIdentity, CmsModel } from "@webiny/api-headless-cms/types"; + +interface MockCmsEntryValues { + title: string; + age: number; +} + +const mockModel: CmsModel = { + locale: "en-US", + tenant: "root", + modelId: "abcdefghijklmn", + name: "Test model", + description: "", + layout: [], + fields: [], + singularApiName: "TestModel", + pluralApiName: "TestModels", + titleFieldId: "id", + group: { + id: "group", + name: "group" + }, + webinyVersion: "0.0.0" +}; +const createdBy: CmsIdentity = { + id: "a", + displayName: "a", + type: "a" +}; + +const mockEntry: CmsEntry = { + id: "abcdefg#0001", + entryId: "abcdefg", + location: { + folderId: "root" + }, + values: { + title: "Initial title", + age: 55 + }, + status: "draft", + locked: false, + locale: "en-US", + tenant: "root", + webinyVersion: "0.0.0", + createdBy, + ownedBy: createdBy, + meta: {}, + createdOn: new Date().toISOString(), + savedOn: new Date().toISOString(), + modelId: mockModel.modelId, + version: 1 +}; + +const getMockData = () => { + return { + model: structuredClone(mockModel), + entry: structuredClone>(mockEntry), + values: structuredClone(mockEntry.values) + }; +}; + +describe("entry values modifier", () => { + it("should modify the original values with a single plugin", async () => { + const { model, values: initialValues, entry } = getMockData(); + let values = structuredClone(initialValues); + + const modifier = createCmsEntryElasticsearchValuesModifier( + async ({ setValues }) => { + setValues(() => { + return { + title: "Test title" + }; + }); + } + ); + + values = await modifier.exec({ + entry, + model, + values + }); + + expect(values).toEqual({ + title: "Test title" + }); + expect(values.age).toBeUndefined(); + }); + + it("should modify the original values with multiple plugins", async () => { + const { model, values: initialValues, entry } = getMockData(); + let values = structuredClone(initialValues); + const titleModifier = createCmsEntryElasticsearchValuesModifier( + async ({ setValues }) => { + setValues(() => { + return { + title: "Test title" + }; + }); + } + ); + + values = await titleModifier.exec({ + entry, + model, + values + }); + + expect(values).toEqual({ + title: "Test title" + }); + expect(values.age).toBeUndefined(); + + const ageModifier = createCmsEntryElasticsearchValuesModifier( + async ({ setValues }) => { + setValues(prev => { + return { + ...prev, + age: 2 + }; + }); + } + ); + + values = await ageModifier.exec({ + entry, + model, + values + }); + expect(values).toEqual({ + title: "Test title", + age: 2 + }); + }); +}); From 26cace7a67068026c996c78153c444f7ab23297d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Thu, 27 Jul 2023 12:55:19 +0200 Subject: [PATCH 4/7] fix(api-headless-cms): permissions checks (#3449) --- packages/api-headless-cms/src/context.ts | 6 + .../src/crud/contentEntry.crud.ts | 121 +++++++++++------- .../src/crud/contentModel.crud.ts | 29 +++-- .../src/crud/contentModelGroup.crud.ts | 21 +-- packages/api-headless-cms/src/index.ts | 1 + packages/api-headless-cms/src/types.ts | 17 +++ packages/api-headless-cms/src/utils/access.ts | 40 ++++++ .../permissions/ModelGroupsPermissions.ts | 15 ++- .../utils/permissions/ModelsPermissions.ts | 27 ++-- 9 files changed, 194 insertions(+), 83 deletions(-) create mode 100644 packages/api-headless-cms/src/utils/access.ts diff --git a/packages/api-headless-cms/src/context.ts b/packages/api-headless-cms/src/context.ts index 85491d00ba6..caff1265391 100644 --- a/packages/api-headless-cms/src/context.ts +++ b/packages/api-headless-cms/src/context.ts @@ -95,6 +95,12 @@ export const createContextPlugin = ({ storageOperations }: CrudParams) => { PREVIEW: type === "preview", MANAGE: type === "manage", storageOperations, + permissions: { + groups: modelGroupsPermissions, + models: modelsPermissions, + entries: entriesPermissions, + settings: settingsPermissions + }, ...createSystemCrud({ context, getTenant, diff --git a/packages/api-headless-cms/src/crud/contentEntry.crud.ts b/packages/api-headless-cms/src/crud/contentEntry.crud.ts index 011217e1959..a5cea47468f 100644 --- a/packages/api-headless-cms/src/crud/contentEntry.crud.ts +++ b/packages/api-headless-cms/src/crud/contentEntry.crud.ts @@ -412,37 +412,35 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm * A helper to delete the entire entry. */ const deleteEntryHelper = async (params: DeleteEntryParams): Promise => { - return context.benchmark.measure("headlessCms.crud.entries.deleteEntry", async () => { - const { model, entry } = params; - try { - await onEntryBeforeDelete.publish({ - entry, - model - }); + const { model, entry } = params; + try { + await onEntryBeforeDelete.publish({ + entry, + model + }); - await storageOperations.entries.delete(model, { - entry - }); + await storageOperations.entries.delete(model, { + entry + }); - await onEntryAfterDelete.publish({ - entry, - model - }); - } catch (ex) { - await onEntryDeleteError.publish({ - entry, - model, - error: ex - }); - throw new WebinyError( - ex.message || "Could not delete entry.", - ex.code || "DELETE_ERROR", - { - entry - } - ); - } - }); + await onEntryAfterDelete.publish({ + entry, + model + }); + } catch (ex) { + await onEntryDeleteError.publish({ + entry, + model, + error: ex + }); + throw new WebinyError( + ex.message || "Could not delete entry.", + ex.code || "DELETE_ERROR", + { + entry + } + ); + } }; /** * A helper to get entries by revision IDs @@ -450,14 +448,23 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm const getEntriesByIds: CmsEntryContext["getEntriesByIds"] = async (model, ids) => { return context.benchmark.measure("headlessCms.crud.entries.getEntriesByIds", async () => { await entriesPermissions.ensure({ rwd: "r" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); const entries = await storageOperations.entries.getByIds(model, { ids }); return filterAsync(entries, async entry => { - return entriesPermissions.ensure({ owns: entry.createdBy }, { throw: false }); + return entriesPermissions.ensure( + { + owns: entry.createdBy + }, + { + throw: false + } + ); }); }); }; @@ -480,7 +487,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm ids ) => { await entriesPermissions.ensure({ rwd: "r" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); const entries = await storageOperations.entries.getPublishedByIds(model, { ids @@ -492,7 +501,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm }; const getLatestEntriesByIds: CmsEntryContext["getLatestEntriesByIds"] = async (model, ids) => { await entriesPermissions.ensure({ rwd: "r" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); const entries = await storageOperations.entries.getLatestByIds(model, { ids @@ -544,7 +555,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm } }); } - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); const { where: initialWhere, limit: initialLimit } = params; const limit = initialLimit && initialLimit > 0 ? initialLimit : 50; @@ -633,7 +646,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm }; const createEntry: CmsEntryContext["createEntry"] = async (model, inputData) => { await entriesPermissions.ensure({ rwd: "w" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); /** * Make sure we only work with fields that are defined in the model. @@ -730,7 +745,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm inputData ) => { await entriesPermissions.ensure({ rwd: "w" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); /** * Make sure we only work with fields that are defined in the model. @@ -855,7 +872,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm }; const updateEntry: CmsEntryContext["updateEntry"] = async (model, id, inputData, metaInput) => { await entriesPermissions.ensure({ rwd: "w" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); /** * Make sure we only work with fields that are defined in the model. @@ -978,7 +997,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm const moveEntry: CmsEntryContext["moveEntry"] = async (model, id, folderId) => { await entriesPermissions.ensure({ rwd: "w" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); /** * The entry we are going to move to another folder. */ @@ -1021,7 +1042,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm const republishEntry: CmsEntryContext["republishEntry"] = async (model, id) => { await entriesPermissions.ensure({ rwd: "w" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); /** * Fetch the entry from the storage. @@ -1116,7 +1139,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm revisionId ) => { await entriesPermissions.ensure({ rwd: "d" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); const { id: entryId, version } = parseIdentifier(revisionId); @@ -1224,7 +1249,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm } await entriesPermissions.ensure({ rwd: "d" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); const { items: entries } = await storageOperations.entries.list(model, { where: { @@ -1277,7 +1304,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm const deleteEntry: CmsEntryContext["deleteEntry"] = async (model, id, options) => { await entriesPermissions.ensure({ rwd: "d" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); const { force } = options || {}; @@ -1319,7 +1348,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm }; const publishEntry: CmsEntryContext["publishEntry"] = async (model, id) => { await entriesPermissions.ensure({ pw: "p" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); const originalStorageEntry = await storageOperations.entries.getRevisionById(model, { id @@ -1455,7 +1486,9 @@ export const createContentEntryCrud = (params: CreateContentEntryCrudParams): Cm const getUniqueFieldValues: CmsEntryContext["getUniqueFieldValues"] = async (model, params) => { await entriesPermissions.ensure({ rwd: "r" }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); const { where: initialWhere, fieldId } = params; diff --git a/packages/api-headless-cms/src/crud/contentModel.crud.ts b/packages/api-headless-cms/src/crud/contentModel.crud.ts index f059c0f39c0..1db85bcdbf9 100644 --- a/packages/api-headless-cms/src/crud/contentModel.crud.ts +++ b/packages/api-headless-cms/src/crud/contentModel.crud.ts @@ -4,23 +4,23 @@ import { CmsContext, CmsModel, CmsModelContext, + CmsModelGroup, CmsModelManager, + CmsModelUpdateInput, HeadlessCmsStorageOperations, - OnModelBeforeCreateTopicParams, + OnModelAfterCreateFromTopicParams, OnModelAfterCreateTopicParams, - OnModelBeforeUpdateTopicParams, - OnModelAfterUpdateTopicParams, - OnModelBeforeDeleteTopicParams, OnModelAfterDeleteTopicParams, - OnModelInitializeParams, + OnModelAfterUpdateTopicParams, OnModelBeforeCreateFromTopicParams, - OnModelAfterCreateFromTopicParams, - CmsModelUpdateInput, + OnModelBeforeCreateTopicParams, + OnModelBeforeDeleteTopicParams, + OnModelBeforeUpdateTopicParams, OnModelCreateErrorTopicParams, OnModelCreateFromErrorParams, - OnModelUpdateErrorTopicParams, OnModelDeleteErrorTopicParams, - CmsModelGroup + OnModelInitializeParams, + OnModelUpdateErrorTopicParams } from "~/types"; import { NotFoundError } from "@webiny/handler-graphql"; import { contentModelManagerFactory } from "./contentModel/contentModelManagerFactory"; @@ -42,9 +42,8 @@ import { createModelCreateValidation, createModelUpdateValidation } from "~/crud/contentModel/validation"; -import { createZodError } from "@webiny/utils"; +import { createZodError, removeUndefinedValues } from "@webiny/utils"; import { assignModelDefaultFields } from "~/crud/contentModel/defaultFields"; -import { removeUndefinedValues } from "@webiny/utils"; import { ensurePluralApiName, ensureSingularApiName @@ -203,7 +202,9 @@ export const createModelsCrud = (params: CreateModelsCrudParams): CmsModelContex return false; } - return modelsPermissions.canAccessModel({ model, locale: getLocale().code }); + return modelsPermissions.canAccessModel({ + model + }); }); }); }; @@ -215,7 +216,9 @@ export const createModelsCrud = (params: CreateModelsCrudParams): CmsModelContex const model = await modelsGet(modelId); await modelsPermissions.ensure({ owns: model.createdBy }); - await modelsPermissions.ensureCanAccessModel({ model, locale: getLocale().code }); + await modelsPermissions.ensureCanAccessModel({ + model + }); return model; }); diff --git a/packages/api-headless-cms/src/crud/contentModelGroup.crud.ts b/packages/api-headless-cms/src/crud/contentModelGroup.crud.ts index b675aeb8251..aae7868d9bb 100644 --- a/packages/api-headless-cms/src/crud/contentModelGroup.crud.ts +++ b/packages/api-headless-cms/src/crud/contentModelGroup.crud.ts @@ -1,20 +1,20 @@ import DataLoader from "dataloader"; import WebinyError from "@webiny/error"; import { + CmsContext, + CmsGroup, CmsGroupContext, CmsGroupListParams, - CmsGroup, - CmsContext, HeadlessCmsStorageOperations, - OnGroupBeforeCreateTopicParams, OnGroupAfterCreateTopicParams, - OnGroupBeforeUpdateTopicParams, + OnGroupAfterDeleteTopicParams, OnGroupAfterUpdateTopicParams, + OnGroupBeforeCreateTopicParams, OnGroupBeforeDeleteTopicParams, - OnGroupAfterDeleteTopicParams, + OnGroupBeforeUpdateTopicParams, OnGroupCreateErrorTopicParams, - OnGroupUpdateErrorTopicParams, - OnGroupDeleteErrorTopicParams + OnGroupDeleteErrorTopicParams, + OnGroupUpdateErrorTopicParams } from "~/types"; import { NotFoundError } from "@webiny/handler-graphql"; import { CmsGroupPlugin } from "~/plugins/CmsGroupPlugin"; @@ -185,7 +185,9 @@ export const createModelGroupsCrud = (params: CreateModelGroupsCrudParams): CmsG const group = await getGroupViaDataLoader(id); await modelGroupsPermissions.ensure({ owns: group.createdBy }); - await modelGroupsPermissions.ensureCanAccessGroup({ group, locale: getLocale().code }); + await modelGroupsPermissions.ensureCanAccessGroup({ + group + }); return group; }; @@ -216,8 +218,7 @@ export const createModelGroupsCrud = (params: CreateModelGroupsCrudParams): CmsG } return await modelGroupsPermissions.canAccessGroup({ - group, - locale: getLocale().code + group }); }); }; diff --git a/packages/api-headless-cms/src/index.ts b/packages/api-headless-cms/src/index.ts index 741719d7bc5..b4a09da0209 100644 --- a/packages/api-headless-cms/src/index.ts +++ b/packages/api-headless-cms/src/index.ts @@ -53,5 +53,6 @@ export const createHeadlessCmsContext = (params: ContentContextParams) => { export * from "~/graphqlFields"; export * from "~/plugins"; export * from "~/utils/incrementEntryIdVersion"; +export * from "~/utils/access"; export * from "./graphql/handleRequest"; export { entryToStorageTransform, entryFieldFromStorageTransform, entryFromStorageTransform }; diff --git a/packages/api-headless-cms/src/types.ts b/packages/api-headless-cms/src/types.ts index 096e1bd6628..8b41896d619 100644 --- a/packages/api-headless-cms/src/types.ts +++ b/packages/api-headless-cms/src/types.ts @@ -10,9 +10,20 @@ import { SecurityPermission } from "@webiny/api-security/types"; import { DbContext } from "@webiny/handler-db/types"; import { Topic } from "@webiny/pubsub/types"; import { CmsModelConverterCallable } from "~/utils/converters/ConverterCollection"; +import { ModelGroupsPermissions } from "~/utils/permissions/ModelGroupsPermissions"; +import { ModelsPermissions } from "~/utils/permissions/ModelsPermissions"; +import { EntriesPermissions } from "~/utils/permissions/EntriesPermissions"; +import { SettingsPermissions } from "~/utils/permissions/SettingsPermissions"; export type ApiEndpoint = "manage" | "preview" | "read"; +interface HeadlessCmsPermissions { + groups: ModelGroupsPermissions; + models: ModelsPermissions; + entries: EntriesPermissions; + settings: SettingsPermissions; +} + export interface HeadlessCms extends CmsSettingsContext, CmsSystemContext, @@ -47,6 +58,12 @@ export interface HeadlessCms * The storage operations loaded for current context. */ storageOperations: HeadlessCmsStorageOperations; + /** + * Permissions for settings, groups, models and entries. + * + * @internal + */ + permissions: HeadlessCmsPermissions; } /** diff --git a/packages/api-headless-cms/src/utils/access.ts b/packages/api-headless-cms/src/utils/access.ts new file mode 100644 index 00000000000..4ea5476f781 --- /dev/null +++ b/packages/api-headless-cms/src/utils/access.ts @@ -0,0 +1,40 @@ +import { CmsContext, CmsGroup, CmsModel } from "~/types"; + +interface PickedCmsContext { + cms: Pick; +} + +type PickedCmsGroup = Pick; +type PickedCmsModel = Pick; + +export const validateGroupAccess = async ( + context: PickedCmsContext, + group: PickedCmsGroup +): Promise => { + const { groups } = context.cms.permissions; + + return groups.canAccessGroup({ + group + }); +}; + +export const validateModelAccess = async ( + context: PickedCmsContext, + model: PickedCmsModel +): Promise => { + const { models } = context.cms.permissions; + return models.canAccessModel({ + model + }); +}; + +export const checkModelAccess = async ( + context: PickedCmsContext, + model: PickedCmsModel +): Promise => { + const { models } = context.cms.permissions; + + await models.ensureCanAccessModel({ + model + }); +}; diff --git a/packages/api-headless-cms/src/utils/permissions/ModelGroupsPermissions.ts b/packages/api-headless-cms/src/utils/permissions/ModelGroupsPermissions.ts index 56fae198c3c..8eb5c665b26 100644 --- a/packages/api-headless-cms/src/utils/permissions/ModelGroupsPermissions.ts +++ b/packages/api-headless-cms/src/utils/permissions/ModelGroupsPermissions.ts @@ -1,22 +1,23 @@ import { AppPermissions, NotAuthorizedError } from "@webiny/api-security"; import { CmsGroup, CmsGroupPermission } from "~/types"; -interface CanAccessGroupParams { - group: CmsGroup; - locale: string; +export interface CanAccessGroupParams { + group: Pick; } export class ModelGroupsPermissions extends AppPermissions { - async canAccessGroup({ group, locale }: CanAccessGroupParams) { + async canAccessGroup({ group }: CanAccessGroupParams) { if (await this.hasFullAccess()) { return true; } const permissions = await this.getPermissions(); - for (let i = 0; i < permissions.length; i++) { - const permission = permissions[i]; + const locale = group.locale; + + for (const permission of permissions) { const { groups } = permission; + // When no groups defined on permission it means user has access to everything. if (!groups) { return true; @@ -28,7 +29,7 @@ export class ModelGroupsPermissions extends AppPermissions { Array.isArray(groups[locale]) === false || groups[locale].includes(group.id) === false ) { - return false; + continue; } return true; } diff --git a/packages/api-headless-cms/src/utils/permissions/ModelsPermissions.ts b/packages/api-headless-cms/src/utils/permissions/ModelsPermissions.ts index 28eb62a873e..53a06368784 100644 --- a/packages/api-headless-cms/src/utils/permissions/ModelsPermissions.ts +++ b/packages/api-headless-cms/src/utils/permissions/ModelsPermissions.ts @@ -1,30 +1,37 @@ import { AppPermissions, AppPermissionsParams, NotAuthorizedError } from "@webiny/api-security"; -import { CmsGroupPermission, CmsModel, CmsModelPermission } from "~/types"; +import { + CmsGroupPermission, + CmsModel as BaseCmsModel, + CmsModelGroup as BaseCmsModelGroup, + CmsModelPermission +} from "~/types"; import { ModelGroupsPermissions } from "~/utils/permissions/ModelGroupsPermissions"; export interface ModelsPermissionsParams extends AppPermissionsParams { modelGroupsPermissions: ModelGroupsPermissions; } +interface PickedCmsModel extends Pick { + group: Pick; +} + export interface CanAccessModelParams { - model: CmsModel; - locale: string; + model: PickedCmsModel; } export interface EnsureModelAccessParams { - model: CmsModel; - locale: string; + model: PickedCmsModel; } export class ModelsPermissions extends AppPermissions { - private modelGroupsPermissions: ModelGroupsPermissions; + private readonly modelGroupsPermissions: ModelGroupsPermissions; - constructor(params: ModelsPermissionsParams) { + public constructor(params: ModelsPermissionsParams) { super(params); this.modelGroupsPermissions = params.modelGroupsPermissions; } - async canAccessModel({ model, locale }: CanAccessModelParams) { + public async canAccessModel({ model }: CanAccessModelParams) { if (await this.hasFullAccess()) { return true; } @@ -47,6 +54,8 @@ export class ModelsPermissions extends AppPermissions { const modelGroupsPermissionsList = await modelGroupsPermissions.getPermissions(); const modelsPermissionsList = await this.getPermissions(); + const locale = model.locale; + for (let i = 0; i < modelGroupsPermissionsList.length; i++) { const modelGroupPermission = modelGroupsPermissionsList[i]; @@ -89,7 +98,7 @@ export class ModelsPermissions extends AppPermissions { return false; } - async ensureCanAccessModel(params: EnsureModelAccessParams) { + public async ensureCanAccessModel(params: EnsureModelAccessParams) { const canAccessModel = await this.canAccessModel(params); if (canAccessModel) { return; From c8c22a4b1bce7e93d2c2a26bc4ae0507853196b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 31 Jul 2023 10:24:38 +0200 Subject: [PATCH 5/7] refactor(api-headless-cms-ddb-es): extract all transformers into a single factory --- jest.config.base.setup.js | 5 +- packages/api-elasticsearch/src/compression.ts | 4 +- ...sEntryElasticsearchValuesModifier.test.ts} | 24 +- .../src/operations/entry/index.ts | 329 ++++-------------- .../entry/transformations/convertEntryKeys.ts | 31 ++ .../operations/entry/transformations/index.ts | 178 ++++++++++ .../transformations/modifyEntryValues.ts | 24 ++ .../transformations/transformEntryKeys.ts | 26 ++ .../transformations/transformEntryToIndex.ts | 34 ++ .../CmsEntryElasticsearchValuesModifier.ts | 43 ++- 10 files changed, 416 insertions(+), 282 deletions(-) rename packages/api-headless-cms-ddb-es/__tests__/plugins/{valuesModifier.test.ts => CmsEntryElasticsearchValuesModifier.test.ts} (81%) create mode 100644 packages/api-headless-cms-ddb-es/src/operations/entry/transformations/convertEntryKeys.ts create mode 100644 packages/api-headless-cms-ddb-es/src/operations/entry/transformations/index.ts create mode 100644 packages/api-headless-cms-ddb-es/src/operations/entry/transformations/modifyEntryValues.ts create mode 100644 packages/api-headless-cms-ddb-es/src/operations/entry/transformations/transformEntryKeys.ts create mode 100644 packages/api-headless-cms-ddb-es/src/operations/entry/transformations/transformEntryToIndex.ts diff --git a/jest.config.base.setup.js b/jest.config.base.setup.js index 99b7b31bc02..7baf05bc251 100644 --- a/jest.config.base.setup.js +++ b/jest.config.base.setup.js @@ -3,4 +3,7 @@ // Runs failed tests five times until they pass or until the max number of retries is exhausted. // https://jestjs.io/docs/jest-object#jestretrytimesnumretries-options -jest.retryTimes(3); +jest.retryTimes(0); +if (process.env.CI === "true") { + jest.retryTimes(3); +} diff --git a/packages/api-elasticsearch/src/compression.ts b/packages/api-elasticsearch/src/compression.ts index 8da6a2766cd..f08879d3aa2 100644 --- a/packages/api-elasticsearch/src/compression.ts +++ b/packages/api-elasticsearch/src/compression.ts @@ -13,7 +13,7 @@ const getCompressionPlugins = (plugins: PluginsContainer): CompressionPlugin[] = export const compress = async ( pluginsContainer: PluginsContainer, data: Record -): Promise> => { +): Promise | string> => { const plugins = getCompressionPlugins(pluginsContainer); if (plugins.length === 0) { console.log("No compression plugins"); @@ -34,7 +34,7 @@ export const compress = async ( export const decompress = async ( pluginsContainer: PluginsContainer, data: Record -): Promise> => { +): Promise | string> => { const plugins = getCompressionPlugins(pluginsContainer); if (plugins.length === 0) { return data; diff --git a/packages/api-headless-cms-ddb-es/__tests__/plugins/valuesModifier.test.ts b/packages/api-headless-cms-ddb-es/__tests__/plugins/CmsEntryElasticsearchValuesModifier.test.ts similarity index 81% rename from packages/api-headless-cms-ddb-es/__tests__/plugins/valuesModifier.test.ts rename to packages/api-headless-cms-ddb-es/__tests__/plugins/CmsEntryElasticsearchValuesModifier.test.ts index 4e761793b3b..d4cb90417e0 100644 --- a/packages/api-headless-cms-ddb-es/__tests__/plugins/valuesModifier.test.ts +++ b/packages/api-headless-cms-ddb-es/__tests__/plugins/CmsEntryElasticsearchValuesModifier.test.ts @@ -77,7 +77,7 @@ describe("entry values modifier", () => { } ); - values = await modifier.exec({ + values = await modifier.modify({ entry, model, values @@ -102,7 +102,7 @@ describe("entry values modifier", () => { } ); - values = await titleModifier.exec({ + values = await titleModifier.modify({ entry, model, values @@ -124,7 +124,7 @@ describe("entry values modifier", () => { } ); - values = await ageModifier.exec({ + values = await ageModifier.modify({ entry, model, values @@ -134,4 +134,22 @@ describe("entry values modifier", () => { age: 2 }); }); + + it("should not modify anything because model is not supported", async () => { + const { model } = getMockData(); + const nothingWillGetModified = + createCmsEntryElasticsearchValuesModifier({ + models: ["nonExisting"], + modifier: async ({ setValues }) => { + setValues(() => { + return { + title: "Test title" + }; + }); + } + }); + + const result = nothingWillGetModified.canModify(model.modelId); + expect(result).toEqual(false); + }); }); diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts index a5789132149..63e4e5bee51 100644 --- a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts @@ -1,25 +1,23 @@ -import lodashCloneDeep from "lodash/cloneDeep"; import WebinyError from "@webiny/error"; import { CmsEntry, CmsModel, - CmsStorageEntry, CONTENT_ENTRY_STATUS, StorageOperationsCmsModel } from "@webiny/api-headless-cms/types"; -import { extractEntriesFromIndex, prepareEntryToIndex } from "~/helpers"; +import { extractEntriesFromIndex } from "~/helpers"; import { configurations } from "~/configurations"; import { Entity } from "dynamodb-toolbox"; import { Client } from "@elastic/elasticsearch"; import { PluginsContainer } from "@webiny/plugins"; import { batchWriteAll, BatchWriteItem } from "@webiny/db-dynamodb/utils/batchWrite"; -import { DataLoadersHandler } from "~/operations/entry/dataLoaders"; +import { DataLoadersHandler } from "./dataLoaders"; import { createLatestSortKey, createPartitionKey, createPublishedSortKey, createRevisionSortKey -} from "~/operations/entry/keys"; +} from "./keys"; import { queryAll, QueryAllParams, @@ -41,108 +39,13 @@ import { SearchBody as ElasticsearchSearchBody } from "@webiny/api-elasticsearch/types"; import { CmsEntryStorageOperations, CmsIndexEntry } from "~/types"; -import { createElasticsearchBody } from "~/operations/entry/elasticsearch/body"; +import { createElasticsearchBody } from "./elasticsearch/body"; import { createLatestRecordType, createPublishedRecordType, createRecordType } from "./recordType"; import { StorageOperationsCmsModelPlugin } from "@webiny/api-headless-cms"; import { DocumentClient } from "aws-sdk/clients/dynamodb"; import { batchReadAll, BatchReadItem } from "@webiny/db-dynamodb"; -import { CmsEntryElasticsearchValuesModifier } from "~/plugins"; -import structuredClone from "@ungap/structured-clone"; - -interface GetElasticsearchEntryDataParams { - container: PluginsContainer; - entry: CmsEntry; - model: CmsModel; -} - -const modifyEntryValues = async (params: GetElasticsearchEntryDataParams) => { - const { entry, model, container } = params; - const plugins = container.byType( - CmsEntryElasticsearchValuesModifier.type - ); - - let values = structuredClone(entry.values); - - for (const plugin of plugins) { - values = plugin.exec({ - model, - entry, - values - }); - } - - entry.values = values; - - return entry; -}; - -const getEntryData = async (params: GetElasticsearchEntryDataParams): Promise => { - const output: any = { - ...params.entry - }; - delete output["PK"]; - delete output["SK"]; - delete output["published"]; - delete output["latest"]; - - return modifyEntryValues({ - ...params, - entry: output - }); -}; - -const getESLatestEntryData = async (params: GetElasticsearchEntryDataParams) => { - const { container } = params; - - const output = await getEntryData(params); - return compress(container, { - ...output, - latest: true, - TYPE: createLatestRecordType(), - __type: createLatestRecordType() - }); -}; - -const getESPublishedEntryData = async (params: GetElasticsearchEntryDataParams) => { - const { container } = params; - const output = await getEntryData(params); - return compress(container, { - ...output, - published: true, - TYPE: createPublishedRecordType(), - __type: createPublishedRecordType() - }); -}; - -interface ConvertStorageEntryParams { - entry: CmsStorageEntry; - model: StorageOperationsCmsModel; -} -const convertEntryKeysToStorage = (params: ConvertStorageEntryParams): CmsStorageEntry => { - const { model, entry } = params; - - const values = model.convertValueKeyToStorage({ - fields: model.fields, - values: entry.values - }); - return { - ...entry, - values - }; -}; - -const convertEntryKeysFromStorage = (params: ConvertStorageEntryParams): CmsStorageEntry => { - const { model, entry } = params; - - const values = model.convertValueKeyFromStorage({ - fields: model.fields, - values: entry.values - }); - return { - ...entry, - values - }; -}; +import { createTransformer } from "./transformations"; +import { convertEntryKeysFromStorage } from "./transformations/convertEntryKeys"; interface ElasticsearchDbRecord { index: string; @@ -188,37 +91,24 @@ export const createEntriesStorageOperations = ( const isPublished = initialEntry.status === "published"; const locked = isPublished ? true : initialEntry.locked; - const entry = convertEntryKeysToStorage({ - model, - entry: initialEntry - }); - const storageEntry = convertEntryKeysToStorage({ - model, - entry: initialStorageEntry - }); + initialEntry.locked = locked; + initialStorageEntry.locked = locked; - const esEntry = prepareEntryToIndex({ + const transformer = createTransformer({ plugins, model, - entry: lodashCloneDeep({ ...entry, locked }), - storageEntry: lodashCloneDeep({ ...storageEntry, locked }) + entry: initialEntry, + storageEntry: initialStorageEntry }); + const { entry, storageEntry } = transformer.transformEntryKeys(); + + const esEntry = transformer.transformToIndex(); + const { index: esIndex } = configurations.es({ model }); - const esLatestData = await getESLatestEntryData({ - container: plugins, - model, - entry: esEntry - }); - const esPublishedData = await getESPublishedEntryData({ - container: plugins, - model, - entry: esEntry - }); - const revisionKeys = { PK: createPartitionKey({ id: entry.id, @@ -292,6 +182,7 @@ export const createEntriesStorageOperations = ( ); } + const esLatestData = await transformer.getElasticsearchLatestEntryData(); const esItems = [ esEntity.putBatch({ ...latestKeys, @@ -300,6 +191,7 @@ export const createEntriesStorageOperations = ( }) ]; if (isPublished) { + const esPublishedData = await transformer.getElasticsearchPublishedEntryData(); esItems.push( esEntity.putBatch({ ...publishedKeys, @@ -336,14 +228,13 @@ export const createEntriesStorageOperations = ( const { entry: initialEntry, storageEntry: initialStorageEntry } = params; const model = getStorageOperationsModel(initialModel); - const entry = convertEntryKeysToStorage({ - model, - entry: initialEntry - }); - const storageEntry = convertEntryKeysToStorage({ + const transformer = createTransformer({ + plugins, model, - entry: initialStorageEntry + entry: initialEntry, + storageEntry: initialStorageEntry }); + const { entry, storageEntry } = transformer.transformEntryKeys(); const revisionKeys = { PK: createPartitionKey({ @@ -362,18 +253,7 @@ export const createEntriesStorageOperations = ( SK: createLatestSortKey() }; - const esEntry = prepareEntryToIndex({ - plugins, - model, - entry: lodashCloneDeep(entry), - storageEntry: lodashCloneDeep(storageEntry) - }); - - const esLatestData = await getESLatestEntryData({ - container: plugins, - model, - entry: esEntry - }); + const esLatestData = await transformer.getElasticsearchLatestEntryData(); const items = [ entity.putBatch({ @@ -439,15 +319,15 @@ export const createEntriesStorageOperations = ( const { entry: initialEntry, storageEntry: initialStorageEntry } = params; const model = getStorageOperationsModel(initialModel); - const entry = convertEntryKeysToStorage({ - model, - entry: initialEntry - }); - const storageEntry = convertEntryKeysToStorage({ + const transformer = createTransformer({ + plugins, model, - entry: initialStorageEntry + entry: initialEntry, + storageEntry: initialStorageEntry }); + const { entry, storageEntry } = transformer.transformEntryKeys(); + const isPublished = entry.status === "published"; const locked = isPublished ? true : entry.locked; @@ -514,14 +394,9 @@ export const createEntriesStorageOperations = ( const { index: esIndex } = configurations.es({ model }); - /** - * Variable for the elasticsearch entry so we do not convert it more than once - */ - let esEntry: CmsIndexEntry | undefined = undefined; /** * If the latest entry is the one being updated, we need to create a new latest entry records. */ - let elasticsearchLatestData: any = null; if (latestStorageEntry?.id === entry.id) { /** * First we update the regular DynamoDB table @@ -536,24 +411,7 @@ export const createEntriesStorageOperations = ( /** * And then update the Elasticsearch table to propagate changes to the Elasticsearch */ - esEntry = prepareEntryToIndex({ - plugins, - model, - entry: lodashCloneDeep({ - ...entry, - locked - }), - storageEntry: lodashCloneDeep({ - ...storageEntry, - locked - }) - }); - - elasticsearchLatestData = await getESLatestEntryData({ - container: plugins, - model, - entry: esEntry - }); + const elasticsearchLatestData = await transformer.getElasticsearchLatestEntryData(); esItems.push( esEntity.putBatch({ @@ -563,40 +421,9 @@ export const createEntriesStorageOperations = ( }) ); } - let elasticsearchPublishedData = null; if (isPublished && publishedStorageEntry?.id === entry.id) { - if (!elasticsearchLatestData) { - /** - * And then update the Elasticsearch table to propagate changes to the Elasticsearch - */ - if (!esEntry) { - esEntry = prepareEntryToIndex({ - plugins, - model, - entry: lodashCloneDeep({ - ...entry, - locked - }), - storageEntry: lodashCloneDeep({ - ...storageEntry, - locked - }) - }); - } - elasticsearchPublishedData = await getESPublishedEntryData({ - container: plugins, - model, - entry: esEntry - }); - } else { - elasticsearchPublishedData = { - ...elasticsearchLatestData, - published: true, - TYPE: createPublishedRecordType(), - __type: createPublishedRecordType() - }; - delete elasticsearchPublishedData.latest; - } + const elasticsearchPublishedData = + await transformer.getElasticsearchPublishedEntryData(); esItems.push( esEntity.putBatch({ ...publishedKeys, @@ -897,7 +724,7 @@ export const createEntriesStorageOperations = ( }) ]; - const esItems = []; + const esItems: BatchWriteItem[] = []; /** * If revision we are deleting is the published one as well, we need to delete those records as well. @@ -917,18 +744,13 @@ export const createEntriesStorageOperations = ( ); } if (latestEntry && latestStorageEntry) { - const esEntry = prepareEntryToIndex({ + const latestTransformer = createTransformer({ plugins, model, - entry: lodashCloneDeep(latestEntry), - storageEntry: lodashCloneDeep(latestStorageEntry) - }); - - const esLatestData = await getESLatestEntryData({ - container: plugins, - model, - entry: esEntry + entry: latestEntry, + storageEntry: latestStorageEntry }); + const esLatestData = await latestTransformer.getElasticsearchLatestEntryData(); /** * In the end we need to set the new latest entry */ @@ -1193,15 +1015,15 @@ export const createEntriesStorageOperations = ( const { entry: initialEntry, storageEntry: initialStorageEntry } = params; const model = getStorageOperationsModel(initialModel); - const entry = convertEntryKeysToStorage({ - model, - entry: initialEntry - }); - const storageEntry = convertEntryKeysToStorage({ + const transformer = createTransformer({ + plugins, model, - entry: initialStorageEntry + entry: initialEntry, + storageEntry: initialStorageEntry }); + const { entry, storageEntry } = transformer.transformEntryKeys(); + /** * We need currently published entry to check if need to remove it. */ @@ -1328,47 +1150,37 @@ export const createEntriesStorageOperations = ( * * No need to transform it for the storage because it was fetched directly from the Elasticsearch table, where it sits transformed. */ - const latestEsEntryDataDecompressed: CmsEntry = (await decompress( + const latestEsEntryDataDecompressed: CmsIndexEntry = (await decompress( plugins, latestEsEntry.data )) as any; - const entryData = { - ...latestEsEntryDataDecompressed, - status: CONTENT_ENTRY_STATUS.PUBLISHED, - locked: true, - savedOn: entry.savedOn, - publishedOn: entry.publishedOn - }; + const latestTransformer = createTransformer({ + plugins, + model, + transformedToIndex: { + ...latestEsEntryDataDecompressed, + status: CONTENT_ENTRY_STATUS.PUBLISHED, + locked: true, + savedOn: entry.savedOn, + publishedOn: entry.publishedOn + } + }); + esItems.push( esEntity.putBatch({ index, PK: createPartitionKey(latestEsEntryDataDecompressed), SK: createLatestSortKey(), - data: await getESLatestEntryData({ - container: plugins, - model, - entry: entryData - }) + data: await latestTransformer.getElasticsearchLatestEntryData() }) ); } - const preparedEntryData = prepareEntryToIndex({ - plugins, - model, - entry: lodashCloneDeep(entry), - storageEntry: lodashCloneDeep(storageEntry) - }); /** * Update the published revision entry in ES. */ - const esPublishedData = await getESPublishedEntryData({ - container: plugins, - model, - entry: preparedEntryData - }); - + const esPublishedData = await transformer.getElasticsearchPublishedEntryData(); esItems.push( esEntity.putBatch({ ...publishedKeys, @@ -1428,14 +1240,13 @@ export const createEntriesStorageOperations = ( const { entry: initialEntry, storageEntry: initialStorageEntry } = params; const model = getStorageOperationsModel(initialModel); - const entry = convertEntryKeysToStorage({ - model, - entry: initialEntry - }); - const storageEntry = convertEntryKeysToStorage({ + const transformer = createTransformer({ + plugins, model, - entry: initialStorageEntry + entry: initialEntry, + storageEntry: initialStorageEntry }); + const { entry, storageEntry } = await transformer.transformEntryKeys(); /** * We need the latest entry to check if it needs to be updated. @@ -1478,18 +1289,7 @@ export const createEntriesStorageOperations = ( model }); - const preparedEntryData = prepareEntryToIndex({ - plugins, - model, - entry: lodashCloneDeep(entry), - storageEntry: lodashCloneDeep(storageEntry) - }); - - const esLatestData = await getESLatestEntryData({ - container: plugins, - model, - entry: preparedEntryData - }); + const esLatestData = await transformer.getElasticsearchLatestEntryData(); esItems.push( esEntity.putBatch({ PK: partitionKey, @@ -1622,6 +1422,7 @@ export const createEntriesStorageOperations = ( model, ids: params.ids }); + return entries.map(entry => { return convertEntryKeysFromStorage({ model, diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/convertEntryKeys.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/convertEntryKeys.ts new file mode 100644 index 00000000000..f3e141907f9 --- /dev/null +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/convertEntryKeys.ts @@ -0,0 +1,31 @@ +import { CmsStorageEntry, StorageOperationsCmsModel } from "@webiny/api-headless-cms/types"; + +interface ConvertStorageEntryParams { + entry: CmsStorageEntry; + model: StorageOperationsCmsModel; +} +export const convertEntryKeysToStorage = (params: ConvertStorageEntryParams): CmsStorageEntry => { + const { model, entry } = params; + + const values = model.convertValueKeyToStorage({ + fields: model.fields, + values: entry.values + }); + return { + ...entry, + values + }; +}; + +export const convertEntryKeysFromStorage = (params: ConvertStorageEntryParams): CmsStorageEntry => { + const { model, entry } = params; + + const values = model.convertValueKeyFromStorage({ + fields: model.fields, + values: entry.values + }); + return { + ...entry, + values + }; +}; diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/index.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/index.ts new file mode 100644 index 00000000000..7859af4eed6 --- /dev/null +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/index.ts @@ -0,0 +1,178 @@ +import { PluginsContainer } from "@webiny/plugins"; +import { + CmsEntry, + CmsStorageEntry, + StorageOperationsCmsModel +} from "@webiny/api-headless-cms/types"; +import { transformEntryKeys } from "./transformEntryKeys"; +import { CmsIndexEntry } from "~/types"; +import { transformEntryToIndex } from "~/operations/entry/transformations/transformEntryToIndex"; +import { CmsEntryElasticsearchValuesModifier } from "~/plugins"; +import { modifyEntryValues as modifyEntryValuesCallable } from "~/operations/entry/transformations/modifyEntryValues"; +import { compress } from "@webiny/api-elasticsearch"; +import { createLatestRecordType, createPublishedRecordType } from "~/operations/entry/recordType"; +import WebinyError from "@webiny/error"; + +interface BaseTransformerParams { + plugins: PluginsContainer; + model: StorageOperationsCmsModel; +} + +interface EntryTransformerParams extends BaseTransformerParams { + entry: CmsEntry; + storageEntry: CmsEntry; + transformedToIndex?: never; +} + +interface TransformedEntryTransformerParams extends BaseTransformerParams { + entry?: never; + storageEntry?: never; + transformedToIndex: CmsIndexEntry; +} + +interface TransformedKeysEntry { + entry: CmsEntry; + storageEntry: CmsEntry; +} + +interface ModifiedEntryValues { + entry: CmsEntry; + storageEntry: CmsEntry; +} + +interface TransformerResult { + transformEntryKeys: () => TransformedKeysEntry; + transformToIndex: () => CmsIndexEntry; + getElasticsearchLatestEntryData: () => Promise; + getElasticsearchPublishedEntryData: () => Promise; +} + +export const createTransformer = ( + params: EntryTransformerParams | TransformedEntryTransformerParams +): TransformerResult => { + const { + plugins, + model, + entry: baseEntry, + storageEntry: baseStorageEntry, + transformedToIndex: initialTransformedEntryToIndex = undefined + } = params; + + let transformedEntryKeys: TransformedKeysEntry | undefined = undefined; + let transformedEntryToIndex: CmsIndexEntry | undefined = initialTransformedEntryToIndex; + let modifiedEntryValues: ModifiedEntryValues | undefined = undefined; + let elasticsearchLatestEntry: any = undefined; + let elasticsearchPublishedEntry: any = undefined; + + const modifierPlugins = plugins + .byType(CmsEntryElasticsearchValuesModifier.type) + .filter(pl => pl.canModify(model.modelId)); + + const modifyEntryValues = () => { + if (initialTransformedEntryToIndex || !baseEntry) { + throw new WebinyError( + `Should not call the "modifyEntryValues" when "transformedToIndex" is provided.`, + "METHOD_NOT_ALLOWED", + { + entry: initialTransformedEntryToIndex + } + ); + } + if (modifiedEntryValues) { + return modifiedEntryValues; + } + const modifiedEntry = modifyEntryValuesCallable({ + plugins: modifierPlugins, + model, + entry: baseEntry + }); + const modifiedStorageEntry = modifyEntryValuesCallable({ + plugins: modifierPlugins, + model, + entry: baseStorageEntry + }); + + return (modifiedEntryValues = transformEntryKeys({ + model, + entry: modifiedEntry, + storageEntry: modifiedStorageEntry + })); + }; + + return { + transformEntryKeys: function () { + if (initialTransformedEntryToIndex || !baseEntry) { + throw new WebinyError( + `Should not call the "modifyEntryValues" when "transformedToIndex" is provided.`, + "METHOD_NOT_ALLOWED", + { + entry: initialTransformedEntryToIndex + } + ); + } + if (transformedEntryKeys) { + return transformedEntryKeys; + } + return (transformedEntryKeys = transformEntryKeys({ + model, + entry: baseEntry, + storageEntry: baseStorageEntry + })); + }, + transformToIndex: function () { + if (transformedEntryToIndex) { + return transformedEntryToIndex; + } + let entry: CmsEntry; + let storageEntry: CmsStorageEntry; + /** + * In case there are value modifier plugins, we need to + * - run modifiers + * - transform keys + */ + if (modifierPlugins.length > 0) { + const result = modifyEntryValues(); + entry = result.entry; + storageEntry = result.storageEntry; + } + // In case there are no modifier plugins, just transform the keys - or used already transformed. + else { + const result = this.transformEntryKeys(); + entry = result.entry; + storageEntry = result.storageEntry; + } + return (transformedEntryToIndex = transformEntryToIndex({ + plugins, + model, + entry, + storageEntry + })); + }, + getElasticsearchLatestEntryData: async function () { + if (elasticsearchLatestEntry) { + return elasticsearchLatestEntry; + } + const entry = this.transformToIndex(); + + return (elasticsearchLatestEntry = await compress(plugins, { + ...entry, + latest: true, + TYPE: createLatestRecordType(), + __type: createLatestRecordType() + })); + }, + getElasticsearchPublishedEntryData: async function () { + if (elasticsearchPublishedEntry) { + return elasticsearchPublishedEntry; + } + const entry = this.transformToIndex(); + + return (elasticsearchPublishedEntry = await compress(plugins, { + ...entry, + published: true, + TYPE: createPublishedRecordType(), + __type: createPublishedRecordType() + })); + } + }; +}; diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/modifyEntryValues.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/modifyEntryValues.ts new file mode 100644 index 00000000000..6992477372d --- /dev/null +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/modifyEntryValues.ts @@ -0,0 +1,24 @@ +import { CmsEntry, StorageOperationsCmsModel } from "@webiny/api-headless-cms/types"; +import { CmsEntryElasticsearchValuesModifier } from "~/plugins"; + +interface Params { + model: StorageOperationsCmsModel; + plugins: CmsEntryElasticsearchValuesModifier[]; + entry: CmsEntry; +} + +export const modifyEntryValues = (params: Params) => { + const { plugins, model, entry } = params; + let values = entry.values; + for (const plugin of plugins) { + values = plugin.modify({ + model, + entry, + values + }); + } + return { + ...entry, + values + }; +}; diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/transformEntryKeys.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/transformEntryKeys.ts new file mode 100644 index 00000000000..a533477f477 --- /dev/null +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/transformEntryKeys.ts @@ -0,0 +1,26 @@ +import { + CmsEntry, + CmsStorageEntry, + StorageOperationsCmsModel +} from "@webiny/api-headless-cms/types"; +import { convertEntryKeysToStorage } from "./convertEntryKeys"; + +interface TransformKeysParams { + model: StorageOperationsCmsModel; + entry: CmsEntry; + storageEntry: CmsStorageEntry; +} + +export const transformEntryKeys = (params: TransformKeysParams) => { + const { model, entry, storageEntry } = params; + return { + entry: convertEntryKeysToStorage({ + model, + entry + }), + storageEntry: convertEntryKeysToStorage({ + model, + entry: storageEntry + }) + }; +}; diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/transformEntryToIndex.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/transformEntryToIndex.ts new file mode 100644 index 00000000000..f122ca502a5 --- /dev/null +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/transformations/transformEntryToIndex.ts @@ -0,0 +1,34 @@ +import { PluginsContainer } from "@webiny/plugins"; +import { + CmsEntry, + CmsStorageEntry, + StorageOperationsCmsModel +} from "@webiny/api-headless-cms/types"; +import { prepareEntryToIndex } from "~/helpers"; +import lodashCloneDeep from "lodash/cloneDeep"; + +interface TransformEntryToIndexParams { + plugins: PluginsContainer; + model: StorageOperationsCmsModel; + entry: CmsEntry; + storageEntry: CmsStorageEntry; +} + +export const transformEntryToIndex = (params: TransformEntryToIndexParams) => { + const { plugins, model, entry, storageEntry } = params; + const result = prepareEntryToIndex({ + plugins, + model, + entry: lodashCloneDeep(entry), + storageEntry: lodashCloneDeep(storageEntry) + }); + + delete result["PK"]; + delete result["SK"]; + delete result["GSI1_PK"]; + delete result["GSI1_SK"]; + delete result["published"]; + delete result["latest"]; + + return result; +}; diff --git a/packages/api-headless-cms-ddb-es/src/plugins/CmsEntryElasticsearchValuesModifier.ts b/packages/api-headless-cms-ddb-es/src/plugins/CmsEntryElasticsearchValuesModifier.ts index bc8acc04f63..498665e7abf 100644 --- a/packages/api-headless-cms-ddb-es/src/plugins/CmsEntryElasticsearchValuesModifier.ts +++ b/packages/api-headless-cms-ddb-es/src/plugins/CmsEntryElasticsearchValuesModifier.ts @@ -13,7 +13,7 @@ interface CmsEntryElasticsearchValuesModifierCbParams { } export interface CmsEntryElasticsearchValuesModifierCb { - (params: CmsEntryElasticsearchValuesModifierCbParams): Promise; + (params: CmsEntryElasticsearchValuesModifierCbParams): void; } export interface CmsEntryElasticsearchValuesModifierExecParams { @@ -22,23 +22,42 @@ export interface CmsEntryElasticsearchValuesModifierExecParams; } +export type CmsEntryElasticsearchValuesModifierParams = + | CmsEntryElasticsearchValuesModifierCb + | { + models: string[]; + modifier: CmsEntryElasticsearchValuesModifierCb; + }; + export class CmsEntryElasticsearchValuesModifier extends Plugin { public static override readonly type: string = "cms.entry.elasticsearch.values.modifier"; + private readonly models?: string[] = undefined; private readonly cb: CmsEntryElasticsearchValuesModifierCb; - public constructor(cb: CmsEntryElasticsearchValuesModifierCb) { + public constructor(params: CmsEntryElasticsearchValuesModifierParams) { super(); - this.cb = cb; + if (typeof params === "function") { + this.cb = params; + } else { + this.cb = params.modifier; + this.models = params.models.length > 0 ? params.models : undefined; + } + } + + public canModify(modelId: string): boolean { + if (!this.models?.length) { + return true; + } + return this.models.includes(modelId); } - public async exec( - params: CmsEntryElasticsearchValuesModifierExecParams - ): Promise> { - let values: any = params.values; - await this.cb({ - model: params.model, - entry: params.entry, + public modify(params: CmsEntryElasticsearchValuesModifierExecParams): Partial { + const { model, entry, values: initialValues } = params; + let values: any = initialValues; + this.cb({ + model, + entry, values, setValues: (cb: CmsEntryElasticsearchValuesModifierCbParamsSetValuesCb) => { values = cb(values); @@ -49,7 +68,7 @@ export class CmsEntryElasticsearchValuesModifier extends Plu } export const createCmsEntryElasticsearchValuesModifier = ( - cb: CmsEntryElasticsearchValuesModifierCb + params: CmsEntryElasticsearchValuesModifierParams ) => { - return new CmsEntryElasticsearchValuesModifier(cb); + return new CmsEntryElasticsearchValuesModifier(params); }; From e342012b028200bb23456334ccd63ef82e292ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 31 Jul 2023 15:58:08 +0200 Subject: [PATCH 6/7] fix(api-elasticsearch): compression and decompression return types --- packages/api-elasticsearch/src/compression.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api-elasticsearch/src/compression.ts b/packages/api-elasticsearch/src/compression.ts index f08879d3aa2..8da6a2766cd 100644 --- a/packages/api-elasticsearch/src/compression.ts +++ b/packages/api-elasticsearch/src/compression.ts @@ -13,7 +13,7 @@ const getCompressionPlugins = (plugins: PluginsContainer): CompressionPlugin[] = export const compress = async ( pluginsContainer: PluginsContainer, data: Record -): Promise | string> => { +): Promise> => { const plugins = getCompressionPlugins(pluginsContainer); if (plugins.length === 0) { console.log("No compression plugins"); @@ -34,7 +34,7 @@ export const compress = async ( export const decompress = async ( pluginsContainer: PluginsContainer, data: Record -): Promise | string> => { +): Promise> => { const plugins = getCompressionPlugins(pluginsContainer); if (plugins.length === 0) { return data; From 0a1729e7f92d609fe986e2299a6c763d6c290587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Tue, 1 Aug 2023 10:11:46 +0200 Subject: [PATCH 7/7] test(api-headless-cms-ddb-es): cms entry values modifier --- packages/api-elasticsearch/src/compression.ts | 1 - .../__tests__/api/entryValuesModifier.test.ts | 228 ++++++++++++++++++ .../api/helpers/fetchFromElasticsearch.ts | 14 ++ .../__tests__/api/mocks/plugins.ts | 76 ++++++ .../__tests__/api/mocks/result.ts | 46 ++++ .../__tests__/graphql/handler.ts | 3 +- packages/api-headless-cms-ddb-es/src/index.ts | 6 +- 7 files changed, 370 insertions(+), 4 deletions(-) create mode 100644 packages/api-headless-cms-ddb-es/__tests__/api/entryValuesModifier.test.ts create mode 100644 packages/api-headless-cms-ddb-es/__tests__/api/helpers/fetchFromElasticsearch.ts create mode 100644 packages/api-headless-cms-ddb-es/__tests__/api/mocks/plugins.ts create mode 100644 packages/api-headless-cms-ddb-es/__tests__/api/mocks/result.ts diff --git a/packages/api-elasticsearch/src/compression.ts b/packages/api-elasticsearch/src/compression.ts index 8da6a2766cd..56cd9301029 100644 --- a/packages/api-elasticsearch/src/compression.ts +++ b/packages/api-elasticsearch/src/compression.ts @@ -16,7 +16,6 @@ export const compress = async ( ): Promise> => { const plugins = getCompressionPlugins(pluginsContainer); if (plugins.length === 0) { - console.log("No compression plugins"); return data; } for (const plugin of plugins) { diff --git a/packages/api-headless-cms-ddb-es/__tests__/api/entryValuesModifier.test.ts b/packages/api-headless-cms-ddb-es/__tests__/api/entryValuesModifier.test.ts new file mode 100644 index 00000000000..69ea18f7a9a --- /dev/null +++ b/packages/api-headless-cms-ddb-es/__tests__/api/entryValuesModifier.test.ts @@ -0,0 +1,228 @@ +/** + * This file tests the CmsEntryElasticsearchValuesModifier plugin. + * It enables a developer to modify the values that are sent to Elasticsearch. + * + * For example, if you want to send just a title of an article into an Elasticsearch index, you can use this plugin to do so. + */ + +import { useHandler } from "~tests/graphql/handler"; +import { createMockPlugins } from "~tests/converters/mocks"; +import { createEntryRawData } from "~tests/converters/mocks/data"; +import { configurations } from "~/configurations"; +import { + createGlobalModifierPlugin, + createGlobalModifierValues, + createNotApplicableModifierPlugin, + createTargetedModifierPlugin, + createTargetedModifierValues +} from "./mocks/plugins"; +import { createExpectedGetResult } from "./mocks/result"; +import { fetchFromElasticsearch } from "~tests/api/helpers/fetchFromElasticsearch"; + +describe("entry values modifier", () => { + const { index: indexName } = configurations.es({ + model: { + tenant: "root", + locale: "en-US", + modelId: "converter" + } + }); + + it("should modify the audit log entry values which are stored into the Elasticsearch - global", async () => { + const { createContext, elasticsearch } = useHandler({ + plugins: [...createMockPlugins(), createGlobalModifierPlugin()] + }); + const context = await createContext(); + + const manager = await context.cms.getEntryManager("converter"); + + const createResult = await manager.create(createEntryRawData()); + + /** + * Check that we are getting everything properly out of the DynamoDB + */ + const getResult = await manager.get(createResult.id); + expect(getResult).toMatchObject(createExpectedGetResult()); + await elasticsearch.indices.refresh({ + index: indexName + }); + /** + * Then check that we are getting everything properly out of the Elasticsearch, via webiny API. + */ + const [[listResult]] = await manager.listLatest({ + where: { + id: createResult.id + } + }); + expect(listResult.values).toEqual(createGlobalModifierValues()); + }); + + it("should modify the audit log entry values which are stored into the Elasticsearch - targeted", async () => { + const { createContext, elasticsearch } = useHandler({ + plugins: [...createMockPlugins(), createTargetedModifierPlugin()] + }); + const context = await createContext(); + + const manager = await context.cms.getEntryManager("converter"); + + const createResult = await manager.create(createEntryRawData()); + + /** + * Check that we are getting everything properly out of the DynamoDB + */ + const getResult = await manager.get(createResult.id); + expect(getResult).toMatchObject(createExpectedGetResult()); + await elasticsearch.indices.refresh({ + index: indexName + }); + /** + * Then check that we are getting everything properly out of the Elasticsearch, via webiny API. + */ + const [[listResult]] = await manager.listLatest({ + where: { + id: createResult.id + } + }); + expect(listResult.values).toEqual(createTargetedModifierValues()); + }); + + it("should modify the audit log entry values which are stored into the Elasticsearch - not applicable", async () => { + const { createContext, elasticsearch } = useHandler({ + plugins: [...createMockPlugins(), createNotApplicableModifierPlugin()] + }); + const context = await createContext(); + + const manager = await context.cms.getEntryManager("converter"); + + const createResult = await manager.create(createEntryRawData()); + + /** + * Check that we are getting everything properly out of the DynamoDB + */ + const getResult = await manager.get(createResult.id); + expect(getResult).toMatchObject(createExpectedGetResult()); + await elasticsearch.indices.refresh({ + index: indexName + }); + /** + * Then check that we are getting everything properly out of the Elasticsearch, via webiny API. + */ + const [[listResult]] = await manager.listLatest({ + where: { + id: createResult.id + } + }); + expect(listResult.values.title).toEqual(createExpectedGetResult().values.title); + }); + + it("should modify the audit log entry values which are stored into the Elasticsearch - targeted, global and not applicable - transform storageId", async () => { + const { createContext, elasticsearch } = useHandler({ + plugins: [ + ...createMockPlugins(), + createGlobalModifierPlugin(), + createTargetedModifierPlugin({ + inherit: true + }), + createNotApplicableModifierPlugin() + ] + }); + const context = await createContext(); + + const manager = await context.cms.getEntryManager("converter"); + + const createResult = await manager.create(createEntryRawData()); + + /** + * Check that we are getting everything properly out of the DynamoDB + */ + const getResult = await manager.get(createResult.id); + expect(getResult).toMatchObject(createExpectedGetResult()); + await elasticsearch.indices.refresh({ + index: indexName + }); + /** + * Then check that we are getting everything properly out of the Elasticsearch, via webiny API. + */ + const [[listResult]] = await manager.listLatest({ + where: { + id: createResult.id + } + }); + expect(listResult.values).toEqual({ + ...createGlobalModifierValues(), + ...createTargetedModifierValues() + }); + + const elasticsearchResult = await fetchFromElasticsearch({ + client: elasticsearch, + index: indexName + }); + expect(elasticsearchResult).not.toBe(null); + expect(elasticsearchResult).not.toBe(undefined); + expect(elasticsearchResult.values).not.toBe(null); + expect(elasticsearchResult.values).not.toBe(undefined); + + expect(elasticsearchResult.values).toEqual({ + // from the global plugin + "number@ageFieldIdWithSomeValue": 25, + // from targeted plugin + "text@titleFieldIdWithSomeValue": "A targeted modifier plugin." + }); + }); + + it("should modify the audit log entry values which are stored into the Elasticsearch - targeted, global and not applicable - disable transform storageId", async () => { + process.env.WEBINY_API_TEST_STORAGE_ID_CONVERSION_DISABLE = "true"; + const { createContext, elasticsearch } = useHandler({ + plugins: [ + ...createMockPlugins(), + createGlobalModifierPlugin(), + createTargetedModifierPlugin({ + inherit: true + }), + createNotApplicableModifierPlugin() + ] + }); + const context = await createContext(); + + const manager = await context.cms.getEntryManager("converter"); + + const createResult = await manager.create(createEntryRawData()); + + /** + * Check that we are getting everything properly out of the DynamoDB + */ + const getResult = await manager.get(createResult.id); + expect(getResult).toMatchObject(createExpectedGetResult()); + await elasticsearch.indices.refresh({ + index: indexName + }); + /** + * Then check that we are getting everything properly out of the Elasticsearch, via webiny API. + */ + const [[listResult]] = await manager.listLatest({ + where: { + id: createResult.id + } + }); + expect(listResult.values).toEqual({ + ...createGlobalModifierValues(), + ...createTargetedModifierValues() + }); + + const elasticsearchResult = await fetchFromElasticsearch({ + client: elasticsearch, + index: indexName + }); + expect(elasticsearchResult).not.toBe(null); + expect(elasticsearchResult).not.toBe(undefined); + expect(elasticsearchResult.values).not.toBe(null); + expect(elasticsearchResult.values).not.toBe(undefined); + + expect(elasticsearchResult.values).toEqual({ + // from the global plugin + age: 25, + // from targeted plugin + title: "A targeted modifier plugin." + }); + }); +}); diff --git a/packages/api-headless-cms-ddb-es/__tests__/api/helpers/fetchFromElasticsearch.ts b/packages/api-headless-cms-ddb-es/__tests__/api/helpers/fetchFromElasticsearch.ts new file mode 100644 index 00000000000..257ad4cb15c --- /dev/null +++ b/packages/api-headless-cms-ddb-es/__tests__/api/helpers/fetchFromElasticsearch.ts @@ -0,0 +1,14 @@ +import { ElasticsearchClient } from "@webiny/project-utils/testing/elasticsearch/createClient"; + +interface Params { + client: ElasticsearchClient; + index: string; +} + +export const fetchFromElasticsearch = async (params: Params) => { + const { client, index } = params; + const result = await client.search({ + index + }); + return result.body?.hits?.hits[0]?._source; +}; diff --git a/packages/api-headless-cms-ddb-es/__tests__/api/mocks/plugins.ts b/packages/api-headless-cms-ddb-es/__tests__/api/mocks/plugins.ts new file mode 100644 index 00000000000..2851e9ad78a --- /dev/null +++ b/packages/api-headless-cms-ddb-es/__tests__/api/mocks/plugins.ts @@ -0,0 +1,76 @@ +import { CmsEntryElasticsearchValuesModifier } from "~/plugins"; + +interface ModifierParams { + inherit?: boolean; +} + +export const createGlobalModifierValues = () => { + return { + title: "A global modifier plugin.", + age: 25 + }; +}; +export const createGlobalModifierPlugin = (params?: ModifierParams) => { + const plugin = new CmsEntryElasticsearchValuesModifier(({ setValues }) => { + setValues(prev => { + if (params?.inherit) { + return { + ...prev, + ...createGlobalModifierValues() + }; + } + return createGlobalModifierValues(); + }); + }); + plugin.name = "headlessCms.test.global.elasticsearchValueModifier"; + return plugin; +}; + +export const createTargetedModifierValues = () => { + return { + title: "A targeted modifier plugin." + }; +}; +export const createTargetedModifierPlugin = (params?: ModifierParams) => { + const plugin = new CmsEntryElasticsearchValuesModifier({ + models: ["converter"], + modifier: ({ setValues }) => { + setValues(prev => { + if (params?.inherit) { + return { + ...prev, + ...createTargetedModifierValues() + }; + } + return createTargetedModifierValues(); + }); + } + }); + plugin.name = "headlessCms.test.targeted.elasticsearchValueModifier"; + return plugin; +}; + +export const createNotApplicableModifierValues = () => { + return { + title: "This title should not be applied." + }; +}; +export const createNotApplicableModifierPlugin = (params?: ModifierParams) => { + const plugin = new CmsEntryElasticsearchValuesModifier({ + models: ["converterNonExisting"], + modifier: ({ setValues }) => { + setValues(prev => { + if (params?.inherit) { + return { + ...prev, + ...createNotApplicableModifierValues() + }; + } + return createNotApplicableModifierValues(); + }); + } + }); + + plugin.name = "headlessCms.test.notApplicable.elasticsearchValueModifier"; + return plugin; +}; diff --git a/packages/api-headless-cms-ddb-es/__tests__/api/mocks/result.ts b/packages/api-headless-cms-ddb-es/__tests__/api/mocks/result.ts new file mode 100644 index 00000000000..8e18ddf6ef0 --- /dev/null +++ b/packages/api-headless-cms-ddb-es/__tests__/api/mocks/result.ts @@ -0,0 +1,46 @@ +export const createExpectedGetResult = () => { + return { + values: { + title: "Title level 0", + age: 123, + isMarried: true, + dateOfBirth: "2020-01-01", + description: { + compression: "gzip", + value: expect.any(String) + }, + body: { + compression: "jsonpack", + value: expect.any(String) + }, + information: { + subtitle: "Title level 1", + subAge: 234, + subIsMarried: false, + subDateOfBirth: "2020-01-02", + subDescription: { + compression: "gzip", + value: expect.any(String) + }, + subBody: { + compression: "jsonpack", + value: expect.any(String) + }, + subInformation: { + subSecondSubtitle: "Title level 2", + subSecondSubAge: 345, + subSecondSubIsMarried: false, + subSecondSubDateOfBirth: "2020-01-03", + subSecondSubDescription: { + compression: "gzip", + value: expect.any(String) + }, + subSecondSubBody: { + compression: "jsonpack", + value: expect.any(String) + } + } + } + } + }; +}; diff --git a/packages/api-headless-cms-ddb-es/__tests__/graphql/handler.ts b/packages/api-headless-cms-ddb-es/__tests__/graphql/handler.ts index 389a0e59a15..6c3256ca213 100644 --- a/packages/api-headless-cms-ddb-es/__tests__/graphql/handler.ts +++ b/packages/api-headless-cms-ddb-es/__tests__/graphql/handler.ts @@ -18,6 +18,7 @@ import { createEntryEntity } from "~/definitions/entry"; interface UseHandlerParams { plugins?: PluginCollection; + path?: "/graphql" | `/cms/manage/${Lowercase}-${Uppercase}`; } export const useHandler = (params: UseHandlerParams = {}) => { @@ -69,7 +70,7 @@ export const useHandler = (params: UseHandlerParams = {}) => { /** * If no path defined, use /graphql as we want to make request to main api */ - path: "/cms/manage/en-US", + path: params.path || "/cms/manage/en-US", headers: { ["x-tenant"]: "root", ["Content-Type"]: "application/json" diff --git a/packages/api-headless-cms-ddb-es/src/index.ts b/packages/api-headless-cms-ddb-es/src/index.ts index bc5afa7d114..6d4666e6ce7 100644 --- a/packages/api-headless-cms-ddb-es/src/index.ts +++ b/packages/api-headless-cms-ddb-es/src/index.ts @@ -30,7 +30,8 @@ import { CmsEntryElasticsearchIndexPlugin, CmsEntryElasticsearchQueryBuilderValueSearchPlugin, CmsEntryElasticsearchQueryModifierPlugin, - CmsEntryElasticsearchSortModifierPlugin + CmsEntryElasticsearchSortModifierPlugin, + CmsEntryElasticsearchValuesModifier } from "~/plugins"; import { createFilterPlugins } from "~/operations/entry/elasticsearch/filtering/plugins"; import { CmsEntryFilterPlugin } from "~/plugins/CmsEntryFilterPlugin"; @@ -156,7 +157,8 @@ export const createStorageOperations: StorageOperationsFactory = params => { CmsEntryElasticsearchQueryModifierPlugin.type, CmsEntryElasticsearchSortModifierPlugin.type, CmsElasticsearchModelFieldPlugin.type, - StorageOperationsCmsModelPlugin.type + StorageOperationsCmsModelPlugin.type, + CmsEntryElasticsearchValuesModifier.type ]; for (const type of types) { plugins.mergeByType(context.plugins, type);