fix(app-page-builder): improve blocks loading and data compression (#…

…3686)
webiny · Nov 10, 2023 · 441ebc8 · 441ebc8
1 parent 511c4d9
commit 441ebc8
Show file tree

Hide file tree

Showing 77 changed files with 2,154 additions and 748 deletions.
diff --git a/packages/api-page-builder-so-ddb-es/src/definitions/pageBlockEntity.ts b/packages/api-page-builder-so-ddb-es/src/definitions/pageBlockEntity.ts
@@ -19,6 +19,12 @@ export const createPageBlockEntity = (params: Params): Entity<any> => {
             SK: {
                 sortKey: true
             },
+            GSI1_PK: {
+                type: "string"
+            },
+            GSI1_SK: {
+                type: "string"
+            },
             TYPE: {
                 type: "string"
             },

diff --git a/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/compression.ts b/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/compression.ts
@@ -0,0 +1,35 @@
+import { compress as gzip, decompress as ungzip } from "@webiny/utils/compression/gzip";
+import { PageBlock } from "@webiny/api-page-builder/types";
+
+const GZIP = "gzip";
+const TO_STORAGE_ENCODING = "base64";
+const FROM_STORAGE_ENCODING = "utf8";
+
+const convertToBuffer = (value: string | Buffer) => {
+    if (typeof value === "string") {
+        return Buffer.from(value, TO_STORAGE_ENCODING);
+    }
+    return value;
+};
+
+export const compress = async (data: any) => {
+    const value = await gzip(JSON.stringify(data));
+
+    return {
+        compression: GZIP,
+        value: value.toString(TO_STORAGE_ENCODING)
+    };
+};
+
+export const decompress = async (pageBlock: PageBlock) => {
+    try {
+        const buf = await ungzip(convertToBuffer(pageBlock.content.value));
+        const value = buf.toString(FROM_STORAGE_ENCODING);
+        return {
+            ...pageBlock,
+            content: JSON.parse(value)
+        };
+    } catch (ex) {
+        return { ...pageBlock, content: null };
+    }
+};
diff --git a/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/dataLoader.ts b/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/dataLoader.ts
@@ -43,7 +43,7 @@ export class PageBlockDataLoader implements DataLoaderInterface {
                     const batched = items.map(item => {
                         return this.entity.getBatch({
                             PK: createPartitionKey(item),
-                            SK: createSortKey(item)
+                            SK: createSortKey()
                         });
                     });
 

diff --git a/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/index.ts b/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/index.ts
@@ -15,8 +15,9 @@ import { PageBlockDataLoader } from "./dataLoader";
 import { createListResponse } from "@webiny/db-dynamodb/utils/listResponse";
 import { PageBlockDynamoDbFieldPlugin } from "~/plugins/definitions/PageBlockDynamoDbFieldPlugin";
 import { PluginsContainer } from "@webiny/plugins";
-import { createPartitionKey, createSortKey } from "./keys";
+import { createPartitionKey, createSortKey, createGSIPartitionKey, createGSISortKey } from "./keys";
 import { PageBlockStorageOperations } from "~/types";
+import { compress, decompress } from "./compression";
 
 const createType = (): string => {
     return "pb.pageBlock";
@@ -38,7 +39,8 @@ export const createPageBlockStorageOperations = ({
         const { where } = params;
 
         try {
-            return await dataLoader.getOne(where);
+            const pageBlock = await dataLoader.getOne(where);
+            return decompress(pageBlock);
         } catch (ex) {
             throw new WebinyError(
                 ex.message || "Could not load page block by given parameters.",
@@ -53,13 +55,19 @@ export const createPageBlockStorageOperations = ({
     const list = async (params: PageBlockStorageOperationsListParams) => {
         const { where, sort, limit } = params;
 
-        const { tenant, locale, ...restWhere } = where;
+        const { tenant, locale, blockCategory, ...restWhere } = where;
         const queryAllParams: QueryAllParams = {
             entity,
-            partitionKey: createPartitionKey({ tenant, locale }),
-            options: {
-                gt: " "
-            }
+            partitionKey: createGSIPartitionKey({ tenant, locale }),
+            options: blockCategory
+                ? {
+                      index: "GSI1",
+                      beginsWith: `${blockCategory}#`
+                  }
+                : {
+                      index: "GSI1",
+                      gt: " "
+                  }
         };
 
         let items: PageBlock[] = [];
@@ -95,7 +103,7 @@ export const createPageBlockStorageOperations = ({
         });
 
         return createListResponse({
-            items: sortedItems,
+            items: await Promise.all(sortedItems.map(item => decompress(item))),
             limit: limit || 100000,
             totalCount: filteredItems.length,
             after: null
@@ -108,16 +116,20 @@ export const createPageBlockStorageOperations = ({
         const keys = {
             PK: createPartitionKey({
                 tenant: pageBlock.tenant,
-                locale: pageBlock.locale
+                locale: pageBlock.locale,
+                id: pageBlock.id
             }),
-            SK: createSortKey(pageBlock)
+            SK: createSortKey(),
+            GSI1_PK: createGSIPartitionKey({ tenant: pageBlock.tenant, locale: pageBlock.locale }),
+            GSI1_SK: createGSISortKey({ blockCategory: pageBlock.blockCategory, id: pageBlock.id })
         };
 
         try {
             await entity.put({
                 ...pageBlock,
                 TYPE: createType(),
-                ...keys
+                ...keys,
+                content: await compress(pageBlock.content)
             });
             /**
              * Always clear data loader cache when modifying the records.
@@ -141,16 +153,20 @@ export const createPageBlockStorageOperations = ({
         const keys = {
             PK: createPartitionKey({
                 tenant: original.tenant,
-                locale: original.locale
+                locale: original.locale,
+                id: pageBlock.id
             }),
-            SK: createSortKey(pageBlock)
+            SK: createSortKey(),
+            GSI1_PK: createGSIPartitionKey({ tenant: pageBlock.tenant, locale: pageBlock.locale }),
+            GSI1_SK: createGSISortKey({ blockCategory: pageBlock.blockCategory, id: pageBlock.id })
         };
 
         try {
             await entity.put({
                 ...pageBlock,
                 TYPE: createType(),
-                ...keys
+                ...keys,
+                content: await compress(pageBlock.content)
             });
             /**
              * Always clear data loader cache when modifying the records.
@@ -176,9 +192,10 @@ export const createPageBlockStorageOperations = ({
         const keys = {
             PK: createPartitionKey({
                 tenant: pageBlock.tenant,
-                locale: pageBlock.locale
+                locale: pageBlock.locale,
+                id: pageBlock.id
             }),
-            SK: createSortKey(pageBlock)
+            SK: createSortKey()
         };
 
         try {
@@ -190,8 +207,6 @@ export const createPageBlockStorageOperations = ({
              * Always clear data loader cache when modifying the records.
              */
             dataLoader.clear();
-
-            return pageBlock;
         } catch (ex) {
             throw new WebinyError(
                 ex.message || "Could not delete page block.",

diff --git a/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/keys.ts b/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/keys.ts
@@ -1,16 +1,37 @@
 export interface PartitionKeyParams {
     tenant: string;
     locale: string;
+    id: string;
 }
+
+export interface GSIPartitionKeyParams {
+    tenant: string;
+    locale: string;
+}
+
+export interface GSISortKeyParams {
+    blockCategory: string;
+    id: string;
+}
+
 export const createPartitionKey = (params: PartitionKeyParams): string => {
+    const { tenant, locale, id } = params;
+    return `T#${tenant}#L#${locale}#PB#BLOCK#${id}`;
+};
+
+export const createGSIPartitionKey = (params: GSIPartitionKeyParams): string => {
     const { tenant, locale } = params;
-    return `T#${tenant}#L#${locale}#PB#B`;
+    return `T#${tenant}#L#${locale}#PB#BLOCKS`;
+};
+
+export const createGSISortKey = (params: GSISortKeyParams): string => {
+    const { id, blockCategory } = params;
+    return `${blockCategory}#${id}`;
 };
 
 export interface SortKeyParams {
     id: string;
 }
-export const createSortKey = (params: SortKeyParams): string => {
-    const { id } = params;
-    return id;
+export const createSortKey = (): string => {
+    return "A";
 };
diff --git a/packages/api-page-builder-so-ddb-es/src/operations/pageElement/index.ts b/packages/api-page-builder-so-ddb-es/src/operations/pageElement/index.ts
@@ -108,7 +108,6 @@ export const createPageElementStorageOperations = ({
 
         try {
             await entity.delete(keys);
-            return pageElement;
         } catch (ex) {
             throw new WebinyError(
                 ex.message || "Could not delete pageElement.",

diff --git a/packages/api-page-builder-so-ddb/src/definitions/pageBlockEntity.ts b/packages/api-page-builder-so-ddb/src/definitions/pageBlockEntity.ts
@@ -19,6 +19,12 @@ export const createPageBlockEntity = (params: Params): Entity<any> => {
             SK: {
                 sortKey: true
             },
+            GSI1_PK: {
+                type: "string"
+            },
+            GSI1_SK: {
+                type: "string"
+            },
             TYPE: {
                 type: "string"
             },

diff --git a/packages/api-page-builder-so-ddb/src/operations/pageBlock/compression.ts b/packages/api-page-builder-so-ddb/src/operations/pageBlock/compression.ts
@@ -0,0 +1,35 @@
+import { compress as gzip, decompress as ungzip } from "@webiny/utils/compression/gzip";
+import { PageBlock } from "@webiny/api-page-builder/types";
+
+const GZIP = "gzip";
+const TO_STORAGE_ENCODING = "base64";
+const FROM_STORAGE_ENCODING = "utf8";
+
+const convertToBuffer = (value: string | Buffer) => {
+    if (typeof value === "string") {
+        return Buffer.from(value, TO_STORAGE_ENCODING);
+    }
+    return value;
+};
+
+export const compress = async (data: any) => {
+    const value = await gzip(JSON.stringify(data));
+
+    return {
+        compression: GZIP,
+        value: value.toString(TO_STORAGE_ENCODING)
+    };
+};
+
+export const decompress = async (pageBlock: PageBlock) => {
+    try {
+        const buf = await ungzip(convertToBuffer(pageBlock.content.value));
+        const value = buf.toString(FROM_STORAGE_ENCODING);
+        return {
+            ...pageBlock,
+            content: JSON.parse(value)
+        };
+    } catch (ex) {
+        return { ...pageBlock, content: null };
+    }
+};
diff --git a/packages/api-page-builder-so-ddb/src/operations/pageBlock/dataLoader.ts b/packages/api-page-builder-so-ddb/src/operations/pageBlock/dataLoader.ts
@@ -43,7 +43,7 @@ export class PageBlockDataLoader implements DataLoaderInterface {
                     const batched = items.map(item => {
                         return this.entity.getBatch({
                             PK: createPartitionKey(item),
-                            SK: createSortKey(item)
+                            SK: createSortKey()
                         });
                     });