chore: enforce limit on storage key to be maximum 128

activepieces · Feb 21, 2024 · 8117785 · 8117785
1 parent cf81d5b
commit 8117785
Show file tree

Hide file tree

Showing 21 changed files with 81 additions and 36 deletions.
diff --git a/packages/engine/src/lib/services/storage.service.ts b/packages/engine/src/lib/services/storage.service.ts
@@ -31,7 +31,7 @@ export const createStorageService = ({ workerToken }: { workerToken: string }) =
                 body: JSON.stringify(request),
             })
             if (!response.ok) {
-                throw new Error('Failed to store entry')
+                throw new Error(JSON.stringify(await response.json()))
             }
             return (await response.json()) ?? null
         },

diff --git a/packages/pieces/community/framework/src/lib/validators/validators.ts b/packages/pieces/community/framework/src/lib/validators/validators.ts
@@ -55,18 +55,18 @@ class Validators {
     return {
       type: ValidationInputType.STRING,
       fn: (property, processedValue, userInput) => {
-        if (isEmpty(processedValue)) return null;
+        if (isEmpty(processedValue)) return 'asd';
 
         const isValid = processedValue.length <= max;
-
+        
         if (!isValid) {
           return formatErrorMessage(ErrorMessages.MAX_LENGTH, {
             userInput,
             length: max.toString(),
           });
         }
 
-        return null;
+        return '123213';
       },
     };
   }

diff --git a/packages/pieces/community/google-gemini/package.json b/packages/pieces/community/google-gemini/package.json
@@ -1,4 +1,4 @@
 {
   "name": "@activepieces/piece-google-gemini",
-  "version": "0.0.5"
+  "version": "0.0.6"
 }
diff --git a/packages/pieces/community/google-gemini/src/lib/actions/chat-gemini.action.ts b/packages/pieces/community/google-gemini/src/lib/actions/chat-gemini.action.ts
@@ -36,14 +36,9 @@ export const chatGemini = createAction({
       required: true,
       description: 'The prompt to generate content from.',
     }),
-    /*image: Property.File({
-            displayName: 'Image',
-            required: false,
-            description: 'The image to generate content from. ONLY when using Gemini Pro Vision.',
-            validators: [Validators.image],
-        }),*/
     memoryKey: Property.ShortText({
       displayName: 'Memory Key',
+      validators: [Validators.maxLength(128)],
       description:
         'A memory key that will keep the chat history. Keep it empty to leave Gemini without memory of previous messages.',
       required: false,

diff --git a/packages/pieces/community/openai/package.json b/packages/pieces/community/openai/package.json
@@ -1,4 +1,4 @@
 {
   "name": "@activepieces/piece-openai",
-  "version": "0.3.20"
+  "version": "0.3.21"
 }
diff --git a/packages/pieces/community/openai/src/lib/actions/ask-assistant.ts b/packages/pieces/community/openai/src/lib/actions/ask-assistant.ts
@@ -56,6 +56,7 @@ export const askAssistant = createAction({
     }),
     memoryKey: Property.ShortText({
       displayName: 'Memory Key',
+      validators: [Validators.maxLength(128)],
       description:
         'A memory key that will keep the chat history shared across runs and flows. Keep it empty to leave your assistant without memory of previous messages.',
       required: false,

diff --git a/packages/pieces/community/openai/src/lib/actions/send-prompt.ts b/packages/pieces/community/openai/src/lib/actions/send-prompt.ts
@@ -103,6 +103,7 @@ export const askOpenAI = createAction({
     }),
     memoryKey: Property.ShortText({
       displayName: 'Memory Key',
+      validators: [Validators.maxLength(128)],
       description:
         'A memory key that will keep the chat history shared across runs and flows. Keep it empty to leave ChatGPT without memory of previous messages.',
       required: false,

diff --git a/packages/pieces/community/store/package.json b/packages/pieces/community/store/package.json
@@ -1,4 +1,4 @@
 {
   "name": "@activepieces/piece-store",
-  "version": "0.4.4"
+  "version": "0.4.5"
 }
diff --git a/packages/pieces/community/store/src/lib/actions/store-add-to-list.ts b/packages/pieces/community/store/src/lib/actions/store-add-to-list.ts
@@ -2,6 +2,7 @@ import {
   createAction,
   Property,
   StoreScope,
+  Validators,
 } from '@activepieces/pieces-framework';
 import deepEqual from 'deep-equal';
 
@@ -21,6 +22,7 @@ export const storageAddtoList = createAction({
     key: Property.ShortText({
       displayName: 'Key',
       required: true,
+      validators: [Validators.maxLength(128)]
     }),
     value: Property.ShortText({
       displayName: 'Value',

diff --git a/packages/pieces/community/store/src/lib/actions/store-append-action.ts b/packages/pieces/community/store/src/lib/actions/store-append-action.ts
@@ -2,6 +2,7 @@ import {
   createAction,
   Property,
   StoreScope,
+  Validators,
 } from '@activepieces/pieces-framework';
 
 export const storageAppendAction = createAction({
@@ -20,6 +21,7 @@ export const storageAppendAction = createAction({
     key: Property.ShortText({
       displayName: 'Key',
       required: true,
+      validators: [Validators.maxLength(128)]
     }),
     value: Property.ShortText({
       displayName: 'Value',

diff --git a/packages/pieces/community/store/src/lib/actions/store-get-action.ts b/packages/pieces/community/store/src/lib/actions/store-get-action.ts
@@ -2,6 +2,7 @@ import {
   createAction,
   Property,
   StoreScope,
+  Validators,
 } from '@activepieces/pieces-framework';
 
 export const storageGetAction = createAction({
@@ -20,6 +21,7 @@ export const storageGetAction = createAction({
     key: Property.ShortText({
       displayName: 'Key',
       required: true,
+      validators: [Validators.maxLength(128)]
     }),
     defaultValue: Property.ShortText({
       displayName: 'Default Value',

diff --git a/packages/pieces/community/store/src/lib/actions/store-put-action.ts b/packages/pieces/community/store/src/lib/actions/store-put-action.ts
@@ -2,6 +2,7 @@ import {
   createAction,
   Property,
   StoreScope,
+  Validators,
 } from '@activepieces/pieces-framework';
 
 export const storagePutAction = createAction({
@@ -20,6 +21,7 @@ export const storagePutAction = createAction({
     key: Property.ShortText({
       displayName: 'Key',
       required: true,
+      validators: [Validators.maxLength(128)]
     }),
     value: Property.ShortText({
       displayName: 'Value',

diff --git a/packages/pieces/community/store/src/lib/actions/store-remove-from-list.ts b/packages/pieces/community/store/src/lib/actions/store-remove-from-list.ts
@@ -2,6 +2,7 @@ import {
   createAction,
   Property,
   StoreScope,
+  Validators,
 } from '@activepieces/pieces-framework';
 import deepEqual from 'deep-equal';
 
@@ -21,6 +22,7 @@ export const storageRemoveFromList = createAction({
     key: Property.ShortText({
       displayName: 'Key',
       required: true,
+      validators: [Validators.maxLength(128)]
     }),
     value: Property.ShortText({
       displayName: 'Value',

diff --git a/packages/pieces/community/store/src/lib/actions/store-remove-value.ts b/packages/pieces/community/store/src/lib/actions/store-remove-value.ts
@@ -2,6 +2,7 @@ import {
   createAction,
   Property,
   StoreScope,
+  Validators,
 } from '@activepieces/pieces-framework';
 
 export const storageRemoveValue = createAction({
@@ -20,6 +21,7 @@ export const storageRemoveValue = createAction({
     key: Property.ShortText({
       displayName: 'Key',
       required: true,
+      validators: [Validators.maxLength(128)]
     }),
     store_scope: Property.StaticDropdown({
       displayName: 'Store Scope',

diff --git a/...s/server/api/src/app/database/migration/postgres/1708455034835-AddUniqueStoreConstrain.ts b/...s/server/api/src/app/database/migration/postgres/1708455034835-AddUniqueStoreConstrain.ts
diff --git a/.../server/api/src/app/database/migration/postgres/1708521505204-AddUniqueStoreConstraint.ts b/.../server/api/src/app/database/migration/postgres/1708521505204-AddUniqueStoreConstraint.ts
@@ -0,0 +1,45 @@
+import { logger } from 'server-shared'
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUniqueStoreConstraint1708521505204 implements MigrationInterface {
+    name = 'AddUniqueStoreConstraint1708521505204'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        logger.info(`${this.name} is up`)
+        // Delete entries larger than 128 characters
+
+        await queryRunner.query(`
+            DELETE FROM "store-entry" AS se WHERE ("se"."projectId", "se"."key", "se"."created") NOT IN (SELECT "projectId", "key", MAX("created") FROM "store-entry" GROUP BY "projectId", "key");
+        `)
+        await queryRunner.query(`
+             DELETE FROM "store-entry"
+             WHERE LENGTH("key") > 128
+         `)
+
+        // Alter the column to change its type
+        await queryRunner.query(`
+            ALTER TABLE "store-entry"
+            ALTER COLUMN "key" TYPE character varying(128) USING "key"::character varying(128)
+        `)
+
+        // Add constraints
+        await queryRunner.query(`
+            ALTER TABLE "store-entry"
+            ADD CONSTRAINT "UQ_6f251cc141de0a8d84d7a4ac17d" UNIQUE ("projectId", "key")
+        `)
+        logger.info(`${this.name} is finished`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        // Drop constraints
+        await queryRunner.query(`
+            ALTER TABLE "store-entry" DROP CONSTRAINT "UQ_6f251cc141de0a8d84d7a4ac17d"
+        `)
+
+        // Alter the column to change its type back to the previous type
+        await queryRunner.query(`
+            ALTER TABLE "store-entry"
+            ALTER COLUMN "key" TYPE character varying USING "key"::character varying
+        `)
+    }
+}
diff --git a/...ges/server/api/src/app/database/migration/sqlite/1708454855876-AddUniqueStoreConstrain.ts b/...ges/server/api/src/app/database/migration/sqlite/1708454855876-AddUniqueStoreConstrain.ts
@@ -4,6 +4,13 @@ export class AddUniqueStoreConstrain1708454855876 implements MigrationInterface
     name = 'AddUniqueStoreConstrain1708454855876'
 
     public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`
+            DELETE FROM "store-entry" AS se WHERE ("se"."projectId", "se"."key", "se"."created") NOT IN (SELECT "projectId", "key", MAX("created") FROM "store-entry" GROUP BY "projectId", "key");
+        `)
+        await queryRunner.query(`
+            DELETE FROM "store-entry"
+            WHERE LENGTH("key") > 128
+        `)
         await queryRunner.query(`
             DROP INDEX "idx_user_platform_id_email"
         `)

diff --git a/packages/server/api/src/app/database/postgres-connection.ts b/packages/server/api/src/app/database/postgres-connection.ts
@@ -101,7 +101,7 @@ import { RemoveUniqueonAppNameAppCredentials1705586178452 } from './migration/po
 import { AddTriggerTestStrategy1707087022764 } from './migration/common/1707087022764-add-trigger-test-strategy'
 import { AddCategoriesToPieceMetadataPostgres1707231704973 } from './migration/postgres/1707231704973-AddCategoriesToPieceMetadataPostgres'
 import { AddAuditEvents1707614902283 } from './migration/postgres/1707614902283-AddAuditEvents'
-import { AddUniqueStoreConstrain1708455034835 } from './migration/postgres/1708455034835-AddUniqueStoreConstrain'
+import { AddUniqueStoreConstraint1708521505204 } from './migration/postgres/1708521505204-AddUniqueStoreConstraint'
 
 const getSslConfig = (): boolean | TlsOptions => {
     const useSsl = system.get(SystemProp.POSTGRES_USE_SSL)
@@ -173,7 +173,7 @@ const getMigrations = (): (new () => MigrationInterface)[] => {
         AddVerifiedAndChangeStatus1703769034497,
         AddTriggerTestStrategy1707087022764,
         AddCategoriesToPieceMetadataPostgres1707231704973,
-        AddUniqueStoreConstrain1708455034835,
+        AddUniqueStoreConstraint1708521505204,
     ]
 
     const edition = getEdition()

diff --git a/packages/server/api/src/app/store-entry/store-entry-entity.ts b/packages/server/api/src/app/store-entry/store-entry-entity.ts
@@ -14,6 +14,7 @@ export const StoreEntryEntity = new EntitySchema<StoreEntrySchema>({
         ...BaseColumnSchemaPart,
         key: {
             type: String,
+            length: 128,
         },
         projectId: ApIdSchema,
         value: {

diff --git a/packages/shared/package.json b/packages/shared/package.json
@@ -1,5 +1,5 @@
 {
   "name": "@activepieces/shared",
-  "version": "0.10.81",
+  "version": "0.10.82",
   "type": "commonjs"
 }
diff --git a/packages/shared/src/lib/store-entry/dto/store-entry-request.ts b/packages/shared/src/lib/store-entry/dto/store-entry-request.ts
@@ -1,7 +1,9 @@
 import { Static, Type } from '@sinclair/typebox'
 
 export const PutStoreEntryRequest = Type.Object({
-    key: Type.String({}),
+    key: Type.String({
+        maxLength: 128,
+    }),
     value: Type.Any({}),
 })