feat(apify): add decryption for input secrets (#83)

Author: drobnikj

Diff for: package-lock.json

@@ -49,7 +49,9 @@
         "lint:fix": "eslint packages/*/src test --fix"
     },
     "devDependencies": {
+        "@apify/consts": "^2.4.1",
         "@apify/eslint-config-ts": "^0.2.3",
+        "@apify/input_secrets": "^1.1.1",
         "@apify/tsconfig": "^0.1.0",
         "@commitlint/config-conventional": "^17.0.3",
         "@types/content-type": "^1.1.5",
@@ -64,15 +66,15 @@
         "@typescript-eslint/parser": "5.39.0",
         "commitlint": "^17.0.3",
         "crawlee": "^3.0.0",
-        "playwright": "*",
-        "puppeteer": "*",
         "eslint": "~8.24.0",
         "fs-extra": "^10.1.0",
         "gen-esm-wrapper": "^1.1.3",
         "husky": "^8.0.1",
         "jest": "^29.1.2",
         "lerna": "^5.0.0",
         "lint-staged": "^13.0.3",
+        "playwright": "*",
+        "puppeteer": "*",
         "rimraf": "^3.0.2",
         "ts-jest": "^29.0.3",
         "ts-node": "^10.8.2",

Diff for: package.json

@@ -54,16 +54,17 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/consts": "^2.3.0",
+        "@apify/consts": "^2.4.1",
+        "@apify/input_secrets": "^1.1.0",
         "@apify/log": "^2.1.4",
         "@apify/timeout": "^0.3.0",
-        "@apify/utilities": "^2.1.5",
+        "@apify/utilities": "^2.2.2",
         "@crawlee/core": "^3.0.0",
         "@crawlee/types": "^3.0.0",
         "@crawlee/utils": "^3.0.0",
-        "semver": "^7.3.7",
         "apify-client": "^2.6.0",
         "ow": "^0.28.1",
+        "semver": "^7.3.7",
         "ws": "^7.5.9"
     }
 }

Diff for: packages/apify/package.json

@@ -1,4 +1,6 @@
 import ow from 'ow';
+import { createPrivateKey } from 'node:crypto';
+import { decryptInputSecrets } from '@apify/input_secrets';
 import { ENV_VARS, INTEGER_ENV_VARS } from '@apify/consts';
 import { addTimeoutToPromise } from '@apify/timeout';
 import log from '@apify/log';
@@ -646,7 +648,17 @@ export class Actor<Data extends Dictionary = Dictionary> {
      * @ignore
      */
     async getInput<T = Dictionary | string | Buffer>(): Promise<T | null> {
-        return this.getValue<T>(this.config.get('inputKey'));
+        const inputSecretsPrivateKeyFile = this.config.get('inputSecretsPrivateKeyFile');
+        const inputSecretsPrivateKeyPassphrase = this.config.get('inputSecretsPrivateKeyPassphrase');
+        const input = await this.getValue<T>(this.config.get('inputKey'));
+        if (ow.isValid(input, ow.object.nonEmpty) && inputSecretsPrivateKeyFile && inputSecretsPrivateKeyPassphrase) {
+            const privateKey = createPrivateKey({
+                key: Buffer.from(inputSecretsPrivateKeyFile, 'base64'),
+                passphrase: inputSecretsPrivateKeyPassphrase,
+            });
+            return decryptInputSecrets<T>({ input, privateKey });
+        }
+        return input;
     }
 
     /**

Diff for: packages/apify/src/actor.ts

@@ -18,6 +18,8 @@ export interface ConfigurationOptions extends CoreConfigurationOptions {
     proxyStatusUrl?: string;
     isAtHome?: boolean;
     userId?: string;
+    inputSecretsPrivateKeyPassphrase?: string;
+    inputSecretsPrivateKeyFile?: string;
 }
 
 /**
@@ -129,6 +131,8 @@ export class Configuration extends CoreConfiguration {
         APIFY_PROXY_PASSWORD: 'proxyPassword',
         APIFY_PROXY_STATUS_URL: 'proxyStatusUrl',
         APIFY_PROXY_PORT: 'proxyPort',
+        APIFY_INPUT_SECRETS_PRIVATE_KEY_FILE: 'inputSecretsPrivateKeyFile',
+        APIFY_INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE: 'inputSecretsPrivateKeyPassphrase',
     };
 
     protected static override INTEGER_VARS = [...super.INTEGER_VARS, 'proxyPort', 'containerPort', 'metamorphAfterSleepMillis'];

Diff for: packages/apify/src/configuration.ts

@@ -1,5 +1,7 @@
+import { createPublicKey } from 'node:crypto';
 import { ACT_JOB_STATUSES, ENV_VARS, KEY_VALUE_STORE_KEYS, WEBHOOK_EVENT_TYPES } from '@apify/consts';
 import log from '@apify/log';
+import { encryptInputSecrets } from '@apify/input_secrets';
 import type { ApifyEnv } from 'apify';
 import { Actor, ProxyConfiguration } from 'apify';
 import type { WebhookUpdateData } from 'apify-client';
@@ -45,6 +47,14 @@ const setEnv = (env: ApifyEnv) => {
     if (env.memoryMbytes) process.env.APIFY_MEMORY_MBYTES = env.memoryMbytes.toString();
 };
 
+const testingPublicKey = createPublicKey({
+    // eslint-disable-next-line max-len
+    key: Buffer.from('LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF0dis3NlNXbklhOFFKWC94RUQxRQpYdnBBQmE3ajBnQnVYenJNUU5adjhtTW1RU0t2VUF0TmpOL2xacUZpQ0haZUQxU2VDcGV1MnFHTm5XbGRxNkhUCnh5cXJpTVZEbFNKaFBNT09QSENISVNVdFI4Tk5lR1Y1MU0wYkxJcENabHcyTU9GUjdqdENWejVqZFRpZ1NvYTIKQWxrRUlRZWQ4UVlDKzk1aGJoOHk5bGcwQ0JxdEdWN1FvMFZQR2xKQ0hGaWNuaWxLVFFZay9MZzkwWVFnUElPbwozbUppeFl5bWFGNmlMZTVXNzg1M0VHWUVFVWdlWmNaZFNjaGVBMEdBMGpRSFVTdnYvMEZjay9adkZNZURJOTVsCmJVQ0JoQjFDbFg4OG4wZUhzUmdWZE5vK0NLMDI4T2IvZTZTK1JLK09VaHlFRVdPTi90alVMdGhJdTJkQWtGcmkKOFFJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==', 'base64'),
+});
+// eslint-disable-next-line max-len
+const testingPrivateKeyFile = 'LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpQcm9jLVR5cGU6IDQsRU5DUllQVEVECkRFSy1JbmZvOiBERVMtRURFMy1DQkMsNTM1QURERjIzNUQ4QkFGOQoKMXFWUzl0S0FhdkVhVUVFMktESnpjM3plMk1lZkc1dmVEd2o1UVJ0ZkRaMXdWNS9VZmIvcU5sVThTSjlNaGhKaQp6RFdrWExueUUzSW0vcEtITVZkS0czYWZkcFRtcis2TmtidXptd0dVMk0vSWpzRjRJZlpad0lGbGJoY09jUnp4CmZmWVIvTlVyaHNrS1RpNGhGV0lBUDlLb3Z6VDhPSzNZY3h6eVZQWUxYNGVWbWt3UmZzeWkwUU5Xb0tGT3d0ZC8KNm9HYzFnd2piRjI5ZDNnUThZQjFGWmRLa1AyMTJGbkt1cTIrUWgvbE1zTUZrTHlTQTRLTGJ3ZG1RSXExbE1QUwpjbUNtZnppV3J1MlBtNEZoM0dmWlQyaE1JWHlIRFdEVzlDTkxKaERodExOZ2RRamFBUFpVT1E4V2hwSkE5MS9vCjJLZzZ3MDd5Z2RCcVd5dTZrc0pXcjNpZ1JpUEJ5QmVNWEpEZU5HY3NhaUZ3Q2c5eFlja1VORXR3NS90WlRsTjIKSEdZV0NpVU5Ed0F2WllMUHR1SHpIOFRFMGxsZm5HR0VuVC9QQlp1UHV4andlZlRleE1mdzFpbGJRU3lkcy9HMgpOOUlKKzkydms0N0ZXR2NOdGh1Q3lCbklva0NpZ0c1ZlBlV2IwQTdpdjk0UGtwRTRJZ3plc0hGQ0ZFQWoxWldLCnpQdFRBQlkwZlJrUzBNc3UwMHYxOXloTTUrdFUwYkVCZWo2eWpzWHRoYzlwS01hcUNIZWlQTC9TSHRkaWsxNVMKQmU4Sml4dVJxZitUeGlYWWVuNTg2aDlzTFpEYzA3cGpkUGp2NVNYRnBYQjhIMlVxQ0tZY2p4R3RvQWpTV0pjWApMNHc3RHNEby80bVg1N0htR09iamlCN1ZyOGhVWEJDdFh2V0dmQXlmcEFZNS9vOXowdm4zREcxaDc1NVVwdDluCkF2MFZrbm9qcmJVYjM1ZlJuU1lYTVltS01LSnpNRlMrdmFvRlpwV0ZjTG10cFRWSWNzc0JGUEYyZEo3V1c0WHMKK0d2Vkl2eFl3S2wyZzFPTE1TTXRZa09vekdlblBXTzdIdU0yMUVKVGIvbHNEZ25GaTkrYWRGZHBLY3R2cm0zdgpmbW1HeG5pRmhLU05GU0xtNms5YStHL2pjK3NVQVBhb2FZNEQ3NHVGajh0WGp0eThFUHdRRGxVUGRVZld3SE9PClF3bVgyMys1REh4V0VoQy91Tm8yNHNNY2ZkQzFGZUpBV281bUNuVU5vUVVmMStNRDVhMzNJdDhhMmlrNUkxUWoKeSs1WGpRaG0xd3RBMWhWTWE4aUxBR0toT09lcFRuK1VBZHpyS0hvNjVtYzNKbGgvSFJDUXJabnVxWkErK0F2WgpjeWU0dWZGWC8xdmRQSTdLb2Q0MEdDM2dlQnhweFFNYnp1OFNUcGpOcElJRkJvRVc5dFRhemUzeHZXWnV6dDc0CnFjZS8xWURuUHBLeW5lM0xGMk94VWoyYWVYUW5YQkpYcGhTZTBVTGJMcWJtUll4bjJKWkl1d09RNHV5dm94NjUKdG9TWGNac054dUs4QTErZXNXR3JSN3pVc0djdU9QQTFERE9Ja2JjcGtmRUxMNjk4RTJRckdqTU9JWnhrcWdxZQoySE5VNktWRmV2NzdZeEJDbm1VcVdXZEhYMjcyU2NPMUYzdWpUdFVnRVBNWGN0aEdBckYzTWxEaUw1Q0k0RkhqCnhHc3pVemxzalRQTmpiY2MzdUE2MjVZS3VVZEI2c1h1Rk5NUHk5UDgwTzBpRWJGTXl3MWxmN2VpdFhvaUUxWVoKc3NhMDVxTUx4M3pPUXZTLzFDdFpqaFp4cVJMRW5pQ3NWa2JVRlVYclpodEU4dG94bGpWSUtpQ25qbitORmtqdwo2bTZ1anpBSytZZHd2Nk5WMFB4S0gwUk5NYVhwb1lmQk1oUmZ3dGlaS3V3Y2hyRFB5UEhBQ2J3WXNZOXdtUE9rCnpwdDNxWi9JdDVYTmVqNDI0RzAzcGpMbk1sd1B1T1VzYmFQUWQ2VHU4TFhsckZReUVjTXJDNHdjUTA1SzFVN3kKM1NNN3RFaTlnbjV3RjY1YVI5eEFBR0grTUtMMk5WNnQrUmlTazJVaWs1clNmeDE4Mk9wYmpSQ2grdmQ4UXhJdwotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=';
+const testingPrivateKeyPassphrase = 'pwd1234';
+
 describe('Actor', () => {
     const localStorageEmulator = new MemoryStorageEmulator();
 
@@ -937,23 +947,43 @@ describe('Actor', () => {
     });
 
     describe('Actor.getInput', () => {
+        const TestingActor = new Actor();
+
         test('should work', async () => {
-            const defaultStore = await KeyValueStore.open();
-            // Uses default value.
-            const oldGet = defaultStore.getValue;
-            // @ts-expect-error TODO use spyOn instead of this
-            defaultStore.getValue = async (key) => expect(key).toEqual(KEY_VALUE_STORE_KEYS.INPUT);
-            await Actor.getInput();
+            const mockGetValue = jest.spyOn(TestingActor, 'getValue');
+            mockGetValue.mockImplementation(async (key) => expect(key).toEqual(KEY_VALUE_STORE_KEYS.INPUT));
+
+            await TestingActor.getInput();
 
             // Uses value from env var.
             process.env[ENV_VARS.INPUT_KEY] = 'some-value';
-            // @ts-expect-error TODO use spyOn instead of this
-            defaultStore.getValue = async (key) => expect(key).toBe('some-value');
-            await Actor.getInput();
+            mockGetValue.mockImplementation(async (key) => expect(key).toBe('some-value'));
+            await TestingActor.getInput();
 
             delete process.env[ENV_VARS.INPUT_KEY];
+            mockGetValue.mockRestore();
+        });
 
-            defaultStore.getValue = oldGet;
+        test('should work with input secrets', async () => {
+            const mockGetValue = jest.spyOn(TestingActor, 'getValue');
+            const originalInput = { secret: 'foo', nonSecret: 'bar' };
+            const likeInputSchema = { properties: { secret: { type: 'string', isSecret: true } }, nonSecret: { type: 'string' } };
+            const encryptedInput = encryptInputSecrets({ input: originalInput, inputSchema: likeInputSchema, publicKey: testingPublicKey });
+            // Checks if encrypts the right value
+            expect(encryptedInput.secret.startsWith('ENCRYPTED_')).toBe(true);
+            expect(encryptedInput.nonSecret).toBe(originalInput.nonSecret);
+
+            mockGetValue.mockImplementation(async (key) => encryptedInput);
+
+            process.env[ENV_VARS.INPUT_SECRETS_PRIVATE_KEY_FILE] = testingPrivateKeyFile;
+            process.env[ENV_VARS.INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE] = testingPrivateKeyPassphrase;
+            const input = await TestingActor.getInput();
+
+            expect(input).toStrictEqual(originalInput);
+
+            delete process.env[ENV_VARS.INPUT_SECRETS_PRIVATE_KEY_FILE];
+            delete process.env[ENV_VARS.INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE];
+            mockGetValue.mockRestore();
         });
     });