From 19c8dd3cefec2c7a13485932d1d69ce094d27a3e Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Fri, 14 Nov 2025 14:00:49 +0100 Subject: [PATCH 01/25] feat: add function to calculate dynamic default Actor memory --- package-lock.json | 108 ++++++++++++ package.json | 3 +- packages/utilities/src/index.ts | 1 + packages/utilities/src/memory_calculator.ts | 185 ++++++++++++++++++++ test/memory_calculator.test.ts | 164 +++++++++++++++++ tsconfig.build.json | 4 +- 6 files changed, 462 insertions(+), 3 deletions(-) create mode 100644 packages/utilities/src/memory_calculator.ts create mode 100644 test/memory_calculator.test.ts diff --git a/package-lock.json b/package-lock.json index 4f12d1ab3..24474de5c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "jest": "^29.7.0", "lerna": "^9.0.0", "lint-staged": "^16.0.0", + "mathjs": "^15.1.0", "nock": "^14.0.0", "strip-ansi": "^6.0.0", "ts-jest": "^29.2.4", @@ -593,6 +594,16 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/runtime": { + "version": "7.28.4", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.4.tgz", + "integrity": "sha512-Q/N6JNWvIvPnLDvjlE1OUBLPQHH6l3CltCEsHIujp45zQUSSh8K+gHnaEX45yAT1nyngnINhvWtzN+Nb9D8RAQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.27.2", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", @@ -6996,6 +7007,20 @@ "dot-prop": "^5.1.0" } }, + "node_modules/complex.js": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.4.2.tgz", + "integrity": "sha512-qtx7HRhPGSCBtGiST4/WGHuW+zeaND/6Ld+db6PbrulIB1i2Ev/2UPiqcmpQNPSyfBKraC0EOvOKCB5dGZKt3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/rawify" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -8255,6 +8280,13 @@ "node": ">=0.10.0" } }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "dev": true, + "license": "MIT" + }, "node_modules/dedent": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.0.tgz", @@ -8831,6 +8863,13 @@ "node": ">=6" } }, + "node_modules/escape-latex": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/escape-latex/-/escape-latex-1.2.0.tgz", + "integrity": "sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==", + "dev": true, + "license": "MIT" + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -9651,6 +9690,20 @@ "node": ">= 6" } }, + "node_modules/fraction.js": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz", + "integrity": "sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/rawify" + } + }, "node_modules/front-matter": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/front-matter/-/front-matter-4.0.2.tgz", @@ -11673,6 +11726,13 @@ "node": ">=10" } }, + "node_modules/javascript-natural-sort": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/javascript-natural-sort/-/javascript-natural-sort-0.7.1.tgz", + "integrity": "sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==", + "dev": true, + "license": "MIT" + }, "node_modules/jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", @@ -14293,6 +14353,30 @@ "node": ">= 0.4" } }, + "node_modules/mathjs": { + "version": "15.1.0", + "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-15.1.0.tgz", + "integrity": "sha512-HfnAcScQm9drGryodlDqeS3WAl4gUTYGDcOtcqL/8s23MZ28Ib1i8XnYK3ZdjNuaW/L4BAp9lIp8vxAMrcuu1w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@babel/runtime": "^7.26.10", + "complex.js": "^2.2.5", + "decimal.js": "^10.4.3", + "escape-latex": "^1.2.0", + "fraction.js": "^5.2.1", + "javascript-natural-sort": "^0.7.1", + "seedrandom": "^3.0.5", + "tiny-emitter": "^2.1.0", + "typed-function": "^4.2.1" + }, + "bin": { + "mathjs": "bin/cli.js" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -17691,6 +17775,13 @@ "dev": true, "license": "MIT" }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", + "dev": true, + "license": "MIT" + }, "node_modules/semver": { "version": "7.7.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", @@ -18711,6 +18802,13 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/tiny-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==", + "dev": true, + "license": "MIT" + }, "node_modules/tinyexec": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.2.tgz", @@ -19275,6 +19373,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typed-function": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-4.2.1.tgz", + "integrity": "sha512-EGjWssW7Tsk4DGfE+5yluuljS1OGYWiI1J6e8puZz9nTMM51Oug8CD5Zo4gWMsOhq5BI+1bF+rWTm4Vbj3ivRA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", diff --git a/package.json b/package.json index 7ba873b9d..e2daac936 100644 --- a/package.json +++ b/package.json @@ -59,11 +59,12 @@ "clone-deep": "^4.0.1", "commitlint": "^20.0.0", "eslint": "^9.24.0", - "husky": "^9.1.4", "globals": "^16.0.0", + "husky": "^9.1.4", "jest": "^29.7.0", "lerna": "^9.0.0", "lint-staged": "^16.0.0", + "mathjs": "^15.1.0", "nock": "^14.0.0", "strip-ansi": "^6.0.0", "ts-jest": "^29.2.4", diff --git a/packages/utilities/src/index.ts b/packages/utilities/src/index.ts index 8f220661b..7dbe885f4 100644 --- a/packages/utilities/src/index.ts +++ b/packages/utilities/src/index.ts @@ -10,3 +10,4 @@ export * from './url_params_utils'; export * from './code_hash_manager'; export * from './hmac'; export * from './storages'; +export * from './memory_calculator'; diff --git a/packages/utilities/src/memory_calculator.ts b/packages/utilities/src/memory_calculator.ts new file mode 100644 index 000000000..b95829459 --- /dev/null +++ b/packages/utilities/src/memory_calculator.ts @@ -0,0 +1,185 @@ +import { all, create, type EvalFunction } from 'mathjs/number'; + +import log from '@apify/log'; + +import type { LruCache } from '../../datastructures/src/lru_cache'; + +type ActorRunOptions = { + build?: string; + timeoutSecs?: number; + memoryMbytes?: number; // probably no one will need it, but let's keep it consistent + diskMbytes?: number; // probably no one will need it, but let's keep it consistent + maxItems?: number; + maxTotalChargeUsd?: number; + restartOnError?: boolean; +} + +type MemoryEvaluationContext = { + runOptions: ActorRunOptions; + input: Record; +} + +export const DEFAULT_MEMORY_MBYTES_MAX_CHARS = 1000; + +/** + * A Set of allowed keys from ActorRunOptions that can be used in + * the {{variable}} syntax. + */ +const ALLOWED_RUN_OPTION_KEYS = new Set([ + 'build', + 'timeoutSecs', + 'memoryMbytes', + 'diskMbytes', + 'maxItems', + 'maxTotalChargeUsd', + 'restartOnError', +]); + +/** + * Create a mathjs instance with all functions, then disable potentially dangerous ones. + * Was taken from official mathjs security recommendations: https://mathjs.org/docs/expressions/security.html + */ +const math = create(all); +const limitedEvaluate = math.evaluate; +const limitedCompile = math.compile; + +// Disable potentially dangerous functions +math.import({ + // most important (hardly any functional impact) + import() { throw new Error('Function import is disabled'); }, + createUnit() { throw new Error('Function createUnit is disabled'); }, + reviver() { throw new Error('Function reviver is disabled'); }, + + // extra (has functional impact) + evaluate() { throw new Error('Function evaluate is disabled'); }, + parse() { throw new Error('Function parse is disabled'); }, + simplify() { throw new Error('Function simplify is disabled'); }, + derivative() { throw new Error('Function derivative is disabled'); }, + resolve() { throw new Error('Function resolve is disabled'); }, +}, { override: true }); + +/** + * Safely retrieves a nested property from an object using a dot-notation string path. + * + * This is custom function designed to be injected into the math expression evaluator, + * allowing expressions like `get(input, 'user.settings.memory', 512)`. + * + * @param obj The source object to search within. + * @param path A dot-separated string representing the nested path (e.g., "input.payload.size"). + * @param defaultVal The value to return if the path is not found or the value is `null` or `undefined`. + * @returns The retrieved value, or `defaultVal` if the path is unreachable. +*/ +const customGetFunc = (obj: any, path: string, defaultVal?: number) => { + return (path.split('.').reduce((current, key) => current?.[key], obj)) ?? defaultVal; +}; + +/** + * Rounds a number to the closest power of 2. + * @param num The number to round. + * @returns The closest power of 2. +*/ +const roundToClosestPowerOf2 = (num: number): number | undefined => { + // Handle 0 or negative values. The smallest power of 2 is 2^7 = 128. + if (num <= 0) { + throw new Error(`Calculated memory value must be a positive number, greater than 0, got: ${num}`); + } + if (typeof num !== 'number' || num <= 0 || Number.isNaN(num)) { + log.warning('Failed to round number to a power of 2.', { num }); + throw new Error(`Failed to round number to a power of 2.`); + } + + const log2n = Math.log2(num); + + const roundedLog = Math.round(log2n); + + return 2 ** roundedLog; +}; + +/** + * Replaces `{{variable}}` placeholders in an expression string with `runOptions.variable`. + * + * This function also validates that the variable is one of the allowed 'runOptions' keys. + * + * @example + * // Returns "runOptions.memoryMbytes + 1024" + * preprocessDefaultMemoryExpression("{{memoryMbytes}} + 1024"); + * + * @param defaultMemoryMbytes The raw string expression, e.g., "{{memoryMbytes}} * 2". + * @returns A safe, processed expression for evaluation, e.g., "runOptions.memoryMbytes * 2". + */ +const preprocessDefaultMemoryExpression = (defaultMemoryMbytes: string): string => { + // This regex captures the variable name inside {{...}} + const variableRegex = /{{\s*([a-zA-Z0-9_]+)\s*}}/g; + + const processedExpression = defaultMemoryMbytes.replace( + variableRegex, + (_, variableName: string) => { + // Check if the captured variable name is in our allowlist + if (!ALLOWED_RUN_OPTION_KEYS.has(variableName as keyof ActorRunOptions)) { + throw new Error( + `Invalid variable '{{${variableName}}}' in expression.`, + ); + } + + return `runOptions.${variableName}`; + }, + ); + + return processedExpression; +}; + +/** + * Evaluates a dynamic string expression to calculate a memory value, + * then rounds the result to the closest power of 2. + * + * This function provides a sandboxed environment for the expression and injects + * a `get(obj, path, defaultVal)` helper to safely access properties + * from the `context` (e.g., `input` and `runOptions`). + * + * @param defaultMemoryMbytes The string expression to evaluate (e.g., "get(input, 'size', 10) * 1024"). + * @param context The `MemoryEvaluationContext` (containing `input` and `runOptions`) available to the expression. + * @returns The calculated memory value rounded to the closest power of 2, + * or `undefined` if the expression fails, is non-numeric, + * or results in a non-positive value. +*/ +export const calculateDefaultMemoryFromExpression = ( + defaultMemoryMbytes: string, + context: MemoryEvaluationContext, + options: { cache: LruCache } | undefined = undefined, +) => { + if (defaultMemoryMbytes.length > DEFAULT_MEMORY_MBYTES_MAX_CHARS) { + throw new Error(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_MAX_CHARS} characters.`); + } + + // Replaces all occurrences of {{variable}} with runOptions.variable + // e.g., "{{memoryMbytes}} + 1024" becomes "runOptions.memoryMbytes + 1024" + const preProcessedExpression = preprocessDefaultMemoryExpression(defaultMemoryMbytes); + + const preparedContext = { + ...context, + get: customGetFunc, + }; + + let finalResult: number | { entries: number[] }; + + if (options?.cache) { + let compiledExpr = options.cache.get(preProcessedExpression); + + if (!compiledExpr) { + compiledExpr = limitedCompile(preProcessedExpression); + options.cache.add(preProcessedExpression, compiledExpr!); + } + + finalResult = compiledExpr.evaluate(preparedContext); + } else { + finalResult = limitedEvaluate(preProcessedExpression, preparedContext); + } + + // Mathjs wraps multi-line expressions in an object, extract the last evaluated entry. + if (finalResult && typeof finalResult === 'object' && 'entries' in finalResult) { + const { entries } = finalResult; + finalResult = entries[entries.length - 1]; + } + + return roundToClosestPowerOf2(finalResult); +}; diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts new file mode 100644 index 000000000..c1cc04d75 --- /dev/null +++ b/test/memory_calculator.test.ts @@ -0,0 +1,164 @@ +import type { EvalFunction } from 'mathjs'; + +import { LruCache } from '@apify/datastructures'; +import { calculateDefaultMemoryFromExpression, DEFAULT_MEMORY_MBYTES_MAX_CHARS } from '@apify/utilities'; + +describe('calculateDefaultMemoryFromExpression', () => { + const emptyContext = { input: {}, runOptions: {} }; + + describe('Basic Evaluation', () => { + it('correctly calculates and rounds memory from one-line expression', () => { + const context = { input: { size: 10 }, runOptions: {} }; + // 10 * 1024 = 10240. log2(10240) ~ 13.32. round(13) -> 2^13 = 8192 + const result = calculateDefaultMemoryFromExpression('input.size * 1024', context); + expect(result).toBe(8192); + }); + + it('correctly calculates and rounds memory from multi-line expression', () => { + const context = { input: { base: 10, multiplier: 1024 }, runOptions: {} }; + const expr = ` + baseVal = input.base; + multVal = input.multiplier; + baseVal * multVal + `; + // 10 * 1024 = 10240. Rounds to 8192. + const result = calculateDefaultMemoryFromExpression(expr, context); + expect(result).toBe(8192); + }); + + it('correctly accesses runOptions from the context', () => { + const context = { input: {}, runOptions: { timeoutSecs: 60 } }; + const expr = 'runOptions.timeoutSecs * 100'; // 60 * 100 = 6000 + // log2(6000) ~ 12.55. round(13) -> 2^13 = 8192 + const result = calculateDefaultMemoryFromExpression(expr, context); + expect(result).toBe(8192); + }); + + it('correctly handles a single number expression', () => { + // 2048 is 2^11 + const result = calculateDefaultMemoryFromExpression('2048', emptyContext); + expect(result).toBe(2048); + }); + + it('correctly handles expressions with custom get() function', () => { + const context = { input: { nested: { value: 20 } }, runOptions: {} }; + const expr = "get(input, 'nested.value', 10) * 50"; // 20 * 50 = 1000 + const result = calculateDefaultMemoryFromExpression(expr, context); + expect(result).toBe(1024); + }); + + it('should use get() default value when path is invalid', () => { + const context = { input: { user: {} }, runOptions: {} }; + const expr = "get(input, 'user.settings.memory', 512)"; + const result = calculateDefaultMemoryFromExpression(expr, context); + expect(result).toBe(512); + }); + }); + + describe('Preprocessing with {{variable}}', () => { + it('correctly replaces {{variable}} with valid runOptions.variable', () => { + const context = { input: {}, runOptions: { memoryMbytes: 16 } }; + const expr = '{{memoryMbytes}} * 1024'; + // 16 * 1024 = 16384, which is 2^14 + const result = calculateDefaultMemoryFromExpression(expr, context); + expect(result).toBe(16384); + }); + + it('should throw error for invalid variable in {{variable}} syntax', () => { + const context = { input: {}, runOptions: { memoryMbytes: 16 } }; + const expr = '{{unexistingVariable}} * 1024'; + expect(() => calculateDefaultMemoryFromExpression(expr, context)) + .toThrow(`Invalid variable '{{unexistingVariable}}' in expression.`); + }); + }); + + describe('Rounding Logic', () => { + it('should round down (e.g., 10240 -> 8192)', () => { + // 2^13 = 8192, 2^14 = 16384. + const result = calculateDefaultMemoryFromExpression('10240', emptyContext); + expect(result).toBe(8192); + }); + + it('should round up (e.g., 13000 -> 16384)', () => { + // 13000 is closer to 16384 than 8192. + const result = calculateDefaultMemoryFromExpression('13000', emptyContext); + expect(result).toBe(16384); + }); + + it('should handle values that are already powers of 2', () => { + // 2^9 = 512 + const result = calculateDefaultMemoryFromExpression('512', emptyContext); + expect(result).toBe(512); + }); + }); + + describe('Invalid/Error Handling', () => { + it('should throw an error if expression length is too long', () => { + const expr = '1'.repeat(DEFAULT_MEMORY_MBYTES_MAX_CHARS + 1); // Assuming max length is 1000 + expect(() => calculateDefaultMemoryFromExpression(expr, emptyContext)) + .toThrow(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_MAX_CHARS} characters.`); + }); + + it('should throw an error for invalid syntax', () => { + const expr = '1 +* 2'; + // The original function does not have a try/catch, so it *should* throw + expect(() => calculateDefaultMemoryFromExpression(expr, emptyContext)) + .toThrow(); + }); + + it('should return undefined for a 0 result', () => { + expect(() => calculateDefaultMemoryFromExpression('10 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: 0`); + }); + + it('should return undefined for a negative result', () => { + expect(() => calculateDefaultMemoryFromExpression('5 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: -5`); + }); + + it('should return undefined for a NaN result', () => { + expect(() => calculateDefaultMemoryFromExpression('0 / 0', emptyContext)).toThrow('Failed to round number to a power of 2.'); + }); + + it('should return undefined for a non-numeric (string) result', () => { + expect(() => calculateDefaultMemoryFromExpression("'hello'", emptyContext)).toThrow('Failed to round number to a power of 2.'); + }); + + it('should return undefined for a non-numeric (object) result', () => { + expect(() => calculateDefaultMemoryFromExpression('{ a: 1, b: 2 }', emptyContext)).toThrow('Failed to round number to a power of 2.'); + }); + + it('should return error when disabled functionality of MathJS is used', () => { + expect(() => calculateDefaultMemoryFromExpression('evaluate(512)', emptyContext)).toThrow('Function evaluate is disabled'); + }); + }); + + describe('Caching', () => { + let cache: LruCache; + const context = { input: { size: 10 }, runOptions: {} }; + const expr = 'input.size * 1024'; + + beforeEach(() => { + cache = new LruCache({ maxLength: 10 }); + }); + + it('correctly works with cache passed in options', () => { + expect(cache.length()).toBe(0); + + // First call - cache miss + const result1 = calculateDefaultMemoryFromExpression(expr, context, { cache }); + expect(result1).toBe(8192); + expect(cache.length()).toBe(1); // Expression is now cached + + // Second call - cache hit + const result2 = calculateDefaultMemoryFromExpression(expr, context, { cache }); + expect(result2).toBe(8192); + expect(cache.length()).toBe(1); // Cache length is unchanged + }); + + it('should cache different expressions separately', () => { + const expr2 = 'input.size * 2048'; // 10 * 2048 = 20480 -> 16384 + calculateDefaultMemoryFromExpression(expr, context, { cache }); + calculateDefaultMemoryFromExpression(expr2, context, { cache }); + expect(cache.length()).toBe(2); + }); + }); +}); diff --git a/tsconfig.build.json b/tsconfig.build.json index c2935cabc..9465ba14c 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,8 +1,8 @@ { "compilerOptions": { - "module": "commonjs", + "module": "nodenext", "target": "es2022", - "moduleResolution": "node", + "moduleResolution": "nodenext", "declaration": true, "sourceMap": false, "strict": true, From 5191978a46200b86bb779b7ac2b2478f6a3e8c68 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Tue, 18 Nov 2025 10:50:44 +0100 Subject: [PATCH 02/25] refactor: improve preprocessing expression --- packages/utilities/src/memory_calculator.ts | 39 +++++++++++++++------ test/memory_calculator.test.ts | 28 +++++++++++---- 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/packages/utilities/src/memory_calculator.ts b/packages/utilities/src/memory_calculator.ts index b95829459..037622f44 100644 --- a/packages/utilities/src/memory_calculator.ts +++ b/packages/utilities/src/memory_calculator.ts @@ -96,32 +96,49 @@ const roundToClosestPowerOf2 = (num: number): number | undefined => { }; /** - * Replaces `{{variable}}` placeholders in an expression string with `runOptions.variable`. - * - * This function also validates that the variable is one of the allowed 'runOptions' keys. + * Replaces `{{variable}}` placeholders in an expression string with the variable name. + * Enforces strict validation to only allow `input.*` paths or whitelisted `runOptions.*` keys. * * @example * // Returns "runOptions.memoryMbytes + 1024" - * preprocessDefaultMemoryExpression("{{memoryMbytes}} + 1024"); + * preprocessDefaultMemoryExpression("{{runOptions.memoryMbytes}} + 1024"); * - * @param defaultMemoryMbytes The raw string expression, e.g., "{{memoryMbytes}} * 2". + * @param defaultMemoryMbytes The raw string expression, e.g., "{{runOptions.memoryMbytes}} * 2". * @returns A safe, processed expression for evaluation, e.g., "runOptions.memoryMbytes * 2". */ const preprocessDefaultMemoryExpression = (defaultMemoryMbytes: string): string => { - // This regex captures the variable name inside {{...}} - const variableRegex = /{{\s*([a-zA-Z0-9_]+)\s*}}/g; + const variableRegex = /{{\s*([a-zA-Z0-9_.]+)\s*}}/g; const processedExpression = defaultMemoryMbytes.replace( variableRegex, (_, variableName: string) => { - // Check if the captured variable name is in our allowlist - if (!ALLOWED_RUN_OPTION_KEYS.has(variableName as keyof ActorRunOptions)) { + // 1. Validate that the variable starts with either 'input.' or 'runOptions.' + if (!variableName.startsWith('runOptions.') && !variableName.startsWith('input.')) { throw new Error( - `Invalid variable '{{${variableName}}}' in expression.`, + `Invalid variable '{{${variableName}}}' in expression. Variables must start with 'input.' or 'runOptions.'.`, ); } - return `runOptions.${variableName}`; + // 2. Check if the variable is accessing Input (e.g. {{input.someValue}}) + // We do not validate the specific property name because input is dynamic. + if (variableName.startsWith('input.')) { + return variableName; + } + + if (variableName.startsWith('runOptions.')) { + const key = variableName.slice('runOptions.'.length); + if (!ALLOWED_RUN_OPTION_KEYS.has(key as keyof ActorRunOptions)) { + throw new Error( + `Invalid variable '{{${variableName}}}' in expression. Only the following runOptions are allowed: ${Array.from(ALLOWED_RUN_OPTION_KEYS).map((k) => `runOptions.${k}`).join(', ')}.`, + ); + } + return variableName; + } + + // 3. Throw error for unrecognized variables (e.g. {{process.env}}) + throw new Error( + `Invalid variable '{{${variableName}}}' in expression.`, + ); }, ); diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index c1cc04d75..6700ee30d 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -56,19 +56,35 @@ describe('calculateDefaultMemoryFromExpression', () => { }); describe('Preprocessing with {{variable}}', () => { - it('correctly replaces {{variable}} with valid runOptions.variable', () => { + it('should throw error if variable doesn\'t start with .runOptions or .input', () => { const context = { input: {}, runOptions: { memoryMbytes: 16 } }; - const expr = '{{memoryMbytes}} * 1024'; + const expr = '{{unexistingVariable}} * 1024'; + expect(() => calculateDefaultMemoryFromExpression(expr, context)) + .toThrow(`Invalid variable '{{unexistingVariable}}' in expression. Variables must start with 'input.' or 'runOptions.'.`); + }); + + it('correctly replaces {{runOptions.variable}} with valid runOptions.variable', () => { + const context = { input: {}, runOptions: { memoryMbytes: 16 } }; + const expr = '{{runOptions.memoryMbytes}} * 1024'; // 16 * 1024 = 16384, which is 2^14 const result = calculateDefaultMemoryFromExpression(expr, context); expect(result).toBe(16384); }); - it('should throw error for invalid variable in {{variable}} syntax', () => { - const context = { input: {}, runOptions: { memoryMbytes: 16 } }; - const expr = '{{unexistingVariable}} * 1024'; + it('correctly replaces {{input.variable}} with valid input.variable', () => { + const context = { input: { value: 16 }, runOptions: { } }; + const expr = '{{input.value}} * 1024'; + // 16 * 1024 = 16384, which is 2^14 + const result = calculateDefaultMemoryFromExpression(expr, context); + expect(result).toBe(16384); + }); + + it('correctly throw error if runOptions variable is invalid', () => { + const context = { input: { value: 16 }, runOptions: { } }; + const expr = '{{runOptions.customVariable}} * 1024'; + // 16 * 1024 = 16384, which is 2^14 expect(() => calculateDefaultMemoryFromExpression(expr, context)) - .toThrow(`Invalid variable '{{unexistingVariable}}' in expression.`); + .toThrow(`Invalid variable '{{runOptions.customVariable}}' in expression. Only the following runOptions are allowed:`); }); }); From 4eef96c5b463f8977ca08d1b112e4475298c13f5 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Tue, 18 Nov 2025 11:36:31 +0100 Subject: [PATCH 03/25] refactor: clamp result to min/max range --- packages/utilities/src/memory_calculator.ts | 14 +++++++++----- test/memory_calculator.test.ts | 10 ++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/utilities/src/memory_calculator.ts b/packages/utilities/src/memory_calculator.ts index 037622f44..864cf6575 100644 --- a/packages/utilities/src/memory_calculator.ts +++ b/packages/utilities/src/memory_calculator.ts @@ -1,5 +1,6 @@ import { all, create, type EvalFunction } from 'mathjs/number'; +import { ACTOR_LIMITS } from '@apify/consts'; import log from '@apify/log'; import type { LruCache } from '../../datastructures/src/lru_cache'; @@ -23,7 +24,7 @@ export const DEFAULT_MEMORY_MBYTES_MAX_CHARS = 1000; /** * A Set of allowed keys from ActorRunOptions that can be used in - * the {{variable}} syntax. + * the {{runOptions.variable}} syntax. */ const ALLOWED_RUN_OPTION_KEYS = new Set([ 'build', @@ -75,8 +76,9 @@ const customGetFunc = (obj: any, path: string, defaultVal?: number) => { /** * Rounds a number to the closest power of 2. + * The result is clamped to the allowed range (ACTOR_LIMITS.MIN_RUN_MEMORY_MBYTES - ACTOR_LIMITS.MAX_RUN_MEMORY_MBYTES). * @param num The number to round. - * @returns The closest power of 2. + * @returns The closest power of 2 within min/max range. */ const roundToClosestPowerOf2 = (num: number): number | undefined => { // Handle 0 or negative values. The smallest power of 2 is 2^7 = 128. @@ -91,13 +93,14 @@ const roundToClosestPowerOf2 = (num: number): number | undefined => { const log2n = Math.log2(num); const roundedLog = Math.round(log2n); + const result = 2 ** roundedLog; - return 2 ** roundedLog; + return Math.max(ACTOR_LIMITS.MIN_RUN_MEMORY_MBYTES, Math.min(result, ACTOR_LIMITS.MAX_RUN_MEMORY_MBYTES)); }; /** * Replaces `{{variable}}` placeholders in an expression string with the variable name. - * Enforces strict validation to only allow `input.*` paths or whitelisted `runOptions.*` keys. + * Enforces strict validation to allow `{{input.*}}` paths or whitelisted `{{runOptions.*}}` keys. * * @example * // Returns "runOptions.memoryMbytes + 1024" @@ -119,12 +122,13 @@ const preprocessDefaultMemoryExpression = (defaultMemoryMbytes: string): string ); } - // 2. Check if the variable is accessing Input (e.g. {{input.someValue}}) + // 2. Check if the variable is accessing input (e.g. {{input.someValue}}) // We do not validate the specific property name because input is dynamic. if (variableName.startsWith('input.')) { return variableName; } + // 3. Check if the variable is accessing runOptions (e.g. {{runOptions.memoryMbytes}}) and validate the keys. if (variableName.startsWith('runOptions.')) { const key = variableName.slice('runOptions.'.length); if (!ALLOWED_RUN_OPTION_KEYS.has(key as keyof ActorRunOptions)) { diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index 6700ee30d..9d295ea1b 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -106,6 +106,16 @@ describe('calculateDefaultMemoryFromExpression', () => { const result = calculateDefaultMemoryFromExpression('512', emptyContext); expect(result).toBe(512); }); + + it('should clamp to the minimum memory limit if the result is too low', () => { + const result = calculateDefaultMemoryFromExpression('64', emptyContext); + expect(result).toBe(128); + }); + + it('should clamp to the maximum memory limit if the result is too high', () => { + const result = calculateDefaultMemoryFromExpression('100000', emptyContext); + expect(result).toBe(32768); + }); }); describe('Invalid/Error Handling', () => { From 95e11b5a78de32231cfd736472cdb35917fed59e Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Tue, 18 Nov 2025 12:51:09 +0100 Subject: [PATCH 04/25] refactor: clean up --- packages/utilities/src/memory_calculator.ts | 12 +++++------ test/memory_calculator.test.ts | 22 +++++++-------------- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/packages/utilities/src/memory_calculator.ts b/packages/utilities/src/memory_calculator.ts index 864cf6575..ca994d2a4 100644 --- a/packages/utilities/src/memory_calculator.ts +++ b/packages/utilities/src/memory_calculator.ts @@ -38,7 +38,7 @@ const ALLOWED_RUN_OPTION_KEYS = new Set([ /** * Create a mathjs instance with all functions, then disable potentially dangerous ones. - * Was taken from official mathjs security recommendations: https://mathjs.org/docs/expressions/security.html + * MathJS security recommendations: https://mathjs.org/docs/expressions/security.html */ const math = create(all); const limitedEvaluate = math.evaluate; @@ -81,7 +81,7 @@ const customGetFunc = (obj: any, path: string, defaultVal?: number) => { * @returns The closest power of 2 within min/max range. */ const roundToClosestPowerOf2 = (num: number): number | undefined => { - // Handle 0 or negative values. The smallest power of 2 is 2^7 = 128. + // Handle 0 or negative values. if (num <= 0) { throw new Error(`Calculated memory value must be a positive number, greater than 0, got: ${num}`); } @@ -159,9 +159,7 @@ const preprocessDefaultMemoryExpression = (defaultMemoryMbytes: string): string * * @param defaultMemoryMbytes The string expression to evaluate (e.g., "get(input, 'size', 10) * 1024"). * @param context The `MemoryEvaluationContext` (containing `input` and `runOptions`) available to the expression. - * @returns The calculated memory value rounded to the closest power of 2, - * or `undefined` if the expression fails, is non-numeric, - * or results in a non-positive value. + * @returns The calculated memory value rounded to the closest power of 2 clamped within allowed limits. */ export const calculateDefaultMemoryFromExpression = ( defaultMemoryMbytes: string, @@ -172,8 +170,8 @@ export const calculateDefaultMemoryFromExpression = ( throw new Error(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_MAX_CHARS} characters.`); } - // Replaces all occurrences of {{variable}} with runOptions.variable - // e.g., "{{memoryMbytes}} + 1024" becomes "runOptions.memoryMbytes + 1024" + // Replaces all occurrences of {{variable}} with variable + // e.g., "{{runOptions.memoryMbytes}} + 1024" becomes "runOptions.memoryMbytes + 1024" const preProcessedExpression = preprocessDefaultMemoryExpression(defaultMemoryMbytes); const preparedContext = { diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index 9d295ea1b..2b6a0936b 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -9,7 +9,7 @@ describe('calculateDefaultMemoryFromExpression', () => { describe('Basic Evaluation', () => { it('correctly calculates and rounds memory from one-line expression', () => { const context = { input: { size: 10 }, runOptions: {} }; - // 10 * 1024 = 10240. log2(10240) ~ 13.32. round(13) -> 2^13 = 8192 + // 10 * 1024 = 10240. log2(10240) ~ 13.32. round(13.32) -> 2^13 = 8192 const result = calculateDefaultMemoryFromExpression('input.size * 1024', context); expect(result).toBe(8192); }); @@ -35,7 +35,6 @@ describe('calculateDefaultMemoryFromExpression', () => { }); it('correctly handles a single number expression', () => { - // 2048 is 2^11 const result = calculateDefaultMemoryFromExpression('2048', emptyContext); expect(result).toBe(2048); }); @@ -66,7 +65,6 @@ describe('calculateDefaultMemoryFromExpression', () => { it('correctly replaces {{runOptions.variable}} with valid runOptions.variable', () => { const context = { input: {}, runOptions: { memoryMbytes: 16 } }; const expr = '{{runOptions.memoryMbytes}} * 1024'; - // 16 * 1024 = 16384, which is 2^14 const result = calculateDefaultMemoryFromExpression(expr, context); expect(result).toBe(16384); }); @@ -74,15 +72,13 @@ describe('calculateDefaultMemoryFromExpression', () => { it('correctly replaces {{input.variable}} with valid input.variable', () => { const context = { input: { value: 16 }, runOptions: { } }; const expr = '{{input.value}} * 1024'; - // 16 * 1024 = 16384, which is 2^14 const result = calculateDefaultMemoryFromExpression(expr, context); expect(result).toBe(16384); }); - it('correctly throw error if runOptions variable is invalid', () => { + it('should throw error if runOptions variable is invalid', () => { const context = { input: { value: 16 }, runOptions: { } }; const expr = '{{runOptions.customVariable}} * 1024'; - // 16 * 1024 = 16384, which is 2^14 expect(() => calculateDefaultMemoryFromExpression(expr, context)) .toThrow(`Invalid variable '{{runOptions.customVariable}}' in expression. Only the following runOptions are allowed:`); }); @@ -132,27 +128,23 @@ describe('calculateDefaultMemoryFromExpression', () => { .toThrow(); }); - it('should return undefined for a 0 result', () => { + it('should throw error if result is 0', () => { expect(() => calculateDefaultMemoryFromExpression('10 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: 0`); }); - it('should return undefined for a negative result', () => { + it('should throw error if result is negative', () => { expect(() => calculateDefaultMemoryFromExpression('5 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: -5`); }); - it('should return undefined for a NaN result', () => { + it('should throw error if result is NaN', () => { expect(() => calculateDefaultMemoryFromExpression('0 / 0', emptyContext)).toThrow('Failed to round number to a power of 2.'); }); - it('should return undefined for a non-numeric (string) result', () => { + it('should throw error if result is a non-numeric (string)', () => { expect(() => calculateDefaultMemoryFromExpression("'hello'", emptyContext)).toThrow('Failed to round number to a power of 2.'); }); - it('should return undefined for a non-numeric (object) result', () => { - expect(() => calculateDefaultMemoryFromExpression('{ a: 1, b: 2 }', emptyContext)).toThrow('Failed to round number to a power of 2.'); - }); - - it('should return error when disabled functionality of MathJS is used', () => { + it('should throw error when disabled functionality of MathJS is used', () => { expect(() => calculateDefaultMemoryFromExpression('evaluate(512)', emptyContext)).toThrow('Function evaluate is disabled'); }); }); From 457f3c1a2a18844400baf490b788236cf0915f0e Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Tue, 18 Nov 2025 13:11:16 +0100 Subject: [PATCH 05/25] refactor: clean up --- packages/utilities/src/memory_calculator.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/utilities/src/memory_calculator.ts b/packages/utilities/src/memory_calculator.ts index ca994d2a4..2618a5687 100644 --- a/packages/utilities/src/memory_calculator.ts +++ b/packages/utilities/src/memory_calculator.ts @@ -1,7 +1,6 @@ import { all, create, type EvalFunction } from 'mathjs/number'; import { ACTOR_LIMITS } from '@apify/consts'; -import log from '@apify/log'; import type { LruCache } from '../../datastructures/src/lru_cache'; @@ -86,7 +85,6 @@ const roundToClosestPowerOf2 = (num: number): number | undefined => { throw new Error(`Calculated memory value must be a positive number, greater than 0, got: ${num}`); } if (typeof num !== 'number' || num <= 0 || Number.isNaN(num)) { - log.warning('Failed to round number to a power of 2.', { num }); throw new Error(`Failed to round number to a power of 2.`); } @@ -150,12 +148,8 @@ const preprocessDefaultMemoryExpression = (defaultMemoryMbytes: string): string }; /** - * Evaluates a dynamic string expression to calculate a memory value, - * then rounds the result to the closest power of 2. - * - * This function provides a sandboxed environment for the expression and injects - * a `get(obj, path, defaultVal)` helper to safely access properties - * from the `context` (e.g., `input` and `runOptions`). + * Evaluates a dynamic memory expression string using the provided context. + * Result is rounded to the closest power of 2 and clamped within allowed limits. * * @param defaultMemoryMbytes The string expression to evaluate (e.g., "get(input, 'size', 10) * 1024"). * @param context The `MemoryEvaluationContext` (containing `input` and `runOptions`) available to the expression. From 5075a6cb5c85db99f4a702bc7edf334e2bb816bb Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Tue, 18 Nov 2025 13:21:31 +0100 Subject: [PATCH 06/25] refactor: update tsconfig.build.json --- tsconfig.build.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tsconfig.build.json b/tsconfig.build.json index 9465ba14c..97067f3f4 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,8 +1,8 @@ { "compilerOptions": { - "module": "nodenext", + "module": "commonjs", "target": "es2022", - "moduleResolution": "nodenext", + "moduleResolution": "node16", "declaration": true, "sourceMap": false, "strict": true, From bd0baa6118417d53fb589f0217edacc51b27213e Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Tue, 18 Nov 2025 14:09:28 +0100 Subject: [PATCH 07/25] fix: build --- tsconfig.build.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tsconfig.build.json b/tsconfig.build.json index 97067f3f4..9465ba14c 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,8 +1,8 @@ { "compilerOptions": { - "module": "commonjs", + "module": "nodenext", "target": "es2022", - "moduleResolution": "node16", + "moduleResolution": "nodenext", "declaration": true, "sourceMap": false, "strict": true, From 640167e544ffd3522777d5098130d0970806cb8c Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Tue, 18 Nov 2025 14:46:21 +0100 Subject: [PATCH 08/25] refactor: clean up --- packages/utilities/src/memory_calculator.ts | 13 ++++++++----- test/memory_calculator.test.ts | 9 +-------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/packages/utilities/src/memory_calculator.ts b/packages/utilities/src/memory_calculator.ts index 2618a5687..bf4ac886e 100644 --- a/packages/utilities/src/memory_calculator.ts +++ b/packages/utilities/src/memory_calculator.ts @@ -1,3 +1,4 @@ +// MathJS bundle with only numbers is ~2x smaller than the default one. import { all, create, type EvalFunction } from 'mathjs/number'; import { ACTOR_LIMITS } from '@apify/consts'; @@ -80,13 +81,14 @@ const customGetFunc = (obj: any, path: string, defaultVal?: number) => { * @returns The closest power of 2 within min/max range. */ const roundToClosestPowerOf2 = (num: number): number | undefined => { + if (typeof num !== 'number' || Number.isNaN(num)) { + throw new Error(`Failed to round number to a power of 2.`); + } + // Handle 0 or negative values. if (num <= 0) { throw new Error(`Calculated memory value must be a positive number, greater than 0, got: ${num}`); } - if (typeof num !== 'number' || num <= 0 || Number.isNaN(num)) { - throw new Error(`Failed to round number to a power of 2.`); - } const log2n = Math.log2(num); @@ -137,7 +139,7 @@ const preprocessDefaultMemoryExpression = (defaultMemoryMbytes: string): string return variableName; } - // 3. Throw error for unrecognized variables (e.g. {{process.env}}) + // 3. Throw error for unrecognized variables (e.g. {{someVariable}}) throw new Error( `Invalid variable '{{${variableName}}}' in expression.`, ); @@ -188,7 +190,8 @@ export const calculateDefaultMemoryFromExpression = ( finalResult = limitedEvaluate(preProcessedExpression, preparedContext); } - // Mathjs wraps multi-line expressions in an object, extract the last evaluated entry. + // Mathjs wraps multi-line expressions in an object, so we need to extract the last entry. + // Note: one-line expressions return a number directly. if (finalResult && typeof finalResult === 'object' && 'entries' in finalResult) { const { entries } = finalResult; finalResult = entries[entries.length - 1]; diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index 2b6a0936b..472a298b0 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -97,12 +97,6 @@ describe('calculateDefaultMemoryFromExpression', () => { expect(result).toBe(16384); }); - it('should handle values that are already powers of 2', () => { - // 2^9 = 512 - const result = calculateDefaultMemoryFromExpression('512', emptyContext); - expect(result).toBe(512); - }); - it('should clamp to the minimum memory limit if the result is too low', () => { const result = calculateDefaultMemoryFromExpression('64', emptyContext); expect(result).toBe(128); @@ -115,7 +109,7 @@ describe('calculateDefaultMemoryFromExpression', () => { }); describe('Invalid/Error Handling', () => { - it('should throw an error if expression length is too long', () => { + it('should throw an error if expression length is greater than DEFAULT_MEMORY_MBYTES_MAX_CHARS', () => { const expr = '1'.repeat(DEFAULT_MEMORY_MBYTES_MAX_CHARS + 1); // Assuming max length is 1000 expect(() => calculateDefaultMemoryFromExpression(expr, emptyContext)) .toThrow(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_MAX_CHARS} characters.`); @@ -123,7 +117,6 @@ describe('calculateDefaultMemoryFromExpression', () => { it('should throw an error for invalid syntax', () => { const expr = '1 +* 2'; - // The original function does not have a try/catch, so it *should* throw expect(() => calculateDefaultMemoryFromExpression(expr, emptyContext)) .toThrow(); }); From 972737353f0fb0b4305f710546625362f46a7791 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:01:42 +0100 Subject: [PATCH 09/25] refactor: move mathjs logic to a new package --- package-lock.json | 121 +++++++++++ packages/math-utils/CHANGELOG.md | 0 packages/math-utils/package.json | 54 +++++ packages/math-utils/src/index.ts | 1 + packages/math-utils/src/memory_calculator.ts | 201 +++++++++++++++++++ packages/math-utils/tsconfig.build.json | 8 + packages/math-utils/tsconfig.json | 4 + packages/math-utils/tsup.config.ts | 4 + packages/utilities/src/index.ts | 1 - test/memory_calculator.test.ts | 2 +- 10 files changed, 394 insertions(+), 2 deletions(-) create mode 100644 packages/math-utils/CHANGELOG.md create mode 100644 packages/math-utils/package.json create mode 100644 packages/math-utils/src/index.ts create mode 100644 packages/math-utils/src/memory_calculator.ts create mode 100644 packages/math-utils/tsconfig.build.json create mode 100644 packages/math-utils/tsconfig.json create mode 100644 packages/math-utils/tsup.config.ts diff --git a/package-lock.json b/package-lock.json index 7cb50d3cb..a956a286e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "jest": "^29.7.0", "lerna": "^9.0.0", "lint-staged": "^16.0.0", + "mathjs": "^15.1.0", "nock": "^14.0.0", "strip-ansi": "^6.0.0", "ts-jest": "^29.2.4", @@ -116,6 +117,10 @@ "resolved": "packages/markdown", "link": true }, + "node_modules/@apify/math-utils": { + "resolved": "packages/math-utils", + "link": true + }, "node_modules/@apify/payment_qr_codes": { "resolved": "packages/payment_qr_codes", "link": true @@ -593,6 +598,16 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/runtime": { + "version": "7.28.4", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.4.tgz", + "integrity": "sha512-Q/N6JNWvIvPnLDvjlE1OUBLPQHH6l3CltCEsHIujp45zQUSSh8K+gHnaEX45yAT1nyngnINhvWtzN+Nb9D8RAQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.27.2", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", @@ -6709,6 +6724,20 @@ "dot-prop": "^5.1.0" } }, + "node_modules/complex.js": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.4.3.tgz", + "integrity": "sha512-UrQVSUur14tNX6tiP4y8T4w4FeJAX3bi2cIv0pu/DTLFNxoq7z2Yh83Vfzztj6Px3X/lubqQ9IrPp7Bpn6p4MQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/rawify" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -7968,6 +7997,13 @@ "node": ">=0.10.0" } }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "dev": true, + "license": "MIT" + }, "node_modules/dedent": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.0.tgz", @@ -8544,6 +8580,13 @@ "node": ">=6" } }, + "node_modules/escape-latex": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/escape-latex/-/escape-latex-1.2.0.tgz", + "integrity": "sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==", + "dev": true, + "license": "MIT" + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -9364,6 +9407,20 @@ "node": ">= 6" } }, + "node_modules/fraction.js": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz", + "integrity": "sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/rawify" + } + }, "node_modules/front-matter": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/front-matter/-/front-matter-4.0.2.tgz", @@ -11389,6 +11446,13 @@ "node": ">=10" } }, + "node_modules/javascript-natural-sort": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/javascript-natural-sort/-/javascript-natural-sort-0.7.1.tgz", + "integrity": "sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==", + "dev": true, + "license": "MIT" + }, "node_modules/jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", @@ -13654,6 +13718,30 @@ "node": ">= 0.4" } }, + "node_modules/mathjs": { + "version": "15.1.0", + "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-15.1.0.tgz", + "integrity": "sha512-HfnAcScQm9drGryodlDqeS3WAl4gUTYGDcOtcqL/8s23MZ28Ib1i8XnYK3ZdjNuaW/L4BAp9lIp8vxAMrcuu1w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@babel/runtime": "^7.26.10", + "complex.js": "^2.2.5", + "decimal.js": "^10.4.3", + "escape-latex": "^1.2.0", + "fraction.js": "^5.2.1", + "javascript-natural-sort": "^0.7.1", + "seedrandom": "^3.0.5", + "tiny-emitter": "^2.1.0", + "typed-function": "^4.2.1" + }, + "bin": { + "mathjs": "bin/cli.js" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -17036,6 +17124,13 @@ "dev": true, "license": "MIT" }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", + "dev": true, + "license": "MIT" + }, "node_modules/semver": { "version": "7.7.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", @@ -18096,6 +18191,13 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/tiny-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==", + "dev": true, + "license": "MIT" + }, "node_modules/tinyexec": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.2.tgz", @@ -18599,6 +18701,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typed-function": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-4.2.1.tgz", + "integrity": "sha512-EGjWssW7Tsk4DGfE+5yluuljS1OGYWiI1J6e8puZz9nTMM51Oug8CD5Zo4gWMsOhq5BI+1bF+rWTm4Vbj3ivRA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", @@ -19437,6 +19549,15 @@ "node": ">= 18.0.0" } }, + "packages/math-utils": { + "name": "@apify/math-utils", + "version": "0.0.1", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.47.0", + "@apify/log": "^2.5.26" + } + }, "packages/payment_qr_codes": { "name": "@apify/payment_qr_codes", "version": "0.2.1", diff --git a/packages/math-utils/CHANGELOG.md b/packages/math-utils/CHANGELOG.md new file mode 100644 index 000000000..e69de29bb diff --git a/packages/math-utils/package.json b/packages/math-utils/package.json new file mode 100644 index 000000000..6874277c8 --- /dev/null +++ b/packages/math-utils/package.json @@ -0,0 +1,54 @@ +{ + "name": "@apify/math-utils", + "version": "0.0.1", + "description": "Mathematical and numerical utility functions.", + "main": "./dist/cjs/index.cjs", + "module": "./dist/esm/index.mjs", + "typings": "./dist/cjs/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/esm/index.d.mts", + "default": "./dist/esm/index.mjs" + }, + "require": { + "types": "./dist/cjs/index.d.ts", + "default": "./dist/cjs/index.cjs" + } + } + }, + "keywords": [ + "apify" + ], + "author": { + "name": "Apify", + "email": "support@apify.com", + "url": "https://apify.com" + }, + "contributors": [ + "Jan Curn ", + "Marek Trunkát " + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/apify/apify-shared-js" + }, + "bugs": { + "url": "https://github.com/apify/apify-shared-js/issues" + }, + "homepage": "https://apify.com", + "scripts": { + "build": "npm run clean && npm run compile && npm run copy", + "clean": "rimraf ./dist", + "compile": "tsup", + "copy": "ts-node -T ../../scripts/copy.ts" + }, + "publishConfig": { + "access": "public" + }, + "dependencies": { + "@apify/consts": "^2.47.0", + "@apify/log": "^2.5.26" + } +} diff --git a/packages/math-utils/src/index.ts b/packages/math-utils/src/index.ts new file mode 100644 index 000000000..84b3d306d --- /dev/null +++ b/packages/math-utils/src/index.ts @@ -0,0 +1 @@ +export * from './memory_calculator'; diff --git a/packages/math-utils/src/memory_calculator.ts b/packages/math-utils/src/memory_calculator.ts new file mode 100644 index 000000000..bf4ac886e --- /dev/null +++ b/packages/math-utils/src/memory_calculator.ts @@ -0,0 +1,201 @@ +// MathJS bundle with only numbers is ~2x smaller than the default one. +import { all, create, type EvalFunction } from 'mathjs/number'; + +import { ACTOR_LIMITS } from '@apify/consts'; + +import type { LruCache } from '../../datastructures/src/lru_cache'; + +type ActorRunOptions = { + build?: string; + timeoutSecs?: number; + memoryMbytes?: number; // probably no one will need it, but let's keep it consistent + diskMbytes?: number; // probably no one will need it, but let's keep it consistent + maxItems?: number; + maxTotalChargeUsd?: number; + restartOnError?: boolean; +} + +type MemoryEvaluationContext = { + runOptions: ActorRunOptions; + input: Record; +} + +export const DEFAULT_MEMORY_MBYTES_MAX_CHARS = 1000; + +/** + * A Set of allowed keys from ActorRunOptions that can be used in + * the {{runOptions.variable}} syntax. + */ +const ALLOWED_RUN_OPTION_KEYS = new Set([ + 'build', + 'timeoutSecs', + 'memoryMbytes', + 'diskMbytes', + 'maxItems', + 'maxTotalChargeUsd', + 'restartOnError', +]); + +/** + * Create a mathjs instance with all functions, then disable potentially dangerous ones. + * MathJS security recommendations: https://mathjs.org/docs/expressions/security.html + */ +const math = create(all); +const limitedEvaluate = math.evaluate; +const limitedCompile = math.compile; + +// Disable potentially dangerous functions +math.import({ + // most important (hardly any functional impact) + import() { throw new Error('Function import is disabled'); }, + createUnit() { throw new Error('Function createUnit is disabled'); }, + reviver() { throw new Error('Function reviver is disabled'); }, + + // extra (has functional impact) + evaluate() { throw new Error('Function evaluate is disabled'); }, + parse() { throw new Error('Function parse is disabled'); }, + simplify() { throw new Error('Function simplify is disabled'); }, + derivative() { throw new Error('Function derivative is disabled'); }, + resolve() { throw new Error('Function resolve is disabled'); }, +}, { override: true }); + +/** + * Safely retrieves a nested property from an object using a dot-notation string path. + * + * This is custom function designed to be injected into the math expression evaluator, + * allowing expressions like `get(input, 'user.settings.memory', 512)`. + * + * @param obj The source object to search within. + * @param path A dot-separated string representing the nested path (e.g., "input.payload.size"). + * @param defaultVal The value to return if the path is not found or the value is `null` or `undefined`. + * @returns The retrieved value, or `defaultVal` if the path is unreachable. +*/ +const customGetFunc = (obj: any, path: string, defaultVal?: number) => { + return (path.split('.').reduce((current, key) => current?.[key], obj)) ?? defaultVal; +}; + +/** + * Rounds a number to the closest power of 2. + * The result is clamped to the allowed range (ACTOR_LIMITS.MIN_RUN_MEMORY_MBYTES - ACTOR_LIMITS.MAX_RUN_MEMORY_MBYTES). + * @param num The number to round. + * @returns The closest power of 2 within min/max range. +*/ +const roundToClosestPowerOf2 = (num: number): number | undefined => { + if (typeof num !== 'number' || Number.isNaN(num)) { + throw new Error(`Failed to round number to a power of 2.`); + } + + // Handle 0 or negative values. + if (num <= 0) { + throw new Error(`Calculated memory value must be a positive number, greater than 0, got: ${num}`); + } + + const log2n = Math.log2(num); + + const roundedLog = Math.round(log2n); + const result = 2 ** roundedLog; + + return Math.max(ACTOR_LIMITS.MIN_RUN_MEMORY_MBYTES, Math.min(result, ACTOR_LIMITS.MAX_RUN_MEMORY_MBYTES)); +}; + +/** + * Replaces `{{variable}}` placeholders in an expression string with the variable name. + * Enforces strict validation to allow `{{input.*}}` paths or whitelisted `{{runOptions.*}}` keys. + * + * @example + * // Returns "runOptions.memoryMbytes + 1024" + * preprocessDefaultMemoryExpression("{{runOptions.memoryMbytes}} + 1024"); + * + * @param defaultMemoryMbytes The raw string expression, e.g., "{{runOptions.memoryMbytes}} * 2". + * @returns A safe, processed expression for evaluation, e.g., "runOptions.memoryMbytes * 2". + */ +const preprocessDefaultMemoryExpression = (defaultMemoryMbytes: string): string => { + const variableRegex = /{{\s*([a-zA-Z0-9_.]+)\s*}}/g; + + const processedExpression = defaultMemoryMbytes.replace( + variableRegex, + (_, variableName: string) => { + // 1. Validate that the variable starts with either 'input.' or 'runOptions.' + if (!variableName.startsWith('runOptions.') && !variableName.startsWith('input.')) { + throw new Error( + `Invalid variable '{{${variableName}}}' in expression. Variables must start with 'input.' or 'runOptions.'.`, + ); + } + + // 2. Check if the variable is accessing input (e.g. {{input.someValue}}) + // We do not validate the specific property name because input is dynamic. + if (variableName.startsWith('input.')) { + return variableName; + } + + // 3. Check if the variable is accessing runOptions (e.g. {{runOptions.memoryMbytes}}) and validate the keys. + if (variableName.startsWith('runOptions.')) { + const key = variableName.slice('runOptions.'.length); + if (!ALLOWED_RUN_OPTION_KEYS.has(key as keyof ActorRunOptions)) { + throw new Error( + `Invalid variable '{{${variableName}}}' in expression. Only the following runOptions are allowed: ${Array.from(ALLOWED_RUN_OPTION_KEYS).map((k) => `runOptions.${k}`).join(', ')}.`, + ); + } + return variableName; + } + + // 3. Throw error for unrecognized variables (e.g. {{someVariable}}) + throw new Error( + `Invalid variable '{{${variableName}}}' in expression.`, + ); + }, + ); + + return processedExpression; +}; + +/** + * Evaluates a dynamic memory expression string using the provided context. + * Result is rounded to the closest power of 2 and clamped within allowed limits. + * + * @param defaultMemoryMbytes The string expression to evaluate (e.g., "get(input, 'size', 10) * 1024"). + * @param context The `MemoryEvaluationContext` (containing `input` and `runOptions`) available to the expression. + * @returns The calculated memory value rounded to the closest power of 2 clamped within allowed limits. +*/ +export const calculateDefaultMemoryFromExpression = ( + defaultMemoryMbytes: string, + context: MemoryEvaluationContext, + options: { cache: LruCache } | undefined = undefined, +) => { + if (defaultMemoryMbytes.length > DEFAULT_MEMORY_MBYTES_MAX_CHARS) { + throw new Error(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_MAX_CHARS} characters.`); + } + + // Replaces all occurrences of {{variable}} with variable + // e.g., "{{runOptions.memoryMbytes}} + 1024" becomes "runOptions.memoryMbytes + 1024" + const preProcessedExpression = preprocessDefaultMemoryExpression(defaultMemoryMbytes); + + const preparedContext = { + ...context, + get: customGetFunc, + }; + + let finalResult: number | { entries: number[] }; + + if (options?.cache) { + let compiledExpr = options.cache.get(preProcessedExpression); + + if (!compiledExpr) { + compiledExpr = limitedCompile(preProcessedExpression); + options.cache.add(preProcessedExpression, compiledExpr!); + } + + finalResult = compiledExpr.evaluate(preparedContext); + } else { + finalResult = limitedEvaluate(preProcessedExpression, preparedContext); + } + + // Mathjs wraps multi-line expressions in an object, so we need to extract the last entry. + // Note: one-line expressions return a number directly. + if (finalResult && typeof finalResult === 'object' && 'entries' in finalResult) { + const { entries } = finalResult; + finalResult = entries[entries.length - 1]; + } + + return roundToClosestPowerOf2(finalResult); +}; diff --git a/packages/math-utils/tsconfig.build.json b/packages/math-utils/tsconfig.build.json new file mode 100644 index 000000000..3f47ba58f --- /dev/null +++ b/packages/math-utils/tsconfig.build.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.build.json", + "compilerOptions": { + "outDir": "./dist", + "emitDeclarationOnly": true + }, + "include": ["src/**/*"] +} diff --git a/packages/math-utils/tsconfig.json b/packages/math-utils/tsconfig.json new file mode 100644 index 000000000..52d43eaaa --- /dev/null +++ b/packages/math-utils/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*"] +} diff --git a/packages/math-utils/tsup.config.ts b/packages/math-utils/tsup.config.ts new file mode 100644 index 000000000..90c4b2c92 --- /dev/null +++ b/packages/math-utils/tsup.config.ts @@ -0,0 +1,4 @@ +import { createTsupConfig } from '../../scripts/tsup.config'; + +// eslint-disable-next-line import/no-default-export +export default createTsupConfig({}); diff --git a/packages/utilities/src/index.ts b/packages/utilities/src/index.ts index 7dbe885f4..8f220661b 100644 --- a/packages/utilities/src/index.ts +++ b/packages/utilities/src/index.ts @@ -10,4 +10,3 @@ export * from './url_params_utils'; export * from './code_hash_manager'; export * from './hmac'; export * from './storages'; -export * from './memory_calculator'; diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index 472a298b0..6cf6e909a 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -1,7 +1,7 @@ import type { EvalFunction } from 'mathjs'; import { LruCache } from '@apify/datastructures'; -import { calculateDefaultMemoryFromExpression, DEFAULT_MEMORY_MBYTES_MAX_CHARS } from '@apify/utilities'; +import { calculateDefaultMemoryFromExpression, DEFAULT_MEMORY_MBYTES_MAX_CHARS } from '@apify/math-utils'; describe('calculateDefaultMemoryFromExpression', () => { const emptyContext = { input: {}, runOptions: {} }; From 2dcee6525b1cae590f6a3e29e35ce5079b5fa07a Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:08:35 +0100 Subject: [PATCH 10/25] fix: tsconfig --- tsconfig.build.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tsconfig.build.json b/tsconfig.build.json index 9465ba14c..0cbd5ddff 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,8 +1,8 @@ { "compilerOptions": { - "module": "nodenext", + "module": "node20", "target": "es2022", - "moduleResolution": "nodenext", + "moduleResolution": "node16", "declaration": true, "sourceMap": false, "strict": true, From 823b1884070298bba5d9e38e766ced6a9c622c96 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Wed, 19 Nov 2025 16:07:46 +0100 Subject: [PATCH 11/25] refactor: clean up --- package-lock.json | 13 +- packages/math-utils/package.json | 3 +- packages/math-utils/src/memory_calculator.ts | 41 +++- packages/utilities/src/memory_calculator.ts | 201 ------------------- test/memory_calculator.test.ts | 30 +++ 5 files changed, 72 insertions(+), 216 deletions(-) delete mode 100644 packages/utilities/src/memory_calculator.ts diff --git a/package-lock.json b/package-lock.json index a956a286e..421d281e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -602,7 +602,6 @@ "version": "7.28.4", "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.4.tgz", "integrity": "sha512-Q/N6JNWvIvPnLDvjlE1OUBLPQHH6l3CltCEsHIujp45zQUSSh8K+gHnaEX45yAT1nyngnINhvWtzN+Nb9D8RAQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -6728,7 +6727,6 @@ "version": "2.4.3", "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.4.3.tgz", "integrity": "sha512-UrQVSUur14tNX6tiP4y8T4w4FeJAX3bi2cIv0pu/DTLFNxoq7z2Yh83Vfzztj6Px3X/lubqQ9IrPp7Bpn6p4MQ==", - "dev": true, "license": "MIT", "engines": { "node": "*" @@ -8001,7 +7999,6 @@ "version": "10.6.0", "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", - "dev": true, "license": "MIT" }, "node_modules/dedent": { @@ -8584,7 +8581,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/escape-latex/-/escape-latex-1.2.0.tgz", "integrity": "sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==", - "dev": true, "license": "MIT" }, "node_modules/escape-string-regexp": { @@ -9411,7 +9407,6 @@ "version": "5.3.4", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz", "integrity": "sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==", - "dev": true, "license": "MIT", "engines": { "node": "*" @@ -11450,7 +11445,6 @@ "version": "0.7.1", "resolved": "https://registry.npmjs.org/javascript-natural-sort/-/javascript-natural-sort-0.7.1.tgz", "integrity": "sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==", - "dev": true, "license": "MIT" }, "node_modules/jest": { @@ -13722,7 +13716,6 @@ "version": "15.1.0", "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-15.1.0.tgz", "integrity": "sha512-HfnAcScQm9drGryodlDqeS3WAl4gUTYGDcOtcqL/8s23MZ28Ib1i8XnYK3ZdjNuaW/L4BAp9lIp8vxAMrcuu1w==", - "dev": true, "license": "Apache-2.0", "dependencies": { "@babel/runtime": "^7.26.10", @@ -17128,7 +17121,6 @@ "version": "3.0.5", "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", - "dev": true, "license": "MIT" }, "node_modules/semver": { @@ -18195,7 +18187,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==", - "dev": true, "license": "MIT" }, "node_modules/tinyexec": { @@ -18705,7 +18696,6 @@ "version": "4.2.1", "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-4.2.1.tgz", "integrity": "sha512-EGjWssW7Tsk4DGfE+5yluuljS1OGYWiI1J6e8puZz9nTMM51Oug8CD5Zo4gWMsOhq5BI+1bF+rWTm4Vbj3ivRA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 18" @@ -19555,7 +19545,8 @@ "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.47.0", - "@apify/log": "^2.5.26" + "@apify/log": "^2.5.26", + "mathjs": "^15.1.0" } }, "packages/payment_qr_codes": { diff --git a/packages/math-utils/package.json b/packages/math-utils/package.json index 6874277c8..36f8f42b6 100644 --- a/packages/math-utils/package.json +++ b/packages/math-utils/package.json @@ -49,6 +49,7 @@ }, "dependencies": { "@apify/consts": "^2.47.0", - "@apify/log": "^2.5.26" + "@apify/log": "^2.5.26", + "mathjs": "^15.1.0" } } diff --git a/packages/math-utils/src/memory_calculator.ts b/packages/math-utils/src/memory_calculator.ts index bf4ac886e..61e186bea 100644 --- a/packages/math-utils/src/memory_calculator.ts +++ b/packages/math-utils/src/memory_calculator.ts @@ -1,5 +1,21 @@ // MathJS bundle with only numbers is ~2x smaller than the default one. -import { all, create, type EvalFunction } from 'mathjs/number'; +import { + addDependencies, + andDependencies, + compileDependencies, + create, + divideDependencies, + type EvalFunction, + evaluateDependencies, + maxDependencies, + minDependencies, + multiplyDependencies, + notDependencies, + nullishDependencies, + orDependencies, + subtractDependencies, + xorDependencies, +} from 'mathjs'; import { ACTOR_LIMITS } from '@apify/consts'; @@ -37,10 +53,29 @@ const ALLOWED_RUN_OPTION_KEYS = new Set([ ]); /** - * Create a mathjs instance with all functions, then disable potentially dangerous ones. + * Create a mathjs instance with selected dependencies, then disable potentially dangerous ones. * MathJS security recommendations: https://mathjs.org/docs/expressions/security.html */ -const math = create(all); +const math = create({ + // arithmetic dependencies + addDependencies, + subtractDependencies, + multiplyDependencies, + divideDependencies, + // expression dependencies + compileDependencies, + evaluateDependencies, + // statistics dependencies + maxDependencies, + minDependencies, + // logical dependencies + andDependencies, + notDependencies, + orDependencies, + xorDependencies, + // without that dependency 'null ?? 5', won't work + nullishDependencies, +}); const limitedEvaluate = math.evaluate; const limitedCompile = math.compile; diff --git a/packages/utilities/src/memory_calculator.ts b/packages/utilities/src/memory_calculator.ts deleted file mode 100644 index bf4ac886e..000000000 --- a/packages/utilities/src/memory_calculator.ts +++ /dev/null @@ -1,201 +0,0 @@ -// MathJS bundle with only numbers is ~2x smaller than the default one. -import { all, create, type EvalFunction } from 'mathjs/number'; - -import { ACTOR_LIMITS } from '@apify/consts'; - -import type { LruCache } from '../../datastructures/src/lru_cache'; - -type ActorRunOptions = { - build?: string; - timeoutSecs?: number; - memoryMbytes?: number; // probably no one will need it, but let's keep it consistent - diskMbytes?: number; // probably no one will need it, but let's keep it consistent - maxItems?: number; - maxTotalChargeUsd?: number; - restartOnError?: boolean; -} - -type MemoryEvaluationContext = { - runOptions: ActorRunOptions; - input: Record; -} - -export const DEFAULT_MEMORY_MBYTES_MAX_CHARS = 1000; - -/** - * A Set of allowed keys from ActorRunOptions that can be used in - * the {{runOptions.variable}} syntax. - */ -const ALLOWED_RUN_OPTION_KEYS = new Set([ - 'build', - 'timeoutSecs', - 'memoryMbytes', - 'diskMbytes', - 'maxItems', - 'maxTotalChargeUsd', - 'restartOnError', -]); - -/** - * Create a mathjs instance with all functions, then disable potentially dangerous ones. - * MathJS security recommendations: https://mathjs.org/docs/expressions/security.html - */ -const math = create(all); -const limitedEvaluate = math.evaluate; -const limitedCompile = math.compile; - -// Disable potentially dangerous functions -math.import({ - // most important (hardly any functional impact) - import() { throw new Error('Function import is disabled'); }, - createUnit() { throw new Error('Function createUnit is disabled'); }, - reviver() { throw new Error('Function reviver is disabled'); }, - - // extra (has functional impact) - evaluate() { throw new Error('Function evaluate is disabled'); }, - parse() { throw new Error('Function parse is disabled'); }, - simplify() { throw new Error('Function simplify is disabled'); }, - derivative() { throw new Error('Function derivative is disabled'); }, - resolve() { throw new Error('Function resolve is disabled'); }, -}, { override: true }); - -/** - * Safely retrieves a nested property from an object using a dot-notation string path. - * - * This is custom function designed to be injected into the math expression evaluator, - * allowing expressions like `get(input, 'user.settings.memory', 512)`. - * - * @param obj The source object to search within. - * @param path A dot-separated string representing the nested path (e.g., "input.payload.size"). - * @param defaultVal The value to return if the path is not found or the value is `null` or `undefined`. - * @returns The retrieved value, or `defaultVal` if the path is unreachable. -*/ -const customGetFunc = (obj: any, path: string, defaultVal?: number) => { - return (path.split('.').reduce((current, key) => current?.[key], obj)) ?? defaultVal; -}; - -/** - * Rounds a number to the closest power of 2. - * The result is clamped to the allowed range (ACTOR_LIMITS.MIN_RUN_MEMORY_MBYTES - ACTOR_LIMITS.MAX_RUN_MEMORY_MBYTES). - * @param num The number to round. - * @returns The closest power of 2 within min/max range. -*/ -const roundToClosestPowerOf2 = (num: number): number | undefined => { - if (typeof num !== 'number' || Number.isNaN(num)) { - throw new Error(`Failed to round number to a power of 2.`); - } - - // Handle 0 or negative values. - if (num <= 0) { - throw new Error(`Calculated memory value must be a positive number, greater than 0, got: ${num}`); - } - - const log2n = Math.log2(num); - - const roundedLog = Math.round(log2n); - const result = 2 ** roundedLog; - - return Math.max(ACTOR_LIMITS.MIN_RUN_MEMORY_MBYTES, Math.min(result, ACTOR_LIMITS.MAX_RUN_MEMORY_MBYTES)); -}; - -/** - * Replaces `{{variable}}` placeholders in an expression string with the variable name. - * Enforces strict validation to allow `{{input.*}}` paths or whitelisted `{{runOptions.*}}` keys. - * - * @example - * // Returns "runOptions.memoryMbytes + 1024" - * preprocessDefaultMemoryExpression("{{runOptions.memoryMbytes}} + 1024"); - * - * @param defaultMemoryMbytes The raw string expression, e.g., "{{runOptions.memoryMbytes}} * 2". - * @returns A safe, processed expression for evaluation, e.g., "runOptions.memoryMbytes * 2". - */ -const preprocessDefaultMemoryExpression = (defaultMemoryMbytes: string): string => { - const variableRegex = /{{\s*([a-zA-Z0-9_.]+)\s*}}/g; - - const processedExpression = defaultMemoryMbytes.replace( - variableRegex, - (_, variableName: string) => { - // 1. Validate that the variable starts with either 'input.' or 'runOptions.' - if (!variableName.startsWith('runOptions.') && !variableName.startsWith('input.')) { - throw new Error( - `Invalid variable '{{${variableName}}}' in expression. Variables must start with 'input.' or 'runOptions.'.`, - ); - } - - // 2. Check if the variable is accessing input (e.g. {{input.someValue}}) - // We do not validate the specific property name because input is dynamic. - if (variableName.startsWith('input.')) { - return variableName; - } - - // 3. Check if the variable is accessing runOptions (e.g. {{runOptions.memoryMbytes}}) and validate the keys. - if (variableName.startsWith('runOptions.')) { - const key = variableName.slice('runOptions.'.length); - if (!ALLOWED_RUN_OPTION_KEYS.has(key as keyof ActorRunOptions)) { - throw new Error( - `Invalid variable '{{${variableName}}}' in expression. Only the following runOptions are allowed: ${Array.from(ALLOWED_RUN_OPTION_KEYS).map((k) => `runOptions.${k}`).join(', ')}.`, - ); - } - return variableName; - } - - // 3. Throw error for unrecognized variables (e.g. {{someVariable}}) - throw new Error( - `Invalid variable '{{${variableName}}}' in expression.`, - ); - }, - ); - - return processedExpression; -}; - -/** - * Evaluates a dynamic memory expression string using the provided context. - * Result is rounded to the closest power of 2 and clamped within allowed limits. - * - * @param defaultMemoryMbytes The string expression to evaluate (e.g., "get(input, 'size', 10) * 1024"). - * @param context The `MemoryEvaluationContext` (containing `input` and `runOptions`) available to the expression. - * @returns The calculated memory value rounded to the closest power of 2 clamped within allowed limits. -*/ -export const calculateDefaultMemoryFromExpression = ( - defaultMemoryMbytes: string, - context: MemoryEvaluationContext, - options: { cache: LruCache } | undefined = undefined, -) => { - if (defaultMemoryMbytes.length > DEFAULT_MEMORY_MBYTES_MAX_CHARS) { - throw new Error(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_MAX_CHARS} characters.`); - } - - // Replaces all occurrences of {{variable}} with variable - // e.g., "{{runOptions.memoryMbytes}} + 1024" becomes "runOptions.memoryMbytes + 1024" - const preProcessedExpression = preprocessDefaultMemoryExpression(defaultMemoryMbytes); - - const preparedContext = { - ...context, - get: customGetFunc, - }; - - let finalResult: number | { entries: number[] }; - - if (options?.cache) { - let compiledExpr = options.cache.get(preProcessedExpression); - - if (!compiledExpr) { - compiledExpr = limitedCompile(preProcessedExpression); - options.cache.add(preProcessedExpression, compiledExpr!); - } - - finalResult = compiledExpr.evaluate(preparedContext); - } else { - finalResult = limitedEvaluate(preProcessedExpression, preparedContext); - } - - // Mathjs wraps multi-line expressions in an object, so we need to extract the last entry. - // Note: one-line expressions return a number directly. - if (finalResult && typeof finalResult === 'object' && 'entries' in finalResult) { - const { entries } = finalResult; - finalResult = entries[entries.length - 1]; - } - - return roundToClosestPowerOf2(finalResult); -}; diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index 6cf6e909a..ce12aeb3f 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -52,6 +52,36 @@ describe('calculateDefaultMemoryFromExpression', () => { const result = calculateDefaultMemoryFromExpression(expr, context); expect(result).toBe(512); }); + + describe('operations supported', () => { + const context = { + input: { A: 200, B: 10, C: 4, nullVal: null, zeroVal: 0, startUrls: [1, 2, 3] }, + runOptions: { timeoutSecs: 60, memoryMbytes: 512 }, + }; + + const cases = [ + { expression: '5 + 5', desc: '+ allowed' }, + { expression: '6 - 5', desc: '- allowed' }, + { expression: '5 / 5', desc: '/ allowed' }, + { expression: '5 * 5', desc: '* allowed' }, + { expression: 'max(1, 2, 3)', desc: 'max() allowed' }, + { expression: 'min(1, 2, 3)', desc: 'min() allowed' }, + { expression: '(true and false) ? 0 : 5', desc: 'and allowed' }, + { expression: '(true or false) ? 5 : 0', desc: 'or allowed' }, + { expression: '(true xor false) ? 5 : 0', desc: 'xor allowed' }, + { expression: 'not(false) ? 5 : 0', desc: 'not allowed' }, + { expression: 'input.nullVal ?? 256', desc: 'nullish coalescing allowed' }, + { expression: 'a = 5', desc: 'variable assignment' }, + ]; + + it.each(cases)( + '$desc', + ({ expression }) => { + // in case operation is not supported, mathjs will throw + expect(calculateDefaultMemoryFromExpression(expression, context)).toBeDefined(); + }, + ); + }); }); describe('Preprocessing with {{variable}}', () => { From 3dbb353eab795dd521655c607a8135e4018b7863 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Wed, 19 Nov 2025 16:40:21 +0100 Subject: [PATCH 12/25] refactor: clean up --- packages/math-utils/src/memory_calculator.ts | 1 + test/memory_calculator.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/math-utils/src/memory_calculator.ts b/packages/math-utils/src/memory_calculator.ts index 61e186bea..2d5e3b761 100644 --- a/packages/math-utils/src/memory_calculator.ts +++ b/packages/math-utils/src/memory_calculator.ts @@ -11,6 +11,7 @@ import { minDependencies, multiplyDependencies, notDependencies, + // @ts-expect-error nullishDependencies is not declared in types. https://github.com/josdejong/mathjs/issues/3597 nullishDependencies, orDependencies, subtractDependencies, diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index ce12aeb3f..a4f91d214 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -55,7 +55,7 @@ describe('calculateDefaultMemoryFromExpression', () => { describe('operations supported', () => { const context = { - input: { A: 200, B: 10, C: 4, nullVal: null, zeroVal: 0, startUrls: [1, 2, 3] }, + input: { }, runOptions: { timeoutSecs: 60, memoryMbytes: 512 }, }; @@ -70,7 +70,7 @@ describe('calculateDefaultMemoryFromExpression', () => { { expression: '(true or false) ? 5 : 0', desc: 'or allowed' }, { expression: '(true xor false) ? 5 : 0', desc: 'xor allowed' }, { expression: 'not(false) ? 5 : 0', desc: 'not allowed' }, - { expression: 'input.nullVal ?? 256', desc: 'nullish coalescing allowed' }, + { expression: 'null ?? 256', desc: 'nullish coalescing allowed' }, { expression: 'a = 5', desc: 'variable assignment' }, ]; From 9b431075ace6e6a5d8555550eda64d2e50865caa Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Wed, 19 Nov 2025 16:46:21 +0100 Subject: [PATCH 13/25] refactor: change tsconfig of math-utils --- packages/math-utils/tsconfig.json | 6 +++++- tsconfig.build.json | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/math-utils/tsconfig.json b/packages/math-utils/tsconfig.json index 52d43eaaa..595b76fe3 100644 --- a/packages/math-utils/tsconfig.json +++ b/packages/math-utils/tsconfig.json @@ -1,4 +1,8 @@ { "extends": "../../tsconfig.json", - "include": ["src/**/*"] + "include": ["src/**/*"], + "compilerOptions": { + "module": "node20", + "moduleResolution": "node16", + } } diff --git a/tsconfig.build.json b/tsconfig.build.json index 0cbd5ddff..c2935cabc 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,8 +1,8 @@ { "compilerOptions": { - "module": "node20", + "module": "commonjs", "target": "es2022", - "moduleResolution": "node16", + "moduleResolution": "node", "declaration": true, "sourceMap": false, "strict": true, From 3b62b9f90ff0e9450f01563ec199dea40915432d Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Thu, 20 Nov 2025 16:07:02 +0100 Subject: [PATCH 14/25] refactor: change names of functions --- packages/math-utils/src/memory_calculator.ts | 14 ++--- test/memory_calculator.test.ts | 54 ++++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/packages/math-utils/src/memory_calculator.ts b/packages/math-utils/src/memory_calculator.ts index 2d5e3b761..f4ac87b94 100644 --- a/packages/math-utils/src/memory_calculator.ts +++ b/packages/math-utils/src/memory_calculator.ts @@ -145,7 +145,7 @@ const roundToClosestPowerOf2 = (num: number): number | undefined => { * @param defaultMemoryMbytes The raw string expression, e.g., "{{runOptions.memoryMbytes}} * 2". * @returns A safe, processed expression for evaluation, e.g., "runOptions.memoryMbytes * 2". */ -const preprocessDefaultMemoryExpression = (defaultMemoryMbytes: string): string => { +const preprocessRunMemoryExpression = (defaultMemoryMbytes: string): string => { const variableRegex = /{{\s*([a-zA-Z0-9_.]+)\s*}}/g; const processedExpression = defaultMemoryMbytes.replace( @@ -193,7 +193,7 @@ const preprocessDefaultMemoryExpression = (defaultMemoryMbytes: string): string * @param context The `MemoryEvaluationContext` (containing `input` and `runOptions`) available to the expression. * @returns The calculated memory value rounded to the closest power of 2 clamped within allowed limits. */ -export const calculateDefaultMemoryFromExpression = ( +export const calculateRunDynamicMemory = ( defaultMemoryMbytes: string, context: MemoryEvaluationContext, options: { cache: LruCache } | undefined = undefined, @@ -204,7 +204,7 @@ export const calculateDefaultMemoryFromExpression = ( // Replaces all occurrences of {{variable}} with variable // e.g., "{{runOptions.memoryMbytes}} + 1024" becomes "runOptions.memoryMbytes + 1024" - const preProcessedExpression = preprocessDefaultMemoryExpression(defaultMemoryMbytes); + const preprocessedExpression = preprocessRunMemoryExpression(defaultMemoryMbytes); const preparedContext = { ...context, @@ -214,16 +214,16 @@ export const calculateDefaultMemoryFromExpression = ( let finalResult: number | { entries: number[] }; if (options?.cache) { - let compiledExpr = options.cache.get(preProcessedExpression); + let compiledExpr = options.cache.get(preprocessedExpression); if (!compiledExpr) { - compiledExpr = limitedCompile(preProcessedExpression); - options.cache.add(preProcessedExpression, compiledExpr!); + compiledExpr = limitedCompile(preprocessedExpression); + options.cache.add(preprocessedExpression, compiledExpr!); } finalResult = compiledExpr.evaluate(preparedContext); } else { - finalResult = limitedEvaluate(preProcessedExpression, preparedContext); + finalResult = limitedEvaluate(preprocessedExpression, preparedContext); } // Mathjs wraps multi-line expressions in an object, so we need to extract the last entry. diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index a4f91d214..e5457b2f9 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -1,7 +1,7 @@ import type { EvalFunction } from 'mathjs'; import { LruCache } from '@apify/datastructures'; -import { calculateDefaultMemoryFromExpression, DEFAULT_MEMORY_MBYTES_MAX_CHARS } from '@apify/math-utils'; +import { calculateRunDynamicMemory, DEFAULT_MEMORY_MBYTES_MAX_CHARS } from '@apify/math-utils'; describe('calculateDefaultMemoryFromExpression', () => { const emptyContext = { input: {}, runOptions: {} }; @@ -10,7 +10,7 @@ describe('calculateDefaultMemoryFromExpression', () => { it('correctly calculates and rounds memory from one-line expression', () => { const context = { input: { size: 10 }, runOptions: {} }; // 10 * 1024 = 10240. log2(10240) ~ 13.32. round(13.32) -> 2^13 = 8192 - const result = calculateDefaultMemoryFromExpression('input.size * 1024', context); + const result = calculateRunDynamicMemory('input.size * 1024', context); expect(result).toBe(8192); }); @@ -22,7 +22,7 @@ describe('calculateDefaultMemoryFromExpression', () => { baseVal * multVal `; // 10 * 1024 = 10240. Rounds to 8192. - const result = calculateDefaultMemoryFromExpression(expr, context); + const result = calculateRunDynamicMemory(expr, context); expect(result).toBe(8192); }); @@ -30,26 +30,26 @@ describe('calculateDefaultMemoryFromExpression', () => { const context = { input: {}, runOptions: { timeoutSecs: 60 } }; const expr = 'runOptions.timeoutSecs * 100'; // 60 * 100 = 6000 // log2(6000) ~ 12.55. round(13) -> 2^13 = 8192 - const result = calculateDefaultMemoryFromExpression(expr, context); + const result = calculateRunDynamicMemory(expr, context); expect(result).toBe(8192); }); it('correctly handles a single number expression', () => { - const result = calculateDefaultMemoryFromExpression('2048', emptyContext); + const result = calculateRunDynamicMemory('2048', emptyContext); expect(result).toBe(2048); }); it('correctly handles expressions with custom get() function', () => { const context = { input: { nested: { value: 20 } }, runOptions: {} }; const expr = "get(input, 'nested.value', 10) * 50"; // 20 * 50 = 1000 - const result = calculateDefaultMemoryFromExpression(expr, context); + const result = calculateRunDynamicMemory(expr, context); expect(result).toBe(1024); }); it('should use get() default value when path is invalid', () => { const context = { input: { user: {} }, runOptions: {} }; const expr = "get(input, 'user.settings.memory', 512)"; - const result = calculateDefaultMemoryFromExpression(expr, context); + const result = calculateRunDynamicMemory(expr, context); expect(result).toBe(512); }); @@ -78,7 +78,7 @@ describe('calculateDefaultMemoryFromExpression', () => { '$desc', ({ expression }) => { // in case operation is not supported, mathjs will throw - expect(calculateDefaultMemoryFromExpression(expression, context)).toBeDefined(); + expect(calculateRunDynamicMemory(expression, context)).toBeDefined(); }, ); }); @@ -88,28 +88,28 @@ describe('calculateDefaultMemoryFromExpression', () => { it('should throw error if variable doesn\'t start with .runOptions or .input', () => { const context = { input: {}, runOptions: { memoryMbytes: 16 } }; const expr = '{{unexistingVariable}} * 1024'; - expect(() => calculateDefaultMemoryFromExpression(expr, context)) + expect(() => calculateRunDynamicMemory(expr, context)) .toThrow(`Invalid variable '{{unexistingVariable}}' in expression. Variables must start with 'input.' or 'runOptions.'.`); }); it('correctly replaces {{runOptions.variable}} with valid runOptions.variable', () => { const context = { input: {}, runOptions: { memoryMbytes: 16 } }; const expr = '{{runOptions.memoryMbytes}} * 1024'; - const result = calculateDefaultMemoryFromExpression(expr, context); + const result = calculateRunDynamicMemory(expr, context); expect(result).toBe(16384); }); it('correctly replaces {{input.variable}} with valid input.variable', () => { const context = { input: { value: 16 }, runOptions: { } }; const expr = '{{input.value}} * 1024'; - const result = calculateDefaultMemoryFromExpression(expr, context); + const result = calculateRunDynamicMemory(expr, context); expect(result).toBe(16384); }); it('should throw error if runOptions variable is invalid', () => { const context = { input: { value: 16 }, runOptions: { } }; const expr = '{{runOptions.customVariable}} * 1024'; - expect(() => calculateDefaultMemoryFromExpression(expr, context)) + expect(() => calculateRunDynamicMemory(expr, context)) .toThrow(`Invalid variable '{{runOptions.customVariable}}' in expression. Only the following runOptions are allowed:`); }); }); @@ -117,23 +117,23 @@ describe('calculateDefaultMemoryFromExpression', () => { describe('Rounding Logic', () => { it('should round down (e.g., 10240 -> 8192)', () => { // 2^13 = 8192, 2^14 = 16384. - const result = calculateDefaultMemoryFromExpression('10240', emptyContext); + const result = calculateRunDynamicMemory('10240', emptyContext); expect(result).toBe(8192); }); it('should round up (e.g., 13000 -> 16384)', () => { // 13000 is closer to 16384 than 8192. - const result = calculateDefaultMemoryFromExpression('13000', emptyContext); + const result = calculateRunDynamicMemory('13000', emptyContext); expect(result).toBe(16384); }); it('should clamp to the minimum memory limit if the result is too low', () => { - const result = calculateDefaultMemoryFromExpression('64', emptyContext); + const result = calculateRunDynamicMemory('64', emptyContext); expect(result).toBe(128); }); it('should clamp to the maximum memory limit if the result is too high', () => { - const result = calculateDefaultMemoryFromExpression('100000', emptyContext); + const result = calculateRunDynamicMemory('100000', emptyContext); expect(result).toBe(32768); }); }); @@ -141,34 +141,34 @@ describe('calculateDefaultMemoryFromExpression', () => { describe('Invalid/Error Handling', () => { it('should throw an error if expression length is greater than DEFAULT_MEMORY_MBYTES_MAX_CHARS', () => { const expr = '1'.repeat(DEFAULT_MEMORY_MBYTES_MAX_CHARS + 1); // Assuming max length is 1000 - expect(() => calculateDefaultMemoryFromExpression(expr, emptyContext)) + expect(() => calculateRunDynamicMemory(expr, emptyContext)) .toThrow(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_MAX_CHARS} characters.`); }); it('should throw an error for invalid syntax', () => { const expr = '1 +* 2'; - expect(() => calculateDefaultMemoryFromExpression(expr, emptyContext)) + expect(() => calculateRunDynamicMemory(expr, emptyContext)) .toThrow(); }); it('should throw error if result is 0', () => { - expect(() => calculateDefaultMemoryFromExpression('10 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: 0`); + expect(() => calculateRunDynamicMemory('10 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: 0`); }); it('should throw error if result is negative', () => { - expect(() => calculateDefaultMemoryFromExpression('5 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: -5`); + expect(() => calculateRunDynamicMemory('5 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: -5`); }); it('should throw error if result is NaN', () => { - expect(() => calculateDefaultMemoryFromExpression('0 / 0', emptyContext)).toThrow('Failed to round number to a power of 2.'); + expect(() => calculateRunDynamicMemory('0 / 0', emptyContext)).toThrow('Failed to round number to a power of 2.'); }); it('should throw error if result is a non-numeric (string)', () => { - expect(() => calculateDefaultMemoryFromExpression("'hello'", emptyContext)).toThrow('Failed to round number to a power of 2.'); + expect(() => calculateRunDynamicMemory("'hello'", emptyContext)).toThrow('Failed to round number to a power of 2.'); }); it('should throw error when disabled functionality of MathJS is used', () => { - expect(() => calculateDefaultMemoryFromExpression('evaluate(512)', emptyContext)).toThrow('Function evaluate is disabled'); + expect(() => calculateRunDynamicMemory('evaluate(512)', emptyContext)).toThrow('Function evaluate is disabled'); }); }); @@ -185,20 +185,20 @@ describe('calculateDefaultMemoryFromExpression', () => { expect(cache.length()).toBe(0); // First call - cache miss - const result1 = calculateDefaultMemoryFromExpression(expr, context, { cache }); + const result1 = calculateRunDynamicMemory(expr, context, { cache }); expect(result1).toBe(8192); expect(cache.length()).toBe(1); // Expression is now cached // Second call - cache hit - const result2 = calculateDefaultMemoryFromExpression(expr, context, { cache }); + const result2 = calculateRunDynamicMemory(expr, context, { cache }); expect(result2).toBe(8192); expect(cache.length()).toBe(1); // Cache length is unchanged }); it('should cache different expressions separately', () => { const expr2 = 'input.size * 2048'; // 10 * 2048 = 20480 -> 16384 - calculateDefaultMemoryFromExpression(expr, context, { cache }); - calculateDefaultMemoryFromExpression(expr2, context, { cache }); + calculateRunDynamicMemory(expr, context, { cache }); + calculateRunDynamicMemory(expr2, context, { cache }); expect(cache.length()).toBe(2); }); }); From a23f3860e9fecb6c5d392f04eb24b5e6a86c67bd Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Fri, 21 Nov 2025 10:20:48 +0100 Subject: [PATCH 15/25] refactor: clean up --- .../CHANGELOG.md | 0 .../package.json | 2 +- .../src/index.ts | 0 .../src/memory_calculator.ts | 95 ++++++++++--------- packages/actor-memory-expression/src/types.ts | 14 +++ .../tsconfig.build.json | 0 .../tsconfig.json | 0 .../tsup.config.ts | 0 test/memory_calculator.test.ts | 58 +++++------ 9 files changed, 94 insertions(+), 75 deletions(-) rename packages/{math-utils => actor-memory-expression}/CHANGELOG.md (100%) rename packages/{math-utils => actor-memory-expression}/package.json (96%) rename packages/{math-utils => actor-memory-expression}/src/index.ts (100%) rename packages/{math-utils => actor-memory-expression}/src/memory_calculator.ts (74%) create mode 100644 packages/actor-memory-expression/src/types.ts rename packages/{math-utils => actor-memory-expression}/tsconfig.build.json (100%) rename packages/{math-utils => actor-memory-expression}/tsconfig.json (100%) rename packages/{math-utils => actor-memory-expression}/tsup.config.ts (100%) diff --git a/packages/math-utils/CHANGELOG.md b/packages/actor-memory-expression/CHANGELOG.md similarity index 100% rename from packages/math-utils/CHANGELOG.md rename to packages/actor-memory-expression/CHANGELOG.md diff --git a/packages/math-utils/package.json b/packages/actor-memory-expression/package.json similarity index 96% rename from packages/math-utils/package.json rename to packages/actor-memory-expression/package.json index 36f8f42b6..ec5c0e39f 100644 --- a/packages/math-utils/package.json +++ b/packages/actor-memory-expression/package.json @@ -1,5 +1,5 @@ { - "name": "@apify/math-utils", + "name": "@apify/actor-memory-expression", "version": "0.0.1", "description": "Mathematical and numerical utility functions.", "main": "./dist/cjs/index.cjs", diff --git a/packages/math-utils/src/index.ts b/packages/actor-memory-expression/src/index.ts similarity index 100% rename from packages/math-utils/src/index.ts rename to packages/actor-memory-expression/src/index.ts diff --git a/packages/math-utils/src/memory_calculator.ts b/packages/actor-memory-expression/src/memory_calculator.ts similarity index 74% rename from packages/math-utils/src/memory_calculator.ts rename to packages/actor-memory-expression/src/memory_calculator.ts index f4ac87b94..bb22833ed 100644 --- a/packages/math-utils/src/memory_calculator.ts +++ b/packages/actor-memory-expression/src/memory_calculator.ts @@ -21,23 +21,12 @@ import { import { ACTOR_LIMITS } from '@apify/consts'; import type { LruCache } from '../../datastructures/src/lru_cache'; +import type { ActorRunOptions, MemoryEvaluationContext } from './types.js'; -type ActorRunOptions = { - build?: string; - timeoutSecs?: number; - memoryMbytes?: number; // probably no one will need it, but let's keep it consistent - diskMbytes?: number; // probably no one will need it, but let's keep it consistent - maxItems?: number; - maxTotalChargeUsd?: number; - restartOnError?: boolean; -} - -type MemoryEvaluationContext = { - runOptions: ActorRunOptions; - input: Record; -} - -export const DEFAULT_MEMORY_MBYTES_MAX_CHARS = 1000; +// In theory, users could create expressions longer than 1000 characters, +// but in practice, it's unlikely anyone would need that much complexity. +// Later we can increase this limit if needed. +export const DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH = 1000; /** * A Set of allowed keys from ActorRunOptions that can be used in @@ -58,14 +47,18 @@ const ALLOWED_RUN_OPTION_KEYS = new Set([ * MathJS security recommendations: https://mathjs.org/docs/expressions/security.html */ const math = create({ + // expression dependencies + // Required for compiling and evaluating root expressions. + // We disable it below to prevent users from calling `evaluate()` inside their expressions. + // For example: defaultMemoryMbytes = "evaluate('2 + 2')" + compileDependencies, + evaluateDependencies, + // arithmetic dependencies addDependencies, subtractDependencies, multiplyDependencies, divideDependencies, - // expression dependencies - compileDependencies, - evaluateDependencies, // statistics dependencies maxDependencies, minDependencies, @@ -77,8 +70,7 @@ const math = create({ // without that dependency 'null ?? 5', won't work nullishDependencies, }); -const limitedEvaluate = math.evaluate; -const limitedCompile = math.compile; +const { compile } = math; // Disable potentially dangerous functions math.import({ @@ -88,6 +80,8 @@ math.import({ reviver() { throw new Error('Function reviver is disabled'); }, // extra (has functional impact) + // We disable evaluate to prevent users from calling it inside their expressions. + // For example: defaultMemoryMbytes = "evaluate('2 + 2')" evaluate() { throw new Error('Function evaluate is disabled'); }, parse() { throw new Error('Function parse is disabled'); }, simplify() { throw new Error('Function simplify is disabled'); }, @@ -118,7 +112,7 @@ const customGetFunc = (obj: any, path: string, defaultVal?: number) => { */ const roundToClosestPowerOf2 = (num: number): number | undefined => { if (typeof num !== 'number' || Number.isNaN(num)) { - throw new Error(`Failed to round number to a power of 2.`); + throw new Error(`Calculated memory value is not a valid number: ${num}.`); } // Handle 0 or negative values. @@ -135,8 +129,14 @@ const roundToClosestPowerOf2 = (num: number): number | undefined => { }; /** - * Replaces `{{variable}}` placeholders in an expression string with the variable name. - * Enforces strict validation to allow `{{input.*}}` paths or whitelisted `{{runOptions.*}}` keys. + * Replaces all `{{variable}}` placeholders in an expression into direct + * property access (e.g. `{{runOptions.memoryMbytes}}` → `runOptions.memoryMbytes`). + * + * Only variables starting with `input.` or whitelisted `runOptions.` keys are allowed. + * All `input.*` values are accepted, while `runOptions.*` are validated (only 7 variables - ALLOWED_RUN_OPTION_KEYS). + * + * Note: this approach allows developers to use a consistent double-brace + * syntax (`{{runOptions.timeoutSecs}}`) across the platform. * * @example * // Returns "runOptions.memoryMbytes + 1024" @@ -145,26 +145,25 @@ const roundToClosestPowerOf2 = (num: number): number | undefined => { * @param defaultMemoryMbytes The raw string expression, e.g., "{{runOptions.memoryMbytes}} * 2". * @returns A safe, processed expression for evaluation, e.g., "runOptions.memoryMbytes * 2". */ -const preprocessRunMemoryExpression = (defaultMemoryMbytes: string): string => { +const processTemplateVariables = (defaultMemoryMbytes: string): string => { const variableRegex = /{{\s*([a-zA-Z0-9_.]+)\s*}}/g; const processedExpression = defaultMemoryMbytes.replace( variableRegex, (_, variableName: string) => { - // 1. Validate that the variable starts with either 'input.' or 'runOptions.' if (!variableName.startsWith('runOptions.') && !variableName.startsWith('input.')) { throw new Error( `Invalid variable '{{${variableName}}}' in expression. Variables must start with 'input.' or 'runOptions.'.`, ); } - // 2. Check if the variable is accessing input (e.g. {{input.someValue}}) - // We do not validate the specific property name because input is dynamic. + // 1. Check if the variable is accessing input (e.g. {{input.someValue}}) + // We do not validate the specific property name because `input` is dynamic. if (variableName.startsWith('input.')) { return variableName; } - // 3. Check if the variable is accessing runOptions (e.g. {{runOptions.memoryMbytes}}) and validate the keys. + // 2. Check if the variable is accessing runOptions (e.g. {{runOptions.memoryMbytes}}) and validate the keys. if (variableName.startsWith('runOptions.')) { const key = variableName.slice('runOptions.'.length); if (!ALLOWED_RUN_OPTION_KEYS.has(key as keyof ActorRunOptions)) { @@ -185,11 +184,26 @@ const preprocessRunMemoryExpression = (defaultMemoryMbytes: string): string => { return processedExpression; }; +const getCompiledExpression = (expression: string, cache: LruCache | undefined): EvalFunction => { + if (!cache) { + return compile(expression); + } + + let compiledExpression = cache.get(expression); + + if (!compiledExpression) { + compiledExpression = compile(expression); + cache.add(expression, compiledExpression!); + } + + return compiledExpression; +}; + /** * Evaluates a dynamic memory expression string using the provided context. * Result is rounded to the closest power of 2 and clamped within allowed limits. * - * @param defaultMemoryMbytes The string expression to evaluate (e.g., "get(input, 'size', 10) * 1024"). + * @param defaultMemoryMbytes The string expression to evaluate (e.g., `get(input, 'urls.length', 10) * 1024` for `input = { urls: ['url1', 'url2'] }`). * @param context The `MemoryEvaluationContext` (containing `input` and `runOptions`) available to the expression. * @returns The calculated memory value rounded to the closest power of 2 clamped within allowed limits. */ @@ -198,33 +212,22 @@ export const calculateRunDynamicMemory = ( context: MemoryEvaluationContext, options: { cache: LruCache } | undefined = undefined, ) => { - if (defaultMemoryMbytes.length > DEFAULT_MEMORY_MBYTES_MAX_CHARS) { - throw new Error(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_MAX_CHARS} characters.`); + if (defaultMemoryMbytes.length > DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH) { + throw new Error(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH} characters.`); } // Replaces all occurrences of {{variable}} with variable // e.g., "{{runOptions.memoryMbytes}} + 1024" becomes "runOptions.memoryMbytes + 1024" - const preprocessedExpression = preprocessRunMemoryExpression(defaultMemoryMbytes); + const preprocessedExpression = processTemplateVariables(defaultMemoryMbytes); const preparedContext = { ...context, get: customGetFunc, }; - let finalResult: number | { entries: number[] }; - - if (options?.cache) { - let compiledExpr = options.cache.get(preprocessedExpression); + const compiledExpression = getCompiledExpression(preprocessedExpression, options?.cache); - if (!compiledExpr) { - compiledExpr = limitedCompile(preprocessedExpression); - options.cache.add(preprocessedExpression, compiledExpr!); - } - - finalResult = compiledExpr.evaluate(preparedContext); - } else { - finalResult = limitedEvaluate(preprocessedExpression, preparedContext); - } + let finalResult: number | { entries: number[] } = compiledExpression.evaluate(preparedContext); // Mathjs wraps multi-line expressions in an object, so we need to extract the last entry. // Note: one-line expressions return a number directly. diff --git a/packages/actor-memory-expression/src/types.ts b/packages/actor-memory-expression/src/types.ts new file mode 100644 index 000000000..0273e836a --- /dev/null +++ b/packages/actor-memory-expression/src/types.ts @@ -0,0 +1,14 @@ +export type ActorRunOptions = { + build?: string; + timeoutSecs?: number; + memoryMbytes?: number; // probably no one will need it, but let's keep it consistent + diskMbytes?: number; // probably no one will need it, but let's keep it consistent + maxItems?: number; + maxTotalChargeUsd?: number; + restartOnError?: boolean; +} + +export type MemoryEvaluationContext = { + runOptions: ActorRunOptions; + input: Record; +} diff --git a/packages/math-utils/tsconfig.build.json b/packages/actor-memory-expression/tsconfig.build.json similarity index 100% rename from packages/math-utils/tsconfig.build.json rename to packages/actor-memory-expression/tsconfig.build.json diff --git a/packages/math-utils/tsconfig.json b/packages/actor-memory-expression/tsconfig.json similarity index 100% rename from packages/math-utils/tsconfig.json rename to packages/actor-memory-expression/tsconfig.json diff --git a/packages/math-utils/tsup.config.ts b/packages/actor-memory-expression/tsup.config.ts similarity index 100% rename from packages/math-utils/tsup.config.ts rename to packages/actor-memory-expression/tsup.config.ts diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index e5457b2f9..db61c22fb 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -1,7 +1,7 @@ import type { EvalFunction } from 'mathjs'; +import { calculateRunDynamicMemory, DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH } from '@apify/actor-memory-expression'; import { LruCache } from '@apify/datastructures'; -import { calculateRunDynamicMemory, DEFAULT_MEMORY_MBYTES_MAX_CHARS } from '@apify/math-utils'; describe('calculateDefaultMemoryFromExpression', () => { const emptyContext = { input: {}, runOptions: {} }; @@ -59,54 +59,56 @@ describe('calculateDefaultMemoryFromExpression', () => { runOptions: { timeoutSecs: 60, memoryMbytes: 512 }, }; + // Note: all results are rounded to the closest power of 2 and clamped within limits. const cases = [ - { expression: '5 + 5', desc: '+ allowed' }, - { expression: '6 - 5', desc: '- allowed' }, - { expression: '5 / 5', desc: '/ allowed' }, - { expression: '5 * 5', desc: '* allowed' }, - { expression: 'max(1, 2, 3)', desc: 'max() allowed' }, - { expression: 'min(1, 2, 3)', desc: 'min() allowed' }, - { expression: '(true and false) ? 0 : 5', desc: 'and allowed' }, - { expression: '(true or false) ? 5 : 0', desc: 'or allowed' }, - { expression: '(true xor false) ? 5 : 0', desc: 'xor allowed' }, - { expression: 'not(false) ? 5 : 0', desc: 'not allowed' }, - { expression: 'null ?? 256', desc: 'nullish coalescing allowed' }, - { expression: 'a = 5', desc: 'variable assignment' }, + { expression: '128 + 5', result: 128, name: '+' }, + { expression: '128 - 5', result: 128, name: '-' }, + { expression: '128 / 5', result: 128, name: '/' }, + { expression: '128 * 5', result: 512, name: '*' }, + { expression: 'max(128, 2, 3)', result: 128, name: 'max()' }, + { expression: 'min(128, 512, 1024)', result: 128, name: 'min()' }, + { expression: '(true and false) ? 0 : 128', result: 128, name: 'and' }, + { expression: '(true or false) ? 128 : 0', result: 128, name: 'or' }, + { expression: '(true xor false) ? 128 : 0', result: 128, name: 'xor' }, + { expression: 'not(false) ? 128 : 0', result: 128, name: 'not' }, + { expression: 'null ?? 256', result: 256, name: 'nullish coalescing' }, + { expression: 'a = 128', result: 128, name: '=' }, ]; it.each(cases)( - '$desc', - ({ expression }) => { + `supports operation '$name'`, + ({ expression, result }) => { // in case operation is not supported, mathjs will throw - expect(calculateRunDynamicMemory(expression, context)).toBeDefined(); + // we round the result to the closest power of 2 and clamp within limits. + expect(calculateRunDynamicMemory(expression, context)).toBe(result); }, ); }); }); - describe('Preprocessing with {{variable}}', () => { - it('should throw error if variable doesn\'t start with .runOptions or .input', () => { + describe('Template {{variables}} support', () => { + it('should throw error if variable doesn\'t start with runOptions. or input.', () => { const context = { input: {}, runOptions: { memoryMbytes: 16 } }; - const expr = '{{unexistingVariable}} * 1024'; + const expr = '{{nonexistentVariable}} * 1024'; expect(() => calculateRunDynamicMemory(expr, context)) - .toThrow(`Invalid variable '{{unexistingVariable}}' in expression. Variables must start with 'input.' or 'runOptions.'.`); + .toThrow(`Invalid variable '{{nonexistentVariable}}' in expression. Variables must start with 'input.' or 'runOptions.'.`); }); - it('correctly replaces {{runOptions.variable}} with valid runOptions.variable', () => { + it('correctly evaluates valid runOptions property', () => { const context = { input: {}, runOptions: { memoryMbytes: 16 } }; const expr = '{{runOptions.memoryMbytes}} * 1024'; const result = calculateRunDynamicMemory(expr, context); expect(result).toBe(16384); }); - it('correctly replaces {{input.variable}} with valid input.variable', () => { + it('correctly evaluates input property', () => { const context = { input: { value: 16 }, runOptions: { } }; const expr = '{{input.value}} * 1024'; const result = calculateRunDynamicMemory(expr, context); expect(result).toBe(16384); }); - it('should throw error if runOptions variable is invalid', () => { + it('should throw error if runOptions property is not supported', () => { const context = { input: { value: 16 }, runOptions: { } }; const expr = '{{runOptions.customVariable}} * 1024'; expect(() => calculateRunDynamicMemory(expr, context)) @@ -114,7 +116,7 @@ describe('calculateDefaultMemoryFromExpression', () => { }); }); - describe('Rounding Logic', () => { + describe('Rounding logic', () => { it('should round down (e.g., 10240 -> 8192)', () => { // 2^13 = 8192, 2^14 = 16384. const result = calculateRunDynamicMemory('10240', emptyContext); @@ -140,9 +142,9 @@ describe('calculateDefaultMemoryFromExpression', () => { describe('Invalid/Error Handling', () => { it('should throw an error if expression length is greater than DEFAULT_MEMORY_MBYTES_MAX_CHARS', () => { - const expr = '1'.repeat(DEFAULT_MEMORY_MBYTES_MAX_CHARS + 1); // Assuming max length is 1000 + const expr = '1'.repeat(DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH + 1); // Assuming max length is 1000 expect(() => calculateRunDynamicMemory(expr, emptyContext)) - .toThrow(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_MAX_CHARS} characters.`); + .toThrow(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH} characters.`); }); it('should throw an error for invalid syntax', () => { @@ -160,11 +162,11 @@ describe('calculateDefaultMemoryFromExpression', () => { }); it('should throw error if result is NaN', () => { - expect(() => calculateRunDynamicMemory('0 / 0', emptyContext)).toThrow('Failed to round number to a power of 2.'); + expect(() => calculateRunDynamicMemory('0 / 0', emptyContext)).toThrow('Calculated memory value is not a valid number: NaN.'); }); it('should throw error if result is a non-numeric (string)', () => { - expect(() => calculateRunDynamicMemory("'hello'", emptyContext)).toThrow('Failed to round number to a power of 2.'); + expect(() => calculateRunDynamicMemory("'hello'", emptyContext)).toThrow('Calculated memory value is not a valid number: hello.'); }); it('should throw error when disabled functionality of MathJS is used', () => { From 543f278bfee1c779ed602b44498cec1dfff6d34c Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Fri, 21 Nov 2025 10:59:00 +0100 Subject: [PATCH 16/25] refactor: clean up --- .../src/memory_calculator.ts | 30 +++++++++++-------- test/memory_calculator.test.ts | 14 ++++----- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/packages/actor-memory-expression/src/memory_calculator.ts b/packages/actor-memory-expression/src/memory_calculator.ts index bb22833ed..f4ba20142 100644 --- a/packages/actor-memory-expression/src/memory_calculator.ts +++ b/packages/actor-memory-expression/src/memory_calculator.ts @@ -75,25 +75,25 @@ const { compile } = math; // Disable potentially dangerous functions math.import({ // most important (hardly any functional impact) - import() { throw new Error('Function import is disabled'); }, - createUnit() { throw new Error('Function createUnit is disabled'); }, - reviver() { throw new Error('Function reviver is disabled'); }, + import() { throw new Error('Function import is disabled.'); }, + createUnit() { throw new Error('Function createUnit is disabled.'); }, + reviver() { throw new Error('Function reviver is disabled.'); }, // extra (has functional impact) // We disable evaluate to prevent users from calling it inside their expressions. // For example: defaultMemoryMbytes = "evaluate('2 + 2')" - evaluate() { throw new Error('Function evaluate is disabled'); }, - parse() { throw new Error('Function parse is disabled'); }, - simplify() { throw new Error('Function simplify is disabled'); }, - derivative() { throw new Error('Function derivative is disabled'); }, - resolve() { throw new Error('Function resolve is disabled'); }, + evaluate() { throw new Error('Function evaluate is disabled.'); }, + parse() { throw new Error('Function parse is disabled.'); }, + simplify() { throw new Error('Function simplify is disabled.'); }, + derivative() { throw new Error('Function derivative is disabled.'); }, + resolve() { throw new Error('Function resolve is disabled.'); }, }, { override: true }); /** * Safely retrieves a nested property from an object using a dot-notation string path. * * This is custom function designed to be injected into the math expression evaluator, - * allowing expressions like `get(input, 'user.settings.memory', 512)`. + * allowing expressions like `get(input, 'user.settings.memory', 512)` or `get(input, 'startUrls.length', 10)` to get array length. * * @param obj The source object to search within. * @param path A dot-separated string representing the nested path (e.g., "input.payload.size"). @@ -117,7 +117,7 @@ const roundToClosestPowerOf2 = (num: number): number | undefined => { // Handle 0 or negative values. if (num <= 0) { - throw new Error(`Calculated memory value must be a positive number, greater than 0, got: ${num}`); + throw new Error(`Calculated memory value must be a positive number, greater than 0, got: ${num}.`); } const log2n = Math.log2(num); @@ -132,11 +132,10 @@ const roundToClosestPowerOf2 = (num: number): number | undefined => { * Replaces all `{{variable}}` placeholders in an expression into direct * property access (e.g. `{{runOptions.memoryMbytes}}` → `runOptions.memoryMbytes`). * - * Only variables starting with `input.` or whitelisted `runOptions.` keys are allowed. * All `input.*` values are accepted, while `runOptions.*` are validated (only 7 variables - ALLOWED_RUN_OPTION_KEYS). * * Note: this approach allows developers to use a consistent double-brace - * syntax (`{{runOptions.timeoutSecs}}`) across the platform. + * syntax `{{runOptions.timeoutSecs}}` across the platform. * * @example * // Returns "runOptions.memoryMbytes + 1024" @@ -184,6 +183,13 @@ const processTemplateVariables = (defaultMemoryMbytes: string): string => { return processedExpression; }; +/* +* Retrieves a compiled expression from the cache or compiles it if not present. +* +* @param expression The expression string to compile. +* @param cache An optional cache to store/retrieve compiled expressions. +* @returns The compiled EvalFunction. +*/ const getCompiledExpression = (expression: string, cache: LruCache | undefined): EvalFunction => { if (!cache) { return compile(expression); diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index db61c22fb..903305e2b 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -6,7 +6,7 @@ import { LruCache } from '@apify/datastructures'; describe('calculateDefaultMemoryFromExpression', () => { const emptyContext = { input: {}, runOptions: {} }; - describe('Basic Evaluation', () => { + describe('Basic evaluation', () => { it('correctly calculates and rounds memory from one-line expression', () => { const context = { input: { size: 10 }, runOptions: {} }; // 10 * 1024 = 10240. log2(10240) ~ 13.32. round(13.32) -> 2^13 = 8192 @@ -91,7 +91,7 @@ describe('calculateDefaultMemoryFromExpression', () => { const context = { input: {}, runOptions: { memoryMbytes: 16 } }; const expr = '{{nonexistentVariable}} * 1024'; expect(() => calculateRunDynamicMemory(expr, context)) - .toThrow(`Invalid variable '{{nonexistentVariable}}' in expression. Variables must start with 'input.' or 'runOptions.'.`); + .toThrow(`Invalid variable '{{nonexistentVariable}}' in expression. Variables must start with 'input.' or 'runOptions.'`); }); it('correctly evaluates valid runOptions property', () => { @@ -140,9 +140,9 @@ describe('calculateDefaultMemoryFromExpression', () => { }); }); - describe('Invalid/Error Handling', () => { + describe('Invalid/error handling', () => { it('should throw an error if expression length is greater than DEFAULT_MEMORY_MBYTES_MAX_CHARS', () => { - const expr = '1'.repeat(DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH + 1); // Assuming max length is 1000 + const expr = '1'.repeat(DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH + 1); expect(() => calculateRunDynamicMemory(expr, emptyContext)) .toThrow(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH} characters.`); }); @@ -154,11 +154,11 @@ describe('calculateDefaultMemoryFromExpression', () => { }); it('should throw error if result is 0', () => { - expect(() => calculateRunDynamicMemory('10 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: 0`); + expect(() => calculateRunDynamicMemory('10 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: 0.`); }); it('should throw error if result is negative', () => { - expect(() => calculateRunDynamicMemory('5 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: -5`); + expect(() => calculateRunDynamicMemory('5 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: -5.`); }); it('should throw error if result is NaN', () => { @@ -170,7 +170,7 @@ describe('calculateDefaultMemoryFromExpression', () => { }); it('should throw error when disabled functionality of MathJS is used', () => { - expect(() => calculateRunDynamicMemory('evaluate(512)', emptyContext)).toThrow('Function evaluate is disabled'); + expect(() => calculateRunDynamicMemory('evaluate(512)', emptyContext)).toThrow('Function evaluate is disabled.'); }); }); From f0d1f54d3d254d5f472c8b3fd087dd90d487fe37 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Fri, 21 Nov 2025 16:37:19 +0100 Subject: [PATCH 17/25] refactor: make cache generic --- .../src/memory_calculator.ts | 14 +++++++------- packages/actor-memory-expression/src/types.ts | 8 ++++++++ test/memory_calculator.test.ts | 10 ++++++++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/actor-memory-expression/src/memory_calculator.ts b/packages/actor-memory-expression/src/memory_calculator.ts index f4ba20142..2cea3d07b 100644 --- a/packages/actor-memory-expression/src/memory_calculator.ts +++ b/packages/actor-memory-expression/src/memory_calculator.ts @@ -20,8 +20,7 @@ import { import { ACTOR_LIMITS } from '@apify/consts'; -import type { LruCache } from '../../datastructures/src/lru_cache'; -import type { ActorRunOptions, MemoryEvaluationContext } from './types.js'; +import type { ActorRunOptions, CompilationCache, MemoryEvaluationContext } from './types.js'; // In theory, users could create expressions longer than 1000 characters, // but in practice, it's unlikely anyone would need that much complexity. @@ -110,7 +109,7 @@ const customGetFunc = (obj: any, path: string, defaultVal?: number) => { * @param num The number to round. * @returns The closest power of 2 within min/max range. */ -const roundToClosestPowerOf2 = (num: number): number | undefined => { +const roundToClosestPowerOf2 = (num: number): number => { if (typeof num !== 'number' || Number.isNaN(num)) { throw new Error(`Calculated memory value is not a valid number: ${num}.`); } @@ -190,7 +189,7 @@ const processTemplateVariables = (defaultMemoryMbytes: string): string => { * @param cache An optional cache to store/retrieve compiled expressions. * @returns The compiled EvalFunction. */ -const getCompiledExpression = (expression: string, cache: LruCache | undefined): EvalFunction => { +const getCompiledExpression = (expression: string, cache: CompilationCache | undefined): EvalFunction => { if (!cache) { return compile(expression); } @@ -199,7 +198,7 @@ const getCompiledExpression = (expression: string, cache: LruCache if (!compiledExpression) { compiledExpression = compile(expression); - cache.add(expression, compiledExpression!); + cache.set(expression, compiledExpression!); } return compiledExpression; @@ -211,12 +210,13 @@ const getCompiledExpression = (expression: string, cache: LruCache * * @param defaultMemoryMbytes The string expression to evaluate (e.g., `get(input, 'urls.length', 10) * 1024` for `input = { urls: ['url1', 'url2'] }`). * @param context The `MemoryEvaluationContext` (containing `input` and `runOptions`) available to the expression. - * @returns The calculated memory value rounded to the closest power of 2 clamped within allowed limits. + * @param options.cache Optional synchronous cache. Since compiled functions cannot be saved to a database/Redis, they are kept in local memory. + * @returns The calculated memory value rounded to the closest power of 2 and clamped within allowed limits. */ export const calculateRunDynamicMemory = ( defaultMemoryMbytes: string, context: MemoryEvaluationContext, - options: { cache: LruCache } | undefined = undefined, + options: { cache: CompilationCache } | undefined = undefined, ) => { if (defaultMemoryMbytes.length > DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH) { throw new Error(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH} characters.`); diff --git a/packages/actor-memory-expression/src/types.ts b/packages/actor-memory-expression/src/types.ts index 0273e836a..f3e1170c6 100644 --- a/packages/actor-memory-expression/src/types.ts +++ b/packages/actor-memory-expression/src/types.ts @@ -1,3 +1,5 @@ +import type { EvalFunction } from 'mathjs'; + export type ActorRunOptions = { build?: string; timeoutSecs?: number; @@ -12,3 +14,9 @@ export type MemoryEvaluationContext = { runOptions: ActorRunOptions; input: Record; } + +export type CompilationCache = { + get: (expression: string) => EvalFunction | null; + set: (expression: string, compilationResult: EvalFunction) => void; + length: () => number; +} diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index 903305e2b..175d3fb64 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -1,4 +1,5 @@ import type { EvalFunction } from 'mathjs'; +import type { CompilationCache } from 'packages/actor-memory-expression/src/types'; import { calculateRunDynamicMemory, DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH } from '@apify/actor-memory-expression'; import { LruCache } from '@apify/datastructures'; @@ -175,12 +176,17 @@ describe('calculateDefaultMemoryFromExpression', () => { }); describe('Caching', () => { - let cache: LruCache; + let cache: CompilationCache; const context = { input: { size: 10 }, runOptions: {} }; const expr = 'input.size * 1024'; beforeEach(() => { - cache = new LruCache({ maxLength: 10 }); + const lruCache = new LruCache({ maxLength: 10 }); + cache = { + get: (expression: string) => lruCache.get(expression), + set: (expression: string, compilationResult: EvalFunction) => lruCache.add(expression, compilationResult), + length: () => lruCache.length(), + }; }); it('correctly works with cache passed in options', () => { From 89543ce47b362bd80086fecf0099cf1e785a57ee Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Fri, 21 Nov 2025 16:38:33 +0100 Subject: [PATCH 18/25] fix: build --- package-lock.json | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 421d281e9..4ff955a76 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,6 +37,10 @@ "underscore": "^1.13.7" } }, + "node_modules/@apify/actor-memory-expression": { + "resolved": "packages/actor-memory-expression", + "link": true + }, "node_modules/@apify/consts": { "resolved": "packages/consts", "link": true @@ -117,10 +121,6 @@ "resolved": "packages/markdown", "link": true }, - "node_modules/@apify/math-utils": { - "resolved": "packages/math-utils", - "link": true - }, "node_modules/@apify/payment_qr_codes": { "resolved": "packages/payment_qr_codes", "link": true @@ -19426,6 +19426,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "packages/actor-memory-expression": { + "name": "@apify/actor-memory-expression", + "version": "0.0.1", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.47.0", + "@apify/log": "^2.5.26", + "mathjs": "^15.1.0" + } + }, "packages/consts": { "name": "@apify/consts", "version": "2.47.0", @@ -19542,6 +19552,7 @@ "packages/math-utils": { "name": "@apify/math-utils", "version": "0.0.1", + "extraneous": true, "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.47.0", From 805365c185425ff0d1207a49b9eba41e8e01e726 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Fri, 21 Nov 2025 16:48:33 +0100 Subject: [PATCH 19/25] refactor: clean up --- packages/actor-memory-expression/package.json | 2 +- packages/actor-memory-expression/src/memory_calculator.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/actor-memory-expression/package.json b/packages/actor-memory-expression/package.json index ec5c0e39f..8e6dcc819 100644 --- a/packages/actor-memory-expression/package.json +++ b/packages/actor-memory-expression/package.json @@ -1,7 +1,7 @@ { "name": "@apify/actor-memory-expression", "version": "0.0.1", - "description": "Mathematical and numerical utility functions.", + "description": "Utility to evaluate dynamic memory expressions for Apify actors.", "main": "./dist/cjs/index.cjs", "module": "./dist/esm/index.mjs", "typings": "./dist/cjs/index.d.ts", diff --git a/packages/actor-memory-expression/src/memory_calculator.ts b/packages/actor-memory-expression/src/memory_calculator.ts index 2cea3d07b..fc0e551e2 100644 --- a/packages/actor-memory-expression/src/memory_calculator.ts +++ b/packages/actor-memory-expression/src/memory_calculator.ts @@ -131,7 +131,7 @@ const roundToClosestPowerOf2 = (num: number): number => { * Replaces all `{{variable}}` placeholders in an expression into direct * property access (e.g. `{{runOptions.memoryMbytes}}` → `runOptions.memoryMbytes`). * - * All `input.*` values are accepted, while `runOptions.*` are validated (only 7 variables - ALLOWED_RUN_OPTION_KEYS). + * All `input.*` values are accepted, while `runOptions.*` are validated (7 variables from ALLOWED_RUN_OPTION_KEYS). * * Note: this approach allows developers to use a consistent double-brace * syntax `{{runOptions.timeoutSecs}}` across the platform. From f6bdccc42d797e8157842a7a39756961d43fc284 Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Fri, 21 Nov 2025 16:50:29 +0100 Subject: [PATCH 20/25] refactor: clean up --- package-lock.json | 1 - package.json | 1 - packages/actor-memory-expression/tsconfig.json | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4ff955a76..58fd9eb4d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,7 +26,6 @@ "jest": "^29.7.0", "lerna": "^9.0.0", "lint-staged": "^16.0.0", - "mathjs": "^15.1.0", "nock": "^14.0.0", "strip-ansi": "^6.0.0", "ts-jest": "^29.2.4", diff --git a/package.json b/package.json index e2daac936..d857640b5 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,6 @@ "jest": "^29.7.0", "lerna": "^9.0.0", "lint-staged": "^16.0.0", - "mathjs": "^15.1.0", "nock": "^14.0.0", "strip-ansi": "^6.0.0", "ts-jest": "^29.2.4", diff --git a/packages/actor-memory-expression/tsconfig.json b/packages/actor-memory-expression/tsconfig.json index 595b76fe3..4a5606763 100644 --- a/packages/actor-memory-expression/tsconfig.json +++ b/packages/actor-memory-expression/tsconfig.json @@ -2,7 +2,7 @@ "extends": "../../tsconfig.json", "include": ["src/**/*"], "compilerOptions": { - "module": "node20", + "module": "node16", "moduleResolution": "node16", } } From e6c340ed4667b9542ae4620330f378e11e46874c Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Fri, 21 Nov 2025 16:51:46 +0100 Subject: [PATCH 21/25] refactor: clean up --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d857640b5..7ba873b9d 100644 --- a/package.json +++ b/package.json @@ -59,8 +59,8 @@ "clone-deep": "^4.0.1", "commitlint": "^20.0.0", "eslint": "^9.24.0", - "globals": "^16.0.0", "husky": "^9.1.4", + "globals": "^16.0.0", "jest": "^29.7.0", "lerna": "^9.0.0", "lint-staged": "^16.0.0", From 5ab9baf74b198d5c2d18e678d62ec2ca623c6fbd Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Fri, 21 Nov 2025 17:07:37 +0100 Subject: [PATCH 22/25] refactor: clean up --- packages/actor-memory-expression/tsconfig.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/actor-memory-expression/tsconfig.json b/packages/actor-memory-expression/tsconfig.json index 4a5606763..52d43eaaa 100644 --- a/packages/actor-memory-expression/tsconfig.json +++ b/packages/actor-memory-expression/tsconfig.json @@ -1,8 +1,4 @@ { "extends": "../../tsconfig.json", - "include": ["src/**/*"], - "compilerOptions": { - "module": "node16", - "moduleResolution": "node16", - } + "include": ["src/**/*"] } From 6a202d472c880a9465e8d95bae9a677792aac48b Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Mon, 24 Nov 2025 14:40:24 +0100 Subject: [PATCH 23/25] refactor: clean up --- .../src/memory_calculator.ts | 23 ++++--------- test/memory_calculator.test.ts | 32 ++++++++++++++++++- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/packages/actor-memory-expression/src/memory_calculator.ts b/packages/actor-memory-expression/src/memory_calculator.ts index fc0e551e2..10f77f891 100644 --- a/packages/actor-memory-expression/src/memory_calculator.ts +++ b/packages/actor-memory-expression/src/memory_calculator.ts @@ -73,19 +73,12 @@ const { compile } = math; // Disable potentially dangerous functions math.import({ - // most important (hardly any functional impact) - import() { throw new Error('Function import is disabled.'); }, - createUnit() { throw new Error('Function createUnit is disabled.'); }, - reviver() { throw new Error('Function reviver is disabled.'); }, - - // extra (has functional impact) // We disable evaluate to prevent users from calling it inside their expressions. // For example: defaultMemoryMbytes = "evaluate('2 + 2')" evaluate() { throw new Error('Function evaluate is disabled.'); }, + compile() { throw new Error('Function compile is disabled.'); }, + // We need to disable it, because compileDependencies imports parseDependencies. parse() { throw new Error('Function parse is disabled.'); }, - simplify() { throw new Error('Function simplify is disabled.'); }, - derivative() { throw new Error('Function derivative is disabled.'); }, - resolve() { throw new Error('Function resolve is disabled.'); }, }, { override: true }); /** @@ -133,8 +126,10 @@ const roundToClosestPowerOf2 = (num: number): number => { * * All `input.*` values are accepted, while `runOptions.*` are validated (7 variables from ALLOWED_RUN_OPTION_KEYS). * - * Note: this approach allows developers to use a consistent double-brace - * syntax `{{runOptions.timeoutSecs}}` across the platform. + * Note: While not really needed for Math.js, this approach allows developers + * to use a consistent double-brace templating syntax `{{runOptions.timeoutSecs}}` + * across the Apify platform. We also want to avoid compiling the expression with the + * actual values as that would make caching less effective. * * @example * // Returns "runOptions.memoryMbytes + 1024" @@ -149,12 +144,6 @@ const processTemplateVariables = (defaultMemoryMbytes: string): string => { const processedExpression = defaultMemoryMbytes.replace( variableRegex, (_, variableName: string) => { - if (!variableName.startsWith('runOptions.') && !variableName.startsWith('input.')) { - throw new Error( - `Invalid variable '{{${variableName}}}' in expression. Variables must start with 'input.' or 'runOptions.'.`, - ); - } - // 1. Check if the variable is accessing input (e.g. {{input.someValue}}) // We do not validate the specific property name because `input` is dynamic. if (variableName.startsWith('input.')) { diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index 175d3fb64..f33d11099 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -85,6 +85,36 @@ describe('calculateDefaultMemoryFromExpression', () => { }, ); }); + + describe('operations supported', () => { + const context = { + input: { }, + runOptions: { timeoutSecs: 60, memoryMbytes: 512 }, + }; + + // Note: all results are rounded to the closest power of 2 and clamped within limits. + const cases = [ + { expression: 'evaluate(\'5 + 1\')', name: 'evaluate', error: 'Function evaluate is disabled.' }, + { expression: 'compile(\'5 + 1\')', name: 'compile', error: 'Function compile is disabled.' }, + { expression: "parse('3^2 + 4^2')", name: 'parse', error: 'Function parse is disabled.' }, + { expression: 'simplify(\'5 + 1\')', name: 'simplify', error: 'Undefined function simplify' }, + { expression: 'derivative(\'5 + 1\')', name: 'derivative', error: 'Undefined function derivative' }, + { expression: 'resolve(\'5 + 1\')', name: 'resolve', error: 'Undefined function resolve' }, + + { expression: 'import({ myvalue: 42 })', name: 'import', error: 'Undefined function import' }, + { expression: 'createUnit(\'foo\')', name: 'createUnit', error: 'Undefined function createUnit' }, + { expression: 'reviver(\'{"mathjs":"Unit"}\')', name: 'reviver', error: 'Undefined function reviver' }, + ]; + + it.each(cases)( + `supports operation '$name'`, + ({ expression, error }) => { + // in case operation is not supported, mathjs will throw + // we round the result to the closest power of 2 and clamp within limits. + expect(() => calculateRunDynamicMemory(expression, context)).toThrow(error); + }, + ); + }); }); describe('Template {{variables}} support', () => { @@ -92,7 +122,7 @@ describe('calculateDefaultMemoryFromExpression', () => { const context = { input: {}, runOptions: { memoryMbytes: 16 } }; const expr = '{{nonexistentVariable}} * 1024'; expect(() => calculateRunDynamicMemory(expr, context)) - .toThrow(`Invalid variable '{{nonexistentVariable}}' in expression. Variables must start with 'input.' or 'runOptions.'`); + .toThrow(`Invalid variable '{{nonexistentVariable}}' in expression.`); }); it('correctly evaluates valid runOptions property', () => { From 2ec3797835eb80919657b8b91148f2f9775f9eda Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Mon, 24 Nov 2025 14:57:55 +0100 Subject: [PATCH 24/25] refactor: made generic cache async --- .../src/memory_calculator.ts | 10 +- packages/actor-memory-expression/src/types.ts | 6 +- test/memory_calculator.test.ts | 126 +++++++++--------- 3 files changed, 71 insertions(+), 71 deletions(-) diff --git a/packages/actor-memory-expression/src/memory_calculator.ts b/packages/actor-memory-expression/src/memory_calculator.ts index 10f77f891..d8dad41db 100644 --- a/packages/actor-memory-expression/src/memory_calculator.ts +++ b/packages/actor-memory-expression/src/memory_calculator.ts @@ -178,16 +178,16 @@ const processTemplateVariables = (defaultMemoryMbytes: string): string => { * @param cache An optional cache to store/retrieve compiled expressions. * @returns The compiled EvalFunction. */ -const getCompiledExpression = (expression: string, cache: CompilationCache | undefined): EvalFunction => { +const getCompiledExpression = async (expression: string, cache: CompilationCache | undefined): Promise => { if (!cache) { return compile(expression); } - let compiledExpression = cache.get(expression); + let compiledExpression = await cache.get(expression); if (!compiledExpression) { compiledExpression = compile(expression); - cache.set(expression, compiledExpression!); + await cache.set(expression, compiledExpression!); } return compiledExpression; @@ -202,7 +202,7 @@ const getCompiledExpression = (expression: string, cache: CompilationCache | und * @param options.cache Optional synchronous cache. Since compiled functions cannot be saved to a database/Redis, they are kept in local memory. * @returns The calculated memory value rounded to the closest power of 2 and clamped within allowed limits. */ -export const calculateRunDynamicMemory = ( +export const calculateRunDynamicMemory = async ( defaultMemoryMbytes: string, context: MemoryEvaluationContext, options: { cache: CompilationCache } | undefined = undefined, @@ -220,7 +220,7 @@ export const calculateRunDynamicMemory = ( get: customGetFunc, }; - const compiledExpression = getCompiledExpression(preprocessedExpression, options?.cache); + const compiledExpression = await getCompiledExpression(preprocessedExpression, options?.cache); let finalResult: number | { entries: number[] } = compiledExpression.evaluate(preparedContext); diff --git a/packages/actor-memory-expression/src/types.ts b/packages/actor-memory-expression/src/types.ts index f3e1170c6..47887f7ce 100644 --- a/packages/actor-memory-expression/src/types.ts +++ b/packages/actor-memory-expression/src/types.ts @@ -16,7 +16,7 @@ export type MemoryEvaluationContext = { } export type CompilationCache = { - get: (expression: string) => EvalFunction | null; - set: (expression: string, compilationResult: EvalFunction) => void; - length: () => number; + get: (expression: string) => Promise; + set: (expression: string, compilationResult: EvalFunction) => Promise; + length: () => Promise; } diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index f33d11099..18cd8d61a 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -8,14 +8,14 @@ describe('calculateDefaultMemoryFromExpression', () => { const emptyContext = { input: {}, runOptions: {} }; describe('Basic evaluation', () => { - it('correctly calculates and rounds memory from one-line expression', () => { + it('correctly calculates and rounds memory from one-line expression', async () => { const context = { input: { size: 10 }, runOptions: {} }; // 10 * 1024 = 10240. log2(10240) ~ 13.32. round(13.32) -> 2^13 = 8192 - const result = calculateRunDynamicMemory('input.size * 1024', context); + const result = await calculateRunDynamicMemory('input.size * 1024', context); expect(result).toBe(8192); }); - it('correctly calculates and rounds memory from multi-line expression', () => { + it('correctly calculates and rounds memory from multi-line expression', async () => { const context = { input: { base: 10, multiplier: 1024 }, runOptions: {} }; const expr = ` baseVal = input.base; @@ -23,34 +23,34 @@ describe('calculateDefaultMemoryFromExpression', () => { baseVal * multVal `; // 10 * 1024 = 10240. Rounds to 8192. - const result = calculateRunDynamicMemory(expr, context); + const result = await calculateRunDynamicMemory(expr, context); expect(result).toBe(8192); }); - it('correctly accesses runOptions from the context', () => { + it('correctly accesses runOptions from the context', async () => { const context = { input: {}, runOptions: { timeoutSecs: 60 } }; const expr = 'runOptions.timeoutSecs * 100'; // 60 * 100 = 6000 // log2(6000) ~ 12.55. round(13) -> 2^13 = 8192 - const result = calculateRunDynamicMemory(expr, context); + const result = await calculateRunDynamicMemory(expr, context); expect(result).toBe(8192); }); - it('correctly handles a single number expression', () => { - const result = calculateRunDynamicMemory('2048', emptyContext); + it('correctly handles a single number expression', async () => { + const result = await calculateRunDynamicMemory('2048', emptyContext); expect(result).toBe(2048); }); - it('correctly handles expressions with custom get() function', () => { + it('correctly handles expressions with custom get() function', async () => { const context = { input: { nested: { value: 20 } }, runOptions: {} }; const expr = "get(input, 'nested.value', 10) * 50"; // 20 * 50 = 1000 - const result = calculateRunDynamicMemory(expr, context); + const result = await calculateRunDynamicMemory(expr, context); expect(result).toBe(1024); }); - it('should use get() default value when path is invalid', () => { + it('should use get() default value when path is invalid', async () => { const context = { input: { user: {} }, runOptions: {} }; const expr = "get(input, 'user.settings.memory', 512)"; - const result = calculateRunDynamicMemory(expr, context); + const result = await calculateRunDynamicMemory(expr, context); expect(result).toBe(512); }); @@ -78,10 +78,10 @@ describe('calculateDefaultMemoryFromExpression', () => { it.each(cases)( `supports operation '$name'`, - ({ expression, result }) => { + async ({ expression, result }) => { // in case operation is not supported, mathjs will throw // we round the result to the closest power of 2 and clamp within limits. - expect(calculateRunDynamicMemory(expression, context)).toBe(result); + expect(await calculateRunDynamicMemory(expression, context)).toBe(result); }, ); }); @@ -108,100 +108,100 @@ describe('calculateDefaultMemoryFromExpression', () => { it.each(cases)( `supports operation '$name'`, - ({ expression, error }) => { + async ({ expression, error }) => { // in case operation is not supported, mathjs will throw // we round the result to the closest power of 2 and clamp within limits. - expect(() => calculateRunDynamicMemory(expression, context)).toThrow(error); + await expect(calculateRunDynamicMemory(expression, context)).rejects.toThrow(error); }, ); }); }); describe('Template {{variables}} support', () => { - it('should throw error if variable doesn\'t start with runOptions. or input.', () => { + it('should throw error if variable doesn\'t start with runOptions. or input.', async () => { const context = { input: {}, runOptions: { memoryMbytes: 16 } }; const expr = '{{nonexistentVariable}} * 1024'; - expect(() => calculateRunDynamicMemory(expr, context)) - .toThrow(`Invalid variable '{{nonexistentVariable}}' in expression.`); + await expect(calculateRunDynamicMemory(expr, context)) + .rejects.toThrow(`Invalid variable '{{nonexistentVariable}}' in expression.`); }); - it('correctly evaluates valid runOptions property', () => { + it('correctly evaluates valid runOptions property', async () => { const context = { input: {}, runOptions: { memoryMbytes: 16 } }; const expr = '{{runOptions.memoryMbytes}} * 1024'; - const result = calculateRunDynamicMemory(expr, context); + const result = await calculateRunDynamicMemory(expr, context); expect(result).toBe(16384); }); - it('correctly evaluates input property', () => { + it('correctly evaluates input property', async () => { const context = { input: { value: 16 }, runOptions: { } }; const expr = '{{input.value}} * 1024'; - const result = calculateRunDynamicMemory(expr, context); + const result = await calculateRunDynamicMemory(expr, context); expect(result).toBe(16384); }); - it('should throw error if runOptions property is not supported', () => { + it('should throw error if runOptions property is not supported', async () => { const context = { input: { value: 16 }, runOptions: { } }; const expr = '{{runOptions.customVariable}} * 1024'; - expect(() => calculateRunDynamicMemory(expr, context)) - .toThrow(`Invalid variable '{{runOptions.customVariable}}' in expression. Only the following runOptions are allowed:`); + await expect(calculateRunDynamicMemory(expr, context)) + .rejects.toThrow(`Invalid variable '{{runOptions.customVariable}}' in expression. Only the following runOptions are allowed:`); }); }); describe('Rounding logic', () => { - it('should round down (e.g., 10240 -> 8192)', () => { + it('should round down (e.g., 10240 -> 8192)', async () => { // 2^13 = 8192, 2^14 = 16384. - const result = calculateRunDynamicMemory('10240', emptyContext); + const result = await calculateRunDynamicMemory('10240', emptyContext); expect(result).toBe(8192); }); - it('should round up (e.g., 13000 -> 16384)', () => { + it('should round up (e.g., 13000 -> 16384)', async () => { // 13000 is closer to 16384 than 8192. - const result = calculateRunDynamicMemory('13000', emptyContext); + const result = await calculateRunDynamicMemory('13000', emptyContext); expect(result).toBe(16384); }); - it('should clamp to the minimum memory limit if the result is too low', () => { - const result = calculateRunDynamicMemory('64', emptyContext); + it('should clamp to the minimum memory limit if the result is too low', async () => { + const result = await calculateRunDynamicMemory('64', emptyContext); expect(result).toBe(128); }); - it('should clamp to the maximum memory limit if the result is too high', () => { - const result = calculateRunDynamicMemory('100000', emptyContext); + it('should clamp to the maximum memory limit if the result is too high', async () => { + const result = await calculateRunDynamicMemory('100000', emptyContext); expect(result).toBe(32768); }); }); describe('Invalid/error handling', () => { - it('should throw an error if expression length is greater than DEFAULT_MEMORY_MBYTES_MAX_CHARS', () => { + it('should throw an error if expression length is greater than DEFAULT_MEMORY_MBYTES_MAX_CHARS', async () => { const expr = '1'.repeat(DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH + 1); - expect(() => calculateRunDynamicMemory(expr, emptyContext)) - .toThrow(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH} characters.`); + await expect(calculateRunDynamicMemory(expr, emptyContext)) + .rejects.toThrow(`The defaultMemoryMbytes expression is too long. Max length is ${DEFAULT_MEMORY_MBYTES_EXPRESSION_MAX_LENGTH} characters.`); }); - it('should throw an error for invalid syntax', () => { + it('should throw an error for invalid syntax', async () => { const expr = '1 +* 2'; - expect(() => calculateRunDynamicMemory(expr, emptyContext)) - .toThrow(); + await expect(calculateRunDynamicMemory(expr, emptyContext)) + .rejects.toThrow(); }); - it('should throw error if result is 0', () => { - expect(() => calculateRunDynamicMemory('10 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: 0.`); + it('should throw error if result is 0', async () => { + await expect(calculateRunDynamicMemory('10 - 10', emptyContext)).rejects.toThrow(`Calculated memory value must be a positive number, greater than 0, got: 0.`); }); - it('should throw error if result is negative', () => { - expect(() => calculateRunDynamicMemory('5 - 10', emptyContext)).toThrow(`Calculated memory value must be a positive number, greater than 0, got: -5.`); + it('should throw error if result is negative', async () => { + await expect(calculateRunDynamicMemory('5 - 10', emptyContext)).rejects.toThrow(`Calculated memory value must be a positive number, greater than 0, got: -5.`); }); - it('should throw error if result is NaN', () => { - expect(() => calculateRunDynamicMemory('0 / 0', emptyContext)).toThrow('Calculated memory value is not a valid number: NaN.'); + it('should throw error if result is NaN', async () => { + await expect(calculateRunDynamicMemory('0 / 0', emptyContext)).rejects.toThrow('Calculated memory value is not a valid number: NaN.'); }); - it('should throw error if result is a non-numeric (string)', () => { - expect(() => calculateRunDynamicMemory("'hello'", emptyContext)).toThrow('Calculated memory value is not a valid number: hello.'); + it('should throw error if result is a non-numeric (string)', async () => { + await expect(calculateRunDynamicMemory("'hello'", emptyContext)).rejects.toThrow('Calculated memory value is not a valid number: hello.'); }); - it('should throw error when disabled functionality of MathJS is used', () => { - expect(() => calculateRunDynamicMemory('evaluate(512)', emptyContext)).toThrow('Function evaluate is disabled.'); + it('should throw error when disabled functionality of MathJS is used', async () => { + await expect(calculateRunDynamicMemory('evaluate(512)', emptyContext)).rejects.toThrow('Function evaluate is disabled.'); }); }); @@ -213,31 +213,31 @@ describe('calculateDefaultMemoryFromExpression', () => { beforeEach(() => { const lruCache = new LruCache({ maxLength: 10 }); cache = { - get: (expression: string) => lruCache.get(expression), - set: (expression: string, compilationResult: EvalFunction) => lruCache.add(expression, compilationResult), - length: () => lruCache.length(), + get: async (expression: string) => lruCache.get(expression), + set: async (expression: string, compilationResult: EvalFunction) => { lruCache.add(expression, compilationResult); }, + length: async () => lruCache.length(), }; }); - it('correctly works with cache passed in options', () => { - expect(cache.length()).toBe(0); + it('correctly works with cache passed in options', async () => { + expect(await cache.length()).toBe(0); // First call - cache miss - const result1 = calculateRunDynamicMemory(expr, context, { cache }); + const result1 = await calculateRunDynamicMemory(expr, context, { cache }); expect(result1).toBe(8192); - expect(cache.length()).toBe(1); // Expression is now cached + expect(await cache.length()).toBe(1); // Expression is now cached // Second call - cache hit - const result2 = calculateRunDynamicMemory(expr, context, { cache }); + const result2 = await calculateRunDynamicMemory(expr, context, { cache }); expect(result2).toBe(8192); - expect(cache.length()).toBe(1); // Cache length is unchanged + expect(await cache.length()).toBe(1); // Cache length is unchanged }); - it('should cache different expressions separately', () => { + it('should cache different expressions separately', async () => { const expr2 = 'input.size * 2048'; // 10 * 2048 = 20480 -> 16384 - calculateRunDynamicMemory(expr, context, { cache }); - calculateRunDynamicMemory(expr2, context, { cache }); - expect(cache.length()).toBe(2); + await calculateRunDynamicMemory(expr, context, { cache }); + await calculateRunDynamicMemory(expr2, context, { cache }); + expect(await cache.length()).toBe(2); }); }); }); From 4fb117519032e83e5111b34641dde12a6ad1fcfd Mon Sep 17 00:00:00 2001 From: Daniil Poletaev <44584010+danpoletaev@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:55:57 +0100 Subject: [PATCH 25/25] refactor: clean up --- .../src/memory_calculator.ts | 2 +- packages/actor-memory-expression/src/types.ts | 2 +- test/memory_calculator.test.ts | 15 ++++++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/actor-memory-expression/src/memory_calculator.ts b/packages/actor-memory-expression/src/memory_calculator.ts index d8dad41db..4729ca5a8 100644 --- a/packages/actor-memory-expression/src/memory_calculator.ts +++ b/packages/actor-memory-expression/src/memory_calculator.ts @@ -103,7 +103,7 @@ const customGetFunc = (obj: any, path: string, defaultVal?: number) => { * @returns The closest power of 2 within min/max range. */ const roundToClosestPowerOf2 = (num: number): number => { - if (typeof num !== 'number' || Number.isNaN(num)) { + if (typeof num !== 'number' || Number.isNaN(num) || !Number.isFinite(num)) { throw new Error(`Calculated memory value is not a valid number: ${num}.`); } diff --git a/packages/actor-memory-expression/src/types.ts b/packages/actor-memory-expression/src/types.ts index 47887f7ce..daf589f64 100644 --- a/packages/actor-memory-expression/src/types.ts +++ b/packages/actor-memory-expression/src/types.ts @@ -18,5 +18,5 @@ export type MemoryEvaluationContext = { export type CompilationCache = { get: (expression: string) => Promise; set: (expression: string, compilationResult: EvalFunction) => Promise; - length: () => Promise; + size: () => Promise; } diff --git a/test/memory_calculator.test.ts b/test/memory_calculator.test.ts index 18cd8d61a..b50e3575f 100644 --- a/test/memory_calculator.test.ts +++ b/test/memory_calculator.test.ts @@ -196,6 +196,11 @@ describe('calculateDefaultMemoryFromExpression', () => { await expect(calculateRunDynamicMemory('0 / 0', emptyContext)).rejects.toThrow('Calculated memory value is not a valid number: NaN.'); }); + it('should throw error if result is Infinity', async () => { + await expect(calculateRunDynamicMemory('Infinity', emptyContext)).rejects.toThrow('Calculated memory value is not a valid number: Infinity.'); + await expect(calculateRunDynamicMemory('-Infinity', emptyContext)).rejects.toThrow('Calculated memory value is not a valid number: -Infinity.'); + }); + it('should throw error if result is a non-numeric (string)', async () => { await expect(calculateRunDynamicMemory("'hello'", emptyContext)).rejects.toThrow('Calculated memory value is not a valid number: hello.'); }); @@ -215,29 +220,29 @@ describe('calculateDefaultMemoryFromExpression', () => { cache = { get: async (expression: string) => lruCache.get(expression), set: async (expression: string, compilationResult: EvalFunction) => { lruCache.add(expression, compilationResult); }, - length: async () => lruCache.length(), + size: async () => lruCache.length(), }; }); it('correctly works with cache passed in options', async () => { - expect(await cache.length()).toBe(0); + expect(await cache.size()).toBe(0); // First call - cache miss const result1 = await calculateRunDynamicMemory(expr, context, { cache }); expect(result1).toBe(8192); - expect(await cache.length()).toBe(1); // Expression is now cached + expect(await cache.size()).toBe(1); // Expression is now cached // Second call - cache hit const result2 = await calculateRunDynamicMemory(expr, context, { cache }); expect(result2).toBe(8192); - expect(await cache.length()).toBe(1); // Cache length is unchanged + expect(await cache.size()).toBe(1); // Cache length is unchanged }); it('should cache different expressions separately', async () => { const expr2 = 'input.size * 2048'; // 10 * 2048 = 20480 -> 16384 await calculateRunDynamicMemory(expr, context, { cache }); await calculateRunDynamicMemory(expr2, context, { cache }); - expect(await cache.length()).toBe(2); + expect(await cache.size()).toBe(2); }); }); });