diff --git a/eslint.config.mjs b/eslint.config.mjs index d9109844c0..fe49f4c35c 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -737,14 +737,15 @@ export default defineConfig([ }, }, { - // Workflow/action/runtime sources are plain JS evaluated outside the TS - // program (QuickJS, skill assets, or generated child-process wrappers), so - // type-aware rules cannot apply. Lint them with core untyped rules so typos - // and dead helpers fail loudly instead of becoming silent sandbox globals. + // Workflow/action/runtime and script helper sources are plain JS evaluated outside the TS + // program (QuickJS, skill assets, generated child-process wrappers, or local tooling), so + // type-aware rules cannot apply. Lint them with core untyped rules so typos and dead helpers + // fail loudly instead of becoming silent globals. files: [ "src/node/builtinSkills/**/*.js", "src/node/builtinWorkflowActions/**/*.js", "src/node/workflowRuntime/*.js", + "scripts/lib/*.js", ], extends: [tseslint.configs.disableTypeChecked], languageOptions: { diff --git a/fmt.mk b/fmt.mk index 0bd87231f4..dfa4e81bc8 100644 --- a/fmt.mk +++ b/fmt.mk @@ -6,7 +6,7 @@ .PHONY: fmt fmt-check fmt-prettier fmt-prettier-check fmt-shell fmt-shell-check fmt-nix fmt-nix-check fmt-python fmt-python-check fmt-sync-docs fmt-sync-docs-check update-flake-hash flake-hash-check # Centralized patterns - single source of truth -PRETTIER_PATTERNS := 'src/**/*.{ts,tsx,json}' 'src/node/workflowRuntime/*.js' 'tests/**/*.ts' 'docs/**/*.mdx' 'package.json' 'tsconfig*.json' 'README.md' +PRETTIER_PATTERNS := 'src/**/*.{ts,tsx,json}' 'src/node/workflowRuntime/*.js' 'scripts/lib/*.js' 'tests/**/*.ts' 'docs/**/*.mdx' 'package.json' 'tsconfig*.json' 'README.md' SHELL_SCRIPTS := scripts PYTHON_DIRS := benchmarks diff --git a/jest.config.js b/jest.config.js index d727218ce4..aa01f2d591 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,18 +1,6 @@ -const os = require("node:os"); +const { workerBudgetFor } = require("./scripts/lib/worker_budget.js"); -// Use cgroup-aware memory when available (containers), fall back to host RAM. -// process.constrainedMemory() returns the cgroup v2 limit (Node 19.6+), -// or 0/undefined outside a cgroup. -const totalMemoryBytes = - (typeof process.constrainedMemory === "function" && - process.constrainedMemory()) || - os.totalmem(); - -const cpuWorkerCap = Math.max(1, Math.floor(os.cpus().length * 0.5)); -const memoryWorkerCap = Math.floor( - totalMemoryBytes / (1024 * 1024 * 1024) / 1.5, -); -const maxWorkers = Math.max(1, Math.min(cpuWorkerCap, memoryWorkerCap)); +const maxWorkers = workerBudgetFor("jest"); /** @type {import('jest').Config} */ module.exports = { @@ -55,9 +43,6 @@ module.exports = { // This is slower but ensures compatibility "node_modules/(?!\\.pnpm)(?!.*)", ], - // High core-count containers with limited cgroup memory (for example 96 cores / - // 32 GB) can OOM if Jest uses CPU-only parallelism, so keep roughly 1.5 GB - // per worker. maxWorkers, // Force exit after tests complete to avoid hanging on lingering handles forceExit: true, diff --git a/scripts/lib/worker_budget.js b/scripts/lib/worker_budget.js new file mode 100644 index 0000000000..33e9b88d1a --- /dev/null +++ b/scripts/lib/worker_budget.js @@ -0,0 +1,233 @@ +"use strict"; + +// Size tool worker pools against the tightest cgroup memory cap, including ancestor caps that +// Node's leaf-only constrainedMemory() check can miss. + +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +const BYTES_PER_GIB = 1024 ** 3; +const DEFAULT_CGROUP_ROOT = "/sys/fs/cgroup"; +const DEFAULT_PROC_SELF_CGROUP = "/proc/self/cgroup"; + +// cgroup v1 spells "unlimited" as a saturated integer rather than "max", so treat implausibly large +// caps as absent instead of trusting them. +const UNLIMITED_BYTES_FLOOR = 2n ** 62n; + +// Peak RSS measured per worker in this repo, rounded up for growth: ESLint's --concurrency lanes are +// worker threads in one process (~2.8GiB per lane), while Jest forks reach ~4.7GiB each. +const PROFILES = { + eslint: { memoryPerWorkerGib: 4, maxWorkers: 4 }, + jest: { memoryPerWorkerGib: 6, maxWorkers: 4 }, +}; + +const CPU_FRACTION = 0.5; + +function readFileOrNull(filePath) { + try { + return fs.readFileSync(filePath, "utf8"); + } catch { + return null; + } +} + +function parseBytes(raw) { + if (raw == null) { + return null; + } + const text = raw.trim(); + if (text === "") { + return null; + } + let value; + try { + value = BigInt(text); + } catch { + return null; + } + if (value < 0n) { + return null; + } + return value; +} + +function parseLimitBytes(raw) { + const value = parseBytes(raw); + if (value == null || value === 0n || value >= UNLIMITED_BYTES_FLOOR) { + return null; + } + return Number(value); +} + +function dirChain(root, cgroupPath) { + const segments = cgroupPath.split("/").filter(Boolean); + const chain = []; + for (let depth = segments.length; depth >= 0; depth--) { + chain.push(path.join(root, ...segments.slice(0, depth))); + } + return chain; +} + +function cgroupMembership(options) { + const raw = readFileOrNull(options.procSelfCgroup ?? DEFAULT_PROC_SELF_CGROUP); + let v2Path = null; + let v1MemoryPath = null; + + for (const line of raw?.split("\n") ?? []) { + const match = line.trim().match(/^\d+:([^:]*):(.*)$/); + if (match == null) { + continue; + } + const controllers = match[1].split(","); + if (controllers.length === 1 && controllers[0] === "") { + v2Path = match[2]; + } else if (controllers.includes("memory")) { + v1MemoryPath = match[2]; + } + } + + return { v2Path, v1MemoryPath }; +} + +// Keep the constraining directory so its ancestor-level co-tenant usage can be subtracted. +function tightestConstraint(dirs, limitFile) { + let constraint = null; + for (const dir of dirs) { + const limitBytes = parseLimitBytes(readFileOrNull(path.join(dir, limitFile))); + if (limitBytes != null && (constraint == null || limitBytes <= constraint.limitBytes)) { + constraint = { dir, limitBytes }; + } + } + return constraint; +} + +function resolveMemoryConstraint(options = {}) { + const cgroupRoot = options.cgroupRoot ?? DEFAULT_CGROUP_ROOT; + const membership = cgroupMembership(options); + if (membership.v2Path != null) { + const constraint = tightestConstraint(dirChain(cgroupRoot, membership.v2Path), "memory.max"); + if (constraint != null) { + return constraint; + } + } + + if (membership.v1MemoryPath == null) { + return null; + } + return tightestConstraint( + dirChain(path.join(cgroupRoot, "memory"), membership.v1MemoryPath), + "memory.limit_in_bytes" + ); +} + +function parseMemoryStat(dir) { + const raw = readFileOrNull(path.join(dir, "memory.stat")); + if (raw == null) { + return null; + } + + const values = new Map(); + for (const line of raw.split("\n")) { + const [key, value] = line.trim().split(/\s+/); + const parsed = parseBytes(value); + if (key && parsed != null) { + values.set(key, Number(parsed)); + } + } + return values; +} + +// Discount inactive file cache, but use the v2 resident-memory fields as a floor when available. +function readCgroupUsageBytes(dir) { + const stat = parseMemoryStat(dir); + const current = parseBytes(readFileOrNull(path.join(dir, "memory.current"))); + if (current != null) { + if (stat == null) { + return Number(current); + } + const resident = + (stat.get("anon") ?? 0) + + (stat.get("kernel") ?? stat.get("slab") ?? 0) + + (stat.get("shmem") ?? 0); + const inactiveFile = stat.get("inactive_file"); + return inactiveFile == null ? resident : Math.max(resident, Number(current) - inactiveFile); + } + + const usage = parseBytes(readFileOrNull(path.join(dir, "memory.usage_in_bytes"))); + if (usage == null) { + return null; + } + const inactiveFile = stat?.get("total_inactive_file") ?? stat?.get("inactive_file"); + return inactiveFile == null ? Number(usage) : Math.max(0, Number(usage) - inactiveFile); +} + +function computeWorkers(input) { + const cpuWorkers = Math.max(1, Math.floor(input.cpuCount * CPU_FRACTION)); + + // Preserve headroom for the parent process, active file cache, and co-tenants growing mid-run. + const reserveBytes = Math.max(2 * BYTES_PER_GIB, input.limitBytes * 0.15); + const usableBytes = Math.max(0, input.limitBytes - input.inUseBytes - reserveBytes); + const memoryWorkers = Math.floor(usableBytes / (input.memoryPerWorkerGib * BYTES_PER_GIB)); + + return Math.max(1, Math.min(cpuWorkers, memoryWorkers, input.maxWorkers)); +} + +function resolveWorkerBudget(profileName, options = {}) { + const profile = PROFILES[profileName]; + if (profile == null) { + throw new Error( + `unknown worker budget profile "${profileName}" (expected one of: ${Object.keys(PROFILES).join(", ")})` + ); + } + + const constraint = resolveMemoryConstraint(options); + // Without a cgroup cap there is no bounded co-tenant usage signal. Host free-memory swings would + // make a busy laptop silently serialize its own test run. + const limitBytes = constraint?.limitBytes ?? os.totalmem(); + const inUseBytes = constraint == null ? 0 : (readCgroupUsageBytes(constraint.dir) ?? 0); + + const input = { + ...profile, + cpuCount: os.availableParallelism?.() ?? os.cpus().length, + limitBytes, + inUseBytes, + }; + return { ...input, cgroupDir: constraint?.dir ?? null, workers: computeWorkers(input) }; +} + +function formatWorkerBudget(budget) { + const gib = (bytes) => `${(bytes / BYTES_PER_GIB).toFixed(1)}GiB`; + return [ + `workers=${budget.workers}`, + `limit=${gib(budget.limitBytes)}`, + `inUse=${gib(budget.inUseBytes)}`, + `perWorker=${budget.memoryPerWorkerGib}GiB`, + `cpus=${budget.cpuCount}`, + `cgroup=${budget.cgroupDir ?? "none"}`, + ].join(" "); +} + +function workerBudgetFor(profileName) { + const budget = resolveWorkerBudget(profileName); + if (process.env.MUX_WORKER_BUDGET_DEBUG) { + process.stderr.write(`[worker-budget] ${profileName} ${formatWorkerBudget(budget)}\n`); + } + return budget.workers; +} + +module.exports = { + computeWorkers, + readCgroupUsageBytes, + resolveMemoryConstraint, + workerBudgetFor, +}; + +if (require.main === module) { + const profileName = process.argv[2]; + const budget = resolveWorkerBudget(profileName); + if (process.argv.includes("--debug") || process.env.MUX_WORKER_BUDGET_DEBUG) { + process.stderr.write(`[worker-budget] ${profileName} ${formatWorkerBudget(budget)}\n`); + } + process.stdout.write(`${budget.workers}\n`); +} diff --git a/scripts/lint.sh b/scripts/lint.sh index e2840366c8..e973ba51ff 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -29,34 +29,10 @@ ESLINT_PATTERNS=( 'src/**/*.{ts,tsx}' 'src/node/builtinSkills/**/*.js' 'src/node/workflowRuntime/*.js' + 'scripts/lib/*.js' ) -get_cpu_count() { - local cpu_count="" - - if command -v getconf >/dev/null 2>&1; then - cpu_count="$(getconf _NPROCESSORS_ONLN 2>/dev/null || true)" - fi - - if [ -z "$cpu_count" ] && command -v nproc >/dev/null 2>&1; then - cpu_count="$(nproc 2>/dev/null || true)" - fi - - if [ -z "$cpu_count" ] && command -v sysctl >/dev/null 2>&1; then - cpu_count="$(sysctl -n hw.ncpu 2>/dev/null || true)" - fi - - if [[ "$cpu_count" =~ ^[0-9]+$ ]] && [ "$cpu_count" -gt 0 ]; then - echo "$cpu_count" - else - echo 2 - fi -} - get_default_eslint_concurrency() { - local cpu_count - local concurrency - # Most local `make static-check` runs are warm-cache validation after a small # edit. ESLint's worker startup/merge overhead dominates that path, so keep it # single-process once the cache exists; cold caches still scale up for CI-like @@ -66,18 +42,15 @@ get_default_eslint_concurrency() { return fi - cpu_count="$(get_cpu_count)" - concurrency=$(((cpu_count + 1) / 2)) - - # User rationale: local static-check should scale up on agent/desktop machines - # without letting ESLint's auto concurrency spawn one worker per core. - if [ "$concurrency" -lt 2 ]; then - concurrency=2 - elif [ "$concurrency" -gt 8 ]; then - concurrency=8 + # Cold type-aware runs scale memory with concurrency, so use cgroup headroom instead of visible + # core count. Keep stderr attached for diagnostics, and fall back only if the helper fails. + local concurrency + if concurrency="$(node "$SCRIPT_DIR/lib/worker_budget.js" eslint)" \ + && [[ "$concurrency" =~ ^[0-9]+$ ]] && [ "$concurrency" -gt 0 ]; then + echo "$concurrency" + else + echo 1 fi - - echo "$concurrency" } ESLINT_CONCURRENCY="${MUX_ESLINT_CONCURRENCY:-$(get_default_eslint_concurrency)}" diff --git a/src/browser/components/AgentModePicker/AgentModePicker.tsx b/src/browser/components/AgentModePicker/AgentModePicker.tsx index 0eae431f5b..13e8cc1fd2 100644 --- a/src/browser/components/AgentModePicker/AgentModePicker.tsx +++ b/src/browser/components/AgentModePicker/AgentModePicker.tsx @@ -7,6 +7,10 @@ import { CUSTOM_EVENTS } from "@/common/constants/events"; import type { AgentDefinitionDescriptor } from "@/common/types/agentDefinition"; import { normalizeAgentId as normalizeStoredAgentId } from "@/common/utils/agentIds"; import { cn } from "@/common/lib/utils"; +import { + COMPOSER_PICKER_PANEL_CLASS, + composerPickerOptionClass, +} from "@/browser/components/composerPickerStyles"; import { DocsLink } from "@/browser/components/DocsLink/DocsLink"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/browser/components/Tooltip/Tooltip"; import { Button } from "@/browser/components/Button/Button"; @@ -380,9 +384,11 @@ export const AgentModePicker: React.FC = (props) => { tabIndex={-1} onKeyDown={handleDropdownKeyDown} // Left alignment prevents the menu from opening beyond the viewport. - className="bg-surface-primary border-border-light absolute bottom-full left-0 z-[1020] mb-1 min-w-52 overflow-hidden rounded border shadow-[0_4px_12px_rgba(0,0,0,0.3)] outline-none" + className={cn( + "absolute bottom-full left-0 z-[1020] mb-1 min-w-52", + COMPOSER_PICKER_PANEL_CLASS + )} > - {/* Agent list — scrollable for long lists */}
{!loaded && options.length === 0 ? (
Loading agents…
@@ -404,11 +410,7 @@ export const AgentModePicker: React.FC = (props) => { tabIndex={-1} data-agent-id={opt.id} data-testid="agent-option" - className={cn( - "flex cursor-pointer items-center gap-2.5 px-2.5 py-1.5 transition-colors duration-100", - isHighlighted ? "bg-hover text-foreground" : "bg-transparent hover:bg-hover", - isSelected ? "text-foreground" : "text-light hover:text-foreground" - )} + className={composerPickerOptionClass({ isHighlighted, isSelected }, "py-1.5")} onMouseEnter={() => setHighlightedIndex(index)} onClick={() => handleSelectAgent(opt.id)} > @@ -418,10 +420,7 @@ export const AgentModePicker: React.FC = (props) => { /> {opt.name} diff --git a/src/browser/components/ModelSelector/ModelSelector.tsx b/src/browser/components/ModelSelector/ModelSelector.tsx index 57890ab026..99fef1544a 100644 --- a/src/browser/components/ModelSelector/ModelSelector.tsx +++ b/src/browser/components/ModelSelector/ModelSelector.tsx @@ -13,8 +13,9 @@ import React, { forwardRef, } from "react"; import { cn } from "@/common/lib/utils"; -import { Check, ChevronDown, Eye, Settings, ShieldCheck, Star } from "lucide-react"; +import { ChevronDown, Eye, Settings, ShieldCheck, Star } from "lucide-react"; +import { COMPOSER_PICKER_PANEL_CLASS, composerPickerOptionClass } from "../composerPickerStyles"; import { ProviderIcon } from "../ProviderIcon/ProviderIcon"; import { Tooltip, TooltipTrigger, TooltipContent } from "../Tooltip/Tooltip"; import { useSettings } from "@/browser/contexts/SettingsContext"; @@ -306,7 +307,7 @@ export const ModelSelector = forwardRef( type="button" className={cn( triggerClassName, - "text-foreground hover:bg-hover flex cursor-pointer items-center justify-between gap-1 px-1.5 py-0.5 transition-colors duration-300" + "text-foreground hover:bg-hover flex cursor-pointer items-center justify-between gap-1 px-1.5 py-0.5 transition-[background-color] duration-150" )} role="combobox" aria-expanded={isOpen} @@ -323,7 +324,12 @@ export const ModelSelector = forwardRef( )} {displayValue} - + ( {/* Dropdown content - rendered inline for testability */} {isOpen && ( -
+
{/* Search input */} -
+
(
{/* Scrollable list */} -
+
{filteredModels.length === 0 ? (
No matching models
) : ( @@ -380,27 +391,27 @@ export const ModelSelector = forwardRef( key={model} data-highlighted={index === highlightedIndex} onMouseEnter={() => setHighlightedIndex(index)} - className={cn( - "flex w-full items-center gap-1.5 rounded-sm px-2 py-0.5 text-xs cursor-pointer", - index === highlightedIndex ? "bg-hover" : "hover:bg-hover", + className={composerPickerOptionClass( + { + isHighlighted: index === highlightedIndex, + isSelected: value === model, + }, + "py-1", hiddenSet.has(model) && "opacity-50" )} onClick={() => handleSelectModel(model)} role="option" aria-selected={value === model} > - - - + {formatModelDisplayName(modelName)} {showProviderLabel && ( @@ -508,7 +519,7 @@ export const ModelSelector = forwardRef( {/* Footer actions (last row in dropdown) */} {(hiddenModels.length > 0 || onOpenSettings) && ( -
+
{hiddenModels.length > 0 && ( )} {policyEnforced && ( -
+
Your settings are controlled by a policy.
@@ -543,7 +554,7 @@ export const ModelSelector = forwardRef( onOpenSettings(); handleCancel(); }} - className="text-muted hover:bg-hover hover:text-foreground flex w-full items-center justify-start gap-1.5 rounded-sm px-2 py-1 text-[11px] transition-colors" + className="text-muted hover:bg-hover hover:text-foreground flex w-full items-center justify-start gap-2.5 px-2.5 py-1 text-[11px] font-medium transition-colors" > Model settings diff --git a/src/browser/components/composerPickerStyles.ts b/src/browser/components/composerPickerStyles.ts new file mode 100644 index 0000000000..689e14c32a --- /dev/null +++ b/src/browser/components/composerPickerStyles.ts @@ -0,0 +1,19 @@ +import type { ClassValue } from "clsx"; + +import { cn } from "@/common/lib/utils"; + +// Keep adjacent composer pickers visually aligned while leaving placement and width to each picker. +export const COMPOSER_PICKER_PANEL_CLASS = + "bg-surface-primary border-border-light overflow-hidden rounded border shadow-[0_4px_12px_rgba(0,0,0,0.3)] outline-none"; + +export function composerPickerOptionClass( + state: { isHighlighted: boolean; isSelected: boolean }, + ...classNames: ClassValue[] +): string { + return cn( + "flex cursor-pointer items-center gap-2.5 px-2.5 text-[11px] font-medium transition-colors duration-100", + state.isHighlighted ? "bg-hover text-foreground" : "bg-transparent hover:bg-hover", + state.isSelected ? "text-foreground" : "text-light hover:text-foreground", + classNames + ); +} diff --git a/tests/workerBudget.test.ts b/tests/workerBudget.test.ts new file mode 100644 index 0000000000..6fc1136e77 --- /dev/null +++ b/tests/workerBudget.test.ts @@ -0,0 +1,192 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; + +// eslint-disable-next-line @typescript-eslint/no-require-imports +const workerBudget = require("../scripts/lib/worker_budget.js"); + +const GIB = 1024 ** 3; + +interface FakeCgroup { + cgroupRoot: string; + procSelfCgroup: string; +} + +function writeFakeCgroup( + root: string, + leafPath: string, + dirs: Record>, + procLine = `0::${leafPath}` +): FakeCgroup { + const cgroupRoot = path.join(root, "cgroup"); + for (const [cgroupPath, files] of Object.entries(dirs)) { + const dir = path.join(cgroupRoot, cgroupPath); + fs.mkdirSync(dir, { recursive: true }); + for (const [name, contents] of Object.entries(files)) { + fs.writeFileSync(path.join(dir, name), contents); + } + } + + const procSelfCgroup = path.join(root, "proc-self-cgroup"); + fs.writeFileSync(procSelfCgroup, `${procLine}\n`); + return { cgroupRoot, procSelfCgroup }; +} + +function memoryStat(overrides: Record): string { + return Object.entries(overrides) + .map(([key, value]) => `${key} ${value}`) + .join("\n"); +} + +describe("worker budget cgroup resolution", () => { + let tmpRoot: string; + + beforeEach(() => { + tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "worker-budget-")); + }); + + afterEach(() => { + fs.rmSync(tmpRoot, { recursive: true, force: true }); + }); + + it("takes the cap from the constraining ancestor when the leaf is unlimited", () => { + const fake = writeFakeCgroup(tmpRoot, "/init.scope", { + "/": { "memory.max": `${32 * GIB}` }, + "/init.scope": { "memory.max": "max" }, + }); + + const constraint = workerBudget.resolveMemoryConstraint(fake); + + expect(constraint.limitBytes).toBe(32 * GIB); + expect(constraint.dir).toBe(fake.cgroupRoot); + }); + + it("prefers the tightest cap when several ancestors impose one", () => { + const fake = writeFakeCgroup(tmpRoot, "/parent/leaf", { + "/": { "memory.max": `${64 * GIB}` }, + "/parent": { "memory.max": `${8 * GIB}` }, + "/parent/leaf": { "memory.max": `${16 * GIB}` }, + }); + + expect(workerBudget.resolveMemoryConstraint(fake).limitBytes).toBe(8 * GIB); + }); + + it("uses the broadest usage scope when equal caps constrain multiple levels", () => { + const fake = writeFakeCgroup(tmpRoot, "/parent/leaf", { + "/": { "memory.max": `${64 * GIB}` }, + "/parent": { "memory.max": `${8 * GIB}` }, + "/parent/leaf": { "memory.max": `${8 * GIB}` }, + }); + + const constraint = workerBudget.resolveMemoryConstraint(fake); + + expect(constraint.limitBytes).toBe(8 * GIB); + expect(constraint.dir).toBe(path.join(fake.cgroupRoot, "parent")); + }); + + it("ignores saturated integers that stand in for 'unlimited'", () => { + const fake = writeFakeCgroup(tmpRoot, "/leaf", { + "/": { "memory.max": "9223372036854771712" }, + "/leaf": { "memory.max": "max" }, + }); + + expect(workerBudget.resolveMemoryConstraint(fake)).toBeNull(); + }); + + it("resolves a nested cgroup v1 memory limit", () => { + const fake = writeFakeCgroup( + tmpRoot, + "/docker/leaf", + { + "/memory": { "memory.limit_in_bytes": `${16 * GIB}` }, + "/memory/docker": { "memory.limit_in_bytes": `${8 * GIB}` }, + "/memory/docker/leaf": { "memory.limit_in_bytes": `${4 * GIB}` }, + }, + "5:cpu,memory:/docker/leaf" + ); + + const constraint = workerBudget.resolveMemoryConstraint(fake); + + expect(constraint.limitBytes).toBe(4 * GIB); + expect(constraint.dir).toBe(path.join(fake.cgroupRoot, "memory/docker/leaf")); + }); + + it("reports no constraint when the host has no cgroup files", () => { + const fake = writeFakeCgroup(tmpRoot, "/", { "/": {} }); + + expect(workerBudget.resolveMemoryConstraint(fake)).toBeNull(); + }); +}); + +describe("worker budget usage accounting", () => { + let tmpRoot: string; + + beforeEach(() => { + tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "worker-budget-usage-")); + }); + + afterEach(() => { + fs.rmSync(tmpRoot, { recursive: true, force: true }); + }); + + it("excludes reclaimable page cache from usage", () => { + const dir = path.join(tmpRoot, "cg"); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, "memory.current"), `${20 * GIB}`); + fs.writeFileSync( + path.join(dir, "memory.stat"), + memoryStat({ anon: 4 * GIB, kernel: GIB, shmem: 0, inactive_file: 15 * GIB }) + ); + + expect(workerBudget.readCgroupUsageBytes(dir)).toBe(5 * GIB); + }); + + it("counts file-backed memory that cannot be reclaimed", () => { + const dir = path.join(tmpRoot, "cg"); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, "memory.current"), `${20 * GIB}`); + fs.writeFileSync( + path.join(dir, "memory.stat"), + memoryStat({ anon: 4 * GIB, kernel: GIB, shmem: 0, inactive_file: 2 * GIB }) + ); + + expect(workerBudget.readCgroupUsageBytes(dir)).toBe(18 * GIB); + }); + + it("subtracts reclaimable cache from cgroup v1 usage", () => { + const dir = path.join(tmpRoot, "cg-v1"); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, "memory.usage_in_bytes"), `${20 * GIB}`); + fs.writeFileSync(path.join(dir, "memory.stat"), memoryStat({ total_inactive_file: 15 * GIB })); + + expect(workerBudget.readCgroupUsageBytes(dir)).toBe(5 * GIB); + }); +}); + +describe("worker budget sizing", () => { + const base = { cpuCount: 96, memoryPerWorkerGib: 4, maxWorkers: 4 }; + + it("sizes against memory rather than the visible core count", () => { + expect(workerBudget.computeWorkers({ ...base, limitBytes: 32 * GIB, inUseBytes: 0 })).toBe(4); + }); + + it("shrinks the pool as co-tenants consume the shared cgroup", () => { + const workersAt = (inUseGib: number) => + workerBudget.computeWorkers({ ...base, limitBytes: 32 * GIB, inUseBytes: inUseGib * GIB }); + + expect(workersAt(16)).toBe(2); + expect(workersAt(22)).toBe(1); + }); + + it("keeps one worker even when the cgroup has no headroom left", () => { + expect( + workerBudget.computeWorkers({ ...base, limitBytes: 32 * GIB, inUseBytes: 32 * GIB }) + ).toBe(1); + }); + + it("never exceeds the core budget on small hosts", () => { + expect( + workerBudget.computeWorkers({ ...base, cpuCount: 2, limitBytes: 64 * GIB, inUseBytes: 0 }) + ).toBe(1); + }); +});