diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8aa8570..8eac624 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,6 +122,11 @@ jobs: - name: Build run: pnpm build + # prepublishOnly enforces this again, but CI should catch growth on the PR + # that introduces it rather than waiting for a maintainer to publish. + - name: Check quality-tools package size + run: pnpm --filter @shiplightai/quality-tools check:size + # The 1% size gate is otherwise only wired into quality-ui's # prepublishOnly, so a regression would surface at publish time rather # than on the PR that caused it. It needs the build output above. diff --git a/AGENTS.md b/AGENTS.md index a403df5..8770a86 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,15 @@ Quality evaluates evidence independently of the systems that produce it. Do not introduce dependencies from the engine into evidence producers or from open-source packages into the Shiplight platform monorepo. +## Release size gate + +The `quality-tools` release artifact may grow by at most 1% in both packed and +unpacked size relative to the current published npm version. A larger increase +requires a human maintainer to add an exact, version-specific +`approvedIncrease`, including their name and reason, to +`packages/quality-tools/package-size.json`. Agents must not add, modify, or +claim this approval on a human's behalf. + ## Extraction discipline During migration, preserve behavior before reorganizing it. Move regression diff --git a/packages/quality-tools/package-size.json b/packages/quality-tools/package-size.json index d6ec1a8..5e0ca99 100644 --- a/packages/quality-tools/package-size.json +++ b/packages/quality-tools/package-size.json @@ -1,6 +1,4 @@ { - "baselineVersion": "0.3.0", - "baselinePackedBytes": 70045, - "baselineUnpackedBytes": 257361, - "maxIncreasePercent": 1 + "maxIncreasePercent": 1, + "approvedIncrease": null } diff --git a/packages/quality-tools/scripts/check-package-size.mjs b/packages/quality-tools/scripts/check-package-size.mjs index d992a9f..d1c909c 100644 --- a/packages/quality-tools/scripts/check-package-size.mjs +++ b/packages/quality-tools/scripts/check-package-size.mjs @@ -1,79 +1,202 @@ #!/usr/bin/env node -/* global console */ +/* global AbortSignal, console, fetch */ import { execFileSync } from "node:child_process"; -import { readFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { createHash } from "node:crypto"; +import { + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, isAbsolute, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { evaluatePackageSize } from "./package-size-policy.mjs"; const scriptDir = dirname(fileURLToPath(import.meta.url)); const packageRoot = resolve(scriptDir, ".."); -const baselinePath = resolve(packageRoot, "package-size.json"); -const baseline = JSON.parse(readFileSync(baselinePath, "utf8")); -const maxIncreasePercent = Number(baseline.maxIncreasePercent); -const baselinePackedSize = Number(baseline.baselinePackedBytes); -const baselineUnpackedSize = Number(baseline.baselineUnpackedBytes); +const packageManifest = JSON.parse(readFileSync(resolve(packageRoot, "package.json"), "utf8")); +const policy = JSON.parse(readFileSync(resolve(packageRoot, "package-size.json"), "utf8")); +const packageName = String(packageManifest.name); +const packageVersion = String(packageManifest.version); -if (!Number.isFinite(maxIncreasePercent) || maxIncreasePercent < 0) { - throw new Error("package-size.json maxIncreasePercent must be a non-negative number."); +// The comparison deliberately fails closed when npm is unavailable: publishing +// without a known previous artifact would bypass the release policy. The error +// distinguishes registry access failures from an actual size violation. +let registryOutput; +try { + registryOutput = execFileSync( + "npm", + [ + "view", + `${packageName}@latest`, + "version", + "dist.tarball", + "dist.unpackedSize", + "dist.integrity", + "versions", + "--json" + ], + { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 30_000 } + ); +} catch (cause) { + throw new Error(`Could not read the published ${packageName} baseline from npm.`, { cause }); +} +const registry = JSON.parse(registryOutput); +const baselineVersion = String(registry.version ?? ""); +const baselineTarball = String(registry["dist.tarball"] ?? ""); +const baselineUnpackedBytes = Number(registry["dist.unpackedSize"]); +const baselineIntegrity = String(registry["dist.integrity"] ?? ""); +if (baselineVersion.length === 0 || baselineTarball.length === 0) { + throw new Error(`npm did not return the current published ${packageName} release.`); +} +if (!Number.isInteger(baselineUnpackedBytes) || baselineUnpackedBytes <= 0) { + throw new Error(`npm did not return the unpacked size for ${packageName}@${baselineVersion}.`); } -if (!Number.isInteger(baselinePackedSize) || baselinePackedSize <= 0) { - throw new Error("package-size.json baselinePackedBytes must be a positive integer."); +const integrityMatch = /^([a-z0-9]+)-([A-Za-z0-9+/=]+)$/u.exec(baselineIntegrity); +if (integrityMatch === null) { + throw new Error(`npm did not return valid integrity metadata for ${packageName}@${baselineVersion}.`); +} + +const publishRegistry = new URL(String(packageManifest.publishConfig?.registry ?? "")); +const baselineUrl = new URL(baselineTarball); +if (baselineUrl.origin !== publishRegistry.origin || baselineUrl.protocol !== "https:") { + throw new Error( + `npm returned a tarball outside the configured HTTPS registry: ${baselineUrl.origin}.` + ); } -if (!Number.isInteger(baselineUnpackedSize) || baselineUnpackedSize <= 0) { - throw new Error("package-size.json baselineUnpackedBytes must be a positive integer."); + +const publishedVersions = registry.versions; +if ( + Array.isArray(publishedVersions) && + publishedVersions.includes(packageVersion) && + packageVersion !== baselineVersion +) { + throw new Error( + `${packageName}@${packageVersion} is older than the current published release ${baselineVersion}.` + ); } -// `npm pack --dry-run` only measures size — no tarball, no workspace:* resolution — so the "never npm pack" publish rule does not apply here. -const output = execFileSync("npm", ["pack", "--dry-run", "--json"], { - cwd: packageRoot, - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"] -}); -const [pack] = JSON.parse(output); -if (pack === undefined || !Number.isInteger(pack.size) || !Number.isInteger(pack.unpackedSize)) { - throw new Error("npm pack --dry-run did not return package sizes."); +let baselineResponse; +try { + baselineResponse = await fetch(baselineUrl, { signal: AbortSignal.timeout(60_000) }); +} catch (cause) { + throw new Error(`Could not download ${packageName}@${baselineVersion} from npm.`, { cause }); +} +if (!baselineResponse.ok) { + throw new Error( + `Could not download ${packageName}@${baselineVersion} for the size comparison: HTTP ${baselineResponse.status}.` + ); } -if (!Array.isArray(pack.files)) { - throw new Error("npm pack --dry-run did not return package file entries."); +const baselineBytes = Buffer.from(await baselineResponse.arrayBuffer()); +const baselineDigest = createHash(integrityMatch[1]).update(baselineBytes).digest("base64"); +if (baselineDigest !== integrityMatch[2]) { + throw new Error(`The downloaded ${packageName}@${baselineVersion} tarball failed its npm integrity check.`); } +const baselinePackedBytes = baselineBytes.byteLength; -const allowedFilePatterns = [ - /^README\.md$/u, - /^package\.json$/u, - /^dist\/[^/]+\.js$/u, - /^dist\/[^/]+\.d\.ts$/u, - // The quality-map JSON Schema shipped as a resolvable asset (exports["./quality-map.schema.json"]). - /^dist\/quality-map\.schema\.json$/u, - // The canonical workflow-observation contract (exports["./quality-observations.schema.json"]). - /^dist\/quality-observations\.schema\.json$/u -]; +const packRoot = mkdtempSync(join(tmpdir(), "quality-tools-size-")); +let pack; +let currentPackedBytes; +let currentUnpackedBytes; +let measurementComplete = false; +try { + const packOutput = execFileSync( + "pnpm", + ["pack", "--pack-destination", packRoot, "--json"], + { cwd: packageRoot, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] } + ); + pack = JSON.parse(packOutput); + const archiveName = String(pack.filename ?? ""); + if (archiveName.length === 0 || !Array.isArray(pack.files)) { + throw new Error("pnpm pack did not return the package archive and file list."); + } + // Current pnpm returns an absolute filename. Resolve a bare filename against + // --pack-destination as well so a harmless output-format change cannot turn + // the release check into an opaque ENOENT. + const archivePath = isAbsolute(archiveName) + ? archiveName + : resolve(packRoot, basename(archiveName)); + currentPackedBytes = statSync(archivePath).size; -for (const file of pack.files) { - const filePath = String(file.path ?? ""); - if (filePath.endsWith(".map")) { - throw new Error(`Source map must not be included in the npm package: ${filePath}`); + // npm-compatible tarballs always extract under package/. + const extractRoot = resolve(packRoot, "package"); + execFileSync("tar", ["-xzf", archivePath, "-C", packRoot], { + stdio: ["ignore", "ignore", "pipe"] + }); + if (!existsSync(extractRoot) || !statSync(extractRoot).isDirectory()) { + throw new Error("pnpm pack produced an archive without the expected package/ directory."); } - if (!allowedFilePatterns.some((pattern) => pattern.test(filePath))) { - throw new Error(`Unexpected file in npm package: ${filePath}`); + function directorySize(path) { + return readdirSync(path, { withFileTypes: true }).reduce((total, entry) => { + const entryPath = resolve(path, entry.name); + if (entry.isSymbolicLink()) { + throw new Error(`Symlink must not be included in the npm package: ${entryPath}`); + } + return total + (entry.isDirectory() ? directorySize(entryPath) : statSync(entryPath).size); + }, 0); } -} + currentUnpackedBytes = directorySize(extractRoot); -function checkSize(label, current, baselineValue) { - const maxSize = Math.floor(baselineValue * (1 + maxIncreasePercent / 100)); - const delta = current - baselineValue; - const percent = (delta / baselineValue) * 100; + const allowedFilePatterns = [ + /^LICENSE$/u, + /^README\.md$/u, + /^package\.json$/u, + /^dist\/[^/]+\.js$/u, + /^dist\/[^/]+\.d\.ts$/u, + /^dist\/quality-map\.schema\.json$/u, + /^dist\/quality-observations\.schema\.json$/u + ]; + for (const file of pack.files) { + const filePath = String(file.path ?? ""); + if (filePath.endsWith(".map")) { + throw new Error(`Source map must not be included in the npm package: ${filePath}`); + } + if (!allowedFilePatterns.some((pattern) => pattern.test(filePath))) { + throw new Error(`Unexpected file in npm package: ${filePath}`); + } + } + measurementComplete = true; +} finally { + // The archive exists only to measure the exact pnpm-published artifact. + // Defer cleanup after a successful measurement because policy evaluation + // below still owns the archive's lifecycle. A failed measurement has no + // second stage, so it cleans up here. + if (!measurementComplete) { + rmSync(packRoot, { recursive: true, force: true }); + } +} - console.log( - `@shiplightai/quality-tools ${label} size: ${current} bytes ` + - `(baseline ${baselineValue}, ${percent >= 0 ? "+" : ""}${percent.toFixed(2)}%, limit ${maxSize})` - ); +try { + const result = evaluatePackageSize({ + packageVersion, + currentPackedBytes, + currentUnpackedBytes, + baseline: { + version: baselineVersion, + packedBytes: baselinePackedBytes, + unpackedBytes: baselineUnpackedBytes + }, + maxIncreasePercent: Number(policy.maxIncreasePercent), + approvedIncrease: policy.approvedIncrease + }); - if (current > maxSize) { - throw new Error( - `${label} size ${current} exceeds the ${maxIncreasePercent}% release limit (${maxSize} bytes).` + for (const measurement of result.measurements) { + console.log( + `${packageName} ${measurement.label} size: ${measurement.current} bytes ` + + `(previous ${baselineVersion}: ${measurement.baseline}, ` + + `${measurement.percent >= 0 ? "+" : ""}${measurement.percent.toFixed(2)}%, ` + + `limit ${measurement.limit})` ); } + if (result.usedApproval) { + console.warn( + `${packageName}@${packageVersion} exceeds the standard size limit and uses the human approval ` + + `recorded by ${policy.approvedIncrease.approvedBy}: ${policy.approvedIncrease.reason}` + ); + } +} finally { + rmSync(packRoot, { recursive: true, force: true }); } - -checkSize("packed", pack.size, baselinePackedSize); -checkSize("unpacked", pack.unpackedSize, baselineUnpackedSize); diff --git a/packages/quality-tools/scripts/package-size-policy.mjs b/packages/quality-tools/scripts/package-size-policy.mjs new file mode 100644 index 0000000..f14f742 --- /dev/null +++ b/packages/quality-tools/scripts/package-size-policy.mjs @@ -0,0 +1,82 @@ +function limitFor(baselineBytes, maxIncreasePercent) { + return Math.floor(baselineBytes * (1 + maxIncreasePercent / 100)); +} + +function approvalProblem(approval, packageVersion, currentPackedBytes, currentUnpackedBytes) { + if (approval === null || approval === undefined) { + return "no approval is recorded"; + } + if (approval.version !== packageVersion) { + return `the recorded approval for ${String(approval.version)} does not apply to ${packageVersion}`; + } + if ( + !Number.isInteger(approval.packedBytes) || + !Number.isInteger(approval.unpackedBytes) || + approval.packedBytes < currentPackedBytes || + approval.unpackedBytes < currentUnpackedBytes + ) { + return "the recorded approval does not cover the measured artifact"; + } + if (typeof approval.approvedBy !== "string" || approval.approvedBy.trim().length === 0) { + return "the recorded approval does not identify its human approver"; + } + if (typeof approval.reason !== "string" || approval.reason.trim().length === 0) { + return "the recorded approval does not explain why the increase is accepted"; + } + return undefined; +} + +export function evaluatePackageSize(input) { + if (!Number.isFinite(input.maxIncreasePercent) || input.maxIncreasePercent < 0) { + throw new Error("package-size.json maxIncreasePercent must be a non-negative number."); + } + for (const key of ["packedBytes", "unpackedBytes"]) { + if (!Number.isInteger(input.baseline[key]) || input.baseline[key] <= 0) { + throw new Error(`baseline ${key} must be a positive integer.`); + } + } + + const measurements = [ + { + label: "packed", + current: input.currentPackedBytes, + baseline: input.baseline.packedBytes + }, + { + label: "unpacked", + current: input.currentUnpackedBytes, + baseline: input.baseline.unpackedBytes + } + ].map((measurement) => ({ + ...measurement, + limit: limitFor(measurement.baseline, input.maxIncreasePercent), + percent: ((measurement.current - measurement.baseline) / measurement.baseline) * 100 + })); + + const exceeded = measurements.filter((measurement) => measurement.current > measurement.limit); + if (exceeded.length === 0) { + return { measurements, usedApproval: false }; + } + + const problem = approvalProblem( + input.approvedIncrease, + input.packageVersion, + input.currentPackedBytes, + input.currentUnpackedBytes + ); + if (problem !== undefined) { + const details = exceeded + .map( + ({ label, current, limit }) => + `${label} size ${current} exceeds the ${input.maxIncreasePercent}% release limit (${limit} bytes)` + ) + .join("; "); + throw new Error( + `${details}; explicit human approval is required because ${problem}. ` + + "A human maintainer may record an exact, version-specific approvedIncrease in package-size.json. " + + "Agents must not create or claim that approval." + ); + } + + return { measurements, usedApproval: true }; +} diff --git a/packages/quality-tools/scripts/package-size-policy.test.ts b/packages/quality-tools/scripts/package-size-policy.test.ts new file mode 100644 index 0000000..dfbae70 --- /dev/null +++ b/packages/quality-tools/scripts/package-size-policy.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest"; +import { evaluatePackageSize } from "./package-size-policy.mjs"; + +const baseline = { + version: "0.3.2", + packedBytes: 10_000, + unpackedBytes: 100_000 +}; + +describe("quality-tools package-size policy", () => { + it("allows growth up to one percent over the published release", () => { + const result = evaluatePackageSize({ + packageVersion: "0.3.3", + currentPackedBytes: 10_100, + currentUnpackedBytes: 101_000, + baseline, + maxIncreasePercent: 1, + approvedIncrease: null + }); + expect(result.usedApproval).toBe(false); + }); + + it("rejects growth above one percent without explicit human approval", () => { + expect(() => + evaluatePackageSize({ + packageVersion: "0.3.3", + currentPackedBytes: 10_101, + currentUnpackedBytes: 100_000, + baseline, + maxIncreasePercent: 1, + approvedIncrease: null + }) + ).toThrow(/explicit human approval/u); + }); + + it("accepts a human approval for the exact release and measured bounds", () => { + const result = evaluatePackageSize({ + packageVersion: "0.3.3", + currentPackedBytes: 10_200, + currentUnpackedBytes: 102_000, + baseline, + maxIncreasePercent: 1, + approvedIncrease: { + version: "0.3.3", + packedBytes: 10_200, + unpackedBytes: 102_000, + approvedBy: "Jane Maintainer", + reason: "Reviewed dependency required for the new command." + } + }); + expect(result.usedApproval).toBe(true); + }); + + it("rejects approval for another version or a smaller artifact", () => { + const common = { + packageVersion: "0.3.3", + currentPackedBytes: 10_200, + currentUnpackedBytes: 102_000, + baseline, + maxIncreasePercent: 1 + }; + expect(() => + evaluatePackageSize({ + ...common, + approvedIncrease: { + version: "0.3.4", + packedBytes: 10_200, + unpackedBytes: 102_000, + approvedBy: "Jane Maintainer", + reason: "Reviewed." + } + }) + ).toThrow(/does not apply to 0\.3\.3/u); + expect(() => + evaluatePackageSize({ + ...common, + approvedIncrease: { + version: "0.3.3", + packedBytes: 10_199, + unpackedBytes: 102_000, + approvedBy: "Jane Maintainer", + reason: "Reviewed." + } + }) + ).toThrow(/does not cover the measured artifact/u); + expect(() => + evaluatePackageSize({ + ...common, + approvedIncrease: { + version: "0.3.3", + packedBytes: 10_200, + unpackedBytes: 101_999, + approvedBy: "Jane Maintainer", + reason: "Reviewed." + } + }) + ).toThrow(/does not cover the measured artifact/u); + }); + + it("requires the human approver and reason to be explicit", () => { + const common = { + packageVersion: "0.3.3", + currentPackedBytes: 10_200, + currentUnpackedBytes: 102_000, + baseline, + maxIncreasePercent: 1 + }; + expect(() => + evaluatePackageSize({ + ...common, + approvedIncrease: { + version: "0.3.3", + packedBytes: 10_200, + unpackedBytes: 102_000, + approvedBy: " ", + reason: "Reviewed." + } + }) + ).toThrow(/does not identify its human approver/u); + expect(() => + evaluatePackageSize({ + ...common, + approvedIncrease: { + version: "0.3.3", + packedBytes: 10_200, + unpackedBytes: 102_000, + approvedBy: "Jane Maintainer", + reason: " " + } + }) + ).toThrow(/does not explain why/u); + }); + + it("rejects a non-positive published baseline", () => { + expect(() => + evaluatePackageSize({ + packageVersion: "0.3.3", + currentPackedBytes: 10_000, + currentUnpackedBytes: 100_000, + baseline: { ...baseline, packedBytes: 0 }, + maxIncreasePercent: 1, + approvedIncrease: null + }) + ).toThrow(/baseline packedBytes must be a positive integer/u); + }); +});