From 241a73b8380ea2af9722f9a7e32955ec2ec44d46 Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Tue, 7 Jul 2026 02:13:37 -0700 Subject: [PATCH] fix(analyzer): bound native-build registry concurrency --- .../src/analyzers/native-build.ts | 30 ++++++++++++++-- review-enrichment/test/native-build.test.ts | 35 +++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/review-enrichment/src/analyzers/native-build.ts b/review-enrichment/src/analyzers/native-build.ts index cd1b60a1be..53395b83d4 100644 --- a/review-enrichment/src/analyzers/native-build.ts +++ b/review-enrichment/src/analyzers/native-build.ts @@ -15,6 +15,7 @@ import { boundedFetchJson } from "../external-fetch.js"; const MAX_QUERIES = 25; const MAX_NPM_VERSION_JSON_BYTES = 256 * 1024; const MAX_PYPI_VERSION_JSON_BYTES = 2 * 1024 * 1024; +const MAX_CONCURRENT_REGISTRY_QUERIES = 4; const INSTALL_HOOKS = ["preinstall", "install", "postinstall"]; // Tokens in an install-lifecycle script that indicate a native toolchain runs on install. const NATIVE_TOOL_RE = @@ -175,6 +176,27 @@ async function fetchJson( return response.ok ? response.data : null; } +async function mapWithConcurrency( + items: T[], + concurrency: number, + fn: (item: T) => Promise, +): Promise { + const results = new Array(items.length); + let nextIndex = 0; + + async function worker(): Promise { + while (nextIndex < items.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await fn(items[index]!); + } + } + + const workerCount = Math.min(concurrency, items.length); + await Promise.all(Array.from({ length: workerCount }, () => worker())); + return results; +} + /** Analyzer entrypoint: added/changed deps → registry metadata → only the versions with a native-build install cost. */ export async function scanNativeBuild( req: EnrichRequest, @@ -186,8 +208,10 @@ export async function scanNativeBuild( const changes = extractDependencyChanges(req.files ?? []) .filter(isQueryable) .slice(0, options.limits?.maxQueries ?? MAX_QUERIES); - const results = await Promise.all( - changes.map(async (change): Promise => { + const results = await mapWithConcurrency( + changes, + MAX_CONCURRENT_REGISTRY_QUERIES, + async (change): Promise => { if (options.signal?.aborted) return null; if (change.ecosystem === "npm") { @@ -233,7 +257,7 @@ export async function scanNativeBuild( } } return null; - }), + }, ); const findings = results.filter((f): f is NativeBuildFinding => f !== null); return findings; diff --git a/review-enrichment/test/native-build.test.ts b/review-enrichment/test/native-build.test.ts index 07c129f805..75608ee654 100644 --- a/review-enrichment/test/native-build.test.ts +++ b/review-enrichment/test/native-build.test.ts @@ -29,6 +29,19 @@ const pypiAdd = (name, version = "1.0.0") => ({ }, ], }); +const pypiAdds = (count) => ({ + repoFullName: "o/r", + prNumber: 1, + files: [ + { + path: "requirements.txt", + patch: `@@ -1,0 +1,${count} @@\n${Array.from( + { length: count }, + (_, i) => `+native${i}==1.0.0`, + ).join("\n")}`, + }, + ], +}); const jsonResponse = (body, init) => new Response(JSON.stringify(body), init); const npmFetch = (meta) => async () => jsonResponse({ @@ -227,6 +240,28 @@ test("scanNativeBuild: the query cap counts only queryable changes (skips don't assert.equal(findings[0].package, "bcrypt"); }); +test("scanNativeBuild bounds concurrent registry fetches below the total query cap", async () => { + let active = 0; + let maxActive = 0; + let started = 0; + const findings = await scanNativeBuild( + pypiAdds(10), + async () => { + started += 1; + active += 1; + maxActive = Math.max(maxActive, active); + await Promise.resolve(); + active -= 1; + return jsonResponse({ urls: [{ packagetype: "sdist" }] }); + }, + { limits: { maxQueries: 10 } }, + ); + + assert.equal(started, 10); + assert.equal(maxActive, 4); + assert.equal(findings.length, 10); +}); + test("scanNativeBuild: a PyPI PEP 440 (non-semver) sdist-only version is flagged", async () => { const findings = await scanNativeBuild( pypiAdd("ujson", "24.1"),