From 3e387965080455290e65bb06f053be1c262fdfe6 Mon Sep 17 00:00:00 2001 From: Andriy Polanski Date: Wed, 22 Jul 2026 14:46:31 +0000 Subject: [PATCH] fix(miner): page contribution-profile label fetches past 100 (#8010) Follow GitHub Link rel=next the same way ci-poller pages check-runs so extractContributionProfile sees the full label set, not only page one. Co-authored-by: Cursor --- .../lib/contribution-profile-extract.ts | 64 +++++- .../unit/contribution-profile-extract.test.ts | 217 ++++++++++++++++++ 2 files changed, 271 insertions(+), 10 deletions(-) diff --git a/packages/loopover-miner/lib/contribution-profile-extract.ts b/packages/loopover-miner/lib/contribution-profile-extract.ts index af928c868a..4a2976fbdb 100644 --- a/packages/loopover-miner/lib/contribution-profile-extract.ts +++ b/packages/loopover-miner/lib/contribution-profile-extract.ts @@ -99,12 +99,12 @@ function githubHeaders(githubToken: string | undefined): Record * falling back to its fail-open contract: returns null on a non-retryable/exhausted HTTP, transport, or parse * failure. `timeoutMs` gives each attempt its own fresh `AbortSignal.timeout` (preserving the per-request bound), * and `sleepFn` is the injectable no-real-timers seam every other `fetchWithRetry` call site exposes. */ -async function getJson( +async function getJsonResponse( url: string, headers: Record, fetchImpl: typeof fetch, sleepFn: ((ms: number) => Promise) | undefined, -): Promise { +): Promise<{ payload: unknown; response: Response } | null> { let response: Response; try { // Cast: the JS always passes `sleepFn` (possibly undefined); EOPT rejects an explicit undefined optional. @@ -118,7 +118,57 @@ async function getJson( return null; } if (!response.ok) return null; - return response.json().catch(() => null); + const payload = await response.json().catch(() => null); + return { payload, response }; +} + +async function getJson( + url: string, + headers: Record, + fetchImpl: typeof fetch, + sleepFn: ((ms: number) => Promise) | undefined, +): Promise { + const result = await getJsonResponse(url, headers, fetchImpl, sleepFn); + return result?.payload ?? null; +} + +/** Same Link-header check as `ci-poller.ts`'s check-run pagination (#8010). */ +function hasNextLink(response: Response): boolean { + const link = + typeof response.headers?.get === "function" ? response.headers.get("link") : null; + return /<[^>]+>;\s*rel="next"/.test(link ?? ""); +} + +/** Cap runaway pagination the way opportunity-fanout caps `maxPages` — 50×100 covers pathological repos + * without inventing a different paging scheme than ci-poller's `page=` loop. */ +const MAX_LABEL_PAGES = 50; + +/** Fetch every label on the repo, following GitHub `Link: rel="next"` the same way `ci-poller.ts` pages + * check-runs (#8010). Fail-open: a failed/malformed page returns whatever was collected so far. */ +async function fetchRepoLabels( + base: string, + target: { owner: string; repo: string }, + headers: Record, + fetchImpl: typeof fetch, + sleepFn: ((ms: number) => Promise) | undefined, +): Promise { + const labels: GithubLabel[] = []; + for (let page = 1; page <= MAX_LABEL_PAGES; page += 1) { + const result = await getJsonResponse( + `${base}/repos/${target.owner}/${target.repo}/labels?per_page=100&page=${page}`, + headers, + fetchImpl, + sleepFn, + ); + if (result === null) return labels; + if (!Array.isArray(result.payload)) return labels; + const pageLabels = result.payload as GithubLabel[]; + labels.push(...pageLabels); + if (!hasNextLink(result.response)) return labels; + if (pageLabels.length === 0) return labels; + } + /* v8 ignore next -- defensive page cap; a real repo never has 5000+ labels. */ + return labels; } /** @@ -256,13 +306,7 @@ export async function extractContributionProfile( ); const sleepFn = options.sleepFn; - const labelsPayload = await getJson( - `${base}/repos/${target.owner}/${target.repo}/labels?per_page=100`, - headers, - fetchImpl, - sleepFn, - ); - const labels = Array.isArray(labelsPayload) ? (labelsPayload as GithubLabel[]) : []; + const labels = await fetchRepoLabels(base, target, headers, fetchImpl, sleepFn); const contributing = await fetchContributing( base, target, diff --git a/test/unit/contribution-profile-extract.test.ts b/test/unit/contribution-profile-extract.test.ts index fcf966b565..8ffaafc3ea 100644 --- a/test/unit/contribution-profile-extract.test.ts +++ b/test/unit/contribution-profile-extract.test.ts @@ -18,6 +18,7 @@ function stubFetch( contributingGithubDir?: string | null; } = {}, ) { + const emptyHeaders = { get: (_name: string) => null }; return asFetch( vi.fn(async (url: string) => { const u = String(url); @@ -26,11 +27,13 @@ function stubFetch( return { ok: false, status: opts.labels, + headers: emptyHeaders, json: async () => ({}), } as unknown as Response; return { ok: true, status: 200, + headers: emptyHeaders, json: async () => opts.labels ?? [], } as unknown as Response; } @@ -39,11 +42,13 @@ function stubFetch( return { ok: false, status: 404, + headers: emptyHeaders, json: async () => ({}), } as unknown as Response; return { ok: true, status: 200, + headers: emptyHeaders, json: async () => ({ encoding: "base64", content: Buffer.from(String(opts.contributing)).toString("base64"), @@ -55,11 +60,13 @@ function stubFetch( return { ok: false, status: 404, + headers: emptyHeaders, json: async () => ({}), } as unknown as Response; return { ok: true, status: 200, + headers: emptyHeaders, json: async () => ({ encoding: "base64", content: Buffer.from(String(opts.contributingGithubDir)).toString( @@ -71,6 +78,7 @@ function stubFetch( return { ok: false, status: 404, + headers: emptyHeaders, json: async () => ({}), } as unknown as Response; }), @@ -246,9 +254,215 @@ describe("extractContributionProfile (#6796)", () => { expect(profile.completeness).toBe("absent"); }); + it("pages through Link rel=next and collects labels past the first 100 (#8010)", async () => { + const emptyHeaders = { get: (_name: string) => null }; + let labelsCalls = 0; + const page1 = Array.from({ length: 100 }, (_, i) => ({ + name: `bulk-${i}`, + description: null as string | null, + })); + const fetchImpl = vi.fn(async (url: string) => { + const u = String(url); + if (u.includes("/labels")) { + labelsCalls += 1; + if (labelsCalls === 1) { + expect(u).toContain("page=1"); + return { + ok: true, + status: 200, + headers: { + get: (name: string) => + name.toLowerCase() === "link" + ? '; rel="next"' + : null, + }, + json: async () => page1, + } as unknown as Response; + } + expect(u).toContain("page=2"); + return { + ok: true, + status: 200, + headers: emptyHeaders, + json: async () => [ + { name: "good first issue", description: "Starter work" }, + { name: "bulk-extra", description: null }, + ], + } as unknown as Response; + } + return { + ok: false, + status: 404, + headers: emptyHeaders, + json: async () => ({}), + } as unknown as Response; + }); + + const profile = await extractContributionProfile("acme/widgets", { + fetchImpl: asFetch(fetchImpl), + generatedAt: AT, + }); + + expect(labelsCalls).toBe(2); + expect(profile.eligibilityLabels.confidence).toBe("explicit"); + expect(profile.eligibilityLabels.value).toEqual([ + { field: "name", contains: "good first issue" }, + ]); + expect(profile.eligibilityLabels.provenance).toEqual([ + { source: "labels", detail: "good first issue" }, + ]); + }); + + it("keeps page-1 labels when a later page fails (fail-open pagination, #8010)", async () => { + const emptyHeaders = { get: (_name: string) => null }; + let labelsCalls = 0; + const fetchImpl = vi.fn(async (url: string) => { + const u = String(url); + if (u.includes("/labels")) { + labelsCalls += 1; + if (labelsCalls === 1) { + return { + ok: true, + status: 200, + headers: { + get: (name: string) => + name.toLowerCase() === "link" + ? '; rel="next"' + : null, + }, + json: async () => [{ name: "help wanted", description: null }], + } as unknown as Response; + } + return { + ok: false, + status: 500, + headers: emptyHeaders, + json: async () => ({}), + } as unknown as Response; + } + return { + ok: false, + status: 404, + headers: emptyHeaders, + json: async () => ({}), + } as unknown as Response; + }); + + const profile = await extractContributionProfile("acme/widgets", { + fetchImpl: asFetch(fetchImpl), + generatedAt: AT, + sleepFn: async () => {}, + }); + + expect(labelsCalls).toBeGreaterThanOrEqual(2); + expect(profile.eligibilityLabels.confidence).toBe("explicit"); + expect(profile.eligibilityLabels.value).toEqual([ + { field: "name", contains: "help wanted" }, + ]); + }); + + it("stops pagination when a next page returns an empty array (#8010)", async () => { + const emptyHeaders = { get: (_name: string) => null }; + let labelsCalls = 0; + const fetchImpl = vi.fn(async (url: string) => { + const u = String(url); + if (u.includes("/labels")) { + labelsCalls += 1; + if (labelsCalls === 1) { + return { + ok: true, + status: 200, + headers: { + get: (name: string) => + name.toLowerCase() === "link" + ? '; rel="next"' + : null, + }, + json: async () => [{ name: "help wanted", description: null }], + } as unknown as Response; + } + // Empty page still advertises a next link — the empty-page guard must stop the loop. + return { + ok: true, + status: 200, + headers: { + get: (name: string) => + name.toLowerCase() === "link" + ? '; rel="next"' + : null, + }, + json: async () => [], + } as unknown as Response; + } + return { + ok: false, + status: 404, + headers: emptyHeaders, + json: async () => ({}), + } as unknown as Response; + }); + + const profile = await extractContributionProfile("acme/widgets", { + fetchImpl: asFetch(fetchImpl), + generatedAt: AT, + }); + + expect(labelsCalls).toBe(2); + expect(profile.eligibilityLabels.value).toEqual([ + { field: "name", contains: "help wanted" }, + ]); + }); + + it("stops pagination when a later page returns a non-array payload (#8010)", async () => { + const emptyHeaders = { get: (_name: string) => null }; + let labelsCalls = 0; + const fetchImpl = vi.fn(async (url: string) => { + const u = String(url); + if (u.includes("/labels")) { + labelsCalls += 1; + if (labelsCalls === 1) { + return { + ok: true, + status: 200, + headers: { + get: (name: string) => + name.toLowerCase() === "link" + ? '; rel="next"' + : null, + }, + json: async () => [{ name: "help wanted", description: null }], + } as unknown as Response; + } + return { + ok: true, + status: 200, + headers: emptyHeaders, + json: async () => ({ message: "weird" }), + } as unknown as Response; + } + return { + ok: false, + status: 404, + headers: emptyHeaders, + json: async () => ({}), + } as unknown as Response; + }); + + const profile = await extractContributionProfile("acme/widgets", { + fetchImpl: asFetch(fetchImpl), + generatedAt: AT, + }); + + expect(labelsCalls).toBe(2); + expect(profile.eligibilityLabels.value).toEqual([ + { field: "name", contains: "help wanted" }, + ]); + }); + it("retries a transient 5xx on the labels fetch and yields the same profile as an immediate success (#7090)", async () => { // A single 5xx blip on the first attempt must NOT degrade the label signal — the retry rides it out and the // resulting profile is identical to one where the labels fetch succeeded immediately. + const emptyHeaders = { get: (_name: string) => null }; const sleeps: number[] = []; let labelsCalls = 0; const fetchImpl = vi.fn(async (url: string) => { @@ -259,11 +473,13 @@ describe("extractContributionProfile (#6796)", () => { return { ok: false, status: 500, + headers: emptyHeaders, json: async () => ({}), } as unknown as Response; return { ok: true, status: 200, + headers: emptyHeaders, json: async () => [ { name: "help wanted", description: "Extra attention is needed" }, ], @@ -272,6 +488,7 @@ describe("extractContributionProfile (#6796)", () => { return { ok: false, status: 404, + headers: emptyHeaders, json: async () => ({}), } as unknown as Response; });