diff --git a/.github/workflows/build_reusable.yml b/.github/workflows/build_reusable.yml index aae09370ce089..1a39146387766 100644 --- a/.github/workflows/build_reusable.yml +++ b/.github/workflows/build_reusable.yml @@ -100,7 +100,9 @@ env: DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }} NEXT_JUNIT_TEST_REPORT: 'true' DD_ENV: 'ci' - TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }} + # Vercel KV Store for test timings + KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }} + KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }} NEXT_TEST_JOB: 1 VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }} VERCEL_TEST_TEAM: vtest314-next-e2e-tests diff --git a/.github/workflows/pull_request_stats.yml b/.github/workflows/pull_request_stats.yml index 14e2e76f8e09b..3b3afeee5da3b 100644 --- a/.github/workflows/pull_request_stats.yml +++ b/.github/workflows/pull_request_stats.yml @@ -16,7 +16,9 @@ env: # we build a dev binary for use in CI so skip downloading # canary next-swc binaries in the monorepo NEXT_SKIP_NATIVE_POSTINSTALL: 1 - TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }} + # Vercel KV Store for test timings + KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }} + KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }} NEXT_TEST_JOB: 1 NEXT_DISABLE_SWC_WASM: 1 diff --git a/package.json b/package.json index 924600c0f660d..6c672e329053f 100644 --- a/package.json +++ b/package.json @@ -152,7 +152,7 @@ "@typescript-eslint/eslint-plugin": "^8.36.0", "@typescript-eslint/parser": "^8.36.0", "@vercel/devlow-bench": "workspace:*", - "@vercel/fetch": "6.1.1", + "@vercel/kv": "3.0.0", "@vercel/og": "0.7.2", "abort-controller": "3.0.0", "alex": "9.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 96e0f28e52e57..5080543b43778 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -225,9 +225,9 @@ importers: '@vercel/devlow-bench': specifier: workspace:* version: link:turbopack/packages/devlow-bench - '@vercel/fetch': - specifier: 6.1.1 - version: 6.1.1(@types/node-fetch@2.6.1)(node-fetch@2.6.7(encoding@0.1.13)) + '@vercel/kv': + specifier: 3.0.0 + version: 3.0.0 '@vercel/og': specifier: 0.7.2 version: 0.7.2 @@ -5463,9 +5463,6 @@ packages: '@types/aria-query@5.0.1': resolution: {integrity: sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==} - '@types/async-retry@1.2.1': - resolution: {integrity: sha512-yMQ6CVgICWtyFNBqJT3zqOc+TnqqEPLo4nKJNPFwcialiylil38Ie6q1ENeFTjvaLOkVim9K5LisHgAKJWidGQ==} - '@types/async-retry@1.4.2': resolution: {integrity: sha512-GUDuJURF0YiJZ+CBjNQA0+vbP/VHlJbB0sFqkzsV7EcOPRfurVonXpXKAt3w8qIjM1TEzpz6hc6POocPvHOS3w==} @@ -5686,9 +5683,6 @@ packages: '@types/long@4.0.1': resolution: {integrity: sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==} - '@types/lru-cache@4.1.1': - resolution: {integrity: sha512-8mNEUG6diOrI6pMqOHrHPDBB1JsrpedeMK9AWGzVCQ7StRRribiT9BRvUmF8aUws9iBbVlgVekOT5Sgzc1MTKw==} - '@types/mdast@3.0.10': resolution: {integrity: sha512-W864tg/Osz1+9f4lrGTZpCSO5/z4608eUp19tbozkq2HJK6i3z1kT0H9tlADXuYIb1YYOBByU4Jsqkk75q48qA==} @@ -5710,9 +5704,6 @@ packages: '@types/mute-stream@0.0.4': resolution: {integrity: sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==} - '@types/node-fetch@2.3.2': - resolution: {integrity: sha512-yW0EOebSsQme9yKu09XbdDfle4/SmWZMK4dfteWcSLCYNQQcF+YOv0kIrvm+9pO11/ghA4E6A+RNQqvYj4Nr3A==} - '@types/node-fetch@2.6.1': resolution: {integrity: sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==} @@ -6028,21 +6019,12 @@ packages: '@ungap/structured-clone@1.2.0': resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} - '@vercel/fetch-cached-dns@2.0.2': - resolution: {integrity: sha512-gDqKEV8CeY2YmCdZpP1rn3tFK1L07Vw2+HYkCK8zpRHOVGr/sP8yhBsW+C/yqGVj0i9z/rIvqIHe5emvRvxwgw==} - peerDependencies: - node-fetch: '*' + '@upstash/redis@1.35.3': + resolution: {integrity: sha512-hSjv66NOuahW3MisRGlSgoszU2uONAY2l5Qo3Sae8OT3/Tng9K+2/cBRuyPBX8egwEGcNNCF9+r0V6grNnhL+w==} - '@vercel/fetch-retry@5.0.3': - resolution: {integrity: sha512-DIIoBY92r+sQ6iHSf5WjKiYvkdsDIMPWKYATlE0KcUAj2RV6SZK9UWpUzBRKsofXqedOqpVjrI0IE6AWL7JRtg==} - peerDependencies: - node-fetch: '*' - - '@vercel/fetch@6.1.1': - resolution: {integrity: sha512-nddCkgpA0aVIqOlzh+qVlzDNcQq0cSnqefM+x6SciGI4GCvVZeaZ7WEowgX8I/HwBAq8Uj5Bdnd+r0+sYsJsig==} - peerDependencies: - '@types/node-fetch': '2' - node-fetch: '2' + '@vercel/kv@3.0.0': + resolution: {integrity: sha512-pKT8fRnfyYk2MgvyB6fn6ipJPCdfZwiKDdw7vB+HL50rjboEBHDVBEcnwfkEpVSp2AjNtoaOUH7zG+bVC/rvSg==} + engines: {node: '>=14.6'} '@vercel/ncc@0.34.0': resolution: {integrity: sha512-G9h5ZLBJ/V57Ou9vz5hI8pda/YQX5HQszCs3AmIus3XzsmRn/0Ptic5otD3xVST8QLKk7AMk7AqpsyQGN7MZ9A==} @@ -6162,9 +6144,6 @@ packages: '@xtuc/long@4.2.2': resolution: {integrity: sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==} - '@zeit/dns-cached-resolve@2.1.2': - resolution: {integrity: sha512-A/5gbBskKPETTBqHwvlaW1Ri2orO62yqoFoXdxna1SQ7A/lXjpWgpJ1wdY3IQEcz5LydpS4sJ8SzI2gFyyLEhg==} - JSONStream@1.3.5: resolution: {integrity: sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==} hasBin: true @@ -6264,10 +6243,6 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} - agentkeepalive@3.4.1: - resolution: {integrity: sha512-MPIwsZU9PP9kOrZpyu2042kYA8Fdt/AedQYkYXucHgF9QoD9dXVp0ypuGnHXSR0hTstBxdt85Xkh4JolYfK5wg==} - engines: {node: '>= 4.0.0'} - agentkeepalive@4.1.4: resolution: {integrity: sha512-+V/rGa3EuU74H6wR04plBb7Ks10FbtUQgRj/FQOG7uUIEuaINI+AiqJR1k6t3SVNs7o7ZjIdus6706qqzVq8jQ==} engines: {node: '>= 8.0.0'} @@ -6778,6 +6753,7 @@ packages: binary-install@1.1.0: resolution: {integrity: sha512-rkwNGW+3aQVSZoD0/o3mfPN6Yxh3Id0R/xzTVBVVpGNlVz8EGwusksxRlbk/A5iKTZt9zkMn3qIqmAt3vpfbzg==} engines: {node: '>=10'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. bindings@1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} @@ -8867,6 +8843,7 @@ packages: eslint-plugin-markdown@3.0.1: resolution: {integrity: sha512-8rqoc148DWdGdmYF6WSQFT3uQ6PO7zXYgeBpHAOAakX/zpq+NvFYbDA/H7PYzHajwtmaOzAwfxyl++x0g1/N9A==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + deprecated: Please use @eslint/markdown instead peerDependencies: eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 @@ -13993,6 +13970,7 @@ packages: engines: {node: '>=0.6.0', teleport: '>=0.2.0'} deprecated: |- You or someone you depend on is using Q, the JavaScript Promise library that gave JavaScript developers strong feelings about promises. They can almost certainly migrate to the native JavaScript promise now. Thank you literally everyone for joining me in this bet against the odds. Be excellent to each other. + (For a CapTP with native promises, see @endo/eventual-send and @endo/captp) qs@6.11.0: @@ -16072,6 +16050,9 @@ packages: unbox-primitive@1.0.2: resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==} + uncrypto@0.1.3: + resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==} + undici-types@6.19.8: resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==} @@ -21388,8 +21369,6 @@ snapshots: '@types/aria-query@5.0.1': {} - '@types/async-retry@1.2.1': {} - '@types/async-retry@1.4.2': dependencies: '@types/retry': 0.12.0 @@ -21648,8 +21627,6 @@ snapshots: '@types/long@4.0.1': {} - '@types/lru-cache@4.1.1': {} - '@types/mdast@3.0.10': dependencies: '@types/unist': 2.0.3 @@ -21670,10 +21647,6 @@ snapshots: dependencies: '@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu) - '@types/node-fetch@2.3.2': - dependencies: - '@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu) - '@types/node-fetch@2.6.1': dependencies: '@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu) @@ -22113,31 +22086,13 @@ snapshots: '@ungap/structured-clone@1.2.0': {} - '@vercel/fetch-cached-dns@2.0.2(node-fetch@2.6.7(encoding@0.1.13))': + '@upstash/redis@1.35.3': dependencies: - '@types/node-fetch': 2.3.2 - '@zeit/dns-cached-resolve': 2.1.2 - node-fetch: 2.6.7(encoding@0.1.13) + uncrypto: 0.1.3 - '@vercel/fetch-retry@5.0.3(node-fetch@2.6.7(encoding@0.1.13))': + '@vercel/kv@3.0.0': dependencies: - async-retry: 1.3.1 - debug: 3.2.7 - node-fetch: 2.6.7(encoding@0.1.13) - transitivePeerDependencies: - - supports-color - - '@vercel/fetch@6.1.1(@types/node-fetch@2.6.1)(node-fetch@2.6.7(encoding@0.1.13))': - dependencies: - '@types/async-retry': 1.2.1 - '@types/node-fetch': 2.6.1 - '@vercel/fetch-cached-dns': 2.0.2(node-fetch@2.6.7(encoding@0.1.13)) - '@vercel/fetch-retry': 5.0.3(node-fetch@2.6.7(encoding@0.1.13)) - agentkeepalive: 3.4.1 - debug: 3.1.0 - node-fetch: 2.6.7(encoding@0.1.13) - transitivePeerDependencies: - - supports-color + '@upstash/redis': 1.35.3 '@vercel/ncc@0.34.0': {} @@ -22331,14 +22286,6 @@ snapshots: '@xtuc/long@4.2.2': {} - '@zeit/dns-cached-resolve@2.1.2': - dependencies: - '@types/async-retry': 1.2.1 - '@types/lru-cache': 4.1.1 - '@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu) - async-retry: 1.2.3 - lru-cache: 5.1.1 - JSONStream@1.3.5: dependencies: jsonparse: 1.3.1 @@ -22423,10 +22370,6 @@ snapshots: transitivePeerDependencies: - supports-color - agentkeepalive@3.4.1: - dependencies: - humanize-ms: 1.2.1 - agentkeepalive@4.1.4: dependencies: debug: 4.1.1 @@ -34731,6 +34674,8 @@ snapshots: has-symbols: 1.0.3 which-boxed-primitive: 1.0.2 + uncrypto@0.1.3: {} + undici-types@6.19.8: {} undici@5.26.3: diff --git a/run-tests.js b/run-tests.js index ea38905c5a25c..94437aa37c79d 100644 --- a/run-tests.js +++ b/run-tests.js @@ -4,10 +4,7 @@ const path = require('path') const _glob = require('glob') const { existsSync } = require('fs') const fsp = require('fs/promises') -const nodeFetch = require('node-fetch') -const vercelFetch = require('@vercel/fetch') -// @ts-expect-error -const fetch = vercelFetch(nodeFetch) +const { createClient } = require('@vercel/kv') const { promisify } = require('util') const { Sema } = require('async-sema') const { spawn, exec: execOrig } = require('child_process') @@ -65,14 +62,47 @@ const shouldContinueTestsOnError = !!process.env.NEXT_TEST_CONTINUE_ON_ERROR const skipRetryTestManifest = process.env.NEXT_TEST_SKIP_RETRY_MANIFEST ? require(process.env.NEXT_TEST_SKIP_RETRY_MANIFEST) : [] -const TIMINGS_API = `https://api.github.com/gists/4500dd89ae2f5d70d9aaceb191f528d1` -const TIMINGS_API_HEADERS = { - Accept: 'application/vnd.github.v3+json', - ...(process.env.TEST_TIMINGS_TOKEN - ? { - Authorization: `Bearer ${process.env.TEST_TIMINGS_TOKEN}`, +const KV_TIMINGS_KEY = 'test-timings' + +const kvClient = + process.env.KV_REST_API_URL && process.env.KV_REST_API_TOKEN + ? createClient({ + url: process.env.KV_REST_API_URL, + token: process.env.KV_REST_API_TOKEN, + }) + : null + +/** + * Retry a KV operation with exponential backoff + * @param {() => Promise} operation - The async operation to retry + * @param {string} operationName - Name of the operation for logging + * @param {number} maxRetries - Maximum number of retries (default: 3) + * @returns {Promise} The result of the operation + */ +async function retryKVOperation(operation, operationName, maxRetries = 3) { + let lastError + let retries = maxRetries + + while (retries > 0) { + try { + return await operation() + } catch (err) { + lastError = err + retries-- + if (retries > 0) { + const delay = (maxRetries - retries + 1) * 5 // 5s, 10s, 15s backoff + console.log( + `KV ${operationName} failed, retrying in ${delay}s. Error:`, + err.message + ) + await new Promise((resolve) => setTimeout(resolve, delay * 1000)) } - : {}), + } + } + + throw new Error( + `Failed to ${operationName} after ${maxRetries} retries: ${lastError?.message}` + ) } const testFilters = { @@ -168,28 +198,19 @@ const isMatchingPattern = (pattern, file) => { } async function getTestTimings() { - let timingsRes - - const doFetch = () => - fetch(TIMINGS_API, { - headers: { - ...TIMINGS_API_HEADERS, - }, - }) - timingsRes = await doFetch() - - if (timingsRes.status === 403) { - const delay = 15 - console.log(`Got 403 response waiting ${delay} seconds before retry`) - await new Promise((resolve) => setTimeout(resolve, delay * 1000)) - timingsRes = await doFetch() + if (!kvClient) { + console.warn('KV client not configured, skipping timing fetch') + return null } - if (!timingsRes.ok) { - throw new Error(`request status: ${timingsRes.status}`) - } - const timingsData = await timingsRes.json() - return JSON.parse(timingsData.files['test-timings.json'].content) + const timings = await retryKVOperation(async () => { + const data = await kvClient.get(KV_TIMINGS_KEY) + if (!data) { + console.log('No timing data found in KV store') + } + return data + }, 'fetch timings') + return timings || null } async function main() { @@ -243,6 +264,10 @@ async function main() { process.env.NEXT_TEST_MODE ) + // Only fetch/update shared timing data during grouped CI runs to avoid + // individual test runs from polluting the timing data + const shouldUseSharedTimings = options.timings && options.group + /** @type TestFile[] */ let tests = argv._.filter((arg) => arg.toString().match(/\.test\.(js|ts|tsx)/) @@ -302,30 +327,45 @@ async function main() { // } - if (options.timings && options.group) { + if (shouldUseSharedTimings) { console.log('Fetching previous timings data') + const timingsFile = path.join(process.cwd(), 'test-timings.json') + try { - const timingsFile = path.join(process.cwd(), 'test-timings.json') - try { - prevTimings = JSON.parse(await fsp.readFile(timingsFile, 'utf8')) - console.log('Loaded test timings from disk successfully') - } catch (_) { - console.error('failed to load from disk', _) - } + prevTimings = JSON.parse(await fsp.readFile(timingsFile, 'utf8')) + console.log('Loaded test timings from disk successfully') + } catch (_) { + console.error( + 'Failed to load test timings from disk. Proceeding to fetch from KV store. Original error: ', + _ + ) + } - if (!prevTimings) { + if (!prevTimings) { + try { prevTimings = await getTestTimings() - console.log('Fetched previous timings data successfully') + if (prevTimings) { + console.log('Fetched previous timings data successfully from KV') + } else { + console.log('No previous timings data available') + } + } catch (kvError) { + console.warn( + 'Failed to fetch timings from KV, continuing without timing data:', + kvError.message + ) + prevTimings = null + } - if (options.writeTimings) { + if (options.writeTimings) { + if (prevTimings) { await fsp.writeFile(timingsFile, JSON.stringify(prevTimings)) console.log('Wrote previous timings data to', timingsFile) - await cleanUpAndExit(0) + } else { + console.log('No timings data to write') } + await cleanUpAndExit(0) } - } catch (err) { - console.log(`Failed to fetch timings data`, err) - await cleanUpAndExit(1) } } @@ -389,6 +429,14 @@ async function main() { // tests tend not to get clustered together tests = tests.filter((_value, idx) => idx % groupTotal === groupPos - 1) console.log('Splitting without timings') + + // Warn in CI that tests are not optimally distributed + if (process.env.GITHUB_ACTIONS) { + core.warning( + `Test timing data unavailable for group ${options.group}. Tests are being distributed round-robin, which may increase CI time. ` + + `Consider checking KV store connectivity if this persists.` + ) + } } } @@ -773,43 +821,34 @@ ${ENDGROUP}`) // junitData += `` // console.log('output timing data to junit.xml') - if (prevTimings && process.env.TEST_TIMINGS_TOKEN) { - try { - const newTimings = { - ...(await getTestTimings()), - ...curTimings, - } + if (shouldUseSharedTimings) { + if (kvClient) { + try { + // Fetch existing timings and merge with new ones + const existingTimings = (await getTestTimings()) || {} + const newTimings = { + ...existingTimings, + ...curTimings, + } - for (const test of Object.keys(newTimings)) { - if (!existsSync(path.join(__dirname, test))) { - console.log('removing stale timing', test) - delete newTimings[test] + // Clean up stale timings for deleted tests + for (const test of Object.keys(newTimings)) { + if (!existsSync(path.join(__dirname, test))) { + console.log('removing stale timing', test) + delete newTimings[test] + } } - } - const timingsRes = await fetch(TIMINGS_API, { - method: 'PATCH', - headers: { - ...TIMINGS_API_HEADERS, - }, - body: JSON.stringify({ - files: { - 'test-timings.json': { - content: JSON.stringify(newTimings), - }, - }, - }), - }) - - if (!timingsRes.ok) { - throw new Error(`request status: ${timingsRes.status}`) + // Update KV store with retries + await retryKVOperation(async () => { + await kvClient.set(KV_TIMINGS_KEY, newTimings) + console.log('Successfully updated test timings in KV store') + }, 'update timings') + } catch (err) { + console.log('Failed to update timings data', err) } - const result = await timingsRes.json() - console.log( - `Sent updated timings successfully. API URL: "${result?.url}" HTML URL: "${result?.html_url}"` - ) - } catch (err) { - console.log('Failed to update timings data', err) + } else { + console.warn('KV client not configured, skipping timing update') } } } diff --git a/turbo.json b/turbo.json index 8c87b7671c847..b7d5d171d1c2b 100644 --- a/turbo.json +++ b/turbo.json @@ -23,7 +23,8 @@ "//#typescript": {}, "//#get-test-timings": { "inputs": ["run-tests.js"], - "outputs": ["test-timings.json"] + "outputs": ["test-timings.json"], + "env": ["KV_REST_API_URL", "KV_REST_API_TOKEN"] } }, "ui": "tui"