diff --git a/README.md b/README.md index e09ebf7..b83e7f2 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Using a static site generator to build your website? Do you simply need to serve ## Prerequisites -Node 18 or newer is required during the build step, as we now rely on its `experimental-fetch` feature. +Although your published application runs on a Fastly Compute service, the publishing process offered by this package requires Node.js 20 or newer. ## How it works diff --git a/package-lock.json b/package-lock.json index 69f8b62..0d34eb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,12 +19,12 @@ "@fastly/js-compute": "^3.0.0", "@types/command-line-args": "^5.2.0", "@types/glob-to-regexp": "^0.4.1", - "@types/node": "^18.0.0", + "@types/node": "^20.0.0", "rimraf": "^4.3.0", "typescript": "^5.0.2" }, "engines": { - "node": ">=18" + "node": ">=20" }, "peerDependencies": { "@fastly/js-compute": "^2.0.0 || ^3.0.0" @@ -1117,10 +1117,14 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.0.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.0.0.tgz", - "integrity": "sha512-cHlGmko4gWLVI27cGJntjs/Sj8th9aYwplmZFwmmgYQQvL5NUsgVJG7OddLvNfLqYS31KFN0s3qlaD9qCaxACA==", - "dev": true + "version": "20.17.27", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.27.tgz", + "integrity": "sha512-U58sbKhDrthHlxHRJw7ZLiLDZGmAUOZUbpw0S6nL27sYUdhvgBLCRu/keSd6qcTsfArd1sRFCCBxzWATGr/0UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.2" + } }, "node_modules/acorn": { "version": "8.14.1", @@ -2368,6 +2372,13 @@ "through": "^2.3.8" } }, + "node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true, + "license": "MIT" + }, "node_modules/unicode-canonical-property-names-ecmascript": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.1.tgz", @@ -3004,10 +3015,13 @@ "dev": true }, "@types/node": { - "version": "18.0.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.0.0.tgz", - "integrity": "sha512-cHlGmko4gWLVI27cGJntjs/Sj8th9aYwplmZFwmmgYQQvL5NUsgVJG7OddLvNfLqYS31KFN0s3qlaD9qCaxACA==", - "dev": true + "version": "20.17.27", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.27.tgz", + "integrity": "sha512-U58sbKhDrthHlxHRJw7ZLiLDZGmAUOZUbpw0S6nL27sYUdhvgBLCRu/keSd6qcTsfArd1sRFCCBxzWATGr/0UA==", + "dev": true, + "requires": { + "undici-types": "~6.19.2" + } }, "acorn": { "version": "8.14.1", @@ -3848,6 +3862,12 @@ "through": "^2.3.8" } }, + "undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true + }, "unicode-canonical-property-names-ecmascript": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.1.tgz", diff --git a/package.json b/package.json index 8b8e5b8..7be042c 100644 --- a/package.json +++ b/package.json @@ -41,12 +41,12 @@ "@fastly/js-compute": "^3.0.0", "@types/command-line-args": "^5.2.0", "@types/glob-to-regexp": "^0.4.1", - "@types/node": "^18.0.0", + "@types/node": "^20.0.0", "rimraf": "^4.3.0", "typescript": "^5.0.2" }, "engines": { - "node": ">=18" + "node": ">=20" }, "files": [ "build", diff --git a/src/cli/commands/build-static.ts b/src/cli/commands/build-static.ts index 76fee97..f5a79f3 100644 --- a/src/cli/commands/build-static.ts +++ b/src/cli/commands/build-static.ts @@ -68,7 +68,7 @@ import { applyDefaults } from "../util/data.js"; import { calculateFileSizeAndHash } from "../util/hash.js"; import { getFiles } from "../util/files.js"; import { generateOrLoadPublishId } from "../util/publish-id.js"; -import { FastlyApiContext, loadApiKey } from "../util/fastly-api.js"; +import { FastlyApiContext, FetchError, loadApiKey } from "../util/fastly-api.js"; import { kvStoreEntryExists, kvStoreSubmitFile } from "../util/kv-store.js"; import { mergeContentTypes, testFileContentType } from "../../util/content-types.js"; import { algs } from "../compression/index.js"; @@ -94,6 +94,9 @@ import type { ContentFileInfoForWasmInline, ContentFileInfoForKVStore, } from "../../types/content-assets.js"; +import { + attemptWithRetries, +} from "../util/retryable.js"; type AssetInfo = ContentTypeTestResult & @@ -123,17 +126,54 @@ type KVStoreItemDesc = { }; async function uploadFilesToKVStore(fastlyApiContext: FastlyApiContext, kvStoreName: string, kvStoreItems: KVStoreItemDesc[]) { - for (const { kvStoreKey, staticFilePath, text } of kvStoreItems) { - if (await kvStoreEntryExists(fastlyApiContext, kvStoreName, kvStoreKey)) { - // Already exists in KV Store - console.log(`✔️ Asset already exists in KV Store with key "${kvStoreKey}".`) - } else { - // Upload to KV Store - const fileData = fs.readFileSync(staticFilePath); - await kvStoreSubmitFile(fastlyApiContext!, kvStoreName!, kvStoreKey, fileData); - console.log(`✔️ Submitted ${text ? 'text' : 'binary'} asset "${staticFilePath}" to KV Store with key "${kvStoreKey}".`) + + const maxConcurrent = 12; + let index = 0; // Shared among workers + + async function worker() { + while (index < kvStoreItems.length) { + const currentIndex = index; + index = index + 1; + const { kvStoreKey, staticFilePath, text } = kvStoreItems[currentIndex]; + + try { + await attemptWithRetries( + async() => { + if (await kvStoreEntryExists(fastlyApiContext, kvStoreName, kvStoreKey)) { + console.log(`✔️ Asset already exists in KV Store with key "${kvStoreKey}".`); + return; + } + const fileData = fs.readFileSync(staticFilePath); + await kvStoreSubmitFile(fastlyApiContext, kvStoreName, kvStoreKey, fileData); + console.log(`✔️ Submitted ${text ? 'text' : 'binary'} asset "${staticFilePath}" to KV Store with key "${kvStoreKey}".`) + }, + { + onAttempt(attempt) { + if (attempt > 0) { + console.log(`Attempt ${attempt + 1} for: ${kvStoreKey}`); + } + }, + onRetry(attempt, err, delay) { + let statusMessage = 'unknown'; + if (err instanceof FetchError) { + statusMessage = `HTTP ${err.status}`; + } else if (err instanceof TypeError) { + statusMessage = 'transport'; + } + console.log(`Attempt ${attempt + 1} for ${kvStoreKey} gave retryable error (${statusMessage}), delaying ${delay} ms`); + }, + } + ); + } catch (err) { + const e = err instanceof Error ? err : new Error(String(err)); + console.error(`❌ Failed: ${kvStoreKey} → ${e.message}`); + console.error(e.stack); + } } } + + const workers = Array.from({ length: maxConcurrent }, () => worker()); + await Promise.all(workers); } function writeKVStoreEntriesToFastlyToml(kvStoreName: string, kvStoreItems: KVStoreItemDesc[]) { @@ -154,7 +194,7 @@ function writeKVStoreEntriesToFastlyToml(kvStoreName: string, kvStoreItems: KVSt if (fastlyToml.indexOf(kvStoreName) !== -1) { // don't do this! - console.error("improperly configured entry for '${kvStoreName}' in fastly.toml"); + console.error(`improperly configured entry for '${kvStoreName}' in fastly.toml`); // TODO: handle thrown exception from callers throw "No"! } diff --git a/src/cli/commands/init-app.ts b/src/cli/commands/init-app.ts index a03e4c3..35a0025 100644 --- a/src/cli/commands/init-app.ts +++ b/src/cli/commands/init-app.ts @@ -543,7 +543,7 @@ ${staticFiles} '@fastly/js-compute': '^3.0.0', }, engines: { - node: '>=18.0.0', + node: '>=20.0.0', }, license: 'UNLICENSED', private: true, diff --git a/src/cli/util/fastly-api.ts b/src/cli/util/fastly-api.ts index 1f46367..d52719c 100644 --- a/src/cli/util/fastly-api.ts +++ b/src/cli/util/fastly-api.ts @@ -1,4 +1,5 @@ import { execSync } from 'child_process'; +import { makeRetryable } from './retryable.js'; export interface FastlyApiContext { apiToken: string, @@ -42,7 +43,33 @@ export function loadApiKey(): LoadApiKeyResult | null { } -export async function callFastlyApi(fastlyApiContext: FastlyApiContext, endpoint: string, queryParams?: URLSearchParams | null, requestInit?: RequestInit): Promise { +const RETRYABLE_STATUS_CODES = [ + 408, // Request Timeout + 409, // Conflict (depends) + 423, // Locked + 429, // Too Many Requests + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout +]; + +export class FetchError extends Error { + constructor(message: string, status: number) { + super(message); + this.name = 'FetchError'; + this.status = status; + } + status: number; +} + +export async function callFastlyApi( + fastlyApiContext: FastlyApiContext, + endpoint: string, + operationName: string, + queryParams?: URLSearchParams | null, + requestInit?: RequestInit, +): Promise { let finalEndpoint = endpoint; if (queryParams != null) { @@ -60,8 +87,24 @@ export async function callFastlyApi(fastlyApiContext: FastlyApiContext, endpoint const request = new Request(url, { ...requestInit, headers, + redirect: 'error', }); - const response = await fetch(request); + let response; + try { + response = await fetch(request); + } catch(err) { + if (err instanceof TypeError) { + throw makeRetryable(err); + } else { + throw err; + } + } + if (!response.ok) { + if (!RETRYABLE_STATUS_CODES.includes(response.status)) { + throw new FetchError(`${operationName} failed: ${response.status}`, response.status); + } + throw makeRetryable(new FetchError(`Retryable ${operationName} error: ${response.status}`, response.status)); + } return response; } diff --git a/src/cli/util/kv-store.ts b/src/cli/util/kv-store.ts index ddaa5b3..3eb2733 100644 --- a/src/cli/util/kv-store.ts +++ b/src/cli/util/kv-store.ts @@ -1,4 +1,4 @@ -import { callFastlyApi, FastlyApiContext } from "./fastly-api.js"; +import { callFastlyApi, FastlyApiContext, FetchError } from "./fastly-api.js"; type KVStoreInfo = { id: string, @@ -22,7 +22,7 @@ type CachedValues = { const cache = new WeakMap(); -export async function getKVStoreIdForNameMap(fastlyApiContext: FastlyApiContext): Promise> { +export async function getKVStoreIdForNameMap(fastlyApiContext: FastlyApiContext) { const cacheEntry = cache.get(fastlyApiContext); let kvStoreNameMap = cacheEntry?.kvStoreNameMap; @@ -42,15 +42,16 @@ export async function getKVStoreIdForNameMap(fastlyApiContext: FastlyApiContext) return kvStoreNameMap; } -export async function getKVStoreIdForName(fastlyApiContext: FastlyApiContext, kvStoreName: string): Promise { +export async function getKVStoreIdForName(fastlyApiContext: FastlyApiContext, kvStoreName: string) { const kvStoreNameMap = await getKVStoreIdForNameMap(fastlyApiContext); return kvStoreNameMap[kvStoreName] ?? null; + } function createArrayGetter() { return function string>(fn: TFn) { - return async function(fastlyApiContext: FastlyApiContext, ...args: Parameters): Promise { + return async function(fastlyApiContext: FastlyApiContext, operationName: string, ...args: Parameters) { const results: TEntry[] = []; let cursor: string | null = null; @@ -62,10 +63,7 @@ function createArrayGetter() { const endpoint = fn(...args); - const response = await callFastlyApi(fastlyApiContext, endpoint, queryParams); - if (response.status !== 200) { - throw new Error('Unexpected data format'); - } + const response = await callFastlyApi(fastlyApiContext, endpoint, operationName, queryParams); const infos = await response.json() as DataAndMeta; @@ -86,7 +84,7 @@ function createArrayGetter() { const _getKVStoreInfos = createArrayGetter()(() => `/resources/stores/kv`); -export async function getKVStoreInfos(fastlyApiContext: FastlyApiContext): Promise { +export async function getKVStoreInfos(fastlyApiContext: FastlyApiContext) { const cacheEntry = cache.get(fastlyApiContext); @@ -95,7 +93,7 @@ export async function getKVStoreInfos(fastlyApiContext: FastlyApiContext): Promi return kvStoreInfos; } - kvStoreInfos = await _getKVStoreInfos(fastlyApiContext); + kvStoreInfos = await _getKVStoreInfos(fastlyApiContext, 'Listing KV Stores'); cache.set(fastlyApiContext, { ...cacheEntry, kvStoreInfos }); @@ -104,17 +102,17 @@ export async function getKVStoreInfos(fastlyApiContext: FastlyApiContext): Promi export const _getKVStoreKeys = createArrayGetter()((kvStoreId: string) => `/resources/stores/kv/${encodeURIComponent(kvStoreId)}/keys`); -export async function getKVStoreKeys(fastlyApiContext: FastlyApiContext, kvStoreName: string): Promise { +export async function getKVStoreKeys(fastlyApiContext: FastlyApiContext, kvStoreName: string) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { return null; } - return await _getKVStoreKeys(fastlyApiContext, kvStoreId); + return await _getKVStoreKeys(fastlyApiContext, `Listing Keys for KV Store [${kvStoreId}] ${kvStoreName}`, kvStoreId); } -export async function kvStoreEntryExists(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string): Promise { +export async function kvStoreEntryExists(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { @@ -123,14 +121,23 @@ export async function kvStoreEntryExists(fastlyApiContext: FastlyApiContext, kvS const endpoint = `/resources/stores/kv/${encodeURIComponent(kvStoreId)}/keys/${encodeURIComponent(key)}`; - const response = await callFastlyApi(fastlyApiContext, endpoint, null, { method: 'HEAD' }); + try { + + await callFastlyApi(fastlyApiContext, endpoint, `Checking existence of [${key}]`, null, { method: 'HEAD' }); + + } catch(err) { + if (err instanceof FetchError && err.status === 404) { + return false; + } + throw err; + } - return response.status === 200; + return true; } const encoder = new TextEncoder(); -export async function kvStoreSubmitFile(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string, data: Uint8Array | string): Promise { +export async function kvStoreSubmitFile(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string, data: Uint8Array | string) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { @@ -140,7 +147,7 @@ export async function kvStoreSubmitFile(fastlyApiContext: FastlyApiContext, kvSt const endpoint = `/resources/stores/kv/${encodeURIComponent(kvStoreId)}/keys/${encodeURIComponent(key)}`; const body = typeof data === 'string' ? encoder.encode(data) : data; - const response = await callFastlyApi(fastlyApiContext, endpoint, null, { + await callFastlyApi(fastlyApiContext, endpoint, `Submitting item [${key}]`, null, { method: 'PUT', headers: { 'content-type': 'application/octet-stream', @@ -148,13 +155,9 @@ export async function kvStoreSubmitFile(fastlyApiContext: FastlyApiContext, kvSt body, }); - if (response.status !== 200) { - throw new Error(`Submitting item ${key} gave error: ${response.status}`); - } - } -export async function kvStoreDeleteFile(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string): Promise { +export async function kvStoreDeleteFile(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { @@ -163,12 +166,8 @@ export async function kvStoreDeleteFile(fastlyApiContext: FastlyApiContext, kvSt const endpoint = `/resources/stores/kv/${encodeURIComponent(kvStoreId)}/keys/${encodeURIComponent(key)}`; - const response = await callFastlyApi(fastlyApiContext, endpoint, null, { + await callFastlyApi(fastlyApiContext, endpoint, `Deleting item [${key}]`, null, { method: 'DELETE', }); - if (response.status < 200 || response.status >= 300) { - throw new Error(`Deleting item '${key}' gave error: ${response.status}`); - } - } diff --git a/src/cli/util/retryable.ts b/src/cli/util/retryable.ts new file mode 100644 index 0000000..fe4e543 --- /dev/null +++ b/src/cli/util/retryable.ts @@ -0,0 +1,62 @@ +let _globalBackoffUntil = 0; + +const retryableSymbol = Symbol(); +export function makeRetryable(error: Error) { + (error as any)[retryableSymbol] = true; + return error; +} +export function isRetryableError(error: unknown) { + if (!(error instanceof Error)) { + return false; + } + return (error as any)[retryableSymbol] ?? false; +} + +type RetryWithBackoffOptions = { + maxRetries?: number, + initialDelay?: number, + onAttempt?: (attempt: number) => void, + onSuccess?: (attempt: number) => void, + onRetry?: (attempt: number, err: unknown, delay: number) => void, +}; +export async function attemptWithRetries( + fn: () => Promise, + options: RetryWithBackoffOptions = {}, +) { + const { + maxRetries = 5, + initialDelay = 60000, + onAttempt, + onSuccess, + onRetry, + } = options; + + let attempt = 0; + let result: TResult; + + while (true) { + const waitTime = _globalBackoffUntil - Date.now(); + if (waitTime > 0) { + await new Promise(res => setTimeout(res, waitTime)); + } + + try { + onAttempt?.(attempt); + result = await fn(); + onSuccess?.(attempt); + break; + } catch (err) { + if (!isRetryableError(err) || attempt >= maxRetries) { + throw err; + } + + const delay = initialDelay * (attempt + 1); + _globalBackoffUntil = Date.now() + delay; + onRetry?.(attempt, err, delay); + } + + attempt++; + } + + return result; +}