From d4c7347f592fe26933bb505590ed32999b4e9476 Mon Sep 17 00:00:00 2001 From: Clay Tercek <30105080+claytercek@users.noreply.github.com> Date: Mon, 11 Aug 2025 16:44:10 -0400 Subject: [PATCH] add abort signal to content sources --- .changeset/mean-trees-share.md | 5 ++ package-lock.json | 42 +++++----- packages/content/src/launchpad-content.ts | 8 +- .../sources/__tests__/airtable-source.test.ts | 3 + .../__tests__/contentful-source.test.ts | 3 + .../src/sources/__tests__/json-source.test.ts | 42 ++++++++++ .../sources/__tests__/sanity-source.test.ts | 47 +++++++++++ .../sources/__tests__/strapi-source.test.ts | 80 +++++++++++++++++++ packages/content/src/sources/json-source.ts | 8 +- packages/content/src/sources/sanity-source.ts | 16 +++- packages/content/src/sources/source.ts | 4 + packages/content/src/sources/strapi-source.ts | 1 + 12 files changed, 233 insertions(+), 26 deletions(-) create mode 100644 .changeset/mean-trees-share.md diff --git a/.changeset/mean-trees-share.md b/.changeset/mean-trees-share.md new file mode 100644 index 00000000..b0dcb891 --- /dev/null +++ b/.changeset/mean-trees-share.md @@ -0,0 +1,5 @@ +--- +"@bluecadet/launchpad-content": minor +--- + +Send abortsignal to sources on content exit. Implemented for json, sanity, and strapi sources. diff --git a/package-lock.json b/package-lock.json index 763d0551..31d5ec42 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,11 +37,11 @@ "vue": "^3.5.12" }, "devDependencies": { - "@bluecadet/launchpad": "2.0.12", - "@bluecadet/launchpad-cli": "2.1.1", - "@bluecadet/launchpad-content": "2.1.3", - "@bluecadet/launchpad-monitor": "2.0.5", - "@bluecadet/launchpad-scaffold": "2.0.0" + "@bluecadet/launchpad": "2.0.13", + "@bluecadet/launchpad-cli": "2.2.0", + "@bluecadet/launchpad-content": "2.2.0", + "@bluecadet/launchpad-monitor": "2.1.0", + "@bluecadet/launchpad-scaffold": "2.0.1" } }, "node_modules/@algolia/autocomplete-core": { @@ -10337,10 +10337,10 @@ }, "packages/cli": { "name": "@bluecadet/launchpad-cli", - "version": "2.1.1", + "version": "2.2.0", "license": "ISC", "dependencies": { - "@bluecadet/launchpad-utils": "~2.0.1", + "@bluecadet/launchpad-utils": "~2.1.0", "chalk": "^5.0.0", "dotenv": "^16.4.5", "jiti": "^2.4.2", @@ -10360,8 +10360,8 @@ "npm": ">=8.5.1" }, "peerDependencies": { - "@bluecadet/launchpad-content": "~2.1.0", - "@bluecadet/launchpad-monitor": "~2.0.0", + "@bluecadet/launchpad-content": "~2.2.0", + "@bluecadet/launchpad-monitor": "~2.1.0", "@bluecadet/launchpad-scaffold": "~2.0.0" }, "peerDependenciesMeta": { @@ -10398,10 +10398,10 @@ }, "packages/content": { "name": "@bluecadet/launchpad-content", - "version": "2.1.3", + "version": "2.2.0", "license": "ISC", "dependencies": { - "@bluecadet/launchpad-utils": "~2.0.1", + "@bluecadet/launchpad-utils": "~2.1.0", "chalk": "^5.0.0", "glob": "^11.0.0", "jsonpath-plus": "^10.3.0", @@ -10488,14 +10488,14 @@ }, "packages/launchpad": { "name": "@bluecadet/launchpad", - "version": "2.0.12", + "version": "2.0.13", "license": "ISC", "dependencies": { - "@bluecadet/launchpad-cli": "2.1.1", - "@bluecadet/launchpad-content": "2.1.3", + "@bluecadet/launchpad-cli": "2.2.0", + "@bluecadet/launchpad-content": "2.2.0", "@bluecadet/launchpad-dashboard": "2.0.0", - "@bluecadet/launchpad-monitor": "2.0.5", - "@bluecadet/launchpad-scaffold": "2.0.0" + "@bluecadet/launchpad-monitor": "2.1.0", + "@bluecadet/launchpad-scaffold": "2.0.1" }, "devDependencies": { "@bluecadet/launchpad-tsconfig": "0.1.0" @@ -10506,10 +10506,10 @@ }, "packages/monitor": { "name": "@bluecadet/launchpad-monitor", - "version": "2.0.5", + "version": "2.1.0", "license": "ISC", "dependencies": { - "@bluecadet/launchpad-utils": "~2.0.0", + "@bluecadet/launchpad-utils": "~2.1.0", "auto-bind": "^5.0.1", "chalk": "^5.0.0", "cross-spawn": "^7.0.3", @@ -10548,10 +10548,10 @@ }, "packages/scaffold": { "name": "@bluecadet/launchpad-scaffold", - "version": "2.0.0", + "version": "2.0.1", "license": "ISC", "dependencies": { - "@bluecadet/launchpad-utils": "~2.0.0", + "@bluecadet/launchpad-utils": "~2.1.0", "sudo-prompt": "^9.2.1" }, "devDependencies": { @@ -10577,7 +10577,7 @@ }, "packages/utils": { "name": "@bluecadet/launchpad-utils", - "version": "2.0.1", + "version": "2.1.0", "license": "ISC", "dependencies": { "@sindresorhus/slugify": "^2.1.0", diff --git a/packages/content/src/launchpad-content.ts b/packages/content/src/launchpad-content.ts index 1f5c03b4..0415aade 100644 --- a/packages/content/src/launchpad-content.ts +++ b/packages/content/src/launchpad-content.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { type Logger, LogManager, PluginDriver } from "@bluecadet/launchpad-utils"; +import { type Logger, LogManager, onExit, PluginDriver } from "@bluecadet/launchpad-utils"; import chalk from "chalk"; import { err, ok, okAsync, Result, ResultAsync } from "neverthrow"; import { @@ -23,6 +23,7 @@ class LaunchpadContent { _rawSources: ConfigContentSource[]; _startDatetime = new Date(); _dataStore: DataStore; + _abortController = new AbortController(); _cwd: string; constructor(config: ContentConfig, parentLogger: Logger, cwd = process.cwd()) { @@ -37,6 +38,10 @@ class LaunchpadContent { // create all sources this._rawSources = this._config.sources; + onExit(() => { + this._abortController.abort(); + }); + const basePluginDriver = new PluginDriver( { logger: this._logger, cwd: this._cwd }, this._config.plugins, @@ -307,6 +312,7 @@ class LaunchpadContent { const initializedFetch = source.fetch({ logger: sourceLogger, dataStore: this._dataStore, + abortSignal: this._abortController.signal, }); const fetchAsArray = Array.isArray(initializedFetch) ? initializedFetch : [initializedFetch]; diff --git a/packages/content/src/sources/__tests__/airtable-source.test.ts b/packages/content/src/sources/__tests__/airtable-source.test.ts index 568c7f70..046f4141 100644 --- a/packages/content/src/sources/__tests__/airtable-source.test.ts +++ b/packages/content/src/sources/__tests__/airtable-source.test.ts @@ -20,9 +20,12 @@ afterAll(() => { afterEach(() => server.resetHandlers()); function createFetchContext() { + const abortController = new AbortController(); return { logger: createMockLogger(), dataStore: new DataStore("/"), + abortSignal: abortController.signal, + _abortController: abortController, }; } diff --git a/packages/content/src/sources/__tests__/contentful-source.test.ts b/packages/content/src/sources/__tests__/contentful-source.test.ts index 6edba409..7920cc9f 100644 --- a/packages/content/src/sources/__tests__/contentful-source.test.ts +++ b/packages/content/src/sources/__tests__/contentful-source.test.ts @@ -20,9 +20,12 @@ afterAll(() => { afterEach(() => server.resetHandlers()); function createFetchContext() { + const abortController = new AbortController(); return { logger: createMockLogger(), dataStore: new DataStore("/"), + abortSignal: abortController.signal, + _abortController: abortController, }; } diff --git a/packages/content/src/sources/__tests__/json-source.test.ts b/packages/content/src/sources/__tests__/json-source.test.ts index 78577bb4..f7e6e096 100644 --- a/packages/content/src/sources/__tests__/json-source.test.ts +++ b/packages/content/src/sources/__tests__/json-source.test.ts @@ -20,9 +20,12 @@ afterAll(() => { afterEach(() => server.resetHandlers()); function createFetchContext() { + const abortController = new AbortController(); return { logger: createMockLogger(), dataStore: new DataStore("/"), + abortSignal: abortController.signal, + _abortController: abortController, }; } @@ -132,4 +135,43 @@ describe("jsonSource", () => { // @ts-expect-error - incomplete config await expect(() => jsonSource({})).toThrow(); }); + + it("should cancel request on abortSignal", async () => { + const ctx = createFetchContext(); + + server.use( + http.get("https://api.example.com/slow", async () => { + await new Promise((resolve) => setTimeout(resolve, 2000)); + return HttpResponse.json({ key: "value" }); + }), + ); + + const source = await jsonSource({ + id: "test-json-timeout", + files: { + "slow.json": "https://api.example.com/slow", + }, + maxTimeout: 200, + }); + + const result = source.fetch(ctx); + + const promise = result[0]!.data; + + const abortReason = "Some abort reason"; + + // Abort the request after a short delay + setTimeout(() => { + ctx._abortController.abort(abortReason); + }, 100); + + vi.runAllTimersAsync(); + + await expect(promise).rejects.toThrowError(abortReason); + }); + + it("should throw on incomplete config", async () => { + // @ts-expect-error - incomplete config + await expect(() => jsonSource({})).toThrow(); + }); }); diff --git a/packages/content/src/sources/__tests__/sanity-source.test.ts b/packages/content/src/sources/__tests__/sanity-source.test.ts index 72934ccc..19d3abdc 100644 --- a/packages/content/src/sources/__tests__/sanity-source.test.ts +++ b/packages/content/src/sources/__tests__/sanity-source.test.ts @@ -20,9 +20,12 @@ afterAll(() => { afterEach(() => server.resetHandlers()); function createFetchContext() { + const abortController = new AbortController(); return { logger: createMockLogger(), dataStore: new DataStore("/"), + abortSignal: abortController.signal, + _abortController: abortController, }; } @@ -267,4 +270,48 @@ describe("sanitySource", () => { expect(data1).toEqual({ _type: "test", title: "Test Document" }); expect(data2).toEqual({ _type: "test", title: "Test Document" }); }); + + it("should cancel request on abortSignal", async () => { + const ctx = createFetchContext(); + + server.use( + http.get("https://test-project.api.sanity.io/v2021-10-21/data/query/production", async () => { + await new Promise((resolve) => setTimeout(resolve, 2000)); + return HttpResponse.json({ + result: { _type: "test", title: "Test Document" }, + ms: 2000, + }); + }), + ); + + const source = await sanitySource({ + id: "test-sanity", + projectId: "test-project", + apiToken: "test-token", + queries: [ + { + id: "custom1", + query: '*[_type == "custom"][0]', + }, + ], + limit: 50, + mergePages: false, + useCdn: false, + }); + + const result = source.fetch(ctx); + + const promise = result[0]!.data; + + const abortReason = "Some abort reason"; + + // Abort the request after a short delay + setTimeout(() => { + ctx._abortController.abort(abortReason); + }, 50); + + vi.runAllTimersAsync(); + + await expect(promise).rejects.toThrowError(abortReason); + }); }); diff --git a/packages/content/src/sources/__tests__/strapi-source.test.ts b/packages/content/src/sources/__tests__/strapi-source.test.ts index 36b10c36..96fdfa14 100644 --- a/packages/content/src/sources/__tests__/strapi-source.test.ts +++ b/packages/content/src/sources/__tests__/strapi-source.test.ts @@ -20,9 +20,12 @@ afterAll(() => { afterEach(() => server.resetHandlers()); function createFetchContext() { + const abortController = new AbortController(); return { logger: createMockLogger(), dataStore: new DataStore("/"), + abortSignal: abortController.signal, + _abortController: abortController, }; } @@ -348,4 +351,81 @@ describe.runIf(majorNodeVersion >= 20)("strapiSource", () => { await expect(async () => (await result[0]!.data.next()).value).rejects.toThrow(); }); + + it("should cancel request on abortSignal", async () => { + const ctx = createFetchContext(); + + server.use( + http.post("http://localhost:1337/auth/local", () => { + return HttpResponse.json({ jwt: "test-token" }); + }), + http.get("http://localhost:1337/api/custom-content", async ({ request }) => { + await new Promise((resolve) => setTimeout(resolve, 2000)); + const url = new URL(request.url); + expect(url.searchParams.get("filters[type][$eq]")).toBe("test"); + expect(url.searchParams.get("sort[0]")).toBe("createdAt:desc"); + + if (url.searchParams.get("pagination[page]") === "2") { + return HttpResponse.json({ + data: [], + meta: { + pagination: { page: 2, pageSize: 100, pageCount: 1, total: 0 }, + }, + }); + } + + return HttpResponse.json({ + data: [ + { + id: 1, + attributes: { + title: "Custom Content", + type: "test", + }, + }, + ], + meta: { + pagination: { + page: 1, + pageSize: 100, + pageCount: 1, + total: 1, + }, + }, + }); + }), + ); + + const source = await strapiSource({ + id: "test-strapi", + version: "4", + baseUrl: "http://localhost:1337", + identifier: "test@example.com", + password: "password", + queries: [ + { + contentType: "custom-content", + params: { + "filters[type][$eq]": "test", + "sort[0]": "createdAt:desc", + }, + }, + ], + }); + + const result = source.fetch(ctx); + + const promise = result[0]!.data.next(); + + const abortReason = "Some abort reason"; + + // Abort the request after a short delay + setTimeout(() => { + ctx._abortController.abort(abortReason); + }, 100); + + vi.runAllTimersAsync(); + + await expect(promise).rejects.toThrowError(abortReason); + }); }); diff --git a/packages/content/src/sources/json-source.ts b/packages/content/src/sources/json-source.ts index ef7ae2ca..71c26f84 100644 --- a/packages/content/src/sources/json-source.ts +++ b/packages/content/src/sources/json-source.ts @@ -20,10 +20,14 @@ export default function jsonSource(options: z.input) { fetch: (ctx) => { return Object.entries(parsedOptions.files).map(([key, url]) => { ctx.logger.debug(`Downloading json ${chalk.blue(url)}`); - return { id: key, - data: ky.get(url, { timeout: parsedOptions.maxTimeout }).json(), + data: ky + .get(url, { + timeout: parsedOptions.maxTimeout, + signal: ctx.abortSignal, + }) + .json(), }; }); }, diff --git a/packages/content/src/sources/sanity-source.ts b/packages/content/src/sources/sanity-source.ts index f229747f..766bf659 100644 --- a/packages/content/src/sources/sanity-source.ts +++ b/packages/content/src/sources/sanity-source.ts @@ -61,7 +61,13 @@ export default async function sanitySource(options: z.input(fullQuery), + data: sanityClient.fetch( + fullQuery, + {}, + { + signal: ctx.abortSignal, + }, + ), }; } @@ -77,7 +83,13 @@ export default async function sanitySource(options: z.input(q); + return sanityClient.fetch( + q, + {}, + { + signal: ctx.abortSignal, + }, + ); } catch (e) { throw new Error(`Could not fetch page with query: '${q}'`, { cause: e }); } diff --git a/packages/content/src/sources/source.ts b/packages/content/src/sources/source.ts index 4f83158d..a3549cb2 100644 --- a/packages/content/src/sources/source.ts +++ b/packages/content/src/sources/source.ts @@ -14,6 +14,10 @@ export type FetchContext = { * Data store instance */ dataStore: DataStore; + /** + * Signals the launchpad process is aborting. Triggered on exception or manual quit. + */ + abortSignal: AbortSignal; }; function asyncIterableSchema(): z.ZodType> { diff --git a/packages/content/src/sources/strapi-source.ts b/packages/content/src/sources/strapi-source.ts index 6a44b85b..3a2c0661 100644 --- a/packages/content/src/sources/strapi-source.ts +++ b/packages/content/src/sources/strapi-source.ts @@ -286,6 +286,7 @@ export default async function strapiSource(options: z.input