From e653cfbe1da1963d4d0ed9fa166336a8fac26dd6 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 10 Jul 2025 22:13:55 -0700 Subject: [PATCH] feat: Adds chunked encoding strategy --- .changeset/twenty-toes-laugh.md | 9 + .eslintignore | 3 +- .prettierignore | 3 + scripts/generate.ts | 4 - src/__generated__/manifest.ts | 57 +- src/__generated__/openapi.ts | 1394 ++++++++++++++++++++++++++++++- src/__generated__/pack.ts | 6 +- src/__generated__/schema.ts | 6 +- src/index.ts | 15 +- src/lib/lifecycle.ts | 32 + src/lib/lifecycle/chunk.ts | 101 +++ src/lib/lifecycle/helpers.ts | 62 ++ src/lib/lifecycle/post.ts | 70 ++ src/lib/lifecycle/pre.ts | 67 ++ src/lib/lifecycle/types.ts | 30 + src/lib/{handler.ts => msw.ts} | 22 +- src/lib/sandbox.ts | 206 +---- src/lib/taskless.ts | 22 +- src/lib/{ => util}/error.ts | 0 src/lib/{ => util}/id.ts | 0 src/lib/{ => util}/logger.ts | 36 +- tsconfig.json | 2 +- 22 files changed, 1925 insertions(+), 222 deletions(-) create mode 100644 .changeset/twenty-toes-laugh.md create mode 100644 src/lib/lifecycle.ts create mode 100644 src/lib/lifecycle/chunk.ts create mode 100644 src/lib/lifecycle/helpers.ts create mode 100644 src/lib/lifecycle/post.ts create mode 100644 src/lib/lifecycle/pre.ts create mode 100644 src/lib/lifecycle/types.ts rename src/lib/{handler.ts => msw.ts} (82%) rename src/lib/{ => util}/error.ts (100%) rename src/lib/{ => util}/id.ts (100%) rename src/lib/{ => util}/logger.ts (54%) diff --git a/.changeset/twenty-toes-laugh.md b/.changeset/twenty-toes-laugh.md new file mode 100644 index 0000000..5a2c969 --- /dev/null +++ b/.changeset/twenty-toes-laugh.md @@ -0,0 +1,9 @@ +--- +"@taskless/loader": patch +--- + +feat: Refactors for support of chunked event-stream + +Previously Taskless Loader only knew of "pre" and "post" lifecycle events. This loader change introduces a new "chunk" event that allows for control over readable stream segments such as those used in event-streams. This is useful for Packs that want to process data in chunks rather than waiting for the entire response such as an MCP integration. + +This change is backwards compatible with existing Packs and the current `pre2` schema. To enable chunk processing, a new `methods` field is added to the `pre2` manifest. Valid options for "methods" are `pre`, `post`, and `chunk`. If a Pack does not declare `chunk` in its manifest, it will continue to function as before, processing the entire response in one go with the standard `pre` and `post` lifecycle events. diff --git a/.eslintignore b/.eslintignore index 8df4a7f..485a521 100644 --- a/.eslintignore +++ b/.eslintignore @@ -2,4 +2,5 @@ __generated__/* *.config.* next-env.d.ts examples/* -test/fixtures/* \ No newline at end of file +test/fixtures/* +src/vendor/* \ No newline at end of file diff --git a/.prettierignore b/.prettierignore index 847bcaf..30f6000 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,3 +7,6 @@ __generated__ # unformattables *.sh *.cfg + +# vendor +vendor \ No newline at end of file diff --git a/scripts/generate.ts b/scripts/generate.ts index bad2cb0..6ebb225 100644 --- a/scripts/generate.ts +++ b/scripts/generate.ts @@ -13,13 +13,9 @@ import { rimraf } from "rimraf"; const base = process.env.TASKLESS_HOST ?? `${TASKLESS_HOST}`; const ROOT = (await packageDirectory())!; const GENERATED = resolve(ROOT, "src/__generated__"); -const WASM = resolve(ROOT, "wasm"); await rimraf(GENERATED); -await rimraf(WASM); - await mkdirp(GENERATED); -await mkdirp(WASM); const prettierOptions = { ...(JSON.parse( diff --git a/src/__generated__/manifest.ts b/src/__generated__/manifest.ts index 1a74e07..d0afb9c 100644 --- a/src/__generated__/manifest.ts +++ b/src/__generated__/manifest.ts @@ -25,6 +25,10 @@ export interface Manifest { * A short description of the pack's functionality */ description: string; + /** + * Supported methods for this pack, defaults to pre and post + */ + methods?: ("pre" | "post" | "chunk")[]; /** * The permissions requested for this pack from the host system */ @@ -34,12 +38,12 @@ export interface Manifest { */ environment?: string[]; /** - * Whether this pack can access the request and response body + * [deprecated] Whether this pack can access the request and response body, always true */ body?: boolean; }; /** - * Default dashboard configurations for this pack + * [deprecated] Default dashboard configurations for this pack - see dashboards */ displays?: { /** @@ -100,4 +104,53 @@ export interface Manifest { default: boolean; } )[]; + /** + * Charts available in this Pack + */ + charts?: { + /** + * The title of the chart + */ + title: string; + /** + * A short description of the chart + */ + description?: string; + /** + * The type of the chart + */ + type: "step" | "pie" | "table"; + definition: { + /** + * Describes the aggregation funciton to use, usually expressed on the Y-axis + */ + aggregate: { + [k: string]: string; + }; + /** + * How should the data be grouped? Usually expressed on the X-axis + */ + bucket: { + [k: string]: string; + }; + /** + * The graph series + */ + series?: + | { + query: string; + } + | { + dimension: string; + /** + * The type of data this is + */ + dimensionType: "string" | "number"; + /** + * The per-series query with optional placeholders + */ + query: string; + }; + }; + }[]; } diff --git a/src/__generated__/openapi.ts b/src/__generated__/openapi.ts index 8fb4a0a..3d8b824 100644 --- a/src/__generated__/openapi.ts +++ b/src/__generated__/openapi.ts @@ -50,11 +50,450 @@ export default { _def: { typeName: "ZodNever", }, + "~standard": { + version: 1, + vendor: "zod", + }, }, typeName: "ZodObject", description: "A valid Taskless cloud configuration", }, - _cached: null, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + schema: { + _def: { + value: "pre1", + typeName: "ZodLiteral", + description: "The config schema version used", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + organizationId: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + packs: { + _def: { + type: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + description: + "A pack delivered from the Taskless cloud, including information on how to retrieve the pack's runtime code", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + schema: { + _def: { + value: "pre1", + typeName: "ZodLiteral", + description: + "The pack schema version used", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + name: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: "The pack name", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + version: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: "The pack version", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + description: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: "The pack description", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + url: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + description: + "When a pack's excutable code is hosted remotely, this object describes how to download and verify it", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + source: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: + "A remote URL for downloading this Pack's executable code", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + signature: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: + "A sha-256 signature of the remote URL's content", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: ["source", "signature"], + }, + }, + capture: { + _def: { + keyType: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + valueType: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + type: { + _def: { + options: [ + { + _def: { + value: "string", + typeName: + "ZodLiteral", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + { + _def: { + value: "number", + typeName: + "ZodLiteral", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + ], + typeName: "ZodUnion", + description: + "The type of data to capture", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + description: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: ["type", "description"], + }, + }, + typeName: "ZodRecord", + description: + "Describes the data this pack intends to capture", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + permissions: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + description: + "The permissions requested for this pack", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + domains: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "The domains this pack is allowed to request data from as regular expressions.", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + environment: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "The environment variables this pack is allowed to access on the host system", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + request: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "During the lifecycle, request access to additional properties such as 'headers' and 'body'", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + response: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "During the lifecycle, response access to additional properties such as 'headers' and 'body'", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: [ + "domains", + "environment", + "request", + "response", + ], + }, + }, + }, + keys: [ + "schema", + "name", + "version", + "description", + "url", + "capture", + "permissions", + ], + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: ["schema", "organizationId", "packs"], + }, }, { _def: { @@ -63,15 +502,27 @@ export default { _def: { typeName: "ZodNever", }, + "~standard": { + version: 1, + vendor: "zod", + }, }, typeName: "ZodObject", description: "A valid Taskless cloud configuration", }, + "~standard": { + version: 1, + vendor: "zod", + }, _cached: null, }, ], typeName: "ZodUnion", }, + "~standard": { + version: 1, + vendor: "zod", + }, }, }, }, @@ -87,9 +538,17 @@ export default { _def: { typeName: "ZodNever", }, + "~standard": { + version: 1, + vendor: "zod", + }, }, typeName: "ZodObject", }, + "~standard": { + version: 1, + vendor: "zod", + }, _cached: null, }, }, @@ -168,10 +627,34 @@ export default { _def: { typeName: "ZodNever", }, + "~standard": { + version: 1, + vendor: "zod", + }, }, typeName: "ZodObject", }, - _cached: null, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + received: { + _def: { + checks: [], + typeName: "ZodNumber", + coerce: false, + description: "The number of events received", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: ["received"], + }, }, }, }, @@ -187,9 +670,17 @@ export default { _def: { typeName: "ZodNever", }, + "~standard": { + version: 1, + vendor: "zod", + }, }, typeName: "ZodObject", }, + "~standard": { + version: 1, + vendor: "zod", + }, _cached: null, }, }, @@ -212,11 +703,447 @@ export default { _def: { typeName: "ZodNever", }, + "~standard": { + version: 1, + vendor: "zod", + }, }, typeName: "ZodObject", description: "A valid Taskless cloud configuration", }, - _cached: null, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + schema: { + _def: { + value: "pre1", + typeName: "ZodLiteral", + description: "The config schema version used", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + organizationId: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + packs: { + _def: { + type: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + description: + "A pack delivered from the Taskless cloud, including information on how to retrieve the pack's runtime code", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + schema: { + _def: { + value: "pre1", + typeName: "ZodLiteral", + description: "The pack schema version used", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + name: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: "The pack name", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + version: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: "The pack version", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + description: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: "The pack description", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + url: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + description: + "When a pack's excutable code is hosted remotely, this object describes how to download and verify it", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + source: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: + "A remote URL for downloading this Pack's executable code", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + signature: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: + "A sha-256 signature of the remote URL's content", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: ["source", "signature"], + }, + }, + capture: { + _def: { + keyType: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + valueType: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + type: { + _def: { + options: [ + { + _def: { + value: "string", + typeName: "ZodLiteral", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + { + _def: { + value: "number", + typeName: "ZodLiteral", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + ], + typeName: "ZodUnion", + description: + "The type of data to capture", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + description: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: ["type", "description"], + }, + }, + typeName: "ZodRecord", + description: + "Describes the data this pack intends to capture", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + permissions: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + description: + "The permissions requested for this pack", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + domains: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "The domains this pack is allowed to request data from as regular expressions.", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + environment: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "The environment variables this pack is allowed to access on the host system", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + request: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "During the lifecycle, request access to additional properties such as 'headers' and 'body'", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + response: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "During the lifecycle, response access to additional properties such as 'headers' and 'body'", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: [ + "domains", + "environment", + "request", + "response", + ], + }, + }, + }, + keys: [ + "schema", + "name", + "version", + "description", + "url", + "capture", + "permissions", + ], + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: ["schema", "organizationId", "packs"], + }, }, }, }, @@ -252,11 +1179,450 @@ export default { _def: { typeName: "ZodNever", }, + "~standard": { + version: 1, + vendor: "zod", + }, }, typeName: "ZodObject", description: "A valid Taskless cloud configuration", }, - _cached: null, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + schema: { + _def: { + value: "pre1", + typeName: "ZodLiteral", + description: "The config schema version used", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + organizationId: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + packs: { + _def: { + type: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + description: + "A pack delivered from the Taskless cloud, including information on how to retrieve the pack's runtime code", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + schema: { + _def: { + value: "pre1", + typeName: "ZodLiteral", + description: + "The pack schema version used", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + name: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: "The pack name", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + version: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: "The pack version", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + description: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: "The pack description", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + url: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + description: + "When a pack's excutable code is hosted remotely, this object describes how to download and verify it", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + source: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: + "A remote URL for downloading this Pack's executable code", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + signature: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + description: + "A sha-256 signature of the remote URL's content", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: ["source", "signature"], + }, + }, + capture: { + _def: { + keyType: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + valueType: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + type: { + _def: { + options: [ + { + _def: { + value: "string", + typeName: + "ZodLiteral", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + { + _def: { + value: "number", + typeName: + "ZodLiteral", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + ], + typeName: "ZodUnion", + description: + "The type of data to capture", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + description: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: ["type", "description"], + }, + }, + typeName: "ZodRecord", + description: + "Describes the data this pack intends to capture", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + permissions: { + _def: { + unknownKeys: "strip", + catchall: { + _def: { + typeName: "ZodNever", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodObject", + description: + "The permissions requested for this pack", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + _cached: { + shape: { + domains: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "The domains this pack is allowed to request data from as regular expressions.", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + environment: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "The environment variables this pack is allowed to access on the host system", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + request: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "During the lifecycle, request access to additional properties such as 'headers' and 'body'", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + response: { + _def: { + innerType: { + _def: { + type: { + _def: { + checks: [], + typeName: "ZodString", + coerce: false, + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + typeName: "ZodOptional", + description: + "During the lifecycle, response access to additional properties such as 'headers' and 'body'", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: [ + "domains", + "environment", + "request", + "response", + ], + }, + }, + }, + keys: [ + "schema", + "name", + "version", + "description", + "url", + "capture", + "permissions", + ], + }, + }, + minLength: null, + maxLength: null, + exactLength: null, + typeName: "ZodArray", + }, + "~standard": { + version: 1, + vendor: "zod", + }, + }, + }, + keys: ["schema", "organizationId", "packs"], + }, }, { _def: { @@ -265,15 +1631,27 @@ export default { _def: { typeName: "ZodNever", }, + "~standard": { + version: 1, + vendor: "zod", + }, }, typeName: "ZodObject", description: "A valid Taskless cloud configuration", }, + "~standard": { + version: 1, + vendor: "zod", + }, _cached: null, }, ], typeName: "ZodUnion", }, + "~standard": { + version: 1, + vendor: "zod", + }, }, }, }, @@ -289,9 +1667,17 @@ export default { _def: { typeName: "ZodNever", }, + "~standard": { + version: 1, + vendor: "zod", + }, }, typeName: "ZodObject", }, + "~standard": { + version: 1, + vendor: "zod", + }, _cached: null, }, }, diff --git a/src/__generated__/pack.ts b/src/__generated__/pack.ts index 08bc053..06dc104 100644 --- a/src/__generated__/pack.ts +++ b/src/__generated__/pack.ts @@ -25,6 +25,10 @@ export interface Pack { * A short description of the pack's functionality */ description: string; + /** + * Supported methods for this pack, defaults to pre and post + */ + methods?: ("pre" | "post" | "chunk")[]; /** * The permissions requested for this pack from the host system */ @@ -34,7 +38,7 @@ export interface Pack { */ environment?: string[]; /** - * Whether this pack can access the request and response body + * [deprecated] Whether this pack can access the request and response body, always true */ body?: boolean; }; diff --git a/src/__generated__/schema.ts b/src/__generated__/schema.ts index 8358465..e0d14ff 100644 --- a/src/__generated__/schema.ts +++ b/src/__generated__/schema.ts @@ -31,6 +31,10 @@ export interface Schema { * A short description of the pack's functionality */ description: string; + /** + * Supported methods for this pack, defaults to pre and post + */ + methods?: ("pre" | "post" | "chunk")[]; /** * The permissions requested for this pack from the host system */ @@ -40,7 +44,7 @@ export interface Schema { */ environment?: string[]; /** - * Whether this pack can access the request and response body + * [deprecated] Whether this pack can access the request and response body, always true */ body?: boolean; }; diff --git a/src/index.ts b/src/index.ts index f8b6156..ffe80ad 100644 --- a/src/index.ts +++ b/src/index.ts @@ -100,12 +100,15 @@ const mergeEnvironments = ( const env: Partial = {}; for (const [key, parser] of Object.entries(environmentParsers)) { for (const source of sources) { - if (source?.[key] !== undefined) { - const parsedValue = parser(source[key]); - if (parsedValue !== undefined) { - // Type assertion is safe here because we know the parser matches the key - (env as Record)[key] = parsedValue; - } + const value = source?.[key]; + if (!value) { + continue; + } + + const parsedValue = parser(value); + if (parsedValue !== undefined) { + // Type assertion is safe here because we know the parser matches the key + (env as Record)[key] = parsedValue; } } } diff --git a/src/lib/lifecycle.ts b/src/lib/lifecycle.ts new file mode 100644 index 0000000..f0c083c --- /dev/null +++ b/src/lib/lifecycle.ts @@ -0,0 +1,32 @@ +import { chunk } from "./lifecycle/chunk.js"; +import { post } from "./lifecycle/post.js"; +import { pre } from "./lifecycle/pre.js"; +import { type LifecycleExecutor } from "./lifecycle/types.js"; + +export const runPackLifecycle: LifecycleExecutor = async ( + requestId, + lifecycle, + packs, + data +) => { + switch (lifecycle) { + case "pre": { + await pre(requestId, lifecycle, packs, data); + break; + } + + case "post": { + await post(requestId, lifecycle, packs, data); + break; + } + + case "chunk": { + await chunk(requestId, lifecycle, packs, data); + break; + } + + default: { + data.logger.error(`Unknown lifecycle "${lifecycle}"`); + } + } +}; diff --git a/src/lib/lifecycle/chunk.ts b/src/lib/lifecycle/chunk.ts new file mode 100644 index 0000000..471101a --- /dev/null +++ b/src/lib/lifecycle/chunk.ts @@ -0,0 +1,101 @@ +import { type ReadableStreamReadResult } from "node:stream/web"; +import { type PluginOutput } from "@~/types.js"; +import { packIdentifier } from "../util/id.js"; +import { createSandbox } from "./helpers.js"; +import { type LifecycleExecutor } from "./types.js"; + +/** + * Read bytes with a callback for every chunk, + * using promises to ensure sequence + **/ +async function getBytes( + stream: ReadableStream, + onChunk: (bytes: Uint8Array) => Promise +) { + const reader = stream.getReader(); + let result: ReadableStreamReadResult; + // eslint-disable-next-line no-await-in-loop + while (!(result = await reader.read()).done) { + // eslint-disable-next-line no-await-in-loop + await onChunk(result.value); + } +} + +const LIFECYCLE_ID = "chunk"; +const EXTISM_FN = "chunk"; + +export const chunk: LifecycleExecutor = async ( + requestId, + _, + packs, + { request, response, context, logger, callbacks } +) => { + const promises = packs.map(async (pack, index) => { + // not in defaults, so an undefined methods collection skips this + if (!pack.methods) { + logger.debug( + `[${requestId}] (${LIFECYCLE_ID}) pack ${pack.name} skipped` + ); + return undefined; + } + + // must be explicitly declared + if (!pack.methods.includes(LIFECYCLE_ID)) { + logger.debug( + `[${requestId}] (${LIFECYCLE_ID}) pack ${pack.name} skipped` + ); + return undefined; + } + + const plugins = await callbacks.getModules(); + const plugin = await plugins.get(packIdentifier(pack)); + + if (!plugin) { + throw new Error( + `[${requestId}] (${LIFECYCLE_ID}) Pack not found in modules` + ); + } + + if (!response?.body) { + return undefined; + } + + await getBytes(response.body, async (bytes) => { + const output = await plugin.call( + EXTISM_FN, + JSON.stringify( + await createSandbox(requestId, pack, { + request, + chunk: bytes, + context: context[`${index}`] ?? {}, + }) + ) + ); + + if (output) { + const result = output?.json() as PluginOutput; + + if (result.context) { + context[`${index}`] = result.context; + } + + await callbacks.onResult(pack, result); + logger.debug( + `[${requestId}] (${LIFECYCLE_ID}) pack ${pack.name} completed` + ); + } else { + logger.debug( + `[${requestId}] (${LIFECYCLE_ID}) pack ${pack.name} no output returned` + ); + } + }); + }); + + const results = await Promise.allSettled(promises); + for (const hook of results) { + if (hook.status === "rejected") { + logger.error(`${hook.reason}`); + callbacks.onError(hook.reason); + } + } +}; diff --git a/src/lib/lifecycle/helpers.ts b/src/lib/lifecycle/helpers.ts new file mode 100644 index 0000000..4ccd635 --- /dev/null +++ b/src/lib/lifecycle/helpers.ts @@ -0,0 +1,62 @@ +import process from "node:process"; +import { type Pack } from "@~/types.js"; + +/** + * Creates a JSON-friendly sandbox payload + * based on the pack's permissions. Responsible for serializing + * each item and supporting binary streams via chunks of + * base64 data + **/ +export const createSandbox = async ( + requestId: string, + pack: Pack, + info: { + request: Request; + response?: Response; + chunk?: Uint8Array; + context: Record; + configuration?: Record; + } +) => { + const extractBody = async (httpObject: Request | Response) => { + const body = await httpObject.clone().text(); + try { + return JSON.parse(body) as unknown; + } catch { + return body; + } + }; + + const environmentKeys = pack.permissions?.environment ?? []; + + const env: Record = {}; + for (const key of environmentKeys) { + // eslint-disable-next-line n/no-process-env + env[key] = process.env[key]; + } + + return { + requestId, + context: info.context, + request: { + url: info.request.url, + domain: new URL(info.request.url).hostname, + path: new URL(info.request.url).pathname, + headers: [...info.request.headers.entries()], + body: pack.permissions?.body + ? await extractBody(info.request) + : undefined, + }, + chunk: info.chunk ? Buffer.from(info.chunk).toString("base64") : undefined, + response: info.response + ? { + status: info.response.status, + headers: [...info.response.headers.entries()], + body: pack.permissions?.body + ? await extractBody(info.response) + : undefined, + } + : undefined, + environment: env, + }; +}; diff --git a/src/lib/lifecycle/post.ts b/src/lib/lifecycle/post.ts new file mode 100644 index 0000000..5f84a5a --- /dev/null +++ b/src/lib/lifecycle/post.ts @@ -0,0 +1,70 @@ +import { type PluginOutput } from "@~/types.js"; +import { packIdentifier } from "../util/id.js"; +import { createSandbox } from "./helpers.js"; +import { type LifecycleExecutor } from "./types.js"; + +const LIFECYCLE_ID = "post"; +const EXTISM_FN = "post"; + +export const post: LifecycleExecutor = async ( + requestId, + _, + packs, + { request, response, context, logger, callbacks } +) => { + // reverse the packs FIFO order + const promises = packs.reverse().map(async (pack, index) => { + if (pack.methods && !pack.methods.includes(LIFECYCLE_ID)) { + logger.debug( + `[${requestId}] (${LIFECYCLE_ID}) pack ${pack.name} skipped` + ); + return undefined; + } + + const plugins = await callbacks.getModules(); + const plugin = await plugins.get(packIdentifier(pack)); + + if (!plugin) { + throw new Error( + `[${requestId}] (${LIFECYCLE_ID}) Pack not found in modules` + ); + } + + const output = await plugin.call( + EXTISM_FN, + JSON.stringify( + await createSandbox(requestId, pack, { + request, + response, // <= post includes a response object + // context is at original (non-reversed) index + context: context[`${packs.length - index - 1}`] ?? {}, + }) + ) + ); + + if (output) { + const result = output?.json() as PluginOutput; + + if (result.context) { + context[`${packs.length - index - 1}`] = result.context; + } + + await callbacks.onResult(pack, result); + logger.debug( + `[${requestId}] (${LIFECYCLE_ID}) pack ${pack.name} completed` + ); + } else { + logger.debug( + `[${requestId}] (${LIFECYCLE_ID}) pack ${pack.name} no output returned` + ); + } + }); + + const results = await Promise.allSettled(promises); + for (const hook of results) { + if (hook.status === "rejected") { + logger.error(`${hook.reason}`); + callbacks.onError(hook.reason); + } + } +}; diff --git a/src/lib/lifecycle/pre.ts b/src/lib/lifecycle/pre.ts new file mode 100644 index 0000000..02482c3 --- /dev/null +++ b/src/lib/lifecycle/pre.ts @@ -0,0 +1,67 @@ +import { type PluginOutput } from "@~/types.js"; +import { packIdentifier } from "../util/id.js"; +import { createSandbox } from "./helpers.js"; +import { type LifecycleExecutor } from "./types.js"; + +const LIFECYCLE_ID = "pre"; +const EXTISM_FN = "pre"; + +export const pre: LifecycleExecutor = async ( + requestId, + _, + packs, + { request, response, context, logger, callbacks } +) => { + const promises = packs.map(async (pack, index) => { + if (pack.methods && !pack.methods.includes(LIFECYCLE_ID)) { + logger.debug( + `[${requestId}] (${LIFECYCLE_ID}) pack ${pack.name} skipped` + ); + return undefined; + } + + const plugins = await callbacks.getModules(); + const plugin = await plugins.get(packIdentifier(pack)); + + if (!plugin) { + throw new Error( + `[${requestId}] (${LIFECYCLE_ID}) Pack not found in modules` + ); + } + + const output = await plugin.call( + EXTISM_FN, + JSON.stringify( + await createSandbox(requestId, pack, { + request, + context: context[`${index}`] ?? {}, + }) + ) + ); + + if (output) { + const result = output?.json() as PluginOutput; + + if (result.context) { + context[`${index}`] = result.context; + } + + await callbacks.onResult(pack, result); + logger.debug( + `[${requestId}] (${LIFECYCLE_ID}) pack ${pack.name} completed` + ); + } else { + logger.debug( + `[${requestId}] (${LIFECYCLE_ID}) pack ${pack.name} no output returned` + ); + } + }); + + const results = await Promise.allSettled(promises); + for (const hook of results) { + if (hook.status === "rejected") { + logger.error(`${hook.reason}`); + callbacks.onError(hook.reason); + } + } +}; diff --git a/src/lib/lifecycle/types.ts b/src/lib/lifecycle/types.ts new file mode 100644 index 0000000..854366c --- /dev/null +++ b/src/lib/lifecycle/types.ts @@ -0,0 +1,30 @@ +import { type Plugin } from "@extism/extism"; +import { type Logger, type Pack, type PluginOutput } from "@~/types.js"; + +type getModulesCallback = () => Promise>>; +type onCompleteCallback = (requestId: string) => void; +type onResultCallback = ( + pack: Pack, + result: NonNullable +) => Promise; +type onErrorCallback = (error: any) => void; + +export type LifecycleCallbacks = { + getModules: getModulesCallback; + onResult: onResultCallback; + onComplete: onCompleteCallback; + onError: onErrorCallback; +}; + +export type LifecycleExecutor = ( + requestId: string, + lifecycle: string, + packs: Pack[], + data: { + request: Request; + response?: Response; + callbacks: LifecycleCallbacks; + logger: Logger; + context: Record>; + } +) => Promise; diff --git a/src/lib/handler.ts b/src/lib/msw.ts similarity index 82% rename from src/lib/handler.ts rename to src/lib/msw.ts index 2e4ad6c..88655e6 100644 --- a/src/lib/handler.ts +++ b/src/lib/msw.ts @@ -7,8 +7,8 @@ import { type PluginOutput, } from "@~/types.js"; import { http, type HttpResponse } from "msw"; -import { id } from "./id.js"; -import { runSandbox } from "./sandbox.js"; +import { run } from "./sandbox.js"; +import { id } from "./util/id.js"; /** Checks if a request is bypassed */ const isBypassed = (request: Request) => { @@ -18,14 +18,12 @@ const isBypassed = (request: Request) => { export const createHandler = ({ loaded, logger, - useLogging, capture, getPacks, getModules, }: { loaded: Promise; logger: Logger; - useLogging: boolean; capture: CaptureCallback; getPacks: () => Promise; getModules: () => Promise>>; @@ -56,7 +54,7 @@ export const createHandler = ({ dimensions: [], }; - const response = await runSandbox( + const response = await run( { requestId, request: info.request }, logger, packs, @@ -66,6 +64,7 @@ export const createHandler = ({ return getModules(); }, async onResult(pack: Pack, result: PluginOutput) { + // logger.debug(`${pack.name} with result: ${JSON.stringify(result)}`); for (const [key, value] of Object.entries(result.capture ?? {})) { // name = @taskless/apm // new key to be = @taskless/apm/durationMs @@ -77,21 +76,26 @@ export const createHandler = ({ dimension: nskey, value: `${value}`, }); + + // the logItem represents a console friendly synchronous payload + // to make it easier to import into existing observability tools + // that like structured JSON logItem.dimensions.push({ name: nskey, value: `${value}`, }); } }, + onComplete(requestId) { + if (logger.data) { + logger.data(JSON.stringify(logItem)); + } + }, onError(error) { logger.error(`[${requestId}] error: ${error.message}`); }, } ); - if (useLogging) { - logger.data(JSON.stringify(logItem)); - } - return response; }); diff --git a/src/lib/sandbox.ts b/src/lib/sandbox.ts index b2cca42..429eac9 100644 --- a/src/lib/sandbox.ts +++ b/src/lib/sandbox.ts @@ -1,76 +1,11 @@ import process from "node:process"; -import { type Plugin } from "@extism/extism"; import { type Pack } from "@~/__generated__/pack.js"; -import { type PluginOutput, type Logger } from "@~/types.js"; -import { packIdentifier } from "./id.js"; - -/** internal: creates a sandbox based on the pack's permissions */ -const createSandbox = async ( - requestId: string, - pack: Pack, - info: { - request: Request; - response?: Response; - context: Record; - configuration?: Record; - } -) => { - const extractBody = async (httpObject: Request | Response) => { - const body = await httpObject.clone().text(); - try { - return JSON.parse(body) as unknown; - } catch { - return body; - } - }; - - const environmentKeys = pack.permissions?.environment ?? []; - - const env: Record = {}; - for (const key of environmentKeys) { - // eslint-disable-next-line n/no-process-env - env[key] = process.env[key]; - } - - return { - requestId, - context: info.context, - request: { - url: info.request.url, - domain: new URL(info.request.url).hostname, - path: new URL(info.request.url).pathname, - headers: [...info.request.headers.entries()], - body: pack.permissions?.body - ? await extractBody(info.request) - : undefined, - }, - response: info.response - ? { - status: info.response.status, - headers: [...info.response.headers.entries()], - body: pack.permissions?.body - ? await extractBody(info.response) - : undefined, - } - : undefined, - environment: env, - }; -}; - -type getModulesCallback = () => Promise>>; -type onResultCallback = ( - pack: Pack, - result: NonNullable -) => Promise; -type onErrorCallback = (error: any) => void; -type Callbacks = { - getModules: getModulesCallback; - onResult: onResultCallback; - onError: onErrorCallback; -}; +import { type Logger } from "@~/types.js"; +import { type LifecycleCallbacks } from "./lifecycle/types.js"; +import { runPackLifecycle } from "./lifecycle.js"; const isDefined = (value: T | undefined): value is T => { - return value !== undefined; + return Boolean(value && value !== undefined); }; /** @@ -78,70 +13,38 @@ const isDefined = (value: T | undefined): value is T => { * This is where the wasm plugins are ran on either side of the request, and simplifies the * handlers logic to focus on the mocking and not on the lifecycle */ -export const runSandbox = async ( +export const run = async ( requestInfo: { requestId: string; request: Request; }, logger: Logger, packs: Pack[], - callbacks: Callbacks + callbacks: LifecycleCallbacks ) => { const { requestId, request } = requestInfo; const context: Record> = {}; - const plugins = await callbacks.getModules(); - // run packs forward + /* + Pre hook handling. Create a collection of promises that receive the + request object and return a promise that resolves when the pack is done. + */ logger.debug(`[${requestId}] (pre) running sandbox`); - const preHooks = await Promise.allSettled( - packs.map(async (pack, index) => { - const plugin = await plugins.get(packIdentifier(pack)); - - if (!plugin) { - throw new Error(`[${requestId}] Plugin not found in modules`); - } - - // logger.debug(`[${requestId}] running pack`); - - const output = await plugin.call( - "pre", - JSON.stringify( - await createSandbox(requestId, pack, { - request, - context: context[`${index}`] ?? {}, - }) - ) - ); - - if (output) { - const result = output?.json() as PluginOutput; - - if (result.context) { - context[`${index}`] = result.context; - } - - await callbacks.onResult(pack, result); - logger.debug(`[${requestId}] (pre) pack ${pack.name} completed`); - } else { - logger.debug( - `[${requestId}] (pre) pack ${pack.name} no output returned` - ); - } - }) - ); - - for (const hook of preHooks) { - if (hook.status === "rejected") { - logger.error(`${hook.reason}`); - callbacks.onError(hook.reason); - } - } + const pre = runPackLifecycle(requestId, "pre", packs, { + request: request.clone(), + // response + logger, + callbacks, + context, + }); - // run request, convert the raw response (remove compression related headers for compatibility with axios, etc) + logger.debug(`[${requestId}] making request`); const finalizedRequest = request.clone(); finalizedRequest.headers.set("x-tskl-bypass", "1"); const rawResponse = await fetch(finalizedRequest); + await pre; + // GZIP headers are being handled by undici and must be stripped const newHeaders: Array<[string, string]> = [...rawResponse.headers.entries()] .map(([key, value]) => { // remove content encoding @@ -157,57 +60,38 @@ export const runSandbox = async ( return [key, value] as [string, string]; }) .filter(isDefined); - - const fetchResponse = new Response(rawResponse.body, { + const cleanedResponse = new Response(rawResponse.body, { status: rawResponse.status, statusText: rawResponse.statusText, headers: newHeaders, }); - // run packs backwards - logger.debug(`[${requestId}] (post) running sandbox`); - const postHooks = await Promise.allSettled( - packs.reverse().map(async (pack, index) => { - const plugin = await plugins.get(packIdentifier(pack)); - - if (!plugin) { - throw new Error(`[${requestId}] Plugin not found in modules`); - } - - const output = await plugin.call( - "post", - JSON.stringify( - await createSandbox(requestId, pack, { - request, - response: fetchResponse, - // context is at original (non-reversed) index - context: context[`${packs.length - index - 1}`] ?? {}, - }) - ) - ); + logger.debug(`[${requestId}] (chunk) running sandbox`); + const chunks = runPackLifecycle(requestId, "chunk", packs, { + request, + response: cleanedResponse.clone(), + logger, + callbacks, + context, + }); - if (output) { - const result = output?.json() as PluginOutput; - if (result.context) { - context[`${index}`] = result.context; - } + logger.debug(`[${requestId}] (post) running sandbox`); + const post = runPackLifecycle(requestId, "post", packs, { + request, + response: cleanedResponse.clone(), + logger, + callbacks, + context, + }); - await callbacks.onResult(pack, result); - logger.debug(`[${requestId}] (post) pack ${pack.name} completed`); - } else { - logger.debug( - `[${requestId}] (post) pack ${pack.name} no output returned` - ); - } + Promise.allSettled([chunks, post]) + .then(() => { + // finalize the request for local logging + callbacks.onComplete(requestId); }) - ); - - for (const hook of postHooks) { - if (hook.status === "rejected") { - logger.error(`${hook.reason}`); - callbacks.onError(hook.reason); - } - } + .catch(() => { + // empty. Errors are handled inside of the lifecycle + }); - return fetchResponse; + return cleanedResponse; }; diff --git a/src/lib/taskless.ts b/src/lib/taskless.ts index 763eb89..9376ebf 100644 --- a/src/lib/taskless.ts +++ b/src/lib/taskless.ts @@ -24,10 +24,10 @@ import { import { createClient, type NormalizeOAS } from "fets"; import { glob } from "glob"; import { setupServer } from "msw/node"; -import { InitializationError } from "./error.js"; -import { createHandler } from "./handler.js"; -import { id, packIdentifier } from "./id.js"; -import { createLogger } from "./logger.js"; +import { createHandler } from "./msw.js"; +import { InitializationError } from "./util/error.js"; +import { id, packIdentifier } from "./util/id.js"; +import { createLogger } from "./util/logger.js"; import type openapi from "../__generated__/openapi.js"; // our on-demand worker code for a synchronous flush @@ -127,7 +127,7 @@ export const taskless = ( /\/$/, "" ); - const logger = createLogger(options?.logLevel, options?.log); + const logger = createLogger(options?.logLevel, options?.log, useLogging); const client = createClient>({ endpoint: activeEndpoint, globalParams: { @@ -218,10 +218,13 @@ export const taskless = ( return networkPayload; }; - /** Flush all pending telemetry asynchronously */ + /** Flush all pending network telemetry asynchronously */ const flush = async () => { logger.trace("Flushing telemetry data"); + + // clear the pending set and save them to entries const entries = pending.splice(0, pending.length); + const networkPayload = useNetwork ? entriesToNetworkJson(entries) : undefined; @@ -497,20 +500,19 @@ export const taskless = ( * creates monotonic increasing values for each telemetry point */ const capture: CaptureCallback = (entry) => { - logger.debug( - `[${entry.requestId}] Captured ${entry.dimension} as ${entry.value}` - ); pending.push({ ...entry, sequenceId: id(), }); + logger.debug( + `[${entry.requestId}] Captured ${entry.dimension} as ${entry.value}` + ); }; // create the msw interceptor const handler = createHandler({ loaded, logger, - useLogging, capture, getPacks: async () => packs, getModules: async () => modules, diff --git a/src/lib/error.ts b/src/lib/util/error.ts similarity index 100% rename from src/lib/error.ts rename to src/lib/util/error.ts diff --git a/src/lib/id.ts b/src/lib/util/id.ts similarity index 100% rename from src/lib/id.ts rename to src/lib/util/id.ts diff --git a/src/lib/logger.ts b/src/lib/util/logger.ts similarity index 54% rename from src/lib/logger.ts rename to src/lib/util/logger.ts index 1e82803..22e7898 100644 --- a/src/lib/logger.ts +++ b/src/lib/util/logger.ts @@ -25,7 +25,8 @@ const defaultLogger: Required = { export const createLogger = ( userLogLevel?: keyof Logger, - userLogger?: Partial + userLogger?: Partial, + enableDataLogging?: boolean ): Required => { const logLevels: Record = { trace: 10, @@ -37,28 +38,19 @@ export const createLogger = ( }; const logLevel = logLevels[userLogLevel ?? "info"]; + const trace = userLogger?.trace ?? defaultLogger.trace; + const debug = userLogger?.debug ?? defaultLogger.debug; + const info = userLogger?.info ?? defaultLogger.info; + const warn = userLogger?.warn ?? defaultLogger.warn; + const error = userLogger?.error ?? defaultLogger.error; + const data = userLogger?.data ?? defaultLogger.data; return { - trace: - logLevel <= logLevels.trace - ? userLogger?.trace ?? defaultLogger.trace - : noop, - debug: - logLevel <= logLevels.debug - ? userLogger?.debug ?? defaultLogger.debug - : noop, - info: - logLevel <= logLevels.info - ? userLogger?.info ?? defaultLogger.info - : noop, - warn: - logLevel <= logLevels.warn - ? userLogger?.warn ?? defaultLogger.warn - : noop, - error: - logLevel <= logLevels.error - ? userLogger?.error ?? defaultLogger.error - : noop, - data: userLogger?.data ?? defaultLogger.data, + trace: logLevel <= logLevels.trace ? trace : noop, + debug: logLevel <= logLevels.debug ? debug : noop, + info: logLevel <= logLevels.info ? info : noop, + warn: logLevel <= logLevels.warn ? warn : noop, + error: logLevel <= logLevels.error ? error : noop, + data: enableDataLogging ? data : noop, }; }; diff --git a/tsconfig.json b/tsconfig.json index db7d975..e1261f2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -33,7 +33,7 @@ "types": ["node", "vite/client"], "allowJs": true, - "lib": ["ES2021"], + "lib": ["ES2021", "DOM", "DOM.Iterable"], "paths": { "@~/*": ["./src/*"],