From 75875b6d399afda4c520cb70b8a72d22d520461e Mon Sep 17 00:00:00 2001 From: anadi45 Date: Sun, 12 Oct 2025 18:23:50 +0530 Subject: [PATCH 1/5] feat: knowledge base indexing poller --- api.md | 1 + .../knowledge-bases/indexing-jobs.ts | 69 +++++++++++++++++++ .../knowledge-bases/indexing-jobs.test.ts | 58 ++++++++++++++++ 3 files changed, 128 insertions(+) diff --git a/api.md b/api.md index bc4367a..2167eda 100644 --- a/api.md +++ b/api.md @@ -733,6 +733,7 @@ Methods: - client.knowledgeBases.indexingJobs.list({ ...params }) -> IndexingJobListResponse - client.knowledgeBases.indexingJobs.retrieveDataSources(indexingJobUuid) -> IndexingJobRetrieveDataSourcesResponse - client.knowledgeBases.indexingJobs.updateCancel(pathUuid, { ...params }) -> IndexingJobUpdateCancelResponse +- client.knowledgeBases.indexingJobs.waitForCompletion(uuid, { ...options }) -> IndexingJobRetrieveResponse # Models diff --git a/src/resources/knowledge-bases/indexing-jobs.ts b/src/resources/knowledge-bases/indexing-jobs.ts index 0a54512..4c2d59c 100644 --- a/src/resources/knowledge-bases/indexing-jobs.ts +++ b/src/resources/knowledge-bases/indexing-jobs.ts @@ -5,6 +5,7 @@ import * as Shared from '../shared'; import { APIPromise } from '../../core/api-promise'; import { RequestOptions } from '../../internal/request-options'; import { path } from '../../internal/utils/path'; +import { sleep } from '../../internal/utils/sleep'; export class IndexingJobs extends APIResource { /** @@ -113,6 +114,74 @@ export class IndexingJobs extends APIResource { ...options, }); } + + /** + * Polls for indexing job completion with configurable interval and timeout. + * Returns the final job state when completed, failed, or cancelled. + * + * @param uuid - The indexing job UUID to poll + * @param options - Polling configuration options + * @returns Promise that resolves with the final job state + * + * @example + * ```ts + * const job = await client.knowledgeBases.indexingJobs.waitForCompletion( + * '123e4567-e89b-12d3-a456-426614174000', + * { interval: 5000, timeout: 300000 } + * ); + * console.log('Job completed with phase:', job.job?.phase); + * ``` + */ + async waitForCompletion( + uuid: string, + options: { + /** + * Polling interval in milliseconds (default: 5000ms) + */ + interval?: number; + /** + * Maximum time to wait in milliseconds (default: 600000ms = 10 minutes) + */ + timeout?: number; + /** + * Request options to pass to each poll request + */ + requestOptions?: RequestOptions; + } = {}, + ): Promise { + const { interval = 5000, timeout = 600000, requestOptions } = options; + const startTime = Date.now(); + + while (true) { + const response = await this.retrieve(uuid, requestOptions); + const job = response.job; + + if (!job) { + throw new Error('Job not found'); + } + + // Check if job is in a terminal state + if (job.phase === 'BATCH_JOB_PHASE_SUCCEEDED') { + return response; + } + + if (job.phase === 'BATCH_JOB_PHASE_FAILED' || job.phase === 'BATCH_JOB_PHASE_ERROR') { + throw new Error(`Indexing job failed with phase: ${job.phase}`); + } + + if (job.phase === 'BATCH_JOB_PHASE_CANCELLED') { + throw new Error('Indexing job was cancelled'); + } + + // Check timeout + if (Date.now() - startTime > timeout) { + throw new Error(`Indexing job polling timed out after ${timeout}ms`); + } + + // Wait before next poll + await sleep(interval); + } + } } export interface APIIndexedDataSource { diff --git a/tests/api-resources/knowledge-bases/indexing-jobs.test.ts b/tests/api-resources/knowledge-bases/indexing-jobs.test.ts index ff34604..fc03c5c 100644 --- a/tests/api-resources/knowledge-bases/indexing-jobs.test.ts +++ b/tests/api-resources/knowledge-bases/indexing-jobs.test.ts @@ -109,4 +109,62 @@ describe('resource indexingJobs', () => { ), ).rejects.toThrow(Gradient.NotFoundError); }); + + describe('waitForCompletion', () => { + // Prism tests are disabled + test.skip('waits for job completion successfully', async () => { + const jobUuid = '123e4567-e89b-12d3-a456-426614174000'; + const responsePromise = client.knowledgeBases.indexingJobs.waitForCompletion(jobUuid, { + interval: 100, + timeout: 1000, + }); + const response = await responsePromise; + expect(response).toBeDefined(); + expect(response.job).toBeDefined(); + }); + + // Prism tests are disabled + test.skip('throws error when job fails', async () => { + const jobUuid = '123e4567-e89b-12d3-a456-426614174000'; + await expect( + client.knowledgeBases.indexingJobs.waitForCompletion(jobUuid, { + interval: 100, + timeout: 1000, + }), + ).rejects.toThrow(); + }); + + // Prism tests are disabled + test.skip('throws error when job is cancelled', async () => { + const jobUuid = '123e4567-e89b-12d3-a456-426614174000'; + await expect( + client.knowledgeBases.indexingJobs.waitForCompletion(jobUuid, { + interval: 100, + timeout: 1000, + }), + ).rejects.toThrow('Indexing job was cancelled'); + }); + + // Prism tests are disabled + test.skip('throws error when timeout is reached', async () => { + const jobUuid = '123e4567-e89b-12d3-a456-426614174000'; + await expect( + client.knowledgeBases.indexingJobs.waitForCompletion(jobUuid, { + interval: 100, + timeout: 50, // Very short timeout + }), + ).rejects.toThrow('Indexing job polling timed out'); + }); + + // Prism tests are disabled + test.skip('throws error when job is not found', async () => { + const jobUuid = 'nonexistent-job-uuid'; + await expect( + client.knowledgeBases.indexingJobs.waitForCompletion(jobUuid, { + interval: 100, + timeout: 1000, + }), + ).rejects.toThrow('Job not found'); + }); + }); }); From 0ff6772284ab1e4f3f2e1d91024ee6243b748b86 Mon Sep 17 00:00:00 2001 From: anadi45 Date: Tue, 21 Oct 2025 22:46:13 +0530 Subject: [PATCH 2/5] fix: add abort controller --- src/resources/knowledge-bases/indexing-jobs.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/resources/knowledge-bases/indexing-jobs.ts b/src/resources/knowledge-bases/indexing-jobs.ts index 4c2d59c..89ec4eb 100644 --- a/src/resources/knowledge-bases/indexing-jobs.ts +++ b/src/resources/knowledge-bases/indexing-jobs.ts @@ -131,6 +131,17 @@ export class IndexingJobs extends APIResource { * ); * console.log('Job completed with phase:', job.job?.phase); * ``` + * + * @example + * ```ts + * const controller = new AbortController(); + * const job = await client.knowledgeBases.indexingJobs.waitForCompletion( + * '123e4567-e89b-12d3-a456-426614174000', + * { requestOptions: { signal: controller.signal } } + * ); + * // Cancel polling after 30 seconds + * setTimeout(() => controller.abort(), 30000); + * ``` */ async waitForCompletion( uuid: string, @@ -153,6 +164,11 @@ export class IndexingJobs extends APIResource { const startTime = Date.now(); while (true) { + // Check if operation was aborted + if (requestOptions?.signal?.aborted) { + throw new Error('Indexing job polling was aborted'); + } + const response = await this.retrieve(uuid, requestOptions); const job = response.job; From 32ba1a794e4eee4cdfa1d2b61d1c23d3a4513428 Mon Sep 17 00:00:00 2001 From: anadi45 Date: Tue, 21 Oct 2025 22:50:42 +0530 Subject: [PATCH 3/5] fix: defined error types --- src/client.ts | 7 ++ src/index.ts | 7 ++ .../knowledge-bases/indexing-jobs.ts | 86 +++++++++++++++++-- 3 files changed, 95 insertions(+), 5 deletions(-) diff --git a/src/client.ts b/src/client.ts index 24c18a0..56df8ad 100644 --- a/src/client.ts +++ b/src/client.ts @@ -809,6 +809,13 @@ export class Gradient { static InternalServerError = Errors.InternalServerError; static PermissionDeniedError = Errors.PermissionDeniedError; static UnprocessableEntityError = Errors.UnprocessableEntityError; + + // Indexing Job Error Types + static IndexingJobAbortedError = API.KnowledgeBases.IndexingJobs.IndexingJobAbortedError; + static IndexingJobNotFoundError = API.KnowledgeBases.IndexingJobs.IndexingJobNotFoundError; + static IndexingJobFailedError = API.KnowledgeBases.IndexingJobs.IndexingJobFailedError; + static IndexingJobCancelledError = API.KnowledgeBases.IndexingJobs.IndexingJobCancelledError; + static IndexingJobTimeoutError = API.KnowledgeBases.IndexingJobs.IndexingJobTimeoutError; static toFile = Uploads.toFile; diff --git a/src/index.ts b/src/index.ts index 0223275..a068645 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,3 +20,10 @@ export { PermissionDeniedError, UnprocessableEntityError, } from './core/error'; +export { + IndexingJobAbortedError, + IndexingJobNotFoundError, + IndexingJobFailedError, + IndexingJobCancelledError, + IndexingJobTimeoutError, +} from './resources/knowledge-bases/indexing-jobs'; diff --git a/src/resources/knowledge-bases/indexing-jobs.ts b/src/resources/knowledge-bases/indexing-jobs.ts index 89ec4eb..676b87f 100644 --- a/src/resources/knowledge-bases/indexing-jobs.ts +++ b/src/resources/knowledge-bases/indexing-jobs.ts @@ -7,6 +7,56 @@ import { RequestOptions } from '../../internal/request-options'; import { path } from '../../internal/utils/path'; import { sleep } from '../../internal/utils/sleep'; +/** + * Error thrown when an indexing job polling operation is aborted + */ +export class IndexingJobAbortedError extends Error { + constructor() { + super('Indexing job polling was aborted'); + this.name = 'IndexingJobAbortedError'; + } +} + +/** + * Error thrown when an indexing job is not found + */ +export class IndexingJobNotFoundError extends Error { + constructor() { + super('Indexing job not found'); + this.name = 'IndexingJobNotFoundError'; + } +} + +/** + * Error thrown when an indexing job fails + */ +export class IndexingJobFailedError extends Error { + constructor(public readonly phase: string) { + super(`Indexing job failed with phase: ${phase}`); + this.name = 'IndexingJobFailedError'; + } +} + +/** + * Error thrown when an indexing job is cancelled + */ +export class IndexingJobCancelledError extends Error { + constructor() { + super('Indexing job was cancelled'); + this.name = 'IndexingJobCancelledError'; + } +} + +/** + * Error thrown when an indexing job polling times out + */ +export class IndexingJobTimeoutError extends Error { + constructor(public readonly timeoutMs: number) { + super(`Indexing job polling timed out after ${timeoutMs}ms`); + this.name = 'IndexingJobTimeoutError'; + } +} + export class IndexingJobs extends APIResource { /** * To start an indexing job for a knowledge base, send a POST request to @@ -122,6 +172,11 @@ export class IndexingJobs extends APIResource { * @param uuid - The indexing job UUID to poll * @param options - Polling configuration options * @returns Promise that resolves with the final job state + * @throws {IndexingJobAbortedError} When the operation is aborted via AbortSignal + * @throws {IndexingJobNotFoundError} When the job is not found + * @throws {IndexingJobFailedError} When the job fails with phase FAILED or ERROR + * @throws {IndexingJobCancelledError} When the job is cancelled + * @throws {IndexingJobTimeoutError} When polling times out * * @example * ```ts @@ -142,6 +197,22 @@ export class IndexingJobs extends APIResource { * // Cancel polling after 30 seconds * setTimeout(() => controller.abort(), 30000); * ``` + * + * @example + * ```ts + * try { + * const job = await client.knowledgeBases.indexingJobs.waitForCompletion(uuid); + * console.log('Job completed successfully'); + * } catch (error) { + * if (error instanceof IndexingJobFailedError) { + * console.log('Job failed with phase:', error.phase); + * } else if (error instanceof IndexingJobTimeoutError) { + * console.log('Job timed out after:', error.timeoutMs, 'ms'); + * } else if (error instanceof IndexingJobAbortedError) { + * console.log('Job polling was aborted'); + * } + * } + * ``` */ async waitForCompletion( uuid: string, @@ -166,14 +237,14 @@ export class IndexingJobs extends APIResource { while (true) { // Check if operation was aborted if (requestOptions?.signal?.aborted) { - throw new Error('Indexing job polling was aborted'); + throw new IndexingJobAbortedError(); } const response = await this.retrieve(uuid, requestOptions); const job = response.job; if (!job) { - throw new Error('Job not found'); + throw new IndexingJobNotFoundError(); } // Check if job is in a terminal state @@ -182,16 +253,16 @@ export class IndexingJobs extends APIResource { } if (job.phase === 'BATCH_JOB_PHASE_FAILED' || job.phase === 'BATCH_JOB_PHASE_ERROR') { - throw new Error(`Indexing job failed with phase: ${job.phase}`); + throw new IndexingJobFailedError(job.phase); } if (job.phase === 'BATCH_JOB_PHASE_CANCELLED') { - throw new Error('Indexing job was cancelled'); + throw new IndexingJobCancelledError(); } // Check timeout if (Date.now() - startTime > timeout) { - throw new Error(`Indexing job polling timed out after ${timeout}ms`); + throw new IndexingJobTimeoutError(timeout); } // Wait before next poll @@ -452,5 +523,10 @@ export declare namespace IndexingJobs { type IndexingJobCreateParams as IndexingJobCreateParams, type IndexingJobListParams as IndexingJobListParams, type IndexingJobUpdateCancelParams as IndexingJobUpdateCancelParams, + IndexingJobAbortedError, + IndexingJobNotFoundError, + IndexingJobFailedError, + IndexingJobCancelledError, + IndexingJobTimeoutError, }; } From 96053a37e86b28a1fac450d70b151183f773449a Mon Sep 17 00:00:00 2001 From: anadi45 Date: Tue, 21 Oct 2025 22:55:58 +0530 Subject: [PATCH 4/5] fix: add exponential backoff while polling --- .../knowledge-bases/indexing-jobs.ts | 50 +++++++++++++++++-- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/src/resources/knowledge-bases/indexing-jobs.ts b/src/resources/knowledge-bases/indexing-jobs.ts index 676b87f..e67ce41 100644 --- a/src/resources/knowledge-bases/indexing-jobs.ts +++ b/src/resources/knowledge-bases/indexing-jobs.ts @@ -167,8 +167,21 @@ export class IndexingJobs extends APIResource { /** * Polls for indexing job completion with configurable interval and timeout. + * Uses exponential backoff to gradually increase polling intervals, reducing API load for long-running jobs. * Returns the final job state when completed, failed, or cancelled. * + * **Exponential Backoff Behavior:** + * - First 2 polls use the initial interval + * - Starting from the 3rd poll, the interval doubles after each poll + * - The interval is capped at the `maxInterval` value + * - Example with default values (interval: 5000ms, maxInterval: 30000ms): + * - Poll 1: 5000ms wait + * - Poll 2: 5000ms wait + * - Poll 3: 10000ms wait (5000 * 2) + * - Poll 4: 20000ms wait (10000 * 2) + * - Poll 5: 30000ms wait (20000 * 1.5, capped at maxInterval) + * - Poll 6+: 30000ms wait (continues at maxInterval) + * * @param uuid - The indexing job UUID to poll * @param options - Polling configuration options * @returns Promise that resolves with the final job state @@ -200,6 +213,16 @@ export class IndexingJobs extends APIResource { * * @example * ```ts + * // Custom exponential backoff configuration + * const job = await client.knowledgeBases.indexingJobs.waitForCompletion(uuid, { + * interval: 2000, // Start with 2 second intervals + * maxInterval: 60000, // Cap at 1 minute intervals + * timeout: 1800000 // 30 minute timeout + * }); + * ``` + * + * @example + * ```ts * try { * const job = await client.knowledgeBases.indexingJobs.waitForCompletion(uuid); * console.log('Job completed successfully'); @@ -218,21 +241,34 @@ export class IndexingJobs extends APIResource { uuid: string, options: { /** - * Polling interval in milliseconds (default: 5000ms) + * Initial polling interval in milliseconds (default: 5000ms). + * This interval will be used for the first few polls, then exponential backoff applies. */ interval?: number; /** * Maximum time to wait in milliseconds (default: 600000ms = 10 minutes) */ timeout?: number; + /** + * Maximum polling interval in milliseconds (default: 30000ms = 30 seconds). + * Exponential backoff will not exceed this value. + */ + maxInterval?: number; /** * Request options to pass to each poll request */ requestOptions?: RequestOptions; } = {}, ): Promise { - const { interval = 5000, timeout = 600000, requestOptions } = options; + const { + interval = 5000, + timeout = 600000, + maxInterval = 30000, + requestOptions + } = options; const startTime = Date.now(); + let currentInterval = interval; + let pollCount = 0; while (true) { // Check if operation was aborted @@ -265,8 +301,14 @@ export class IndexingJobs extends APIResource { throw new IndexingJobTimeoutError(timeout); } - // Wait before next poll - await sleep(interval); + // Wait before next poll with exponential backoff + await sleep(currentInterval); + + // Apply exponential backoff: double the interval after each poll, up to maxInterval + pollCount++; + if (pollCount > 2) { // Start exponential backoff after 2 polls + currentInterval = Math.min(currentInterval * 2, maxInterval); + } } } } From fc252811e33e83bf76db45d1fc7e5558e1f5c486 Mon Sep 17 00:00:00 2001 From: anadi45 Date: Wed, 22 Oct 2025 23:38:00 +0530 Subject: [PATCH 5/5] fix: lints --- README.md | 3 +-- src/client.ts | 2 +- src/resources/knowledge-bases/indexing-jobs.ts | 12 ++++-------- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 950a8b2..797ac88 100644 --- a/README.md +++ b/README.md @@ -401,7 +401,6 @@ If you are interested in other runtime environments, please open or upvote an is See [the contributing documentation](./CONTRIBUTING.md). - ## License -Licensed under the Apache License 2.0. See [LICENSE.](./LICENSE) \ No newline at end of file +Licensed under the Apache License 2.0. See [LICENSE.](./LICENSE) diff --git a/src/client.ts b/src/client.ts index 56df8ad..bdb9d87 100644 --- a/src/client.ts +++ b/src/client.ts @@ -809,7 +809,7 @@ export class Gradient { static InternalServerError = Errors.InternalServerError; static PermissionDeniedError = Errors.PermissionDeniedError; static UnprocessableEntityError = Errors.UnprocessableEntityError; - + // Indexing Job Error Types static IndexingJobAbortedError = API.KnowledgeBases.IndexingJobs.IndexingJobAbortedError; static IndexingJobNotFoundError = API.KnowledgeBases.IndexingJobs.IndexingJobNotFoundError; diff --git a/src/resources/knowledge-bases/indexing-jobs.ts b/src/resources/knowledge-bases/indexing-jobs.ts index e67ce41..25fdd8b 100644 --- a/src/resources/knowledge-bases/indexing-jobs.ts +++ b/src/resources/knowledge-bases/indexing-jobs.ts @@ -260,12 +260,7 @@ export class IndexingJobs extends APIResource { requestOptions?: RequestOptions; } = {}, ): Promise { - const { - interval = 5000, - timeout = 600000, - maxInterval = 30000, - requestOptions - } = options; + const { interval = 5000, timeout = 600000, maxInterval = 30000, requestOptions } = options; const startTime = Date.now(); let currentInterval = interval; let pollCount = 0; @@ -303,10 +298,11 @@ export class IndexingJobs extends APIResource { // Wait before next poll with exponential backoff await sleep(currentInterval); - + // Apply exponential backoff: double the interval after each poll, up to maxInterval pollCount++; - if (pollCount > 2) { // Start exponential backoff after 2 polls + if (pollCount > 2) { + // Start exponential backoff after 2 polls currentInterval = Math.min(currentInterval * 2, maxInterval); } }