diff --git a/src/base/resource_collection_client.ts b/src/base/resource_collection_client.ts index cc476397..ac42268a 100644 --- a/src/base/resource_collection_client.ts +++ b/src/base/resource_collection_client.ts @@ -1,3 +1,4 @@ +import type { PaginatedResponse, PaginationOptions } from '../utils'; import { parseDateFields, pluckData } from '../utils'; import { ApiClient } from './api_client'; @@ -18,6 +19,41 @@ export class ResourceCollectionClient extends ApiClient { return parseDateFields(pluckData(response.data)) as R; } + /** + * Returns async iterator to paginate through all items and first page of results is returned immediately as well. + */ + protected _getPaginatedIterator>( + options: T = {} as T, + ): AsyncIterable & Promise { + const getPaginatedList = this._list.bind(this); + const paginatedListPromise = getPaginatedList(options); + + async function* asyncGenerator() { + let currentPage = await paginatedListPromise; + yield* currentPage.items; + const offset = options.offset ?? 0; + const limit = Math.min(options.limit || currentPage.total, currentPage.total); + + let currentOffset = offset + currentPage.items.length; + let remainingItems = Math.min(currentPage.total - offset, limit) - currentPage.items.length; + + while ( + currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page. + remainingItems > 0 + ) { + const newOptions = { ...options, limit: remainingItems, offset: currentOffset }; + currentPage = await getPaginatedList(newOptions); + yield* currentPage.items; + currentOffset += currentPage.items.length; + remainingItems -= currentPage.items.length; + } + } + + return Object.defineProperty(paginatedListPromise, Symbol.asyncIterator, { + value: asyncGenerator, + }) as unknown as AsyncIterable & Promise; + } + protected async _create(resource: D): Promise { const response = await this.httpClient.call({ url: this._url(), diff --git a/src/resource_clients/actor_collection.ts b/src/resource_clients/actor_collection.ts index 43ad2656..47bd075c 100644 --- a/src/resource_clients/actor_collection.ts +++ b/src/resource_clients/actor_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedIterator, PaginatedList, PaginationOptions } from '../utils'; import type { Actor, ActorDefaultRunOptions, ActorExampleRunInput, ActorStandby } from './actor'; import type { ActorVersion } from './actor_version'; @@ -19,8 +19,15 @@ export class ActorCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/actors/actor-collection/get-list-of-actors + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list(options: ActorCollectionListOptions = {}): Promise { + list(options: ActorCollectionListOptions = {}): PaginatedIterator { ow( options, ow.object.exactShape({ @@ -32,7 +39,7 @@ export class ActorCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } /** @@ -50,11 +57,8 @@ export enum ActorListSortBy { LAST_RUN_STARTED_AT = 'stats.lastRunStartedAt', } -export interface ActorCollectionListOptions { +export interface ActorCollectionListOptions extends PaginationOptions { my?: boolean; - limit?: number; - offset?: number; - desc?: boolean; sortBy?: ActorListSortBy; } diff --git a/src/resource_clients/actor_env_var_collection.ts b/src/resource_clients/actor_env_var_collection.ts index 0a959230..f06ae774 100644 --- a/src/resource_clients/actor_env_var_collection.ts +++ b/src/resource_clients/actor_env_var_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedList, PaginationOptions } from '../utils'; import type { ActorEnvironmentVariable } from './actor_version'; export class ActorEnvVarCollectionClient extends ResourceCollectionClient { @@ -18,8 +18,17 @@ export class ActorEnvVarCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/actors/environment-variable-collection/get-list-of-environment-variables + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list(options: ActorEnvVarCollectionListOptions = {}): Promise { + list( + options: ActorEnvVarCollectionListOptions = {}, + ): Promise & AsyncIterable { ow( options, ow.object.exactShape({ @@ -28,7 +37,7 @@ export class ActorEnvVarCollectionClient extends ResourceCollectionClient { desc: ow.optional.boolean, }), ); - return this._list(options); + return this._getPaginatedIterator(options); } /** @@ -40,9 +49,7 @@ export class ActorEnvVarCollectionClient extends ResourceCollectionClient { } } -export interface ActorEnvVarCollectionListOptions { - limit?: number; - offset?: number; +export interface ActorEnvVarCollectionListOptions extends PaginationOptions { desc?: boolean; } diff --git a/src/resource_clients/actor_version_collection.ts b/src/resource_clients/actor_version_collection.ts index 2cfd6656..1b49b801 100644 --- a/src/resource_clients/actor_version_collection.ts +++ b/src/resource_clients/actor_version_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedList, PaginationOptions } from '../utils'; import type { ActorVersion, FinalActorVersion } from './actor_version'; export class ActorVersionCollectionClient extends ResourceCollectionClient { @@ -18,8 +18,17 @@ export class ActorVersionCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/actors/version-collection/get-list-of-versions + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list(options: ActorVersionCollectionListOptions = {}): Promise { + list( + options: ActorVersionCollectionListOptions = {}, + ): Promise & AsyncIterable { ow( options, ow.object.exactShape({ @@ -29,7 +38,7 @@ export class ActorVersionCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } /** @@ -42,9 +51,7 @@ export class ActorVersionCollectionClient extends ResourceCollectionClient { } } -export interface ActorVersionCollectionListOptions { - limit?: number; - offset?: number; +export interface ActorVersionCollectionListOptions extends PaginationOptions { desc?: boolean; } diff --git a/src/resource_clients/build_collection.ts b/src/resource_clients/build_collection.ts index d4866914..d695b2b4 100644 --- a/src/resource_clients/build_collection.ts +++ b/src/resource_clients/build_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedIterator, PaginatedList, PaginationOptions } from '../utils'; import type { Build } from './build'; export class BuildCollectionClient extends ResourceCollectionClient { @@ -18,8 +18,15 @@ export class BuildCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/actors/build-collection/get-list-of-builds + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list(options: BuildCollectionClientListOptions = {}): Promise { + list(options: BuildCollectionClientListOptions = {}): PaginatedIterator { ow( options, ow.object.exactShape({ @@ -29,13 +36,11 @@ export class BuildCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } } -export interface BuildCollectionClientListOptions { - limit?: number; - offset?: number; +export interface BuildCollectionClientListOptions extends PaginationOptions { desc?: boolean; } diff --git a/src/resource_clients/dataset_collection.ts b/src/resource_clients/dataset_collection.ts index e9b972c8..21a5ed01 100644 --- a/src/resource_clients/dataset_collection.ts +++ b/src/resource_clients/dataset_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedIterator, PaginatedList, PaginationOptions } from '../utils'; import type { Dataset } from './dataset'; export class DatasetCollectionClient extends ResourceCollectionClient { @@ -18,8 +18,15 @@ export class DatasetCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/datasets/dataset-collection/get-list-of-datasets + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list(options: DatasetCollectionClientListOptions = {}): Promise { + list(options: DatasetCollectionClientListOptions = {}): PaginatedIterator { ow( options, ow.object.exactShape({ @@ -30,7 +37,7 @@ export class DatasetCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } /** @@ -44,10 +51,8 @@ export class DatasetCollectionClient extends ResourceCollectionClient { } } -export interface DatasetCollectionClientListOptions { +export interface DatasetCollectionClientListOptions extends PaginationOptions { unnamed?: boolean; - limit?: number; - offset?: number; desc?: boolean; } diff --git a/src/resource_clients/key_value_store_collection.ts b/src/resource_clients/key_value_store_collection.ts index 54faeaed..6a2f14d5 100644 --- a/src/resource_clients/key_value_store_collection.ts +++ b/src/resource_clients/key_value_store_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedList, PaginationOptions } from '../utils'; import type { KeyValueStore } from './key_value_store'; export class KeyValueStoreCollectionClient extends ResourceCollectionClient { @@ -18,10 +18,17 @@ export class KeyValueStoreCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/key-value-stores/store-collection/get-list-of-key-value-stores + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list( + list( options: KeyValueStoreCollectionClientListOptions = {}, - ): Promise> { + ): Promise> & AsyncIterable { ow( options, ow.object.exactShape({ @@ -32,7 +39,7 @@ export class KeyValueStoreCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } /** @@ -49,10 +56,8 @@ export class KeyValueStoreCollectionClient extends ResourceCollectionClient { } } -export interface KeyValueStoreCollectionClientListOptions { +export interface KeyValueStoreCollectionClientListOptions extends PaginationOptions { unnamed?: boolean; - limit?: number; - offset?: number; desc?: boolean; } diff --git a/src/resource_clients/request_queue_collection.ts b/src/resource_clients/request_queue_collection.ts index 54f57628..634f861a 100644 --- a/src/resource_clients/request_queue_collection.ts +++ b/src/resource_clients/request_queue_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedList, PaginationOptions } from '../utils'; import type { RequestQueue } from './request_queue'; export class RequestQueueCollectionClient extends ResourceCollectionClient { @@ -18,8 +18,17 @@ export class RequestQueueCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/request-queues/queue-collection/get-list-of-request-queues + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list(options: RequestQueueCollectionListOptions = {}): Promise { + list( + options: RequestQueueCollectionListOptions = {}, + ): Promise & AsyncIterable { ow( options, ow.object.exactShape({ @@ -30,7 +39,7 @@ export class RequestQueueCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } /** @@ -43,10 +52,8 @@ export class RequestQueueCollectionClient extends ResourceCollectionClient { } } -export interface RequestQueueCollectionListOptions { +export interface RequestQueueCollectionListOptions extends PaginationOptions { unnamed?: boolean; - limit?: number; - offset?: number; desc?: boolean; } diff --git a/src/resource_clients/run_collection.ts b/src/resource_clients/run_collection.ts index b5192534..d321763d 100644 --- a/src/resource_clients/run_collection.ts +++ b/src/resource_clients/run_collection.ts @@ -4,7 +4,7 @@ import { ACTOR_JOB_STATUSES } from '@apify/consts'; import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedIterator, PaginationOptions } from '../utils'; import type { ActorRunListItem } from './actor'; export class RunCollectionClient extends ResourceCollectionClient { @@ -20,8 +20,15 @@ export class RunCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/actors/run-collection/get-list-of-runs + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list(options: RunCollectionListOptions = {}): Promise> { + list(options: RunCollectionListOptions = {}): PaginatedIterator { ow( options, ow.object.exactShape({ @@ -37,13 +44,11 @@ export class RunCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } } -export interface RunCollectionListOptions { - limit?: number; - offset?: number; +export interface RunCollectionListOptions extends PaginationOptions { desc?: boolean; status?: | (typeof ACTOR_JOB_STATUSES)[keyof typeof ACTOR_JOB_STATUSES] diff --git a/src/resource_clients/schedule_collection.ts b/src/resource_clients/schedule_collection.ts index 3c9bb712..6cf2bde1 100644 --- a/src/resource_clients/schedule_collection.ts +++ b/src/resource_clients/schedule_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedIterator, PaginationOptions } from '../utils'; import type { Schedule, ScheduleCreateOrUpdateData } from './schedule'; export class ScheduleCollectionClient extends ResourceCollectionClient { @@ -18,8 +18,15 @@ export class ScheduleCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/schedules/schedules-collection/get-list-of-schedules + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list(options: ScheduleCollectionListOptions = {}): Promise> { + list(options: ScheduleCollectionListOptions = {}): PaginatedIterator { ow( options, ow.object.exactShape({ @@ -29,7 +36,7 @@ export class ScheduleCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } /** @@ -42,8 +49,6 @@ export class ScheduleCollectionClient extends ResourceCollectionClient { } } -export interface ScheduleCollectionListOptions { - limit?: number; - offset?: number; +export interface ScheduleCollectionListOptions extends PaginationOptions { desc?: boolean; } diff --git a/src/resource_clients/store_collection.ts b/src/resource_clients/store_collection.ts index 6dcae28e..5ced435d 100644 --- a/src/resource_clients/store_collection.ts +++ b/src/resource_clients/store_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedIterator, PaginationOptions } from '../utils'; import type { ActorStats } from './actor'; export class StoreCollectionClient extends ResourceCollectionClient { @@ -18,8 +18,15 @@ export class StoreCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2/#/reference/store/store-actors-collection/get-list-of-actors-in-store + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list(options: StoreCollectionListOptions = {}): Promise> { + list(options: StoreCollectionListOptions = {}): PaginatedIterator { ow( options, ow.object.exactShape({ @@ -33,7 +40,7 @@ export class StoreCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } } @@ -54,9 +61,7 @@ export interface ActorStoreList { url: string; } -export interface StoreCollectionListOptions { - limit?: number; - offset?: number; +export interface StoreCollectionListOptions extends PaginationOptions { search?: string; sortBy?: string; category?: string; diff --git a/src/resource_clients/task_collection.ts b/src/resource_clients/task_collection.ts index 48a71461..21111f28 100644 --- a/src/resource_clients/task_collection.ts +++ b/src/resource_clients/task_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedIterator, PaginationOptions } from '../utils'; import type { Task, TaskUpdateData } from './task'; export class TaskCollectionClient extends ResourceCollectionClient { @@ -22,9 +22,16 @@ export class TaskCollectionClient extends ResourceCollectionClient { * @param {number} [options.limit] * @param {number} [options.offset] * @param {boolean} [options.desc] - * @return {Promise} + * @return {PaginatedIterator} + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list(options: TaskCollectionListOptions = {}): Promise> { + list(options: TaskCollectionListOptions = {}): PaginatedIterator { ow( options, ow.object.exactShape({ @@ -34,7 +41,7 @@ export class TaskCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } /** @@ -47,9 +54,7 @@ export class TaskCollectionClient extends ResourceCollectionClient { } } -export interface TaskCollectionListOptions { - limit?: number; - offset?: number; +export interface TaskCollectionListOptions extends PaginationOptions { desc?: boolean; } diff --git a/src/resource_clients/webhook_collection.ts b/src/resource_clients/webhook_collection.ts index 7865134b..5bc2a488 100644 --- a/src/resource_clients/webhook_collection.ts +++ b/src/resource_clients/webhook_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedIterator, PaginationOptions } from '../utils'; import type { Webhook, WebhookUpdateData } from './webhook'; export class WebhookCollectionClient extends ResourceCollectionClient { @@ -18,10 +18,18 @@ export class WebhookCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/webhooks/webhook-collection/get-list-of-webhooks + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list( + + list( options: WebhookCollectionListOptions = {}, - ): Promise>> { + ): PaginatedIterator> { ow( options, ow.object.exactShape({ @@ -31,7 +39,7 @@ export class WebhookCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } /** @@ -44,8 +52,6 @@ export class WebhookCollectionClient extends ResourceCollectionClient { } } -export interface WebhookCollectionListOptions { - limit?: number; - offset?: number; +export interface WebhookCollectionListOptions extends PaginationOptions { desc?: boolean; } diff --git a/src/resource_clients/webhook_dispatch_collection.ts b/src/resource_clients/webhook_dispatch_collection.ts index bed32ebb..15145050 100644 --- a/src/resource_clients/webhook_dispatch_collection.ts +++ b/src/resource_clients/webhook_dispatch_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { PaginatedIterator, PaginationOptions } from '../utils'; import type { WebhookDispatch } from './webhook_dispatch'; export class WebhookDispatchCollectionClient extends ResourceCollectionClient { @@ -18,8 +18,15 @@ export class WebhookDispatchCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/webhook-dispatches/webhook-dispatches-collection/get-list-of-webhook-dispatches + * + * Use as a promise. It will always do only 1 API call: + * const paginatedList = await client.list(options); + * + * Use as an async iterator. It can do multiple API calls if needed: + * for await (const singleItem of client.list(options)) {...} + * */ - async list(options: WebhookDispatchCollectionListOptions = {}): Promise> { + list(options: WebhookDispatchCollectionListOptions = {}): PaginatedIterator { ow( options, ow.object.exactShape({ @@ -29,12 +36,10 @@ export class WebhookDispatchCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getPaginatedIterator(options); } } -export interface WebhookDispatchCollectionListOptions { - limit?: number; - offset?: number; +export interface WebhookDispatchCollectionListOptions extends PaginationOptions { desc?: boolean; } diff --git a/src/utils.ts b/src/utils.ts index e5981c98..55b5775a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -233,6 +233,20 @@ export interface PaginationIteratorOptions { exclusiveStartId?: string; } +export interface PaginationOptions { + /** Position of the first returned entry. */ + offset?: number; + /** Maximum number of entries requested. */ + limit?: number; +} + +export interface PaginatedResponse { + /** Total count of entries. */ + total: number; + /** Entries. */ + items: Data[]; +} + export interface PaginatedList { /** Total count of entries in the dataset. */ total: number; @@ -248,6 +262,8 @@ export interface PaginatedList { items: Data[]; } +export type PaginatedIterator = Promise> & AsyncIterable; + export function cast(input: unknown): T { return input as T; } diff --git a/test/datasets.test.js b/test/datasets.test.js index d0441b7d..4734c2ad 100644 --- a/test/datasets.test.js +++ b/test/datasets.test.js @@ -438,3 +438,122 @@ describe('Dataset methods', () => { }); }); }); + +describe('actor.store.list as async iterable', () => { + // Test using store().list() as an async iterable + const client = new ApifyClient(); + const maxItemsPerPage = 1000; + + const range = (start, end, step = 1) => { + // Inclusive range, ordered based on start and end values + return Array.from( + { + length: Math.abs(end - start) + 1, + }, + (_, i) => start + Math.sign(end - start) * step * i, + ); + }; + + const testCases = [ + { + testName: 'No offset, no limit', + userDefinedOptions: {}, + expectedItems: range(0, 2499), + }, + { + testName: 'No offset, user limit', + userDefinedOptions: { limit: 1100 }, + expectedItems: range(0, 1099), + }, + { + testName: 'User offset, no limit', + userDefinedOptions: { offset: 1000 }, + expectedItems: range(1000, 2499), + }, + { + testName: 'User offset, user limit', + userDefinedOptions: { offset: 1000, limit: 1100 }, + expectedItems: range(1000, 2099), + }, + { + testName: 'User offset, user limit, descending', + userDefinedOptions: { offset: 1000, limit: 1100, desc: true }, + expectedItems: range(1500, 401), + }, + { + testName: 'User offset, user limit, descending, unnamed included', + userDefinedOptions: { offset: 50, limit: 1100, desc: true, unnamed: true }, + expectedItems: range(2550, 1451), + }, + { + testName: 'User out of range offset, no limit', + userDefinedOptions: { offset: 3000 }, + expectedItems: [], + }, + { + testName: 'User no offset, out of range limit', + userDefinedOptions: { limit: 3000 }, + expectedItems: range(0, 2499), + }, + { + testName: 'User no offset, out of range limit', + userDefinedOptions: { limit: 3000 }, + expectedItems: range(0, 2499), + }, + ]; + + test.each(testCases)('$testName', async ({ userDefinedOptions, expectedItems }) => { + const mockedPlatformLogic = async (request) => { + // Simulated platform logic for pagination. + // There are 2500 named items in the collection and additional 100 unnamed items. + // Items are simple number for easy verification 0..2499 -> named, 2500..2599 -> unnamed + + const namedItems = 2500; // represent named items + const unnamedItems = 100; // additional unnamed items + + const totalItems = request.params.unnamed ? namedItems + unnamedItems : namedItems; + + const offset = request.params.offset ? request.params.offset : 0; + const limit = request.params.limit ? request.params.limit : 0; + const desc = request.params.desc ? request.params.desc : false; + + const items = range(desc ? totalItems : 0, desc ? 0 : totalItems); + + if (offset < 0 || limit < 0) { + throw new Error('Offset and limit must be non-negative'); + } + + const lowerIndex = Math.min(offset, totalItems); + const upperIndex = Math.min(offset + (limit || totalItems), totalItems); + const returnedItemsCount = Math.min(upperIndex - lowerIndex, maxItemsPerPage); + + return { + data: { + data: { + total: namedItems, + count: returnedItemsCount, + offset, + limit: returnedItemsCount, + desc: false, + items: items.slice(lowerIndex, Math.min(upperIndex, lowerIndex + maxItemsPerPage)), + }, + }, + }; + }; + + const datasetsClient = client.datasets(); + + const mockedClient = jest.spyOn(datasetsClient.httpClient, 'call').mockImplementation(mockedPlatformLogic); + + try { + const totalItems = []; + for await (const page of datasetsClient.list(userDefinedOptions)) { + totalItems.push(page); + } + expect(totalItems).toEqual(expectedItems); + expect(mockedClient).toHaveBeenCalledTimes(Math.max(Math.ceil(expectedItems.length / maxItemsPerPage), 1)); + } finally { + mockedClient.mockRestore(); + } + }); +});