From 8c38f9449b4db4d4875df6a3261aec5244ab64dd Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 20 Nov 2025 11:22:19 +0100 Subject: [PATCH 1/9] Draft, working. TODO: generalize for other endpoints --- src/resource_clients/dataset.ts | 30 ++++++++++++++++++++---- src/resource_clients/store_collection.ts | 30 +++++++++++++++++++++--- src/utils.ts | 14 ++++++++++- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/src/resource_clients/dataset.ts b/src/resource_clients/dataset.ts index 220b3409..e7f7ba18 100644 --- a/src/resource_clients/dataset.ts +++ b/src/resource_clients/dataset.ts @@ -12,7 +12,7 @@ import { SMALL_TIMEOUT_MILLIS, } from '../base/resource_client'; import type { ApifyRequestConfig, ApifyResponse } from '../http_client'; -import type { PaginatedList } from '../utils'; +import { IterablePaginatedList, PaginatedList } from "../utils"; import { applyQueryParamsToUrl, cast, catchNotFoundOrThrow, pluckData } from '../utils'; export class DatasetClient< @@ -54,7 +54,7 @@ export class DatasetClient< /** * https://docs.apify.com/api/v2#/reference/datasets/item-collection/get-items */ - async listItems(options: DatasetClientListItemOptions = {}): Promise> { + async listItems(options: DatasetClientListItemOptions = {}): Promise> { ow( options, ow.object.exactShape({ @@ -73,14 +73,36 @@ export class DatasetClient< }), ); - const response = await this.httpClient.call({ + // TODO: Should empty limit return all, or current behavior, which is max per request 1000? + let chunksIterated = 0; + const self=this; + const firstResponse = await self.httpClient.call({ url: this._url('items'), method: 'GET', params: this._params(options), timeout: DEFAULT_TIMEOUT_MILLIS, }); - return this._createPaginationList(response, options.desc ?? false); + let currentPage = this._createPaginationList(firstResponse, options.desc ?? false); + + return { + ...currentPage, + async *[Symbol.asyncIterator](){ + while (currentPage.items.length!=0 || chunksIterated===0) { + if (chunksIterated > 0) { + const response = await self.httpClient.call({ + url: self._url('items'), + method: 'GET', + params: self._params(options), + timeout: DEFAULT_TIMEOUT_MILLIS, + }); + currentPage = self._createPaginationList(response, options.desc ?? false); + } + chunksIterated++; + yield currentPage; + } + } + }; } /** diff --git a/src/resource_clients/store_collection.ts b/src/resource_clients/store_collection.ts index 6dcae28e..4f29e2dc 100644 --- a/src/resource_clients/store_collection.ts +++ b/src/resource_clients/store_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { PaginatedList } from '../utils'; +import type { IterablePaginatedList , PaginatedList} from "../utils"; import type { ActorStats } from './actor'; export class StoreCollectionClient extends ResourceCollectionClient { @@ -19,7 +19,7 @@ export class StoreCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2/#/reference/store/store-actors-collection/get-list-of-actors-in-store */ - async list(options: StoreCollectionListOptions = {}): Promise> { + async list(options: StoreCollectionListOptions = {}): Promise> { ow( options, ow.object.exactShape({ @@ -32,8 +32,32 @@ export class StoreCollectionClient extends ResourceCollectionClient { pricingModel: ow.optional.string, }), ); + const getPaginatedList = this._list.bind(this); - return this._list(options); + let currentPage = await getPaginatedList>(options) + + + + return { + ...currentPage, + async *[Symbol.asyncIterator](){ + yield currentPage; + let itemsFetched = currentPage.items.length; + let currentLimit= options.limit !== undefined ? options.limit-itemsFetched: undefined; + let currentOffset= options.offset ?? 0 + itemsFetched; + + while (currentPage.items.length>0 && (currentLimit === undefined || currentLimit>0)) { + + const newOptions = { ...options, limit: currentLimit, offset: currentOffset}; + currentPage = await getPaginatedList>(newOptions) + yield currentPage; + itemsFetched += currentPage.items.length + currentLimit= options.limit !== undefined ? options.limit-itemsFetched: undefined; + currentOffset= options.offset ?? 0 + itemsFetched; + + } + } + }; } } diff --git a/src/utils.ts b/src/utils.ts index e5981c98..a9fe0e71 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -233,7 +233,7 @@ export interface PaginationIteratorOptions { exclusiveStartId?: string; } -export interface PaginatedList { +export interface PaginatedList{ /** Total count of entries in the dataset. */ total: number; /** Count of dataset entries returned in this set. */ @@ -248,6 +248,18 @@ export interface PaginatedList { items: Data[]; } +export interface IterablePagination{ + /** Position of the first returned entry in the dataset. */ + offset: number; + /** Maximum number of dataset entries requested. */ + limit: number; + /** Should the results be in descending order. */ + items: Data[]; +} + +export interface IterablePaginatedList extends PaginatedList, AsyncIterable>{ +} + export function cast(input: unknown): T { return input as T; } From 68ed7a3c51ab34be117e844ae276139bc547bcb7 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 20 Nov 2025 16:08:00 +0100 Subject: [PATCH 2/9] Generalized form. Applied only to `StoreCollectionClient` --- src/base/resource_collection_client.ts | 39 +++++- src/resource_clients/dataset.ts | 30 +---- src/resource_clients/store_collection.ts | 28 +--- src/utils.ts | 32 +++-- test/store.test.ts | 163 +++++++++++++++++++++-- 5 files changed, 212 insertions(+), 80 deletions(-) diff --git a/src/base/resource_collection_client.ts b/src/base/resource_collection_client.ts index cc476397..c910ef2e 100644 --- a/src/base/resource_collection_client.ts +++ b/src/base/resource_collection_client.ts @@ -1,4 +1,4 @@ -import { parseDateFields, pluckData } from '../utils'; +import type { PaginatedResponse, PaginationOptions , parseDateFields, pluckData } from '../utils'; import { ApiClient } from './api_client'; /** @@ -18,6 +18,43 @@ export class ResourceCollectionClient extends ApiClient { return parseDateFields(pluckData(response.data)) as R; } + /** + * Returns async iterator to paginate through all pages and first page of results is returned immediately as well. + * @private + */ + protected async _getIterablePagination( + options: T = {} as T, + ): Promise> { + const getPaginatedList = this._list.bind(this); + + let currentPage = await getPaginatedList(options); + + return { + ...currentPage, + async *[Symbol.asyncIterator]() { + yield currentPage; + let itemsFetched = currentPage.items.length; + let currentLimit = options.limit !== undefined ? options.limit - itemsFetched : undefined; + let currentOffset = options.offset ?? 0 + itemsFetched; + const maxRelevantItems = + currentPage.total === undefined ? undefined : currentPage.total - (options.offset || 0); + + while ( + currentPage.items.length > 0 && // Some items were returned in last page + (currentLimit === undefined || currentLimit > 0) && // User defined a limit, and we have not yet exhausted it + (maxRelevantItems === undefined || maxRelevantItems > itemsFetched) // We know total and we did not get it yet + ) { + const newOptions = { ...options, limit: currentLimit, offset: currentOffset }; + currentPage = await getPaginatedList(newOptions); + yield currentPage; + itemsFetched += currentPage.items.length; + currentLimit = options.limit !== undefined ? options.limit - itemsFetched : undefined; + currentOffset = options.offset ?? 0 + itemsFetched; + } + }, + }; + } + protected async _create(resource: D): Promise { const response = await this.httpClient.call({ url: this._url(), diff --git a/src/resource_clients/dataset.ts b/src/resource_clients/dataset.ts index e7f7ba18..220b3409 100644 --- a/src/resource_clients/dataset.ts +++ b/src/resource_clients/dataset.ts @@ -12,7 +12,7 @@ import { SMALL_TIMEOUT_MILLIS, } from '../base/resource_client'; import type { ApifyRequestConfig, ApifyResponse } from '../http_client'; -import { IterablePaginatedList, PaginatedList } from "../utils"; +import type { PaginatedList } from '../utils'; import { applyQueryParamsToUrl, cast, catchNotFoundOrThrow, pluckData } from '../utils'; export class DatasetClient< @@ -54,7 +54,7 @@ export class DatasetClient< /** * https://docs.apify.com/api/v2#/reference/datasets/item-collection/get-items */ - async listItems(options: DatasetClientListItemOptions = {}): Promise> { + async listItems(options: DatasetClientListItemOptions = {}): Promise> { ow( options, ow.object.exactShape({ @@ -73,36 +73,14 @@ export class DatasetClient< }), ); - // TODO: Should empty limit return all, or current behavior, which is max per request 1000? - let chunksIterated = 0; - const self=this; - const firstResponse = await self.httpClient.call({ + const response = await this.httpClient.call({ url: this._url('items'), method: 'GET', params: this._params(options), timeout: DEFAULT_TIMEOUT_MILLIS, }); - let currentPage = this._createPaginationList(firstResponse, options.desc ?? false); - - return { - ...currentPage, - async *[Symbol.asyncIterator](){ - while (currentPage.items.length!=0 || chunksIterated===0) { - if (chunksIterated > 0) { - const response = await self.httpClient.call({ - url: self._url('items'), - method: 'GET', - params: self._params(options), - timeout: DEFAULT_TIMEOUT_MILLIS, - }); - currentPage = self._createPaginationList(response, options.desc ?? false); - } - chunksIterated++; - yield currentPage; - } - } - }; + return this._createPaginationList(response, options.desc ?? false); } /** diff --git a/src/resource_clients/store_collection.ts b/src/resource_clients/store_collection.ts index 4f29e2dc..a592398f 100644 --- a/src/resource_clients/store_collection.ts +++ b/src/resource_clients/store_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { IterablePaginatedList , PaginatedList} from "../utils"; +import type { IterablePaginatedList } from '../utils'; import type { ActorStats } from './actor'; export class StoreCollectionClient extends ResourceCollectionClient { @@ -32,32 +32,8 @@ export class StoreCollectionClient extends ResourceCollectionClient { pricingModel: ow.optional.string, }), ); - const getPaginatedList = this._list.bind(this); - let currentPage = await getPaginatedList>(options) - - - - return { - ...currentPage, - async *[Symbol.asyncIterator](){ - yield currentPage; - let itemsFetched = currentPage.items.length; - let currentLimit= options.limit !== undefined ? options.limit-itemsFetched: undefined; - let currentOffset= options.offset ?? 0 + itemsFetched; - - while (currentPage.items.length>0 && (currentLimit === undefined || currentLimit>0)) { - - const newOptions = { ...options, limit: currentLimit, offset: currentOffset}; - currentPage = await getPaginatedList>(newOptions) - yield currentPage; - itemsFetched += currentPage.items.length - currentLimit= options.limit !== undefined ? options.limit-itemsFetched: undefined; - currentOffset= options.offset ?? 0 + itemsFetched; - - } - } - }; + return this._getIterablePagination(options); } } diff --git a/src/utils.ts b/src/utils.ts index a9fe0e71..ca062373 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -233,32 +233,36 @@ export interface PaginationIteratorOptions { exclusiveStartId?: string; } -export interface PaginatedList{ +export interface PaginationOptions { + /** Position of the first returned entry. */ + offset?: number; + /** Maximum number of entries requested. */ + limit?: number; +} + +export interface PaginatedResponse { + /** Total count of entries. */ + total?: number; + /** Entries. */ + items: unknown[]; +} + +export interface PaginatedList { /** Total count of entries in the dataset. */ total: number; /** Count of dataset entries returned in this set. */ count: number; - /** Position of the first returned entry in the dataset. */ - offset: number; - /** Maximum number of dataset entries requested. */ - limit: number; /** Should the results be in descending order. */ desc: boolean; - /** Dataset entries based on chosen format parameter. */ - items: Data[]; -} - -export interface IterablePagination{ - /** Position of the first returned entry in the dataset. */ + /** Position of the first returned entry. */ offset: number; /** Maximum number of dataset entries requested. */ limit: number; - /** Should the results be in descending order. */ + /** Dataset entries based on chosen format parameter. */ items: Data[]; } -export interface IterablePaginatedList extends PaginatedList, AsyncIterable>{ -} +export interface IterablePaginatedList extends PaginatedList, AsyncIterable> {} export function cast(input: unknown): T { return input as T; diff --git a/test/store.test.ts b/test/store.test.ts index daa07da9..c5e8efa2 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -1,8 +1,16 @@ +import { setTimeout } from 'node:timers/promises'; + +import c from 'ansi-colors'; import type { StoreCollectionListOptions } from 'apify-client'; -import { ApifyClient } from 'apify-client'; +import { ApifyClient, LoggerActorRedirect } from 'apify-client'; +import express from 'express'; + +import { LEVELS, Log } from '@apify/log'; +import type { ApifyResponse } from '../src/http_client'; import { Browser, DEFAULT_OPTIONS, validateRequest } from './_helper'; -import { mockServer } from './mock_server/server'; +import { createDefaultApp, mockServer } from './mock_server/server'; +import { MOCKED_ACTOR_LOGS_PROCESSED, StatusGenerator } from './mock_server/test_utils'; describe('Store', () => { let baseUrl: string | undefined; @@ -33,17 +41,17 @@ describe('Store', () => { page.close().catch(() => {}); }); - test('list() works', async () => { - const opts = { - limit: 5, - offset: 3, - search: 'my search', - sortBy: 'my sort', - category: 'my category', - username: 'my username', - pricingModel: 'my pricing model', - }; + const opts = { + limit: 5, + offset: 3, + search: 'my search', + sortBy: 'my sort', + category: 'my category', + username: 'my username', + pricingModel: 'my pricing model', + }; + test('list() works', async () => { const res: any = client && (await client.store().list(opts)); expect(res.id).toEqual('store-list'); validateRequest(opts); @@ -53,7 +61,136 @@ describe('Store', () => { opts, ); expect(browserRes.id).toEqual('store-list'); - expect(browserRes).toEqual(res); + const { [Symbol.asyncIterator]: _, ...expectedResponse } = res; + expect(browserRes).toEqual(expectedResponse); validateRequest(opts); }); }); + +describe('actor.store.list as async iterable', () => { + // Test using store().list() as an async iterable + const client: ApifyClient = new ApifyClient(); + + const createItems = (count: number) => { + return new Array(count).fill('some actor details'); + }; + + const exampleResponseData = { + total: 2500, + count: 0, + offset: 0, + limit: 0, + desc: false, + items: createItems(1000), + }; + + const testCases = [ + { + testName: 'Known total items, no offset, no limit', + options: {}, + responseDataOverrides: [ + { count: 1000, limit: 1000 }, + { count: 1000, limit: 1000, offset: 1000 }, + { count: 500, limit: 1000, offset: 2000, items: createItems(500) }, + ], + expectedItems: 2500, + }, + { + testName: 'Known total items, user offset, no limit', + options: { offset: 1000 }, + responseDataOverrides: [ + { count: 1000, limit: 1000, offset: 1000 }, + { count: 500, limit: 1000, offset: 2000, items: createItems(500) }, + ], + expectedItems: 1500, + }, + { + testName: 'Known total items, no offset, user limit', + options: { limit: 1100 }, + responseDataOverrides: [ + { count: 1000, limit: 1000 }, + { count: 100, limit: 100, offset: 1000, items: createItems(100) }, + ], + expectedItems: 1100, + }, + { + testName: 'Known total items, user offset, user limit', + options: { offset: 1000, limit: 1100 }, + responseDataOverrides: [ + { count: 1000, limit: 1000, offset: 1000 }, + { count: 100, limit: 100, offset: 2000, items: createItems(100) }, + ], + expectedItems: 1100, + }, + { + testName: 'Unknown total items, no offset, no limit', + options: {}, + responseDataOverrides: [ + { total: undefined, count: 1000, limit: 1000 }, + { total: undefined, count: 1000, limit: 1000, offset: 1000 }, + { total: undefined, count: 500, limit: 1000, offset: 2000, items: createItems(500) }, + { total: undefined, count: 0, limit: 1000, offset: 2500, items: [] }, // In this case, iterator had to try as it does not know if there is more or not and there is no user limit. + ], + expectedItems: 2500, + }, + { + testName: 'Unknown total items, user offset, no limit', + options: { offset: 1000 }, + responseDataOverrides: [ + { total: undefined, count: 1000, limit: 1000, offset: 1000 }, + { total: undefined, count: 500, limit: 1000, offset: 2000, items: createItems(500) }, + { total: undefined, count: 0, limit: 1000, offset: 2500, items: [] }, // In this case, iterator had to try as it does not know if there is more or not and there is no user limit. + ], + expectedItems: 1500, + }, + { + testName: 'Unknown total items, no offset, user limit', + options: { limit: 1100 }, + responseDataOverrides: [ + { total: undefined, count: 1000, limit: 1000 }, + { total: undefined, count: 100, limit: 100, offset: 1000, items: createItems(100) }, + ], + expectedItems: 1100, + }, + { + testName: 'Unknown total items, user offset, user limit', + options: { offset: 1000, limit: 1100 }, + responseDataOverrides: [ + { total: undefined, count: 1000, limit: 1000, offset: 1000 }, + { total: undefined, count: 100, limit: 100, offset: 2000, items: createItems(100) }, + ], + expectedItems: 1100, + }, + ]; + + test.each(testCases)('$testName', async ({ options, responseDataOverrides, expectedItems }) => { + // Simulate 2500 actors in store and 8 possible combinations of user options and API responses. + + function* mockedResponsesGenerator() { + for (const responseDataOverride of responseDataOverrides) { + yield { data: { data: { ...exampleResponseData, ...responseDataOverride } } } as ApifyResponse; + } + } + + const mockedResponses = mockedResponsesGenerator(); + + const storeClient = client.store(); + const mockedClient = jest.spyOn(storeClient.httpClient, 'call').mockImplementation(async () => { + const next = mockedResponses.next(); + if (next.done) { + // Return a default or dummy ApifyResponse here + return { data: {} } as ApifyResponse; + } + return next.value; + }); + + const pages = await client.store().list(options); + + const totalItems: any[] = []; + for await (const page of pages) { + totalItems.push(...page.items); + } + mockedClient.mockRestore(); + expect(totalItems.length).toBe(expectedItems); + }); +}); From de2c43f9a25b42e39855805950815df503b95c79 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 20 Nov 2025 16:18:04 +0100 Subject: [PATCH 3/9] Remove unintended changes --- src/base/resource_collection_client.ts | 3 ++- src/utils.ts | 6 +++--- test/store.test.ts | 30 ++++++++++---------------- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/base/resource_collection_client.ts b/src/base/resource_collection_client.ts index c910ef2e..f0368fc8 100644 --- a/src/base/resource_collection_client.ts +++ b/src/base/resource_collection_client.ts @@ -1,4 +1,5 @@ -import type { PaginatedResponse, PaginationOptions , parseDateFields, pluckData } from '../utils'; +import type { PaginatedResponse, PaginationOptions } from '../utils'; +import { parseDateFields, pluckData } from '../utils'; import { ApiClient } from './api_client'; /** diff --git a/src/utils.ts b/src/utils.ts index ca062373..5201031d 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -252,12 +252,12 @@ export interface PaginatedList { total: number; /** Count of dataset entries returned in this set. */ count: number; - /** Should the results be in descending order. */ - desc: boolean; - /** Position of the first returned entry. */ + /** Position of the first returned entry in the dataset. */ offset: number; /** Maximum number of dataset entries requested. */ limit: number; + /** Should the results be in descending order. */ + desc: boolean; /** Dataset entries based on chosen format parameter. */ items: Data[]; } diff --git a/test/store.test.ts b/test/store.test.ts index c5e8efa2..f708cc57 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -1,16 +1,9 @@ -import { setTimeout } from 'node:timers/promises'; - -import c from 'ansi-colors'; import type { StoreCollectionListOptions } from 'apify-client'; -import { ApifyClient, LoggerActorRedirect } from 'apify-client'; -import express from 'express'; - -import { LEVELS, Log } from '@apify/log'; +import { ApifyClient } from 'apify-client'; import type { ApifyResponse } from '../src/http_client'; import { Browser, DEFAULT_OPTIONS, validateRequest } from './_helper'; -import { createDefaultApp, mockServer } from './mock_server/server'; -import { MOCKED_ACTOR_LOGS_PROCESSED, StatusGenerator } from './mock_server/test_utils'; +import { mockServer } from './mock_server/server'; describe('Store', () => { let baseUrl: string | undefined; @@ -41,17 +34,16 @@ describe('Store', () => { page.close().catch(() => {}); }); - const opts = { - limit: 5, - offset: 3, - search: 'my search', - sortBy: 'my sort', - category: 'my category', - username: 'my username', - pricingModel: 'my pricing model', - }; - test('list() works', async () => { + const opts = { + limit: 5, + offset: 3, + search: 'my search', + sortBy: 'my sort', + category: 'my category', + username: 'my username', + pricingModel: 'my pricing model', + }; const res: any = client && (await client.store().list(opts)); expect(res.id).toEqual('store-list'); validateRequest(opts); From d9f28d3a10188a30feb53f8f743bfc682705a209 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 24 Nov 2025 15:43:57 +0100 Subject: [PATCH 4/9] Review comments --- src/base/resource_collection_client.ts | 59 +++++++++++++----------- src/resource_clients/store_collection.ts | 4 +- src/utils.ts | 4 +- test/store.test.ts | 6 +-- 4 files changed, 39 insertions(+), 34 deletions(-) diff --git a/src/base/resource_collection_client.ts b/src/base/resource_collection_client.ts index f0368fc8..d13be1d2 100644 --- a/src/base/resource_collection_client.ts +++ b/src/base/resource_collection_client.ts @@ -20,40 +20,45 @@ export class ResourceCollectionClient extends ApiClient { } /** - * Returns async iterator to paginate through all pages and first page of results is returned immediately as well. - * @private + * Returns async iterator to paginate through all items and first page of results is returned immediately as well. */ - protected async _getIterablePagination( + protected _getIterablePagination>( options: T = {} as T, - ): Promise> { + ): AsyncIterable & Promise { const getPaginatedList = this._list.bind(this); - let currentPage = await getPaginatedList(options); + const paginatedListPromise = getPaginatedList(options); - return { - ...currentPage, - async *[Symbol.asyncIterator]() { - yield currentPage; - let itemsFetched = currentPage.items.length; - let currentLimit = options.limit !== undefined ? options.limit - itemsFetched : undefined; - let currentOffset = options.offset ?? 0 + itemsFetched; - const maxRelevantItems = - currentPage.total === undefined ? undefined : currentPage.total - (options.offset || 0); + async function* asyncGenerator() { + let currentPage = await paginatedListPromise; + let itemsFetched = currentPage.items.length; + let currentLimit = options.limit !== undefined ? options.limit - itemsFetched : undefined; + let currentOffset = options.offset ?? 0 + itemsFetched; + const maxRelevantItems = + currentPage.total === undefined ? undefined : currentPage.total - (options.offset || 0); + for (const item of currentPage.items) { + yield item; + } - while ( - currentPage.items.length > 0 && // Some items were returned in last page - (currentLimit === undefined || currentLimit > 0) && // User defined a limit, and we have not yet exhausted it - (maxRelevantItems === undefined || maxRelevantItems > itemsFetched) // We know total and we did not get it yet - ) { - const newOptions = { ...options, limit: currentLimit, offset: currentOffset }; - currentPage = await getPaginatedList(newOptions); - yield currentPage; - itemsFetched += currentPage.items.length; - currentLimit = options.limit !== undefined ? options.limit - itemsFetched : undefined; - currentOffset = options.offset ?? 0 + itemsFetched; + while ( + currentPage.items.length > 0 && // Some items were returned in last page + (currentLimit === undefined || currentLimit > 0) && // User defined a limit, and we have not yet exhausted it + (maxRelevantItems === undefined || maxRelevantItems > itemsFetched) // We know total and we did not get it yet + ) { + const newOptions = { ...options, limit: currentLimit, offset: currentOffset }; + currentPage = await getPaginatedList(newOptions); + for (const item of currentPage.items) { + yield item; } - }, - }; + itemsFetched += currentPage.items.length; + currentLimit = options.limit !== undefined ? options.limit - itemsFetched : undefined; + currentOffset = options.offset ?? 0 + itemsFetched; + } + } + + return Object.defineProperty(paginatedListPromise, Symbol.asyncIterator, { + value: asyncGenerator, + }) as unknown as AsyncIterable & Promise; } protected async _create(resource: D): Promise { diff --git a/src/resource_clients/store_collection.ts b/src/resource_clients/store_collection.ts index a592398f..46e7f61d 100644 --- a/src/resource_clients/store_collection.ts +++ b/src/resource_clients/store_collection.ts @@ -19,7 +19,9 @@ export class StoreCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2/#/reference/store/store-actors-collection/get-list-of-actors-in-store */ - async list(options: StoreCollectionListOptions = {}): Promise> { + list( + options: StoreCollectionListOptions = {}, + ): Promise> & AsyncIterable { ow( options, ow.object.exactShape({ diff --git a/src/utils.ts b/src/utils.ts index 5201031d..e081ce5b 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -240,11 +240,11 @@ export interface PaginationOptions { limit?: number; } -export interface PaginatedResponse { +export interface PaginatedResponse { /** Total count of entries. */ total?: number; /** Entries. */ - items: unknown[]; + items: Data[]; } export interface PaginatedList { diff --git a/test/store.test.ts b/test/store.test.ts index f708cc57..c8c55a3a 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -176,11 +176,9 @@ describe('actor.store.list as async iterable', () => { return next.value; }); - const pages = await client.store().list(options); - const totalItems: any[] = []; - for await (const page of pages) { - totalItems.push(...page.items); + for await (const page of client.store().list(options)) { + totalItems.push(page); } mockedClient.mockRestore(); expect(totalItems.length).toBe(expectedItems); From 5e4cb9993185386edc6d9eac8bb7b36f77ee4017 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 24 Nov 2025 16:03:43 +0100 Subject: [PATCH 5/9] Update type hint --- src/resource_clients/store_collection.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resource_clients/store_collection.ts b/src/resource_clients/store_collection.ts index 46e7f61d..6e9dca2d 100644 --- a/src/resource_clients/store_collection.ts +++ b/src/resource_clients/store_collection.ts @@ -2,7 +2,7 @@ import ow from 'ow'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceCollectionClient } from '../base/resource_collection_client'; -import type { IterablePaginatedList } from '../utils'; +import type { PaginatedList } from '../utils'; import type { ActorStats } from './actor'; export class StoreCollectionClient extends ResourceCollectionClient { @@ -21,7 +21,7 @@ export class StoreCollectionClient extends ResourceCollectionClient { */ list( options: StoreCollectionListOptions = {}, - ): Promise> & AsyncIterable { + ): Promise> & AsyncIterable { ow( options, ow.object.exactShape({ From 9e108d91d1afc59dfe6f446d8aab54dfa6ca8f3f Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 25 Nov 2025 10:28:39 +0100 Subject: [PATCH 6/9] Simplify fetching more pages --- src/base/resource_collection_client.ts | 31 ++++++---------- src/utils.ts | 2 +- test/store.test.ts | 51 +++++++------------------- 3 files changed, 27 insertions(+), 57 deletions(-) diff --git a/src/base/resource_collection_client.ts b/src/base/resource_collection_client.ts index d13be1d2..fac2efba 100644 --- a/src/base/resource_collection_client.ts +++ b/src/base/resource_collection_client.ts @@ -26,33 +26,26 @@ export class ResourceCollectionClient extends ApiClient { options: T = {} as T, ): AsyncIterable & Promise { const getPaginatedList = this._list.bind(this); - const paginatedListPromise = getPaginatedList(options); async function* asyncGenerator() { let currentPage = await paginatedListPromise; - let itemsFetched = currentPage.items.length; - let currentLimit = options.limit !== undefined ? options.limit - itemsFetched : undefined; - let currentOffset = options.offset ?? 0 + itemsFetched; - const maxRelevantItems = - currentPage.total === undefined ? undefined : currentPage.total - (options.offset || 0); - for (const item of currentPage.items) { - yield item; - } + yield* currentPage.items; + const offset = options.offset || 0; + const limit = Math.min(options.limit || currentPage.total, currentPage.total); + + let currentOffset = offset + currentPage.items.length; + let remainingItems = Math.min(currentPage.total - offset, limit) - currentPage.items.length; while ( - currentPage.items.length > 0 && // Some items were returned in last page - (currentLimit === undefined || currentLimit > 0) && // User defined a limit, and we have not yet exhausted it - (maxRelevantItems === undefined || maxRelevantItems > itemsFetched) // We know total and we did not get it yet + currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page. + remainingItems > 0 ) { - const newOptions = { ...options, limit: currentLimit, offset: currentOffset }; + const newOptions = { ...options, limit: remainingItems, offset: currentOffset }; currentPage = await getPaginatedList(newOptions); - for (const item of currentPage.items) { - yield item; - } - itemsFetched += currentPage.items.length; - currentLimit = options.limit !== undefined ? options.limit - itemsFetched : undefined; - currentOffset = options.offset ?? 0 + itemsFetched; + yield* currentPage.items; + currentOffset += currentPage.items.length; + remainingItems -= currentPage.items.length; } } diff --git a/src/utils.ts b/src/utils.ts index e081ce5b..14d21735 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -242,7 +242,7 @@ export interface PaginationOptions { export interface PaginatedResponse { /** Total count of entries. */ - total?: number; + total: number; /** Entries. */ items: Data[]; } diff --git a/test/store.test.ts b/test/store.test.ts index c8c55a3a..6a69f794 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -78,7 +78,7 @@ describe('actor.store.list as async iterable', () => { const testCases = [ { - testName: 'Known total items, no offset, no limit', + testName: 'No offset, no limit', options: {}, responseDataOverrides: [ { count: 1000, limit: 1000 }, @@ -88,7 +88,7 @@ describe('actor.store.list as async iterable', () => { expectedItems: 2500, }, { - testName: 'Known total items, user offset, no limit', + testName: 'User offset, no limit', options: { offset: 1000 }, responseDataOverrides: [ { count: 1000, limit: 1000, offset: 1000 }, @@ -97,7 +97,7 @@ describe('actor.store.list as async iterable', () => { expectedItems: 1500, }, { - testName: 'Known total items, no offset, user limit', + testName: 'No offset, user limit', options: { limit: 1100 }, responseDataOverrides: [ { count: 1000, limit: 1000 }, @@ -106,7 +106,7 @@ describe('actor.store.list as async iterable', () => { expectedItems: 1100, }, { - testName: 'Known total items, user offset, user limit', + testName: 'User offset, user limit', options: { offset: 1000, limit: 1100 }, responseDataOverrides: [ { count: 1000, limit: 1000, offset: 1000 }, @@ -115,43 +115,20 @@ describe('actor.store.list as async iterable', () => { expectedItems: 1100, }, { - testName: 'Unknown total items, no offset, no limit', - options: {}, - responseDataOverrides: [ - { total: undefined, count: 1000, limit: 1000 }, - { total: undefined, count: 1000, limit: 1000, offset: 1000 }, - { total: undefined, count: 500, limit: 1000, offset: 2000, items: createItems(500) }, - { total: undefined, count: 0, limit: 1000, offset: 2500, items: [] }, // In this case, iterator had to try as it does not know if there is more or not and there is no user limit. - ], - expectedItems: 2500, - }, - { - testName: 'Unknown total items, user offset, no limit', - options: { offset: 1000 }, - responseDataOverrides: [ - { total: undefined, count: 1000, limit: 1000, offset: 1000 }, - { total: undefined, count: 500, limit: 1000, offset: 2000, items: createItems(500) }, - { total: undefined, count: 0, limit: 1000, offset: 2500, items: [] }, // In this case, iterator had to try as it does not know if there is more or not and there is no user limit. - ], - expectedItems: 1500, - }, - { - testName: 'Unknown total items, no offset, user limit', - options: { limit: 1100 }, - responseDataOverrides: [ - { total: undefined, count: 1000, limit: 1000 }, - { total: undefined, count: 100, limit: 100, offset: 1000, items: createItems(100) }, - ], - expectedItems: 1100, + testName: 'User out of range offset, no limit', + options: { offset: 3000 }, + responseDataOverrides: [{ count: 0, limit: 0, offset: 3000, items: [] }], + expectedItems: 0, }, { - testName: 'Unknown total items, user offset, user limit', - options: { offset: 1000, limit: 1100 }, + testName: 'User no offset, out of range limit', + options: { limit: 3000 }, responseDataOverrides: [ - { total: undefined, count: 1000, limit: 1000, offset: 1000 }, - { total: undefined, count: 100, limit: 100, offset: 2000, items: createItems(100) }, + { count: 1000, limit: 1000 }, + { count: 1000, limit: 1000, offset: 1000 }, + { count: 500, limit: 1000, offset: 2000, items: createItems(500) }, ], - expectedItems: 1100, + expectedItems: 2500, }, ]; From bd4a534702acfce52a2686702bc1e9530fc37c7d Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 25 Nov 2025 11:28:22 +0100 Subject: [PATCH 7/9] Improve tests --- test/store.test.ts | 104 +++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 64 deletions(-) diff --git a/test/store.test.ts b/test/store.test.ts index 6a69f794..a77d007d 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -1,7 +1,8 @@ import type { StoreCollectionListOptions } from 'apify-client'; import { ApifyClient } from 'apify-client'; -import type { ApifyResponse } from '../src/http_client'; +import type { ApifyRequestConfig, ApifyResponse } from '../src/http_client'; +import { PaginationOptions } from '../src/utils'; import { Browser, DEFAULT_OPTIONS, validateRequest } from './_helper'; import { mockServer } from './mock_server/server'; @@ -63,98 +64,73 @@ describe('actor.store.list as async iterable', () => { // Test using store().list() as an async iterable const client: ApifyClient = new ApifyClient(); - const createItems = (count: number) => { - return new Array(count).fill('some actor details'); - }; - - const exampleResponseData = { - total: 2500, - count: 0, - offset: 0, - limit: 0, - desc: false, - items: createItems(1000), - }; - const testCases = [ - { - testName: 'No offset, no limit', - options: {}, - responseDataOverrides: [ - { count: 1000, limit: 1000 }, - { count: 1000, limit: 1000, offset: 1000 }, - { count: 500, limit: 1000, offset: 2000, items: createItems(500) }, - ], - expectedItems: 2500, - }, { testName: 'User offset, no limit', - options: { offset: 1000 }, - responseDataOverrides: [ - { count: 1000, limit: 1000, offset: 1000 }, - { count: 500, limit: 1000, offset: 2000, items: createItems(500) }, - ], + userDefinedOptions: { offset: 1000 }, expectedItems: 1500, }, + { + testName: 'No offset, no limit', + userDefinedOptions: {}, + expectedItems: 2500, + }, { testName: 'No offset, user limit', - options: { limit: 1100 }, - responseDataOverrides: [ - { count: 1000, limit: 1000 }, - { count: 100, limit: 100, offset: 1000, items: createItems(100) }, - ], + userDefinedOptions: { limit: 1100 }, expectedItems: 1100, }, { testName: 'User offset, user limit', - options: { offset: 1000, limit: 1100 }, - responseDataOverrides: [ - { count: 1000, limit: 1000, offset: 1000 }, - { count: 100, limit: 100, offset: 2000, items: createItems(100) }, - ], + userDefinedOptions: { offset: 1000, limit: 1100 }, expectedItems: 1100, }, { testName: 'User out of range offset, no limit', - options: { offset: 3000 }, - responseDataOverrides: [{ count: 0, limit: 0, offset: 3000, items: [] }], + userDefinedOptions: { offset: 3000 }, expectedItems: 0, }, { testName: 'User no offset, out of range limit', - options: { limit: 3000 }, - responseDataOverrides: [ - { count: 1000, limit: 1000 }, - { count: 1000, limit: 1000, offset: 1000 }, - { count: 500, limit: 1000, offset: 2000, items: createItems(500) }, - ], + userDefinedOptions: { limit: 3000 }, expectedItems: 2500, }, ]; - test.each(testCases)('$testName', async ({ options, responseDataOverrides, expectedItems }) => { - // Simulate 2500 actors in store and 8 possible combinations of user options and API responses. - - function* mockedResponsesGenerator() { - for (const responseDataOverride of responseDataOverrides) { - yield { data: { data: { ...exampleResponseData, ...responseDataOverride } } } as ApifyResponse; + test.each(testCases)('$testName', async ({ userDefinedOptions, expectedItems }) => { + const mockedPlatformLogic = async (request: ApifyRequestConfig) => { + // Simulated platform logic for pagination when there are 2500 actors in store. + const maxItems = 2500; + const maxItemsPerPage = 1000; + const offset = request.params.offset ? request.params.offset : 0; + const limit = request.params.limit ? request.params.limit : 0; + if (offset < 0 || limit < 0) { + throw new Error('Offset and limit must be non-negative'); } - } - const mockedResponses = mockedResponsesGenerator(); + const lowerIndex = Math.min(offset, maxItems); + const upperIndex = Math.min(offset + (limit || maxItems), maxItems); + const returnedItemsCount = Math.min(upperIndex - lowerIndex, maxItemsPerPage); + + return { + data: { + data: { + total: maxItems, + count: returnedItemsCount, + offset, + limit: returnedItemsCount, + desc: false, + items: new Array(returnedItemsCount).fill('some actor details'), + }, + }, + } as ApifyResponse; + }; const storeClient = client.store(); - const mockedClient = jest.spyOn(storeClient.httpClient, 'call').mockImplementation(async () => { - const next = mockedResponses.next(); - if (next.done) { - // Return a default or dummy ApifyResponse here - return { data: {} } as ApifyResponse; - } - return next.value; - }); + const mockedClient = jest.spyOn(storeClient.httpClient, 'call').mockImplementation(mockedPlatformLogic); const totalItems: any[] = []; - for await (const page of client.store().list(options)) { + for await (const page of client.store().list(userDefinedOptions)) { totalItems.push(page); } mockedClient.mockRestore(); From 32c13ce8fbc94371b97d4d99bd5a035d911b5b25 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 26 Nov 2025 13:31:51 +0100 Subject: [PATCH 8/9] Improve test to include desc and unnamed. Apply to other endpoints. --- src/resource_clients/actor_collection.ts | 6 +- .../actor_env_var_collection.ts | 6 +- .../actor_version_collection.ts | 6 +- src/resource_clients/build_collection.ts | 6 +- src/resource_clients/dataset_collection.ts | 6 +- .../key_value_store_collection.ts | 6 +- .../request_queue_collection.ts | 6 +- src/resource_clients/run_collection.ts | 6 +- src/resource_clients/schedule_collection.ts | 4 +- src/resource_clients/task_collection.ts | 4 +- src/resource_clients/webhook_collection.ts | 7 +- .../webhook_dispatch_collection.ts | 6 +- test/datasets.test.js | 114 ++++++++++++++++++ test/store.test.ts | 80 ------------ 14 files changed, 157 insertions(+), 106 deletions(-) diff --git a/src/resource_clients/actor_collection.ts b/src/resource_clients/actor_collection.ts index 43ad2656..ecd81490 100644 --- a/src/resource_clients/actor_collection.ts +++ b/src/resource_clients/actor_collection.ts @@ -20,7 +20,9 @@ export class ActorCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/actors/actor-collection/get-list-of-actors */ - async list(options: ActorCollectionListOptions = {}): Promise { + list( + options: ActorCollectionListOptions = {}, + ): Promise> & AsyncIterable { ow( options, ow.object.exactShape({ @@ -32,7 +34,7 @@ export class ActorCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getIterablePagination(options); } /** diff --git a/src/resource_clients/actor_env_var_collection.ts b/src/resource_clients/actor_env_var_collection.ts index 0a959230..cd8d45c2 100644 --- a/src/resource_clients/actor_env_var_collection.ts +++ b/src/resource_clients/actor_env_var_collection.ts @@ -19,7 +19,9 @@ export class ActorEnvVarCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/actors/environment-variable-collection/get-list-of-environment-variables */ - async list(options: ActorEnvVarCollectionListOptions = {}): Promise { + list( + options: ActorEnvVarCollectionListOptions = {}, + ): Promise & AsyncIterable { ow( options, ow.object.exactShape({ @@ -28,7 +30,7 @@ export class ActorEnvVarCollectionClient extends ResourceCollectionClient { desc: ow.optional.boolean, }), ); - return this._list(options); + return this._getIterablePagination(options); } /** diff --git a/src/resource_clients/actor_version_collection.ts b/src/resource_clients/actor_version_collection.ts index 2cfd6656..550e3993 100644 --- a/src/resource_clients/actor_version_collection.ts +++ b/src/resource_clients/actor_version_collection.ts @@ -19,7 +19,9 @@ export class ActorVersionCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/actors/version-collection/get-list-of-versions */ - async list(options: ActorVersionCollectionListOptions = {}): Promise { + list( + options: ActorVersionCollectionListOptions = {}, + ): Promise & AsyncIterable { ow( options, ow.object.exactShape({ @@ -29,7 +31,7 @@ export class ActorVersionCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getIterablePagination(options); } /** diff --git a/src/resource_clients/build_collection.ts b/src/resource_clients/build_collection.ts index d4866914..9cc4e8fc 100644 --- a/src/resource_clients/build_collection.ts +++ b/src/resource_clients/build_collection.ts @@ -19,7 +19,9 @@ export class BuildCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/actors/build-collection/get-list-of-builds */ - async list(options: BuildCollectionClientListOptions = {}): Promise { + list( + options: BuildCollectionClientListOptions = {}, + ): Promise> & AsyncIterable { ow( options, ow.object.exactShape({ @@ -29,7 +31,7 @@ export class BuildCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getIterablePagination(options); } } diff --git a/src/resource_clients/dataset_collection.ts b/src/resource_clients/dataset_collection.ts index e9b972c8..29b575b6 100644 --- a/src/resource_clients/dataset_collection.ts +++ b/src/resource_clients/dataset_collection.ts @@ -19,7 +19,9 @@ export class DatasetCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/datasets/dataset-collection/get-list-of-datasets */ - async list(options: DatasetCollectionClientListOptions = {}): Promise { + list( + options: DatasetCollectionClientListOptions = {}, + ): Promise> & AsyncIterable { ow( options, ow.object.exactShape({ @@ -30,7 +32,7 @@ export class DatasetCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getIterablePagination(options); } /** diff --git a/src/resource_clients/key_value_store_collection.ts b/src/resource_clients/key_value_store_collection.ts index 54faeaed..cfb88feb 100644 --- a/src/resource_clients/key_value_store_collection.ts +++ b/src/resource_clients/key_value_store_collection.ts @@ -19,9 +19,9 @@ export class KeyValueStoreCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/key-value-stores/store-collection/get-list-of-key-value-stores */ - async list( + list( options: KeyValueStoreCollectionClientListOptions = {}, - ): Promise> { + ): Promise> & AsyncIterable { ow( options, ow.object.exactShape({ @@ -32,7 +32,7 @@ export class KeyValueStoreCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getIterablePagination(options); } /** diff --git a/src/resource_clients/request_queue_collection.ts b/src/resource_clients/request_queue_collection.ts index 54f57628..a11150b3 100644 --- a/src/resource_clients/request_queue_collection.ts +++ b/src/resource_clients/request_queue_collection.ts @@ -19,7 +19,9 @@ export class RequestQueueCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/request-queues/queue-collection/get-list-of-request-queues */ - async list(options: RequestQueueCollectionListOptions = {}): Promise { + list( + options: RequestQueueCollectionListOptions = {}, + ): Promise & AsyncIterable { ow( options, ow.object.exactShape({ @@ -30,7 +32,7 @@ export class RequestQueueCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getIterablePagination(options); } /** diff --git a/src/resource_clients/run_collection.ts b/src/resource_clients/run_collection.ts index b5192534..e7d040da 100644 --- a/src/resource_clients/run_collection.ts +++ b/src/resource_clients/run_collection.ts @@ -21,7 +21,9 @@ export class RunCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/actors/run-collection/get-list-of-runs */ - async list(options: RunCollectionListOptions = {}): Promise> { + list( + options: RunCollectionListOptions = {}, + ): Promise> & AsyncIterable { ow( options, ow.object.exactShape({ @@ -37,7 +39,7 @@ export class RunCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getIterablePagination(options); } } diff --git a/src/resource_clients/schedule_collection.ts b/src/resource_clients/schedule_collection.ts index 3c9bb712..79ca4253 100644 --- a/src/resource_clients/schedule_collection.ts +++ b/src/resource_clients/schedule_collection.ts @@ -19,7 +19,7 @@ export class ScheduleCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/schedules/schedules-collection/get-list-of-schedules */ - async list(options: ScheduleCollectionListOptions = {}): Promise> { + list(options: ScheduleCollectionListOptions = {}): Promise> & AsyncIterable { ow( options, ow.object.exactShape({ @@ -29,7 +29,7 @@ export class ScheduleCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getIterablePagination(options); } /** diff --git a/src/resource_clients/task_collection.ts b/src/resource_clients/task_collection.ts index 48a71461..46987044 100644 --- a/src/resource_clients/task_collection.ts +++ b/src/resource_clients/task_collection.ts @@ -24,7 +24,7 @@ export class TaskCollectionClient extends ResourceCollectionClient { * @param {boolean} [options.desc] * @return {Promise} */ - async list(options: TaskCollectionListOptions = {}): Promise> { + list(options: TaskCollectionListOptions = {}): Promise> & AsyncIterable { ow( options, ow.object.exactShape({ @@ -34,7 +34,7 @@ export class TaskCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getIterablePagination(options); } /** diff --git a/src/resource_clients/webhook_collection.ts b/src/resource_clients/webhook_collection.ts index 7865134b..19a34253 100644 --- a/src/resource_clients/webhook_collection.ts +++ b/src/resource_clients/webhook_collection.ts @@ -19,9 +19,10 @@ export class WebhookCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/webhooks/webhook-collection/get-list-of-webhooks */ - async list( + list( options: WebhookCollectionListOptions = {}, - ): Promise>> { + ): Promise>> & + AsyncIterable> { ow( options, ow.object.exactShape({ @@ -31,7 +32,7 @@ export class WebhookCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getIterablePagination(options); } /** diff --git a/src/resource_clients/webhook_dispatch_collection.ts b/src/resource_clients/webhook_dispatch_collection.ts index bed32ebb..a7b60d41 100644 --- a/src/resource_clients/webhook_dispatch_collection.ts +++ b/src/resource_clients/webhook_dispatch_collection.ts @@ -19,7 +19,9 @@ export class WebhookDispatchCollectionClient extends ResourceCollectionClient { /** * https://docs.apify.com/api/v2#/reference/webhook-dispatches/webhook-dispatches-collection/get-list-of-webhook-dispatches */ - async list(options: WebhookDispatchCollectionListOptions = {}): Promise> { + list( + options: WebhookDispatchCollectionListOptions = {}, + ): Promise> & AsyncIterable { ow( options, ow.object.exactShape({ @@ -29,7 +31,7 @@ export class WebhookDispatchCollectionClient extends ResourceCollectionClient { }), ); - return this._list(options); + return this._getIterablePagination(options); } } diff --git a/test/datasets.test.js b/test/datasets.test.js index d0441b7d..3a44f54a 100644 --- a/test/datasets.test.js +++ b/test/datasets.test.js @@ -438,3 +438,117 @@ describe('Dataset methods', () => { }); }); }); + +describe('actor.store.list as async iterable', () => { + // Test using store().list() as an async iterable + const client = new ApifyClient(); + + const range = (start, end, step = 1) => { + // Inclusive range, ordered based on start and end values + return Array.from( + { + length: Math.abs(end - start) + 1, + }, + (_, i) => start + Math.sign(end - start) * step * i, + ); + }; + + const testCases = [ + { + testName: 'No offset, no limit', + userDefinedOptions: {}, + expectedItems: range(0, 2499), + }, + { + testName: 'No offset, user limit', + userDefinedOptions: { limit: 1100 }, + expectedItems: range(0, 1099), + }, + { + testName: 'User offset, no limit', + userDefinedOptions: { offset: 1000 }, + expectedItems: range(1000, 2499), + }, + { + testName: 'User offset, user limit', + userDefinedOptions: { offset: 1000, limit: 1100 }, + expectedItems: range(1000, 2099), + }, + { + testName: 'User offset, user limit, descending', + userDefinedOptions: { offset: 1000, limit: 1100, desc: true }, + expectedItems: range(1500, 401), + }, + { + testName: 'User offset, user limit, descending, unnamed included', + userDefinedOptions: { offset: 50, limit: 1100, desc: true, unnamed: true }, + expectedItems: range(2550, 1451), + }, + { + testName: 'User out of range offset, no limit', + userDefinedOptions: { offset: 3000 }, + expectedItems: [], + }, + { + testName: 'User no offset, out of range limit', + userDefinedOptions: { limit: 3000 }, + expectedItems: range(0, 2499), + }, + { + testName: 'User no offset, out of range limit', + userDefinedOptions: { limit: 3000 }, + expectedItems: range(0, 2499), + }, + ]; + + test.each(testCases)('$testName', async ({ userDefinedOptions, expectedItems }) => { + const mockedPlatformLogic = async (request) => { + // Simulated platform logic for pagination. + // There are 2500 named items in the collection and additional 100 unnamed items. + // Items are simple number for easy verification 0..2499 -> named, 2500..2599 -> unnamed + + const namedItems = 2500; // represent named items + const unnamedItems = 100; // additional unnamed items + + const totalItems = request.params.unnamed ? namedItems + unnamedItems : namedItems; + + const maxItemsPerPage = 1000; + const offset = request.params.offset ? request.params.offset : 0; + const limit = request.params.limit ? request.params.limit : 0; + const desc = request.params.desc ? request.params.desc : false; + + const items = range(desc ? totalItems : 0, desc ? 0 : totalItems); + + if (offset < 0 || limit < 0) { + throw new Error('Offset and limit must be non-negative'); + } + + const lowerIndex = Math.min(offset, totalItems); + const upperIndex = Math.min(offset + (limit || totalItems), totalItems); + const returnedItemsCount = Math.min(upperIndex - lowerIndex, maxItemsPerPage); + + return { + data: { + data: { + total: namedItems, + count: returnedItemsCount, + offset, + limit: returnedItemsCount, + desc: false, + items: items.slice(lowerIndex, upperIndex), + }, + }, + }; + }; + + const datasetsClient = client.datasets(); + const mockedClient = jest.spyOn(datasetsClient.httpClient, 'call').mockImplementation(mockedPlatformLogic); + + const totalItems = []; + for await (const page of datasetsClient.list(userDefinedOptions)) { + totalItems.push(page); + } + mockedClient.mockRestore(); + expect(totalItems).toEqual(expectedItems); + }); +}); diff --git a/test/store.test.ts b/test/store.test.ts index a77d007d..78a764b0 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -1,8 +1,6 @@ import type { StoreCollectionListOptions } from 'apify-client'; import { ApifyClient } from 'apify-client'; -import type { ApifyRequestConfig, ApifyResponse } from '../src/http_client'; -import { PaginationOptions } from '../src/utils'; import { Browser, DEFAULT_OPTIONS, validateRequest } from './_helper'; import { mockServer } from './mock_server/server'; @@ -59,81 +57,3 @@ describe('Store', () => { validateRequest(opts); }); }); - -describe('actor.store.list as async iterable', () => { - // Test using store().list() as an async iterable - const client: ApifyClient = new ApifyClient(); - - const testCases = [ - { - testName: 'User offset, no limit', - userDefinedOptions: { offset: 1000 }, - expectedItems: 1500, - }, - { - testName: 'No offset, no limit', - userDefinedOptions: {}, - expectedItems: 2500, - }, - { - testName: 'No offset, user limit', - userDefinedOptions: { limit: 1100 }, - expectedItems: 1100, - }, - { - testName: 'User offset, user limit', - userDefinedOptions: { offset: 1000, limit: 1100 }, - expectedItems: 1100, - }, - { - testName: 'User out of range offset, no limit', - userDefinedOptions: { offset: 3000 }, - expectedItems: 0, - }, - { - testName: 'User no offset, out of range limit', - userDefinedOptions: { limit: 3000 }, - expectedItems: 2500, - }, - ]; - - test.each(testCases)('$testName', async ({ userDefinedOptions, expectedItems }) => { - const mockedPlatformLogic = async (request: ApifyRequestConfig) => { - // Simulated platform logic for pagination when there are 2500 actors in store. - const maxItems = 2500; - const maxItemsPerPage = 1000; - const offset = request.params.offset ? request.params.offset : 0; - const limit = request.params.limit ? request.params.limit : 0; - if (offset < 0 || limit < 0) { - throw new Error('Offset and limit must be non-negative'); - } - - const lowerIndex = Math.min(offset, maxItems); - const upperIndex = Math.min(offset + (limit || maxItems), maxItems); - const returnedItemsCount = Math.min(upperIndex - lowerIndex, maxItemsPerPage); - - return { - data: { - data: { - total: maxItems, - count: returnedItemsCount, - offset, - limit: returnedItemsCount, - desc: false, - items: new Array(returnedItemsCount).fill('some actor details'), - }, - }, - } as ApifyResponse; - }; - - const storeClient = client.store(); - const mockedClient = jest.spyOn(storeClient.httpClient, 'call').mockImplementation(mockedPlatformLogic); - - const totalItems: any[] = []; - for await (const page of client.store().list(userDefinedOptions)) { - totalItems.push(page); - } - mockedClient.mockRestore(); - expect(totalItems.length).toBe(expectedItems); - }); -}); From 11ba52aba4a4708afb7cb9424be39c7c5c3e29d5 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 26 Nov 2025 14:12:12 +0100 Subject: [PATCH 9/9] Remove left over change from previous implementation --- test/store.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/store.test.ts b/test/store.test.ts index 78a764b0..daa07da9 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -43,6 +43,7 @@ describe('Store', () => { username: 'my username', pricingModel: 'my pricing model', }; + const res: any = client && (await client.store().list(opts)); expect(res.id).toEqual('store-list'); validateRequest(opts); @@ -52,8 +53,7 @@ describe('Store', () => { opts, ); expect(browserRes.id).toEqual('store-list'); - const { [Symbol.asyncIterator]: _, ...expectedResponse } = res; - expect(browserRes).toEqual(expectedResponse); + expect(browserRes).toEqual(res); validateRequest(opts); }); });