feat: Add asyncIterator to the StoreCollectionClient.list return value

src/base/resource_collection_client.ts

@@ -1,3 +1,4 @@
+import type { PaginatedResponse, PaginationOptions } from '../utils';
 import { parseDateFields, pluckData } from '../utils';
 import { ApiClient } from './api_client';
 
@@ -18,6 +19,41 @@ export class ResourceCollectionClient extends ApiClient {
         return parseDateFields(pluckData(response.data)) as R;
     }
 
+    /**
+     * Returns async iterator to paginate through all items and first page of results is returned immediately as well.
+     */
+    protected _getIterablePagination<T extends PaginationOptions, Data, R extends PaginatedResponse<Data>>(
+        options: T = {} as T,
+    ): AsyncIterable<Data> & Promise<R> {
+        const getPaginatedList = this._list.bind(this);
+        const paginatedListPromise = getPaginatedList<T, R>(options);
+
+        async function* asyncGenerator() {
+            let currentPage = await paginatedListPromise;
+            yield* currentPage.items;
+            const offset = options.offset || 0;
+            const limit = Math.min(options.limit || currentPage.total, currentPage.total);
+
+            let currentOffset = offset + currentPage.items.length;
+            let remainingItems = Math.min(currentPage.total - offset, limit) - currentPage.items.length;
+
+            while (
+                currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page.
+                remainingItems > 0
+            ) {
+                const newOptions = { ...options, limit: remainingItems, offset: currentOffset };
+                currentPage = await getPaginatedList<T, R>(newOptions);
+                yield* currentPage.items;
+                currentOffset += currentPage.items.length;
+                remainingItems -= currentPage.items.length;
+            }
+        }
+
+        return Object.defineProperty(paginatedListPromise, Symbol.asyncIterator, {
+            value: asyncGenerator,
+        }) as unknown as AsyncIterable<Data> & Promise<R>;
+    }
+
     protected async _create<D, R>(resource: D): Promise<R> {
         const response = await this.httpClient.call({
             url: this._url(),

src/resource_clients/store_collection.ts

@@ -19,7 +19,9 @@ export class StoreCollectionClient extends ResourceCollectionClient {
     /**
      * https://docs.apify.com/api/v2/#/reference/store/store-actors-collection/get-list-of-actors-in-store
      */
-    async list(options: StoreCollectionListOptions = {}): Promise<PaginatedList<ActorStoreList>> {
+    list(
+        options: StoreCollectionListOptions = {},
+    ): Promise<PaginatedList<ActorStoreList>> & AsyncIterable<ActorStoreList> {
         ow(
             options,
             ow.object.exactShape({
@@ -33,7 +35,7 @@ export class StoreCollectionClient extends ResourceCollectionClient {
             }),
         );
 
-        return this._list(options);
+        return this._getIterablePagination(options);
     }
 }
 

src/utils.ts

@@ -233,6 +233,20 @@ export interface PaginationIteratorOptions {
     exclusiveStartId?: string;
 }
 
+export interface PaginationOptions {
+    /** Position of the first returned entry. */
+    offset?: number;
+    /** Maximum number of entries requested. */
+    limit?: number;
+}
+
+export interface PaginatedResponse<Data> {
+    /** Total count of entries. */
+    total: number;
+    /** Entries. */
+    items: Data[];
+}
+
 export interface PaginatedList<Data> {
     /** Total count of entries in the dataset. */
     total: number;
@@ -248,6 +262,8 @@ export interface PaginatedList<Data> {
     items: Data[];
 }
 
+export interface IterablePaginatedList<Data> extends PaginatedList<Data>, AsyncIterable<PaginatedList<Data>> {}
+
 export function cast<T>(input: unknown): T {
     return input as T;
 }

test/store.test.ts

@@ -1,6 +1,8 @@
 import type { StoreCollectionListOptions } from 'apify-client';
 import { ApifyClient } from 'apify-client';
 
+import type { ApifyRequestConfig, ApifyResponse } from '../src/http_client';
+import { PaginationOptions } from '../src/utils';
 import { Browser, DEFAULT_OPTIONS, validateRequest } from './_helper';
 import { mockServer } from './mock_server/server';
 
@@ -43,7 +45,6 @@ describe('Store', () => {
             username: 'my username',
             pricingModel: 'my pricing model',
         };
-
         const res: any = client && (await client.store().list(opts));
         expect(res.id).toEqual('store-list');
         validateRequest(opts);
@@ -53,7 +54,86 @@ describe('Store', () => {
             opts,
         );
         expect(browserRes.id).toEqual('store-list');
-        expect(browserRes).toEqual(res);
+        const { [Symbol.asyncIterator]: _, ...expectedResponse } = res;
+        expect(browserRes).toEqual(expectedResponse);
         validateRequest(opts);
     });
 });
+
+describe('actor.store.list as async iterable', () => {
+    // Test using store().list() as an async iterable
+    const client: ApifyClient = new ApifyClient();
+
+    const testCases = [
+        {
+            testName: 'User offset, no limit',
+            userDefinedOptions: { offset: 1000 },
+            expectedItems: 1500,
+        },
+        {
+            testName: 'No offset, no limit',
+            userDefinedOptions: {},
+            expectedItems: 2500,
+        },
+        {
+            testName: 'No offset, user limit',
+            userDefinedOptions: { limit: 1100 },
+            expectedItems: 1100,
+        },
+        {
+            testName: 'User offset, user limit',
+            userDefinedOptions: { offset: 1000, limit: 1100 },
+            expectedItems: 1100,
+        },
+        {
+            testName: 'User out of range offset, no limit',
+            userDefinedOptions: { offset: 3000 },
+            expectedItems: 0,
+        },
+        {
+            testName: 'User no offset, out of range limit',
+            userDefinedOptions: { limit: 3000 },
+            expectedItems: 2500,
+        },
+    ];
+
+    test.each(testCases)('$testName', async ({ userDefinedOptions, expectedItems }) => {
+        const mockedPlatformLogic = async (request: ApifyRequestConfig) => {
+            // Simulated platform logic for pagination when there are 2500 actors in store.
+            const maxItems = 2500;
+            const maxItemsPerPage = 1000;
+            const offset = request.params.offset ? request.params.offset : 0;
+            const limit = request.params.limit ? request.params.limit : 0;
+            if (offset < 0 || limit < 0) {
+                throw new Error('Offset and limit must be non-negative');
+            }
+
+            const lowerIndex = Math.min(offset, maxItems);
+            const upperIndex = Math.min(offset + (limit || maxItems), maxItems);
+            const returnedItemsCount = Math.min(upperIndex - lowerIndex, maxItemsPerPage);
+
+            return {
+                data: {
+                    data: {
+                        total: maxItems,
+                        count: returnedItemsCount,
+                        offset,
+                        limit: returnedItemsCount,
+                        desc: false,
+                        items: new Array(returnedItemsCount).fill('some actor details'),
+                    },
+                },
+            } as ApifyResponse;
+        };
+
+        const storeClient = client.store();
+        const mockedClient = jest.spyOn(storeClient.httpClient, 'call').mockImplementation(mockedPlatformLogic);
+
+        const totalItems: any[] = [];
+        for await (const page of client.store().list(userDefinedOptions)) {
+            totalItems.push(page);
+        }
+        mockedClient.mockRestore();
+        expect(totalItems.length).toBe(expectedItems);
+    });
+});