diff --git a/redisinsight/api/config/default.ts b/redisinsight/api/config/default.ts index 894914b3b8..4286e2bacd 100644 --- a/redisinsight/api/config/default.ts +++ b/redisinsight/api/config/default.ts @@ -451,4 +451,8 @@ export default { queryMaxNestedElements: parseInt(process.env.RI_AI_QUERY_MAX_NESTED_ELEMENTS, 10) || 25, }, + bulk_actions: { + summaryKeysLimit: + parseInt(process.env.RI_BULK_ACTIONS_SUMMARY_KEYS_LIMIT, 10) || 10_000, + }, }; diff --git a/redisinsight/api/src/modules/bulk-actions/bulk-import.service.spec.ts b/redisinsight/api/src/modules/bulk-actions/bulk-import.service.spec.ts index 0468e09ef5..2a4f4b4f4b 100644 --- a/redisinsight/api/src/modules/bulk-actions/bulk-import.service.spec.ts +++ b/redisinsight/api/src/modules/bulk-actions/bulk-import.service.spec.ts @@ -176,12 +176,11 @@ describe('BulkImportService', () => { }); it('should return all failed in case of global error', async () => { mockStandaloneRedisClient.sendPipeline.mockRejectedValueOnce(new Error()); - expect( - await service['executeBatch']( - mockStandaloneRedisClient, - mockBatchCommands, - ), - ).toEqual({ + const result = await service['executeBatch']( + mockStandaloneRedisClient, + mockBatchCommands, + ); + expect(result.getOverview()).toEqual({ ...mockSummary.getOverview(), succeed: 0, failed: mockSummary.getOverview().processed, diff --git a/redisinsight/api/src/modules/bulk-actions/models/bulk-action-summary.spec.ts b/redisinsight/api/src/modules/bulk-actions/models/bulk-action-summary.spec.ts index e6b925378c..8d84b9a0fa 100644 --- a/redisinsight/api/src/modules/bulk-actions/models/bulk-action-summary.spec.ts +++ b/redisinsight/api/src/modules/bulk-actions/models/bulk-action-summary.spec.ts @@ -81,6 +81,98 @@ describe('BulkActionSummary', () => { expect(summary['errors']).toEqual(generateErrors(500)); }); }); + + describe('addKeys', () => { + it('should add keys when under limit', async () => { + const keys = [Buffer.from('key1'), Buffer.from('key2')]; + + summary.addKeys(keys); + + expect(summary['keys']).toEqual(keys); + expect(summary['totalKeysProcessed']).toEqual(2); + expect(summary['hasMoreKeys']).toBe(false); + }); + + it('should limit stored keys when exceeding default limit', async () => { + // Create many keys to exceed the default limit of 10,000 + const keys = Array.from({ length: 15000 }, (_, i) => + Buffer.from(`key${i}`), + ); + + summary.addKeys(keys); + + expect(summary['keys']).toHaveLength(10000); // Default limit + expect(summary['totalKeysProcessed']).toEqual(15000); + expect(summary['hasMoreKeys']).toBe(true); + }); + + it('should handle multiple addKeys calls with default limit', async () => { + // Add keys in batches that exceed the default limit + const batch1 = Array.from({ length: 8000 }, (_, i) => + Buffer.from(`key${i}`), + ); + const batch2 = Array.from({ length: 5000 }, (_, i) => + Buffer.from(`key${i + 8000}`), + ); + + summary.addKeys(batch1); + expect(summary['keys']).toHaveLength(8000); + expect(summary['totalKeysProcessed']).toEqual(8000); + expect(summary['hasMoreKeys']).toBe(false); + + summary.addKeys(batch2); + expect(summary['keys']).toHaveLength(10000); // Default limit + expect(summary['totalKeysProcessed']).toEqual(13000); + expect(summary['hasMoreKeys']).toBe(true); + }); + + it('should handle partial batch when approaching default limit', async () => { + // Add keys close to the limit + const batch1 = Array.from({ length: 9999 }, (_, i) => + Buffer.from(`key${i}`), + ); + const batch2 = [Buffer.from('key9999'), Buffer.from('key10000')]; + + summary.addKeys(batch1); + expect(summary['keys']).toHaveLength(9999); + expect(summary['totalKeysProcessed']).toEqual(9999); + expect(summary['hasMoreKeys']).toBe(false); + + summary.addKeys(batch2); + expect(summary['keys']).toHaveLength(10000); // Default limit + expect(summary['totalKeysProcessed']).toEqual(10001); + expect(summary['hasMoreKeys']).toBe(true); + }); + + it('should use default limit from configuration', async () => { + expect(summary['maxStoredKeys']).toEqual(10_000); + }); + + it('should handle empty keys array', async () => { + summary.addKeys([]); + + expect(summary['keys']).toEqual([]); + expect(summary['totalKeysProcessed']).toEqual(0); + expect(summary['hasMoreKeys']).toBe(false); + }); + + it('should handle large number of keys efficiently', async () => { + const largeKeySet = Array.from({ length: 15000 }, (_, i) => + Buffer.from(`key${i}`), + ); + + summary.addKeys(largeKeySet); + + expect(summary['keys']).toHaveLength(10000); // Default limit + expect(summary['totalKeysProcessed']).toEqual(15000); + expect(summary['hasMoreKeys']).toBe(true); + + // Verify only first 10000 keys are stored + expect(summary['keys'][0]).toEqual(Buffer.from('key0')); + expect(summary['keys'][9999]).toEqual(Buffer.from('key9999')); + }); + }); + describe('getOverview', () => { it('should get overview and clear errors', async () => { expect(summary['processed']).toEqual(0); @@ -105,5 +197,58 @@ describe('BulkActionSummary', () => { expect(summary['failed']).toEqual(1000); expect(summary['errors']).toEqual([]); }); + + it('should return only stored keys in overview (not internal fields)', async () => { + const keys = Array.from({ length: 15000 }, (_, i) => + Buffer.from(`key${i}`), + ); + + summary.addKeys(keys); + summary.addProcessed(15000); + summary.addSuccess(15000); + + const overview = summary.getOverview(); + + // Should only contain the interface fields, not internal tracking fields + expect(overview.processed).toEqual(15000); + expect(overview.succeed).toEqual(15000); + expect(overview.failed).toEqual(0); + expect(overview.errors).toEqual([]); + expect(overview.keys).toHaveLength(10000); // Default limit + + // Internal fields should not be exposed + expect(overview).not.toHaveProperty('totalKeysProcessed'); + expect(overview).not.toHaveProperty('hasMoreKeys'); + expect(overview).not.toHaveProperty('maxStoredKeys'); + }); + }); + + describe('memory management integration', () => { + it('should maintain consistent state across operations', async () => { + const batch1 = Array.from({ length: 5000 }, (_, i) => + Buffer.from(`key${i}`), + ); + summary.addKeys(batch1); + summary.addProcessed(5000); + summary.addSuccess(5000); + + // Add more keys that exceed default limit + const batch2 = Array.from({ length: 8000 }, (_, i) => + Buffer.from(`key${i + 5000}`), + ); + summary.addKeys(batch2); + summary.addProcessed(8000); + summary.addSuccess(7000); + summary.addFailed(1000); + + const overview = summary.getOverview(); + + expect(overview.processed).toEqual(13000); + expect(overview.succeed).toEqual(12000); + expect(overview.failed).toEqual(1000); + expect(overview.keys).toHaveLength(10000); // Default limit + expect(summary['totalKeysProcessed']).toEqual(13000); + expect(summary['hasMoreKeys']).toBe(true); + }); }); }); diff --git a/redisinsight/api/src/modules/bulk-actions/models/bulk-action-summary.ts b/redisinsight/api/src/modules/bulk-actions/models/bulk-action-summary.ts index 39da1dee57..8898f565b6 100644 --- a/redisinsight/api/src/modules/bulk-actions/models/bulk-action-summary.ts +++ b/redisinsight/api/src/modules/bulk-actions/models/bulk-action-summary.ts @@ -1,5 +1,10 @@ import { RedisString } from 'src/common/constants'; import { IBulkActionSummaryOverview } from 'src/modules/bulk-actions/interfaces/bulk-action-summary-overview.interface'; +import config, { Config } from 'src/utils/config'; + +const BULK_ACTIONS_CONFIG = config.get( + 'bulk_actions', +) as Config['bulk_actions']; export class BulkActionSummary { private processed: number = 0; @@ -12,6 +17,12 @@ export class BulkActionSummary { private keys: Array = []; + private hasMoreKeys: boolean = false; + + private totalKeysProcessed: number = 0; + + private readonly maxStoredKeys: number = BULK_ACTIONS_CONFIG.summaryKeysLimit; + addProcessed(count: number) { this.processed += count; } @@ -33,7 +44,18 @@ export class BulkActionSummary { } addKeys(keys: Array) { - this.keys.push(...keys); + this.totalKeysProcessed += keys.length; + + const remaining = this.maxStoredKeys - this.keys.length; + + if (remaining > 0) { + const keysToStore = keys.slice(0, remaining); + this.keys.push(...keysToStore); + } + + if (this.totalKeysProcessed > this.maxStoredKeys) { + this.hasMoreKeys = true; + } } getOverview(): IBulkActionSummaryOverview {