From 32a900555a3e9e320d1e899c3f5c0b83b17bc912 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 8 Oct 2025 17:18:38 +0530 Subject: [PATCH] feat: Added Dynamic Filters --- .../get-auto-import-job-data.consumer.ts | 91 ++++++++++++++++++- .../repositories/user-job/user-job.entity.ts | 4 + .../repositories/user-job/user-job.schema.ts | 14 +++ .../src/entities/UserJob/Userjob.interface.ts | 15 +++ 4 files changed, 122 insertions(+), 2 deletions(-) diff --git a/apps/queue-manager/src/consumers/get-auto-import-job-data.consumer.ts b/apps/queue-manager/src/consumers/get-auto-import-job-data.consumer.ts index cc20cd9f2..73dfbbfef 100644 --- a/apps/queue-manager/src/consumers/get-auto-import-job-data.consumer.ts +++ b/apps/queue-manager/src/consumers/get-auto-import-job-data.consumer.ts @@ -5,6 +5,8 @@ import { ITemplateSchemaItem, ColumnDelimiterEnum, UserJobImportStatusEnum, + IFilter, + FilterOperationEnum, } from '@impler/shared'; import { SendImportJobDataConsumer } from './send-import-job-data.consumer'; @@ -88,8 +90,15 @@ export class SendAutoImportJobDataConsumer extends SendImportJobDataConsumer { const validationResult = await this.validateData(_jobId, mappedData); - // Send data to webhook with validation status - this.sendDataImportData(_jobId, mappedData, 1, undefined, validationResult.hasInvalidRecords); + if (validationResult.filteredData.length > 0) { + this.sendDataImportData( + _jobId, + validationResult.filteredData, + 1, + undefined, + validationResult.hasInvalidRecords + ); + } if (validationResult.hasInvalidRecords) { await this.userJobRepository.update({ _id: _jobId }, { $set: { isInvalidRecords: true } }); @@ -117,6 +126,7 @@ export class SendAutoImportJobDataConsumer extends SendImportJobDataConsumer { invalidRecords: number; validData: Record[]; invalidData: Record[]; + filteredData: Record[]; }> { try { const userJob = await this.userJobRepository.findOne({ _id: _jobId }); @@ -134,6 +144,7 @@ export class SendAutoImportJobDataConsumer extends SendImportJobDataConsumer { invalidRecords: 0, validData: [], invalidData: [], + filteredData: [], }; } @@ -150,7 +161,19 @@ export class SendAutoImportJobDataConsumer extends SendImportJobDataConsumer { const validData: Record[] = []; const invalidData: Record[] = []; + const filters = userJob.filters || []; + + const filteredData: Record[] = []; + for (const recordData of mappedData) { + const passesFilters = await this.applyDynamicFilters(recordData, filters); + + if (passesFilters) { + filteredData.push(recordData); + } + } + + for (const recordData of filteredData) { const checkRecord: Record = this.formatRecord({ record: { record: recordData }, multiSelectColumnHeadings, @@ -181,6 +204,7 @@ export class SendAutoImportJobDataConsumer extends SendImportJobDataConsumer { invalidRecords, validData, invalidData, + filteredData, }; } catch (error) { return { @@ -190,6 +214,7 @@ export class SendAutoImportJobDataConsumer extends SendImportJobDataConsumer { invalidRecords: 0, validData: [], invalidData: [], + filteredData: [], }; } } @@ -287,6 +312,68 @@ export class SendAutoImportJobDataConsumer extends SendImportJobDataConsumer { } } + private async applyDynamicFilters(record: Record, filters: IFilter[]): Promise { + try { + if (!filters || filters.length === 0) { + return true; + } + + for (const filter of filters) { + const fieldValue = record[filter.field]; + + if (fieldValue === undefined || fieldValue === null) { + return false; + } + + const stringValue = String(fieldValue); + const filterValue = String(filter.value); + + const operationType = filter.operation; + + let operationPassed = false; + + switch (operationType) { + case FilterOperationEnum.CONTAINS: + operationPassed = stringValue.includes(filterValue); + break; + + case FilterOperationEnum.EQUALS: + operationPassed = stringValue === filterValue; + break; + + case FilterOperationEnum.STARTSWITH: + operationPassed = stringValue.startsWith(filterValue); + break; + + case FilterOperationEnum.ENDSWITH: + operationPassed = stringValue.endsWith(filterValue); + break; + + case FilterOperationEnum.MATCHES: + try { + const regex = new RegExp(filter.value, 'i'); + operationPassed = regex.test(stringValue); + } catch (regexError) { + operationPassed = false; + } + break; + + default: + operationPassed = false; + break; + } + + if (!operationPassed) { + return false; + } + } + + return true; + } catch (error) { + return true; + } + } + private async finalizeAutoImportJob( _jobId: string, validationResult: IValidationResult, diff --git a/libs/dal/src/repositories/user-job/user-job.entity.ts b/libs/dal/src/repositories/user-job/user-job.entity.ts index b9ec6320c..2311542f4 100644 --- a/libs/dal/src/repositories/user-job/user-job.entity.ts +++ b/libs/dal/src/repositories/user-job/user-job.entity.ts @@ -1,3 +1,5 @@ +import { IFilter } from '@impler/shared'; + export class UserJobEntity { _id?: string; @@ -34,4 +36,6 @@ export class UserJobEntity { validRecords?: number; invalidRecords?: number; + + filters?: IFilter[]; } diff --git a/libs/dal/src/repositories/user-job/user-job.schema.ts b/libs/dal/src/repositories/user-job/user-job.schema.ts index 50fa76cb3..7e0a27705 100644 --- a/libs/dal/src/repositories/user-job/user-job.schema.ts +++ b/libs/dal/src/repositories/user-job/user-job.schema.ts @@ -1,6 +1,7 @@ import { model, models, Schema, Model } from 'mongoose'; import { schemaOptions } from '../schema-default.options'; import { UserJobEntity } from './user-job.entity'; +import { FilterOperationEnum } from '@impler/shared'; const userJobSchema = new Schema( { @@ -62,6 +63,19 @@ const userJobSchema = new Schema( type: Schema.Types.Number, default: 0, }, + filters: { + type: [ + { + field: { type: Schema.Types.String }, + operation: { + type: Schema.Types.String, + enum: Object.values(FilterOperationEnum), + }, + value: { type: Schema.Types.String }, + }, + ], + default: [], + }, }, { ...schemaOptions } ); diff --git a/libs/shared/src/entities/UserJob/Userjob.interface.ts b/libs/shared/src/entities/UserJob/Userjob.interface.ts index e40a13e3e..a7f6e0f59 100644 --- a/libs/shared/src/entities/UserJob/Userjob.interface.ts +++ b/libs/shared/src/entities/UserJob/Userjob.interface.ts @@ -1,3 +1,17 @@ +export enum FilterOperationEnum { + CONTAINS = 'contains', + EQUALS = 'equals', + STARTSWITH = 'startsWith', + ENDSWITH = 'endsWith', + MATCHES = 'matches', +} + +export interface IFilter { + field: string; + operation: FilterOperationEnum; + value: string; +} + export interface IUserJob { _id: string; url: string; @@ -7,4 +21,5 @@ export interface IUserJob { endsOn?: Date; headings: string[]; _templateId: string; + filters?: IFilter[]; }