From c560fa0052461e8c6fa3f5a45eebfba9c2749fe3 Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Thu, 28 May 2026 10:25:23 +0000 Subject: [PATCH] feat: implement AI settings and widgets creation with streaming response --- .../src/entities/ai/ai-use-cases.interface.ts | 4 +- .../request-ai-settings-creation.ds.ts | 6 + ...-settings-and-widgets-creation.use.case.ts | 42 +++- .../ai/user-ai-requests-v2.controller.ts | 9 +- .../use-cases/create-connection.use.case.ts | 8 +- .../shared-jobs/shared-jobs.service.ts | 42 +++- .../utils/emit-settings-messages.util.ts | 35 +++ ...as-ai-settings-creation-stream-e2e.test.ts | 233 ++++++++++++++++++ .../non-saas-emit-settings-messages.test.ts | 90 +++++++ 9 files changed, 455 insertions(+), 14 deletions(-) create mode 100644 backend/src/entities/ai/application/data-structures/request-ai-settings-creation.ds.ts create mode 100644 backend/src/entities/shared-jobs/utils/emit-settings-messages.util.ts create mode 100644 backend/test/ava-tests/non-saas-tests/non-saas-ai-settings-creation-stream-e2e.test.ts create mode 100644 backend/test/ava-tests/non-saas-tests/non-saas-emit-settings-messages.test.ts diff --git a/backend/src/entities/ai/ai-use-cases.interface.ts b/backend/src/entities/ai/ai-use-cases.interface.ts index 18ba3b42d..ce85fee03 100644 --- a/backend/src/entities/ai/ai-use-cases.interface.ts +++ b/backend/src/entities/ai/ai-use-cases.interface.ts @@ -1,5 +1,5 @@ import { InTransactionEnum } from '../../enums/in-transaction.enum.js'; -import { FindOneConnectionDs } from '../connection/application/data-structures/find-one-connection.ds.js'; +import { RequestAISettingsCreationDs } from './application/data-structures/request-ai-settings-creation.ds.js'; import { RequestInfoFromTableDSV2 } from './application/data-structures/request-info-from-table.ds.js'; export interface IRequestInfoFromTableV2 { @@ -7,5 +7,5 @@ export interface IRequestInfoFromTableV2 { } export interface IAISettingsAndWidgetsCreation { - execute(connectionData: FindOneConnectionDs, inTransaction: InTransactionEnum): Promise; + execute(inputData: RequestAISettingsCreationDs, inTransaction: InTransactionEnum): Promise; } diff --git a/backend/src/entities/ai/application/data-structures/request-ai-settings-creation.ds.ts b/backend/src/entities/ai/application/data-structures/request-ai-settings-creation.ds.ts new file mode 100644 index 000000000..42fa1c2cc --- /dev/null +++ b/backend/src/entities/ai/application/data-structures/request-ai-settings-creation.ds.ts @@ -0,0 +1,6 @@ +import { Response } from 'express'; +import { FindOneConnectionDs } from '../../../connection/application/data-structures/find-one-connection.ds.js'; + +export class RequestAISettingsCreationDs extends FindOneConnectionDs { + response: Response; +} diff --git a/backend/src/entities/ai/use-cases/request-ai-settings-and-widgets-creation.use.case.ts b/backend/src/entities/ai/use-cases/request-ai-settings-and-widgets-creation.use.case.ts index 81bd06e0d..c75618ded 100644 --- a/backend/src/entities/ai/use-cases/request-ai-settings-and-widgets-creation.use.case.ts +++ b/backend/src/entities/ai/use-cases/request-ai-settings-and-widgets-creation.use.case.ts @@ -1,15 +1,18 @@ import { BadRequestException, Inject, Injectable, Scope } from '@nestjs/common'; +import Sentry from '@sentry/minimal'; +import { Response } from 'express'; import AbstractUseCase from '../../../common/abstract-use.case.js'; import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; import { BaseType } from '../../../common/data-injection.tokens.js'; import { Messages } from '../../../exceptions/text/messages.js'; -import { FindOneConnectionDs } from '../../connection/application/data-structures/find-one-connection.ds.js'; +import { getErrorMessage } from '../../../helpers/get-error-message.js'; import { SharedJobsService } from '../../shared-jobs/shared-jobs.service.js'; import { IAISettingsAndWidgetsCreation } from '../ai-use-cases.interface.js'; +import { RequestAISettingsCreationDs } from '../application/data-structures/request-ai-settings-creation.ds.js'; @Injectable({ scope: Scope.REQUEST }) export class RequestAISettingsAndWidgetsCreationUseCase - extends AbstractUseCase + extends AbstractUseCase implements IAISettingsAndWidgetsCreation { constructor( @@ -20,14 +23,43 @@ export class RequestAISettingsAndWidgetsCreationUseCase super(); } - public async implementation(connectionData: FindOneConnectionDs): Promise { - const { connectionId, masterPwd } = connectionData; + public async implementation(inputData: RequestAISettingsCreationDs): Promise { + const { connectionId, masterPwd, response } = inputData; const connection = await this._dbContext.connectionRepository.findAndDecryptConnection(connectionId, masterPwd); if (!connection) { throw new BadRequestException(Messages.CONNECTION_NOT_FOUND); } - await this.sharedJobsService.scanDatabaseAndCreateSettingsAndWidgetsWithAI(connection); + this.setupResponseHeaders(response); + + try { + await this.sharedJobsService.scanDatabaseAndCreateSettingsAndWidgetsWithAI(connection, (chunk) => + this.writeChunk(response, chunk), + ); + this.writeChunk(response, { type: 'complete' }); + response.end(); + } catch (error) { + Sentry.captureException(error); + if (!response.headersSent) { + response.status(500).send({ error: 'An error occurred while processing your request.' }); + return; + } + this.writeChunk(response, { type: 'error', message: getErrorMessage(error) }); + response.end(); + } + } + + private setupResponseHeaders(response: Response): void { + response.setHeader('Content-Type', 'text/event-stream'); + response.setHeader('Cache-Control', 'no-cache'); + response.setHeader('Connection', 'keep-alive'); + } + + private writeChunk( + response: Response, + chunk: { type: 'message'; text: string } | { type: 'complete' } | { type: 'error'; message: string }, + ): void { + response.write(JSON.stringify(chunk) + '\n'); } } diff --git a/backend/src/entities/ai/user-ai-requests-v2.controller.ts b/backend/src/entities/ai/user-ai-requests-v2.controller.ts index 59b7954e3..281419d07 100644 --- a/backend/src/entities/ai/user-ai-requests-v2.controller.ts +++ b/backend/src/entities/ai/user-ai-requests-v2.controller.ts @@ -25,6 +25,7 @@ import { isTest } from '../../helpers/app/is-test.js'; import { ValidationHelper } from '../../helpers/validators/validation-helper.js'; import { SentryInterceptor } from '../../interceptors/sentry.interceptor.js'; import { IAISettingsAndWidgetsCreation, IRequestInfoFromTableV2 } from './ai-use-cases.interface.js'; +import { RequestAISettingsCreationDs } from './application/data-structures/request-ai-settings-creation.ds.js'; import { RequestInfoFromTableDSV2 } from './application/data-structures/request-info-from-table.ds.js'; import { RequestInfoFromTableBodyDTO } from './application/dto/request-info-from-table-body.dto.js'; @@ -88,7 +89,7 @@ export class UserAIRequestsControllerV2 { }) @ApiResponse({ status: 200, - description: 'AI settings and widgets creation job has been queued.', + description: 'Streams progress of the AI settings and widgets creation job as newline-delimited JSON chunks.', }) @UseGuards(ConnectionEditGuard) @Timeout(!isTest() ? TimeoutDefaults.AI : TimeoutDefaults.AI_TEST) @@ -97,12 +98,14 @@ export class UserAIRequestsControllerV2 { @SlugUuid('connectionId') connectionId: string, @MasterPassword() masterPassword: string, @UserId() userId: string, + @Res({ passthrough: true }) response: Response, ): Promise { - const connectionData = { + const inputData: RequestAISettingsCreationDs = { connectionId, masterPwd: masterPassword, cognitoUserName: userId, + response, }; - return await this.requestAISettingsAndWidgetsCreationUseCase.execute(connectionData, InTransactionEnum.OFF); + return await this.requestAISettingsAndWidgetsCreationUseCase.execute(inputData, InTransactionEnum.OFF); } } diff --git a/backend/src/entities/connection/use-cases/create-connection.use.case.ts b/backend/src/entities/connection/use-cases/create-connection.use.case.ts index 04b47281c..a35868be4 100644 --- a/backend/src/entities/connection/use-cases/create-connection.use.case.ts +++ b/backend/src/entities/connection/use-cases/create-connection.use.case.ts @@ -6,6 +6,7 @@ import { IGlobalDatabaseContext } from '../../../common/application/global-datab import { BaseType } from '../../../common/data-injection.tokens.js'; import { AccessLevelEnum } from '../../../enums/access-level.enum.js'; import { Messages } from '../../../exceptions/text/messages.js'; +import { isTest } from '../../../helpers/app/is-test.js'; import { Encryptor } from '../../../helpers/encryption/encryptor.js'; import { isConnectionTypeAgent } from '../../../helpers/is-connection-entity-agent.js'; import { slackPostMessage } from '../../../helpers/slack/slack-post-message.js'; @@ -125,7 +126,12 @@ export class CreateConnectionUseCase const connectionRO = buildCreatedConnectionDs(savedConnection, token, masterPwd); return connectionRO; } finally { - if (connectionCopy && isConnectionTestedSuccessfully && !isConnectionTypeAgent(connectionCopy.type)) { + if ( + connectionCopy && + isConnectionTestedSuccessfully && + !isConnectionTypeAgent(connectionCopy.type) && + !isTest() + ) { // Fire-and-forget: run AI scan in background without blocking response this.sharedJobsService.scanDatabaseAndCreateSettingsAndWidgetsWithAI(connectionCopy).catch((error) => { console.error('Background AI scan failed:', error); diff --git a/backend/src/entities/shared-jobs/shared-jobs.service.ts b/backend/src/entities/shared-jobs/shared-jobs.service.ts index 02c579a71..e02a5359b 100644 --- a/backend/src/entities/shared-jobs/shared-jobs.service.ts +++ b/backend/src/entities/shared-jobs/shared-jobs.service.ts @@ -9,7 +9,6 @@ import PQueue from 'p-queue'; import { IGlobalDatabaseContext } from '../../common/application/global-database-context.interface.js'; import { BaseType } from '../../common/data-injection.tokens.js'; import { WidgetTypeEnum } from '../../enums/widget-type.enum.js'; -import { isTest } from '../../helpers/app/is-test.js'; import { getErrorMessage } from '../../helpers/get-error-message.js'; import { ValidationHelper } from '../../helpers/validators/validation-helper.js'; import { AiService } from '../ai/ai.service.js'; @@ -18,6 +17,11 @@ import { TableSettingsEntity } from '../table-settings/common-table-settings/tab import { buildEmptyTableSettings } from '../table-settings/common-table-settings/utils/build-empty-table-settings.js'; import { buildNewTableSettingsEntity } from '../table-settings/common-table-settings/utils/build-new-table-settings-entity.js'; import { TableWidgetEntity } from '../widget/table-widget.entity.js'; +import { emitSettingsMessages } from './utils/emit-settings-messages.util.js'; + +export type AIScanProgressChunk = { type: 'message'; text: string }; + +export type AIScanProgressCallback = (chunk: AIScanProgressChunk) => void; @Injectable() export class SharedJobsService { @@ -27,11 +31,22 @@ export class SharedJobsService { private readonly aiService: AiService, ) {} - public async scanDatabaseAndCreateSettingsAndWidgetsWithAI(connection: ConnectionEntity): Promise { - if (!connection || isTest()) { + public async scanDatabaseAndCreateSettingsAndWidgetsWithAI( + connection: ConnectionEntity, + onProgress?: AIScanProgressCallback, + ): Promise { + if (!connection) { return; } + const emit: AIScanProgressCallback = (chunk) => { + if (onProgress) { + onProgress(chunk); + } + }; + const message = (text: string): void => emit({ type: 'message', text }); + console.info(`Starting AI scan for connection with id "${connection.id}"`); + message(`Starting AI scan for connection "${connection.title || connection.id}"`); try { const existingTableSettings = await this._dbContext.tableSettingsRepository.find({ where: { @@ -43,14 +58,18 @@ export class SharedJobsService { const existingTableNames = new Set(existingTableSettings.map((setting) => setting.table_name)); const dao = getDataAccessObject(connection); + message('Fetching tables from database'); const tables: Array = await dao.getTablesFromDB(); const tablesToScan = tables.filter((table) => !existingTableNames.has(table.tableName)); if (tablesToScan.length === 0) { console.info(`No new tables to scan for connection with id "${connection.id}"`); + message('No new tables to scan — all tables already have settings'); return; } + message(`Found ${tablesToScan.length} new ${tablesToScan.length === 1 ? 'table' : 'tables'} to scan`); + const queue = new PQueue({ concurrency: 4 }); const tablesInformationResults = await Promise.all( tablesToScan.map((table) => @@ -59,6 +78,7 @@ export class SharedJobsService { const structure = await dao.getTableStructure(table.tableName, null); const primaryColumns = await dao.getTablePrimaryColumns(table.tableName, null); const foreignKeys = await dao.getTableForeignKeys(table.tableName, null); + message(`Inspected structure of table "${table.tableName}"`); return { table_name: table.tableName, structure, @@ -66,6 +86,7 @@ export class SharedJobsService { foreignKeys, }; } catch (error) { + message(`Failed to inspect table "${table.tableName}": ${getErrorMessage(error)}`); console.error(`Error getting table information for "${table.tableName}": ${getErrorMessage(error)}`); return null; } @@ -77,18 +98,26 @@ export class SharedJobsService { if (tablesInformation.length === 0) { console.info(`No valid tables to process for connection with id "${connection.id}"`); + message('No valid tables to process'); return; } console.info(`Processing ${tablesInformation.length} tables with AI for connection "${connection.id}"`); + message( + `Generating settings with AI for ${tablesInformation.length} ${tablesInformation.length === 1 ? 'table' : 'tables'}`, + ); const generatedTableSettings = await this.aiService.generateNewTableSettingsWithAI(tablesInformation); if (generatedTableSettings.length === 0) { console.info(`No table settings generated by AI for connection with id "${connection.id}"`); + message('AI did not produce any table settings'); return; } console.info(`AI generated settings for ${generatedTableSettings.length} tables`); + message( + `AI returned settings for ${generatedTableSettings.length} ${generatedTableSettings.length === 1 ? 'table' : 'tables'}`, + ); const widgetsByTable = new Map>(); for (const setting of generatedTableSettings) { @@ -106,6 +135,7 @@ export class SharedJobsService { const validateSettingsDS = buildValidateTableSettingsDS(setting); const errors = await dao.validateSettings(validateSettingsDS, setting.table_name, undefined); if (errors.length > 0) { + message(`Validation failed for table "${setting.table_name}", skipping`); console.error(`Validation errors for table "${setting.table_name}":`, errors); return null; } @@ -119,6 +149,7 @@ export class SharedJobsService { const savedSettings = await this._dbContext.tableSettingsRepository.save(settingsToSave); const widgetsToSave: Array = []; for (const savedSetting of savedSettings) { + emitSettingsMessages(savedSetting, message); const widgets = widgetsByTable.get(savedSetting.table_name); if (widgets && widgets.length > 0) { for (const widget of widgets) { @@ -131,6 +162,9 @@ export class SharedJobsService { widgetEntity.description = widget.description || null; widgetEntity.settings = savedSetting; widgetsToSave.push(widgetEntity); + message( + `Added ${widget.widget_type} widget for table "${savedSetting.table_name}" on column "${widget.field_name}"`, + ); } } } @@ -138,10 +172,12 @@ export class SharedJobsService { if (widgetsToSave.length > 0) { await this._dbContext.tableWidgetsRepository.save(widgetsToSave); } + message(`Finished setup for ${savedSettings.length} ${savedSettings.length === 1 ? 'table' : 'tables'}`); } } catch (error) { console.error('Error during AI scan and creation of settings/widgets: ', error); Sentry.captureException(error); + throw error; } } diff --git a/backend/src/entities/shared-jobs/utils/emit-settings-messages.util.ts b/backend/src/entities/shared-jobs/utils/emit-settings-messages.util.ts new file mode 100644 index 000000000..a00b8ce53 --- /dev/null +++ b/backend/src/entities/shared-jobs/utils/emit-settings-messages.util.ts @@ -0,0 +1,35 @@ +import { TableSettingsEntity } from '../../table-settings/common-table-settings/table-settings.entity.js'; + +export function emitSettingsMessages(setting: TableSettingsEntity, emit: (text: string) => void): void { + const tableName = setting.table_name; + const params: Array<[string, string]> = []; + if (setting.display_name) { + params.push(['display_name', `"${setting.display_name}"`]); + } + if (setting.search_fields && setting.search_fields.length > 0) { + params.push(['search_fields', setting.search_fields.join(', ')]); + } + if (setting.readonly_fields && setting.readonly_fields.length > 0) { + params.push(['readonly_fields', setting.readonly_fields.join(', ')]); + } + if (setting.columns_view && setting.columns_view.length > 0) { + params.push(['columns_view', setting.columns_view.join(', ')]); + } + if (setting.ordering) { + params.push(['ordering', String(setting.ordering)]); + } + if (setting.ordering_field) { + params.push(['ordering_field', `"${setting.ordering_field}"`]); + } + if (setting.identity_column) { + params.push(['identity_column', `"${setting.identity_column}"`]); + } + + if (params.length === 0) { + emit(`Set up settings for table "${tableName}" with default parameters`); + return; + } + for (const [name, value] of params) { + emit(`Set up settings for table "${tableName}", ${name} parameter set to ${value}`); + } +} diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-ai-settings-creation-stream-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-ai-settings-creation-stream-e2e.test.ts new file mode 100644 index 000000000..134057f93 --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-ai-settings-creation-stream-e2e.test.ts @@ -0,0 +1,233 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/services/ai-core.service.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { createTestPostgresTableWithSchema } from '../../utils/create-test-table.js'; +import { dropTestTables } from '../../utils/drop-test-tables.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const testTables: Array = []; + +function buildIterableStream(chunks: Array>) { + return { + async *[Symbol.asyncIterator]() { + for (const chunk of chunks) { + yield chunk; + } + }, + }; +} + +function buildAIBatchResponse(prompt: string): string { + const tableNames = [...prompt.matchAll(/^Table: (\S+)/gm)].map((m) => m[1]); + const tables = tableNames.map((tableName) => ({ + table_name: tableName, + display_name: tableName.charAt(0).toUpperCase() + tableName.slice(1), + search_fields: ['name', 'email'], + readonly_fields: ['id', 'created_at'], + columns_view: ['id', 'name', 'email', 'created_at'], + ordering: 'DESC', + ordering_field: 'created_at', + identity_column: 'name', + widgets: [ + { + field_name: 'id', + widget_type: 'Readonly', + widget_params: {}, + name: 'ID', + description: 'Unique identifier', + }, + { + field_name: 'email', + widget_type: 'String', + widget_params: { validate: 'isEmail' }, + name: 'Email', + description: 'Email address', + }, + ], + })); + return JSON.stringify({ tables }); +} + +const mockAICoreService = { + completeWithProvider: async (_provider: unknown, prompt: string, _config: unknown) => buildAIBatchResponse(prompt), + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + chatWithProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => buildIterableStream([{ type: 'text', content: 'mocked' }]), + streamChatWithProvider: async () => buildIterableStream([{ type: 'text', content: 'mocked' }]), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => buildIterableStream([{ type: 'text', content: 'mocked' }]), + streamChatWithToolsAndProvider: async () => buildIterableStream([{ type: 'text', content: 'mocked' }]), + getDefaultProvider: () => 'bedrock', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +function parseNdjsonChunks(body: string): Array> { + return body + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line)); +} + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after.always(async () => { + try { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + await dropTestTables(testTables, connectionToTestDB); + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +const currentTest = 'GET /ai/v2/setup/:connectionId'; + +test.serial(`${currentTest} streams human-readable progress and ends with a complete chunk`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + const { token } = await registerUserAndReturnUserInfo(app); + const { testTableName } = await createTestPostgresTableWithSchema(connectionToTestDB); + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + t.is(createConnectionResponse.status, 201); + const connectionRO = JSON.parse(createConnectionResponse.text); + + const setupResponse = await request(app.getHttpServer()) + .get(`/ai/v2/setup/${connectionRO.id}`) + .set('Cookie', token) + .buffer(true) + .parse((res, callback) => { + let data = ''; + res.setEncoding('utf8'); + res.on('data', (chunk: string) => { + data += chunk; + }); + res.on('end', () => callback(null, data)); + }); + + t.is(setupResponse.status, 200); + t.regex(String(setupResponse.headers['content-type']), /text\/event-stream/); + + const chunks = parseNdjsonChunks(setupResponse.body as string); + t.true(chunks.length > 0, 'expected at least one chunk'); + + const types = chunks.map((c) => c.type); + t.true(types.includes('message'), 'expected at least one "message" chunk'); + t.is(types[types.length - 1], 'complete', 'last chunk should be "complete"'); + + const messages = chunks.filter((c) => c.type === 'message').map((c) => String(c.text)); + t.true( + messages.some((m) => m.startsWith('Starting AI scan')), + 'expected an initial "Starting AI scan" message', + ); + t.true( + messages.some((m) => m.includes(`table "${testTableName}"`)), + `expected at least one message mentioning the test table "${testTableName}"`, + ); + t.true( + messages.some((m) => m.includes('display_name parameter set to')), + 'expected a per-parameter "display_name parameter set to" message', + ); + t.true( + messages.some((m) => /Added \w+ widget for table/.test(m)), + 'expected at least one "Added widget for table" message', + ); +}); + +test.serial(`${currentTest} streams a no-new-tables message when settings already exist`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + const { token } = await registerUserAndReturnUserInfo(app); + const { testTableName } = await createTestPostgresTableWithSchema(connectionToTestDB); + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + t.is(createConnectionResponse.status, 201); + const connectionRO = JSON.parse(createConnectionResponse.text); + + const fetchStream = async () => + request(app.getHttpServer()) + .get(`/ai/v2/setup/${connectionRO.id}`) + .set('Cookie', token) + .buffer(true) + .parse((res, callback) => { + let data = ''; + res.setEncoding('utf8'); + res.on('data', (chunk: string) => { + data += chunk; + }); + res.on('end', () => callback(null, data)); + }); + + await fetchStream(); + const secondResponse = await fetchStream(); + + t.is(secondResponse.status, 200); + const chunks = parseNdjsonChunks(secondResponse.body as string); + const messages = chunks.filter((c) => c.type === 'message').map((c) => String(c.text)); + t.true( + messages.some((m) => m.toLowerCase().includes('no new tables')), + 'expected a "No new tables" status message on second invocation', + ); + t.is(chunks[chunks.length - 1].type, 'complete'); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-emit-settings-messages.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-emit-settings-messages.test.ts new file mode 100644 index 000000000..f1d895910 --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-emit-settings-messages.test.ts @@ -0,0 +1,90 @@ +import test from 'ava'; +import { emitSettingsMessages } from '../../../src/entities/shared-jobs/utils/emit-settings-messages.util.js'; +import { TableSettingsEntity } from '../../../src/entities/table-settings/common-table-settings/table-settings.entity.js'; +import { QueryOrderingEnum } from '../../../src/enums/query-ordering.enum.js'; + +function buildSetting(overrides: Partial = {}): TableSettingsEntity { + const setting = new TableSettingsEntity(); + setting.table_name = 'users'; + return Object.assign(setting, overrides); +} + +function collect(setting: TableSettingsEntity): Array { + const lines: Array = []; + emitSettingsMessages(setting, (text) => lines.push(text)); + return lines; +} + +test('emits one line per populated parameter', (t) => { + const setting = buildSetting({ + display_name: 'Users', + search_fields: ['name', 'email'], + readonly_fields: ['id', 'created_at'], + columns_view: ['id', 'name', 'email'], + ordering: QueryOrderingEnum.DESC, + ordering_field: 'created_at', + identity_column: 'email', + }); + + const lines = collect(setting); + + t.deepEqual(lines, [ + 'Set up settings for table "users", display_name parameter set to "Users"', + 'Set up settings for table "users", search_fields parameter set to name, email', + 'Set up settings for table "users", readonly_fields parameter set to id, created_at', + 'Set up settings for table "users", columns_view parameter set to id, name, email', + 'Set up settings for table "users", ordering parameter set to DESC', + 'Set up settings for table "users", ordering_field parameter set to "created_at"', + 'Set up settings for table "users", identity_column parameter set to "email"', + ]); +}); + +test('skips parameters that are null, undefined, or empty arrays', (t) => { + const setting = buildSetting({ + display_name: 'Users', + search_fields: [], + readonly_fields: null, + columns_view: undefined, + ordering: null, + ordering_field: null, + identity_column: null, + }); + + const lines = collect(setting); + + t.deepEqual(lines, ['Set up settings for table "users", display_name parameter set to "Users"']); +}); + +test('emits default-parameters fallback when nothing is populated', (t) => { + const setting = buildSetting({ + table_name: 'orders', + }); + + const lines = collect(setting); + + t.deepEqual(lines, ['Set up settings for table "orders" with default parameters']); +}); + +test('uses the table name from the setting in every line', (t) => { + const setting = buildSetting({ + table_name: 'products', + display_name: 'Products', + ordering: QueryOrderingEnum.ASC, + }); + + const lines = collect(setting); + + t.true(lines.every((line) => line.includes('"products"'))); + t.is(lines.length, 2); +}); + +test('joins array-valued parameters with comma+space', (t) => { + const setting = buildSetting({ + search_fields: ['first_name', 'last_name', 'email'], + }); + + const lines = collect(setting); + + t.is(lines.length, 1); + t.is(lines[0], 'Set up settings for table "users", search_fields parameter set to first_name, last_name, email'); +});