From 71b111e160081fcd3339735aa9241076d023bef0 Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Mon, 11 May 2026 10:37:32 +0000 Subject: [PATCH] feat: add GET_TABLE_STRUCTURE_WITHOUT_CACHE use case and endpoint - Introduced a new use case `GetTableStructureWithoutCacheUseCase` to fetch table structure directly from the database without using cache. - Updated `TableController` to include a new endpoint `/table/structure/no-cache/:connectionId` for accessing the new use case. - Enhanced `TableModule` to provide the new use case for dependency injection. - Updated interfaces to include `IGetTableStructureWithoutCache`. - Modified existing data access objects to support fetching table structure without cache. - Added comprehensive tests for the new endpoint in both MySQL and PostgreSQL E2E test files. --- backend/src/common/data-injection.tokens.ts | 1 + .../src/entities/table/table.controller.ts | 39 ++++ backend/src/entities/table/table.module.ts | 6 + ...-table-structure-without-cache.use.case.ts | 109 ++++++++++ .../use-cases/table-use-cases.interface.ts | 4 + .../non-saas-table-mysql-e2e.test.ts | 200 ++++++++++++++++++ ...non-saas-table-postgres-schema-e2e.test.ts | 200 ++++++++++++++++++ .../data-access-object-agent.ts | 13 +- .../data-access-object-cassandra.ts | 4 + .../data-access-object-clickhouse.ts | 7 +- .../data-access-object-dynamodb.ts | 4 + .../data-access-object-elasticsearch.ts | 4 + .../data-access-object-ibmdb2.ts | 4 + .../data-access-object-mongodb.ts | 4 + .../data-access-object-mssql.ts | 7 +- .../data-access-object-mysql.ts | 10 +- .../data-access-object-oracle.ts | 7 +- .../data-access-object-postgres.ts | 7 +- .../data-access-object-redis.ts | 4 + .../data-access-object-agent.interface.ts | 6 +- .../data-access-object.interface.ts | 6 +- 21 files changed, 632 insertions(+), 14 deletions(-) create mode 100644 backend/src/entities/table/use-cases/get-table-structure-without-cache.use.case.ts diff --git a/backend/src/common/data-injection.tokens.ts b/backend/src/common/data-injection.tokens.ts index 2f485053d..f55f49a74 100644 --- a/backend/src/common/data-injection.tokens.ts +++ b/backend/src/common/data-injection.tokens.ts @@ -92,6 +92,7 @@ export enum UseCaseType { FIND_TABLES_IN_CONNECTION_V2 = 'FIND_TABLES_IN_CONNECTION_V2', GET_ALL_TABLE_ROWS = 'GET_ALL_TABLE_ROWS', GET_TABLE_STRUCTURE = 'GET_TABLE_STRUCTURE', + GET_TABLE_STRUCTURE_WITHOUT_CACHE = 'GET_TABLE_STRUCTURE_WITHOUT_CACHE', ADD_ROW_IN_TABLE = 'ADD_ROW_IN_TABLE', UPDATE_ROW_IN_TABLE = 'UPDATE_ROW_IN_TABLE', BULK_UPDATE_ROWS_IN_TABLE = 'BULK_UPDATE_ROWS_IN_TABLE', diff --git a/backend/src/entities/table/table.controller.ts b/backend/src/entities/table/table.controller.ts index 8b7d3bbca..dd3889902 100644 --- a/backend/src/entities/table/table.controller.ts +++ b/backend/src/entities/table/table.controller.ts @@ -69,6 +69,7 @@ import { IGetRowByPrimaryKey, IGetTableRows, IGetTableStructure, + IGetTableStructureWithoutCache, IImportCSVFinTable, IUpdateRowInTable, } from './use-cases/table-use-cases.interface.js'; @@ -90,6 +91,8 @@ export class TableController { private readonly getTableRowsUseCase: IGetTableRows, @Inject(UseCaseType.GET_TABLE_STRUCTURE) private readonly getTableStructureUseCase: IGetTableStructure, + @Inject(UseCaseType.GET_TABLE_STRUCTURE_WITHOUT_CACHE) + private readonly getTableStructureWithoutCacheUseCase: IGetTableStructureWithoutCache, @Inject(UseCaseType.ADD_ROW_IN_TABLE) private readonly addRowInTableUseCase: IAddRowInTable, @Inject(UseCaseType.UPDATE_ROW_IN_TABLE) @@ -344,6 +347,42 @@ export class TableController { return await this.getTableStructureUseCase.execute(inputData, InTransactionEnum.OFF); } + @ApiOperation({ + summary: 'Get table structure without cache. API+', + description: + 'Return table structure fetched directly from the database, bypassing the structure cache. Support access with api key.', + }) + @ApiResponse({ + status: 200, + description: 'Table structure found.', + type: TableStructureDs, + }) + @ApiQuery({ name: 'tableName', required: true }) + @UseGuards(TableReadGuard) + @Get('/table/structure/no-cache/:connectionId') + async getTableStructureWithoutCache( + @QueryTableName() tableName: string, + @UserId() userId: string, + @SlugUuid('connectionId') connectionId: string, + @MasterPassword() masterPwd: string, + ): Promise { + if (!connectionId) { + throw new HttpException( + { + message: Messages.CONNECTION_ID_MISSING, + }, + HttpStatus.BAD_REQUEST, + ); + } + const inputData: GetTableStructureDs = { + connectionId: connectionId, + masterPwd: masterPwd, + tableName: tableName, + userId: userId, + }; + return await this.getTableStructureWithoutCacheUseCase.execute(inputData, InTransactionEnum.OFF); + } + @ApiOperation({ summary: 'Add new row in table. API+', description: 'Add new row in table. Support access with api key.', diff --git a/backend/src/entities/table/table.module.ts b/backend/src/entities/table/table.module.ts index 75ad84d90..1261bb57f 100644 --- a/backend/src/entities/table/table.module.ts +++ b/backend/src/entities/table/table.module.ts @@ -26,6 +26,7 @@ import { FindTablesInConnectionV2UseCase } from './use-cases/find-tables-in-conn import { GetRowByPrimaryKeyUseCase } from './use-cases/get-row-by-primary-key.use.case.js'; import { GetTableRowsUseCase } from './use-cases/get-table-rows.use.case.js'; import { GetTableStructureUseCase } from './use-cases/get-table-structure.use.case.js'; +import { GetTableStructureWithoutCacheUseCase } from './use-cases/get-table-structure-without-cache.use.case.js'; import { ImportCSVInTableUseCase } from './use-cases/import-csv-in-table-user.case.js'; import { UpdateRowInTableUseCase } from './use-cases/update-row-in-table.use.case.js'; @@ -67,6 +68,10 @@ import { UpdateRowInTableUseCase } from './use-cases/update-row-in-table.use.cas provide: UseCaseType.GET_TABLE_STRUCTURE, useClass: GetTableStructureUseCase, }, + { + provide: UseCaseType.GET_TABLE_STRUCTURE_WITHOUT_CACHE, + useClass: GetTableStructureWithoutCacheUseCase, + }, { provide: UseCaseType.ADD_ROW_IN_TABLE, useClass: AddRowInTableUseCase, @@ -115,6 +120,7 @@ export class TableModule { { path: '/table/rows/:connectionId', method: RequestMethod.GET }, { path: '/table/rows/find/:connectionId', method: RequestMethod.POST }, { path: '/table/structure/:connectionId', method: RequestMethod.GET }, + { path: '/table/structure/no-cache/:connectionId', method: RequestMethod.GET }, { path: '/table/row/:connectionId', method: RequestMethod.POST }, { path: '/table/row/:connectionId', method: RequestMethod.PUT }, { path: '/table/row/:connectionId', method: RequestMethod.DELETE }, diff --git a/backend/src/entities/table/use-cases/get-table-structure-without-cache.use.case.ts b/backend/src/entities/table/use-cases/get-table-structure-without-cache.use.case.ts new file mode 100644 index 000000000..4ea9b4e6b --- /dev/null +++ b/backend/src/entities/table/use-cases/get-table-structure-without-cache.use.case.ts @@ -0,0 +1,109 @@ +import { HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common'; +import { getDataAccessObject } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/create-data-access-object.js'; +import { ForeignKeyWithAutocompleteColumnsDS } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/data-structures/foreign-key-with-autocomplete-columns.ds.js'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { ExceptionOperations } from '../../../exceptions/custom-exceptions/exception-operation.js'; +import { UnknownSQLException } from '../../../exceptions/custom-exceptions/unknown-sql-exception.js'; +import { Messages } from '../../../exceptions/text/messages.js'; +import { CedarPermissionsService } from '../../cedar-authorization/cedar-permissions.service.js'; +import { buildFoundTableWidgetDs } from '../../widget/utils/build-found-table-widget-ds.js'; +import { GetTableStructureDs } from '../application/data-structures/get-table-structure-ds.js'; +import { TableStructureDs } from '../table-datastructures.js'; +import { attachForeignColumnNames } from '../utils/attach-foreign-column-names.util.js'; +import { extractForeignKeysFromWidgets } from '../utils/extract-foreign-keys-from-widgets.util.js'; +import { filterForeignKeysByReadPermission } from '../utils/filter-foreign-keys-by-permission.util.js'; +import { formFullTableStructure } from '../utils/form-full-table-structure.js'; +import { getUserEmailForAgent, validateConnection } from '../utils/validate-connection.util.js'; +import { IGetTableStructureWithoutCache } from './table-use-cases.interface.js'; + +@Injectable() +export class GetTableStructureWithoutCacheUseCase + extends AbstractUseCase + implements IGetTableStructureWithoutCache +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + private readonly cedarPermissions: CedarPermissionsService, + ) { + super(); + } + + protected async implementation(inputData: GetTableStructureDs): Promise { + const { connectionId, masterPwd, tableName, userId } = inputData; + const foundConnection = await this._dbContext.connectionRepository.findAndDecryptConnection( + connectionId, + masterPwd, + ); + validateConnection(foundConnection); + + try { + const dao = getDataAccessObject(foundConnection); + const foundTalesInConnection = await dao.getTablesFromDB(); + if (!foundTalesInConnection.find((el) => el.tableName === tableName)) { + throw new HttpException( + { + message: Messages.TABLE_NOT_FOUND, + }, + HttpStatus.BAD_REQUEST, + ); + } + const userEmail = await getUserEmailForAgent(foundConnection, userId, this._dbContext.userRepository); + + // eslint-disable-next-line prefer-const + let [tableSettings, personalTableSettings, tablePrimaryColumns, tableForeignKeys, tableStructure, tableWidgets] = + await Promise.all([ + this._dbContext.tableSettingsRepository.findTableSettings(connectionId, tableName), + this._dbContext.personalTableSettingsRepository.findUserTableSettings(userId, connectionId, tableName), + dao.getTablePrimaryColumns(tableName, userEmail), + dao.getTableForeignKeys(tableName, userEmail), + dao.getTableStructureWithoutCache(tableName, userEmail), + this._dbContext.tableWidgetsRepository.findTableWidgets(connectionId, tableName), + ]); + const foreignKeysFromWidgets = extractForeignKeysFromWidgets(tableWidgets); + + tableForeignKeys = tableForeignKeys.concat(foreignKeysFromWidgets); + let transformedTableForeignKeys: Array = []; + tableForeignKeys = await filterForeignKeysByReadPermission( + tableForeignKeys, + userId, + connectionId, + masterPwd, + this.cedarPermissions, + ); + + if (tableForeignKeys && tableForeignKeys.length > 0) { + transformedTableForeignKeys = await Promise.all( + tableForeignKeys.map((el) => + attachForeignColumnNames( + el, + userEmail, + connectionId, + dao, + this._dbContext.tableSettingsRepository.findTableSettings.bind(this._dbContext.tableSettingsRepository), + ).catch(() => el as ForeignKeyWithAutocompleteColumnsDS), + ), + ); + } + const readonly_fields = tableSettings?.readonly_fields?.length > 0 ? tableSettings.readonly_fields : []; + const formedTableStructure = formFullTableStructure(tableStructure, tableSettings); + return { + structure: formedTableStructure, + primaryColumns: tablePrimaryColumns, + foreignKeys: transformedTableForeignKeys, + readonly_fields: readonly_fields, + table_widgets: tableWidgets?.length > 0 ? tableWidgets.map((widget) => buildFoundTableWidgetDs(widget)) : [], + list_fields: personalTableSettings?.list_fields ? personalTableSettings.list_fields : [], + display_name: tableSettings?.display_name ? tableSettings.display_name : null, + excluded_fields: tableSettings?.excluded_fields ? tableSettings.excluded_fields : [], + }; + } catch (e) { + if (e instanceof HttpException) { + throw e; + } + throw new UnknownSQLException(e.message, ExceptionOperations.FAILED_TO_GET_TABLE_STRUCTURE); + } + } +} diff --git a/backend/src/entities/table/use-cases/table-use-cases.interface.ts b/backend/src/entities/table/use-cases/table-use-cases.interface.ts index 55e387210..5de564c88 100644 --- a/backend/src/entities/table/use-cases/table-use-cases.interface.ts +++ b/backend/src/entities/table/use-cases/table-use-cases.interface.ts @@ -34,6 +34,10 @@ export interface IGetTableStructure { execute(inputData: GetTableStructureDs, inTransaction: InTransactionEnum): Promise; } +export interface IGetTableStructureWithoutCache { + execute(inputData: GetTableStructureDs, inTransaction: InTransactionEnum): Promise; +} + export interface IAddRowInTable { execute(inputData: AddRowInTableDs, inTransaction: InTransactionEnum): Promise; } diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-mysql-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-mysql-e2e.test.ts index 8a78c7d74..3de807f4e 100644 --- a/backend/test/ava-tests/non-saas-tests/non-saas-table-mysql-e2e.test.ts +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-mysql-e2e.test.ts @@ -2294,6 +2294,206 @@ test.serial(`${currentTest} should throw an exception when tableName passed in r t.is(message, Messages.TABLE_NOT_FOUND); }); +currentTest = 'GET /table/structure/no-cache/:slug'; +test.serial(`${currentTest} should return table structure without using cache`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToMySQL; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestTable(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + + const getTableStructure = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${testTableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(getTableStructure.status, 200); + const getTableStructureRO = JSON.parse(getTableStructure.text); + + t.is(typeof getTableStructureRO, 'object'); + t.is(typeof getTableStructureRO.structure, 'object'); + t.is(getTableStructureRO.structure.length, 5); + + for (const element of getTableStructureRO.structure) { + t.is(Object.hasOwn(element, 'column_name'), true); + t.is(Object.hasOwn(element, 'column_default'), true); + t.is(Object.hasOwn(element, 'data_type'), true); + t.is(Object.hasOwn(element, 'isExcluded'), true); + t.is(Object.hasOwn(element, 'isSearched'), true); + } + + t.is(Object.hasOwn(getTableStructureRO, 'primaryColumns'), true); + t.is(Object.hasOwn(getTableStructureRO, 'foreignKeys'), true); + + for (const element of getTableStructureRO.primaryColumns) { + t.is(Object.hasOwn(element, 'column_name'), true); + t.is(Object.hasOwn(element, 'data_type'), true); + } + + for (const element of getTableStructureRO.foreignKeys) { + t.is(Object.hasOwn(element, 'referenced_column_name'), true); + t.is(Object.hasOwn(element, 'referenced_table_name'), true); + t.is(Object.hasOwn(element, 'constraint_name'), true); + t.is(Object.hasOwn(element, 'column_name'), true); + } +}); + +test.serial(`${currentTest} should return the same payload shape as cached endpoint for the same table`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToMySQL; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestTable(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + + const cachedResponse = await request(app.getHttpServer()) + .get(`/table/structure/${createConnectionRO.id}?tableName=${testTableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + const noCacheResponse = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${testTableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(cachedResponse.status, 200); + t.is(noCacheResponse.status, 200); + + const cachedRO = JSON.parse(cachedResponse.text); + const noCacheRO = JSON.parse(noCacheResponse.text); + + t.deepEqual( + noCacheRO.structure.map((column) => column.column_name).sort(), + cachedRO.structure.map((column) => column.column_name).sort(), + ); + t.is(noCacheRO.structure.length, cachedRO.structure.length); + t.deepEqual(Object.keys(noCacheRO).sort(), Object.keys(cachedRO).sort()); +}); + +test.serial(`${currentTest} should reject access when connection id is not passed in request`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToMySQL; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestTable(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + createConnectionRO.id = ''; + const getTableStructure = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${testTableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(getTableStructure.status, 403); + const { message } = JSON.parse(getTableStructure.text); + t.is(message, Messages.DONT_HAVE_PERMISSIONS); +}); + +test.serial(`${currentTest} should return 403 when connection id is incorrect`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToMySQL; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestTable(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + + createConnectionRO.id = faker.string.uuid(); + const getTableStructure = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${testTableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(getTableStructure.status, 403); + const { message } = JSON.parse(getTableStructure.text); + t.is(message, Messages.DONT_HAVE_PERMISSIONS); +}); + +test.serial(`${currentTest} should throw an exception when tableName is not passed in request`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToMySQL; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestTable(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + const tableName = ''; + const getTableStructure = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${tableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(getTableStructure.status, 400); + const { message } = JSON.parse(getTableStructure.text); + t.is(message, Messages.TABLE_NAME_MISSING); +}); + +test.serial(`${currentTest} should throw an exception when tableName is incorrect`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToMySQL; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestTable(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + const tableName = faker.lorem.words(1); + const getTableStructure = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${tableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(getTableStructure.status, 400); + const { message } = JSON.parse(getTableStructure.text); + t.is(message, Messages.TABLE_NOT_FOUND); +}); + currentTest = 'POST /table/row/:slug'; test.serial(`${currentTest} should add row in table and return result`, async (t) => { diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-postgres-schema-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-postgres-schema-e2e.test.ts index 28847e959..3033efbbc 100644 --- a/backend/test/ava-tests/non-saas-tests/non-saas-table-postgres-schema-e2e.test.ts +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-postgres-schema-e2e.test.ts @@ -2295,6 +2295,206 @@ test.serial(`${currentTest} should throw an exception when tableName passed in r t.is(message, Messages.TABLE_NOT_FOUND); }); +currentTest = 'GET /table/structure/no-cache/:slug'; +test.serial(`${currentTest} should return table structure without using cache`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestPostgresTableWithSchema(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + + const getTableStructure = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${testTableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(getTableStructure.status, 200); + const getTableStructureRO = JSON.parse(getTableStructure.text); + + t.is(typeof getTableStructureRO, 'object'); + t.is(typeof getTableStructureRO.structure, 'object'); + t.is(getTableStructureRO.structure.length, 5); + + for (const element of getTableStructureRO.structure) { + t.is(Object.hasOwn(element, 'column_name'), true); + t.is(Object.hasOwn(element, 'column_default'), true); + t.is(Object.hasOwn(element, 'data_type'), true); + t.is(Object.hasOwn(element, 'isExcluded'), true); + t.is(Object.hasOwn(element, 'isSearched'), true); + } + + t.is(Object.hasOwn(getTableStructureRO, 'primaryColumns'), true); + t.is(Object.hasOwn(getTableStructureRO, 'foreignKeys'), true); + + for (const element of getTableStructureRO.primaryColumns) { + t.is(Object.hasOwn(element, 'column_name'), true); + t.is(Object.hasOwn(element, 'data_type'), true); + } + + for (const element of getTableStructureRO.foreignKeys) { + t.is(Object.hasOwn(element, 'referenced_column_name'), true); + t.is(Object.hasOwn(element, 'referenced_table_name'), true); + t.is(Object.hasOwn(element, 'constraint_name'), true); + t.is(Object.hasOwn(element, 'column_name'), true); + } +}); + +test.serial(`${currentTest} should return the same payload shape as cached endpoint for the same table`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestPostgresTableWithSchema(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + + const cachedResponse = await request(app.getHttpServer()) + .get(`/table/structure/${createConnectionRO.id}?tableName=${testTableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + const noCacheResponse = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${testTableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(cachedResponse.status, 200); + t.is(noCacheResponse.status, 200); + + const cachedRO = JSON.parse(cachedResponse.text); + const noCacheRO = JSON.parse(noCacheResponse.text); + + t.deepEqual( + noCacheRO.structure.map((column) => column.column_name).sort(), + cachedRO.structure.map((column) => column.column_name).sort(), + ); + t.is(noCacheRO.structure.length, cachedRO.structure.length); + t.deepEqual(Object.keys(noCacheRO).sort(), Object.keys(cachedRO).sort()); +}); + +test.serial(`${currentTest} should reject access when connection id is not passed in request`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestPostgresTableWithSchema(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + createConnectionRO.id = ''; + const getTableStructure = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${testTableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(getTableStructure.status, 403); + const { message } = JSON.parse(getTableStructure.text); + t.is(message, Messages.DONT_HAVE_PERMISSIONS); +}); + +test.serial(`${currentTest} should return 403 when connection id is incorrect`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestPostgresTableWithSchema(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + + createConnectionRO.id = faker.string.uuid(); + const getTableStructure = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${testTableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(getTableStructure.status, 403); + const { message } = JSON.parse(getTableStructure.text); + t.is(message, Messages.DONT_HAVE_PERMISSIONS); +}); + +test.serial(`${currentTest} should throw an exception when tableName is not passed in request`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestPostgresTableWithSchema(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + const tableName = ''; + const getTableStructure = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${tableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(getTableStructure.status, 400); + const { message } = JSON.parse(getTableStructure.text); + t.is(message, Messages.TABLE_NAME_MISSING); +}); + +test.serial(`${currentTest} should throw an exception when tableName is incorrect`, async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName } = await createTestPostgresTableWithSchema(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + const tableName = faker.lorem.words(1); + const getTableStructure = await request(app.getHttpServer()) + .get(`/table/structure/no-cache/${createConnectionRO.id}?tableName=${tableName}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(getTableStructure.status, 400); + const { message } = JSON.parse(getTableStructure.text); + t.is(message, Messages.TABLE_NOT_FOUND); +}); + currentTest = 'POST /table/row/:slug'; test.serial(`${currentTest} should add row in table and return result`, async (t) => { diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-agent.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-agent.ts index 6668c9fb5..c3aa5f3e1 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-agent.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-agent.ts @@ -385,14 +385,20 @@ export class DataAccessObjectAgent implements IDataAccessObjectAgent { } public async getTableStructure(tableName: string, userEmail: string): Promise { - const jwtAuthToken = this.generateJWT(this.connection.token); - axios.defaults.headers.common.Authorization = `Bearer ${jwtAuthToken}`; - const cachedTableStructure = LRUStorage.getTableStructureCache(this.connection, tableName); if (cachedTableStructure) { return cachedTableStructure; } + const commandResult = await this.getTableStructureWithoutCache(tableName, userEmail); + LRUStorage.setTableStructureCache(this.connection, tableName, commandResult); + return commandResult; + } + + public async getTableStructureWithoutCache(tableName: string, userEmail: string): Promise { + const jwtAuthToken = this.generateJWT(this.connection.token); + axios.defaults.headers.common.Authorization = `Bearer ${jwtAuthToken}`; + return this.executeWithRetry(async () => { try { const { data: { commandResult } = {} } = await axios.post(this.serverAddress, { @@ -409,7 +415,6 @@ export class DataAccessObjectAgent implements IDataAccessObjectAgent { throw new Error(ERROR_MESSAGES.NO_DATA_RETURNED_FROM_AGENT); } - LRUStorage.setTableStructureCache(this.connection, tableName, commandResult); return commandResult; } catch (e) { if (axios.isAxiosError(e)) { diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-cassandra.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-cassandra.ts index 1c3ee0344..fe42f75ce 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-cassandra.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-cassandra.ts @@ -545,6 +545,10 @@ export class DataAccessObjectCassandra extends BasicDataAccessObject implements } } + public async getTableStructureWithoutCache(tableName: string): Promise> { + return this.getTableStructure(tableName); + } + public async testConnect(): Promise { try { const client = await this.getCassandraClient(); diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts index 1807efecd..dd46a9d37 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts @@ -400,6 +400,12 @@ export class DataAccessObjectClickHouse extends BasicDataAccessObject implements return cachedTableStructure; } + const structure = await this.getTableStructureWithoutCache(tableName); + LRUStorage.setTableStructureCache(this.connection, tableName, structure); + return structure; + } + + public async getTableStructureWithoutCache(tableName: string): Promise> { const client = await this.getClickHouseClient(); try { const database = this.connection.database || 'default'; @@ -440,7 +446,6 @@ export class DataAccessObjectClickHouse extends BasicDataAccessObject implements extra: this.buildExtraInfo(column.is_in_primary_key, column.default_expression), })); - LRUStorage.setTableStructureCache(this.connection, tableName, structure); return structure; } finally { await client.close(); diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-dynamodb.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-dynamodb.ts index 9927fe1f8..a4099097e 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-dynamodb.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-dynamodb.ts @@ -481,6 +481,10 @@ export class DataAccessObjectDynamoDB extends BasicDataAccessObject implements I return this.getTableStructureOrReturnPrimaryKeysIfNothingToScan(tableName); } + public async getTableStructureWithoutCache(tableName: string): Promise> { + return this.getTableStructureOrReturnPrimaryKeysIfNothingToScan(tableName); + } + public async testConnect(): Promise { const { dynamoDb } = this.getDynamoDb(); try { diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-elasticsearch.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-elasticsearch.ts index 4615b9af4..cf3a3a2e4 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-elasticsearch.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-elasticsearch.ts @@ -456,6 +456,10 @@ export class DataAccessObjectElasticsearch extends BasicDataAccessObject impleme return structure; } + public async getTableStructureWithoutCache(tableName: string): Promise> { + return this.getTableStructure(tableName); + } + public async testConnect(): Promise { const client = this.getElasticClient(); diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts index e6c1427d9..41d066f03 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts @@ -498,6 +498,10 @@ ORDER BY }); } + public async getTableStructureWithoutCache(tableName: string): Promise { + return this.getTableStructure(tableName); + } + public async testConnect(): Promise { if (!this.connection.id) { this.connection.id = nanoid(6); diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-mongodb.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-mongodb.ts index 5d7c814cf..9d6a44386 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-mongodb.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-mongodb.ts @@ -355,6 +355,10 @@ export class DataAccessObjectMongo extends BasicDataAccessObject implements IDat return await this.getTableStructureOrReturnPrimaryKeysIfNothingToScan(tableName); } + public async getTableStructureWithoutCache(tableName: string): Promise { + return await this.getTableStructureOrReturnPrimaryKeysIfNothingToScan(tableName); + } + public async testConnect(): Promise { if (!this.connection.id) { this.connection.id = nanoid(6); diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-mssql.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-mssql.ts index 0b64730cb..446b30948 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-mssql.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-mssql.ts @@ -353,6 +353,12 @@ WHERE TABLE_TYPE = 'VIEW' if (cachedTableStructure) { return cachedTableStructure; } + const structureColumnsInLowercase = await this.getTableStructureWithoutCache(tableName); + LRUStorage.setTableStructureCache(this.connection, tableName, structureColumnsInLowercase); + return structureColumnsInLowercase; + } + + public async getTableStructureWithoutCache(tableName: string): Promise { const { database } = this.connection; const knex = await this.configureKnex(); const schema = await this.getSchemaNameWithoutBrackets(tableName); @@ -387,7 +393,6 @@ WHERE TABLE_TYPE = 'VIEW' } return column; }); - LRUStorage.setTableStructureCache(this.connection, tableName, structureColumnsInLowercase as TableStructureDS[]); return structureColumnsInLowercase as TableStructureDS[]; } diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-mysql.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-mysql.ts index a69784039..9b23f5ef1 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-mysql.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-mysql.ts @@ -395,6 +395,14 @@ export class DataAccessObjectMysql extends BasicDataAccessObject implements IDat return cachedTableStructure; } + const structureColumnsInLowercase = await this.getTableStructureWithoutCache(tableName); + + LRUStorage.setTableStructureCache(this.connection, tableName, structureColumnsInLowercase); + + return structureColumnsInLowercase; + } + + public async getTableStructureWithoutCache(tableName: string): Promise { const knex = await this.configureKnex(); const { database } = this.connection; @@ -433,8 +441,6 @@ export class DataAccessObjectMysql extends BasicDataAccessObject implements IDat element.character_maximum_length ?? getNumbersFromString(element.column_type) ?? null; }); - LRUStorage.setTableStructureCache(this.connection, tableName, structureColumnsInLowercase as TableStructureDS[]); - return structureColumnsInLowercase as TableStructureDS[]; } diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts index 1a784c8bc..a84ab36ee 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts @@ -515,6 +515,12 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa if (cachedTableStructure) { return cachedTableStructure; } + const resultColumns = await this.getTableStructureWithoutCache(tableName); + LRUStorage.setTableStructureCache(this.connection, tableName, resultColumns); + return resultColumns; + } + + public async getTableStructureWithoutCache(tableName: string): Promise { const knex = await this.configureKnex(); const schema = this.connection.schema ?? this.connection.username.toUpperCase(); const structureColumns = await knex @@ -534,7 +540,6 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa } return objectKeysToLowercase(column); }) as TableStructureDS[]; - LRUStorage.setTableStructureCache(this.connection, tableName, resultColumns); return resultColumns; } diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-postgres.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-postgres.ts index e11f55334..652978190 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-postgres.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-postgres.ts @@ -366,6 +366,12 @@ export class DataAccessObjectPostgres extends BasicDataAccessObject implements I if (cachedTableStructure) { return cachedTableStructure; } + const result = await this.getTableStructureWithoutCache(tableName); + LRUStorage.setTableStructureCache(this.connection, tableName, result); + return result; + } + + public async getTableStructureWithoutCache(tableName: string): Promise { const knex = await this.configureKnex(); let result = await knex('information_schema.columns') .select( @@ -435,7 +441,6 @@ export class DataAccessObjectPostgres extends BasicDataAccessObject implements I } } - LRUStorage.setTableStructureCache(this.connection, tableName, result); return result; } diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-redis.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-redis.ts index 94ec0702f..822e17109 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-redis.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-redis.ts @@ -1130,6 +1130,10 @@ export class DataAccessObjectRedis extends BasicDataAccessObject implements IDat return this.getPrefixedTableStructure(tableName); } + public async getTableStructureWithoutCache(tableName: string): Promise> { + return this.getTableStructure(tableName); + } + private getStandaloneCollectionStructure(): Array { return [ { diff --git a/shared-code/src/shared/interfaces/data-access-object-agent.interface.ts b/shared-code/src/shared/interfaces/data-access-object-agent.interface.ts index 683a7e7bf..eaedfebd5 100644 --- a/shared-code/src/shared/interfaces/data-access-object-agent.interface.ts +++ b/shared-code/src/shared/interfaces/data-access-object-agent.interface.ts @@ -1,15 +1,15 @@ +import { Stream } from 'node:stream'; import { AutocompleteFieldsDS } from '../../data-access-layer/shared/data-structures/autocomplete-fields.ds.js'; import { FilteringFieldsDS } from '../../data-access-layer/shared/data-structures/filtering-fields.ds.js'; import { ForeignKeyDS } from '../../data-access-layer/shared/data-structures/foreign-key.ds.js'; import { FoundRowsDS } from '../../data-access-layer/shared/data-structures/found-rows.ds.js'; import { PrimaryKeyDS } from '../../data-access-layer/shared/data-structures/primary-key.ds.js'; import { ReferencedTableNamesAndColumnsDS } from '../../data-access-layer/shared/data-structures/referenced-table-names-columns.ds.js'; +import { TableDS } from '../../data-access-layer/shared/data-structures/table.ds.js'; import { TableSettingsDS } from '../../data-access-layer/shared/data-structures/table-settings.ds.js'; import { TableStructureDS } from '../../data-access-layer/shared/data-structures/table-structure.ds.js'; -import { TableDS } from '../../data-access-layer/shared/data-structures/table.ds.js'; import { TestConnectionResultDS } from '../../data-access-layer/shared/data-structures/test-result-connection.ds.js'; import { ValidateTableSettingsDS } from '../../data-access-layer/shared/data-structures/validate-table-settings.ds.js'; -import { Stream } from 'node:stream'; export interface IDataAccessObjectAgent { addRowInTable( @@ -66,6 +66,8 @@ export interface IDataAccessObjectAgent { getTableStructure(tableName: string, userEmail: string): Promise>; + getTableStructureWithoutCache(tableName: string, userEmail: string): Promise>; + testConnect(): Promise; updateRowInTable( diff --git a/shared-code/src/shared/interfaces/data-access-object.interface.ts b/shared-code/src/shared/interfaces/data-access-object.interface.ts index 8326660d4..2698454aa 100644 --- a/shared-code/src/shared/interfaces/data-access-object.interface.ts +++ b/shared-code/src/shared/interfaces/data-access-object.interface.ts @@ -1,15 +1,15 @@ +import { Stream } from 'node:stream'; import { AutocompleteFieldsDS } from '../../data-access-layer/shared/data-structures/autocomplete-fields.ds.js'; import { FilteringFieldsDS } from '../../data-access-layer/shared/data-structures/filtering-fields.ds.js'; import { ForeignKeyDS } from '../../data-access-layer/shared/data-structures/foreign-key.ds.js'; import { FoundRowsDS } from '../../data-access-layer/shared/data-structures/found-rows.ds.js'; import { PrimaryKeyDS } from '../../data-access-layer/shared/data-structures/primary-key.ds.js'; import { ReferencedTableNamesAndColumnsDS } from '../../data-access-layer/shared/data-structures/referenced-table-names-columns.ds.js'; +import { TableDS } from '../../data-access-layer/shared/data-structures/table.ds.js'; import { TableSettingsDS } from '../../data-access-layer/shared/data-structures/table-settings.ds.js'; import { TableStructureDS } from '../../data-access-layer/shared/data-structures/table-structure.ds.js'; import { TestConnectionResultDS } from '../../data-access-layer/shared/data-structures/test-result-connection.ds.js'; import { ValidateTableSettingsDS } from '../../data-access-layer/shared/data-structures/validate-table-settings.ds.js'; -import { TableDS } from '../../data-access-layer/shared/data-structures/table.ds.js'; -import { Stream } from 'node:stream'; export interface IDataAccessObject { addRowInTable(tableName: string, row: Record): Promise | number>; @@ -54,6 +54,8 @@ export interface IDataAccessObject { getTableStructure(tableName: string): Promise>; + getTableStructureWithoutCache(tableName: string): Promise>; + testConnect(): Promise; updateRowInTable(