From 7e89be2b11f299b7b23f3ed9baf37562dda865e2 Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Mon, 20 May 2024 15:44:24 +0200 Subject: [PATCH 1/9] Introduce foreign table service --- .../foreign-table/foreign-table.module.ts | 17 + .../foreign-table/foreign-table.service.ts | 127 +++++++ .../get-foreign-table-column-name.util.ts | 0 .../remote-table/remote-table.module.ts | 6 +- .../remote-table/remote-table.resolver.ts | 2 +- .../remote-table/remote-table.service.ts | 353 ++++++------------ .../utils/fetch-table-columns.util.ts | 25 ++ 7 files changed, 289 insertions(+), 241 deletions(-) create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts rename packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/{ => foreign-table}/utils/get-foreign-table-column-name.util.ts (100%) create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util.ts diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module.ts new file mode 100644 index 000000000000..c40aeb4201f8 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module.ts @@ -0,0 +1,17 @@ +import { Module } from '@nestjs/common'; + +import { ForeignTableService } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service'; +import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; +import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; +import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; + +@Module({ + imports: [ + WorkspaceMigrationModule, + WorkspaceMigrationRunnerModule, + WorkspaceDataSourceModule, + ], + providers: [ForeignTableService], + exports: [ForeignTableService], +}) +export class ForeignTableModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts new file mode 100644 index 000000000000..c5973ea8c93c --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts @@ -0,0 +1,127 @@ +import { BadRequestException, Injectable } from '@nestjs/common'; + +import { + RemoteServerEntity, + RemoteServerType, +} from 'src/engine/metadata-modules/remote-server/remote-server.entity'; +import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util'; +import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; +import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; +import { + ReferencedTable, + WorkspaceMigrationTableActionType, + WorkspaceMigrationForeignColumnDefinition, + WorkspaceMigrationForeignTable, +} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service'; +import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; +import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; + +@Injectable() +export class ForeignTableService { + constructor( + private readonly workspaceMigrationService: WorkspaceMigrationService, + private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, + private readonly workspaceDataSourceService: WorkspaceDataSourceService, + ) {} + + public async fetchForeignTableNamesWithinWorkspace( + workspaceId: string, + foreignDataWrapperId: string, + ): Promise { + const workspaceDataSource = + await this.workspaceDataSourceService.connectToWorkspaceDataSource( + workspaceId, + ); + + return ( + await workspaceDataSource.query( + `SELECT foreign_table_name, foreign_server_name FROM information_schema.foreign_tables WHERE foreign_server_name = '${foreignDataWrapperId}'`, + ) + ).map((foreignTable) => foreignTable.foreign_table_name); + } + + public async createForeignTable( + workspaceId: string, + localTableName: string, + remoteServer: RemoteServerEntity, + distantTableName: string, + distantTableColumns: PostgresTableSchemaColumn[], + ) { + const referencedTable: ReferencedTable = this.buildReferencedTable( + remoteServer, + distantTableName, + ); + + const workspaceMigration = + await this.workspaceMigrationService.createCustomMigration( + generateMigrationName(`create-foreign-table-${localTableName}`), + workspaceId, + [ + { + name: localTableName, + action: WorkspaceMigrationTableActionType.CREATE_FOREIGN_TABLE, + foreignTable: { + columns: distantTableColumns.map( + (column) => + ({ + columnName: getForeignTableColumnName(column.columnName), + columnType: column.dataType, + distantColumnName: column.columnName, + }) satisfies WorkspaceMigrationForeignColumnDefinition, + ), + referencedTable, + foreignDataWrapperId: remoteServer.foreignDataWrapperId, + } satisfies WorkspaceMigrationForeignTable, + }, + ], + ); + + // TODO: This should be done in a transaction. Waiting for a global refactoring of transaction management. + try { + await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( + workspaceId, + ); + } catch (exception) { + this.workspaceMigrationService.deleteById(workspaceMigration.id); + + throw new BadRequestException( + 'Could not create foreign table. The table may already exists or a column type may not be supported.', + ); + } + } + + public async deleteForeignTable(tableName: string, workspaceId: string) { + await this.workspaceMigrationService.createCustomMigration( + generateMigrationName(`drop-foreign-table-${tableName}`), + workspaceId, + [ + { + name: tableName, + action: WorkspaceMigrationTableActionType.DROP_FOREIGN_TABLE, + }, + ], + ); + + return this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( + workspaceId, + ); + } + + private buildReferencedTable( + remoteServer: RemoteServerEntity, + tableName: string, + ): ReferencedTable { + switch (remoteServer.foreignDataWrapperType) { + case RemoteServerType.POSTGRES_FDW: + return { + table_name: tableName, + schema_name: remoteServer.schema, + }; + case RemoteServerType.STRIPE_FDW: + return { object: tableName }; + default: + throw new BadRequestException('Foreign data wrapper not supported'); + } + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-foreign-table-column-name.util.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util.ts similarity index 100% rename from packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-foreign-table-column-name.util.ts rename to packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util.ts diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts index 56640927f526..f56846dc071e 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts @@ -6,13 +6,12 @@ import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; import { DistantTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.module'; +import { ForeignTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module'; import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity'; import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver'; import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service'; import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; -import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; -import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; @Module({ imports: [ @@ -25,9 +24,8 @@ import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/wor ObjectMetadataModule, FieldMetadataModule, WorkspaceCacheVersionModule, - WorkspaceMigrationModule, - WorkspaceMigrationRunnerModule, WorkspaceDataSourceModule, + ForeignTableModule, ], providers: [RemoteTableService, RemoteTableResolver], exports: [RemoteTableService], diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts index 8a7fa549a196..311d8bad3b83 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts @@ -19,7 +19,7 @@ export class RemoteTableResolver { @Args('input') input: FindManyRemoteTablesInput, @AuthWorkspace() { id: workspaceId }: Workspace, ) { - return this.remoteTableService.findDistantTablesByServerId( + return this.remoteTableService.findDistantTablesWithStatusByServerId( input.id, workspaceId, ); diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts index fef23b0020b7..5e6aefa21d0f 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts @@ -25,22 +25,15 @@ import { CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dto import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; import { camelCase } from 'src/utils/camel-case'; import { camelToTitleCase } from 'src/utils/camel-to-title-case'; -import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service'; -import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; -import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; -import { - ReferencedTable, - WorkspaceMigrationForeignColumnDefinition, - WorkspaceMigrationForeignTable, - WorkspaceMigrationTableActionType, -} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity'; import { getRemoteTableLocalName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util'; import { DistantTableService } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service'; import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table'; -import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-foreign-table-column-name.util'; +import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util'; import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; +import { fetchTableColumns } from 'src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util'; +import { ForeignTableService } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service'; export class RemoteTableService { private readonly logger = new Logger(RemoteTableService.name); @@ -57,12 +50,14 @@ export class RemoteTableService { private readonly objectMetadataService: ObjectMetadataService, private readonly fieldMetadataService: FieldMetadataService, private readonly distantTableService: DistantTableService, - private readonly workspaceMigrationService: WorkspaceMigrationService, - private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, + private readonly foreignTableService: ForeignTableService, private readonly workspaceDataSourceService: WorkspaceDataSourceService, ) {} - public async findDistantTablesByServerId(id: string, workspaceId: string) { + public async findDistantTablesWithStatusByServerId( + id: string, + workspaceId: string, + ) { const remoteServer = await this.remoteServerRepository.findOne({ where: { id, @@ -110,109 +105,6 @@ export class RemoteTableService { }); } - private async getDistantTablesWithUpdates({ - remoteServerSchema, - workspaceId, - remoteTables, - distantTables, - }: { - remoteServerSchema: string; - workspaceId: string; - remoteTables: RemoteTableEntity[]; - distantTables: DistantTables; - }) { - const schemaPendingUpdates = - await this.getSchemaUpdatesBetweenForeignAndDistantTables({ - workspaceId, - remoteTables, - distantTables, - }); - - const remoteTablesDistantNames = remoteTables.map( - (remoteTable) => remoteTable.distantTableName, - ); - - const distantTablesWithUpdates = Object.keys(distantTables).map( - (tableName) => ({ - name: tableName, - schema: remoteServerSchema, - status: remoteTablesDistantNames.includes(tableName) - ? RemoteTableStatus.SYNCED - : RemoteTableStatus.NOT_SYNCED, - schemaPendingUpdates: schemaPendingUpdates[tableName], - }), - ); - - const deletedTables = Object.entries(schemaPendingUpdates) - .filter(([_tableName, updates]) => - updates.includes(DistantTableUpdate.TABLE_DELETED), - ) - .map(([tableName, updates]) => ({ - name: tableName, - schema: remoteServerSchema, - status: RemoteTableStatus.SYNCED, - schemaPendingUpdates: updates, - })); - - return distantTablesWithUpdates.concat(deletedTables); - } - - private async getSchemaUpdatesBetweenForeignAndDistantTables({ - workspaceId, - remoteTables, - distantTables, - }: { - workspaceId: string; - remoteTables: RemoteTableEntity[]; - distantTables: DistantTables; - }): Promise<{ [tablename: string]: DistantTableUpdate[] }> { - const updates = {}; - - for (const remoteTable of remoteTables) { - const distantTable = distantTables[remoteTable.distantTableName]; - const tableName = remoteTable.distantTableName; - - if (!distantTable) { - updates[tableName] = [DistantTableUpdate.TABLE_DELETED]; - continue; - } - - const distantTableColumnNames = new Set( - distantTable.map((column) => - getForeignTableColumnName(column.columnName), - ), - ); - const foreignTableColumnNames = new Set( - ( - await this.fetchTableColumns(workspaceId, remoteTable.localTableName) - ).map((column) => column.columnName), - ); - - const columnsAdded = [...distantTableColumnNames].filter( - (columnName) => !foreignTableColumnNames.has(columnName), - ); - - const columnsDeleted = [...foreignTableColumnNames].filter( - (columnName) => !distantTableColumnNames.has(columnName), - ); - - if (columnsAdded.length > 0) { - updates[tableName] = [ - ...(updates[tableName] || []), - DistantTableUpdate.COLUMNS_ADDED, - ]; - } - if (columnsDeleted.length > 0) { - updates[tableName] = [ - ...(updates[tableName] || []), - DistantTableUpdate.COLUMNS_DELETED, - ]; - } - } - - return updates; - } - public async findRemoteTablesByServerId({ remoteServerId, workspaceId, @@ -295,11 +187,11 @@ export class RemoteTableService { throw new BadRequestException('Remote table must have an id column'); } - await this.createForeignTable( + await this.foreignTableService.createForeignTable( workspaceId, localTableName, - input, remoteServer, + input.name, distantTableColumns, ); @@ -379,7 +271,7 @@ export class RemoteTableService { remoteServer: RemoteServerEntity, ) { const currentForeignTableNames = - await this.fetchForeignTableNamesWithinWorkspace( + await this.foreignTableService.fetchForeignTableNamesWithinWorkspace( workspaceId, remoteServer.foreignDataWrapperId, ); @@ -400,18 +292,8 @@ export class RemoteTableService { ); } - await this.workspaceMigrationService.createCustomMigration( - generateMigrationName(`drop-foreign-table-${remoteTable.localTableName}`), - workspaceId, - [ - { - name: remoteTable.localTableName, - action: WorkspaceMigrationTableActionType.DROP_FOREIGN_TABLE, - }, - ], - ); - - await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( + await this.foreignTableService.deleteForeignTable( + remoteTable.localTableName, workspaceId, ); @@ -420,97 +302,6 @@ export class RemoteTableService { await this.workspaceCacheVersionService.incrementVersion(workspaceId); } - private async fetchForeignTableNamesWithinWorkspace( - workspaceId: string, - foreignDataWrapperId: string, - ): Promise { - const workspaceDataSource = - await this.workspaceDataSourceService.connectToWorkspaceDataSource( - workspaceId, - ); - - return ( - await workspaceDataSource.query( - `SELECT foreign_table_name, foreign_server_name FROM information_schema.foreign_tables WHERE foreign_server_name = '${foreignDataWrapperId}'`, - ) - ).map((foreignTable) => foreignTable.foreign_table_name); - } - - private async fetchTableColumns( - workspaceId: string, - tableName: string, - ): Promise { - const workspaceDataSource = - await this.workspaceDataSourceService.connectToWorkspaceDataSource( - workspaceId, - ); - - const schemaName = - this.workspaceDataSourceService.getSchemaName(workspaceId); - - const res = await workspaceDataSource.query( - `SELECT column_name, data_type, udt_name - FROM information_schema.columns - WHERE table_schema = '${schemaName}' AND table_name = '${tableName}'`, - ); - - return res.map((column) => ({ - columnName: column.column_name, - dataType: column.data_type, - udtName: column.udt_name, - })); - } - - private async createForeignTable( - workspaceId: string, - localTableName: string, - remoteTableInput: RemoteTableInput, - remoteServer: RemoteServerEntity, - distantTableColumns: PostgresTableSchemaColumn[], - ) { - const referencedTable: ReferencedTable = this.buildReferencedTable( - remoteServer, - remoteTableInput, - ); - - const workspaceMigration = - await this.workspaceMigrationService.createCustomMigration( - generateMigrationName(`create-foreign-table-${localTableName}`), - workspaceId, - [ - { - name: localTableName, - action: WorkspaceMigrationTableActionType.CREATE_FOREIGN_TABLE, - foreignTable: { - columns: distantTableColumns.map( - (column) => - ({ - columnName: getForeignTableColumnName(column.columnName), - columnType: column.dataType, - distantColumnName: column.columnName, - }) satisfies WorkspaceMigrationForeignColumnDefinition, - ), - referencedTable, - foreignDataWrapperId: remoteServer.foreignDataWrapperId, - } satisfies WorkspaceMigrationForeignTable, - }, - ], - ); - - // TODO: This should be done in a transaction. Waiting for a global refactoring of transaction management. - try { - await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( - workspaceId, - ); - } catch (exception) { - this.workspaceMigrationService.deleteById(workspaceMigration.id); - - throw new BadRequestException( - 'Could not create foreign table. The table may already exists or a column type may not be supported.', - ); - } - } - private async createRemoteTableMetadata( workspaceId: string, localTableBaseName: string, @@ -574,20 +365,110 @@ export class RemoteTableService { } } - private buildReferencedTable( - remoteServer: RemoteServerEntity, - remoteTableInput: RemoteTableInput, - ): ReferencedTable { - switch (remoteServer.foreignDataWrapperType) { - case RemoteServerType.POSTGRES_FDW: - return { - table_name: remoteTableInput.name, - schema_name: remoteServer.schema, - }; - case RemoteServerType.STRIPE_FDW: - return { object: remoteTableInput.name }; - default: - throw new BadRequestException('Foreign data wrapper not supported'); + private async getDistantTablesWithUpdates({ + remoteServerSchema, + workspaceId, + remoteTables, + distantTables, + }: { + remoteServerSchema: string; + workspaceId: string; + remoteTables: RemoteTableEntity[]; + distantTables: DistantTables; + }) { + const schemaPendingUpdates = + await this.getSchemaUpdatesBetweenForeignAndDistantTables({ + workspaceId, + remoteTables, + distantTables, + }); + + const remoteTablesDistantNames = remoteTables.map( + (remoteTable) => remoteTable.distantTableName, + ); + + const distantTablesWithUpdates = Object.keys(distantTables).map( + (tableName) => ({ + name: tableName, + schema: remoteServerSchema, + status: remoteTablesDistantNames.includes(tableName) + ? RemoteTableStatus.SYNCED + : RemoteTableStatus.NOT_SYNCED, + schemaPendingUpdates: schemaPendingUpdates[tableName], + }), + ); + + const deletedTables = Object.entries(schemaPendingUpdates) + .filter(([_tableName, updates]) => + updates.includes(DistantTableUpdate.TABLE_DELETED), + ) + .map(([tableName, updates]) => ({ + name: tableName, + schema: remoteServerSchema, + status: RemoteTableStatus.SYNCED, + schemaPendingUpdates: updates, + })); + + return distantTablesWithUpdates.concat(deletedTables); + } + + private async getSchemaUpdatesBetweenForeignAndDistantTables({ + workspaceId, + remoteTables, + distantTables, + }: { + workspaceId: string; + remoteTables: RemoteTableEntity[]; + distantTables: DistantTables; + }): Promise<{ [tablename: string]: DistantTableUpdate[] }> { + const updates = {}; + + for (const remoteTable of remoteTables) { + const distantTable = distantTables[remoteTable.distantTableName]; + const tableName = remoteTable.distantTableName; + + if (!distantTable) { + updates[tableName] = [DistantTableUpdate.TABLE_DELETED]; + continue; + } + + const distantTableColumnNames = new Set( + distantTable.map((column) => + getForeignTableColumnName(column.columnName), + ), + ); + const foreignTableColumnNames = new Set( + ( + await fetchTableColumns( + this.workspaceDataSourceService, + workspaceId, + remoteTable.localTableName, + ) + ).map((column) => column.columnName), + ); + + const columnsAdded = [...distantTableColumnNames].filter( + (columnName) => !foreignTableColumnNames.has(columnName), + ); + + const columnsDeleted = [...foreignTableColumnNames].filter( + (columnName) => !distantTableColumnNames.has(columnName), + ); + + if (columnsAdded.length > 0) { + updates[tableName] = [ + ...(updates[tableName] || []), + DistantTableUpdate.COLUMNS_ADDED, + ]; + } + if (columnsDeleted.length > 0) { + updates[tableName] = [ + ...(updates[tableName] || []), + DistantTableUpdate.COLUMNS_DELETED, + ]; + } } + + return updates; } } diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util.ts new file mode 100644 index 000000000000..a6bd4ca1288e --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util.ts @@ -0,0 +1,25 @@ +import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; +import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; + +export const fetchTableColumns = async ( + workspaceDataSourceService: WorkspaceDataSourceService, + workspaceId: string, + tableName: string, +): Promise => { + const workspaceDataSource = + await workspaceDataSourceService.connectToWorkspaceDataSource(workspaceId); + + const schemaName = workspaceDataSourceService.getSchemaName(workspaceId); + + const res = await workspaceDataSource.query( + `SELECT column_name, data_type, udt_name + FROM information_schema.columns + WHERE table_schema = '${schemaName}' AND table_name = '${tableName}'`, + ); + + return res.map((column) => ({ + columnName: column.column_name, + dataType: column.data_type, + udtName: column.udt_name, + })); +}; From 29d0b3e578bae83fd2292f4a8ba696fe4bf2e054 Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Mon, 20 May 2024 16:14:17 +0200 Subject: [PATCH 2/9] only fetch distant table schema if shouldFetchPendingSchemaUpdates --- .../twenty-front/src/generated-metadata/graphql.ts | 2 ++ .../databases/hooks/useGetDatabaseConnectionTables.ts | 3 +++ ...ttingsIntegrationDatabaseConnectionSummaryCard.tsx | 1 + ...ettingsIntegrationDatabaseConnectionSyncStatus.tsx | 3 +++ .../hooks/useDatabaseConnection.ts | 1 + .../remote-server/remote-server.entity.ts | 4 ---- .../dtos/find-many-remote-tables-input.ts | 11 ++++++++++- .../remote-table/remote-table.resolver.ts | 1 + .../remote-table/remote-table.service.ts | 3 ++- 9 files changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts index e57c79ee388a..3af8d5b9c5c6 100644 --- a/packages/twenty-front/src/generated-metadata/graphql.ts +++ b/packages/twenty-front/src/generated-metadata/graphql.ts @@ -338,6 +338,8 @@ export enum FileFolder { export type FindManyRemoteTablesInput = { /** The id of the remote server. */ id: Scalars['ID']['input']; + /** Indicates if pending schema updates status should be computed. */ + shouldFetchPendingSchemaUpdates?: InputMaybe; }; export type FullName = { diff --git a/packages/twenty-front/src/modules/databases/hooks/useGetDatabaseConnectionTables.ts b/packages/twenty-front/src/modules/databases/hooks/useGetDatabaseConnectionTables.ts index dd50582bb792..99484670f389 100644 --- a/packages/twenty-front/src/modules/databases/hooks/useGetDatabaseConnectionTables.ts +++ b/packages/twenty-front/src/modules/databases/hooks/useGetDatabaseConnectionTables.ts @@ -10,11 +10,13 @@ import { type UseGetDatabaseConnectionTablesParams = { connectionId: string; skip?: boolean; + shouldFetchPendingSchemaUpdates?: boolean; }; export const useGetDatabaseConnectionTables = ({ connectionId, skip, + shouldFetchPendingSchemaUpdates, }: UseGetDatabaseConnectionTablesParams) => { const apolloMetadataClient = useApolloMetadataClient(); @@ -27,6 +29,7 @@ export const useGetDatabaseConnectionTables = ({ variables: { input: { id: connectionId, + shouldFetchPendingSchemaUpdates, }, }, }); diff --git a/packages/twenty-front/src/modules/settings/integrations/database-connection/components/SettingsIntegrationDatabaseConnectionSummaryCard.tsx b/packages/twenty-front/src/modules/settings/integrations/database-connection/components/SettingsIntegrationDatabaseConnectionSummaryCard.tsx index 4fe124377ff6..e2d14b794e2c 100644 --- a/packages/twenty-front/src/modules/settings/integrations/database-connection/components/SettingsIntegrationDatabaseConnectionSummaryCard.tsx +++ b/packages/twenty-front/src/modules/settings/integrations/database-connection/components/SettingsIntegrationDatabaseConnectionSummaryCard.tsx @@ -53,6 +53,7 @@ export const SettingsIntegrationDatabaseConnectionSummaryCard = ({ <> { const { tables, error } = useGetDatabaseConnectionTables({ connectionId, skip, + shouldFetchPendingSchemaUpdates, }); if (isDefined(error)) { diff --git a/packages/twenty-front/src/modules/settings/integrations/database-connection/hooks/useDatabaseConnection.ts b/packages/twenty-front/src/modules/settings/integrations/database-connection/hooks/useDatabaseConnection.ts index 1a44195771e3..0079630765e0 100644 --- a/packages/twenty-front/src/modules/settings/integrations/database-connection/hooks/useDatabaseConnection.ts +++ b/packages/twenty-front/src/modules/settings/integrations/database-connection/hooks/useDatabaseConnection.ts @@ -42,6 +42,7 @@ export const useDatabaseConnection = () => { const { tables } = useGetDatabaseConnectionTables({ connectionId, skip: !connection, + shouldFetchPendingSchemaUpdates: true, }); return { connection, integration, databaseKey, tables }; diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.entity.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.entity.ts index 91493501bf86..672e3424b550 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.entity.ts @@ -11,7 +11,6 @@ import { import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity'; import { UserMappingOptions } from 'src/engine/metadata-modules/remote-server/types/user-mapping-options'; -import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table'; export enum RemoteServerType { POSTGRES_FDW = 'postgres_fdw', @@ -59,9 +58,6 @@ export class RemoteServerEntity { @Column({ nullable: false, type: 'uuid' }) workspaceId: string; - @Column({ type: 'jsonb', nullable: true }) - availableTables: DistantTables; - @OneToMany(() => RemoteTableEntity, (table) => table.server, { cascade: true, }) diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/find-many-remote-tables-input.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/find-many-remote-tables-input.ts index f74519cb933c..0e5c9e55568b 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/find-many-remote-tables-input.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/find-many-remote-tables-input.ts @@ -1,9 +1,18 @@ -import { InputType, ID } from '@nestjs/graphql'; +import { InputType, ID, Field } from '@nestjs/graphql'; import { IDField } from '@ptc-org/nestjs-query-graphql'; +import { IsOptional } from 'class-validator'; @InputType() export class FindManyRemoteTablesInput { @IDField(() => ID, { description: 'The id of the remote server.' }) id!: string; + + @IsOptional() + @Field(() => Boolean, { + description: + 'Indicates if pending schema updates status should be computed.', + nullable: true, + }) + shouldFetchPendingSchemaUpdates?: boolean; } diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts index 311d8bad3b83..e80f4a1cabfa 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts @@ -22,6 +22,7 @@ export class RemoteTableResolver { return this.remoteTableService.findDistantTablesWithStatusByServerId( input.id, workspaceId, + input.shouldFetchPendingSchemaUpdates, ); } diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts index 5e6aefa21d0f..9209e1c1c556 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts @@ -57,6 +57,7 @@ export class RemoteTableService { public async findDistantTablesWithStatusByServerId( id: string, workspaceId: string, + shouldFetchPendingSchemaUpdates?: boolean, ) { const remoteServer = await this.remoteServerRepository.findOne({ where: { @@ -83,7 +84,7 @@ export class RemoteTableService { workspaceId, ); - if (currentRemoteTables.length === 0) { + if (currentRemoteTables.length === 0 || !shouldFetchPendingSchemaUpdates) { const distantTablesWithStatus = Object.keys(distantTables).map( (tableName) => ({ name: tableName, From cb91fd31d249f35a4088aba9478ba75b7b21cb9c Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Mon, 20 May 2024 16:43:03 +0200 Subject: [PATCH 3/9] Replace availableTable with schema fetch of single table when syncing a table --- .../distant-table/distant-table.service.ts | 57 +++++++++---------- .../remote-table/remote-table.service.ts | 10 ++-- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts index e10b1cc65904..1240db8f7ee6 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts @@ -23,43 +23,42 @@ export class DistantTableService { >, ) {} - public getDistantTableColumns( - remoteServer: RemoteServerEntity, - tableName: string, - ): PostgresTableSchemaColumn[] { - if (!remoteServer.availableTables) { - throw new BadRequestException( - 'Remote server available tables are not defined', - ); - } - - return remoteServer.availableTables[tableName]; - } - public async fetchDistantTables( remoteServer: RemoteServerEntity, workspaceId: string, ): Promise { - return this.createAvailableTables(remoteServer, workspaceId); + if (remoteServer.schema) { + return this.getDistantTablesFromDynamicSchema(remoteServer, workspaceId); + } + + return this.getDistantTablesFromStaticSchema(remoteServer); } - private async createAvailableTables( + public async getDistantTableColumns( remoteServer: RemoteServerEntity, workspaceId: string, - ): Promise { + tableName: string, + ): Promise { if (remoteServer.schema) { - return this.createAvailableTablesFromDynamicSchema( + const distantTableInList = await this.getDistantTablesFromDynamicSchema( remoteServer, workspaceId, + tableName, ); + + return distantTableInList[tableName]; } - return this.createAvailableTablesFromStaticSchema(remoteServer); + const distantTableInList = + await this.getDistantTablesFromStaticSchema(remoteServer); + + return distantTableInList[tableName]; } - private async createAvailableTablesFromDynamicSchema( + private async getDistantTablesFromDynamicSchema( remoteServer: RemoteServerEntity, workspaceId: string, + tableName?: string, ): Promise { if (!remoteServer.schema) { throw new BadRequestException('Remote server schema is not defined'); @@ -73,12 +72,16 @@ export class DistantTableService { workspaceId, ); - const availableTables = await workspaceDataSource.transaction( + const distantTables = await workspaceDataSource.transaction( async (entityManager: EntityManager) => { await entityManager.query(`CREATE SCHEMA "${tmpSchemaName}"`); + const tableLimitationsOptions = tableName + ? ` LIMIT TO (${tableName})` + : ''; + await entityManager.query( - `IMPORT FOREIGN SCHEMA "${remoteServer.schema}" FROM SERVER "${remoteServer.foreignDataWrapperId}" INTO "${tmpSchemaName}"`, + `IMPORT FOREIGN SCHEMA "${remoteServer.schema}"${tableLimitationsOptions} FROM SERVER "${remoteServer.foreignDataWrapperId}" INTO "${tmpSchemaName}"`, ); const createdForeignTableNames = await entityManager.query( @@ -106,22 +109,14 @@ export class DistantTableService { }, ); - await this.remoteServerRepository.update(remoteServer.id, { - availableTables, - }); - - return availableTables; + return distantTables; } - private async createAvailableTablesFromStaticSchema( + private async getDistantTablesFromStaticSchema( remoteServer: RemoteServerEntity, ): Promise { switch (remoteServer.foreignDataWrapperType) { case RemoteServerType.STRIPE_FDW: - this.remoteServerRepository.update(remoteServer.id, { - availableTables: STRIPE_DISTANT_TABLES, - }); - return STRIPE_DISTANT_TABLES; default: throw new BadRequestException( diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts index 9209e1c1c556..f17b822fdf2c 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts @@ -174,10 +174,12 @@ export class RemoteTableService { remoteServerId: remoteServer.id, }); - const distantTableColumns = this.distantTableService.getDistantTableColumns( - remoteServer, - input.name, - ); + const distantTableColumns = + await this.distantTableService.getDistantTableColumns( + remoteServer, + workspaceId, + input.name, + ); // We only support remote tables with an id column for now. const distantTableIdColumn = distantTableColumns.find( From 6a544e1f5704bb0e5b5619a4cfb8de1f2b26e268 Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Tue, 21 May 2024 14:37:36 +0200 Subject: [PATCH 4/9] alter foreign table according to distant table schema --- .../foreign-table/foreign-table.module.ts | 2 + .../foreign-table/foreign-table.service.ts | 42 +++++ .../remote-table/remote-table.resolver.ts | 11 ++ .../remote-table/remote-table.service.ts | 163 +++++++++++++++--- .../workspace-migration.entity.ts | 6 + .../workspace-migration-runner.service.ts | 32 ++++ 6 files changed, 236 insertions(+), 20 deletions(-) diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module.ts index c40aeb4201f8..bcfc2e109258 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module.ts @@ -1,6 +1,7 @@ import { Module } from '@nestjs/common'; import { ForeignTableService } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service'; +import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; @@ -10,6 +11,7 @@ import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/wor WorkspaceMigrationModule, WorkspaceMigrationRunnerModule, WorkspaceDataSourceModule, + WorkspaceCacheVersionModule, ], providers: [ForeignTableService], exports: [ForeignTableService], diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts index c5973ea8c93c..256b2fa2c07d 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts @@ -4,14 +4,17 @@ import { RemoteServerEntity, RemoteServerType, } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; +import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util'; import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; +import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { ReferencedTable, WorkspaceMigrationTableActionType, WorkspaceMigrationForeignColumnDefinition, WorkspaceMigrationForeignTable, + WorkspaceMigrationAlterForeignTableAlteration, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; @@ -23,6 +26,7 @@ export class ForeignTableService { private readonly workspaceMigrationService: WorkspaceMigrationService, private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, private readonly workspaceDataSourceService: WorkspaceDataSourceService, + private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, ) {} public async fetchForeignTableNamesWithinWorkspace( @@ -91,6 +95,44 @@ export class ForeignTableService { } } + public async updateForeignTable( + tableName: string, + workspaceId: string, + alterations?: WorkspaceMigrationAlterForeignTableAlteration[], + ) { + const workspaceMigration = + await this.workspaceMigrationService.createCustomMigration( + generateMigrationName(`alter-foreign-table-${tableName}`), + workspaceId, + [ + { + name: tableName, + action: WorkspaceMigrationTableActionType.ALTER_FOREIGN_TABLE, + foreignTableAlterations: alterations, + }, + ], + ); + + // TODO: This should be done in a transaction. Waiting for a global refactoring of transaction management. + try { + await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( + workspaceId, + ); + + await this.workspaceCacheVersionService.incrementVersion(workspaceId); + + return { + name: tableName, + status: RemoteTableStatus.SYNCED, + schemaPendingUpdates: [], + }; + } catch (exception) { + this.workspaceMigrationService.deleteById(workspaceMigration.id); + + throw new BadRequestException('Could not alter foreign table.'); + } + } + public async deleteForeignTable(tableName: string, workspaceId: string) { await this.workspaceMigrationService.createCustomMigration( generateMigrationName(`drop-foreign-table-${tableName}`), diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts index e80f4a1cabfa..c9ead15f1d4a 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts @@ -41,4 +41,15 @@ export class RemoteTableResolver { ) { return this.remoteTableService.unsyncRemoteTable(input, workspaceId); } + + @Mutation(() => RemoteTableDTO) + async updateRemoteTabletoDistantTable( + @Args('input') input: RemoteTableInput, + @AuthWorkspace() { id: workspaceId }: Workspace, + ) { + return this.remoteTableService.updateRemoteTableToDistantTable( + input, + workspaceId, + ); + } } diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts index f17b822fdf2c..8ca1ef66c212 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts @@ -3,6 +3,7 @@ import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; import { plural } from 'pluralize'; +import isEmpty from 'lodash.isempty'; import { RemoteServerType, @@ -30,10 +31,16 @@ import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/rem import { getRemoteTableLocalName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util'; import { DistantTableService } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service'; import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table'; -import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util'; +import { getForeignTableColumnName as convertToForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util'; import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; import { fetchTableColumns } from 'src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util'; import { ForeignTableService } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service'; +import { + WorkspaceMigrationAlterForeignTableAlteration, + WorkspaceMigrationColumnActionType, + WorkspaceMigrationColumnCreate, + WorkspaceMigrationColumnDrop, +} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; export class RemoteTableService { private readonly logger = new Logger(RemoteTableService.name); @@ -268,6 +275,78 @@ export class RemoteTableService { } } + public async updateRemoteTableToDistantTable( + input: RemoteTableInput, + workspaceId: string, + ) { + const remoteServer = await this.remoteServerRepository.findOne({ + where: { + id: input.remoteServerId, + workspaceId, + }, + }); + + if (!remoteServer) { + throw new NotFoundException('Remote server does not exist'); + } + + const remoteTable = await this.remoteTableRepository.findOne({ + where: { + distantTableName: input.name, + remoteServerId: remoteServer.id, + workspaceId, + }, + }); + + if (!remoteTable) { + throw new NotFoundException('Remote table does not exist'); + } + + const distantTableColumns = + await this.distantTableService.getDistantTableColumns( + remoteServer, + workspaceId, + remoteTable.distantTableName, + ); + + if (!distantTableColumns) { + await this.unsyncOne(workspaceId, remoteTable, remoteServer); + + return {}; + } + + const foreignTableColumns = await fetchTableColumns( + this.workspaceDataSourceService, + workspaceId, + remoteTable.localTableName, + ); + + const alterations = this.computeForeignTableAlterations( + foreignTableColumns, + distantTableColumns, + ); + + if (isEmpty(alterations)) { + this.logger.log( + `No update to perform on table "${remoteTable.localTableName}" for workspace ${workspaceId}`, + ); + + return { + name: remoteTable.localTableName, + status: RemoteTableStatus.SYNCED, + schemaPendingUpdates: [], + }; + } + + const updatedTable = await this.foreignTableService.updateForeignTable( + remoteTable.localTableName, + workspaceId, + alterations, + ); + + return updatedTable; + } + private async unsyncOne( workspaceId: string, remoteTable: RemoteTableEntity, @@ -435,27 +514,15 @@ export class RemoteTableService { continue; } - const distantTableColumnNames = new Set( - distantTable.map((column) => - getForeignTableColumnName(column.columnName), - ), - ); - const foreignTableColumnNames = new Set( - ( - await fetchTableColumns( - this.workspaceDataSourceService, - workspaceId, - remoteTable.localTableName, - ) - ).map((column) => column.columnName), - ); - - const columnsAdded = [...distantTableColumnNames].filter( - (columnName) => !foreignTableColumnNames.has(columnName), + const foreignTable = await fetchTableColumns( + this.workspaceDataSourceService, + workspaceId, + remoteTable.localTableName, ); - const columnsDeleted = [...foreignTableColumnNames].filter( - (columnName) => !distantTableColumnNames.has(columnName), + const { columnsAdded, columnsDeleted } = this.compareForeignTableColumns( + foreignTable, + distantTable, ); if (columnsAdded.length > 0) { @@ -474,4 +541,60 @@ export class RemoteTableService { return updates; } + + private compareForeignTableColumns = ( + foreignTableColumns: PostgresTableSchemaColumn[], + distantTableColumns: PostgresTableSchemaColumn[], + ) => { + const foreignTableColumnNames = foreignTableColumns.map( + (column) => column.columnName, + ); + const distantTableColumnsWithConvertedName = distantTableColumns.map( + (column) => { + return { + name: convertToForeignTableColumnName(column.columnName), + type: column.dataType, + }; + }, + ); + + const columnsAdded = distantTableColumnsWithConvertedName.filter( + (column) => !foreignTableColumnNames.includes(column.name), + ); + const columnsDeleted = foreignTableColumnNames.filter( + (columnName) => + !distantTableColumnsWithConvertedName + .map((column) => column.name) + .includes(columnName), + ); + + return { + columnsAdded, + columnsDeleted, + }; + }; + + private computeForeignTableAlterations = ( + foreignTableColumns: PostgresTableSchemaColumn[], + distantTableColumns: PostgresTableSchemaColumn[], + ): WorkspaceMigrationAlterForeignTableAlteration[] => { + const { columnsAdded, columnsDeleted } = this.compareForeignTableColumns( + foreignTableColumns, + distantTableColumns, + ); + const columnsAddedAlterations: WorkspaceMigrationColumnCreate[] = + columnsAdded.map((columnAdded) => ({ + action: WorkspaceMigrationColumnActionType.CREATE, + columnName: columnAdded.name, + columnType: columnAdded.type, + })); + + const columnsDeletedAlterations: WorkspaceMigrationColumnDrop[] = + columnsDeleted.map((columnDeleted) => ({ + action: WorkspaceMigrationColumnActionType.DROP, + columnName: columnDeleted, + })); + + return [...columnsAddedAlterations, ...columnsDeletedAlterations]; + }; } diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts index a524f1d98a82..ec2e9f462e1f 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts @@ -95,6 +95,10 @@ export type WorkspaceMigrationColumnAction = { | WorkspaceMigrationCreateComment ); +export type WorkspaceMigrationAlterForeignTableAlteration = + | WorkspaceMigrationColumnDrop + | WorkspaceMigrationColumnCreate; + /** * Enum values are lowercase to avoid issues with already existing enum values */ @@ -104,6 +108,7 @@ export enum WorkspaceMigrationTableActionType { DROP = 'drop', CREATE_FOREIGN_TABLE = 'create_foreign_table', DROP_FOREIGN_TABLE = 'drop_foreign_table', + ALTER_FOREIGN_TABLE = 'alter_foreign_table', } export type WorkspaceMigrationTableAction = { @@ -112,6 +117,7 @@ export type WorkspaceMigrationTableAction = { action: WorkspaceMigrationTableActionType; columns?: WorkspaceMigrationColumnAction[]; foreignTable?: WorkspaceMigrationForeignTable; + foreignTableAlterations?: WorkspaceMigrationAlterForeignTableAlteration[]; }; @Entity('workspaceMigration') diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts index cb2e3c8d804a..544837af9660 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts @@ -20,6 +20,7 @@ import { WorkspaceMigrationColumnDropRelation, WorkspaceMigrationTableActionType, WorkspaceMigrationForeignTable, + WorkspaceMigrationAlterForeignTableAlteration, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service'; @@ -155,6 +156,14 @@ export class WorkspaceMigrationRunnerService { `DROP FOREIGN TABLE ${schemaName}."${tableMigration.name}"`, ); break; + case 'alter_foreign_table': + await this.alterForeignTable( + queryRunner, + schemaName, + tableMigration.name, + tableMigration.foreignTableAlterations, + ); + break; default: throw new Error( `Migration table action ${tableMigration.action} not supported`, @@ -507,4 +516,27 @@ export class WorkspaceMigrationRunnerService { COMMENT ON FOREIGN TABLE "${schemaName}"."${name}" IS '@graphql({"primary_key_columns": ["id"], "totalCount": {"enabled": true}})'; `); } + + private async alterForeignTable( + queryRunner: QueryRunner, + schemaName: string, + name: string, + alterations: WorkspaceMigrationAlterForeignTableAlteration[] | undefined, + ) { + const alterationsQuery = alterations + ?.map((alteration) => { + if (alteration.action === WorkspaceMigrationColumnActionType.DROP) { + return `DROP COLUMN "${alteration.columnName}"`; + } else if ( + alteration.action === WorkspaceMigrationColumnActionType.CREATE + ) { + return `ADD COLUMN "${alteration.columnName}" ${alteration.columnType}`; // Add OPTIONS (column_name '${column.distantColumnName}')` ? + } + }) + .join(', '); + + await queryRunner.query( + `ALTER FOREIGN TABLE ${schemaName}."${name}" ${alterationsQuery};`, + ); + } } From 43c6587a9a42feb07774cacb9dcd15769aa1e59e Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Tue, 21 May 2024 15:06:24 +0200 Subject: [PATCH 5/9] remove comment --- .../workspace-migration-runner.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts index 544837af9660..0b8bdad0a4c5 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts @@ -530,7 +530,7 @@ export class WorkspaceMigrationRunnerService { } else if ( alteration.action === WorkspaceMigrationColumnActionType.CREATE ) { - return `ADD COLUMN "${alteration.columnName}" ${alteration.columnType}`; // Add OPTIONS (column_name '${column.distantColumnName}')` ? + return `ADD COLUMN "${alteration.columnName}" ${alteration.columnType}`; } }) .join(', '); From b3b7caa6fde05501568f28ce7aa0c3fea3588ee1 Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Tue, 21 May 2024 18:24:02 +0200 Subject: [PATCH 6/9] Improve code quality --- .../distant-table/distant-table.service.ts | 8 +- .../distant-table/types/distant-table.ts | 2 +- .../foreign-table/foreign-table.service.ts | 32 +-- .../remote-table-schema-update.module.ts | 11 ++ .../remote-table-schema-update.service.ts | 176 +++++++++++++++++ .../remote-table/remote-table.module.ts | 2 + .../remote-table/remote-table.resolver.ts | 4 +- .../remote-table/remote-table.service.ts | 183 ++---------------- .../utils/fetch-table-columns.util.ts | 9 +- .../utils/get-remote-table-local-name.util.ts | 2 +- .../workspace-migration.entity.ts | 5 - .../workspace-migration-runner.service.ts | 27 +-- 12 files changed, 245 insertions(+), 216 deletions(-) create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.module.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts index 1240db8f7ee6..40c88bfb27f4 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts @@ -40,19 +40,19 @@ export class DistantTableService { tableName: string, ): Promise { if (remoteServer.schema) { - const distantTableInList = await this.getDistantTablesFromDynamicSchema( + const distantTables = await this.getDistantTablesFromDynamicSchema( remoteServer, workspaceId, tableName, ); - return distantTableInList[tableName]; + return distantTables[tableName]; } - const distantTableInList = + const distantTables = await this.getDistantTablesFromStaticSchema(remoteServer); - return distantTableInList[tableName]; + return distantTables[tableName]; } private async getDistantTablesFromDynamicSchema( diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table.ts index aeeb52512cbd..21031b195dfe 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table.ts @@ -1,5 +1,5 @@ import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; export type DistantTables = { - [tableName: string]: PostgresTableSchemaColumn[]; + [distantTableName: string]: PostgresTableSchemaColumn[]; }; diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts index 256b2fa2c07d..b99d555e4731 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts @@ -14,7 +14,7 @@ import { WorkspaceMigrationTableActionType, WorkspaceMigrationForeignColumnDefinition, WorkspaceMigrationForeignTable, - WorkspaceMigrationAlterForeignTableAlteration, + WorkspaceMigrationColumnAction, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; @@ -40,7 +40,8 @@ export class ForeignTableService { return ( await workspaceDataSource.query( - `SELECT foreign_table_name, foreign_server_name FROM information_schema.foreign_tables WHERE foreign_server_name = '${foreignDataWrapperId}'`, + `SELECT foreign_table_name, foreign_server_name FROM information_schema.foreign_tables WHERE foreign_server_name = '$1'`, + [foreignDataWrapperId], ) ).map((foreignTable) => foreignTable.foreign_table_name); } @@ -96,19 +97,19 @@ export class ForeignTableService { } public async updateForeignTable( - tableName: string, + foreignTableName: string, workspaceId: string, - alterations?: WorkspaceMigrationAlterForeignTableAlteration[], + columnsUpdates?: WorkspaceMigrationColumnAction[], ) { const workspaceMigration = await this.workspaceMigrationService.createCustomMigration( - generateMigrationName(`alter-foreign-table-${tableName}`), + generateMigrationName(`alter-foreign-table-${foreignTableName}`), workspaceId, [ { - name: tableName, + name: foreignTableName, action: WorkspaceMigrationTableActionType.ALTER_FOREIGN_TABLE, - foreignTableAlterations: alterations, + columns: columnsUpdates, }, ], ); @@ -122,7 +123,7 @@ export class ForeignTableService { await this.workspaceCacheVersionService.incrementVersion(workspaceId); return { - name: tableName, + name: foreignTableName, status: RemoteTableStatus.SYNCED, schemaPendingUpdates: [], }; @@ -133,13 +134,16 @@ export class ForeignTableService { } } - public async deleteForeignTable(tableName: string, workspaceId: string) { + public async deleteForeignTable( + foreignTableName: string, + workspaceId: string, + ) { await this.workspaceMigrationService.createCustomMigration( - generateMigrationName(`drop-foreign-table-${tableName}`), + generateMigrationName(`drop-foreign-table-${foreignTableName}`), workspaceId, [ { - name: tableName, + name: foreignTableName, action: WorkspaceMigrationTableActionType.DROP_FOREIGN_TABLE, }, ], @@ -152,16 +156,16 @@ export class ForeignTableService { private buildReferencedTable( remoteServer: RemoteServerEntity, - tableName: string, + distantTableName: string, ): ReferencedTable { switch (remoteServer.foreignDataWrapperType) { case RemoteServerType.POSTGRES_FDW: return { - table_name: tableName, + table_name: distantTableName, schema_name: remoteServer.schema, }; case RemoteServerType.STRIPE_FDW: - return { object: tableName }; + return { object: distantTableName }; default: throw new BadRequestException('Foreign data wrapper not supported'); } diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.module.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.module.ts new file mode 100644 index 000000000000..c1f26fc9b044 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; + +import { RemoteTableSchemaUpdateService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service'; +import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; + +@Module({ + imports: [WorkspaceDataSourceModule], + providers: [RemoteTableSchemaUpdateService], + exports: [RemoteTableSchemaUpdateService], +}) +export class RemoteTableSchemaUpdateModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts new file mode 100644 index 000000000000..8b080b07b5bc --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts @@ -0,0 +1,176 @@ +import { Injectable } from '@nestjs/common'; + +import { getForeignTableColumnName as convertToForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util'; +import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table'; +import { + RemoteTableStatus, + DistantTableUpdate, +} from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; +import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity'; +import { fetchTableColumns } from 'src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util'; +import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; +import { + WorkspaceMigrationColumnAction, + WorkspaceMigrationColumnCreate, + WorkspaceMigrationColumnActionType, + WorkspaceMigrationColumnDrop, +} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; + +@Injectable() +export class RemoteTableSchemaUpdateService { + constructor( + private readonly workspaceDataSourceService: WorkspaceDataSourceService, + ) {} + + public async getDistantTablesWithUpdates({ + remoteServerSchema, + workspaceId, + remoteTables, + distantTables, + }: { + remoteServerSchema: string; + workspaceId: string; + remoteTables: RemoteTableEntity[]; + distantTables: DistantTables; + }) { + const schemaPendingUpdates = + await this.getSchemaUpdatesBetweenForeignAndDistantTables({ + workspaceId, + remoteTables, + distantTables, + }); + + const remoteTablesDistantNames = remoteTables.map( + (remoteTable) => remoteTable.distantTableName, + ); + + const distantTablesWithUpdates = Object.keys(distantTables).map( + (tableName) => ({ + name: tableName, + schema: remoteServerSchema, + status: remoteTablesDistantNames.includes(tableName) + ? RemoteTableStatus.SYNCED + : RemoteTableStatus.NOT_SYNCED, + schemaPendingUpdates: schemaPendingUpdates[tableName], + }), + ); + + const deletedTables = Object.entries(schemaPendingUpdates) + .filter(([_tableName, updates]) => + updates.includes(DistantTableUpdate.TABLE_DELETED), + ) + .map(([tableName, updates]) => ({ + name: tableName, + schema: remoteServerSchema, + status: RemoteTableStatus.SYNCED, + schemaPendingUpdates: updates, + })); + + return distantTablesWithUpdates.concat(deletedTables); + } + + public computeForeignTableColumnsUpdates = ( + foreignTableColumns: PostgresTableSchemaColumn[], + distantTableColumns: PostgresTableSchemaColumn[], + ): WorkspaceMigrationColumnAction[] => { + const { columnsAdded, columnsDeleted } = this.compareForeignTableColumns( + foreignTableColumns, + distantTableColumns, + ); + const columnsAddedUpdates: WorkspaceMigrationColumnCreate[] = + columnsAdded.map((columnAdded) => ({ + action: WorkspaceMigrationColumnActionType.CREATE, + columnName: columnAdded.name, + columnType: columnAdded.type, + })); + + const columnsDeletedUpdates: WorkspaceMigrationColumnDrop[] = + columnsDeleted.map((columnDeleted) => ({ + action: WorkspaceMigrationColumnActionType.DROP, + columnName: columnDeleted, + })); + + return [...columnsAddedUpdates, ...columnsDeletedUpdates]; + }; + + private async getSchemaUpdatesBetweenForeignAndDistantTables({ + workspaceId, + remoteTables, + distantTables, + }: { + workspaceId: string; + remoteTables: RemoteTableEntity[]; + distantTables: DistantTables; + }): Promise<{ [tablename: string]: DistantTableUpdate[] }> { + const updates = {}; + + for (const remoteTable of remoteTables) { + const distantTable = distantTables[remoteTable.distantTableName]; + const tableName = remoteTable.distantTableName; + + if (!distantTable) { + updates[tableName] = [DistantTableUpdate.TABLE_DELETED]; + continue; + } + + const foreignTable = await fetchTableColumns( + this.workspaceDataSourceService, + workspaceId, + remoteTable.localTableName, + ); + + const { columnsAdded, columnsDeleted } = this.compareForeignTableColumns( + foreignTable, + distantTable, + ); + + if (columnsAdded.length > 0) { + updates[tableName] = [ + ...(updates[tableName] || []), + DistantTableUpdate.COLUMNS_ADDED, + ]; + } + if (columnsDeleted.length > 0) { + updates[tableName] = [ + ...(updates[tableName] || []), + DistantTableUpdate.COLUMNS_DELETED, + ]; + } + } + + return updates; + } + + private compareForeignTableColumns = ( + foreignTableColumns: PostgresTableSchemaColumn[], + distantTableColumns: PostgresTableSchemaColumn[], + ) => { + const foreignTableColumnNames = foreignTableColumns.map( + (column) => column.columnName, + ); + const distantTableColumnsWithConvertedName = distantTableColumns.map( + (column) => { + return { + name: convertToForeignTableColumnName(column.columnName), + type: column.dataType, + }; + }, + ); + + const columnsAdded = distantTableColumnsWithConvertedName.filter( + (column) => !foreignTableColumnNames.includes(column.name), + ); + const columnsDeleted = foreignTableColumnNames.filter( + (columnName) => + !distantTableColumnsWithConvertedName + .map((column) => column.name) + .includes(columnName), + ); + + return { + columnsAdded, + columnsDeleted, + }; + }; +} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts index f56846dc071e..720b8a466b47 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts @@ -7,6 +7,7 @@ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadat import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; import { DistantTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.module'; import { ForeignTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module'; +import { RemoteTableSchemaUpdateModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.module'; import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity'; import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver'; import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service'; @@ -26,6 +27,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works WorkspaceCacheVersionModule, WorkspaceDataSourceModule, ForeignTableModule, + RemoteTableSchemaUpdateModule, ], providers: [RemoteTableService, RemoteTableResolver], exports: [RemoteTableService], diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts index c9ead15f1d4a..b49d913a713f 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts @@ -43,11 +43,11 @@ export class RemoteTableResolver { } @Mutation(() => RemoteTableDTO) - async updateRemoteTabletoDistantTable( + async syncRemoteTableSchemaChanges( @Args('input') input: RemoteTableInput, @AuthWorkspace() { id: workspaceId }: Workspace, ) { - return this.remoteTableService.updateRemoteTableToDistantTable( + return this.remoteTableService.syncRemoteTableSchemaChanges( input, workspaceId, ); diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts index 8ca1ef66c212..7e283a8a00fc 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts @@ -9,10 +9,7 @@ import { RemoteServerType, RemoteServerEntity, } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; -import { - RemoteTableStatus, - DistantTableUpdate, -} from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; +import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; import { mapUdtNameToFieldType, mapUdtNameToFieldSettings, @@ -30,17 +27,10 @@ import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/work import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity'; import { getRemoteTableLocalName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util'; import { DistantTableService } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service'; -import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table'; -import { getForeignTableColumnName as convertToForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util'; import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; import { fetchTableColumns } from 'src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util'; import { ForeignTableService } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service'; -import { - WorkspaceMigrationAlterForeignTableAlteration, - WorkspaceMigrationColumnActionType, - WorkspaceMigrationColumnCreate, - WorkspaceMigrationColumnDrop, -} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { RemoteTableSchemaUpdateService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service'; export class RemoteTableService { private readonly logger = new Logger(RemoteTableService.name); @@ -59,6 +49,7 @@ export class RemoteTableService { private readonly distantTableService: DistantTableService, private readonly foreignTableService: ForeignTableService, private readonly workspaceDataSourceService: WorkspaceDataSourceService, + private readonly remoteTableSchemaUpdateService: RemoteTableSchemaUpdateService, ) {} public async findDistantTablesWithStatusByServerId( @@ -105,7 +96,7 @@ export class RemoteTableService { return distantTablesWithStatus; } - return this.getDistantTablesWithUpdates({ + return this.remoteTableSchemaUpdateService.getDistantTablesWithUpdates({ remoteServerSchema: remoteServer.schema, workspaceId, remoteTables: currentRemoteTables, @@ -275,7 +266,7 @@ export class RemoteTableService { } } - public async updateRemoteTableToDistantTable( + public async syncRemoteTableSchemaChanges( input: RemoteTableInput, workspaceId: string, ) { @@ -321,12 +312,13 @@ export class RemoteTableService { remoteTable.localTableName, ); - const alterations = this.computeForeignTableAlterations( - foreignTableColumns, - distantTableColumns, - ); + const columnsUpdates = + this.remoteTableSchemaUpdateService.computeForeignTableColumnsUpdates( + foreignTableColumns, + distantTableColumns, + ); - if (isEmpty(alterations)) { + if (isEmpty(columnsUpdates)) { this.logger.log( `No update to perform on table "${remoteTable.localTableName}" for workspace ${workspaceId}`, ); @@ -341,7 +333,7 @@ export class RemoteTableService { const updatedTable = await this.foreignTableService.updateForeignTable( remoteTable.localTableName, workspaceId, - alterations, + columnsUpdates, ); return updatedTable; @@ -446,155 +438,4 @@ export class RemoteTableService { } } } - - private async getDistantTablesWithUpdates({ - remoteServerSchema, - workspaceId, - remoteTables, - distantTables, - }: { - remoteServerSchema: string; - workspaceId: string; - remoteTables: RemoteTableEntity[]; - distantTables: DistantTables; - }) { - const schemaPendingUpdates = - await this.getSchemaUpdatesBetweenForeignAndDistantTables({ - workspaceId, - remoteTables, - distantTables, - }); - - const remoteTablesDistantNames = remoteTables.map( - (remoteTable) => remoteTable.distantTableName, - ); - - const distantTablesWithUpdates = Object.keys(distantTables).map( - (tableName) => ({ - name: tableName, - schema: remoteServerSchema, - status: remoteTablesDistantNames.includes(tableName) - ? RemoteTableStatus.SYNCED - : RemoteTableStatus.NOT_SYNCED, - schemaPendingUpdates: schemaPendingUpdates[tableName], - }), - ); - - const deletedTables = Object.entries(schemaPendingUpdates) - .filter(([_tableName, updates]) => - updates.includes(DistantTableUpdate.TABLE_DELETED), - ) - .map(([tableName, updates]) => ({ - name: tableName, - schema: remoteServerSchema, - status: RemoteTableStatus.SYNCED, - schemaPendingUpdates: updates, - })); - - return distantTablesWithUpdates.concat(deletedTables); - } - - private async getSchemaUpdatesBetweenForeignAndDistantTables({ - workspaceId, - remoteTables, - distantTables, - }: { - workspaceId: string; - remoteTables: RemoteTableEntity[]; - distantTables: DistantTables; - }): Promise<{ [tablename: string]: DistantTableUpdate[] }> { - const updates = {}; - - for (const remoteTable of remoteTables) { - const distantTable = distantTables[remoteTable.distantTableName]; - const tableName = remoteTable.distantTableName; - - if (!distantTable) { - updates[tableName] = [DistantTableUpdate.TABLE_DELETED]; - continue; - } - - const foreignTable = await fetchTableColumns( - this.workspaceDataSourceService, - workspaceId, - remoteTable.localTableName, - ); - - const { columnsAdded, columnsDeleted } = this.compareForeignTableColumns( - foreignTable, - distantTable, - ); - - if (columnsAdded.length > 0) { - updates[tableName] = [ - ...(updates[tableName] || []), - DistantTableUpdate.COLUMNS_ADDED, - ]; - } - if (columnsDeleted.length > 0) { - updates[tableName] = [ - ...(updates[tableName] || []), - DistantTableUpdate.COLUMNS_DELETED, - ]; - } - } - - return updates; - } - - private compareForeignTableColumns = ( - foreignTableColumns: PostgresTableSchemaColumn[], - distantTableColumns: PostgresTableSchemaColumn[], - ) => { - const foreignTableColumnNames = foreignTableColumns.map( - (column) => column.columnName, - ); - const distantTableColumnsWithConvertedName = distantTableColumns.map( - (column) => { - return { - name: convertToForeignTableColumnName(column.columnName), - type: column.dataType, - }; - }, - ); - - const columnsAdded = distantTableColumnsWithConvertedName.filter( - (column) => !foreignTableColumnNames.includes(column.name), - ); - const columnsDeleted = foreignTableColumnNames.filter( - (columnName) => - !distantTableColumnsWithConvertedName - .map((column) => column.name) - .includes(columnName), - ); - - return { - columnsAdded, - columnsDeleted, - }; - }; - - private computeForeignTableAlterations = ( - foreignTableColumns: PostgresTableSchemaColumn[], - distantTableColumns: PostgresTableSchemaColumn[], - ): WorkspaceMigrationAlterForeignTableAlteration[] => { - const { columnsAdded, columnsDeleted } = this.compareForeignTableColumns( - foreignTableColumns, - distantTableColumns, - ); - const columnsAddedAlterations: WorkspaceMigrationColumnCreate[] = - columnsAdded.map((columnAdded) => ({ - action: WorkspaceMigrationColumnActionType.CREATE, - columnName: columnAdded.name, - columnType: columnAdded.type, - })); - - const columnsDeletedAlterations: WorkspaceMigrationColumnDrop[] = - columnsDeleted.map((columnDeleted) => ({ - action: WorkspaceMigrationColumnActionType.DROP, - columnName: columnDeleted, - })); - - return [...columnsAddedAlterations, ...columnsDeletedAlterations]; - }; } diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util.ts index a6bd4ca1288e..89422bb4ffe7 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util.ts @@ -6,15 +6,14 @@ export const fetchTableColumns = async ( workspaceId: string, tableName: string, ): Promise => { - const workspaceDataSource = - await workspaceDataSourceService.connectToWorkspaceDataSource(workspaceId); - const schemaName = workspaceDataSourceService.getSchemaName(workspaceId); - const res = await workspaceDataSource.query( + const res = await workspaceDataSourceService.executeRawQuery( `SELECT column_name, data_type, udt_name FROM information_schema.columns - WHERE table_schema = '${schemaName}' AND table_name = '${tableName}'`, + WHERE table_schema = $1 AND table_name = $2`, + [schemaName, tableName], + workspaceId, ); return res.map((column) => ({ diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util.ts index 92008e169d00..45d07604e05b 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util.ts @@ -21,7 +21,7 @@ const isNameAvailable = async ( await workspaceDataSource.query( `SELECT count(table_name) FROM information_schema.tables WHERE table_name LIKE '${tableName}' AND table_schema IN ('core', 'metadata', '${workspaceSchemaName}')`, ) - )[0].count; + )[0].count; // parameterize this? return numberOfTablesWithSameName === 0; }; diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts index ec2e9f462e1f..8116e0b3dc9e 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts @@ -95,10 +95,6 @@ export type WorkspaceMigrationColumnAction = { | WorkspaceMigrationCreateComment ); -export type WorkspaceMigrationAlterForeignTableAlteration = - | WorkspaceMigrationColumnDrop - | WorkspaceMigrationColumnCreate; - /** * Enum values are lowercase to avoid issues with already existing enum values */ @@ -117,7 +113,6 @@ export type WorkspaceMigrationTableAction = { action: WorkspaceMigrationTableActionType; columns?: WorkspaceMigrationColumnAction[]; foreignTable?: WorkspaceMigrationForeignTable; - foreignTableAlterations?: WorkspaceMigrationAlterForeignTableAlteration[]; }; @Entity('workspaceMigration') diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts index 0b8bdad0a4c5..560d9fff4995 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts @@ -20,7 +20,6 @@ import { WorkspaceMigrationColumnDropRelation, WorkspaceMigrationTableActionType, WorkspaceMigrationForeignTable, - WorkspaceMigrationAlterForeignTableAlteration, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service'; @@ -156,12 +155,12 @@ export class WorkspaceMigrationRunnerService { `DROP FOREIGN TABLE ${schemaName}."${tableMigration.name}"`, ); break; - case 'alter_foreign_table': + case WorkspaceMigrationTableActionType.ALTER_FOREIGN_TABLE: await this.alterForeignTable( queryRunner, schemaName, tableMigration.name, - tableMigration.foreignTableAlterations, + tableMigration.columns, ); break; default: @@ -521,22 +520,24 @@ export class WorkspaceMigrationRunnerService { queryRunner: QueryRunner, schemaName: string, name: string, - alterations: WorkspaceMigrationAlterForeignTableAlteration[] | undefined, + columns: WorkspaceMigrationColumnAction[] | undefined, ) { - const alterationsQuery = alterations - ?.map((alteration) => { - if (alteration.action === WorkspaceMigrationColumnActionType.DROP) { - return `DROP COLUMN "${alteration.columnName}"`; - } else if ( - alteration.action === WorkspaceMigrationColumnActionType.CREATE - ) { - return `ADD COLUMN "${alteration.columnName}" ${alteration.columnType}`; + const columnUpdatesQuery = columns + ?.map((column) => { + switch (column.action) { + case WorkspaceMigrationColumnActionType.DROP: + return `DROP COLUMN "${column.columnName}"`; + case WorkspaceMigrationColumnActionType.CREATE: + return `ADD COLUMN "${column.columnName}" ${column.columnType}`; + default: + return ''; } }) + .filter(Boolean) .join(', '); await queryRunner.query( - `ALTER FOREIGN TABLE ${schemaName}."${name}" ${alterationsQuery};`, + `ALTER FOREIGN TABLE ${schemaName}."${name}" ${columnUpdatesQuery};`, ); } } From 8b83fe536d8d63c5e5e16ee700acb4168adcc7a8 Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Tue, 21 May 2024 19:06:02 +0200 Subject: [PATCH 7/9] Remove availableTables from remoteServer --- .../1716310822694-removeAvailableTables.ts | 17 +++++++++++++++++ .../utils/get-remote-table-local-name.util.ts | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 packages/twenty-server/src/database/typeorm/metadata/migrations/1716310822694-removeAvailableTables.ts diff --git a/packages/twenty-server/src/database/typeorm/metadata/migrations/1716310822694-removeAvailableTables.ts b/packages/twenty-server/src/database/typeorm/metadata/migrations/1716310822694-removeAvailableTables.ts new file mode 100644 index 000000000000..a55ce837a1c7 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/metadata/migrations/1716310822694-removeAvailableTables.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class RemoveAvailableTables1716310822694 implements MigrationInterface { + name = 'RemoveAvailableTables1716310822694'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."remoteServer" DROP COLUMN "availableTables"`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."remoteServer" ADD "availableTables" jsonb`, + ); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util.ts index 45d07604e05b..92008e169d00 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util.ts @@ -21,7 +21,7 @@ const isNameAvailable = async ( await workspaceDataSource.query( `SELECT count(table_name) FROM information_schema.tables WHERE table_name LIKE '${tableName}' AND table_schema IN ('core', 'metadata', '${workspaceSchemaName}')`, ) - )[0].count; // parameterize this? + )[0].count; return numberOfTablesWithSameName === 0; }; From 93ddbdd7bd3b34f9c1ccbba06fb7687dc53f8f3d Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Tue, 21 May 2024 19:26:09 +0200 Subject: [PATCH 8/9] Fix parameterized query --- .../remote-table/foreign-table/foreign-table.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts index b99d555e4731..d660626dcd07 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts @@ -40,7 +40,7 @@ export class ForeignTableService { return ( await workspaceDataSource.query( - `SELECT foreign_table_name, foreign_server_name FROM information_schema.foreign_tables WHERE foreign_server_name = '$1'`, + `SELECT foreign_table_name, foreign_server_name FROM information_schema.foreign_tables WHERE foreign_server_name = $1`, [foreignDataWrapperId], ) ).map((foreignTable) => foreignTable.foreign_table_name); From d4460e2c5da767fce32cfb67e4669f3963dbcfa6 Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Tue, 21 May 2024 20:06:16 +0200 Subject: [PATCH 9/9] Handle undefined cases and improve code quality --- .../distant-table/distant-table.service.ts | 26 +++++++++---------- .../remote-table-schema-update.service.ts | 18 ++++++------- .../remote-table/remote-table.service.ts | 2 +- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts index 40c88bfb27f4..20d5fb7cbec6 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts @@ -26,9 +26,14 @@ export class DistantTableService { public async fetchDistantTables( remoteServer: RemoteServerEntity, workspaceId: string, + tableName?: string, ): Promise { if (remoteServer.schema) { - return this.getDistantTablesFromDynamicSchema(remoteServer, workspaceId); + return this.getDistantTablesFromDynamicSchema( + remoteServer, + workspaceId, + tableName, + ); } return this.getDistantTablesFromStaticSchema(remoteServer); @@ -39,20 +44,13 @@ export class DistantTableService { workspaceId: string, tableName: string, ): Promise { - if (remoteServer.schema) { - const distantTables = await this.getDistantTablesFromDynamicSchema( - remoteServer, - workspaceId, - tableName, - ); - - return distantTables[tableName]; - } - - const distantTables = - await this.getDistantTablesFromStaticSchema(remoteServer); + const distantTables = await this.fetchDistantTables( + remoteServer, + workspaceId, + tableName, + ); - return distantTables[tableName]; + return distantTables[tableName] || []; } private async getDistantTablesFromDynamicSchema( diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts index 8b080b07b5bc..afbd64f8fab6 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts @@ -41,18 +41,18 @@ export class RemoteTableSchemaUpdateService { distantTables, }); - const remoteTablesDistantNames = remoteTables.map( - (remoteTable) => remoteTable.distantTableName, + const remoteTablesDistantNames = new Set( + remoteTables.map((remoteTable) => remoteTable.distantTableName), ); const distantTablesWithUpdates = Object.keys(distantTables).map( (tableName) => ({ name: tableName, schema: remoteServerSchema, - status: remoteTablesDistantNames.includes(tableName) + status: remoteTablesDistantNames.has(tableName) ? RemoteTableStatus.SYNCED : RemoteTableStatus.NOT_SYNCED, - schemaPendingUpdates: schemaPendingUpdates[tableName], + schemaPendingUpdates: schemaPendingUpdates[tableName] || [], }), ); @@ -67,7 +67,7 @@ export class RemoteTableSchemaUpdateService { schemaPendingUpdates: updates, })); - return distantTablesWithUpdates.concat(deletedTables); + return [...distantTablesWithUpdates, ...deletedTables]; } public computeForeignTableColumnsUpdates = ( @@ -146,8 +146,8 @@ export class RemoteTableSchemaUpdateService { foreignTableColumns: PostgresTableSchemaColumn[], distantTableColumns: PostgresTableSchemaColumn[], ) => { - const foreignTableColumnNames = foreignTableColumns.map( - (column) => column.columnName, + const foreignTableColumnNames = new Set( + foreignTableColumns.map((column) => column.columnName), ); const distantTableColumnsWithConvertedName = distantTableColumns.map( (column) => { @@ -159,9 +159,9 @@ export class RemoteTableSchemaUpdateService { ); const columnsAdded = distantTableColumnsWithConvertedName.filter( - (column) => !foreignTableColumnNames.includes(column.name), + (column) => !foreignTableColumnNames.has(column.name), ); - const columnsDeleted = foreignTableColumnNames.filter( + const columnsDeleted = Array.from(foreignTableColumnNames).filter( (columnName) => !distantTableColumnsWithConvertedName .map((column) => column.name) diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts index 7e283a8a00fc..ba44cc41fffd 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts @@ -300,7 +300,7 @@ export class RemoteTableService { remoteTable.distantTableName, ); - if (!distantTableColumns) { + if (isEmpty(distantTableColumns)) { await this.unsyncOne(workspaceId, remoteTable, remoteServer); return {};