From a3b5f01a4de81c6c2ee47da90a5c9f635d48d9ee Mon Sep 17 00:00:00 2001 From: Joe Flateau Date: Wed, 18 Aug 2021 17:30:53 -0400 Subject: [PATCH] feat: add upsert methods for the drivers that support it This adds EntityManager#upsert, BaseEntity#upsert and EntityManager#upsert Closes: #2363 --- src/entity-manager/EntityManager.ts | 47 +++++++++++++++- src/query-builder/InsertQueryBuilder.ts | 9 ++- src/repository/BaseEntity.ts | 10 ++++ src/repository/Repository.ts | 10 ++++ src/repository/UpsertOptions.ts | 10 ++++ test/functional/entity-model/entity-model.ts | 21 ++++++- test/functional/entity-model/entity/Post.ts | 3 + .../repository/basic-methods/entity/Post.ts | 21 ++++++- .../basic-methods/repository-basic-methods.ts | 55 +++++++++++++++++-- test/utils/test-utils.ts | 1 + 10 files changed, 175 insertions(+), 12 deletions(-) create mode 100644 src/repository/UpsertOptions.ts diff --git a/src/entity-manager/EntityManager.ts b/src/entity-manager/EntityManager.ts index e30dd1b1995..589581df83f 100644 --- a/src/entity-manager/EntityManager.ts +++ b/src/entity-manager/EntityManager.ts @@ -6,8 +6,10 @@ import {EntityNotFoundError} from "../error/EntityNotFoundError"; import {QueryRunnerProviderAlreadyReleasedError} from "../error/QueryRunnerProviderAlreadyReleasedError"; import {FindOneOptions} from "../find-options/FindOneOptions"; import {DeepPartial} from "../common/DeepPartial"; +import {ColumnMetadata} from "../metadata/ColumnMetadata"; import {RemoveOptions} from "../repository/RemoveOptions"; import {SaveOptions} from "../repository/SaveOptions"; +import {UpsertOptions} from "../repository/UpsertOptions"; import {NoNeedToReleaseEntityManagerError} from "../error/NoNeedToReleaseEntityManagerError"; import {MongoRepository} from "../repository/MongoRepository"; import {TreeRepository} from "../repository/TreeRepository"; @@ -37,7 +39,7 @@ import {ObjectUtils} from "../util/ObjectUtils"; import {EntitySchema} from "../entity-schema/EntitySchema"; import {ObjectLiteral} from "../common/ObjectLiteral"; import {getMetadataArgsStorage} from "../globals"; -import { TypeORMError } from "../error"; +import {EntityColumnNotFound,TypeORMError} from "../error"; /** * Entity manager supposed to work with any entity, automatically find its repository and call its methods, @@ -478,6 +480,49 @@ export class EntityManager { .execute(); } + async upsert(target: EntityTarget, entity: QueryDeepPartialEntity | (QueryDeepPartialEntity[]), options: UpsertOptions = {}): Promise { + const metadata = this.connection.getMetadata(target); + + const conflictColumns = + options.conflictProperties?.map( + (prop) => + { + const column = metadata.columns.find(col => col.propertyName === prop); + if (column == null) { + throw new EntityColumnNotFound(prop); + } + return column.databaseName; + } + ) ?? + metadata.columns + .filter((col) => col.isPrimary) + .map((col) => col.databaseName); + + + const insertQueryBuilder = this.createQueryBuilder() + .insert() + .into(target) + .values(entity); + + const insertedColumns: ColumnMetadata[] = (insertQueryBuilder as any).getInsertedColumns(); + + const overwriteColumns = insertedColumns + .filter( + (col) => + !conflictColumns.includes(col.databaseName) && + !col.isCreateDate && + !col.isGenerated + ) + .map((col) => col.databaseName); + + insertQueryBuilder + .orUpdate(overwriteColumns, conflictColumns); + + const insertResult = await insertQueryBuilder.execute(); + + return insertResult; + } + /** * Updates entity partially. Entity can be found by a given condition(s). * Unlike save method executes a primitive operation without cascades, relations and other operations included. diff --git a/src/query-builder/InsertQueryBuilder.ts b/src/query-builder/InsertQueryBuilder.ts index 5f51af38720..09fd925086c 100644 --- a/src/query-builder/InsertQueryBuilder.ts +++ b/src/query-builder/InsertQueryBuilder.ts @@ -18,6 +18,7 @@ import {BroadcasterResult} from "../subscriber/BroadcasterResult"; import {EntitySchema} from "../entity-schema/EntitySchema"; import {OracleDriver} from "../driver/oracle/OracleDriver"; import {AuroraDataApiDriver} from "../driver/aurora-data-api/AuroraDataApiDriver"; +import { TypeORMError } from "../error"; /** * Allows to build complex sql queries in a fashion way and execute those queries. @@ -378,9 +379,7 @@ export class InsertQueryBuilder extends QueryBuilder { query += " "; } } - } - - if (this.connection.driver instanceof MysqlDriver || this.connection.driver instanceof AuroraDataApiDriver) { + } else if (this.connection.driver instanceof MysqlDriver || this.connection.driver instanceof AuroraDataApiDriver) { if (this.expressionMap.onUpdate) { const { overwrite, columns } = this.expressionMap.onUpdate; @@ -394,6 +393,10 @@ export class InsertQueryBuilder extends QueryBuilder { query += " "; } } + } else { + if (this.expressionMap.onUpdate) { + throw new TypeORMError(`onUpdate is not supported by the current database driver`); + } } // add RETURNING expression diff --git a/src/repository/BaseEntity.ts b/src/repository/BaseEntity.ts index a414533a9ea..a7ea79f3972 100644 --- a/src/repository/BaseEntity.ts +++ b/src/repository/BaseEntity.ts @@ -15,6 +15,7 @@ import {DeleteResult} from "../query-builder/result/DeleteResult"; import {ObjectID} from "../driver/mongodb/typings"; import {ObjectUtils} from "../util/ObjectUtils"; import {QueryDeepPartialEntity} from "../query-builder/QueryPartialEntity"; +import {UpsertOptions} from "./UpsertOptions"; /** * Base abstract entity for all entities, used in ActiveRecord patterns. @@ -249,6 +250,15 @@ export class BaseEntity { return (this as any).getRepository().update(criteria, partialEntity, options); } + /** + * Inserts a given entity into the database, unless a unique constraint conflicts then updates the entity + * Unlike save method executes a primitive operation without cascades, relations and other operations included. + * Executes fast and efficient INSERT ... ON CONFLICT DO UPDATE/ON DUPLICATE KEY UPDATE query. + */ + static upsert(this: ObjectType & typeof BaseEntity, entity: QueryDeepPartialEntity | (QueryDeepPartialEntity[]), options?: UpsertOptions): Promise { + return this.getRepository().upsert(entity, options); + } + /** * Deletes entities by a given criteria. * Unlike remove method executes a primitive operation without cascades, relations and other operations included. diff --git a/src/repository/Repository.ts b/src/repository/Repository.ts index 6050de50de2..fe6ce15aed9 100644 --- a/src/repository/Repository.ts +++ b/src/repository/Repository.ts @@ -14,6 +14,7 @@ import {InsertResult} from "../query-builder/result/InsertResult"; import {QueryDeepPartialEntity} from "../query-builder/QueryPartialEntity"; import {ObjectID} from "../driver/mongodb/typings"; import {FindConditions} from "../find-options/FindConditions"; +import {UpsertOptions} from "./UpsertOptions"; /** * Repository is supposed to work with your entity objects. Find entities, insert, update, delete, etc. @@ -241,6 +242,15 @@ export class Repository { return this.manager.update(this.metadata.target as any, criteria as any, partialEntity); } + /** + * Inserts a given entity into the database, unless a unique constraint conflicts then updates the entity + * Unlike save method executes a primitive operation without cascades, relations and other operations included. + * Executes fast and efficient INSERT ... ON CONFLICT DO UPDATE/ON DUPLICATE KEY UPDATE query. + */ + upsert(entity: QueryDeepPartialEntity | (QueryDeepPartialEntity[]), options?: UpsertOptions): Promise { + return this.manager.upsert(this.metadata.target as any, entity, options); + } + /** * Deletes entities by a given criteria. * Unlike save method executes a primitive operation without cascades, relations and other operations included. diff --git a/src/repository/UpsertOptions.ts b/src/repository/UpsertOptions.ts new file mode 100644 index 00000000000..39ec6b8a965 --- /dev/null +++ b/src/repository/UpsertOptions.ts @@ -0,0 +1,10 @@ +/** + * Special options passed to Repository#upsert + */ +export interface UpsertOptions { + + /** + * Define which columns to conflict on + */ + conflictProperties?: ((keyof T) & string)[]; +} diff --git a/test/functional/entity-model/entity-model.ts b/test/functional/entity-model/entity-model.ts index baf250b4732..b8c0315beb7 100644 --- a/test/functional/entity-model/entity-model.ts +++ b/test/functional/entity-model/entity-model.ts @@ -29,6 +29,26 @@ describe("entity-model", () => { loadedPost!.id.should.be.eql(post.id); loadedPost!.title.should.be.eql("About ActiveRecord"); loadedPost!.text.should.be.eql("Huge discussion how good or bad ActiveRecord is."); + + const externalId = "external-entity"; + + await Post.upsert({ + externalId, + title: "External post" + }, { + conflictProperties: ["externalId"] + }); + const upsertInsertedExternalPost = await Post.findOneOrFail({ externalId }); + await Post.upsert({ + externalId, + title: "External post 2" + }, { + conflictProperties: ["externalId"] + }); + const upsertUpdatedExternalPost = await Post.findOneOrFail({ externalId }); + + upsertInsertedExternalPost.id.should.be.equal(upsertUpdatedExternalPost.id); + upsertInsertedExternalPost.title.should.not.be.equal(upsertUpdatedExternalPost.title); } }); @@ -73,5 +93,4 @@ describe("entity-model", () => { }); } }); - }); diff --git a/test/functional/entity-model/entity/Post.ts b/test/functional/entity-model/entity/Post.ts index fc77fe36fcd..5caee481ae1 100644 --- a/test/functional/entity-model/entity/Post.ts +++ b/test/functional/entity-model/entity/Post.ts @@ -11,6 +11,9 @@ export class Post extends BaseEntity { @PrimaryGeneratedColumn() id: number; + @Column() + externalId?: string; + @Column() title: string; diff --git a/test/functional/repository/basic-methods/entity/Post.ts b/test/functional/repository/basic-methods/entity/Post.ts index 63d10c8180d..03b5ce887aa 100644 --- a/test/functional/repository/basic-methods/entity/Post.ts +++ b/test/functional/repository/basic-methods/entity/Post.ts @@ -1,6 +1,7 @@ import {Entity} from "../../../../../src/decorator/entity/Entity"; import {PrimaryGeneratedColumn} from "../../../../../src/decorator/columns/PrimaryGeneratedColumn"; import {Column} from "../../../../../src/decorator/columns/Column"; +import {CreateDateColumn,UpdateDateColumn} from "../../../../../src"; @Entity() export class Post { @@ -8,15 +9,31 @@ export class Post { @PrimaryGeneratedColumn() id: number|undefined|null|string; + @Column({ + nullable: true, + unique: true, + name: "eXtErNal___id" // makes sure we test handling differing property/database names where necessary + }) + externalId?: string; + @Column() title: string; @Column({ + nullable: true, type: "date", transformer: { from: (value: any) => new Date(value), - to: (value: Date) => value.toISOString(), + to: (value?: Date) => value?.toISOString(), } }) - dateAdded: Date; + dateAdded?: Date; + + @CreateDateColumn() + createdAt!: Date; + + @UpdateDateColumn({ + name: "uPdAtEd___At" // makes sure we test handling differing property/database names where necessary + }) + updatedAt!: Date; } \ No newline at end of file diff --git a/test/functional/repository/basic-methods/repository-basic-methods.ts b/test/functional/repository/basic-methods/repository-basic-methods.ts index a37d09358aa..4190a5fd2a1 100644 --- a/test/functional/repository/basic-methods/repository-basic-methods.ts +++ b/test/functional/repository/basic-methods/repository-basic-methods.ts @@ -1,5 +1,5 @@ import "reflect-metadata"; -import {closeTestingConnections, createTestingConnections, reloadTestingDatabases} from "../../../utils/test-utils"; +import {closeTestingConnections, createTestingConnections, reloadTestingDatabases, sleep} from "../../../utils/test-utils"; import {Connection} from "../../../../src/connection/Connection"; import {Post} from "./entity/Post"; import {QueryBuilder} from "../../../../src/query-builder/QueryBuilder"; @@ -341,8 +341,8 @@ describe("repository > basic methods", () => { const dbPost = await postRepository.findOne(post.id) as Post; dbPost.should.be.instanceOf(Post); - dbPost.dateAdded.should.be.instanceOf(Date); - dbPost.dateAdded.getTime().should.be.equal(date.getTime()); + dbPost.dateAdded!.should.be.instanceOf(Date); + dbPost.dateAdded!.getTime().should.be.equal(date.getTime()); dbPost.title = "New title"; const saved = await postRepository.save(dbPost); @@ -351,8 +351,53 @@ describe("repository > basic methods", () => { saved.id!.should.be.equal(1); saved.title.should.be.equal("New title"); - saved.dateAdded.should.be.instanceof(Date); - saved.dateAdded.getTime().should.be.equal(date.getTime()); + saved.dateAdded!.should.be.instanceof(Date); + saved.dateAdded!.getTime().should.be.equal(date.getTime()); + }))); + }); + + describe("upsert", function () { + it("should first create then update an entity", () => Promise.all(connections.map(async connection => { + const postRepository = connection.getRepository(Post); + const externalId = "external-1"; + + // create a new post and insert it + const post = new Post(); + post.externalId = externalId; + post.title = "Post title initial"; + await postRepository.upsert(post, { + conflictProperties: ["externalId"] + }); + const initial = await postRepository.findOneOrFail({ + externalId + }); + initial.title.should.be.equal("Post title initial"); + + await sleep(1000); // need to let a second pass for the updatedAt time to have a new value + + // update post with externalId + const update = new Post(); + update.externalId = externalId; + update.title = "Post title updated"; + + await postRepository.upsert(update, { + conflictProperties: ["externalId"] + }); + const updated = await postRepository.findOneOrFail({ + externalId + }); + // title should have changed + updated.title.should.be.equal("Post title updated"); + // id should not have changed + updated.id!.should.be.equal(initial.id); + + updated.createdAt.getTime().should.be.equal(initial.createdAt.getTime(), "created time should be the same"); + + updated.updatedAt.getTime().should.not.be.equal(initial.updatedAt.getTime(), "updated time should not be the same"); + + // unique constraint on externalId already enforces this, but test it anyways + const count = await postRepository.find({ externalId }); + count.length.should.be.equal(1); }))); }); diff --git a/test/utils/test-utils.ts b/test/utils/test-utils.ts index 58f2f8b8341..153ea500606 100644 --- a/test/utils/test-utils.ts +++ b/test/utils/test-utils.ts @@ -214,6 +214,7 @@ export function setupTestingConnections(options?: TestingOptions): ConnectionOpt subscribers: options && options.subscribers ? options.subscribers : [], dropSchema: options && options.dropSchema !== undefined ? options.dropSchema : false, cache: options ? options.cache : undefined, + logging: process.env.TYPEORM_LOGGING === "1" }); if (options && options.driverSpecific) newOptions = Object.assign({}, options.driverSpecific, newOptions);