feat: add upsert methods for the drivers that support onUpdate (#8104)

* feat: add upsert methods for the drivers that support it

This adds EntityManager#upsert, BaseEntity#upsert and EntityManager#upsert

Closes: #2363

* docs: Document which drivers support upsert operations

* docs: fix typo in entity manager upsert many example

* refactor: remove mongodb style upsert signature, enforce types of conflict paths

* docs: add note to repository docs specifying which drivers support upsert

* refactor: cannot staticly type conflict paths because that would break typescript pre-4

* refactor: remove test utility methods in favor of some repeated checks

docs/entity-manager-api.md

@@ -147,6 +147,22 @@ await manager.update(User, 1, { firstName: "Rizzrak" });
 // executes UPDATE user SET firstName = Rizzrak WHERE id = 1
 ```
 
+* `upsert` - Inserts a new entity or array of entities unless they already exist in which case they are updated instead. Supported by AuroraDataApi, Cockroach, Mysql, Postgres, and Sqlite database drivers.
+
+```typescript
+await manager.upsert(User, [
+    { externalId:"abc123", firstName: "Rizzrak" },
+    { externalId:"bca321", firstName: "Karzzir" },
+], ["externalId"]);
+/** executes 
+ *  INSERT INTO user 
+ *  VALUES 
+ *      (externalId = abc123, firstName = Rizzrak),
+ *      (externalId = cba321, firstName = Karzzir),
+ *  ON CONFLICT (externalId) DO UPDATE firstName = EXCLUDED.firstName
+ **/
+```
+
 * `delete` - Deletes entities by entity id, ids or given conditions:
 
 ```typescript

docs/repository-api.md

@@ -154,6 +154,22 @@ await repository.update(1, { firstName: "Rizzrak" });
 // executes UPDATE user SET firstName = Rizzrak WHERE id = 1
 ```
 
+* `upsert` - Inserts a new entity or array of entities unless they already exist in which case they are updated instead. Supported by AuroraDataApi, Cockroach, Mysql, Postgres, and Sqlite database drivers.
+
+```typescript
+await repository.update([
+    { externalId:"abc123", firstName: "Rizzrak" },
+    { externalId:"bca321", firstName: "Karzzir" },
+], ["externalId"]);
+/** executes 
+ *  INSERT INTO user 
+ *  VALUES 
+ *      (externalId = abc123, firstName = Rizzrak),
+ *      (externalId = cba321, firstName = Karzzir),
+ *  ON CONFLICT (externalId) DO UPDATE firstName = EXCLUDED.firstName
+ **/
+```
+
 * `delete` - Deletes entities by entity id, ids or given conditions:
 
 ```typescript

src/driver/Driver.ts

@@ -12,6 +12,7 @@ import {ReplicationMode} from "./types/ReplicationMode";
 import { Table } from "../schema-builder/table/Table";
 import { View } from "../schema-builder/view/View";
 import { TableForeignKey } from "../schema-builder/table/TableForeignKey";
+import { UpsertType } from "./types/UpsertType";
 
 /**
  * Driver organizes TypeORM communication with specific database management system.
@@ -50,6 +51,11 @@ export interface Driver {
      */
     supportedDataTypes: ColumnType[];
 
+    /**
+     * Returns type of upsert supported by driver if any
+     */
+    supportedUpsertType?: UpsertType;
+
     /**
      * Default values of length, precision and scale depends on column data type.
      * Used in the cases when length/precision/scale is not specified by user.

src/driver/aurora-data-api/AuroraDataApiDriver.ts

@@ -142,6 +142,11 @@ export class AuroraDataApiDriver implements Driver {
         "geometrycollection"
     ];
 
+    /**
+     * Returns type of upsert supported by driver if any
+     */
+    readonly supportedUpsertType = "on-duplicate-key-update";
+
     /**
      * Gets list of spatial column data types.
      */

src/driver/cockroachdb/CockroachDriver.ts

@@ -148,6 +148,11 @@ export class CockroachDriver implements Driver {
         "uuid",
     ];
 
+    /**
+     * Returns type of upsert supported by driver if any
+     */
+    readonly supportedUpsertType = "on-conflict-do-update";
+
     /**
      * Gets list of spatial column data types.
      */

src/driver/mysql/MysqlDriver.ts

@@ -141,6 +141,11 @@ export class MysqlDriver implements Driver {
         "geometrycollection"
     ];
 
+    /**
+     * Returns type of upsert supported by driver if any
+     */
+    readonly supportedUpsertType = "on-duplicate-key-update";
+
     /**
      * Gets list of spatial column data types.
      */

src/driver/postgres/PostgresDriver.ts

@@ -173,6 +173,11 @@ export class PostgresDriver implements Driver {
         "ltree"
     ];
 
+    /**
+     * Returns type of upsert supported by driver if any
+     */
+    readonly supportedUpsertType = "on-conflict-do-update";
+
     /**
      * Gets list of spatial column data types.
      */

src/driver/sqlite-abstract/AbstractSqliteDriver.ts

@@ -113,6 +113,11 @@ export abstract class AbstractSqliteDriver implements Driver {
         "datetime"
     ];
 
+    /**
+     * Returns type of upsert supported by driver if any
+     */
+    readonly supportedUpsertType = "on-conflict-do-update";
+
     /**
      * Gets list of column data types that support length by a driver.
      */

src/driver/types/UpsertType.ts

@@ -0,0 +1 @@
+export type UpsertType = "on-conflict-do-update" | "on-duplicate-key-update";

src/entity-manager/EntityManager.ts

@@ -37,7 +37,8 @@ import {ObjectUtils} from "../util/ObjectUtils";
 import {EntitySchema} from "../entity-schema/EntitySchema";
 import {ObjectLiteral} from "../common/ObjectLiteral";
 import {getMetadataArgsStorage} from "../globals";
-import { TypeORMError } from "../error";
+import {TypeORMError} from "../error";
+import {UpsertOptions} from "../repository/UpsertOptions";
 
 /**
  * Entity manager supposed to work with any entity, automatically find its repository and call its methods,
@@ -478,6 +479,61 @@ export class EntityManager {
             .execute();
     }
 
+    async upsert<Entity>(
+        target: EntityTarget<Entity>,
+        entityOrEntities: QueryDeepPartialEntity<Entity> | (QueryDeepPartialEntity<Entity>[]),
+        conflictPathsOrOptions: string[] | UpsertOptions<Entity>): Promise<InsertResult> {
+        const metadata = this.connection.getMetadata(target);
+
+        let options: UpsertOptions<Entity>;
+
+        if (Array.isArray(conflictPathsOrOptions)) {
+            options = {
+                conflictPaths: conflictPathsOrOptions
+            };
+        } else {
+            options = conflictPathsOrOptions;
+        }
+
+        const uniqueColumnConstraints = [
+            metadata.primaryColumns,
+            ...metadata.indices.filter(ix => ix.isUnique).map(ix => ix.columns),
+            ...metadata.uniques.map(uq => uq.columns)
+        ];
+
+        const useIndex = uniqueColumnConstraints.find((ix) =>
+            ix.length === options.conflictPaths.length &&
+            options.conflictPaths.every((conflictPropertyPath) => ix.some((col) => col.propertyPath === conflictPropertyPath))
+        );
+
+        if (useIndex == null) {
+            throw new TypeORMError(`An upsert requires conditions that have a unique constraint but none was found for conflict properties: ${options.conflictPaths.join(", ")}`);
+        }
+
+        let entities: QueryDeepPartialEntity<Entity>[];
+
+        if (!Array.isArray(entityOrEntities)) {
+            entities = [entityOrEntities];
+        } else {
+            entities = entityOrEntities;
+        }
+
+        const conflictColumns = metadata.mapPropertyPathsToColumns(options.conflictPaths);
+
+        const overwriteColumns = metadata.columns
+            .filter((col) => (!conflictColumns.includes(col)) && entities.some(entity => typeof col.getEntityValue(entity) !== "undefined"));
+
+        return this.createQueryBuilder()
+            .insert()
+            .into(target)
+            .values(entities)
+            .orUpdate(
+                [...conflictColumns, ...overwriteColumns].map((col) => col.databaseName),
+                conflictColumns.map((col) => col.databaseName)
+            )
+            .execute();
+    }
+
     /**
      * Updates entity partially. Entity can be found by a given condition(s).
      * Unlike save method executes a primitive operation without cascades, relations and other operations included.

src/metadata/EntityMetadata.ts

@@ -1,4 +1,4 @@
-import {QueryRunner, SelectQueryBuilder} from "..";
+import {EntityColumnNotFound, QueryRunner, SelectQueryBuilder} from "..";
 import {ObjectLiteral} from "../common/ObjectLiteral";
 import {Connection} from "../connection/Connection";
 import {CannotCreateEntityIdMapError} from "../error/CannotCreateEntityIdMapError";
@@ -720,6 +720,19 @@ export class EntityMetadata {
         return this.allEmbeddeds.find(embedded => embedded.propertyPath === propertyPath);
     }
 
+    /**
+     * Returns an array of databaseNames mapped from provided propertyPaths
+     */
+    mapPropertyPathsToColumns(propertyPaths: string[]) {
+        return propertyPaths.map(propertyPath => {
+            const column = this.findColumnWithPropertyPath(propertyPath);
+            if (column == null) {
+                throw new EntityColumnNotFound(propertyPath);
+            }
+            return column;
+        });
+    }
+
     /**
      * Iterates through entity and finds and extracts all values from relations in the entity.
      * If relation value is an array its being flattened.

src/query-builder/InsertQueryBuilder.ts

@@ -18,6 +18,7 @@ import {BroadcasterResult} from "../subscriber/BroadcasterResult";
 import {EntitySchema} from "../entity-schema/EntitySchema";
 import {OracleDriver} from "../driver/oracle/OracleDriver";
 import {AuroraDataApiDriver} from "../driver/aurora-data-api/AuroraDataApiDriver";
+import { TypeORMError } from "../error";
 
 /**
  * Allows to build complex sql queries in a fashion way and execute those queries.
@@ -352,7 +353,7 @@ export class InsertQueryBuilder<Entity> extends QueryBuilder<Entity> {
                 query += ` DEFAULT VALUES`;
             }
         }
-        if (this.connection.driver instanceof PostgresDriver || this.connection.driver instanceof AbstractSqliteDriver || this.connection.driver instanceof CockroachDriver) {
+        if (this.connection.driver.supportedUpsertType === "on-conflict-do-update") {
             if (this.expressionMap.onIgnore) {
                 query += " ON CONFLICT DO NOTHING ";
             } else if (this.expressionMap.onConflict) {
@@ -378,9 +379,7 @@ export class InsertQueryBuilder<Entity> extends QueryBuilder<Entity> {
                     query += " ";
                 }
             }
-        }
-
-        if (this.connection.driver instanceof MysqlDriver || this.connection.driver instanceof AuroraDataApiDriver) {
+        } else if (this.connection.driver.supportedUpsertType === "on-duplicate-key-update") {
             if (this.expressionMap.onUpdate) {
                 const { overwrite, columns } = this.expressionMap.onUpdate;
 
@@ -394,6 +393,10 @@ export class InsertQueryBuilder<Entity> extends QueryBuilder<Entity> {
                     query += " ";
                 }
             }
+        } else {
+            if (this.expressionMap.onUpdate) {
+                throw new TypeORMError(`onUpdate is not supported by the current database driver`);
+            }
         }
 
         // add RETURNING expression

src/repository/BaseEntity.ts

@@ -15,6 +15,7 @@ import {DeleteResult} from "../query-builder/result/DeleteResult";
 import {ObjectID} from "../driver/mongodb/typings";
 import {ObjectUtils} from "../util/ObjectUtils";
 import {QueryDeepPartialEntity} from "../query-builder/QueryPartialEntity";
+import {UpsertOptions} from "./UpsertOptions";
 
 /**
  * Base abstract entity for all entities, used in ActiveRecord patterns.
@@ -249,6 +250,17 @@ export class BaseEntity {
         return (this as any).getRepository().update(criteria, partialEntity, options);
     }
 
+    /**
+     * Inserts a given entity into the database, unless a unique constraint conflicts then updates the entity
+     * Unlike save method executes a primitive operation without cascades, relations and other operations included.
+     * Executes fast and efficient INSERT ... ON CONFLICT DO UPDATE/ON DUPLICATE KEY UPDATE query.
+     */
+    static upsert<T extends BaseEntity>(this: ObjectType<T> & typeof BaseEntity, 
+        entityOrEntities: QueryDeepPartialEntity<T> | (QueryDeepPartialEntity<T>[]),
+        conflictPathsOrOptions: string[] | UpsertOptions<T>): Promise<InsertResult> {
+        return this.getRepository<T>().upsert(entityOrEntities, conflictPathsOrOptions);
+    }
+
     /**
      * Deletes entities by a given criteria.
      * Unlike remove method executes a primitive operation without cascades, relations and other operations included.

src/repository/Repository.ts

@@ -14,6 +14,7 @@ import {InsertResult} from "../query-builder/result/InsertResult";
 import {QueryDeepPartialEntity} from "../query-builder/QueryPartialEntity";
 import {ObjectID} from "../driver/mongodb/typings";
 import {FindConditions} from "../find-options/FindConditions";
+import {UpsertOptions} from "./UpsertOptions";
 
 /**
  * Repository is supposed to work with your entity objects. Find entities, insert, update, delete, etc.
@@ -241,6 +242,17 @@ export class Repository<Entity extends ObjectLiteral> {
         return this.manager.update(this.metadata.target as any, criteria as any, partialEntity);
     }
 
+    /**
+     * Inserts a given entity into the database, unless a unique constraint conflicts then updates the entity
+     * Unlike save method executes a primitive operation without cascades, relations and other operations included.
+     * Executes fast and efficient INSERT ... ON CONFLICT DO UPDATE/ON DUPLICATE KEY UPDATE query.
+     */
+    upsert(
+        entityOrEntities: QueryDeepPartialEntity<Entity> | (QueryDeepPartialEntity<Entity>[]),
+        conflictPathsOrOptions: string[] | UpsertOptions<Entity>): Promise<InsertResult> {
+        return this.manager.upsert(this.metadata.target as any, entityOrEntities, conflictPathsOrOptions);
+    }
+
     /**
      * Deletes entities by a given criteria.
      * Unlike save method executes a primitive operation without cascades, relations and other operations included.

src/repository/UpsertOptions.ts

@@ -0,0 +1,7 @@
+/**
+ * Special options passed to Repository#upsert
+ */
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+export interface UpsertOptions<Entity> {
+    conflictPaths: string[]
+}

src/util/OrmUtils.ts

@@ -237,6 +237,14 @@ export class OrmUtils {
         });
     }
 
+    static areMutuallyExclusive<T>(...lists: T[][]): boolean {
+        const haveSharedObjects = lists.some(list => {
+            const otherLists = lists.filter(otherList => otherList !== list);
+            return list.some(item => otherLists.some(otherList => otherList.includes(item)));
+        });
+        return !haveSharedObjects;
+    }
+
     // -------------------------------------------------------------------------
     // Private methods
     // -------------------------------------------------------------------------

test/functional/entity-model/entity-model.ts

@@ -32,6 +32,26 @@ describe("entity-model", () => {
         }
     });
 
+    describe("upsert", function () {
+        it("should upsert successfully", async () => {
+            // These must run sequentially as we have the global context of the `Post` ActiveRecord class
+            for (const connection of connections.filter((c) => c.driver.supportedUpsertType != null)) {
+                Post.useConnection(connection); // change connection each time because of AR specifics
+
+                const externalId = "external-entity";
+
+                await Post.upsert({ externalId, title: "External post" }, ["externalId"]);
+                const upsertInsertedExternalPost = await Post.findOneOrFail({ externalId });
+
+                await Post.upsert({ externalId, title: "External post 2" }, ["externalId"]);
+                const upsertUpdatedExternalPost = await Post.findOneOrFail({ externalId });
+
+                upsertInsertedExternalPost.id.should.be.equal(upsertUpdatedExternalPost.id);
+                upsertInsertedExternalPost.title.should.not.be.equal(upsertUpdatedExternalPost.title);
+            }
+        });
+    });
+
     it("should reload given entity successfully", async () => {
         // These must run sequentially as we have the global context of the `Post` ActiveRecord class
         for (const connection of connections) {
@@ -73,5 +93,4 @@ describe("entity-model", () => {
             });
         }
     });
-
 });

test/functional/entity-model/entity/Post.ts

@@ -11,6 +11,12 @@ export class Post extends BaseEntity {
     @PrimaryGeneratedColumn()
     id: number;
 
+    @Column({
+        nullable: true,
+        unique: true
+    })
+    externalId?: string;
+
     @Column()
     title: string;