From f238f604196ee2d5a9d7a2d4d8273e2a77e383af Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Mon, 20 Apr 2026 15:09:24 +0200 Subject: [PATCH 1/3] feat(db): add optional hintShardKey parameter to insertIgnoreConflict Signed-off-by: Thomas Citharel --- lib/private/DB/Adapter.php | 5 ++++- lib/private/DB/AdapterMySQL.php | 7 +++++-- lib/private/DB/AdapterPgSql.php | 5 ++++- lib/private/DB/AdapterSqlite.php | 7 +++++-- lib/private/DB/Connection.php | 4 ++-- lib/private/DB/ConnectionAdapter.php | 4 ++-- lib/public/IDBConnection.php | 4 +++- 7 files changed, 25 insertions(+), 11 deletions(-) diff --git a/lib/private/DB/Adapter.php b/lib/private/DB/Adapter.php index 6ebcfdc34f242..43bf6822ee9f9 100644 --- a/lib/private/DB/Adapter.php +++ b/lib/private/DB/Adapter.php @@ -111,13 +111,16 @@ public function insertIfNotExist($table, $input, ?array $compare = null) { /** * @throws \OCP\DB\Exception */ - public function insertIgnoreConflict(string $table, array $values) : int { + public function insertIgnoreConflict(string $table, array $values, array $hintShardKey = []) : int { try { $builder = $this->conn->getQueryBuilder(); $builder->insert($table); foreach ($values as $key => $value) { $builder->setValue($key, $builder->createNamedParameter($value)); } + if (isset($hintShardKey['column'], $hintShardKey['value'])) { + $builder->hintShardKey($hintShardKey['column'], $hintShardKey['value'], $hintShardKey['overwrite'] ?? false); + } return $builder->executeStatement(); } catch (DbalException $e) { if ($e->getReason() === \OCP\DB\Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { diff --git a/lib/private/DB/AdapterMySQL.php b/lib/private/DB/AdapterMySQL.php index 63c75607379a3..bbb2e23f8ce70 100644 --- a/lib/private/DB/AdapterMySQL.php +++ b/lib/private/DB/AdapterMySQL.php @@ -36,14 +36,17 @@ protected function getCollation(): string { return $this->collation; } - public function insertIgnoreConflict(string $table, array $values): int { + public function insertIgnoreConflict(string $table, array $values, array $hintShardKey = []): int { $builder = $this->conn->getQueryBuilder(); $builder->insert($table); - $updates = []; foreach ($values as $key => $value) { $builder->setValue($key, $builder->createNamedParameter($value)); } + if (isset($hintShardKey['column'], $hintShardKey['value'])) { + $builder->hintShardKey($hintShardKey['column'], $hintShardKey['value'], $hintShardKey['overwrite'] ?? false); + } + /* * We can't use ON DUPLICATE KEY UPDATE here because Nextcloud use the CLIENT_FOUND_ROWS flag * With this flag the MySQL returns the number of selected rows diff --git a/lib/private/DB/AdapterPgSql.php b/lib/private/DB/AdapterPgSql.php index db48c81c2c54c..aa1a5e9f42d40 100644 --- a/lib/private/DB/AdapterPgSql.php +++ b/lib/private/DB/AdapterPgSql.php @@ -23,7 +23,7 @@ public function fixupStatement($statement) { return $statement; } - public function insertIgnoreConflict(string $table, array $values) : int { + public function insertIgnoreConflict(string $table, array $values, array $hintShardKey = []) : int { // "upsert" is only available since PgSQL 9.5, but the generic way // would leave error logs in the DB. $builder = $this->conn->getQueryBuilder(); @@ -31,6 +31,9 @@ public function insertIgnoreConflict(string $table, array $values) : int { foreach ($values as $key => $value) { $builder->setValue($key, $builder->createNamedParameter($value)); } + if (isset($hintShardKey['column'], $hintShardKey['value'])) { + $builder->hintShardKey($hintShardKey['column'], $hintShardKey['value'], $hintShardKey['overwrite'] ?? false); + } $queryString = $builder->getSQL() . ' ON CONFLICT DO NOTHING'; return $this->conn->executeUpdate($queryString, $builder->getParameters(), $builder->getParameterTypes()); } diff --git a/lib/private/DB/AdapterSqlite.php b/lib/private/DB/AdapterSqlite.php index aeadf55ecf7b5..8b18494f64260 100644 --- a/lib/private/DB/AdapterSqlite.php +++ b/lib/private/DB/AdapterSqlite.php @@ -77,14 +77,17 @@ public function insertIfNotExist($table, $input, ?array $compare = null) { } } - public function insertIgnoreConflict(string $table, array $values): int { + public function insertIgnoreConflict(string $table, array $values, array $hintShardKey = []): int { $builder = $this->conn->getQueryBuilder(); $builder->insert($table); - $updates = []; foreach ($values as $key => $value) { $builder->setValue($key, $builder->createNamedParameter($value)); } + if (isset($hintShardKey['column'], $hintShardKey['value'])) { + $builder->hintShardKey($hintShardKey['column'], $hintShardKey['value'], $hintShardKey['overwrite'] ?? false); + } + return $this->conn->executeStatement( $builder->getSQL() . ' ON CONFLICT DO NOTHING', $builder->getParameters(), diff --git a/lib/private/DB/Connection.php b/lib/private/DB/Connection.php index 3eef60c6dafb7..f583e4a03ba3d 100644 --- a/lib/private/DB/Connection.php +++ b/lib/private/DB/Connection.php @@ -548,9 +548,9 @@ public function insertIfNotExist($table, $input, ?array $compare = null) { } } - public function insertIgnoreConflict(string $table, array $values) : int { + public function insertIgnoreConflict(string $table, array $values, array $hintShardKey = []) : int { try { - return $this->adapter->insertIgnoreConflict($table, $values); + return $this->adapter->insertIgnoreConflict($table, $values, $hintShardKey); } catch (\Exception $e) { $this->logDatabaseException($e); throw $e; diff --git a/lib/private/DB/ConnectionAdapter.php b/lib/private/DB/ConnectionAdapter.php index d9e3e7ec5490b..82b2a2e513298 100644 --- a/lib/private/DB/ConnectionAdapter.php +++ b/lib/private/DB/ConnectionAdapter.php @@ -89,9 +89,9 @@ public function insertIfNotExist(string $table, array $input, ?array $compare = } } - public function insertIgnoreConflict(string $table, array $values): int { + public function insertIgnoreConflict(string $table, array $values, array $hintShardKey = []): int { try { - return $this->inner->insertIgnoreConflict($table, $values); + return $this->inner->insertIgnoreConflict($table, $values, $hintShardKey); } catch (Exception $e) { throw DbalException::wrap($e); } diff --git a/lib/public/IDBConnection.php b/lib/public/IDBConnection.php index 4526f71ceeb47..108821e2f2f06 100644 --- a/lib/public/IDBConnection.php +++ b/lib/public/IDBConnection.php @@ -168,10 +168,12 @@ public function insertIfNotExist(string $table, array $input, ?array $compare = * * @param string $table The table name (will replace *PREFIX* with the actual prefix) * @param array $values data that should be inserted into the table (column name => value) + * @param array{column: string, value: mixed, overwrite?: bool}|array{} $hintShardKey An array representing the shard key to hint * @return int number of inserted rows + * @since 34.0.0 Parameter $hintShardKey was added * @since 16.0.0 */ - public function insertIgnoreConflict(string $table, array $values) : int; + public function insertIgnoreConflict(string $table, array $values, array $hintShardKey = []) : int; /** * Insert or update a row value From ab3d4c6c18359880383c492479cc0065de7a6aa8 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Sun, 26 Apr 2026 17:50:33 +0200 Subject: [PATCH 2/3] feat(db): introduce a ignoreConflictsOnInsert method on the QueryBuilder Since the DB adapters insertIgnoreConflict methods are using the conn directly to alter the produced SQL, bypassing the ShardedQueryBuilder logic, hinting for sharded keys didn't work. We need to expose to the QueryBuilder the information that we want to ignore conflicts, and how to handle it. Signed-off-by: Thomas Citharel --- lib/private/DB/Adapter.php | 4 +++ lib/private/DB/AdapterMySQL.php | 33 +++++++++---------- lib/private/DB/AdapterPgSql.php | 8 +++-- lib/private/DB/AdapterSqlite.php | 11 ++++--- lib/private/DB/Connection.php | 4 +++ lib/private/DB/ConnectionAdapter.php | 4 +++ .../DB/QueryBuilder/ExtendedQueryBuilder.php | 5 +++ lib/private/DB/QueryBuilder/QueryBuilder.php | 14 +++++++- lib/public/DB/QueryBuilder/IQueryBuilder.php | 8 +++++ 9 files changed, 66 insertions(+), 25 deletions(-) diff --git a/lib/private/DB/Adapter.php b/lib/private/DB/Adapter.php index 43bf6822ee9f9..ea4f5cd771327 100644 --- a/lib/private/DB/Adapter.php +++ b/lib/private/DB/Adapter.php @@ -129,4 +129,8 @@ public function insertIgnoreConflict(string $table, array $values, array $hintSh throw $e; } } + + public function getInsertIgnoreSqlTransformer(): ?callable { + return null; + } } diff --git a/lib/private/DB/AdapterMySQL.php b/lib/private/DB/AdapterMySQL.php index bbb2e23f8ce70..57a794b790fd1 100644 --- a/lib/private/DB/AdapterMySQL.php +++ b/lib/private/DB/AdapterMySQL.php @@ -47,23 +47,22 @@ public function insertIgnoreConflict(string $table, array $values, array $hintSh $builder->hintShardKey($hintShardKey['column'], $hintShardKey['value'], $hintShardKey['overwrite'] ?? false); } - /* - * We can't use ON DUPLICATE KEY UPDATE here because Nextcloud use the CLIENT_FOUND_ROWS flag - * With this flag the MySQL returns the number of selected rows - * instead of the number of affected/modified rows - * It's impossible to change this behaviour at runtime or for a single query - * Then, the result is 1 if a row is inserted and also 1 if a row is updated with same or different values - * - * With INSERT IGNORE, the result is 1 when a row is inserted, 0 otherwise - * - * Risk: it can also ignore other errors like type mismatch or truncated data… - */ - $res = $this->conn->executeStatement( - preg_replace('/^INSERT/i', 'INSERT IGNORE', $builder->getSQL()), - $builder->getParameters(), - $builder->getParameterTypes() - ); + $builder->ignoreConflictsOnInsert(); + return $builder->executeStatement(); + } - return $res; + /** + * We can't use ON DUPLICATE KEY UPDATE here because Nextcloud use the CLIENT_FOUND_ROWS flag + * With this flag the MySQL returns the number of selected rows + * instead of the number of affected/modified rows + * It's impossible to change this behaviour at runtime or for a single query + * Then, the result is 1 if a row is inserted and also 1 if a row is updated with same or different values + * + * With INSERT IGNORE, the result is 1 when a row is inserted, 0 otherwise + * + * Risk: it can also ignore other errors like type mismatch or truncated data… + */ + public function getInsertIgnoreSqlTransformer(): callable { + return fn (string $sql) => preg_replace('/^INSERT/i', 'INSERT IGNORE', $sql); } } diff --git a/lib/private/DB/AdapterPgSql.php b/lib/private/DB/AdapterPgSql.php index aa1a5e9f42d40..af1e0a2f089f1 100644 --- a/lib/private/DB/AdapterPgSql.php +++ b/lib/private/DB/AdapterPgSql.php @@ -34,7 +34,11 @@ public function insertIgnoreConflict(string $table, array $values, array $hintSh if (isset($hintShardKey['column'], $hintShardKey['value'])) { $builder->hintShardKey($hintShardKey['column'], $hintShardKey['value'], $hintShardKey['overwrite'] ?? false); } - $queryString = $builder->getSQL() . ' ON CONFLICT DO NOTHING'; - return $this->conn->executeUpdate($queryString, $builder->getParameters(), $builder->getParameterTypes()); + $builder->ignoreConflictsOnInsert(); + return $builder->executeStatement(); + } + + public function getInsertIgnoreSqlTransformer(): callable { + return fn (string $sql) => $sql . ' ON CONFLICT DO NOTHING'; } } diff --git a/lib/private/DB/AdapterSqlite.php b/lib/private/DB/AdapterSqlite.php index 8b18494f64260..d9768572924cf 100644 --- a/lib/private/DB/AdapterSqlite.php +++ b/lib/private/DB/AdapterSqlite.php @@ -88,10 +88,11 @@ public function insertIgnoreConflict(string $table, array $values, array $hintSh $builder->hintShardKey($hintShardKey['column'], $hintShardKey['value'], $hintShardKey['overwrite'] ?? false); } - return $this->conn->executeStatement( - $builder->getSQL() . ' ON CONFLICT DO NOTHING', - $builder->getParameters(), - $builder->getParameterTypes() - ); + $builder->ignoreConflictsOnInsert(); + return $builder->executeStatement(); + } + + public function getInsertIgnoreSqlTransformer(): callable { + return fn (string $sql) => $sql . ' ON CONFLICT DO NOTHING'; } } diff --git a/lib/private/DB/Connection.php b/lib/private/DB/Connection.php index f583e4a03ba3d..c817a11bd4d53 100644 --- a/lib/private/DB/Connection.php +++ b/lib/private/DB/Connection.php @@ -961,4 +961,8 @@ public function getShardDefinition(string $name): ?ShardDefinition { public function getCrossShardMoveHelper(): CrossShardMoveHelper { return new CrossShardMoveHelper($this->shardConnectionManager); } + + public function getInsertIgnoreSqlTransformer(): ?callable { + return $this->adapter->getInsertIgnoreSqlTransformer(); + } } diff --git a/lib/private/DB/ConnectionAdapter.php b/lib/private/DB/ConnectionAdapter.php index 82b2a2e513298..d8af700de69c1 100644 --- a/lib/private/DB/ConnectionAdapter.php +++ b/lib/private/DB/ConnectionAdapter.php @@ -265,4 +265,8 @@ public function getShardDefinition(string $name): ?ShardDefinition { public function getCrossShardMoveHelper(): CrossShardMoveHelper { return $this->inner->getCrossShardMoveHelper(); } + + public function getInsertIgnoreSqlTransformer(): ?callable { + return $this->inner->getInsertIgnoreSqlTransformer(); + } } diff --git a/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php b/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php index 33901ace1d4ee..dcb8bc1f9c792 100644 --- a/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php +++ b/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php @@ -47,6 +47,11 @@ public function getState() { return $this->builder->getState(); } + public function ignoreConflictsOnInsert(): self { + $this->builder->ignoreConflictsOnInsert(); + return $this; + } + public function getSQL() { return $this->builder->getSQL(); } diff --git a/lib/private/DB/QueryBuilder/QueryBuilder.php b/lib/private/DB/QueryBuilder/QueryBuilder.php index 1129970265c66..bf8854ad6d314 100644 --- a/lib/private/DB/QueryBuilder/QueryBuilder.php +++ b/lib/private/DB/QueryBuilder/QueryBuilder.php @@ -39,6 +39,7 @@ class QueryBuilder extends TypedQueryBuilder { private bool $nonEmptyWhere = false; protected ?string $lastInsertedTable = null; private array $selectedColumns = []; + private bool $insertIgnoreConflicts = false; /** * Initializes a new QueryBuilder. @@ -275,6 +276,13 @@ public function executeStatement(?IDBConnection $connection = null): int { ); } + public function ignoreConflictsOnInsert(): self { + if ($this->getType() !== \Doctrine\DBAL\Query\QueryBuilder::INSERT) { + throw new \LogicException('ignoreConflictsOnInsert() can only be used on INSERT queries'); + } + $this->insertIgnoreConflicts = true; + return $this; + } /** * Gets the complete SQL string formed by the current specifications of this QueryBuilder. @@ -289,7 +297,11 @@ public function executeStatement(?IDBConnection $connection = null): int { * @return string The SQL query string. */ public function getSQL() { - return $this->queryBuilder->getSQL(); + $sql = $this->queryBuilder->getSQL(); + if ($this->insertIgnoreConflicts && $this->connection->getInsertIgnoreSqlTransformer() !== null) { + return ($this->connection->getInsertIgnoreSqlTransformer())($sql); + } + return $sql; } /** diff --git a/lib/public/DB/QueryBuilder/IQueryBuilder.php b/lib/public/DB/QueryBuilder/IQueryBuilder.php index 32a505292c20e..bb5dcc2f27d1a 100644 --- a/lib/public/DB/QueryBuilder/IQueryBuilder.php +++ b/lib/public/DB/QueryBuilder/IQueryBuilder.php @@ -216,6 +216,14 @@ public function executeQuery(?IDBConnection $connection = null): IResult; */ public function executeStatement(?IDBConnection $connection = null): int; + /** + * Set to ignore conflicts on insert + * + * @since 34.0.0 + * @return self + */ + public function ignoreConflictsOnInsert(): self; + /** * Gets the complete SQL string formed by the current specifications of this QueryBuilder. * From fe788721c0a685a9b4cd0e11057433f1aa85ff79 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Mon, 20 Apr 2026 15:12:37 +0200 Subject: [PATCH 3/3] feat(cache): use the insertIgnoreConflict connexion method so that conflicts are properly handled In the FileCache, use insertIgnoreConflict to avoid conflict error messages on PostgreSQL. Use the new optional hintShardKey parameter Closes #19494 Signed-off-by: Thomas Citharel --- lib/private/Files/Cache/Cache.php | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 96b0c413e4db5..5875511e9eb89 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -394,21 +394,8 @@ public function update($id, array $data) { } if (count($extensionValues)) { - try { - $query = $this->getQueryBuilder(); - $query->insert('filecache_extended'); - $query->hintShardKey('storage', $this->getNumericStorageId()); - - $query->setValue('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)); - foreach ($extensionValues as $column => $value) { - $query->setValue($column, $query->createNamedParameter($value)); - } - - $query->executeStatement(); - } catch (Exception $e) { - if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { - throw $e; - } + $insertCount = $this->connection->insertIgnoreConflict('filecache_extended', array_merge(['fileid' => $id], $extensionValues), ['column' => 'storage', 'value' => $this->getNumericStorageId()]); + if ($insertCount === 0) { $query = $this->getQueryBuilder(); $query->update('filecache_extended') ->whereFileId($id)