Skip to content

Commit 5ded64e

Browse files
AlessioGrAlessioGr
authored
feat(db-*): support atomic array $push db updates (#13453)
This PR adds **atomic** `$push` **support for array fields**. It makes it possible to safely append new items to arrays, which is especially useful when running tasks in parallel (like job queues) where multiple processes might update the same record at the same time. By handling pushes atomically, we avoid race conditions and keep data consistent - especially on postgres, where the current implementation would nuke the entire array table before re-inserting every single array item. The feature works for both localized and unlocalized arrays, and supports pushing either single or multiple items at once. This PR is a requirement for reliably running parallel tasks in the job queue - see #13452. Alongside documenting `$push`, this PR also adds documentation for `$inc`. ## Changes to updatedAt behavior #13335 allows us to override the updatedAt property instead of the db always setting it to the current date. However, we are not able to skip updating the updatedAt property completely. This means, usage of $push results in 2 postgres db calls: 1. set updatedAt in main row 2. append array row in arrays table This PR changes the behavior to only automatically set updatedAt if it's undefined. If you explicitly set it to `null`, this now allows you to skip the db adapter automatically setting updatedAt. => This allows us to use $push in just one single db call ## Usage Examples ### Pushing a single item to an array ```ts const post = (await payload.db.updateOne({ data: { array: { $push: { text: 'some text 2', id: new mongoose.Types.ObjectId().toHexString(), }, }, }, collection: 'posts', id: post.id, })) ``` ### Pushing a single item to a localized array ```ts const post = (await payload.db.updateOne({ data: { arrayLocalized: { $push: { en: { text: 'some text 2', id: new mongoose.Types.ObjectId().toHexString(), }, es: { text: 'some text 2 es', id: new mongoose.Types.ObjectId().toHexString(), }, }, }, }, collection: 'posts', id: post.id, })) ``` ### Pushing multiple items to an array ```ts const post = (await payload.db.updateOne({ data: { array: { $push: [ { text: 'some text 2', id: new mongoose.Types.ObjectId().toHexString(), }, { text: 'some text 3', id: new mongoose.Types.ObjectId().toHexString(), }, ], }, }, collection: 'posts', id: post.id, })) ``` ### Pushing multiple items to a localized array ```ts const post = (await payload.db.updateOne({ data: { arrayLocalized: { $push: { en: { text: 'some text 2', id: new mongoose.Types.ObjectId().toHexString(), }, es: [ { text: 'some text 2 es', id: new mongoose.Types.ObjectId().toHexString(), }, { text: 'some text 3 es', id: new mongoose.Types.ObjectId().toHexString(), }, ], }, }, }, collection: 'posts', id: post.id, })) ``` --- - To see the specific tasks where the Asana app for GitHub is being used, see below: - https://app.asana.com/0/0/1211110462564647
1 parent a670438 commit 5ded64e

File tree

14 files changed

+685
-65
lines changed

14 files changed

+685
-65
lines changed

‎packages/db-mongodb/src/updateOne.ts

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,24 @@ export const updateOne: UpdateOne = async function updateOne(
5252

5353
let result
5454

55-
const $inc: Record<string, number> = {}
5655
let updateData: UpdateQuery<any> = data
57-
transform({ $inc, adapter: this, data, fields, operation: 'write' })
56+
57+
const $inc: Record<string, number> = {}
58+
const $push: Record<string, { $each: any[] } | any> = {}
59+
60+
transform({ $inc, $push, adapter: this, data, fields, operation: 'write' })
61+
62+
const updateOps: UpdateQuery<any> = {}
63+
5864
if (Object.keys($inc).length) {
59-
updateData = { $inc, $set: updateData }
65+
updateOps.$inc = $inc
66+
}
67+
if (Object.keys($push).length) {
68+
updateOps.$push = $push
69+
}
70+
if (Object.keys(updateOps).length) {
71+
updateOps.$set = updateData
72+
updateData = updateOps
6073
}
6174

6275
try {

‎packages/db-mongodb/src/utilities/transform.ts

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ const sanitizeDate = ({
209209

210210
type Args = {
211211
$inc?: Record<string, number>
212+
$push?: Record<string, { $each: any[] } | any>
212213
/** instance of the adapter */
213214
adapter: MongooseAdapter
214215
/** data to transform, can be an array of documents or a single document */
@@ -398,6 +399,7 @@ const stripFields = ({
398399

399400
export const transform = ({
400401
$inc,
402+
$push,
401403
adapter,
402404
data,
403405
fields,
@@ -412,7 +414,16 @@ export const transform = ({
412414

413415
if (Array.isArray(data)) {
414416
for (const item of data) {
415-
transform({ $inc, adapter, data: item, fields, globalSlug, operation, validateRelationships })
417+
transform({
418+
$inc,
419+
$push,
420+
adapter,
421+
data: item,
422+
fields,
423+
globalSlug,
424+
operation,
425+
validateRelationships,
426+
})
416427
}
417428
return
418429
}
@@ -470,6 +481,39 @@ export const transform = ({
470481
}
471482
}
472483

484+
if (
485+
$push &&
486+
field.type === 'array' &&
487+
operation === 'write' &&
488+
field.name in ref &&
489+
ref[field.name]
490+
) {
491+
const value = ref[field.name]
492+
if (value && typeof value === 'object' && '$push' in value) {
493+
const push = value.$push
494+
495+
if (config.localization && fieldShouldBeLocalized({ field, parentIsLocalized })) {
496+
if (typeof push === 'object' && push !== null) {
497+
Object.entries(push).forEach(([localeKey, localeData]) => {
498+
if (Array.isArray(localeData)) {
499+
$push[`${parentPath}${field.name}.${localeKey}`] = { $each: localeData }
500+
} else if (typeof localeData === 'object') {
501+
$push[`${parentPath}${field.name}.${localeKey}`] = localeData
502+
}
503+
})
504+
}
505+
} else {
506+
if (Array.isArray(push)) {
507+
$push[`${parentPath}${field.name}`] = { $each: push }
508+
} else if (typeof push === 'object') {
509+
$push[`${parentPath}${field.name}`] = push
510+
}
511+
}
512+
513+
delete ref[field.name]
514+
}
515+
}
516+
473517
if (field.type === 'date' && operation === 'read' && field.name in ref && ref[field.name]) {
474518
if (config.localization && fieldShouldBeLocalized({ field, parentIsLocalized })) {
475519
const fieldRef = ref[field.name] as Record<string, unknown>
@@ -550,8 +594,13 @@ export const transform = ({
550594
})
551595

552596
if (operation === 'write') {
553-
if (!data.updatedAt) {
597+
if (typeof data.updatedAt === 'undefined') {
598+
// If data.updatedAt is explicitly set to `null` we should not set it - this means we don't want to change the value of updatedAt.
554599
data.updatedAt = new Date().toISOString()
600+
} else if (data.updatedAt === null) {
601+
// `updatedAt` may be explicitly set to null to disable updating it - if that is the case, we need to delete the property. Keeping it as null will
602+
// cause the database to think we want to set it to null, which we don't.
603+
delete data.updatedAt
555604
}
556605
}
557606
}

‎packages/drizzle/src/transform/write/array.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ export const transformArray = ({
7171
data.forEach((arrayRow, i) => {
7272
const newRow: ArrayRowToInsert = {
7373
arrays: {},
74+
arraysToPush: {},
7475
locales: {},
7576
row: {
7677
_order: i + 1,
@@ -104,6 +105,7 @@ export const transformArray = ({
104105
traverseFields({
105106
adapter,
106107
arrays: newRow.arrays,
108+
arraysToPush: newRow.arraysToPush,
107109
baseTableName,
108110
blocks,
109111
blocksToDelete,

‎packages/drizzle/src/transform/write/blocks.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ export const transformBlocks = ({
7878

7979
const newRow: BlockRowToInsert = {
8080
arrays: {},
81+
arraysToPush: {},
8182
locales: {},
8283
row: {
8384
_order: i + 1,
@@ -116,6 +117,7 @@ export const transformBlocks = ({
116117
traverseFields({
117118
adapter,
118119
arrays: newRow.arrays,
120+
arraysToPush: newRow.arraysToPush,
119121
baseTableName,
120122
blocks,
121123
blocksToDelete,

‎packages/drizzle/src/transform/write/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ export const transformForWrite = ({
2727
// Split out the incoming data into rows to insert / delete
2828
const rowToInsert: RowToInsert = {
2929
arrays: {},
30+
arraysToPush: {},
3031
blocks: {},
3132
blocksToDelete: new Set(),
3233
locales: {},
@@ -45,6 +46,7 @@ export const transformForWrite = ({
4546
traverseFields({
4647
adapter,
4748
arrays: rowToInsert.arrays,
49+
arraysToPush: rowToInsert.arraysToPush,
4850
baseTableName: tableName,
4951
blocks: rowToInsert.blocks,
5052
blocksToDelete: rowToInsert.blocksToDelete,

‎packages/drizzle/src/transform/write/traverseFields.ts

Lines changed: 63 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,7 @@ import { fieldIsVirtual, fieldShouldBeLocalized } from 'payload/shared'
44
import toSnakeCase from 'to-snake-case'
55

66
import type { DrizzleAdapter } from '../../types.js'
7-
import type {
8-
ArrayRowToInsert,
9-
BlockRowToInsert,
10-
NumberToDelete,
11-
RelationshipToDelete,
12-
TextToDelete,
13-
} from './types.js'
7+
import type { NumberToDelete, RelationshipToDelete, RowToInsert, TextToDelete } from './types.js'
148

159
import { isArrayOfRows } from '../../utilities/isArrayOfRows.js'
1610
import { resolveBlockTableName } from '../../utilities/validateExistingBlockIsIdentical.js'
@@ -23,16 +17,20 @@ import { transformTexts } from './texts.js'
2317

2418
type Args = {
2519
adapter: DrizzleAdapter
26-
arrays: {
27-
[tableName: string]: ArrayRowToInsert[]
28-
}
20+
/**
21+
* This will delete the array table and then re-insert all the new array rows.
22+
*/
23+
arrays: RowToInsert['arrays']
24+
/**
25+
* Array rows to push to the existing array. This will simply create
26+
* a new row in the array table.
27+
*/
28+
arraysToPush: RowToInsert['arraysToPush']
2929
/**
3030
* This is the name of the base table
3131
*/
3232
baseTableName: string
33-
blocks: {
34-
[blockType: string]: BlockRowToInsert[]
35-
}
33+
blocks: RowToInsert['blocks']
3634
blocksToDelete: Set<string>
3735
/**
3836
* A snake-case field prefix, representing prior fields
@@ -82,6 +80,7 @@ type Args = {
8280
export const traverseFields = ({
8381
adapter,
8482
arrays,
83+
arraysToPush,
8584
baseTableName,
8685
blocks,
8786
blocksToDelete,
@@ -129,13 +128,24 @@ export const traverseFields = ({
129128
if (field.type === 'array') {
130129
const arrayTableName = adapter.tableNameMap.get(`${parentTableName}_${columnName}`)
131130

132-
if (!arrays[arrayTableName]) {
133-
arrays[arrayTableName] = []
134-
}
135-
136131
if (isLocalized) {
137-
if (typeof data[field.name] === 'object' && data[field.name] !== null) {
138-
Object.entries(data[field.name]).forEach(([localeKey, localeData]) => {
132+
let value: {
133+
[locale: string]: unknown[]
134+
} = data[field.name] as any
135+
136+
let push = false
137+
if (typeof value === 'object' && '$push' in value) {
138+
value = value.$push as any
139+
push = true
140+
}
141+
142+
if (typeof value === 'object' && value !== null) {
143+
Object.entries(value).forEach(([localeKey, _localeData]) => {
144+
let localeData = _localeData
145+
if (push && !Array.isArray(localeData)) {
146+
localeData = [localeData]
147+
}
148+
139149
if (Array.isArray(localeData)) {
140150
const newRows = transformArray({
141151
adapter,
@@ -158,18 +168,35 @@ export const traverseFields = ({
158168
withinArrayOrBlockLocale: localeKey,
159169
})
160170

161-
arrays[arrayTableName] = arrays[arrayTableName].concat(newRows)
171+
if (push) {
172+
if (!arraysToPush[arrayTableName]) {
173+
arraysToPush[arrayTableName] = []
174+
}
175+
arraysToPush[arrayTableName] = arraysToPush[arrayTableName].concat(newRows)
176+
} else {
177+
if (!arrays[arrayTableName]) {
178+
arrays[arrayTableName] = []
179+
}
180+
arrays[arrayTableName] = arrays[arrayTableName].concat(newRows)
181+
}
162182
}
163183
})
164184
}
165185
} else {
186+
let value = data[field.name]
187+
let push = false
188+
if (typeof value === 'object' && '$push' in value) {
189+
value = Array.isArray(value.$push) ? value.$push : [value.$push]
190+
push = true
191+
}
192+
166193
const newRows = transformArray({
167194
adapter,
168195
arrayTableName,
169196
baseTableName,
170197
blocks,
171198
blocksToDelete,
172-
data: data[field.name],
199+
data: value,
173200
field,
174201
numbers,
175202
numbersToDelete,
@@ -183,7 +210,17 @@ export const traverseFields = ({
183210
withinArrayOrBlockLocale,
184211
})
185212

186-
arrays[arrayTableName] = arrays[arrayTableName].concat(newRows)
213+
if (push) {
214+
if (!arraysToPush[arrayTableName]) {
215+
arraysToPush[arrayTableName] = []
216+
}
217+
arraysToPush[arrayTableName] = arraysToPush[arrayTableName].concat(newRows)
218+
} else {
219+
if (!arrays[arrayTableName]) {
220+
arrays[arrayTableName] = []
221+
}
222+
arrays[arrayTableName] = arrays[arrayTableName].concat(newRows)
223+
}
187224
}
188225

189226
return
@@ -264,6 +301,7 @@ export const traverseFields = ({
264301
traverseFields({
265302
adapter,
266303
arrays,
304+
arraysToPush,
267305
baseTableName,
268306
blocks,
269307
blocksToDelete,
@@ -298,6 +336,7 @@ export const traverseFields = ({
298336
traverseFields({
299337
adapter,
300338
arrays,
339+
arraysToPush,
301340
baseTableName,
302341
blocks,
303342
blocksToDelete,
@@ -547,8 +586,8 @@ export const traverseFields = ({
547586
let formattedValue = value
548587

549588
if (field.type === 'date') {
550-
if (fieldName === 'updatedAt' && !formattedValue) {
551-
// let the db handle this
589+
if (fieldName === 'updatedAt' && typeof formattedValue === 'undefined') {
590+
// let the db handle this. If formattedValue is explicitly set to `null` we should not set it - this means we don't want to change the value of updatedAt.
552591
formattedValue = new Date().toISOString()
553592
} else {
554593
if (typeof value === 'number' && !Number.isNaN(value)) {

‎packages/drizzle/src/transform/write/types.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ export type ArrayRowToInsert = {
22
arrays: {
33
[tableName: string]: ArrayRowToInsert[]
44
}
5+
arraysToPush: {
6+
[tableName: string]: ArrayRowToInsert[]
7+
}
58
locales: {
69
[locale: string]: Record<string, unknown>
710
}
@@ -12,6 +15,9 @@ export type BlockRowToInsert = {
1215
arrays: {
1316
[tableName: string]: ArrayRowToInsert[]
1417
}
18+
arraysToPush: {
19+
[tableName: string]: ArrayRowToInsert[]
20+
}
1521
locales: {
1622
[locale: string]: Record<string, unknown>
1723
}
@@ -37,6 +43,9 @@ export type RowToInsert = {
3743
arrays: {
3844
[tableName: string]: ArrayRowToInsert[]
3945
}
46+
arraysToPush: {
47+
[tableName: string]: ArrayRowToInsert[]
48+
}
4049
blocks: {
4150
[tableName: string]: BlockRowToInsert[]
4251
}

0 commit comments

Comments
 (0)