feat(db-mongodb): strip keys from the data that don't exist in the schema from read results (#11558)

This change makes so that data that exists in MongoDB but isn't defined
in the Payload config won't be included to `payload.find` /
`payload.db.find` calls. Now we strip all the additional keys.

Consider you have a field named `secretField` that's also `hidden: true`
(or `read: () => false`) that contains some sensitive data. Then you
removed this field from the database and as for now with the MongoDB
adapter this field will be included to the Local API / REST API results
without any consideration, as Payload doesn't know about it anymore.

This also fixes #11542 if
you removed / renamed a relationship field from the schema, Payload
won't sanitize ObjectIDs back to strings anymore.

Ideally you should create a migration script that completely removes the
deleted field from the database with `$unset`, but people rarely do
this.

If you still need to keep those fields to the result, this PR allows you
to do this with the new `allowAdditionalKeys: true` flag.

Author: r1tsuu

docs/database/mongodb.mdx

@@ -34,11 +34,12 @@ export default buildConfig({
 | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | `autoPluralization`        | Tell Mongoose to auto-pluralize any collection names if it encounters any singular words used as collection `slug`s.                                                                                                                                                                                         |
 | `connectOptions`           | Customize MongoDB connection options. Payload will connect to your MongoDB database using default options which you can override and extend to include all the [options](https://mongoosejs.com/docs/connections.html#options) available to mongoose.                                                        |
-| `collectionsSchemaOptions` | Customize Mongoose schema options for collections.                                                                                                                                                                                                                                                                                                             |
+| `collectionsSchemaOptions` | Customize Mongoose schema options for collections.                                                                                                                                                                                                                                                           |
 | `disableIndexHints`        | Set to true to disable hinting to MongoDB to use 'id' as index. This is currently done when counting documents for pagination, as it increases the speed of the count function used in that query. Disabling this optimization might fix some problems with AWS DocumentDB. Defaults to false                |
 | `migrationDir`             | Customize the directory that migrations are stored.                                                                                                                                                                                                                                                          |
 | `transactionOptions`       | An object with configuration properties used in [transactions](https://www.mongodb.com/docs/manual/core/transactions/) or `false` which will disable the use of transactions.                                                                                                                                |
 | `collation`                | Enable language-specific string comparison with customizable options. Available on MongoDB 3.4+. Defaults locale to "en". Example: `{ strength: 3 }`. For a full list of collation options and their definitions, see the [MongoDB documentation](https://www.mongodb.com/docs/manual/reference/collation/). |
+| `allowAdditionalKeys`      | By default, Payload strips all additional keys from MongoDB data that don't exist in the Payload schema. If you have some data that you want to include to the result but it doesn't exist in Payload, you can set this to `true`. Be careful as Payload access control _won't_ work for this data.          |
 
 ## Access to Mongoose models
 

packages/db-mongodb/src/index.ts

@@ -63,6 +63,13 @@ import { upsert } from './upsert.js'
 export type { MigrateDownArgs, MigrateUpArgs } from './types.js'
 
 export interface Args {
+  /**
+   * By default, Payload strips all additional keys from MongoDB data that don't exist
+   * in the Payload schema. If you have some data that you want to include to the result
+   * but it doesn't exist in Payload, you can enable this flag
+   * @default false
+   */
+  allowAdditionalKeys?: boolean
   /** Set to false to disable auto-pluralization of collection names, Defaults to true */
   autoPluralization?: boolean
   /**
@@ -89,6 +96,7 @@ export interface Args {
    * Defaults to disabled.
    */
   collation?: Omit<CollationOptions, 'locale'>
+
   collectionsSchemaOptions?: Partial<Record<CollectionSlug, SchemaOptions>>
 
   /** Extra configuration options */
@@ -174,6 +182,7 @@ declare module 'payload' {
 }
 
 export function mongooseAdapter({
+  allowAdditionalKeys = false,
   autoPluralization = true,
   collectionsSchemaOptions = {},
   connectOptions,
@@ -210,6 +219,7 @@ export function mongooseAdapter({
       url,
       versions: {},
       // DatabaseAdapter
+      allowAdditionalKeys,
       beginTransaction: transactionOptions === false ? defaultBeginTransaction() : beginTransaction,
       collectionsSchemaOptions,
       commitTransaction,

packages/db-mongodb/src/utilities/transform.ts

@@ -2,6 +2,8 @@ import type {
   CollectionConfig,
   DateField,
   Field,
+  FlattenedBlock,
+  FlattenedField,
   JoinField,
   RelationshipField,
   SanitizedConfig,
@@ -10,7 +12,7 @@ import type {
 } from 'payload'
 
 import { Types } from 'mongoose'
-import { traverseFields } from 'payload'
+import { flattenAllFields, traverseFields } from 'payload'
 import { fieldAffectsData, fieldShouldBeLocalized } from 'payload/shared'
 
 import type { MongooseAdapter } from '../index.js'
@@ -228,6 +230,141 @@ type Args = {
   validateRelationships?: boolean
 }
 
+const stripFields = ({
+  config,
+  data,
+  fields,
+  reservedKeys = [],
+}: {
+  config: SanitizedConfig
+  data: any
+  fields: FlattenedField[]
+  reservedKeys?: string[]
+}) => {
+  for (const k in data) {
+    if (!fields.some((field) => field.name === k) && !reservedKeys.includes(k)) {
+      delete data[k]
+    }
+  }
+
+  for (const field of fields) {
+    reservedKeys = []
+    const fieldData = data[field.name]
+    if (!fieldData || typeof fieldData !== 'object') {
+      continue
+    }
+
+    if (field.type === 'blocks') {
+      reservedKeys.push('blockType')
+    }
+
+    if ('flattenedFields' in field || 'blocks' in field) {
+      if (field.localized && config.localization) {
+        for (const localeKey in fieldData) {
+          if (!config.localization.localeCodes.some((code) => code === localeKey)) {
+            delete fieldData[localeKey]
+            continue
+          }
+
+          const localeData = fieldData[localeKey]
+
+          if (!localeData || typeof localeData !== 'object') {
+            continue
+          }
+
+          if (field.type === 'array' || field.type === 'blocks') {
+            if (!Array.isArray(localeData)) {
+              continue
+            }
+
+            for (const data of localeData) {
+              let fields: FlattenedField[] | null = null
+
+              if (field.type === 'array') {
+                fields = field.flattenedFields
+              } else {
+                let maybeBlock: FlattenedBlock | undefined = undefined
+
+                if (field.blockReferences) {
+                  const maybeBlockReference = field.blockReferences.find(
+                    (each) => typeof each === 'object' && each.slug === data.blockType,
+                  )
+                  if (maybeBlockReference && typeof maybeBlockReference === 'object') {
+                    maybeBlock = maybeBlockReference
+                  }
+                }
+
+                if (!maybeBlock) {
+                  maybeBlock = field.blocks.find((each) => each.slug === data.blockType)
+                }
+
+                if (maybeBlock) {
+                  fields = maybeBlock.flattenedFields
+                }
+              }
+
+              if (!fields) {
+                continue
+              }
+
+              stripFields({ config, data, fields, reservedKeys })
+            }
+
+            continue
+          } else {
+            stripFields({ config, data: localeData, fields: field.flattenedFields, reservedKeys })
+          }
+        }
+        continue
+      }
+
+      if (field.type === 'array' || field.type === 'blocks') {
+        if (!Array.isArray(fieldData)) {
+          continue
+        }
+
+        for (const data of fieldData) {
+          let fields: FlattenedField[] | null = null
+
+          if (field.type === 'array') {
+            fields = field.flattenedFields
+          } else {
+            let maybeBlock: FlattenedBlock | undefined = undefined
+
+            if (field.blockReferences) {
+              const maybeBlockReference = field.blockReferences.find(
+                (each) => typeof each === 'object' && each.slug === data.blockType,
+              )
+
+              if (maybeBlockReference && typeof maybeBlockReference === 'object') {
+                maybeBlock = maybeBlockReference
+              }
+            }
+
+            if (!maybeBlock) {
+              maybeBlock = field.blocks.find((each) => each.slug === data.blockType)
+            }
+
+            if (maybeBlock) {
+              fields = maybeBlock.flattenedFields
+            }
+          }
+
+          if (!fields) {
+            continue
+          }
+
+          stripFields({ config, data, fields, reservedKeys })
+        }
+
+        continue
+      } else {
+        stripFields({ config, data: fieldData, fields: field.flattenedFields, reservedKeys })
+      }
+    }
+  }
+}
+
 export const transform = ({
   adapter,
   data,
@@ -256,6 +393,15 @@ export const transform = ({
     if (data.id instanceof Types.ObjectId) {
       data.id = data.id.toHexString()
     }
+
+    if (!adapter.allowAdditionalKeys) {
+      stripFields({
+        config,
+        data,
+        fields: flattenAllFields({ cache: true, fields }),
+        reservedKeys: ['id', 'globalType'],
+      })
+    }
   }
 
   if (operation === 'write' && globalSlug) {

packages/payload/src/utilities/flattenAllFields.ts

@@ -16,7 +16,23 @@ export const flattenBlock = ({ block }: { block: Block }): FlattenedBlock => {
   }
 }
 
-export const flattenAllFields = ({ fields }: { fields: Field[] }): FlattenedField[] => {
+const flattenedFieldsCache = new Map<Field[], FlattenedField[]>()
+
+export const flattenAllFields = ({
+  cache,
+  fields,
+}: {
+  /** Allows you to get FlattenedField[] from Field[] anywhere without performance overhead by caching. */
+  cache?: boolean
+  fields: Field[]
+}): FlattenedField[] => {
+  if (cache) {
+    const maybeFields = flattenedFieldsCache.get(fields)
+    if (maybeFields) {
+      return maybeFields
+    }
+  }
+
   const result: FlattenedField[] = []
 
   for (const field of fields) {
@@ -97,5 +113,7 @@ export const flattenAllFields = ({ fields }: { fields: Field[] }): FlattenedFiel
     }
   }
 
+  flattenedFieldsCache.set(fields, result)
+
   return result
 }

test/database/int.spec.ts

@@ -1790,4 +1790,49 @@ describe('database', () => {
     expect(query2.totalDocs).toEqual(1)
     expect(query3.totalDocs).toEqual(1)
   })
+
+  it('mongodb additional keys stripping', async () => {
+    // eslint-disable-next-line jest/no-conditional-in-test
+    if (payload.db.name !== 'mognoose') {
+      return
+    }
+
+    const arrItemID = randomUUID()
+    const res = await payload.db.collections[postsSlug]?.collection.insertOne({
+      SECRET_FIELD: 'secret data',
+      arrayWithIDs: [
+        {
+          id: arrItemID,
+          additionalKeyInArray: 'true',
+          text: 'existing key',
+        },
+      ],
+    })
+
+    let payloadRes: any = await payload.findByID({
+      collection: postsSlug,
+      id: res!.insertedId.toHexString(),
+    })
+
+    expect(payloadRes.id).toBe(res!.insertedId.toHexString())
+    expect(payloadRes['SECRET_FIELD']).toBeUndefined()
+    expect(payloadRes.arrayWithIDs).toBeDefined()
+    expect(payloadRes.arrayWithIDs[0].id).toBe(arrItemID)
+    expect(payloadRes.arrayWithIDs[0].text).toBe('existing key')
+    expect(payloadRes.arrayWithIDs[0].additionalKeyInArray).toBeUndefined()
+
+    // But allows when allowAdditionaKeys is true
+    payload.db.allowAdditionalKeys = true
+
+    payloadRes = await payload.findByID({
+      collection: postsSlug,
+      id: res!.insertedId.toHexString(),
+    })
+
+    expect(payloadRes.id).toBe(res!.insertedId.toHexString())
+    expect(payloadRes['SECRET_FIELD']).toBe('secret data')
+    expect(payloadRes.arrayWithIDs[0].additionalKeyInArray).toBe('true')
+
+    payload.db.allowAdditionalKeys = false
+  })
 })