Skip to content

Commit 8f4c442

Browse files
DanRibbensDanRibbens
authored
feat(plugin-import-export): add custom toCSV function on fields (#12533)
This makes it possible to add custom logic into how we map the document data into the CSV data on a field-by-field basis. - Allow custom data transformation to be added to `custom.['plugin-import-export'].toCSV inside the field config - Add type declaration to FieldCustom to improve types - Export with `depth: 1` Example: ```ts { name: 'customRelationship', type: 'relationship', relationTo: 'users', custom: { 'plugin-import-export': { toCSV: ({ value, columnName, row, siblingDoc, doc }) => { row[`${columnName}_id`] = value.id row[`${columnName}_email`] = value.email }, }, }, }, ```
1 parent 773e4ad commit 8f4c442

File tree

17 files changed

+349
-33
lines changed

17 files changed

+349
-33
lines changed

‎docs/plugins/form-builder.mdx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -551,4 +551,4 @@ Below are some common troubleshooting tips. To help other developers, please con
551551

552552
![screenshot 5](https://github.com/payloadcms/plugin-form-builder/blob/main/images/screenshot-5.jpg?raw=true)
553553

554-
![screenshot 6](https://github.com/payloadcms/plugin-form-builder/blob/main/images/screenshot-6.jpg?raw=true)
554+
![screenshot 6](https://github.com/payloadcms/plugin-form-builder/blob/main/images/screenshot-6.jpg?raw=true)

‎packages/payload/src/fields/config/types.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ import type {
128128
CollectionSlug,
129129
DateFieldValidation,
130130
EmailFieldValidation,
131+
FieldCustom,
131132
JSONFieldValidation,
132133
PointFieldValidation,
133134
RadioFieldValidation,
@@ -482,7 +483,7 @@ export interface FieldBase {
482483
}
483484
admin?: Admin
484485
/** Extension point to add your custom data. Server only. */
485-
custom?: Record<string, any>
486+
custom?: FieldCustom
486487
defaultValue?: DefaultValue
487488
hidden?: boolean
488489
hooks?: {

‎packages/payload/src/index.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1238,6 +1238,9 @@ export {
12381238
} from './fields/config/client.js'
12391239

12401240
export { sanitizeFields } from './fields/config/sanitize.js'
1241+
1242+
export interface FieldCustom extends Record<string, any> {}
1243+
12411244
export type {
12421245
AdminClient,
12431246
ArrayField,

‎packages/payload/src/utilities/traverseFields.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -304,12 +304,12 @@ export const traverseFields = ({
304304
return
305305
}
306306

307-
if (field.type !== 'tab' && (fieldHasSubFields(field) || field.type === 'blocks')) {
307+
if (field.type === 'tab' || fieldHasSubFields(field) || field.type === 'blocks') {
308308
if ('name' in field && field.name) {
309309
currentParentRef = currentRef
310310
if (!ref[field.name]) {
311311
if (fillEmpty) {
312-
if (field.type === 'group') {
312+
if (field.type === 'group' || field.type === 'tab') {
313313
if (fieldShouldBeLocalized({ field, parentIsLocalized: parentIsLocalized! })) {
314314
ref[field.name] = {
315315
en: {},
@@ -334,7 +334,7 @@ export const traverseFields = ({
334334
}
335335

336336
if (
337-
field.type === 'group' &&
337+
(field.type === 'tab' || field.type === 'group') &&
338338
fieldShouldBeLocalized({ field, parentIsLocalized: parentIsLocalized! }) &&
339339
currentRef &&
340340
typeof currentRef === 'object'

‎packages/plugin-import-export/src/export/createExport.ts

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { APIError } from 'payload'
66
import { Readable } from 'stream'
77

88
import { flattenObject } from './flattenObject.js'
9+
import { getCustomFieldFunctions } from './getCustomFieldFunctions.js'
910
import { getFilename } from './getFilename.js'
1011
import { getSelect } from './getSelect.js'
1112

@@ -79,20 +80,21 @@ export const createExport = async (args: CreateExportArgs) => {
7980

8081
const name = `${nameArg ?? `${getFilename()}-${collectionSlug}`}.${format}`
8182
const isCSV = format === 'csv'
83+
const select = Array.isArray(fields) && fields.length > 0 ? getSelect(fields) : undefined
8284

8385
if (debug) {
8486
req.payload.logger.info({ message: 'Export configuration:', name, isCSV, locale })
8587
}
8688

8789
const findArgs = {
8890
collection: collectionSlug,
89-
depth: 0,
91+
depth: 1,
9092
draft: drafts === 'yes',
9193
limit: 100,
9294
locale,
9395
overrideAccess: false,
9496
page: 0,
95-
select: Array.isArray(fields) && fields.length > 0 ? getSelect(fields) : undefined,
97+
select,
9698
sort,
9799
user,
98100
where,
@@ -104,6 +106,11 @@ export const createExport = async (args: CreateExportArgs) => {
104106

105107
let result: PaginatedDocs = { hasNextPage: true } as PaginatedDocs
106108

109+
const toCSVFunctions = getCustomFieldFunctions({
110+
fields: collectionConfig.flattenedFields,
111+
select,
112+
})
113+
107114
if (download) {
108115
if (debug) {
109116
req.payload.logger.info('Starting download stream')
@@ -120,7 +127,7 @@ export const createExport = async (args: CreateExportArgs) => {
120127
`Processing batch ${findArgs.page + 1} with ${result.docs.length} documents`,
121128
)
122129
}
123-
const csvInput = result.docs.map((doc) => flattenObject({ doc, fields }))
130+
const csvInput = result.docs.map((doc) => flattenObject({ doc, fields, toCSVFunctions }))
124131
const csvString = stringify(csvInput, { header: isFirstBatch })
125132
this.push(encoder.encode(csvString))
126133
isFirstBatch = false
@@ -164,7 +171,7 @@ export const createExport = async (args: CreateExportArgs) => {
164171
}
165172

166173
if (isCSV) {
167-
const csvInput = result.docs.map((doc) => flattenObject({ doc, fields }))
174+
const csvInput = result.docs.map((doc) => flattenObject({ doc, fields, toCSVFunctions }))
168175
outputData.push(stringify(csvInput, { header: isFirstBatch }))
169176
isFirstBatch = false
170177
} else {

‎packages/plugin-import-export/src/export/flattenObject.ts

Lines changed: 51 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,69 @@
11
import type { Document } from 'payload'
22

3+
import type { ToCSVFunction } from '../types.js'
4+
35
type Args = {
46
doc: Document
57
fields?: string[]
68
prefix?: string
9+
toCSVFunctions: Record<string, ToCSVFunction>
710
}
811

9-
export const flattenObject = ({ doc, fields, prefix }: Args): Record<string, unknown> => {
10-
const result: Record<string, unknown> = {}
12+
export const flattenObject = ({
13+
doc,
14+
fields,
15+
prefix,
16+
toCSVFunctions,
17+
}: Args): Record<string, unknown> => {
18+
const row: Record<string, unknown> = {}
1119

12-
const flatten = (doc: Document, prefix?: string) => {
13-
Object.entries(doc).forEach(([key, value]) => {
20+
const flatten = (siblingDoc: Document, prefix?: string) => {
21+
Object.entries(siblingDoc).forEach(([key, value]) => {
1422
const newKey = prefix ? `${prefix}_${key}` : key
1523

1624
if (Array.isArray(value)) {
1725
value.forEach((item, index) => {
1826
if (typeof item === 'object' && item !== null) {
1927
flatten(item, `${newKey}_${index}`)
2028
} else {
21-
result[`${newKey}_${index}`] = item
29+
if (toCSVFunctions?.[newKey]) {
30+
const columnName = `${newKey}_${index}`
31+
row[columnName] = toCSVFunctions[newKey]({
32+
columnName,
33+
doc,
34+
row,
35+
siblingDoc,
36+
value: item,
37+
})
38+
} else {
39+
row[`${newKey}_${index}`] = item
40+
}
2241
}
2342
})
2443
} else if (typeof value === 'object' && value !== null) {
25-
flatten(value, newKey)
44+
if (!toCSVFunctions?.[newKey]) {
45+
flatten(value, newKey)
46+
} else {
47+
row[newKey] = toCSVFunctions[newKey]({
48+
columnName: newKey,
49+
doc,
50+
row,
51+
siblingDoc,
52+
value,
53+
})
54+
}
2655
} else {
27-
result[newKey] = value
56+
if (toCSVFunctions?.[newKey]) {
57+
row[newKey] = toCSVFunctions[newKey]({
58+
columnName: newKey,
59+
doc,
60+
row,
61+
siblingDoc,
62+
value,
63+
})
64+
} else {
65+
row[newKey] = value
66+
}
2867
}
2968
})
3069
}
@@ -41,14 +80,14 @@ export const flattenObject = ({ doc, fields, prefix }: Args): Record<string, unk
4180
}
4281

4382
fields.forEach((field) => {
44-
if (result[field.replace(/\./g, '_')]) {
83+
if (row[field.replace(/\./g, '_')]) {
4584
const sanitizedField = field.replace(/\./g, '_')
46-
orderedResult[sanitizedField] = result[sanitizedField]
85+
orderedResult[sanitizedField] = row[sanitizedField]
4786
} else {
4887
const regex = fieldToRegex(field)
49-
Object.keys(result).forEach((key) => {
88+
Object.keys(row).forEach((key) => {
5089
if (regex.test(key)) {
51-
orderedResult[key] = result[key]
90+
orderedResult[key] = row[key]
5291
}
5392
})
5493
}
@@ -57,5 +96,5 @@ export const flattenObject = ({ doc, fields, prefix }: Args): Record<string, unk
5796
return orderedResult
5897
}
5998

60-
return result
99+
return row
61100
}

‎packages/plugin-import-export/src/export/getCustomFieldFunctions.ts

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
import {
2+
type FlattenedField,
3+
type SelectIncludeType,
4+
traverseFields,
5+
type TraverseFieldsCallback,
6+
} from 'payload'
7+
8+
import type { ToCSVFunction } from '../types.js'
9+
10+
type Args = {
11+
fields: FlattenedField[]
12+
select: SelectIncludeType | undefined
13+
}
14+
15+
export const getCustomFieldFunctions = ({
16+
fields,
17+
select,
18+
}: Args): Record<string, ToCSVFunction> => {
19+
const result: Record<string, ToCSVFunction> = {}
20+
21+
const buildCustomFunctions: TraverseFieldsCallback = ({ field, parentRef, ref }) => {
22+
// @ts-expect-error ref is untyped
23+
ref.prefix = parentRef.prefix || ''
24+
if (field.type === 'group' || field.type === 'tab') {
25+
// @ts-expect-error ref is untyped
26+
const parentPrefix = parentRef?.prefix ? `${parentRef.prefix}_` : ''
27+
// @ts-expect-error ref is untyped
28+
ref.prefix = `${parentPrefix}${field.name}_`
29+
}
30+
31+
if (typeof field.custom?.['plugin-import-export']?.toCSV === 'function') {
32+
// @ts-expect-error ref is untyped
33+
result[`${ref.prefix}${field.name}`] = field.custom['plugin-import-export']?.toCSV
34+
} else if (field.type === 'relationship' || field.type === 'upload') {
35+
if (field.hasMany !== true) {
36+
if (!Array.isArray(field.relationTo)) {
37+
// monomorphic single
38+
// @ts-expect-error ref is untyped
39+
result[`${ref.prefix}${field.name}`] = ({ value }) =>
40+
typeof value === 'object' && value && 'id' in value ? value.id : value
41+
} else {
42+
// polymorphic single
43+
// @ts-expect-error ref is untyped
44+
result[`${ref.prefix}${field.name}`] = ({ data, value }) => {
45+
// @ts-expect-error ref is untyped
46+
data[`${ref.prefix}${field.name}_id`] = value.id
47+
// @ts-expect-error ref is untyped
48+
data[`${ref.prefix}${field.name}_relationTo`] = value.relationTo
49+
return undefined
50+
}
51+
}
52+
} else {
53+
if (!Array.isArray(field.relationTo)) {
54+
// monomorphic many
55+
// @ts-expect-error ref is untyped
56+
result[`${ref.prefix}${field.name}`] = ({
57+
value,
58+
}: {
59+
value: Record<string, unknown>[]
60+
}) =>
61+
value.map((val: number | Record<string, unknown> | string) =>
62+
typeof val === 'object' ? val.id : val,
63+
)
64+
} else {
65+
// polymorphic many
66+
// @ts-expect-error ref is untyped
67+
result[`${ref.prefix}${field.name}`] = ({
68+
data,
69+
value,
70+
}: {
71+
data: Record<string, unknown>
72+
value: Record<string, unknown>[]
73+
}) =>
74+
value.map((val: number | Record<string, unknown> | string, i) => {
75+
// @ts-expect-error ref is untyped
76+
data[`${ref.prefix}${field.name}_${i}_id`] = val.id
77+
// @ts-expect-error ref is untyped
78+
data[`${ref.prefix}${field.name}_${i}_relationTo`] = val.relationTo
79+
return undefined
80+
})
81+
}
82+
}
83+
}
84+
85+
// TODO: do this so we only return the functions needed based on the select used
86+
////@ts-expect-error ref is untyped
87+
// ref.select = typeof select !== 'undefined' || select[field.name] ? select : {}
88+
}
89+
90+
traverseFields({ callback: buildCustomFunctions, fields })
91+
92+
return result
93+
}

‎packages/plugin-import-export/src/export/getSelect.ts

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,13 @@
1-
import type { SelectType } from 'payload'
1+
import type { SelectIncludeType } from 'payload'
22

33
/**
44
* Takes an input of array of string paths in dot notation and returns a select object
55
* example args: ['id', 'title', 'group.value', 'createdAt', 'updatedAt']
66
*/
7-
export const getSelect = (fields: string[]): SelectType => {
8-
const select: SelectType = {}
7+
export const getSelect = (fields: string[]): SelectIncludeType => {
8+
const select: SelectIncludeType = {}
99

1010
fields.forEach((field) => {
11-
// TODO: this can likely be removed, the form was not saving, leaving in for now
12-
if (!field) {
13-
return
14-
}
1511
const segments = field.split('.')
1612
let selectRef = select
1713

@@ -22,7 +18,7 @@ export const getSelect = (fields: string[]): SelectType => {
2218
if (!selectRef[segment]) {
2319
selectRef[segment] = {}
2420
}
25-
selectRef = selectRef[segment] as SelectType
21+
selectRef = selectRef[segment] as SelectIncludeType
2622
}
2723
})
2824
})

‎packages/plugin-import-export/src/exports/types.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
export type { ImportExportPluginConfig } from '../types.js'
1+
export type { ImportExportPluginConfig, ToCSVFunction } from '../types.js'

‎packages/plugin-import-export/src/getExportCollection.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import type {
22
CollectionAfterChangeHook,
3-
CollectionBeforeChangeHook,
43
CollectionBeforeOperationHook,
54
CollectionConfig,
65
Config,

‎packages/plugin-import-export/src/index.ts

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import type { Config, JobsConfig } from 'payload'
33
import { deepMergeSimple } from 'payload'
44

55
import type { PluginDefaultTranslationsObject } from './translations/types.js'
6-
import type { ImportExportPluginConfig } from './types.js'
6+
import type { ImportExportPluginConfig, ToCSVFunction } from './types.js'
77

88
import { getCreateCollectionExportTask } from './export/getCreateExportCollectionTask.js'
99
import { getExportCollection } from './getExportCollection.js'
@@ -91,3 +91,11 @@ export const importExportPlugin =
9191

9292
return config
9393
}
94+
95+
declare module 'payload' {
96+
export interface FieldCustom {
97+
'plugin-import-export'?: {
98+
toCSVFunction?: ToCSVFunction
99+
}
100+
}
101+
}

0 commit comments

Comments
 (0)